pax_global_header00006660000000000000000000000064147752462710014531gustar00rootroot0000000000000052 comment=6b830a942419a2ccc0250bcf4b7c7a8d89a5f03b nats-server-2.10.27/000077500000000000000000000000001477524627100141535ustar00rootroot00000000000000nats-server-2.10.27/.coveralls.yml000066400000000000000000000000311477524627100167400ustar00rootroot00000000000000service_name: travis-pro nats-server-2.10.27/.github/000077500000000000000000000000001477524627100155135ustar00rootroot00000000000000nats-server-2.10.27/.github/CODEOWNERS000066400000000000000000000000221477524627100171000ustar00rootroot00000000000000* @nats-io/server nats-server-2.10.27/.github/FUNDING.yml000066400000000000000000000001241477524627100173250ustar00rootroot00000000000000# These are supported funding model platforms # NATS.io community_bridge: nats-io nats-server-2.10.27/.github/ISSUE_TEMPLATE/000077500000000000000000000000001477524627100176765ustar00rootroot00000000000000nats-server-2.10.27/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000005151477524627100216670ustar00rootroot00000000000000blank_issues_enabled: false contact_links: - name: Discussion url: https://github.com/nats-io/nats-server/discussions about: Ideal for ideas, feedback, or longer form questions. - name: Chat url: https://slack.nats.io about: Ideal for short, one-off questions, general conversation, and meeting other NATS users! nats-server-2.10.27/.github/ISSUE_TEMPLATE/defect.yml000066400000000000000000000030071477524627100216530ustar00rootroot00000000000000--- name: Defect description: Report a defect, such as a bug or regression. labels: - defect body: - type: textarea id: observed attributes: label: Observed behavior description: Describe the unexpected behavior or performance regression you are observing. validations: required: true - type: textarea id: expected attributes: label: Expected behavior description: Describe the expected behavior or performance characteristics. validations: required: true - type: textarea id: versions attributes: label: Server and client version description: |- Provide the versions you were using when the detect was observed. For the server, use `nats-server --version`, check the startup log output, or the image tag pulled from Docker. For the CLI client, use `nats --version`. For language-specific clients, check the version downloaded by the language dependency manager. validations: required: true - type: textarea id: environment attributes: label: Host environment description: |- Specify any relevant details about the host environment the server and/or client was running in, such as operating system, CPU architecture, container runtime, etc. validations: required: false - type: textarea id: steps attributes: label: Steps to reproduce description: Provide as many concrete steps to reproduce the defect. validations: required: false nats-server-2.10.27/.github/ISSUE_TEMPLATE/proposal.yml000066400000000000000000000013241477524627100222600ustar00rootroot00000000000000--- name: Proposal description: Propose an enhancement or new feature. labels: - proposal body: - type: textarea id: change attributes: label: Proposed change description: This could be a behavior change, enhanced API, or a new feature. validations: required: true - type: textarea id: usecase attributes: label: Use case description: What is the use case or general motivation for this proposal? validations: required: true - type: textarea id: contribute attributes: label: Contribution description: |- Are you intending or interested in contributing code for this proposal if accepted? validations: required: false nats-server-2.10.27/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000005461477524627100213210ustar00rootroot00000000000000 Signed-off-by: Your Name nats-server-2.10.27/.github/actions/000077500000000000000000000000001477524627100171535ustar00rootroot00000000000000nats-server-2.10.27/.github/actions/nightly-release/000077500000000000000000000000001477524627100222475ustar00rootroot00000000000000nats-server-2.10.27/.github/actions/nightly-release/action.yaml000066400000000000000000000020051477524627100244050ustar00rootroot00000000000000name: Nightly Docker Releaser description: Builds nightly docker images inputs: hub_username: description: Docker hub username required: true hub_password: description: Docker hub password required: true workdir: description: The working directory for actions requiring it required: true runs: using: composite steps: - name: Log in to Docker Hub shell: bash run: docker login -u "${{ inputs.hub_username }}" -p "${{ inputs.hub_password }}" - name: Set up Go uses: actions/setup-go@v5 with: go-version: stable - name: Build and push Docker images # Use commit hash here to avoid a re-tagging attack, as this is a third-party action # Commit 9ed2f89a662bf1735a48bc8557fd212fa902bebf = tag v6.1.0 uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf with: workdir: "${{ inputs.workdir }}" version: ~> v2 args: release --skip=announce,validate --config .goreleaser-nightly.yml nats-server-2.10.27/.github/dependabot.yml000066400000000000000000000003161477524627100203430ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "gomod" directory: "/" schedule: interval: "weekly" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly"nats-server-2.10.27/.github/workflows/000077500000000000000000000000001477524627100175505ustar00rootroot00000000000000nats-server-2.10.27/.github/workflows/cov.yaml000066400000000000000000000030471477524627100212270ustar00rootroot00000000000000name: NATS Server Code Coverage on: workflow_dispatch: {} schedule: - cron: "40 4 * * *" jobs: nightly_coverage: runs-on: ubuntu-latest env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" steps: - name: Checkout code uses: actions/checkout@v4 with: path: src/github.com/nats-io/nats-server - name: Set up Go uses: actions/setup-go@v5 with: go-version: stable cache-dependency-path: src/github.com/nats-io/nats-server/go.sum - name: Run code coverage shell: bash --noprofile --norc -x -eo pipefail {0} run: | set -e cd src/github.com/nats-io/nats-server ./scripts/cov.sh upload set +e - name: Convert coverage.out to coverage.lcov # Use commit hash here to avoid a re-tagging attack, as this is a third-party action # Commit 4e1989767862652e6ca8d3e2e61aabe6d43be28b = tag v1.1.1 uses: jandelgado/gcov2lcov-action@4e1989767862652e6ca8d3e2e61aabe6d43be28b with: infile: acc.out working-directory: src/github.com/nats-io/nats-server - name: Coveralls # Use commit hash here to avoid a re-tagging attack, as this is a third-party action # Commit 648a8eb78e6d50909eff900e4ec85cab4524a45b = tag v2.3.6 uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b with: github-token: ${{ secrets.github_token }} file: src/github.com/nats-io/nats-server/coverage.lcov nats-server-2.10.27/.github/workflows/go-test.yaml000066400000000000000000000022671477524627100220250ustar00rootroot00000000000000name: NATS Server Testing on: [push, pull_request] jobs: test: strategy: matrix: go: ["1.21"] env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 with: path: src/github.com/nats-io/nats-server - name: Setup Go uses: actions/setup-go@v5 with: go-version: ${{matrix.go}} - name: Install deps shell: bash --noprofile --norc -x -eo pipefail {0} run: | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3 - name: Lint shell: bash --noprofile --norc -x -eo pipefail {0} run: | golangci-lint run - name: Run tests shell: bash --noprofile --norc -x -eo pipefail {0} run: | set -e go test -vet=off -i ./... # go test -vet=off -v -run=TestNoRace --failfast -p=1 ./... # coverage via cov.sh disabled while just testing the waters # Also disable race since we are overwhelming the GHA runners. go test -vet=off -v -p=1 --failfast ./... set +e nats-server-2.10.27/.github/workflows/long-tests.yaml000066400000000000000000000014401477524627100225320ustar00rootroot00000000000000name: NATS Server Long Tests on: # Allow manual trigger (any branch) workflow_dispatch: # Run daily at 12:30 on default branch schedule: - cron: "30 12 * * *" concurrency: # At most one of these workflow per ref running group: ${{ github.workflow }}-${{ github.ref }} # New one cancels in-progress one cancel-in-progress: true jobs: js-long: name: Long JetStream tests runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run tests run: go test -race -v -run='^TestLong.*' ./server -tags=include_js_long_tests -count=1 -vet=off -timeout=60m -shuffle on -p 1 -failfast nats-server-2.10.27/.github/workflows/mqtt-test.yaml000066400000000000000000000030661477524627100224030ustar00rootroot00000000000000name: MQTT External Tests on: [pull_request] jobs: test: env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} steps: - name: Checkout code uses: actions/checkout@v4 with: path: src/github.com/nats-io/nats-server - name: Setup Go uses: actions/setup-go@v5 with: go-version: stable cache-dependency-path: src/github.com/nats-io/nats-server/go.sum - name: Set up testing tools and environment shell: bash --noprofile --norc -eo pipefail {0} id: setup run: | wget https://github.com/hivemq/mqtt-cli/releases/download/v4.20.0/mqtt-cli-4.20.0.deb sudo apt install ./mqtt-cli-4.20.0.deb go install github.com/ConnectEverything/mqtt-test@v0.1.0 - name: Run tests (3 times to detect flappers) shell: bash --noprofile --norc -eo pipefail {0} run: | cd src/github.com/nats-io/nats-server go test -v --count=3 --run='TestXMQTT' ./server - name: Run tests with --race shell: bash --noprofile --norc -eo pipefail {0} run: | cd src/github.com/nats-io/nats-server go test -v --race --failfast --run='TestXMQTT' ./server - name: Run benchmarks shell: bash --noprofile --norc -eo pipefail {0} run: | cd src/github.com/nats-io/nats-server go test --run='-' --count=3 --bench 'BenchmarkXMQTT' --benchtime=100x ./server # TODO: compare benchmarks nats-server-2.10.27/.github/workflows/nightly.yaml000066400000000000000000000014271477524627100221160ustar00rootroot00000000000000name: Docker Nightly on: workflow_dispatch: inputs: target: description: "Override source branch (optional)" type: string required: false schedule: - cron: "40 4 * * *" jobs: run: runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }} steps: - name: Checkout code uses: actions/checkout@v4 with: path: src/github.com/nats-io/nats-server ref: ${{ inputs.target || 'main' }} fetch-depth: 0 fetch-tags: true - uses: ./src/github.com/nats-io/nats-server/.github/actions/nightly-release with: workdir: src/github.com/nats-io/nats-server hub_username: "${{ secrets.DOCKER_USERNAME }}" hub_password: "${{ secrets.DOCKER_PASSWORD }}" nats-server-2.10.27/.github/workflows/release.yaml000066400000000000000000000023311477524627100220530ustar00rootroot00000000000000name: NATS Server Releases on: push: tags: - v* permissions: contents: write jobs: run: name: GitHub Release runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }} steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 fetch-tags: true - name: Set up Go uses: actions/setup-go@v5 with: go-version: "stable" - name: Check version matches tag env: TRAVIS_TAG: ${{ github.ref_name }} run: | go test -race -v -run=TestVersionMatchesTag ./server -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -count=1 -vet=off - name: Install cosign uses: sigstore/cosign-installer@v3.8.1 - name: Install syft uses: anchore/sbom-action/download-syft@v0.18.0 with: syft-version: "v1.20.0" - name: Create release uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser version: "~> v2" args: release --clean env: GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} nats-server-2.10.27/.github/workflows/stale-issues.yaml000066400000000000000000000007721477524627100230630ustar00rootroot00000000000000name: Stale Issues on: schedule: - cron: "30 1 * * *" permissions: issues: write pull-requests: write jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v9 with: stale-issue-label: stale stale-pr-label: stale days-before-stale: 56 # Mark stale after 8 weeks (56 days) of inactivity days-before-close: -1 # Disable auto-closing exempt-all-milestones: true # Any issue/PR within a milestone will be omitted nats-server-2.10.27/.github/workflows/tests.yaml000066400000000000000000000161221477524627100216000ustar00rootroot00000000000000name: NATS Server Tests on: push: pull_request: workflow_dispatch: env: RACE: ${{ (github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') && github.event_name != 'pull_request') && '-race' || '' }} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }} jobs: lint: name: Lint runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} continue-on-error: true steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run golangci-lint uses: golangci/golangci-lint-action@v6 with: version: v1.64.8 skip-cache: true skip-save-cache: true args: --timeout=5m --config=.golangci.yml build-latest: name: Build (Latest Go) runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Build NATS Server run: go build build-supported: name: Build (Minimum Go) runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version-file: "go.mod" - name: Build NATS Server run: go build # Using GitHub-supplied workers for Windows for now. # Note that the below testing steps depend on the Linux build # only, as the Windows builds take a fair bit longer to set up. build-windows: name: Build (Minimum Go, ${{ matrix.os }}) strategy: fail-fast: false matrix: os: [windows-2019, windows-2022] runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version-file: "go.mod" - name: Build NATS Server run: go build store: name: Test Stores needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh store_tests js-no-cluster: name: Test JetStream needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_tests js-cluster-1: name: Test JetStream Cluster 1 needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_cluster_tests_1 js-cluster-2: name: Test JetStream Cluster 2 needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_cluster_tests_2 js-cluster-3: name: Test JetStream Cluster 3 needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_cluster_tests_3 js-cluster-4: name: Test JetStream Cluster 4 needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_cluster_tests_4 js-supercluster: name: Test JetStream Supercluster needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh js_super_cluster_tests no-race: name: Test No-Race needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh no_race_tests mqtt: name: Test MQTT needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh mqtt_tests msgtrace: name: Test Message Tracing needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh msgtrace_tests server-pkg-non-js: name: Test Remaining Server Tests needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh srv_pkg_non_js_tests timeout-minutes: 15 non-server-pkg: name: Test Other Packages needs: [build-latest, build-supported, lint] runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v5 with: go-version: stable - name: Run unit tests run: ./scripts/runTestsOnTravis.sh non_srv_pkg_tests nats-server-2.10.27/.gitignore000066400000000000000000000007521477524627100161470ustar00rootroot00000000000000# Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test dist # Configuration Files *.conf *.cfg # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe # Eclipse .project # IntelliJ .idea/ # Emacs *~ \#*\# .\#* # Visual Studio Code .vscode # Mac .DS_Store # bin nats-server gnatsd check # coverage coverage.out # Cross compiled binaries pkg nats-server-2.10.27/.golangci.yml000066400000000000000000000027671477524627100165530ustar00rootroot00000000000000# Config file for golangci-lint run: concurrency: 4 issues-exit-code: 1 tests: true modules-download-mode: readonly output: formats: - format: colored-line-number path: stdout print-issued-lines: true print-linter-name: true linters: disable-all: true enable: # - errcheck - forbidigo - goimports - gosimple - govet - ineffassign # - maligned - misspell # - prealloc - staticcheck # - unparam - unused linters-settings: errcheck: check-type-assertions: false check-blank: false forbidigo: forbid: - ^fmt\.Print(f|ln)?$ govet: settings: printf: funcs: - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf misspell: locale: US unparam: check-exported: false prealloc: simple: true range-loops: true for-loops: true issues: exclude-dirs: - .github - doc - docker - logos - scripts - util exclude-rules: - path: "main.go" # Excludes main usage linters: [forbidigo] - source: "nats-server: v%s" # Excludes PrintServerAndExit linters: [forbidigo] - path: "server/opts.go" # Excludes TLS usage options linters: [forbidigo] - path: "_test.go" # Excludes unit tests linters: [forbidigo] nats-server-2.10.27/.goreleaser-nightly.yml000066400000000000000000000021741477524627100205640ustar00rootroot00000000000000project_name: nats-server version: 2 builds: - main: . id: nats-server binary: nats-server ldflags: - -w -X github.com/nats-io/nats-server/v2/server.gitCommit={{.ShortCommit}} env: - GO111MODULE=on - CGO_ENABLED=0 goos: - linux goarch: - amd64 mod_timestamp: "{{ .CommitTimestamp }}" release: disable: true dockers: - goos: linux goarch: amd64 dockerfile: docker/Dockerfile.nightly skip_push: false build_flag_templates: - '--build-arg=VERSION={{ if ne .Branch "main" }}{{ replace .Branch "/" "-" }}{{ else }}nightly{{ end }}-{{ time "20060102" }}' image_templates: - synadia/nats-server:{{ if ne .Branch "main" }}{{ replace .Branch "/" "-" }}{{ else }}nightly{{ end }} - synadia/nats-server:{{ if ne .Branch "main" }}{{ replace .Branch "/" "-" }}{{ else }}nightly{{ end }}-{{ time "20060102" }} extra_files: - docker/nats-server.conf checksum: name_template: "SHA256SUMS" algorithm: sha256 snapshot: version_template: '{{ if ne .Branch "main" }}{{ replace .Branch "/" "-" }}{{ else }}nightly{{ end }}-{{ time "20060102" }}' nats-server-2.10.27/.goreleaser.yml000066400000000000000000000045351477524627100171130ustar00rootroot00000000000000project_name: nats-server version: 2 release: github: owner: '{{ envOrDefault "GITHUB_REPOSITORY_OWNER" "nats-io" }}' name: '{{ envOrDefault "GITHUB_REPOSITORY_NAME" "nats-server" }}' name_template: "Release {{.Tag}}" draft: true changelog: disable: true builds: - main: . binary: nats-server flags: - -trimpath ldflags: - -w -X 'github.com/nats-io/nats-server/v2/server.gitCommit={{.ShortCommit}}' -X 'github.com/nats-io/nats-server/v2/server.serverVersion={{.Tag}}' env: # This is the toolchain version we use for releases. To override, set the env var, e.g.: # GORELEASER_TOOLCHAIN="go1.22.8" TARGET='linux_amd64' goreleaser build --snapshot --clean --single-target - GOTOOLCHAIN={{ envOrDefault "GORELEASER_TOOLCHAIN" "go1.24.1" }} - GO111MODULE=on - CGO_ENABLED=0 goos: - darwin - linux - windows - freebsd goarch: - amd64 - arm - arm64 - 386 - mips64le - s390x - ppc64le goarm: - 6 - 7 ignore: - goos: darwin goarch: 386 - goos: freebsd goarch: arm - goos: freebsd goarch: arm64 - goos: freebsd goarch: 386 mod_timestamp: "{{ .CommitTimestamp }}" nfpms: - file_name_template: "{{.ProjectName}}-{{.Tag}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}" homepage: https://nats.io description: High-Performance server for NATS, the cloud native messaging system. maintainer: Ivan Kozlovic license: Apache 2.0 vendor: Synadia Inc. formats: - deb - rpm mtime: "{{ .CommitDate }}" contents: - src: /usr/bin/nats-server dst: /usr/local/bin/nats-server type: "symlink" archives: - name_template: "{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}" id: targz-archives wrap_in_directory: true formats: ["tar.gz"] format_overrides: - goos: windows formats: ["zip"] files: - src: README.md info: mtime: "{{ .CommitDate }}" - src: LICENSE info: mtime: "{{ .CommitDate }}" checksum: name_template: "SHA256SUMS" algorithm: sha256 sboms: - artifacts: binary documents: [ "{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}.sbom.spdx.json", ] nats-server-2.10.27/.travis.yml000066400000000000000000000033311477524627100162640ustar00rootroot00000000000000os: linux dist: focal vm: size: 2x-large language: go go: # This should be quoted or use .x, but should not be unquoted. # Remember that a YAML bare float drops trailing zeroes. - "1.23.6" go_import_path: github.com/nats-io/nats-server addons: apt: packages: - rpm jobs: include: - name: "Compile and various other checks" env: TEST_SUITE=compile - name: "Run TestNoRace tests" env: TEST_SUITE=no_race_tests - name: "Run Store tests" env: TEST_SUITE=store_tests - name: "Run JetStream tests" env: TEST_SUITE=js_tests - name: "Run JetStream cluster tests (1)" env: TEST_SUITE=js_cluster_tests_1 - name: "Run JetStream cluster tests (2)" env: TEST_SUITE=js_cluster_tests_2 - name: "Run JetStream cluster tests (3)" env: TEST_SUITE=js_cluster_tests_3 - name: "Run JetStream cluster tests (4)" env: TEST_SUITE=js_cluster_tests_4 - name: "Run JetStream super cluster tests" env: TEST_SUITE=js_super_cluster_tests - name: "Run MQTT tests" env: TEST_SUITE=mqtt_tests - name: "Run Message Tracing tests" env: TEST_SUITE=msgtrace_tests - name: "Run all other tests from the server package" env: TEST_SUITE=srv_pkg_non_js_tests - name: "Run all tests from all other packages" env: TEST_SUITE=non_srv_pkg_tests script: ./scripts/runTestsOnTravis.sh $TEST_SUITE deploy: provider: script cleanup: true script: curl -o /tmp/goreleaser.tar.gz -sLf https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/goreleaser_Linux_x86_64.tar.gz && tar -xvf /tmp/goreleaser.tar.gz -C /tmp/ && /tmp/goreleaser on: tags: true condition: ($TRAVIS_GO_VERSION =~ 1.23) && ($TEST_SUITE = "compile") nats-server-2.10.27/CODE-OF-CONDUCT.md000066400000000000000000000002121477524627100166010ustar00rootroot00000000000000## Community Code of Conduct NATS follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). nats-server-2.10.27/CONTRIBUTING.md000066400000000000000000000066601477524627100164140ustar00rootroot00000000000000# Contributing Thanks for your interest in contributing! This document contains `nats-io/nats-server` specific contributing details. If you are a first-time contributor, please refer to the general [NATS Contributor Guide](https://nats.io/contributing/) to get a comprehensive overview of contributing to the NATS project. ## Getting started There are there general ways you can contribute to this repo: - Proposing an enhancement or new feature - Reporting a bug or regression - Contributing changes to the source code For the first two, refer to the [GitHub Issues](https://github.com/nats-io/nats-server/issues/new/choose) which guides you through the available options along with the needed information to collect. ## Contributing changes _Prior to opening a pull request, it is recommended to open an issue first to ensure the maintainers can review intended changes. Exceptions to this rule include fixing non-functional source such as code comments, documentation or other supporting files._ Proposing source code changes is done through GitHub's standard pull request workflow. If your branch is a work-in-progress then please start by creating your pull requests as draft, by clicking the down-arrow next to the `Create pull request` button and instead selecting `Create draft pull request`. This will defer the automatic process of requesting a review from the NATS team and significantly reduces noise until you are ready. Once you are happy, you can click the `Ready for review` button. ### Guidelines A good pull request includes: - A high-level description of the changes, including links to any issues that are related by adding comments like `Resolves #NNN` to your description. See [Linking a Pull Request to an Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) for more information. - An up-to-date parent commit. Please make sure you are pulling in the latest `main` branch and rebasing your work on top of it, i.e. `git rebase main`. - Unit tests where appropriate. Bug fixes will benefit from the addition of regression tests. New features will not be accepted without suitable test coverage! - No more commits than necessary. Sometimes having multiple commits is useful for telling a story or isolating changes from one another, but please squash down any unnecessary commits that may just be for clean-up, comments or small changes. - No additional external dependencies that aren't absolutely essential. Please do everything you can to avoid pulling in additional libraries/dependencies into `go.mod` as we will be very critical of these. ### Sign-off In order to accept a contribution, you will first need to certify that the contribution is your original work and that you license the work to the project under the [Apache-2.0 license](https://github.com/nats-io/nats-server/blob/main/LICENSE). This is done by using `Signed-off-by` statements, which should appear in **both** your commit messages and your PR description. Please note that we can only accept sign-offs under a legal name. Nicknames and aliases are not permitted. To perform a sign-off with `git`, use `git commit -s` (or `--signoff`). ## Get help If you have questions about the contribution process, please start a [GitHub discussion](https://github.com/nats-io/nats-server/discussions), join the [NATS Slack](https://slack.nats.io/), or send your question to the [NATS Google Group](https://groups.google.com/forum/#!forum/natsio). nats-server-2.10.27/DEPENDENCIES.md000066400000000000000000000021471477524627100163270ustar00rootroot00000000000000# External Dependencies This file lists the dependencies used in this repository. | Dependency | License | |-|-| | Go | BSD 3-Clause "New" or "Revised" License | | github.com/nats-io/nats-server/v2 | Apache License 2.0 | | github.com/golang/protobuf v1.4.2 | BSD 3-Clause "New" or "Revised" License | | github.com/nats-io/jwt/v2 v2.5.2 | Apache License 2.0 | | github.com/nats-io/nats.go v1.30.2 | Apache License 2.0 | | github.com/nats-io/nkeys v0.4.5 | Apache License 2.0 | | github.com/nats-io/nuid v1.0.1 | Apache License 2.0 | | golang.org/x/crypto v0.13.0 | BSD 3-Clause "New" or "Revised" License | | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 | BSD 3-Clause "New" or "Revised" License | | golang.org/x/sys v0.12.0 | BSD 3-Clause "New" or "Revised" License | | golang.org/x/time v0.3.0 | BSD 3-Clause "New" or "Revised" License | | google.golang.org/protobuf v1.23.0 | BSD 3-Clause "New" or "Revised" License | | github.com/klauspost/compress/s2 v1.17.0 | BSD 3-Clause "New" or "Revised" License | | github.com/minio/highwayhash v1.0.2 | Apache License 2.0 | | go.uber.org/automaxprocs v1.5.3 | MIT License | nats-server-2.10.27/GOVERNANCE.md000066400000000000000000000002541477524627100161250ustar00rootroot00000000000000# NATS Server Governance NATS Server is part of the NATS project and is subject to the [NATS Governance](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md). nats-server-2.10.27/LICENSE000066400000000000000000000261351477524627100151670ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. nats-server-2.10.27/MAINTAINERS.md000066400000000000000000000011001477524627100162370ustar00rootroot00000000000000# Maintainers Maintainership is on a per project basis. Reference [NATS Governance Model](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md). ### Maintainers - Derek Collison [@derekcollison](https://github.com/derekcollison) - Ivan Kozlovic [@kozlovic](https://github.com/kozlovic) - Waldemar Quevedo [@wallyqs](https://github.com/wallyqs) - Oleg Shaldybin [@olegshaldybin](https://github.com/olegshaldybin) - R.I. Pienaar [@ripienaar](https://github.com/ripienaar) nats-server-2.10.27/README.md000066400000000000000000000100661477524627100154350ustar00rootroot00000000000000

NATS Logo

[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)). NATS has over [40 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems. [![License][License-Image]][License-Url] [![Build][Build-Status-Image]][Build-Status-Url] [![Release][Release-Image]][Release-Url] [![Slack][Slack-Image]][Slack-Url] [![Coverage][Coverage-Image]][Coverage-Url] [![Docker Downloads][Docker-Image]][Docker-Url] [![CII Best Practices][CIIBestPractices-Image]][CIIBestPractices-Url] [![Artifact Hub][ArtifactHub-Image]][ArtifactHub-Url] ## Documentation - [Official Website](https://nats.io) - [Official Documentation](https://docs.nats.io) - [FAQ](https://docs.nats.io/reference/faq) - Watch [a video overview](https://rethink.synadia.com/episodes/1/) of NATS. - Watch [this video from SCALE 13x](https://www.youtube.com/watch?v=sm63oAVPqAM) to learn more about its origin story and design philosophy. ## Contact - [Twitter](https://twitter.com/nats_io): Follow us on Twitter! - [Google Groups](https://groups.google.com/forum/#!forum/natsio): Where you can ask questions - [Slack](https://natsio.slack.com): Click [here](https://slack.nats.io) to join. You can ask question to our maintainers and to the rich and active community. ## Contributing If you are interested in contributing to NATS, read about our... - [Contributing guide](./CONTRIBUTING.md) - [Report issues or propose Pull Requests](https://github.com/nats-io) [License-Url]: https://www.apache.org/licenses/LICENSE-2.0 [License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg [Docker-Image]: https://img.shields.io/docker/pulls/_/nats.svg [Docker-Url]: https://hub.docker.com/_/nats [Slack-Image]: https://img.shields.io/badge/chat-on%20slack-green [Slack-Url]: https://slack.nats.io [Fossa-Url]: https://app.fossa.io/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server?ref=badge_shield [Fossa-Image]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server.svg?type=shield [Build-Status-Url]: https://travis-ci.com/github/nats-io/nats-server [Build-Status-Image]: https://travis-ci.com/nats-io/nats-server.svg?branch=main [Release-Url]: https://github.com/nats-io/nats-server/releases/tag/v2.10.23 [Release-image]: https://img.shields.io/badge/release-v2.10.23-1eb0fc.svg [Coverage-Url]: https://coveralls.io/r/nats-io/nats-server?branch=main [Coverage-image]: https://coveralls.io/repos/github/nats-io/nats-server/badge.svg?branch=main [ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-server [ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-server [CIIBestPractices-Url]: https://bestpractices.coreinfrastructure.org/projects/1895 [CIIBestPractices-Image]: https://bestpractices.coreinfrastructure.org/projects/1895/badge [ArtifactHub-Url]: https://artifacthub.io/packages/helm/nats/nats [ArtifactHub-Image]: https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/nats [github-release]: https://github.com/nats-io/nats-server/releases/ ## Roadmap The NATS product roadmap can be found [here](https://nats.io/about/#roadmap). ## Adopters Who uses NATS? See our [list of users](https://nats.io/#who-uses-nats) on [https://nats.io](https://nats.io). ## Security ### Security Audit A third party security audit was performed by Cure53, you can see the full report [here](https://github.com/nats-io/nats-general/blob/main/reports/Cure53_NATS_Audit.pdf). ### Reporting Security Vulnerabilities If you've found a vulnerability or a potential vulnerability in the NATS server, please let us know at [nats-security](mailto:security@nats.io). ## License Unless otherwise noted, the NATS source files are distributed under the Apache Version 2.0 license found in the LICENSE file. nats-server-2.10.27/TODO.md000066400000000000000000000043161477524627100152460ustar00rootroot00000000000000 # General - [ ] Auth for queue groups? - [ ] Blacklist or ERR escalation to close connection for auth/permissions - [ ] Protocol updates, MAP, MPUB, etc - [ ] Multiple listen endpoints - [ ] Websocket / HTTP2 strategy - [ ] T series reservations - [ ] _SYS. server events? - [ ] No downtime restart - [ ] Signal based reload of configuration - [ ] brew, apt-get, rpm, chocately (windows) - [ ] IOVec pools and writev for high fanout? - [ ] Modify cluster support for single message across routes between pub/sub and d-queue - [ ] Memory limits/warnings? - [ ] Limit number of subscriptions a client can have, total memory usage etc. - [ ] Multi-tenant accounts with isolation of subject space - [ ] Pedantic state - [X] _SYS.> reserved for server events? - [X] Listen configure key vs addr and port - [X] Add ENV and variable support to dconf? ucl? - [X] Buffer pools/sync pools? - [X] Multiple Authorization / Access - [X] Write dynamic socket buffer sizes - [X] Read dynamic socket buffer sizes - [X] Info updates contain other implicit route servers - [X] Sublist better at high concurrency, cache uses writelock always currently - [X] Switch to 1.4/1.5 and use maps vs hashmaps in sublist - [X] NewSource on Rand to lower lock contention on QueueSubs, or redesign! - [X] Default sort by cid on connz - [X] Track last activity time per connection? - [X] Add total connections to varz so we won't miss spikes, etc. - [X] Add starttime and uptime to connz list. - [X] Gossip Protocol for discovery for clustering - [X] Add in HTTP requests to varz? - [X] Add favico and help link for monitoring? - [X] Better user/pass support using bcrypt etc. - [X] SSL/TLS support - [X] Add support for / to point to varz, connz, etc.. - [X] Support sort options for /connz via nats-top - [X] Dropped message statistics (slow consumers) - [X] Add current time to each monitoring endpoint - [X] varz uptime do days and only integer secs - [X] Place version in varz (same info sent to clients) - [X] Place server ID/UUID in varz - [X] nats-top equivalent, utils - [X] Connz report routes (/routez) - [X] Docker - [X] Remove reliance on `ps` - [X] Syslog support - [X] Client support for language and version - [X] Fix benchmarks on linux - [X] Daemon mode? Won't fix nats-server-2.10.27/conf/000077500000000000000000000000001477524627100151005ustar00rootroot00000000000000nats-server-2.10.27/conf/fuzz.go000066400000000000000000000013511477524627100164250ustar00rootroot00000000000000// Copyright 2020-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build gofuzz // +build gofuzz package conf func Fuzz(data []byte) int { _, err := Parse(string(data)) if err != nil { return 0 } return 1 } nats-server-2.10.27/conf/includes/000077500000000000000000000000001477524627100167065ustar00rootroot00000000000000nats-server-2.10.27/conf/includes/passwords.conf000066400000000000000000000002551477524627100216040ustar00rootroot00000000000000# Just foo & bar for testing ALICE_PASS: $2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q BOB_PASS: $2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly nats-server-2.10.27/conf/includes/users.conf000066400000000000000000000002121477524627100207110ustar00rootroot00000000000000# Users configuration include ./passwords.conf; users = [ {user: alice, password: $ALICE_PASS} {user: bob, password: $BOB_PASS} ] nats-server-2.10.27/conf/lex.go000066400000000000000000001015771477524627100162320ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Customized heavily from // https://github.com/BurntSushi/toml/blob/master/lex.go, which is based on // Rob Pike's talk: http://cuddle.googlecode.com/hg/talk/lex.html // The format supported is less restrictive than today's formats. // Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //) // Also supports key value assignments using '=' or ':' or whiteSpace() // e.g. foo = 2, foo : 2, foo 2 // maps can be assigned with no key separator as well // semicolons as value terminators in key/value assignments are optional // // see lex_test.go for more examples. package conf import ( "encoding/hex" "fmt" "strings" "unicode" "unicode/utf8" ) type itemType int const ( itemError itemType = iota itemNIL // used in the parser to indicate no type itemEOF itemKey itemText itemString itemBool itemInteger itemFloat itemDatetime itemArrayStart itemArrayEnd itemMapStart itemMapEnd itemCommentStart itemVariable itemInclude ) const ( eof = 0 mapStart = '{' mapEnd = '}' keySepEqual = '=' keySepColon = ':' arrayStart = '[' arrayEnd = ']' arrayValTerm = ',' mapValTerm = ',' commentHashStart = '#' commentSlashStart = '/' dqStringStart = '"' dqStringEnd = '"' sqStringStart = '\'' sqStringEnd = '\'' optValTerm = ';' topOptStart = '{' topOptValTerm = ',' topOptTerm = '}' blockStart = '(' blockEnd = ')' mapEndString = string(mapEnd) ) type stateFn func(lx *lexer) stateFn type lexer struct { input string start int pos int width int line int state stateFn items chan item // A stack of state functions used to maintain context. // The idea is to reuse parts of the state machine in various places. // For example, values can appear at the top level or within arbitrarily // nested arrays. The last state on the stack is used after a value has // been lexed. Similarly for comments. stack []stateFn // Used for processing escapable substrings in double-quoted and raw strings stringParts []string stringStateFn stateFn // lstart is the start position of the current line. lstart int // ilstart is the start position of the line from the current item. ilstart int } type item struct { typ itemType val string line int pos int } func (lx *lexer) nextItem() item { for { select { case item := <-lx.items: return item default: lx.state = lx.state(lx) } } } func lex(input string) *lexer { lx := &lexer{ input: input, state: lexTop, line: 1, items: make(chan item, 10), stack: make([]stateFn, 0, 10), stringParts: []string{}, } return lx } func (lx *lexer) push(state stateFn) { lx.stack = append(lx.stack, state) } func (lx *lexer) pop() stateFn { if len(lx.stack) == 0 { return lx.errorf("BUG in lexer: no states to pop.") } li := len(lx.stack) - 1 last := lx.stack[li] lx.stack = lx.stack[0:li] return last } func (lx *lexer) emit(typ itemType) { val := strings.Join(lx.stringParts, "") + lx.input[lx.start:lx.pos] // Position of item in line where it started. pos := lx.pos - lx.ilstart - len(val) lx.items <- item{typ, val, lx.line, pos} lx.start = lx.pos lx.ilstart = lx.lstart } func (lx *lexer) emitString() { var finalString string if len(lx.stringParts) > 0 { finalString = strings.Join(lx.stringParts, "") + lx.input[lx.start:lx.pos] lx.stringParts = []string{} } else { finalString = lx.input[lx.start:lx.pos] } // Position of string in line where it started. pos := lx.pos - lx.ilstart - len(finalString) lx.items <- item{itemString, finalString, lx.line, pos} lx.start = lx.pos lx.ilstart = lx.lstart } func (lx *lexer) addCurrentStringPart(offset int) { lx.stringParts = append(lx.stringParts, lx.input[lx.start:lx.pos-offset]) lx.start = lx.pos } func (lx *lexer) addStringPart(s string) stateFn { lx.stringParts = append(lx.stringParts, s) lx.start = lx.pos return lx.stringStateFn } func (lx *lexer) hasEscapedParts() bool { return len(lx.stringParts) > 0 } func (lx *lexer) next() (r rune) { if lx.pos >= len(lx.input) { lx.width = 0 return eof } if lx.input[lx.pos] == '\n' { lx.line++ // Mark start position of current line. lx.lstart = lx.pos } r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) lx.pos += lx.width return r } // ignore skips over the pending input before this point. func (lx *lexer) ignore() { lx.start = lx.pos lx.ilstart = lx.lstart } // backup steps back one rune. Can be called only once per call of next. func (lx *lexer) backup() { lx.pos -= lx.width if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { lx.line-- } } // peek returns but does not consume the next rune in the input. func (lx *lexer) peek() rune { r := lx.next() lx.backup() return r } // errorf stops all lexing by emitting an error and returning `nil`. // Note that any value that is a character is escaped if it's a special // character (new lines, tabs, etc.). func (lx *lexer) errorf(format string, values ...any) stateFn { for i, value := range values { if v, ok := value.(rune); ok { values[i] = escapeSpecial(v) } } // Position of error in current line. pos := lx.pos - lx.lstart lx.items <- item{ itemError, fmt.Sprintf(format, values...), lx.line, pos, } return nil } // lexTop consumes elements at the top level of data structure. func lexTop(lx *lexer) stateFn { r := lx.next() if unicode.IsSpace(r) { return lexSkip(lx, lexTop) } switch r { case topOptStart: lx.push(lexTop) return lexSkip(lx, lexBlockStart) case commentHashStart: lx.push(lexTop) return lexCommentStart case commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexTop) return lexCommentStart } lx.backup() fallthrough case eof: if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } // At this point, the only valid item can be a key, so we back up // and let the key lexer do the rest. lx.backup() lx.push(lexTopValueEnd) return lexKeyStart } // lexTopValueEnd is entered whenever a top-level value has been consumed. // It must see only whitespace, and will turn back to lexTop upon a new line. // If it sees EOF, it will quit the lexer successfully. func lexTopValueEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentHashStart: // a comment will read to a new line for us. lx.push(lexTop) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexTop) return lexCommentStart } lx.backup() fallthrough case isWhitespace(r): return lexTopValueEnd case isNL(r) || r == eof || r == optValTerm || r == topOptValTerm || r == topOptTerm: lx.ignore() return lexTop } return lx.errorf("Expected a top-level value to end with a new line, "+ "comment or EOF, but got '%v' instead.", r) } func lexBlockStart(lx *lexer) stateFn { r := lx.next() if unicode.IsSpace(r) { return lexSkip(lx, lexBlockStart) } switch r { case topOptStart: lx.push(lexBlockEnd) return lexSkip(lx, lexBlockStart) case topOptTerm: lx.ignore() return lx.pop() case commentHashStart: lx.push(lexBlockStart) return lexCommentStart case commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexBlockStart) return lexCommentStart } lx.backup() fallthrough case eof: if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } // At this point, the only valid item can be a key, so we back up // and let the key lexer do the rest. lx.backup() lx.push(lexBlockValueEnd) return lexKeyStart } // lexBlockValueEnd is entered whenever a block-level value has been consumed. // It must see only whitespace, and will turn back to lexBlockStart upon a new line. // If it sees EOF, it will quit the lexer successfully. func lexBlockValueEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentHashStart: // a comment will read to a new line for us. lx.push(lexBlockValueEnd) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexBlockValueEnd) return lexCommentStart } lx.backup() fallthrough case isWhitespace(r): return lexBlockValueEnd case isNL(r) || r == optValTerm || r == topOptValTerm: lx.ignore() return lexBlockStart case r == topOptTerm: lx.backup() return lexBlockEnd } return lx.errorf("Expected a block-level value to end with a new line, "+ "comment or EOF, but got '%v' instead.", r) } // lexBlockEnd is entered whenever a block-level value has been consumed. // It must see only whitespace, and will turn back to lexTop upon a "}". func lexBlockEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentHashStart: // a comment will read to a new line for us. lx.push(lexBlockStart) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexBlockStart) return lexCommentStart } lx.backup() fallthrough case isNL(r) || isWhitespace(r): return lexBlockEnd case r == optValTerm || r == topOptValTerm: lx.ignore() return lexBlockStart case r == topOptTerm: lx.ignore() return lx.pop() } return lx.errorf("Expected a block-level to end with a '}', but got '%v' instead.", r) } // lexKeyStart consumes a key name up until the first non-whitespace character. // lexKeyStart will ignore whitespace. It will also eat enclosing quotes. func lexKeyStart(lx *lexer) stateFn { r := lx.peek() switch { case isKeySeparator(r): return lx.errorf("Unexpected key separator '%v'", r) case unicode.IsSpace(r): lx.next() return lexSkip(lx, lexKeyStart) case r == dqStringStart: lx.next() return lexSkip(lx, lexDubQuotedKey) case r == sqStringStart: lx.next() return lexSkip(lx, lexQuotedKey) } lx.ignore() lx.next() return lexKey } // lexDubQuotedKey consumes the text of a key between quotes. func lexDubQuotedKey(lx *lexer) stateFn { r := lx.peek() if r == dqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexKeyEnd) } else if r == eof { if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } lx.next() return lexDubQuotedKey } // lexQuotedKey consumes the text of a key between quotes. func lexQuotedKey(lx *lexer) stateFn { r := lx.peek() if r == sqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexKeyEnd) } else if r == eof { if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } lx.next() return lexQuotedKey } // keyCheckKeyword will check for reserved keywords as the key value when the key is // separated with a space. func (lx *lexer) keyCheckKeyword(fallThrough, push stateFn) stateFn { key := strings.ToLower(lx.input[lx.start:lx.pos]) switch key { case "include": lx.ignore() if push != nil { lx.push(push) } return lexIncludeStart } lx.emit(itemKey) return fallThrough } // lexIncludeStart will consume the whitespace til the start of the value. func lexIncludeStart(lx *lexer) stateFn { r := lx.next() if isWhitespace(r) { return lexSkip(lx, lexIncludeStart) } lx.backup() return lexInclude } // lexIncludeQuotedString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. It will not interpret any // internal contents. func lexIncludeQuotedString(lx *lexer) stateFn { r := lx.next() switch { case r == sqStringEnd: lx.backup() lx.emit(itemInclude) lx.next() lx.ignore() return lx.pop() case r == eof: return lx.errorf("Unexpected EOF in quoted include") } return lexIncludeQuotedString } // lexIncludeDubQuotedString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. It will not interpret any // internal contents. func lexIncludeDubQuotedString(lx *lexer) stateFn { r := lx.next() switch { case r == dqStringEnd: lx.backup() lx.emit(itemInclude) lx.next() lx.ignore() return lx.pop() case r == eof: return lx.errorf("Unexpected EOF in double quoted include") } return lexIncludeDubQuotedString } // lexIncludeString consumes the inner contents of a raw string. func lexIncludeString(lx *lexer) stateFn { r := lx.next() switch { case isNL(r) || r == eof || r == optValTerm || r == mapEnd || isWhitespace(r): lx.backup() lx.emit(itemInclude) return lx.pop() case r == sqStringEnd: lx.backup() lx.emit(itemInclude) lx.next() lx.ignore() return lx.pop() } return lexIncludeString } // lexInclude will consume the include value. func lexInclude(lx *lexer) stateFn { r := lx.next() switch { case r == sqStringStart: lx.ignore() // ignore the " or ' return lexIncludeQuotedString case r == dqStringStart: lx.ignore() // ignore the " or ' return lexIncludeDubQuotedString case r == arrayStart: return lx.errorf("Expected include value but found start of an array") case r == mapStart: return lx.errorf("Expected include value but found start of a map") case r == blockStart: return lx.errorf("Expected include value but found start of a block") case unicode.IsDigit(r), r == '-': return lx.errorf("Expected include value but found start of a number") case r == '\\': return lx.errorf("Expected include value but found escape sequence") case isNL(r): return lx.errorf("Expected include value but found new line") } lx.backup() return lexIncludeString } // lexKey consumes the text of a key. Assumes that the first character (which // is not whitespace) has already been consumed. func lexKey(lx *lexer) stateFn { r := lx.peek() if unicode.IsSpace(r) { // Spaces signal we could be looking at a keyword, e.g. include. // Keywords will eat the keyword and set the appropriate return stateFn. return lx.keyCheckKeyword(lexKeyEnd, nil) } else if isKeySeparator(r) || r == eof { lx.emit(itemKey) return lexKeyEnd } lx.next() return lexKey } // lexKeyEnd consumes the end of a key (up to the key separator). // Assumes that the first whitespace character after a key (or the '=' or ':' // separator) has NOT been consumed. func lexKeyEnd(lx *lexer) stateFn { r := lx.next() switch { case unicode.IsSpace(r): return lexSkip(lx, lexKeyEnd) case isKeySeparator(r): return lexSkip(lx, lexValue) case r == eof: lx.emit(itemEOF) return nil } // We start the value here lx.backup() return lexValue } // lexValue starts the consumption of a value anywhere a value is expected. // lexValue will ignore whitespace. // After a value is lexed, the last state on the next is popped and returned. func lexValue(lx *lexer) stateFn { // We allow whitespace to precede a value, but NOT new lines. // In array syntax, the array states are responsible for ignoring new lines. r := lx.next() if isWhitespace(r) { return lexSkip(lx, lexValue) } switch { case r == arrayStart: lx.ignore() lx.emit(itemArrayStart) return lexArrayValue case r == mapStart: lx.ignore() lx.emit(itemMapStart) return lexMapKeyStart case r == sqStringStart: lx.ignore() // ignore the " or ' return lexQuotedString case r == dqStringStart: lx.ignore() // ignore the " or ' lx.stringStateFn = lexDubQuotedString return lexDubQuotedString case r == '-': return lexNegNumberStart case r == blockStart: lx.ignore() return lexBlock case unicode.IsDigit(r): lx.backup() // avoid an extra state and use the same as above return lexNumberOrDateOrStringOrIPStart case r == '.': // special error case, be kind to users return lx.errorf("Floats must start with a digit") case isNL(r): return lx.errorf("Expected value but found new line") } lx.backup() lx.stringStateFn = lexString return lexString } // lexArrayValue consumes one value in an array. It assumes that '[' or ',' // have already been consumed. All whitespace and new lines are ignored. func lexArrayValue(lx *lexer) stateFn { r := lx.next() switch { case unicode.IsSpace(r): return lexSkip(lx, lexArrayValue) case r == commentHashStart: lx.push(lexArrayValue) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexArrayValue) return lexCommentStart } lx.backup() fallthrough case r == arrayValTerm: return lx.errorf("Unexpected array value terminator '%v'.", arrayValTerm) case r == arrayEnd: return lexArrayEnd } lx.backup() lx.push(lexArrayValueEnd) return lexValue } // lexArrayValueEnd consumes the cruft between values of an array. Namely, // it ignores whitespace and expects either a ',' or a ']'. func lexArrayValueEnd(lx *lexer) stateFn { r := lx.next() switch { case isWhitespace(r): return lexSkip(lx, lexArrayValueEnd) case r == commentHashStart: lx.push(lexArrayValueEnd) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexArrayValueEnd) return lexCommentStart } lx.backup() fallthrough case r == arrayValTerm || isNL(r): return lexSkip(lx, lexArrayValue) // Move onto next case r == arrayEnd: return lexArrayEnd } return lx.errorf("Expected an array value terminator %q or an array "+ "terminator %q, but got '%v' instead.", arrayValTerm, arrayEnd, r) } // lexArrayEnd finishes the lexing of an array. It assumes that a ']' has // just been consumed. func lexArrayEnd(lx *lexer) stateFn { lx.ignore() lx.emit(itemArrayEnd) return lx.pop() } // lexMapKeyStart consumes a key name up until the first non-whitespace // character. // lexMapKeyStart will ignore whitespace. func lexMapKeyStart(lx *lexer) stateFn { r := lx.peek() switch { case isKeySeparator(r): return lx.errorf("Unexpected key separator '%v'.", r) case r == arrayEnd: return lx.errorf("Unexpected array end '%v' processing map.", r) case unicode.IsSpace(r): lx.next() return lexSkip(lx, lexMapKeyStart) case r == mapEnd: lx.next() return lexSkip(lx, lexMapEnd) case r == commentHashStart: lx.next() lx.push(lexMapKeyStart) return lexCommentStart case r == commentSlashStart: lx.next() rn := lx.next() if rn == commentSlashStart { lx.push(lexMapKeyStart) return lexCommentStart } lx.backup() case r == sqStringStart: lx.next() return lexSkip(lx, lexMapQuotedKey) case r == dqStringStart: lx.next() return lexSkip(lx, lexMapDubQuotedKey) case r == eof: return lx.errorf("Unexpected EOF processing map.") } lx.ignore() lx.next() return lexMapKey } // lexMapQuotedKey consumes the text of a key between quotes. func lexMapQuotedKey(lx *lexer) stateFn { if r := lx.peek(); r == eof { return lx.errorf("Unexpected EOF processing quoted map key.") } else if r == sqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexMapKeyEnd) } lx.next() return lexMapQuotedKey } // lexMapDubQuotedKey consumes the text of a key between quotes. func lexMapDubQuotedKey(lx *lexer) stateFn { if r := lx.peek(); r == eof { return lx.errorf("Unexpected EOF processing double quoted map key.") } else if r == dqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexMapKeyEnd) } lx.next() return lexMapDubQuotedKey } // lexMapKey consumes the text of a key. Assumes that the first character (which // is not whitespace) has already been consumed. func lexMapKey(lx *lexer) stateFn { if r := lx.peek(); r == eof { return lx.errorf("Unexpected EOF processing map key.") } else if unicode.IsSpace(r) { // Spaces signal we could be looking at a keyword, e.g. include. // Keywords will eat the keyword and set the appropriate return stateFn. return lx.keyCheckKeyword(lexMapKeyEnd, lexMapValueEnd) } else if isKeySeparator(r) { lx.emit(itemKey) return lexMapKeyEnd } lx.next() return lexMapKey } // lexMapKeyEnd consumes the end of a key (up to the key separator). // Assumes that the first whitespace character after a key (or the '=' // separator) has NOT been consumed. func lexMapKeyEnd(lx *lexer) stateFn { r := lx.next() switch { case unicode.IsSpace(r): return lexSkip(lx, lexMapKeyEnd) case isKeySeparator(r): return lexSkip(lx, lexMapValue) } // We start the value here lx.backup() return lexMapValue } // lexMapValue consumes one value in a map. It assumes that '{' or ',' // have already been consumed. All whitespace and new lines are ignored. // Map values can be separated by ',' or simple NLs. func lexMapValue(lx *lexer) stateFn { r := lx.next() switch { case unicode.IsSpace(r): return lexSkip(lx, lexMapValue) case r == mapValTerm: return lx.errorf("Unexpected map value terminator %q.", mapValTerm) case r == mapEnd: return lexSkip(lx, lexMapEnd) } lx.backup() lx.push(lexMapValueEnd) return lexValue } // lexMapValueEnd consumes the cruft between values of a map. Namely, // it ignores whitespace and expects either a ',' or a '}'. func lexMapValueEnd(lx *lexer) stateFn { r := lx.next() switch { case isWhitespace(r): return lexSkip(lx, lexMapValueEnd) case r == commentHashStart: lx.push(lexMapValueEnd) return lexCommentStart case r == commentSlashStart: rn := lx.next() if rn == commentSlashStart { lx.push(lexMapValueEnd) return lexCommentStart } lx.backup() fallthrough case r == optValTerm || r == mapValTerm || isNL(r): return lexSkip(lx, lexMapKeyStart) // Move onto next case r == mapEnd: return lexSkip(lx, lexMapEnd) } return lx.errorf("Expected a map value terminator %q or a map "+ "terminator %q, but got '%v' instead.", mapValTerm, mapEnd, r) } // lexMapEnd finishes the lexing of a map. It assumes that a '}' has // just been consumed. func lexMapEnd(lx *lexer) stateFn { lx.ignore() lx.emit(itemMapEnd) return lx.pop() } // Checks if the unquoted string was actually a boolean func (lx *lexer) isBool() bool { str := strings.ToLower(lx.input[lx.start:lx.pos]) return str == "true" || str == "false" || str == "on" || str == "off" || str == "yes" || str == "no" } // Check if the unquoted string is a variable reference, starting with $. func (lx *lexer) isVariable() bool { if lx.start >= len(lx.input) { return false } if lx.input[lx.start] == '$' { lx.start += 1 return true } return false } // lexQuotedString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. It will not interpret any // internal contents. func lexQuotedString(lx *lexer) stateFn { r := lx.next() switch { case r == sqStringEnd: lx.backup() lx.emit(itemString) lx.next() lx.ignore() return lx.pop() case r == eof: if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } return lexQuotedString } // lexDubQuotedString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. It will not interpret any // internal contents. func lexDubQuotedString(lx *lexer) stateFn { r := lx.next() switch { case r == '\\': lx.addCurrentStringPart(1) return lexStringEscape case r == dqStringEnd: lx.backup() lx.emitString() lx.next() lx.ignore() return lx.pop() case r == eof: if lx.pos > lx.start { return lx.errorf("Unexpected EOF.") } lx.emit(itemEOF) return nil } return lexDubQuotedString } // lexString consumes the inner contents of a raw string. func lexString(lx *lexer) stateFn { r := lx.next() switch { case r == '\\': lx.addCurrentStringPart(1) return lexStringEscape // Termination of non-quoted strings case isNL(r) || r == eof || r == optValTerm || r == arrayValTerm || r == arrayEnd || r == mapEnd || isWhitespace(r): lx.backup() if lx.hasEscapedParts() { lx.emitString() } else if lx.isBool() { lx.emit(itemBool) } else if lx.isVariable() { lx.emit(itemVariable) } else { lx.emitString() } return lx.pop() case r == sqStringEnd: lx.backup() lx.emitString() lx.next() lx.ignore() return lx.pop() } return lexString } // lexBlock consumes the inner contents as a string. It assumes that the // beginning '(' has already been consumed and ignored. It will continue // processing until it finds a ')' on a new line by itself. func lexBlock(lx *lexer) stateFn { r := lx.next() switch { case r == blockEnd: lx.backup() lx.backup() // Looking for a ')' character on a line by itself, if the previous // character isn't a new line, then break so we keep processing the block. if lx.next() != '\n' { lx.next() break } lx.next() // Make sure the next character is a new line or an eof. We want a ')' on a // bare line by itself. switch lx.next() { case '\n', eof: lx.backup() lx.backup() lx.emit(itemString) lx.next() lx.ignore() return lx.pop() } lx.backup() case r == eof: return lx.errorf("Unexpected EOF processing block.") } return lexBlock } // lexStringEscape consumes an escaped character. It assumes that the preceding // '\\' has already been consumed. func lexStringEscape(lx *lexer) stateFn { r := lx.next() switch r { case 'x': return lexStringBinary case 't': return lx.addStringPart("\t") case 'n': return lx.addStringPart("\n") case 'r': return lx.addStringPart("\r") case '"': return lx.addStringPart("\"") case '\\': return lx.addStringPart("\\") } return lx.errorf("Invalid escape character '%v'. Only the following "+ "escape characters are allowed: \\xXX, \\t, \\n, \\r, \\\", \\\\.", r) } // lexStringBinary consumes two hexadecimal digits following '\x'. It assumes // that the '\x' has already been consumed. func lexStringBinary(lx *lexer) stateFn { r := lx.next() if isNL(r) { return lx.errorf("Expected two hexadecimal digits after '\\x', but hit end of line") } r = lx.next() if isNL(r) { return lx.errorf("Expected two hexadecimal digits after '\\x', but hit end of line") } offset := lx.pos - 2 byteString, err := hex.DecodeString(lx.input[offset:lx.pos]) if err != nil { return lx.errorf("Expected two hexadecimal digits after '\\x', but got '%s'", lx.input[offset:lx.pos]) } lx.addStringPart(string(byteString)) return lx.stringStateFn } // lexNumberOrDateOrStringOrIPStart consumes either a (positive) // integer, a float, a datetime, or IP, or String that started with a // number. It assumes that NO negative sign has been consumed, that // is triggered above. func lexNumberOrDateOrStringOrIPStart(lx *lexer) stateFn { r := lx.next() if !unicode.IsDigit(r) { if r == '.' { return lx.errorf("Floats must start with a digit, not '.'.") } return lx.errorf("Expected a digit but got '%v'.", r) } return lexNumberOrDateOrStringOrIP } // lexNumberOrDateOrStringOrIP consumes either a (positive) integer, // float, datetime, IP or string without quotes that starts with a // number. func lexNumberOrDateOrStringOrIP(lx *lexer) stateFn { r := lx.next() switch { case r == '-': if lx.pos-lx.start != 5 { return lx.errorf("All ISO8601 dates must be in full Zulu form.") } return lexDateAfterYear case unicode.IsDigit(r): return lexNumberOrDateOrStringOrIP case r == '.': // Assume float at first, but could be IP return lexFloatStart case isNumberSuffix(r): return lexConvenientNumber case !(isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r)): // Treat it as a string value once we get a rune that // is not a number. lx.stringStateFn = lexString return lexString } lx.backup() lx.emit(itemInteger) return lx.pop() } // lexConvenientNumber is when we have a suffix, e.g. 1k or 1Mb func lexConvenientNumber(lx *lexer) stateFn { r := lx.next() switch { case r == 'b' || r == 'B' || r == 'i' || r == 'I': return lexConvenientNumber } lx.backup() if isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r) { lx.emit(itemInteger) return lx.pop() } // This is not a number, so treat it as a string. lx.stringStateFn = lexString return lexString } // lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. // It assumes that "YYYY-" has already been consumed. func lexDateAfterYear(lx *lexer) stateFn { formats := []rune{ // digits are '0'. // everything else is direct equality. '0', '0', '-', '0', '0', 'T', '0', '0', ':', '0', '0', ':', '0', '0', 'Z', } for _, f := range formats { r := lx.next() if f == '0' { if !unicode.IsDigit(r) { return lx.errorf("Expected digit in ISO8601 datetime, "+ "but found '%v' instead.", r) } } else if f != r { return lx.errorf("Expected '%v' in ISO8601 datetime, "+ "but found '%v' instead.", f, r) } } lx.emit(itemDatetime) return lx.pop() } // lexNegNumberStart consumes either an integer or a float. It assumes that a // negative sign has already been read, but that *no* digits have been consumed. // lexNegNumberStart will move to the appropriate integer or float states. func lexNegNumberStart(lx *lexer) stateFn { // we MUST see a digit. Even floats have to start with a digit. r := lx.next() if !unicode.IsDigit(r) { if r == '.' { return lx.errorf("Floats must start with a digit, not '.'.") } return lx.errorf("Expected a digit but got '%v'.", r) } return lexNegNumber } // lexNegNumber consumes a negative integer or a float after seeing the first digit. func lexNegNumber(lx *lexer) stateFn { r := lx.next() switch { case unicode.IsDigit(r): return lexNegNumber case r == '.': return lexFloatStart case isNumberSuffix(r): return lexConvenientNumber } lx.backup() lx.emit(itemInteger) return lx.pop() } // lexFloatStart starts the consumption of digits of a float after a '.'. // Namely, at least one digit is required. func lexFloatStart(lx *lexer) stateFn { r := lx.next() if !unicode.IsDigit(r) { return lx.errorf("Floats must have a digit after the '.', but got "+ "'%v' instead.", r) } return lexFloat } // lexFloat consumes the digits of a float after a '.'. // Assumes that one digit has been consumed after a '.' already. func lexFloat(lx *lexer) stateFn { r := lx.next() if unicode.IsDigit(r) { return lexFloat } // Not a digit, if its another '.', need to see if we falsely assumed a float. if r == '.' { return lexIPAddr } lx.backup() lx.emit(itemFloat) return lx.pop() } // lexIPAddr consumes IP addrs, like 127.0.0.1:4222 func lexIPAddr(lx *lexer) stateFn { r := lx.next() if unicode.IsDigit(r) || r == '.' || r == ':' || r == '-' { return lexIPAddr } lx.backup() lx.emit(itemString) return lx.pop() } // lexCommentStart begins the lexing of a comment. It will emit // itemCommentStart and consume no characters, passing control to lexComment. func lexCommentStart(lx *lexer) stateFn { lx.ignore() lx.emit(itemCommentStart) return lexComment } // lexComment lexes an entire comment. It assumes that '#' has been consumed. // It will consume *up to* the first new line character, and pass control // back to the last state on the stack. func lexComment(lx *lexer) stateFn { r := lx.peek() if isNL(r) || r == eof { lx.emit(itemText) return lx.pop() } lx.next() return lexComment } // lexSkip ignores all slurped input and moves on to the next state. func lexSkip(lx *lexer, nextState stateFn) stateFn { return func(lx *lexer) stateFn { lx.ignore() return nextState } } // Tests to see if we have a number suffix func isNumberSuffix(r rune) bool { return r == 'k' || r == 'K' || r == 'm' || r == 'M' || r == 'g' || r == 'G' || r == 't' || r == 'T' || r == 'p' || r == 'P' || r == 'e' || r == 'E' } // Tests for both key separators func isKeySeparator(r rune) bool { return r == keySepEqual || r == keySepColon } // isWhitespace returns true if `r` is a whitespace character according // to the spec. func isWhitespace(r rune) bool { return r == '\t' || r == ' ' } func isNL(r rune) bool { return r == '\n' || r == '\r' } func (itype itemType) String() string { switch itype { case itemError: return "Error" case itemNIL: return "NIL" case itemEOF: return "EOF" case itemText: return "Text" case itemString: return "String" case itemBool: return "Bool" case itemInteger: return "Integer" case itemFloat: return "Float" case itemDatetime: return "DateTime" case itemKey: return "Key" case itemArrayStart: return "ArrayStart" case itemArrayEnd: return "ArrayEnd" case itemMapStart: return "MapStart" case itemMapEnd: return "MapEnd" case itemCommentStart: return "CommentStart" case itemVariable: return "Variable" case itemInclude: return "Include" } panic(fmt.Sprintf("BUG: Unknown type '%s'.", itype.String())) } func (item item) String() string { return fmt.Sprintf("(%s, '%s', %d, %d)", item.typ.String(), item.val, item.line, item.pos) } func escapeSpecial(c rune) string { switch c { case '\n': return "\\n" } return string(c) } nats-server-2.10.27/conf/lex_test.go000066400000000000000000001074121477524627100172630ustar00rootroot00000000000000package conf import "testing" // Test to make sure we get what we expect. func expect(t *testing.T, lx *lexer, items []item) { t.Helper() for i := 0; i < len(items); i++ { item := lx.nextItem() _ = item.String() if item.typ == itemEOF { break } if item != items[i] { t.Fatalf("Testing: '%s'\nExpected %q, received %q\n", lx.input, items[i], item) } if item.typ == itemError { break } } } func TestPlainValue(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemEOF, "", 1, 0}, } lx := lex("foo") expect(t, lx, expectedItems) } func TestSimpleKeyStringValues(t *testing.T) { // Double quotes expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 7}, {itemEOF, "", 1, 0}, } lx := lex("foo = \"bar\"") expect(t, lx, expectedItems) // Single quotes expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 7}, {itemEOF, "", 1, 0}, } lx = lex("foo = 'bar'") expect(t, lx, expectedItems) // No spaces expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 5}, {itemEOF, "", 1, 0}, } lx = lex("foo='bar'") expect(t, lx, expectedItems) // NL expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 5}, {itemEOF, "", 1, 0}, } lx = lex("foo='bar'\r\n") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo=\t'bar'\t") expect(t, lx, expectedItems) } func TestComplexStringValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "bar\\r\\n \\t", 1, 7}, {itemEOF, "", 2, 0}, } lx := lex("foo = 'bar\\r\\n \\t'") expect(t, lx, expectedItems) } func TestStringStartingWithNumber(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "3xyz", 1, 6}, {itemEOF, "", 2, 0}, } lx := lex(`foo = 3xyz`) expect(t, lx, expectedItems) lx = lex(`foo = 3xyz,`) expect(t, lx, expectedItems) lx = lex(`foo = 3xyz;`) expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 2, 9}, {itemString, "3xyz", 2, 15}, {itemEOF, "", 2, 0}, } content := ` foo = 3xyz ` lx = lex(content) expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "map", 2, 9}, {itemMapStart, "", 2, 14}, {itemKey, "foo", 3, 11}, {itemString, "3xyz", 3, 17}, {itemMapEnd, "", 3, 22}, {itemEOF, "", 2, 0}, } content = ` map { foo = 3xyz} ` lx = lex(content) expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "map", 2, 9}, {itemMapStart, "", 2, 14}, {itemKey, "foo", 3, 11}, {itemString, "3xyz", 3, 17}, {itemMapEnd, "", 4, 10}, {itemEOF, "", 2, 0}, } content = ` map { foo = 3xyz; } ` lx = lex(content) expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "map", 2, 9}, {itemMapStart, "", 2, 14}, {itemKey, "foo", 3, 11}, {itemString, "3xyz", 3, 17}, {itemKey, "bar", 4, 11}, {itemString, "4wqs", 4, 17}, {itemMapEnd, "", 5, 10}, {itemEOF, "", 2, 0}, } content = ` map { foo = 3xyz, bar = 4wqs } ` lx = lex(content) expect(t, lx, expectedItems) } func TestBinaryString(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "e", 1, 9}, {itemEOF, "", 1, 0}, } lx := lex("foo = \\x65") expect(t, lx, expectedItems) } func TestBinaryStringLatin1(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "\xe9", 1, 9}, {itemEOF, "", 1, 0}, } lx := lex("foo = \\xe9") expect(t, lx, expectedItems) } func TestSimpleKeyIntegerValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = 123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 4}, {itemEOF, "", 1, 0}, } lx = lex("foo=123") expect(t, lx, expectedItems) lx = lex("foo=123\r\n") expect(t, lx, expectedItems) } func TestSimpleKeyNegativeIntegerValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "-123", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = -123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "-123", 1, 4}, {itemEOF, "", 1, 0}, } lx = lex("foo=-123") expect(t, lx, expectedItems) lx = lex("foo=-123\r\n") expect(t, lx, expectedItems) } func TestConvenientIntegerValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1k", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = 1k") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1K", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1K") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1m", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1m") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1M", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1M") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1g", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1g") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1G", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1G") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1MB", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1MB") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "1Gb", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1Gb") expect(t, lx, expectedItems) // Negative versions expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "-1m", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = -1m") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "-1GB", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = -1GB ") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "1Ghz", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 1Ghz") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "2Pie", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 2Pie") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemString, "3Mbs", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo = 3Mbs,") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "4Gb", 1, 6}, {itemKey, "bar", 1, 11}, {itemString, "5Gø", 1, 17}, {itemEOF, "", 1, 0}, } lx = lex("foo = 4Gb, bar = 5Gø") expect(t, lx, expectedItems) } func TestSimpleKeyFloatValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemFloat, "22.2", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = 22.2") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemFloat, "22.2", 1, 4}, {itemEOF, "", 1, 0}, } lx = lex("foo=22.2") expect(t, lx, expectedItems) lx = lex("foo=22.2\r\n") expect(t, lx, expectedItems) } func TestBadBinaryStringEndingAfterZeroHexChars(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Expected two hexadecimal digits after '\\x', but hit end of line", 2, 1}, {itemEOF, "", 1, 0}, } lx := lex("foo = xyz\\x\n") expect(t, lx, expectedItems) } func TestBadBinaryStringEndingAfterOneHexChar(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Expected two hexadecimal digits after '\\x', but hit end of line", 2, 1}, {itemEOF, "", 1, 0}, } lx := lex("foo = xyz\\xF\n") expect(t, lx, expectedItems) } func TestBadBinaryStringWithZeroHexChars(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Expected two hexadecimal digits after '\\x', but got ']\"'", 1, 12}, {itemEOF, "", 1, 0}, } lx := lex(`foo = "[\x]"`) expect(t, lx, expectedItems) } func TestBadBinaryStringWithOneHexChar(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Expected two hexadecimal digits after '\\x', but got 'e]'", 1, 12}, {itemEOF, "", 1, 0}, } lx := lex(`foo = "[\xe]"`) expect(t, lx, expectedItems) } func TestBadFloatValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Floats must start with a digit", 1, 7}, {itemEOF, "", 1, 0}, } lx := lex("foo = .2") expect(t, lx, expectedItems) } func TestBadKey(t *testing.T) { expectedItems := []item{ {itemError, "Unexpected key separator ':'", 1, 1}, {itemEOF, "", 1, 0}, } lx := lex(" :foo = 22") expect(t, lx, expectedItems) } func TestSimpleKeyBoolValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemBool, "true", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = true") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemBool, "true", 1, 4}, {itemEOF, "", 1, 0}, } lx = lex("foo=true") expect(t, lx, expectedItems) lx = lex("foo=true\r\n") expect(t, lx, expectedItems) } func TestComments(t *testing.T) { expectedItems := []item{ {itemCommentStart, "", 1, 1}, {itemText, " This is a comment", 1, 1}, {itemEOF, "", 1, 0}, } lx := lex("# This is a comment") expect(t, lx, expectedItems) lx = lex("# This is a comment\r\n") expect(t, lx, expectedItems) expectedItems = []item{ {itemCommentStart, "", 1, 2}, {itemText, " This is a comment", 1, 2}, {itemEOF, "", 1, 0}, } lx = lex("// This is a comment\r\n") expect(t, lx, expectedItems) } func TestTopValuesWithComments(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 6}, {itemCommentStart, "", 1, 12}, {itemText, " This is a comment", 1, 12}, {itemEOF, "", 1, 0}, } lx := lex("foo = 123 // This is a comment") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 4}, {itemCommentStart, "", 1, 12}, {itemText, " This is a comment", 1, 12}, {itemEOF, "", 1, 0}, } lx = lex("foo=123 # This is a comment") expect(t, lx, expectedItems) } func TestRawString(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "bar", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = bar") expect(t, lx, expectedItems) lx = lex(`foo = bar' `) expect(t, lx, expectedItems) } func TestDateValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemDatetime, "2016-05-04T18:53:41Z", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo = 2016-05-04T18:53:41Z") expect(t, lx, expectedItems) } func TestVariableValues(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemVariable, "bar", 1, 7}, {itemEOF, "", 1, 0}, } lx := lex("foo = $bar") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemVariable, "bar", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo =$bar") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemVariable, "bar", 1, 5}, {itemEOF, "", 1, 0}, } lx = lex("foo $bar") expect(t, lx, expectedItems) } func TestArrays(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemArrayStart, "", 1, 7}, {itemInteger, "1", 1, 7}, {itemInteger, "2", 1, 10}, {itemInteger, "3", 1, 13}, {itemString, "bar", 1, 17}, {itemArrayEnd, "", 1, 22}, {itemEOF, "", 1, 0}, } lx := lex("foo = [1, 2, 3, 'bar']") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemArrayStart, "", 1, 7}, {itemInteger, "1", 1, 7}, {itemInteger, "2", 1, 9}, {itemInteger, "3", 1, 11}, {itemString, "bar", 1, 14}, {itemArrayEnd, "", 1, 19}, {itemEOF, "", 1, 0}, } lx = lex("foo = [1,2,3,'bar']") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemArrayStart, "", 1, 7}, {itemInteger, "1", 1, 7}, {itemInteger, "2", 1, 10}, {itemInteger, "3", 1, 12}, {itemString, "bar", 1, 15}, {itemArrayEnd, "", 1, 20}, {itemEOF, "", 1, 0}, } lx = lex("foo = [1, 2,3,'bar']") expect(t, lx, expectedItems) } var mlArray = ` # top level comment foo = [ 1, # One 2, // Two 3 # Three 'bar' , "bar" ] ` func TestMultilineArrays(t *testing.T) { expectedItems := []item{ {itemCommentStart, "", 2, 2}, {itemText, " top level comment", 2, 2}, {itemKey, "foo", 3, 1}, {itemArrayStart, "", 3, 8}, {itemInteger, "1", 4, 2}, {itemCommentStart, "", 4, 6}, {itemText, " One", 4, 6}, {itemInteger, "2", 5, 2}, {itemCommentStart, "", 5, 7}, {itemText, " Two", 5, 7}, {itemInteger, "3", 6, 2}, {itemCommentStart, "", 6, 5}, {itemText, " Three", 6, 5}, {itemString, "bar", 7, 3}, {itemString, "bar", 8, 3}, {itemArrayEnd, "", 9, 2}, {itemEOF, "", 9, 0}, } lx := lex(mlArray) expect(t, lx, expectedItems) } var mlArrayNoSep = ` # top level comment foo = [ 1 // foo 2 3 'bar' "bar" ] ` func TestMultilineArraysNoSep(t *testing.T) { expectedItems := []item{ {itemCommentStart, "", 2, 2}, {itemText, " top level comment", 2, 2}, {itemKey, "foo", 3, 1}, {itemArrayStart, "", 3, 8}, {itemInteger, "1", 4, 2}, {itemCommentStart, "", 4, 6}, {itemText, " foo", 4, 6}, {itemInteger, "2", 5, 2}, {itemInteger, "3", 6, 2}, {itemString, "bar", 7, 3}, {itemString, "bar", 8, 3}, {itemArrayEnd, "", 9, 2}, {itemEOF, "", 9, 0}, } lx := lex(mlArrayNoSep) expect(t, lx, expectedItems) } func TestSimpleMap(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 7}, {itemKey, "ip", 1, 7}, {itemString, "127.0.0.1", 1, 11}, {itemKey, "port", 1, 23}, {itemInteger, "4242", 1, 30}, {itemMapEnd, "", 1, 35}, {itemEOF, "", 1, 0}, } lx := lex("foo = {ip='127.0.0.1', port = 4242}") expect(t, lx, expectedItems) } var mlMap = ` foo = { ip = '127.0.0.1' # the IP port= 4242 // the port } ` func TestMultilineMap(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemMapStart, "", 2, 8}, {itemKey, "ip", 3, 3}, {itemString, "127.0.0.1", 3, 9}, {itemCommentStart, "", 3, 21}, {itemText, " the IP", 3, 21}, {itemKey, "port", 4, 3}, {itemInteger, "4242", 4, 9}, {itemCommentStart, "", 4, 16}, {itemText, " the port", 4, 16}, {itemMapEnd, "", 5, 2}, {itemEOF, "", 5, 0}, } lx := lex(mlMap) expect(t, lx, expectedItems) } var nestedMap = ` foo = { host = { ip = '127.0.0.1' port= 4242 } } ` func TestNestedMaps(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemMapStart, "", 2, 8}, {itemKey, "host", 3, 3}, {itemMapStart, "", 3, 11}, {itemKey, "ip", 4, 5}, {itemString, "127.0.0.1", 4, 11}, {itemKey, "port", 5, 5}, {itemInteger, "4242", 5, 11}, {itemMapEnd, "", 6, 4}, {itemMapEnd, "", 7, 2}, {itemEOF, "", 7, 0}, } lx := lex(nestedMap) expect(t, lx, expectedItems) } func TestQuotedKeys(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo : 123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 1}, {itemInteger, "123", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("'foo' : 123") expect(t, lx, expectedItems) lx = lex("\"foo\" : 123") expect(t, lx, expectedItems) } func TestQuotedKeysWithSpace(t *testing.T) { expectedItems := []item{ {itemKey, " foo", 1, 1}, {itemInteger, "123", 1, 9}, {itemEOF, "", 1, 0}, } lx := lex("' foo' : 123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, " foo", 1, 1}, {itemInteger, "123", 1, 9}, {itemEOF, "", 1, 0}, } lx = lex("\" foo\" : 123") expect(t, lx, expectedItems) } func TestColonKeySep(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 6}, {itemEOF, "", 1, 0}, } lx := lex("foo : 123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 4}, {itemEOF, "", 1, 0}, } lx = lex("foo:123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 5}, {itemEOF, "", 1, 0}, } lx = lex("foo: 123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 6}, {itemEOF, "", 1, 0}, } lx = lex("foo: 123\r\n") expect(t, lx, expectedItems) } func TestWhitespaceKeySep(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 4}, {itemEOF, "", 1, 0}, } lx := lex("foo 123") expect(t, lx, expectedItems) lx = lex("foo 123") expect(t, lx, expectedItems) lx = lex("foo\t123") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemInteger, "123", 1, 5}, {itemEOF, "", 1, 0}, } lx = lex("foo\t\t123\r\n") expect(t, lx, expectedItems) } var escString = ` foo = \t bar = \r baz = \n q = \" bs = \\ ` func TestEscapedString(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemString, "\t", 2, 9}, {itemKey, "bar", 3, 1}, {itemString, "\r", 3, 9}, {itemKey, "baz", 4, 1}, {itemString, "\n", 4, 9}, {itemKey, "q", 5, 1}, {itemString, "\"", 5, 9}, {itemKey, "bs", 6, 1}, {itemString, "\\", 6, 9}, {itemEOF, "", 6, 0}, } lx := lex(escString) expect(t, lx, expectedItems) } func TestCompoundStringES(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "\\end", 1, 8}, {itemEOF, "", 2, 0}, } lx := lex(`foo = "\\end"`) expect(t, lx, expectedItems) } func TestCompoundStringSE(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "start\\", 1, 8}, {itemEOF, "", 2, 0}, } lx := lex(`foo = "start\\"`) expect(t, lx, expectedItems) } func TestCompoundStringEE(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "Eq", 1, 12}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \x45\x71`) expect(t, lx, expectedItems) } func TestCompoundStringSEE(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "startEq", 1, 12}, {itemEOF, "", 2, 0}, } lx := lex(`foo = start\x45\x71`) expect(t, lx, expectedItems) } func TestCompoundStringSES(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "start|end", 1, 9}, {itemEOF, "", 2, 0}, } lx := lex(`foo = start\x7Cend`) expect(t, lx, expectedItems) } func TestCompoundStringEES(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "<>end", 1, 12}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \x3c\x3eend`) expect(t, lx, expectedItems) } func TestCompoundStringESE(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "", 1, 12}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \x3cmiddle\x3E`) expect(t, lx, expectedItems) } func TestBadStringEscape(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemError, "Invalid escape character 'y'. Only the following escape characters are allowed: \\xXX, \\t, \\n, \\r, \\\", \\\\.", 1, 8}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \y`) expect(t, lx, expectedItems) } func TestNonBool(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "\\true", 1, 7}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \\true`) expect(t, lx, expectedItems) } func TestNonVariable(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "\\$var", 1, 7}, {itemEOF, "", 2, 0}, } lx := lex(`foo = \\$var`) expect(t, lx, expectedItems) } func TestEmptyStringDQ(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "", 1, 7}, {itemEOF, "", 2, 0}, } lx := lex(`foo = ""`) expect(t, lx, expectedItems) } func TestEmptyStringSQ(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "", 1, 7}, {itemEOF, "", 2, 0}, } lx := lex(`foo = ''`) expect(t, lx, expectedItems) } var nestedWhitespaceMap = ` foo { host { ip = '127.0.0.1' port= 4242 } } ` func TestNestedWhitespaceMaps(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemMapStart, "", 2, 7}, {itemKey, "host", 3, 3}, {itemMapStart, "", 3, 10}, {itemKey, "ip", 4, 5}, {itemString, "127.0.0.1", 4, 11}, {itemKey, "port", 5, 5}, {itemInteger, "4242", 5, 11}, {itemMapEnd, "", 6, 4}, {itemMapEnd, "", 7, 2}, {itemEOF, "", 7, 0}, } lx := lex(nestedWhitespaceMap) expect(t, lx, expectedItems) } var semicolons = ` foo = 123; bar = 'baz'; baz = 'boo' map { id = 1; } ` func TestOptionalSemicolons(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemInteger, "123", 2, 7}, {itemKey, "bar", 3, 1}, {itemString, "baz", 3, 8}, {itemKey, "baz", 4, 1}, {itemString, "boo", 4, 8}, {itemKey, "map", 5, 1}, {itemMapStart, "", 5, 6}, {itemKey, "id", 6, 2}, {itemInteger, "1", 6, 7}, {itemMapEnd, "", 7, 2}, {itemEOF, "", 8, 0}, } lx := lex(semicolons) expect(t, lx, expectedItems) } func TestSemicolonChaining(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemString, "1", 1, 5}, {itemKey, "bar", 1, 9}, {itemFloat, "2.2", 1, 13}, {itemKey, "baz", 1, 18}, {itemBool, "true", 1, 22}, {itemEOF, "", 1, 0}, } lx := lex("foo='1'; bar=2.2; baz=true;") expect(t, lx, expectedItems) } var noquotes = ` foo = 123 bar = baz baz=boo map { id:one id2 : onetwo } t true f false tstr "true" tkey = two fkey = five # This should be a string ` func TestNonQuotedStrings(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemInteger, "123", 2, 7}, {itemKey, "bar", 3, 1}, {itemString, "baz", 3, 7}, {itemKey, "baz", 4, 1}, {itemString, "boo", 4, 5}, {itemKey, "map", 5, 1}, {itemMapStart, "", 5, 6}, {itemKey, "id", 6, 2}, {itemString, "one", 6, 5}, {itemKey, "id2", 7, 2}, {itemString, "onetwo", 7, 8}, {itemMapEnd, "", 8, 2}, {itemKey, "t", 9, 1}, {itemBool, "true", 9, 3}, {itemKey, "f", 10, 1}, {itemBool, "false", 10, 3}, {itemKey, "tstr", 11, 1}, {itemString, "true", 11, 7}, {itemKey, "tkey", 12, 1}, {itemString, "two", 12, 8}, {itemKey, "fkey", 13, 1}, {itemString, "five", 13, 8}, {itemCommentStart, "", 13, 14}, {itemText, " This should be a string", 13, 14}, {itemEOF, "", 14, 0}, } lx := lex(noquotes) expect(t, lx, expectedItems) } var danglingquote = ` listen: "localhost:4242 http: localhost:8222 ` func TestDanglingQuotedString(t *testing.T) { expectedItems := []item{ {itemKey, "listen", 2, 1}, {itemError, "Unexpected EOF.", 5, 1}, } lx := lex(danglingquote) expect(t, lx, expectedItems) } var keydanglingquote = ` foo = " listen: " http: localhost:8222 " ` func TestKeyDanglingQuotedString(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemString, "\nlisten: ", 3, 8}, {itemKey, "http", 5, 1}, {itemString, "localhost:8222", 5, 7}, {itemError, "Unexpected EOF.", 8, 1}, } lx := lex(keydanglingquote) expect(t, lx, expectedItems) } var danglingsquote = ` listen: 'localhost:4242 http: localhost:8222 ` func TestDanglingSingleQuotedString(t *testing.T) { expectedItems := []item{ {itemKey, "listen", 2, 1}, {itemError, "Unexpected EOF.", 5, 1}, } lx := lex(danglingsquote) expect(t, lx, expectedItems) } var keydanglingsquote = ` foo = ' listen: ' http: localhost:8222 ' ` func TestKeyDanglingSingleQuotedString(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 2, 1}, {itemString, "\nlisten: ", 3, 8}, {itemKey, "http", 5, 1}, {itemString, "localhost:8222", 5, 7}, {itemError, "Unexpected EOF.", 8, 1}, } lx := lex(keydanglingsquote) expect(t, lx, expectedItems) } var mapdanglingbracket = ` listen = 4222 cluster = { foo = bar ` func TestMapDanglingBracket(t *testing.T) { expectedItems := []item{ {itemKey, "listen", 2, 1}, {itemInteger, "4222", 2, 10}, {itemKey, "cluster", 4, 1}, {itemMapStart, "", 4, 12}, {itemKey, "foo", 6, 3}, {itemString, "bar", 6, 9}, {itemError, "Unexpected EOF processing map.", 8, 1}, } lx := lex(mapdanglingbracket) expect(t, lx, expectedItems) } var blockdanglingparens = ` listen = 4222 quote = ( foo = bar ` func TestBlockDanglingParens(t *testing.T) { expectedItems := []item{ {itemKey, "listen", 2, 1}, {itemInteger, "4222", 2, 10}, {itemKey, "quote", 4, 1}, {itemError, "Unexpected EOF processing block.", 8, 1}, } lx := lex(blockdanglingparens) expect(t, lx, expectedItems) } func TestMapQuotedKeys(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 7}, {itemKey, "bar", 1, 8}, {itemInteger, "4242", 1, 15}, {itemMapEnd, "", 1, 20}, {itemEOF, "", 1, 0}, } lx := lex("foo = {'bar' = 4242}") expect(t, lx, expectedItems) lx = lex("foo = {\"bar\" = 4242}") expect(t, lx, expectedItems) } func TestSpecialCharsMapQuotedKeys(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 7}, {itemKey, "bar-1.2.3", 1, 8}, {itemMapStart, "", 1, 22}, {itemKey, "port", 1, 23}, {itemInteger, "4242", 1, 28}, {itemMapEnd, "", 1, 34}, {itemMapEnd, "", 1, 35}, {itemEOF, "", 1, 0}, } lx := lex("foo = {'bar-1.2.3' = { port:4242 }}") expect(t, lx, expectedItems) lx = lex("foo = {\"bar-1.2.3\" = { port:4242 }}") expect(t, lx, expectedItems) } var mlnestedmap = ` systems { allinone { description: "This is a description." } } ` func TestDoubleNestedMapsNewLines(t *testing.T) { expectedItems := []item{ {itemKey, "systems", 2, 1}, {itemMapStart, "", 2, 10}, {itemKey, "allinone", 3, 3}, {itemMapStart, "", 3, 13}, {itemKey, "description", 4, 5}, {itemString, "This is a description.", 4, 19}, {itemMapEnd, "", 5, 4}, {itemMapEnd, "", 6, 2}, {itemEOF, "", 7, 0}, } lx := lex(mlnestedmap) expect(t, lx, expectedItems) } var blockexample = ` numbers ( 1234567890 ) ` func TestBlockString(t *testing.T) { expectedItems := []item{ {itemKey, "numbers", 2, 1}, {itemString, "\n1234567890\n", 4, 10}, } lx := lex(blockexample) expect(t, lx, expectedItems) } func TestBlockStringEOF(t *testing.T) { expectedItems := []item{ {itemKey, "numbers", 2, 1}, {itemString, "\n1234567890\n", 4, 10}, } blockbytes := []byte(blockexample[0 : len(blockexample)-1]) blockbytes = append(blockbytes, 0) lx := lex(string(blockbytes)) expect(t, lx, expectedItems) } var mlblockexample = ` numbers ( 12(34)56 ( 7890 ) ) ` func TestBlockStringMultiLine(t *testing.T) { expectedItems := []item{ {itemKey, "numbers", 2, 1}, {itemString, "\n 12(34)56\n (\n 7890\n )\n", 7, 10}, } lx := lex(mlblockexample) expect(t, lx, expectedItems) } func TestUnquotedIPAddr(t *testing.T) { expectedItems := []item{ {itemKey, "listen", 1, 0}, {itemString, "127.0.0.1:4222", 1, 8}, {itemEOF, "", 1, 0}, } lx := lex("listen: 127.0.0.1:4222") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemString, "127.0.0.1", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("listen: 127.0.0.1") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemString, "apcera.me:80", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("listen: apcera.me:80") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemString, "nats.io:-1", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("listen: nats.io:-1") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemInteger, "-1", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("listen: -1") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemString, ":-1", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("listen: :-1") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemString, ":80", 1, 9}, {itemEOF, "", 1, 0}, } lx = lex("listen = :80") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "listen", 1, 0}, {itemArrayStart, "", 1, 10}, {itemString, "localhost:4222", 1, 10}, {itemString, "localhost:4333", 1, 26}, {itemArrayEnd, "", 1, 41}, {itemEOF, "", 1, 0}, } lx = lex("listen = [localhost:4222, localhost:4333]") expect(t, lx, expectedItems) } var arrayOfMaps = ` authorization { users = [ {user: alice, password: foo} {user: bob, password: bar} ] timeout: 0.5 } ` func TestArrayOfMaps(t *testing.T) { expectedItems := []item{ {itemKey, "authorization", 2, 1}, {itemMapStart, "", 2, 16}, {itemKey, "users", 3, 5}, {itemArrayStart, "", 3, 14}, {itemMapStart, "", 4, 8}, {itemKey, "user", 4, 8}, {itemString, "alice", 4, 14}, {itemKey, "password", 4, 21}, {itemString, "foo", 4, 31}, {itemMapEnd, "", 4, 35}, {itemMapStart, "", 5, 8}, {itemKey, "user", 5, 8}, {itemString, "bob", 5, 14}, {itemKey, "password", 5, 21}, {itemString, "bar", 5, 31}, {itemMapEnd, "", 5, 35}, {itemArrayEnd, "", 6, 6}, {itemKey, "timeout", 7, 5}, {itemFloat, "0.5", 7, 14}, {itemMapEnd, "", 8, 2}, {itemEOF, "", 9, 0}, } lx := lex(arrayOfMaps) expect(t, lx, expectedItems) } func TestInclude(t *testing.T) { expectedItems := []item{ {itemInclude, "users.conf", 1, 9}, {itemEOF, "", 1, 0}, } lx := lex("include \"users.conf\"") expect(t, lx, expectedItems) lx = lex("include 'users.conf'") expect(t, lx, expectedItems) expectedItems = []item{ {itemInclude, "users.conf", 1, 8}, {itemEOF, "", 1, 0}, } lx = lex("include users.conf") expect(t, lx, expectedItems) } func TestMapInclude(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 5}, {itemInclude, "users.conf", 1, 14}, {itemMapEnd, "", 1, 26}, {itemEOF, "", 1, 0}, } lx := lex("foo { include users.conf }") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 5}, {itemInclude, "users.conf", 1, 13}, {itemMapEnd, "", 1, 24}, {itemEOF, "", 1, 0}, } lx = lex("foo {include users.conf}") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 5}, {itemInclude, "users.conf", 1, 15}, {itemMapEnd, "", 1, 28}, {itemEOF, "", 1, 0}, } lx = lex("foo { include 'users.conf' }") expect(t, lx, expectedItems) expectedItems = []item{ {itemKey, "foo", 1, 0}, {itemMapStart, "", 1, 5}, {itemInclude, "users.conf", 1, 15}, {itemMapEnd, "", 1, 27}, {itemEOF, "", 1, 0}, } lx = lex("foo { include \"users.conf\"}") expect(t, lx, expectedItems) } func TestJSONCompat(t *testing.T) { for _, test := range []struct { name string input string expected []item }{ { name: "should omit initial and final brackets at top level with a single item", input: ` { "http_port": 8223 } `, expected: []item{ {itemKey, "http_port", 3, 28}, {itemInteger, "8223", 3, 40}, {itemKey, "}", 4, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should omit trailing commas at top level with two items", input: ` { "http_port": 8223, "port": 4223 } `, expected: []item{ {itemKey, "http_port", 3, 28}, {itemInteger, "8223", 3, 40}, {itemKey, "port", 4, 28}, {itemInteger, "4223", 4, 35}, {itemKey, "}", 5, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should omit trailing commas at top level with multiple items", input: ` { "http_port": 8223, "port": 4223, "max_payload": "5MB", "debug": true, "max_control_line": 1024 } `, expected: []item{ {itemKey, "http_port", 3, 28}, {itemInteger, "8223", 3, 40}, {itemKey, "port", 4, 28}, {itemInteger, "4223", 4, 35}, {itemKey, "max_payload", 5, 28}, {itemString, "5MB", 5, 43}, {itemKey, "debug", 6, 28}, {itemBool, "true", 6, 36}, {itemKey, "max_control_line", 7, 28}, {itemInteger, "1024", 7, 47}, {itemKey, "}", 8, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should support JSON not prettified", input: `{"http_port": 8224,"port": 4224} `, expected: []item{ {itemKey, "http_port", 1, 2}, {itemInteger, "8224", 1, 14}, {itemKey, "port", 1, 20}, {itemInteger, "4224", 1, 27}, {itemEOF, "", 0, 0}, }, }, { name: "should support JSON not prettified with final bracket after newline", input: `{"http_port": 8225,"port": 4225 } `, expected: []item{ {itemKey, "http_port", 1, 2}, {itemInteger, "8225", 1, 14}, {itemKey, "port", 1, 20}, {itemInteger, "4225", 1, 27}, {itemKey, "}", 2, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should support uglified JSON with inner blocks", input: `{"http_port": 8227,"port": 4227,"write_deadline": "1h","cluster": {"port": 6222,"routes": ["nats://127.0.0.1:4222","nats://127.0.0.1:4223","nats://127.0.0.1:4224"]}} `, expected: []item{ {itemKey, "http_port", 1, 2}, {itemInteger, "8227", 1, 14}, {itemKey, "port", 1, 20}, {itemInteger, "4227", 1, 27}, {itemKey, "write_deadline", 1, 33}, {itemString, "1h", 1, 51}, {itemKey, "cluster", 1, 56}, {itemMapStart, "", 1, 67}, {itemKey, "port", 1, 68}, {itemInteger, "6222", 1, 75}, {itemKey, "routes", 1, 81}, {itemArrayStart, "", 1, 91}, {itemString, "nats://127.0.0.1:4222", 1, 92}, {itemString, "nats://127.0.0.1:4223", 1, 116}, {itemString, "nats://127.0.0.1:4224", 1, 140}, {itemArrayEnd, "", 1, 163}, {itemMapEnd, "", 1, 164}, {itemKey, "}", 14, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should support prettified JSON with inner blocks", input: ` { "http_port": 8227, "port": 4227, "write_deadline": "1h", "cluster": { "port": 6222, "routes": [ "nats://127.0.0.1:4222", "nats://127.0.0.1:4223", "nats://127.0.0.1:4224" ] } } `, expected: []item{ {itemKey, "http_port", 3, 28}, {itemInteger, "8227", 3, 40}, {itemKey, "port", 4, 28}, {itemInteger, "4227", 4, 35}, {itemKey, "write_deadline", 5, 28}, {itemString, "1h", 5, 46}, {itemKey, "cluster", 6, 28}, {itemMapStart, "", 6, 39}, {itemKey, "port", 7, 30}, {itemInteger, "6222", 7, 37}, {itemKey, "routes", 8, 30}, {itemArrayStart, "", 8, 40}, {itemString, "nats://127.0.0.1:4222", 9, 32}, {itemString, "nats://127.0.0.1:4223", 10, 32}, {itemString, "nats://127.0.0.1:4224", 11, 32}, {itemArrayEnd, "", 12, 30}, {itemMapEnd, "", 13, 28}, {itemKey, "}", 14, 25}, {itemEOF, "", 0, 0}, }, }, { name: "should support JSON with blocks", input: `{ "jetstream": { "store_dir": "/tmp/nats" "max_mem": 1000000, }, "port": 4222, "server_name": "nats1" } `, expected: []item{ {itemKey, "jetstream", 2, 28}, {itemMapStart, "", 2, 41}, {itemKey, "store_dir", 3, 30}, {itemString, "/tmp/nats", 3, 43}, {itemKey, "max_mem", 4, 30}, {itemInteger, "1000000", 4, 40}, {itemMapEnd, "", 5, 28}, {itemKey, "port", 6, 28}, {itemInteger, "4222", 6, 35}, {itemKey, "server_name", 7, 28}, {itemString, "nats1", 7, 43}, {itemKey, "}", 8, 25}, {itemEOF, "", 0, 0}, }, }, } { t.Run(test.name, func(t *testing.T) { lx := lex(test.input) expect(t, lx, test.expected) }) } } nats-server-2.10.27/conf/parse.go000066400000000000000000000253731477524627100165530ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package conf supports a configuration file format used by gnatsd. It is // a flexible format that combines the best of traditional // configuration formats and newer styles such as JSON and YAML. package conf // The format supported is less restrictive than today's formats. // Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //) // Also supports key value assignments using '=' or ':' or whiteSpace() // e.g. foo = 2, foo : 2, foo 2 // maps can be assigned with no key separator as well // semicolons as value terminators in key/value assignments are optional // // see parse_test.go for more examples. import ( "fmt" "os" "path/filepath" "strconv" "strings" "time" "unicode" ) type parser struct { mapping map[string]any lx *lexer // The current scoped context, can be array or map ctx any // stack of contexts, either map or array/slice stack ctxs []any // Keys stack keys []string // Keys stack as items ikeys []item // The config file path, empty by default. fp string // pedantic reports error when configuration is not correct. pedantic bool } // Parse will return a map of keys to any, although concrete types // underly them. The values supported are string, bool, int64, float64, DateTime. // Arrays and nested Maps are also supported. func Parse(data string) (map[string]any, error) { p, err := parse(data, "", false) if err != nil { return nil, err } return p.mapping, nil } // ParseFile is a helper to open file, etc. and parse the contents. func ParseFile(fp string) (map[string]any, error) { data, err := os.ReadFile(fp) if err != nil { return nil, fmt.Errorf("error opening config file: %v", err) } p, err := parse(string(data), fp, false) if err != nil { return nil, err } return p.mapping, nil } // ParseFileWithChecks is equivalent to ParseFile but runs in pedantic mode. func ParseFileWithChecks(fp string) (map[string]any, error) { data, err := os.ReadFile(fp) if err != nil { return nil, err } p, err := parse(string(data), fp, true) if err != nil { return nil, err } return p.mapping, nil } type token struct { item item value any usedVariable bool sourceFile string } func (t *token) Value() any { return t.value } func (t *token) Line() int { return t.item.line } func (t *token) IsUsedVariable() bool { return t.usedVariable } func (t *token) SourceFile() string { return t.sourceFile } func (t *token) Position() int { return t.item.pos } func parse(data, fp string, pedantic bool) (p *parser, err error) { p = &parser{ mapping: make(map[string]any), lx: lex(data), ctxs: make([]any, 0, 4), keys: make([]string, 0, 4), ikeys: make([]item, 0, 4), fp: filepath.Dir(fp), pedantic: pedantic, } p.pushContext(p.mapping) var prevItem item for { it := p.next() if it.typ == itemEOF { // Here we allow the final character to be a bracket '}' // in order to support JSON like configurations. if prevItem.typ == itemKey && prevItem.val != mapEndString { return nil, fmt.Errorf("config is invalid (%s:%d:%d)", fp, it.line, it.pos) } break } prevItem = it if err := p.processItem(it, fp); err != nil { return nil, err } } return p, nil } func (p *parser) next() item { return p.lx.nextItem() } func (p *parser) pushContext(ctx any) { p.ctxs = append(p.ctxs, ctx) p.ctx = ctx } func (p *parser) popContext() any { if len(p.ctxs) == 0 { panic("BUG in parser, context stack empty") } li := len(p.ctxs) - 1 last := p.ctxs[li] p.ctxs = p.ctxs[0:li] p.ctx = p.ctxs[len(p.ctxs)-1] return last } func (p *parser) pushKey(key string) { p.keys = append(p.keys, key) } func (p *parser) popKey() string { if len(p.keys) == 0 { panic("BUG in parser, keys stack empty") } li := len(p.keys) - 1 last := p.keys[li] p.keys = p.keys[0:li] return last } func (p *parser) pushItemKey(key item) { p.ikeys = append(p.ikeys, key) } func (p *parser) popItemKey() item { if len(p.ikeys) == 0 { panic("BUG in parser, item keys stack empty") } li := len(p.ikeys) - 1 last := p.ikeys[li] p.ikeys = p.ikeys[0:li] return last } func (p *parser) processItem(it item, fp string) error { setValue := func(it item, v any) { if p.pedantic { p.setValue(&token{it, v, false, fp}) } else { p.setValue(v) } } switch it.typ { case itemError: return fmt.Errorf("Parse error on line %d: '%s'", it.line, it.val) case itemKey: // Keep track of the keys as items and strings, // we do this in order to be able to still support // includes without many breaking changes. p.pushKey(it.val) if p.pedantic { p.pushItemKey(it) } case itemMapStart: newCtx := make(map[string]any) p.pushContext(newCtx) case itemMapEnd: setValue(it, p.popContext()) case itemString: // FIXME(dlc) sanitize string? setValue(it, it.val) case itemInteger: lastDigit := 0 for _, r := range it.val { if !unicode.IsDigit(r) && r != '-' { break } lastDigit++ } numStr := it.val[:lastDigit] num, err := strconv.ParseInt(numStr, 10, 64) if err != nil { if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { return fmt.Errorf("integer '%s' is out of the range", it.val) } return fmt.Errorf("expected integer, but got '%s'", it.val) } // Process a suffix suffix := strings.ToLower(strings.TrimSpace(it.val[lastDigit:])) switch suffix { case "": setValue(it, num) case "k": setValue(it, num*1000) case "kb", "ki", "kib": setValue(it, num*1024) case "m": setValue(it, num*1000*1000) case "mb", "mi", "mib": setValue(it, num*1024*1024) case "g": setValue(it, num*1000*1000*1000) case "gb", "gi", "gib": setValue(it, num*1024*1024*1024) case "t": setValue(it, num*1000*1000*1000*1000) case "tb", "ti", "tib": setValue(it, num*1024*1024*1024*1024) case "p": setValue(it, num*1000*1000*1000*1000*1000) case "pb", "pi", "pib": setValue(it, num*1024*1024*1024*1024*1024) case "e": setValue(it, num*1000*1000*1000*1000*1000*1000) case "eb", "ei", "eib": setValue(it, num*1024*1024*1024*1024*1024*1024) } case itemFloat: num, err := strconv.ParseFloat(it.val, 64) if err != nil { if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { return fmt.Errorf("float '%s' is out of the range", it.val) } return fmt.Errorf("expected float, but got '%s'", it.val) } setValue(it, num) case itemBool: switch strings.ToLower(it.val) { case "true", "yes", "on": setValue(it, true) case "false", "no", "off": setValue(it, false) default: return fmt.Errorf("expected boolean value, but got '%s'", it.val) } case itemDatetime: dt, err := time.Parse("2006-01-02T15:04:05Z", it.val) if err != nil { return fmt.Errorf( "expected Zulu formatted DateTime, but got '%s'", it.val) } setValue(it, dt) case itemArrayStart: var array = make([]any, 0) p.pushContext(array) case itemArrayEnd: array := p.ctx p.popContext() setValue(it, array) case itemVariable: value, found, err := p.lookupVariable(it.val) if err != nil { return fmt.Errorf("variable reference for '%s' on line %d could not be parsed: %s", it.val, it.line, err) } if !found { return fmt.Errorf("variable reference for '%s' on line %d can not be found", it.val, it.line) } if p.pedantic { switch tk := value.(type) { case *token: // Mark the looked up variable as used, and make // the variable reference become handled as a token. tk.usedVariable = true p.setValue(&token{it, tk.Value(), false, fp}) default: // Special case to add position context to bcrypt references. p.setValue(&token{it, value, false, fp}) } } else { p.setValue(value) } case itemInclude: var ( m map[string]any err error ) if p.pedantic { m, err = ParseFileWithChecks(filepath.Join(p.fp, it.val)) } else { m, err = ParseFile(filepath.Join(p.fp, it.val)) } if err != nil { return fmt.Errorf("error parsing include file '%s', %v", it.val, err) } for k, v := range m { p.pushKey(k) if p.pedantic { switch tk := v.(type) { case *token: p.pushItemKey(tk.item) } } p.setValue(v) } } return nil } // Used to map an environment value into a temporary map to pass to secondary Parse call. const pkey = "pk" // We special case raw strings here that are bcrypt'd. This allows us not to force quoting the strings const bcryptPrefix = "2a$" // lookupVariable will lookup a variable reference. It will use block scoping on keys // it has seen before, with the top level scoping being the environment variables. We // ignore array contexts and only process the map contexts.. // // Returns true for ok if it finds something, similar to map. func (p *parser) lookupVariable(varReference string) (any, bool, error) { // Do special check to see if it is a raw bcrypt string. if strings.HasPrefix(varReference, bcryptPrefix) { return "$" + varReference, true, nil } // Loop through contexts currently on the stack. for i := len(p.ctxs) - 1; i >= 0; i-- { ctx := p.ctxs[i] // Process if it is a map context if m, ok := ctx.(map[string]any); ok { if v, ok := m[varReference]; ok { return v, ok, nil } } } // If we are here, we have exhausted our context maps and still not found anything. // Parse from the environment. if vStr, ok := os.LookupEnv(varReference); ok { // Everything we get here will be a string value, so we need to process as a parser would. if vmap, err := Parse(fmt.Sprintf("%s=%s", pkey, vStr)); err == nil { v, ok := vmap[pkey] return v, ok, nil } else { return nil, false, err } } return nil, false, nil } func (p *parser) setValue(val any) { // Test to see if we are on an array or a map // Array processing if ctx, ok := p.ctx.([]any); ok { p.ctx = append(ctx, val) p.ctxs[len(p.ctxs)-1] = p.ctx } // Map processing if ctx, ok := p.ctx.(map[string]any); ok { key := p.popKey() if p.pedantic { // Change the position to the beginning of the key // since more useful when reporting errors. switch v := val.(type) { case *token: it := p.popItemKey() v.item.pos = it.pos v.item.line = it.line ctx[key] = v } } else { // FIXME(dlc), make sure to error if redefining same key? ctx[key] = val } } } nats-server-2.10.27/conf/parse_test.go000066400000000000000000000451001477524627100176000ustar00rootroot00000000000000package conf import ( "fmt" "os" "path/filepath" "reflect" "strings" "testing" "time" ) // Test to make sure we get what we expect. func test(t *testing.T, data string, ex map[string]any) { t.Helper() m, err := Parse(data) if err != nil { t.Fatalf("Received err: %v\n", err) } if m == nil { t.Fatal("Received nil map") } if !reflect.DeepEqual(m, ex) { t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, ex) } } func TestSimpleTopLevel(t *testing.T) { ex := map[string]any{ "foo": "1", "bar": float64(2.2), "baz": true, "boo": int64(22), } test(t, "foo='1'; bar=2.2; baz=true; boo=22", ex) } func TestBools(t *testing.T) { ex := map[string]any{ "foo": true, } test(t, "foo=true", ex) test(t, "foo=TRUE", ex) test(t, "foo=true", ex) test(t, "foo=yes", ex) test(t, "foo=on", ex) } var varSample = ` index = 22 foo = $index ` func TestSimpleVariable(t *testing.T) { ex := map[string]any{ "index": int64(22), "foo": int64(22), } test(t, varSample, ex) } var varNestedSample = ` index = 22 nest { index = 11 foo = $index } bar = $index ` func TestNestedVariable(t *testing.T) { ex := map[string]any{ "index": int64(22), "nest": map[string]any{ "index": int64(11), "foo": int64(11), }, "bar": int64(22), } test(t, varNestedSample, ex) } func TestMissingVariable(t *testing.T) { _, err := Parse("foo=$index") if err == nil { t.Fatalf("Expected an error for a missing variable, got none") } if !strings.HasPrefix(err.Error(), "variable reference") { t.Fatalf("Wanted a variable reference err, got %q\n", err) } } func TestEnvVariable(t *testing.T) { ex := map[string]any{ "foo": int64(22), } evar := "__UNIQ22__" os.Setenv(evar, "22") defer os.Unsetenv(evar) test(t, fmt.Sprintf("foo = $%s", evar), ex) } func TestEnvVariableString(t *testing.T) { ex := map[string]any{ "foo": "xyz", } evar := "__UNIQ22__" os.Setenv(evar, "xyz") defer os.Unsetenv(evar) test(t, fmt.Sprintf("foo = $%s", evar), ex) } func TestEnvVariableStringStartingWithNumber(t *testing.T) { evar := "__UNIQ22__" os.Setenv(evar, "3xyz") defer os.Unsetenv(evar) _, err := Parse("foo = $%s") if err == nil { t.Fatalf("Expected err not being able to process string: %v\n", err) } } func TestEnvVariableStringStartingWithNumberAndSizeUnit(t *testing.T) { ex := map[string]any{ "foo": "3Gyz", } evar := "__UNIQ22__" os.Setenv(evar, "3Gyz") defer os.Unsetenv(evar) test(t, fmt.Sprintf("foo = $%s", evar), ex) } func TestEnvVariableStringStartingWithNumberUsingQuotes(t *testing.T) { ex := map[string]any{ "foo": "3xyz", } evar := "__UNIQ22__" os.Setenv(evar, "'3xyz'") defer os.Unsetenv(evar) test(t, fmt.Sprintf("foo = $%s", evar), ex) } func TestBcryptVariable(t *testing.T) { ex := map[string]any{ "password": "$2a$11$ooo", } test(t, "password: $2a$11$ooo", ex) } var easynum = ` k = 8k kb = 4kb ki = 3ki kib = 4ki m = 1m mb = 2MB mi = 2Mi mib = 64MiB g = 2g gb = 22GB gi = 22Gi gib = 22GiB tb = 22TB ti = 22Ti tib = 22TiB pb = 22PB pi = 22Pi pib = 22PiB ` func TestConvenientNumbers(t *testing.T) { ex := map[string]any{ "k": int64(8 * 1000), "kb": int64(4 * 1024), "ki": int64(3 * 1024), "kib": int64(4 * 1024), "m": int64(1000 * 1000), "mb": int64(2 * 1024 * 1024), "mi": int64(2 * 1024 * 1024), "mib": int64(64 * 1024 * 1024), "g": int64(2 * 1000 * 1000 * 1000), "gb": int64(22 * 1024 * 1024 * 1024), "gi": int64(22 * 1024 * 1024 * 1024), "gib": int64(22 * 1024 * 1024 * 1024), "tb": int64(22 * 1024 * 1024 * 1024 * 1024), "ti": int64(22 * 1024 * 1024 * 1024 * 1024), "tib": int64(22 * 1024 * 1024 * 1024 * 1024), "pb": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), "pi": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), "pib": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024), } test(t, easynum, ex) } var sample1 = ` foo { host { ip = '127.0.0.1' port = 4242 } servers = [ "a.com", "b.com", "c.com"] } ` func TestSample1(t *testing.T) { ex := map[string]any{ "foo": map[string]any{ "host": map[string]any{ "ip": "127.0.0.1", "port": int64(4242), }, "servers": []any{"a.com", "b.com", "c.com"}, }, } test(t, sample1, ex) } var cluster = ` cluster { port: 4244 authorization { user: route_user password: top_secret timeout: 1 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. // Test both styles of comments routes = [ nats-route://foo:bar@apcera.me:4245 nats-route://foo:bar@apcera.me:4246 ] } ` func TestSample2(t *testing.T) { ex := map[string]any{ "cluster": map[string]any{ "port": int64(4244), "authorization": map[string]any{ "user": "route_user", "password": "top_secret", "timeout": int64(1), }, "routes": []any{ "nats-route://foo:bar@apcera.me:4245", "nats-route://foo:bar@apcera.me:4246", }, }, } test(t, cluster, ex) } var sample3 = ` foo { expr = '(true == "false")' text = 'This is a multi-line text block.' } ` func TestSample3(t *testing.T) { ex := map[string]any{ "foo": map[string]any{ "expr": "(true == \"false\")", "text": "This is a multi-line\ntext block.", }, } test(t, sample3, ex) } var sample4 = ` array [ { abc: 123 } { xyz: "word" } ] ` func TestSample4(t *testing.T) { ex := map[string]any{ "array": []any{ map[string]any{"abc": int64(123)}, map[string]any{"xyz": "word"}, }, } test(t, sample4, ex) } var sample5 = ` now = 2016-05-04T18:53:41Z gmt = false ` func TestSample5(t *testing.T) { dt, _ := time.Parse("2006-01-02T15:04:05Z", "2016-05-04T18:53:41Z") ex := map[string]any{ "now": dt, "gmt": false, } test(t, sample5, ex) } func TestIncludes(t *testing.T) { ex := map[string]any{ "listen": "127.0.0.1:4222", "authorization": map[string]any{ "ALICE_PASS": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q", "BOB_PASS": "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly", "users": []any{ map[string]any{ "user": "alice", "password": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q"}, map[string]any{ "user": "bob", "password": "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly"}, }, "timeout": float64(0.5), }, } m, err := ParseFile("simple.conf") if err != nil { t.Fatalf("Received err: %v\n", err) } if m == nil { t.Fatal("Received nil map") } if !reflect.DeepEqual(m, ex) { t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, ex) } } var varIncludedVariablesSample = ` authorization { include "./includes/passwords.conf" CAROL_PASS: foo users = [ {user: alice, password: $ALICE_PASS} {user: bob, password: $BOB_PASS} {user: carol, password: $CAROL_PASS} ] } ` func TestIncludeVariablesWithChecks(t *testing.T) { p, err := parse(varIncludedVariablesSample, "", true) if err != nil { t.Fatalf("Received err: %v\n", err) } key := "authorization" m, ok := p.mapping[key] if !ok { t.Errorf("Expected %q to be in the config", key) } expectKeyVal := func(t *testing.T, m any, expectedKey string, expectedVal string, expectedLine, expectedPos int) { t.Helper() tk := m.(*token) v := tk.Value() vv := v.(map[string]any) value, ok := vv[expectedKey] if !ok { t.Errorf("Expected key %q", expectedKey) } tk, ok = value.(*token) if !ok { t.Fatalf("Expected token %v", value) } if tk.Line() != expectedLine { t.Errorf("Expected token to be at line %d, got: %d", expectedLine, tk.Line()) } if tk.Position() != expectedPos { t.Errorf("Expected token to be at position %d, got: %d", expectedPos, tk.Position()) } v = tk.Value() if v != expectedVal { t.Errorf("Expected %q, got: %s", expectedVal, v) } } expectKeyVal(t, m, "ALICE_PASS", "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q", 2, 1) expectKeyVal(t, m, "BOB_PASS", "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly", 3, 1) expectKeyVal(t, m, "CAROL_PASS", "foo", 6, 3) } func TestParserNoInfiniteLoop(t *testing.T) { for _, test := range []string{`A@@Føøøø?˛ø:{øøøø˙˙`, `include "9/�`} { if _, err := Parse(test); err == nil { t.Fatal("expected an error") } else if !strings.Contains(err.Error(), "Unexpected EOF") { t.Fatal("expected unexpected eof error") } } } func TestParseWithNoValuesAreInvalid(t *testing.T) { for _, test := range []struct { name string conf string err string }{ { "invalid key without values", `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, "config is invalid (:1:41)", }, { "invalid untrimmed key without values", ` aaaaaaaaaaaaaaaaaaaaaaaaaaa`, "config is invalid (:1:41)", }, { "invalid untrimmed key without values", ` aaaaaaaaaaaaaaaaaaaaaaaaaaa `, "config is invalid (:1:41)", }, { "invalid keys after comments", ` # with comments and no spaces to create key values # is also an invalid config. aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa `, "config is invalid (:5:25)", }, { "comma separated without values are invalid", ` a,a,a,a,a,a,a,a,a,a,a `, "config is invalid (:3:25)", }, } { t.Run(test.name, func(t *testing.T) { if _, err := parse(test.conf, "", true); err == nil { t.Error("expected an error") } else if !strings.Contains(err.Error(), test.err) { t.Errorf("expected invalid conf error, got: %v", err) } }) } } func TestParseWithNoValuesEmptyConfigsAreValid(t *testing.T) { for _, test := range []struct { name string conf string }{ { "empty conf", "", }, { "empty conf with line breaks", ` `, }, { "just comments with no values", ` # just comments with no values # is still valid. `, }, } { t.Run(test.name, func(t *testing.T) { if _, err := parse(test.conf, "", true); err != nil { t.Errorf("unexpected error: %v", err) } }) } } func TestParseWithTrailingBracketsAreValid(t *testing.T) { for _, test := range []struct { name string conf string }{ { "empty conf", "{}", }, { "just comments with no values", ` { # comments in the body } `, }, { // trailing brackets accidentally can become keys, // this is valid since needed to support JSON like configs.. "trailing brackets after config", ` accounts { users = [{}]} } `, }, { "wrapped in brackets", `{ accounts { users = [{}]} } `, }, } { t.Run(test.name, func(t *testing.T) { if _, err := parse(test.conf, "", true); err != nil { t.Errorf("unexpected error: %v", err) } }) } } func TestParseWithNoValuesIncludes(t *testing.T) { for _, test := range []struct { input string includes map[string]string err string linepos string }{ { `# includes accounts { foo { include 'foo.conf'} bar { users = [{user = "bar"}] } quux { include 'quux.conf'} } `, map[string]string{ "foo.conf": ``, "quux.conf": `?????????????`, }, "error parsing include file 'quux.conf', config is invalid", "quux.conf:1:1", }, { `# includes accounts { foo { include 'foo.conf'} bar { include 'bar.conf'} quux { include 'quux.conf'} } `, map[string]string{ "foo.conf": ``, // Empty configs are ok "bar.conf": `AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`, "quux.conf": ` # just some comments, # and no key values also ok. `, }, "error parsing include file 'bar.conf', config is invalid", "bar.conf:1:34", }, } { t.Run("", func(t *testing.T) { sdir := t.TempDir() f, err := os.CreateTemp(sdir, "nats.conf-") if err != nil { t.Fatal(err) } if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { t.Error(err) } if test.includes != nil { for includeFile, contents := range test.includes { inf, err := os.Create(filepath.Join(sdir, includeFile)) if err != nil { t.Fatal(err) } if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil { t.Error(err) } } } if _, err := parse(test.input, f.Name(), true); err == nil { t.Error("expected an error") } else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) { t.Errorf("expected invalid conf error, got: %v", err) } }) } } func TestJSONParseCompat(t *testing.T) { for _, test := range []struct { name string input string includes map[string]string expected map[string]any }{ { "JSON with nested blocks", ` { "http_port": 8227, "port": 4227, "write_deadline": "1h", "cluster": { "port": 6222, "routes": [ "nats://127.0.0.1:4222", "nats://127.0.0.1:4223", "nats://127.0.0.1:4224" ] } } `, nil, map[string]any{ "http_port": int64(8227), "port": int64(4227), "write_deadline": "1h", "cluster": map[string]any{ "port": int64(6222), "routes": []any{ "nats://127.0.0.1:4222", "nats://127.0.0.1:4223", "nats://127.0.0.1:4224", }, }, }, }, { "JSON with nested blocks", `{ "jetstream": { "store_dir": "/tmp/nats" "max_mem": 1000000, }, "port": 4222, "server_name": "nats1" } `, nil, map[string]any{ "jetstream": map[string]any{ "store_dir": "/tmp/nats", "max_mem": int64(1_000_000), }, "port": int64(4222), "server_name": "nats1", }, }, { "JSON empty object in one line", `{}`, nil, map[string]any{}, }, { "JSON empty object with line breaks", ` { } `, nil, map[string]any{}, }, { "JSON includes", ` accounts { foo { include 'foo.json' } bar { include 'bar.json' } quux { include 'quux.json' } } `, map[string]string{ "foo.json": `{ "users": [ {"user": "foo"} ] }`, "bar.json": `{ "users": [ {"user": "bar"} ] }`, "quux.json": `{}`, }, map[string]any{ "accounts": map[string]any{ "foo": map[string]any{ "users": []any{ map[string]any{ "user": "foo", }, }, }, "bar": map[string]any{ "users": []any{ map[string]any{ "user": "bar", }, }, }, "quux": map[string]any{}, }, }, }, } { t.Run(test.name, func(t *testing.T) { sdir := t.TempDir() f, err := os.CreateTemp(sdir, "nats.conf-") if err != nil { t.Fatal(err) } if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { t.Error(err) } if test.includes != nil { for includeFile, contents := range test.includes { inf, err := os.Create(filepath.Join(sdir, includeFile)) if err != nil { t.Fatal(err) } if err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil { t.Error(err) } } } m, err := ParseFile(f.Name()) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !reflect.DeepEqual(m, test.expected) { t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, test.expected) } }) } } func TestBlocks(t *testing.T) { for _, test := range []struct { name string input string expected map[string]any err string linepos string }{ { "inline block", `{ listen: 0.0.0.0:4222 }`, map[string]any{ "listen": "0.0.0.0:4222", }, "", "", }, { "newline block", `{ listen: 0.0.0.0:4222 }`, map[string]any{ "listen": "0.0.0.0:4222", }, "", "", }, { "newline block with trailing comment", ` { listen: 0.0.0.0:4222 } # wibble `, map[string]any{ "listen": "0.0.0.0:4222", }, "", "", }, { "nested newline blocks with trailing comment", ` { { listen: 0.0.0.0:4222 // random comment } # wibble1 } # wibble2 `, map[string]any{ "listen": "0.0.0.0:4222", }, "", "", }, { "top line values in block scope", ` { "debug": False "prof_port": 8221 "server_name": "aws-useast2-natscj1-1" } `, map[string]any{ "debug": false, "prof_port": int64(8221), "server_name": "aws-useast2-natscj1-1", }, "", "", }, { "comment in block scope after value parse", ` { "debug": False "server_name": "gcp-asianortheast3-natscj1-1" # Profile port specification. "prof_port": 8221 } `, map[string]any{ "debug": false, "prof_port": int64(8221), "server_name": "gcp-asianortheast3-natscj1-1", }, "", "", }, } { t.Run(test.name, func(t *testing.T) { f, err := os.CreateTemp(t.TempDir(), "nats.conf-") if err != nil { t.Fatal(err) } if err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil { t.Error(err) } if m, err := ParseFile(f.Name()); err == nil { if !reflect.DeepEqual(m, test.expected) { t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, test.expected) } } else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) { t.Errorf("expected invalid conf error, got: %v", err) } else if err != nil { t.Error(err) } }) } } nats-server-2.10.27/conf/simple.conf000066400000000000000000000001551477524627100172410ustar00rootroot00000000000000listen: 127.0.0.1:4222 authorization { include 'includes/users.conf' # Pull in from file timeout: 0.5 } nats-server-2.10.27/doc/000077500000000000000000000000001477524627100147205ustar00rootroot00000000000000nats-server-2.10.27/doc/README.md000066400000000000000000000002271477524627100162000ustar00rootroot00000000000000# Architecture Decision Records The NATS ADR documents have moved to their [own repository](https://github.com/nats-io/nats-architecture-and-design/) nats-server-2.10.27/docker/000077500000000000000000000000001477524627100154225ustar00rootroot00000000000000nats-server-2.10.27/docker/Dockerfile.nightly000066400000000000000000000013231477524627100210700ustar00rootroot00000000000000FROM golang:alpine AS builder ARG VERSION="nightly" RUN apk add --update git RUN mkdir -p src/github.com/nats-io && \ cd src/github.com/nats-io/ && \ git clone https://github.com/nats-io/natscli.git && \ cd natscli/nats && \ go build -ldflags "-w -X main.version=${VERSION}" -o /nats RUN go install github.com/nats-io/nsc/v2@latest FROM alpine:latest RUN apk add --update ca-certificates && mkdir -p /nats/bin && mkdir /nats/conf COPY docker/nats-server.conf /nats/conf/nats-server.conf COPY nats-server /bin/nats-server COPY --from=builder /nats /bin/nats COPY --from=builder /go/bin/nsc /bin/nsc EXPOSE 4222 8222 6222 5222 ENTRYPOINT ["/bin/nats-server"] CMD ["-c", "/nats/conf/nats-server.conf"] nats-server-2.10.27/docker/nats-server.conf000066400000000000000000000012441477524627100205430ustar00rootroot00000000000000 # Client port of 4222 on all interfaces port: 4222 # HTTP monitoring port monitor_port: 8222 # This is for clustering multiple servers together. cluster { # Route connections to be received on any interface on port 6222 port: 6222 # Routes are protected, so need to use them with --routes flag # e.g. --routes=nats-route://ruser:T0pS3cr3t@otherdockerhost:6222 authorization { user: ruser password: T0pS3cr3t timeout: 2 } # Routes are actively solicited and connected to from this server. # This Docker image has none by default, but you can pass a # flag to the nats-server docker image to create one to an existing server. routes = [] } nats-server-2.10.27/go.mod000066400000000000000000000006341477524627100152640ustar00rootroot00000000000000module github.com/nats-io/nats-server/v2 go 1.23.0 toolchain go1.23.6 require ( github.com/klauspost/compress v1.18.0 github.com/minio/highwayhash v1.0.3 github.com/nats-io/jwt/v2 v2.7.3 github.com/nats-io/nats.go v1.39.1 github.com/nats-io/nkeys v0.4.10 github.com/nats-io/nuid v1.0.1 go.uber.org/automaxprocs v1.6.0 golang.org/x/crypto v0.34.0 golang.org/x/sys v0.30.0 golang.org/x/time v0.10.0 ) nats-server-2.10.27/go.sum000066400000000000000000000050351477524627100153110ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.7.3 h1:6bNPK+FXgBeAqdj4cYQ0F8ViHRbi7woQLq4W29nUAzE= github.com/nats-io/jwt/v2 v2.7.3/go.mod h1:GvkcbHhKquj3pkioy5put1wvPxs78UlZ7D/pY+BgZk4= github.com/nats-io/nats.go v1.39.1 h1:oTkfKBmz7W047vRxV762M67ZdXeOtUgvbBaNoQ+3PPk= github.com/nats-io/nats.go v1.39.1/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM= github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc= github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= golang.org/x/crypto v0.34.0 h1:+/C6tk6rf/+t5DhUketUbD1aNGqiSX3j15Z6xuIDlBA= golang.org/x/crypto v0.34.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nats-server-2.10.27/internal/000077500000000000000000000000001477524627100157675ustar00rootroot00000000000000nats-server-2.10.27/internal/fastrand/000077500000000000000000000000001477524627100175715ustar00rootroot00000000000000nats-server-2.10.27/internal/fastrand/LICENSE000066400000000000000000000027161477524627100206040ustar00rootroot00000000000000Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.nats-server-2.10.27/internal/fastrand/fastrand.go000066400000000000000000000011641477524627100217240ustar00rootroot00000000000000// Copyright 2020-2023 The LevelDB-Go, Pebble and NATS Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. package fastrand import _ "unsafe" // required by go:linkname // Uint32 returns a lock free uint32 value. // //go:linkname Uint32 runtime.fastrand func Uint32() uint32 // Uint32n returns a lock free uint32 value in the interval [0, n). // //go:linkname Uint32n runtime.fastrandn func Uint32n(n uint32) uint32 // Uint64 returns a lock free uint64 value. func Uint64() uint64 { v := uint64(Uint32()) return v<<32 | uint64(Uint32()) } nats-server-2.10.27/internal/fastrand/fastrand_test.go000066400000000000000000000022661477524627100227670ustar00rootroot00000000000000// Copyright 2020-23 The LevelDB-Go, Pebble and NATS Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. package fastrand import ( "math/rand" "sync" "testing" "time" ) type defaultRand struct { mu sync.Mutex src rand.Source64 } func newDefaultRand() *defaultRand { r := &defaultRand{ src: rand.New(rand.NewSource(time.Now().UnixNano())), } return r } func (r *defaultRand) Uint32() uint32 { r.mu.Lock() i := uint32(r.src.Uint64()) r.mu.Unlock() return i } func (r *defaultRand) Uint64() uint64 { r.mu.Lock() i := uint64(r.src.Uint64()) r.mu.Unlock() return i } func BenchmarkFastRand32(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { Uint32() } }) } func BenchmarkFastRand64(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { Uint64() } }) } func BenchmarkDefaultRand32(b *testing.B) { r := newDefaultRand() b.RunParallel(func(pb *testing.PB) { for pb.Next() { r.Uint32() } }) } func BenchmarkDefaultRand64(b *testing.B) { r := newDefaultRand() b.RunParallel(func(pb *testing.PB) { for pb.Next() { r.Uint64() } }) } nats-server-2.10.27/internal/ldap/000077500000000000000000000000001477524627100167075ustar00rootroot00000000000000nats-server-2.10.27/internal/ldap/dn.go000066400000000000000000000205471477524627100176470ustar00rootroot00000000000000// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) // Portions copyright (c) 2015-2016 go-ldap Authors package ldap import ( "bytes" "crypto/x509/pkix" "encoding/asn1" enchex "encoding/hex" "errors" "fmt" "strings" ) var attributeTypeNames = map[string]string{ "2.5.4.3": "CN", "2.5.4.5": "SERIALNUMBER", "2.5.4.6": "C", "2.5.4.7": "L", "2.5.4.8": "ST", "2.5.4.9": "STREET", "2.5.4.10": "O", "2.5.4.11": "OU", "2.5.4.17": "POSTALCODE", // FIXME: Add others. "0.9.2342.19200300.100.1.25": "DC", } // AttributeTypeAndValue represents an attributeTypeAndValue from https://tools.ietf.org/html/rfc4514 type AttributeTypeAndValue struct { // Type is the attribute type Type string // Value is the attribute value Value string } // RelativeDN represents a relativeDistinguishedName from https://tools.ietf.org/html/rfc4514 type RelativeDN struct { Attributes []*AttributeTypeAndValue } // DN represents a distinguishedName from https://tools.ietf.org/html/rfc4514 type DN struct { RDNs []*RelativeDN } // FromCertSubject takes a pkix.Name from a cert and returns a DN // that uses the same set. Does not support multi value RDNs. func FromCertSubject(subject pkix.Name) (*DN, error) { dn := &DN{ RDNs: make([]*RelativeDN, 0), } for i := len(subject.Names) - 1; i >= 0; i-- { name := subject.Names[i] oidString := name.Type.String() typeName, ok := attributeTypeNames[oidString] if !ok { return nil, fmt.Errorf("invalid type name: %+v", name) } v, ok := name.Value.(string) if !ok { return nil, fmt.Errorf("invalid type value: %+v", v) } rdn := &RelativeDN{ Attributes: []*AttributeTypeAndValue{ { Type: typeName, Value: v, }, }, } dn.RDNs = append(dn.RDNs, rdn) } return dn, nil } // FromRawCertSubject takes a raw subject from a certificate // and uses asn1.Unmarshal to get the individual RDNs in the // original order, including multi-value RDNs. func FromRawCertSubject(rawSubject []byte) (*DN, error) { dn := &DN{ RDNs: make([]*RelativeDN, 0), } var rdns pkix.RDNSequence _, err := asn1.Unmarshal(rawSubject, &rdns) if err != nil { return nil, err } for i := len(rdns) - 1; i >= 0; i-- { rdn := rdns[i] if len(rdn) == 0 { continue } r := &RelativeDN{} attrs := make([]*AttributeTypeAndValue, 0) for j := len(rdn) - 1; j >= 0; j-- { atv := rdn[j] typeName := "" name := atv.Type.String() typeName, ok := attributeTypeNames[name] if !ok { return nil, fmt.Errorf("invalid type name: %+v", name) } value, ok := atv.Value.(string) if !ok { return nil, fmt.Errorf("invalid type value: %+v", atv.Value) } attr := &AttributeTypeAndValue{ Type: typeName, Value: value, } attrs = append(attrs, attr) } r.Attributes = attrs dn.RDNs = append(dn.RDNs, r) } return dn, nil } // ParseDN returns a distinguishedName or an error. // The function respects https://tools.ietf.org/html/rfc4514 func ParseDN(str string) (*DN, error) { dn := new(DN) dn.RDNs = make([]*RelativeDN, 0) rdn := new(RelativeDN) rdn.Attributes = make([]*AttributeTypeAndValue, 0) buffer := bytes.Buffer{} attribute := new(AttributeTypeAndValue) escaping := false unescapedTrailingSpaces := 0 stringFromBuffer := func() string { s := buffer.String() s = s[0 : len(s)-unescapedTrailingSpaces] buffer.Reset() unescapedTrailingSpaces = 0 return s } for i := 0; i < len(str); i++ { char := str[i] switch { case escaping: unescapedTrailingSpaces = 0 escaping = false switch char { case ' ', '"', '#', '+', ',', ';', '<', '=', '>', '\\': buffer.WriteByte(char) continue } // Not a special character, assume hex encoded octet if len(str) == i+1 { return nil, errors.New("got corrupted escaped character") } dst := []byte{0} n, err := enchex.Decode([]byte(dst), []byte(str[i:i+2])) if err != nil { return nil, fmt.Errorf("failed to decode escaped character: %s", err) } else if n != 1 { return nil, fmt.Errorf("expected 1 byte when un-escaping, got %d", n) } buffer.WriteByte(dst[0]) i++ case char == '\\': unescapedTrailingSpaces = 0 escaping = true case char == '=': attribute.Type = stringFromBuffer() // Special case: If the first character in the value is # the following data // is BER encoded. Throw an error since not supported right now. if len(str) > i+1 && str[i+1] == '#' { return nil, errors.New("unsupported BER encoding") } case char == ',' || char == '+': // We're done with this RDN or value, push it if len(attribute.Type) == 0 { return nil, errors.New("incomplete type, value pair") } attribute.Value = stringFromBuffer() rdn.Attributes = append(rdn.Attributes, attribute) attribute = new(AttributeTypeAndValue) if char == ',' { dn.RDNs = append(dn.RDNs, rdn) rdn = new(RelativeDN) rdn.Attributes = make([]*AttributeTypeAndValue, 0) } case char == ' ' && buffer.Len() == 0: // ignore unescaped leading spaces continue default: if char == ' ' { // Track unescaped spaces in case they are trailing and we need to remove them unescapedTrailingSpaces++ } else { // Reset if we see a non-space char unescapedTrailingSpaces = 0 } buffer.WriteByte(char) } } if buffer.Len() > 0 { if len(attribute.Type) == 0 { return nil, errors.New("DN ended with incomplete type, value pair") } attribute.Value = stringFromBuffer() rdn.Attributes = append(rdn.Attributes, attribute) dn.RDNs = append(dn.RDNs, rdn) } return dn, nil } // Equal returns true if the DNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch). // Returns true if they have the same number of relative distinguished names // and corresponding relative distinguished names (by position) are the same. func (d *DN) Equal(other *DN) bool { if len(d.RDNs) != len(other.RDNs) { return false } for i := range d.RDNs { if !d.RDNs[i].Equal(other.RDNs[i]) { return false } } return true } // RDNsMatch returns true if the individual RDNs of the DNs // are the same regardless of ordering. func (d *DN) RDNsMatch(other *DN) bool { if len(d.RDNs) != len(other.RDNs) { return false } CheckNextRDN: for _, irdn := range d.RDNs { for _, ordn := range other.RDNs { if (len(irdn.Attributes) == len(ordn.Attributes)) && (irdn.hasAllAttributes(ordn.Attributes) && ordn.hasAllAttributes(irdn.Attributes)) { // Found the RDN, check if next one matches. continue CheckNextRDN } } // Could not find a matching individual RDN, auth fails. return false } return true } // AncestorOf returns true if the other DN consists of at least one RDN followed by all the RDNs of the current DN. // "ou=widgets,o=acme.com" is an ancestor of "ou=sprockets,ou=widgets,o=acme.com" // "ou=widgets,o=acme.com" is not an ancestor of "ou=sprockets,ou=widgets,o=foo.com" // "ou=widgets,o=acme.com" is not an ancestor of "ou=widgets,o=acme.com" func (d *DN) AncestorOf(other *DN) bool { if len(d.RDNs) >= len(other.RDNs) { return false } // Take the last `len(d.RDNs)` RDNs from the other DN to compare against otherRDNs := other.RDNs[len(other.RDNs)-len(d.RDNs):] for i := range d.RDNs { if !d.RDNs[i].Equal(otherRDNs[i]) { return false } } return true } // Equal returns true if the RelativeDNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch). // Relative distinguished names are the same if and only if they have the same number of AttributeTypeAndValues // and each attribute of the first RDN is the same as the attribute of the second RDN with the same attribute type. // The order of attributes is not significant. // Case of attribute types is not significant. func (r *RelativeDN) Equal(other *RelativeDN) bool { if len(r.Attributes) != len(other.Attributes) { return false } return r.hasAllAttributes(other.Attributes) && other.hasAllAttributes(r.Attributes) } func (r *RelativeDN) hasAllAttributes(attrs []*AttributeTypeAndValue) bool { for _, attr := range attrs { found := false for _, myattr := range r.Attributes { if myattr.Equal(attr) { found = true break } } if !found { return false } } return true } // Equal returns true if the AttributeTypeAndValue is equivalent to the specified AttributeTypeAndValue // Case of the attribute type is not significant func (a *AttributeTypeAndValue) Equal(other *AttributeTypeAndValue) bool { return strings.EqualFold(a.Type, other.Type) && a.Value == other.Value } nats-server-2.10.27/internal/ldap/dn_test.go000066400000000000000000000141331477524627100207000ustar00rootroot00000000000000// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) // Portions copyright (c) 2015-2016 go-ldap Authors // Static-Check Fixes Copyright 2024 The NATS Authors package ldap import ( "reflect" "testing" ) func TestSuccessfulDNParsing(t *testing.T) { testcases := map[string]DN{ "": {[]*RelativeDN{}}, "cn=Jim\\2C \\22Hasse Hö\\22 Hansson!,dc=dummy,dc=com": {[]*RelativeDN{ {[]*AttributeTypeAndValue{{"cn", "Jim, \"Hasse Hö\" Hansson!"}}}, {[]*AttributeTypeAndValue{{"dc", "dummy"}}}, {[]*AttributeTypeAndValue{{"dc", "com"}}}}}, "UID=jsmith,DC=example,DC=net": {[]*RelativeDN{ {[]*AttributeTypeAndValue{{"UID", "jsmith"}}}, {[]*AttributeTypeAndValue{{"DC", "example"}}}, {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, "OU=Sales+CN=J. Smith,DC=example,DC=net": {[]*RelativeDN{ {[]*AttributeTypeAndValue{ {"OU", "Sales"}, {"CN", "J. Smith"}}}, {[]*AttributeTypeAndValue{{"DC", "example"}}}, {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, // // "1.3.6.1.4.1.1466.0=#04024869": {[]*RelativeDN{ // {[]*AttributeTypeAndValue{{"1.3.6.1.4.1.1466.0", "Hi"}}}}}, // "1.3.6.1.4.1.1466.0=#04024869,DC=net": {[]*RelativeDN{ // {[]*AttributeTypeAndValue{{"1.3.6.1.4.1.1466.0", "Hi"}}}, // {[]*AttributeTypeAndValue{{"DC", "net"}}}}}, "CN=Lu\\C4\\8Di\\C4\\87": {[]*RelativeDN{ {[]*AttributeTypeAndValue{{"CN", "LuÄić"}}}}}, " CN = Lu\\C4\\8Di\\C4\\87 ": {[]*RelativeDN{ {[]*AttributeTypeAndValue{{"CN", "LuÄić"}}}}}, ` A = 1 , B = 2 `: {[]*RelativeDN{ {[]*AttributeTypeAndValue{{"A", "1"}}}, {[]*AttributeTypeAndValue{{"B", "2"}}}}}, ` A = 1 + B = 2 `: {[]*RelativeDN{ {[]*AttributeTypeAndValue{ {"A", "1"}, {"B", "2"}}}}}, ` \ \ A\ \ = \ \ 1\ \ , \ \ B\ \ = \ \ 2\ \ `: {[]*RelativeDN{ {[]*AttributeTypeAndValue{{" A ", " 1 "}}}, {[]*AttributeTypeAndValue{{" B ", " 2 "}}}}}, ` \ \ A\ \ = \ \ 1\ \ + \ \ B\ \ = \ \ 2\ \ `: {[]*RelativeDN{ {[]*AttributeTypeAndValue{ {" A ", " 1 "}, {" B ", " 2 "}}}}}, } for test, answer := range testcases { dn, err := ParseDN(test) if err != nil { t.Error(err.Error()) continue } if !reflect.DeepEqual(dn, &answer) { t.Errorf("Parsed DN %s is not equal to the expected structure", test) t.Logf("Expected:") for _, rdn := range answer.RDNs { for _, attribs := range rdn.Attributes { t.Logf("#%v\n", attribs) } } t.Logf("Actual:") for _, rdn := range dn.RDNs { for _, attribs := range rdn.Attributes { t.Logf("#%v\n", attribs) } } } } } func TestErrorDNParsing(t *testing.T) { testcases := map[string]string{ "*": "DN ended with incomplete type, value pair", "cn=Jim\\0Test": "failed to decode escaped character: encoding/hex: invalid byte: U+0054 'T'", "cn=Jim\\0": "got corrupted escaped character", "DC=example,=net": "DN ended with incomplete type, value pair", // "1=#0402486": "failed to decode BER encoding: encoding/hex: odd length hex string", "test,DC=example,DC=com": "incomplete type, value pair", "=test,DC=example,DC=com": "incomplete type, value pair", } for test, answer := range testcases { _, err := ParseDN(test) if err == nil { t.Errorf("Expected %s to fail parsing but succeeded\n", test) } else if err.Error() != answer { t.Errorf("Unexpected error on %s:\n%s\nvs.\n%s\n", test, answer, err.Error()) } } } func TestDNEqual(t *testing.T) { testcases := []struct { A string B string Equal bool }{ // Exact match {"", "", true}, {"o=A", "o=A", true}, {"o=A", "o=B", false}, {"o=A,o=B", "o=A,o=B", true}, {"o=A,o=B", "o=A,o=C", false}, {"o=A+o=B", "o=A+o=B", true}, {"o=A+o=B", "o=A+o=C", false}, // Case mismatch in type is ignored {"o=A", "O=A", true}, {"o=A,o=B", "o=A,O=B", true}, {"o=A+o=B", "o=A+O=B", true}, // Case mismatch in value is significant {"o=a", "O=A", false}, {"o=a,o=B", "o=A,O=B", false}, {"o=a+o=B", "o=A+O=B", false}, // Multi-valued RDN order mismatch is ignored {"o=A+o=B", "O=B+o=A", true}, // Number of RDN attributes is significant {"o=A+o=B", "O=B+o=A+O=B", false}, // Missing values are significant {"o=A+o=B", "O=B+o=A+O=C", false}, // missing values matter {"o=A+o=B+o=C", "O=B+o=A", false}, // missing values matter // Whitespace tests // Matching { "cn=John Doe, ou=People, dc=sun.com", "cn=John Doe, ou=People, dc=sun.com", true, }, // Difference in leading/trailing chars is ignored { "cn=John Doe, ou=People, dc=sun.com", "cn=John Doe,ou=People,dc=sun.com", true, }, // Difference in values is significant { "cn=John Doe, ou=People, dc=sun.com", "cn=John Doe, ou=People, dc=sun.com", false, }, } for i, tc := range testcases { a, err := ParseDN(tc.A) if err != nil { t.Errorf("%d: %v", i, err) continue } b, err := ParseDN(tc.B) if err != nil { t.Errorf("%d: %v", i, err) continue } if expected, actual := tc.Equal, a.Equal(b); expected != actual { t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) continue } if expected, actual := tc.Equal, b.Equal(a); expected != actual { t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) continue } } } func TestDNAncestor(t *testing.T) { testcases := []struct { A string B string Ancestor bool }{ // Exact match returns false {"", "", false}, {"o=A", "o=A", false}, {"o=A,o=B", "o=A,o=B", false}, {"o=A+o=B", "o=A+o=B", false}, // Mismatch {"ou=C,ou=B,o=A", "ou=E,ou=D,ou=B,o=A", false}, // Descendant {"ou=C,ou=B,o=A", "ou=E,ou=C,ou=B,o=A", true}, } for i, tc := range testcases { a, err := ParseDN(tc.A) if err != nil { t.Errorf("%d: %v", i, err) continue } b, err := ParseDN(tc.B) if err != nil { t.Errorf("%d: %v", i, err) continue } if expected, actual := tc.Ancestor, a.AncestorOf(b); expected != actual { t.Errorf("%d: when comparing '%s' and '%s' expected %v, got %v", i, tc.A, tc.B, expected, actual) continue } } } nats-server-2.10.27/internal/ocsp/000077500000000000000000000000001477524627100167335ustar00rootroot00000000000000nats-server-2.10.27/internal/ocsp/ocsp.go000066400000000000000000000163371477524627100202400ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testhelper import ( "crypto" "crypto/tls" "crypto/x509" "encoding/base64" "encoding/pem" "fmt" "io" "net/http" "os" "strconv" "strings" "sync" "testing" "time" "golang.org/x/crypto/ocsp" ) const ( defaultResponseTTL = 4 * time.Second defaultAddress = "127.0.0.1:8888" ) func NewOCSPResponderCustomAddress(t *testing.T, issuerCertPEM, issuerKeyPEM string, addr string) *http.Server { t.Helper() return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, addr, defaultResponseTTL, "") } func NewOCSPResponder(t *testing.T, issuerCertPEM, issuerKeyPEM string) *http.Server { t.Helper() return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, "") } func NewOCSPResponderDesignatedCustomAddress(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, addr string) *http.Server { t.Helper() return NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, addr, defaultResponseTTL, "") } func NewOCSPResponderPreferringHTTPMethod(t *testing.T, issuerCertPEM, issuerKeyPEM, method string) *http.Server { t.Helper() return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, method) } func NewOCSPResponderCustomTimeout(t *testing.T, issuerCertPEM, issuerKeyPEM string, responseTTL time.Duration) *http.Server { t.Helper() return NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, responseTTL, "") } func NewOCSPResponderBase(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, embed bool, addr string, responseTTL time.Duration, method string) *http.Server { t.Helper() var mu sync.Mutex status := make(map[string]int) issuerCert := parseCertPEM(t, issuerCertPEM) respCert := parseCertPEM(t, respCertPEM) respKey := parseKeyPEM(t, respKeyPEM) mux := http.NewServeMux() // The "/statuses/" endpoint is for directly setting a key-value pair in // the CA's status database. mux.HandleFunc("/statuses/", func(rw http.ResponseWriter, r *http.Request) { defer r.Body.Close() key := r.URL.Path[len("/statuses/"):] switch r.Method { case "GET": mu.Lock() n, ok := status[key] if !ok { n = ocsp.Unknown } mu.Unlock() fmt.Fprintf(rw, "%s %d", key, n) case "POST": data, err := io.ReadAll(r.Body) if err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) return } n, err := strconv.Atoi(string(data)) if err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) return } mu.Lock() status[key] = n mu.Unlock() fmt.Fprintf(rw, "%s %d", key, n) default: http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) return } }) // The "/" endpoint is for normal OCSP requests. This actually parses an // OCSP status request and signs a response with a CA. Lightly based off: // https://www.ietf.org/rfc/rfc2560.txt mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { var reqData []byte var err error switch { case r.Method == "GET": if method != "" && r.Method != method { http.Error(rw, "", http.StatusBadRequest) return } reqData, err = base64.StdEncoding.DecodeString(r.URL.Path[1:]) case r.Method == "POST": if method != "" && r.Method != method { http.Error(rw, "", http.StatusBadRequest) return } reqData, err = io.ReadAll(r.Body) r.Body.Close() default: http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) return } if err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) return } ocspReq, err := ocsp.ParseRequest(reqData) if err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) return } mu.Lock() n, ok := status[ocspReq.SerialNumber.String()] if !ok { n = ocsp.Unknown } mu.Unlock() tmpl := ocsp.Response{ Status: n, SerialNumber: ocspReq.SerialNumber, ThisUpdate: time.Now(), } if responseTTL != 0 { tmpl.NextUpdate = tmpl.ThisUpdate.Add(responseTTL) } if embed { tmpl.Certificate = respCert } respData, err := ocsp.CreateResponse(issuerCert, respCert, tmpl, respKey) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return } rw.Header().Set("Content-Type", "application/ocsp-response") rw.Header().Set("Content-Length", fmt.Sprint(len(respData))) fmt.Fprint(rw, string(respData)) }) srv := &http.Server{ Addr: addr, Handler: mux, ReadTimeout: time.Second * 5, } go srv.ListenAndServe() time.Sleep(1 * time.Second) return srv } func parseCertPEM(t *testing.T, certPEM string) *x509.Certificate { t.Helper() block := parsePEM(t, certPEM) cert, err := x509.ParseCertificate(block.Bytes) if err != nil { t.Fatalf("failed to parse cert '%s': %s", certPEM, err) } return cert } func parseKeyPEM(t *testing.T, keyPEM string) crypto.Signer { t.Helper() block := parsePEM(t, keyPEM) key, err := x509.ParsePKCS8PrivateKey(block.Bytes) if err != nil { key, err = x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { t.Fatalf("failed to parse ikey %s: %s", keyPEM, err) } } keyc := key.(crypto.Signer) return keyc } func parsePEM(t *testing.T, pemPath string) *pem.Block { t.Helper() data, err := os.ReadFile(pemPath) if err != nil { t.Fatal(err) } block, _ := pem.Decode(data) if block == nil { t.Fatalf("failed to decode PEM %s", pemPath) } return block } func GetOCSPStatus(s tls.ConnectionState) (*ocsp.Response, error) { if len(s.VerifiedChains) == 0 { return nil, fmt.Errorf("missing TLS verified chains") } chain := s.VerifiedChains[0] if got, want := len(chain), 2; got < want { return nil, fmt.Errorf("incomplete cert chain, got %d, want at least %d", got, want) } leaf, issuer := chain[0], chain[1] resp, err := ocsp.ParseResponseForCert(s.OCSPResponse, leaf, issuer) if err != nil { return nil, fmt.Errorf("failed to parse OCSP response: %w", err) } if err := resp.CheckSignatureFrom(issuer); err != nil { return resp, err } return resp, nil } func SetOCSPStatus(t *testing.T, ocspURL, certPEM string, status int) { t.Helper() cert := parseCertPEM(t, certPEM) hc := &http.Client{Timeout: 10 * time.Second} resp, err := hc.Post( fmt.Sprintf("%s/statuses/%s", ocspURL, cert.SerialNumber), "", strings.NewReader(fmt.Sprint(status)), ) if err != nil { t.Fatal(err) } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("failed to read OCSP HTTP response body: %s", err) } if got, want := resp.Status, "200 OK"; got != want { t.Error(strings.TrimSpace(string(data))) t.Fatalf("unexpected OCSP HTTP set status, got %q, want %q", got, want) } } nats-server-2.10.27/internal/testhelper/000077500000000000000000000000001477524627100201465ustar00rootroot00000000000000nats-server-2.10.27/internal/testhelper/logging.go000066400000000000000000000062221477524627100221250ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testhelper // These routines need to be accessible in both the server and test // directories, and tests importing a package don't get exported symbols from // _test.go files in the imported package, so we put them here where they can // be used freely. import ( "fmt" "strings" "sync" "testing" ) type DummyLogger struct { sync.Mutex Msg string AllMsgs []string } func (l *DummyLogger) CheckContent(t *testing.T, expectedStr string) { t.Helper() l.Lock() defer l.Unlock() if l.Msg != expectedStr { t.Fatalf("Expected log to be: %v, got %v", expectedStr, l.Msg) } } func (l *DummyLogger) aggregate() { if l.AllMsgs != nil { l.AllMsgs = append(l.AllMsgs, l.Msg) } } func (l *DummyLogger) Noticef(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } func (l *DummyLogger) Errorf(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } func (l *DummyLogger) Warnf(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } func (l *DummyLogger) Fatalf(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } func (l *DummyLogger) Debugf(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } func (l *DummyLogger) Tracef(format string, v ...any) { l.Lock() defer l.Unlock() l.Msg = fmt.Sprintf(format, v...) l.aggregate() } // NewDummyLogger creates a dummy logger and allows to ask for logs to be // retained instead of just keeping the most recent. Use retain to provide an // initial size estimate on messages (not to provide a max capacity). func NewDummyLogger(retain uint) *DummyLogger { l := &DummyLogger{} if retain > 0 { l.AllMsgs = make([]string, 0, retain) } return l } func (l *DummyLogger) Drain() { l.Lock() defer l.Unlock() if l.AllMsgs == nil { return } l.AllMsgs = make([]string, 0, len(l.AllMsgs)) } func (l *DummyLogger) CheckForProhibited(t *testing.T, reason, needle string) { t.Helper() l.Lock() defer l.Unlock() if l.AllMsgs == nil { t.Fatal("DummyLogger.CheckForProhibited called without AllMsgs being collected") } // Collect _all_ matches, rather than have to re-test repeatedly. // This will particularly help with less deterministic tests with multiple matches. shouldFail := false for i := range l.AllMsgs { if strings.Contains(l.AllMsgs[i], needle) { t.Errorf("log contains %s: %v", reason, l.AllMsgs[i]) shouldFail = true } } if shouldFail { t.FailNow() } } nats-server-2.10.27/locksordering.txt000066400000000000000000000023671477524627100175710ustar00rootroot00000000000000Here is the list of some established lock ordering. In this list, A -> B means that you can have A.Lock() then B.Lock(), not the opposite. jetStream -> jsAccount -> Server -> client -> Account jetStream -> jsAccount -> stream -> consumer A lock to protect jetstream account's usage has been introduced: jsAccount.usageMu. This lock is independent and can be invoked under any other lock: jsAccount -> jsa.usageMu, stream -> jsa.usageMu, etc... A lock to protect the account's leafnodes list was also introduced to allow that lock to be held and the acquire a client lock which is not possible with the normal account lock. accountLeafList -> client AccountResolver interface has various implementations, but assume: AccountResolver -> Server A reloadMu lock was added to prevent newly connecting clients racing with the configuration reload. This must be taken out as soon as a reload is about to happen before any other locks: reloadMu -> Server reloadMu -> optsMu The "jscmMu" lock in the Account is used to serialise calls to checkJetStreamMigrate and clearObserverState so that they cannot interleave which would leave Raft nodes in inconsistent observer states. jscmMu -> Account -> jsAccount jscmMu -> stream.clsMu jscmMu -> RaftNodenats-server-2.10.27/logger/000077500000000000000000000000001477524627100154325ustar00rootroot00000000000000nats-server-2.10.27/logger/log.go000066400000000000000000000230341477524627100165440ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package logger provides logging facilities for the NATS server package logger import ( "fmt" "log" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" ) // Default file permissions for log files. const defaultLogPerms = os.FileMode(0640) // Logger is the server logger type Logger struct { sync.Mutex logger *log.Logger debug bool trace bool infoLabel string warnLabel string errorLabel string fatalLabel string debugLabel string traceLabel string fl *fileLogger } type LogOption interface { isLoggerOption() } // LogUTC controls whether timestamps in the log output should be UTC or local time. type LogUTC bool func (l LogUTC) isLoggerOption() {} func logFlags(time bool, opts ...LogOption) int { flags := 0 if time { flags = log.LstdFlags | log.Lmicroseconds } for _, opt := range opts { switch v := opt.(type) { case LogUTC: if time && bool(v) { flags |= log.LUTC } } } return flags } // NewStdLogger creates a logger with output directed to Stderr func NewStdLogger(time, debug, trace, colors, pid bool, opts ...LogOption) *Logger { flags := logFlags(time, opts...) pre := "" if pid { pre = pidPrefix() } l := &Logger{ logger: log.New(os.Stderr, pre, flags), debug: debug, trace: trace, } if colors { setColoredLabelFormats(l) } else { setPlainLabelFormats(l) } return l } // NewFileLogger creates a logger with output directed to a file func NewFileLogger(filename string, time, debug, trace, pid bool, opts ...LogOption) *Logger { flags := logFlags(time, opts...) pre := "" if pid { pre = pidPrefix() } fl, err := newFileLogger(filename, pre, time) if err != nil { log.Fatalf("error opening file: %v", err) return nil } l := &Logger{ logger: log.New(fl, pre, flags), debug: debug, trace: trace, fl: fl, } fl.Lock() fl.l = l fl.Unlock() setPlainLabelFormats(l) return l } type writerAndCloser interface { Write(b []byte) (int, error) Close() error Name() string } type fileLogger struct { out int64 canRotate int32 sync.Mutex l *Logger f writerAndCloser limit int64 olimit int64 pid string time bool closed bool maxNumFiles int } func newFileLogger(filename, pidPrefix string, time bool) (*fileLogger, error) { fileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE f, err := os.OpenFile(filename, fileflags, defaultLogPerms) if err != nil { return nil, err } stats, err := f.Stat() if err != nil { f.Close() return nil, err } fl := &fileLogger{ canRotate: 0, f: f, out: stats.Size(), pid: pidPrefix, time: time, } return fl, nil } func (l *fileLogger) setLimit(limit int64) { l.Lock() l.olimit, l.limit = limit, limit atomic.StoreInt32(&l.canRotate, 1) rotateNow := l.out > l.limit l.Unlock() if rotateNow { l.l.Noticef("Rotating logfile...") } } func (l *fileLogger) setMaxNumFiles(max int) { l.Lock() l.maxNumFiles = max l.Unlock() } func (l *fileLogger) logDirect(label, format string, v ...any) int { var entrya = [256]byte{} var entry = entrya[:0] if l.pid != "" { entry = append(entry, l.pid...) } if l.time { now := time.Now() year, month, day := now.Date() hour, min, sec := now.Clock() microsec := now.Nanosecond() / 1000 entry = append(entry, fmt.Sprintf("%04d/%02d/%02d %02d:%02d:%02d.%06d ", year, month, day, hour, min, sec, microsec)...) } entry = append(entry, label...) entry = append(entry, fmt.Sprintf(format, v...)...) entry = append(entry, '\r', '\n') l.f.Write(entry) return len(entry) } func (l *fileLogger) logPurge(fname string) { var backups []string lDir := filepath.Dir(fname) lBase := filepath.Base(fname) entries, err := os.ReadDir(lDir) if err != nil { l.logDirect(l.l.errorLabel, "Unable to read directory %q for log purge (%v), will attempt next rotation", lDir, err) return } for _, entry := range entries { if entry.IsDir() || entry.Name() == lBase || !strings.HasPrefix(entry.Name(), lBase) { continue } if stamp, found := strings.CutPrefix(entry.Name(), fmt.Sprintf("%s%s", lBase, ".")); found { _, err := time.Parse("2006:01:02:15:04:05.999999999", strings.Replace(stamp, ".", ":", 5)) if err == nil { backups = append(backups, entry.Name()) } } } currBackups := len(backups) maxBackups := l.maxNumFiles - 1 if currBackups > maxBackups { // backups sorted oldest to latest based on timestamped lexical filename (ReadDir) for i := 0; i < currBackups-maxBackups; i++ { if err := os.Remove(filepath.Join(lDir, string(os.PathSeparator), backups[i])); err != nil { l.logDirect(l.l.errorLabel, "Unable to remove backup log file %q (%v), will attempt next rotation", backups[i], err) // Bail fast, we'll try again next rotation return } l.logDirect(l.l.infoLabel, "Purged log file %q", backups[i]) } } } func (l *fileLogger) Write(b []byte) (int, error) { if atomic.LoadInt32(&l.canRotate) == 0 { n, err := l.f.Write(b) if err == nil { atomic.AddInt64(&l.out, int64(n)) } return n, err } l.Lock() n, err := l.f.Write(b) if err == nil { l.out += int64(n) if l.out > l.limit { if err := l.f.Close(); err != nil { l.limit *= 2 l.logDirect(l.l.errorLabel, "Unable to close logfile for rotation (%v), will attempt next rotation at size %v", err, l.limit) l.Unlock() return n, err } fname := l.f.Name() now := time.Now() bak := fmt.Sprintf("%s.%04d.%02d.%02d.%02d.%02d.%02d.%09d", fname, now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()) os.Rename(fname, bak) fileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE f, err := os.OpenFile(fname, fileflags, defaultLogPerms) if err != nil { l.Unlock() panic(fmt.Sprintf("Unable to re-open the logfile %q after rotation: %v", fname, err)) } l.f = f n := l.logDirect(l.l.infoLabel, "Rotated log, backup saved as %q", bak) l.out = int64(n) l.limit = l.olimit if l.maxNumFiles > 0 { l.logPurge(fname) } } } l.Unlock() return n, err } func (l *fileLogger) close() error { l.Lock() if l.closed { l.Unlock() return nil } l.closed = true l.Unlock() return l.f.Close() } // SetSizeLimit sets the size of a logfile after which a backup // is created with the file name + "year.month.day.hour.min.sec.nanosec" // and the current log is truncated. func (l *Logger) SetSizeLimit(limit int64) error { l.Lock() if l.fl == nil { l.Unlock() return fmt.Errorf("can set log size limit only for file logger") } fl := l.fl l.Unlock() fl.setLimit(limit) return nil } // SetMaxNumFiles sets the number of archived log files that will be retained func (l *Logger) SetMaxNumFiles(max int) error { l.Lock() if l.fl == nil { l.Unlock() return fmt.Errorf("can set log max number of files only for file logger") } fl := l.fl l.Unlock() fl.setMaxNumFiles(max) return nil } // NewTestLogger creates a logger with output directed to Stderr with a prefix. // Useful for tracing in tests when multiple servers are in the same pid func NewTestLogger(prefix string, time bool) *Logger { flags := 0 if time { flags = log.LstdFlags | log.Lmicroseconds } l := &Logger{ logger: log.New(os.Stderr, prefix, flags), debug: true, trace: true, } setColoredLabelFormats(l) return l } // Close implements the io.Closer interface to clean up // resources in the server's logger implementation. // Caller must ensure threadsafety. func (l *Logger) Close() error { if l.fl != nil { return l.fl.close() } return nil } // Generate the pid prefix string func pidPrefix() string { return fmt.Sprintf("[%d] ", os.Getpid()) } func setPlainLabelFormats(l *Logger) { l.infoLabel = "[INF] " l.debugLabel = "[DBG] " l.warnLabel = "[WRN] " l.errorLabel = "[ERR] " l.fatalLabel = "[FTL] " l.traceLabel = "[TRC] " } func setColoredLabelFormats(l *Logger) { colorFormat := "[\x1b[%sm%s\x1b[0m] " l.infoLabel = fmt.Sprintf(colorFormat, "32", "INF") l.debugLabel = fmt.Sprintf(colorFormat, "36", "DBG") l.warnLabel = fmt.Sprintf(colorFormat, "0;93", "WRN") l.errorLabel = fmt.Sprintf(colorFormat, "31", "ERR") l.fatalLabel = fmt.Sprintf(colorFormat, "31", "FTL") l.traceLabel = fmt.Sprintf(colorFormat, "33", "TRC") } // Noticef logs a notice statement func (l *Logger) Noticef(format string, v ...any) { l.logger.Printf(l.infoLabel+format, v...) } // Warnf logs a notice statement func (l *Logger) Warnf(format string, v ...any) { l.logger.Printf(l.warnLabel+format, v...) } // Errorf logs an error statement func (l *Logger) Errorf(format string, v ...any) { l.logger.Printf(l.errorLabel+format, v...) } // Fatalf logs a fatal error func (l *Logger) Fatalf(format string, v ...any) { l.logger.Fatalf(l.fatalLabel+format, v...) } // Debugf logs a debug statement func (l *Logger) Debugf(format string, v ...any) { if l.debug { l.logger.Printf(l.debugLabel+format, v...) } } // Tracef logs a trace statement func (l *Logger) Tracef(format string, v ...any) { if l.trace { l.logger.Printf(l.traceLabel+format, v...) } } nats-server-2.10.27/logger/log_test.go000066400000000000000000000206751477524627100176130ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logger import ( "bytes" "fmt" "io" "log" "os" "path/filepath" "regexp" "strings" "testing" ) func TestStdLogger(t *testing.T) { logger := NewStdLogger(false, false, false, false, false) flags := logger.logger.Flags() if flags != 0 { t.Fatalf("Expected %q, received %q\n", 0, flags) } if logger.debug { t.Fatalf("Expected %t, received %t\n", false, logger.debug) } if logger.trace { t.Fatalf("Expected %t, received %t\n", false, logger.trace) } } func TestStdLoggerWithDebugTraceAndTime(t *testing.T) { logger := NewStdLogger(true, true, true, false, false) flags := logger.logger.Flags() if flags != log.LstdFlags|log.Lmicroseconds { t.Fatalf("Expected %d, received %d\n", log.LstdFlags, flags) } if !logger.debug { t.Fatalf("Expected %t, received %t\n", true, logger.debug) } if !logger.trace { t.Fatalf("Expected %t, received %t\n", true, logger.trace) } } func TestStdLoggerNotice(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, false, false, false, false) logger.Noticef("foo") }, "[INF] foo\n") } func TestStdLoggerNoticeWithColor(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, false, false, true, false) logger.Noticef("foo") }, "[\x1b[32mINF\x1b[0m] foo\n") } func TestStdLoggerDebug(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, true, false, false, false) logger.Debugf("foo %s", "bar") }, "[DBG] foo bar\n") } func TestStdLoggerDebugWithOutDebug(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, false, false, false, false) logger.Debugf("foo") }, "") } func TestStdLoggerTrace(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, false, true, false, false) logger.Tracef("foo") }, "[TRC] foo\n") } func TestStdLoggerTraceWithOutDebug(t *testing.T) { expectOutput(t, func() { logger := NewStdLogger(false, false, false, false, false) logger.Tracef("foo") }, "") } func TestFileLogger(t *testing.T) { tmpDir := t.TempDir() file := createFileAtDir(t, tmpDir, "nats-server:log_") file.Close() logger := NewFileLogger(file.Name(), false, false, false, false) defer logger.Close() logger.Noticef("foo") buf, err := os.ReadFile(file.Name()) if err != nil { t.Fatalf("Could not read logfile: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length logfile") } if string(buf) != "[INF] foo\n" { t.Fatalf("Expected '%s', received '%s'\n", "[INFO] foo", string(buf)) } file = createFileAtDir(t, tmpDir, "nats-server:log_") file.Close() logger = NewFileLogger(file.Name(), true, false, true, true) defer logger.Close() logger.Errorf("foo") buf, err = os.ReadFile(file.Name()) if err != nil { t.Fatalf("Could not read logfile: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length logfile") } str := string(buf) errMsg := fmt.Sprintf("Expected '%s', received '%s'\n", "[pid] [ERR] foo", str) pidEnd := strings.Index(str, " ") infoStart := strings.LastIndex(str, "[ERR]") if pidEnd == -1 || infoStart == -1 { t.Fatalf("%v", errMsg) } pid := str[0:pidEnd] if pid[0] != '[' || pid[len(pid)-1] != ']' { t.Fatalf("%v", errMsg) } date := str[pidEnd:infoStart] dateRegExp := "[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}" reg, err := regexp.Compile(dateRegExp) if err != nil { t.Fatalf("Compile date regexp error: %v", err) } if matched := reg.Match([]byte(date)); !matched { t.Fatalf("Date string '%s' does not match '%s'", date, dateRegExp) } if !strings.HasSuffix(str, "[ERR] foo\n") { t.Fatalf("%v", errMsg) } } func TestFileLoggerSizeLimit(t *testing.T) { // Create std logger logger := NewStdLogger(true, false, false, false, true) if err := logger.SetSizeLimit(1000); err == nil || !strings.Contains(err.Error(), "only for file logger") { t.Fatalf("Expected error about being able to use only for file logger, got %v", err) } logger.Close() tmpDir := t.TempDir() file := createFileAtDir(t, tmpDir, "log_") file.Close() logger = NewFileLogger(file.Name(), true, false, false, true) defer logger.Close() logger.SetSizeLimit(1000) for i := 0; i < 50; i++ { logger.Noticef("This is a line in the log file") } files, err := os.ReadDir(tmpDir) if err != nil { t.Fatalf("Error reading logs dir: %v", err) } if len(files) == 1 { t.Fatalf("Expected file to have been rotated") } lastBackup := files[len(files)-1] if err := logger.Close(); err != nil { t.Fatalf("Error closing log: %v", err) } content, err := os.ReadFile(file.Name()) if err != nil { t.Fatalf("Error loading latest log: %v", err) } if !bytes.Contains(content, []byte("Rotated log")) || !bytes.Contains(content, []byte(lastBackup.Name())) { t.Fatalf("Should be statement about rotated log and backup name, got %s", content) } tmpDir = t.TempDir() // Recreate logger and don't set a limit file = createFileAtDir(t, tmpDir, "log_") file.Close() logger = NewFileLogger(file.Name(), true, false, false, true) defer logger.Close() for i := 0; i < 50; i++ { logger.Noticef("This is line %d in the log file", i+1) } files, err = os.ReadDir(tmpDir) if err != nil { t.Fatalf("Error reading logs dir: %v", err) } if len(files) != 1 { t.Fatalf("Expected file to not be rotated") } // Now set a limit that is below current size logger.SetSizeLimit(1000) // Should have triggered rotation files, err = os.ReadDir(tmpDir) if err != nil { t.Fatalf("Error reading logs dir: %v", err) } if len(files) <= 1 { t.Fatalf("Expected file to have been rotated") } if err := logger.Close(); err != nil { t.Fatalf("Error closing log: %v", err) } lastBackup = files[len(files)-1] content, err = os.ReadFile(file.Name()) if err != nil { t.Fatalf("Error loading latest log: %v", err) } if !bytes.Contains(content, []byte("Rotated log")) || !bytes.Contains(content, []byte(lastBackup.Name())) { t.Fatalf("Should be statement about rotated log and backup name, got %s", content) } logger = NewFileLogger(file.Name(), true, false, false, true) defer logger.Close() logger.SetSizeLimit(1000) // Check error on rotate. logger.Lock() logger.fl.Lock() failClose := &fileLogFailClose{logger.fl.f, true} logger.fl.f = failClose logger.fl.Unlock() logger.Unlock() // Write a big line that will force rotation. // Since we fail to close the log file, we should have bumped the limit to 2000 logger.Noticef("This is a big line: %v", make([]byte, 1000)) // Remove the failure failClose.fail = false // Write a big line that makes rotation happen logger.Noticef("This is a big line: %v", make([]byte, 2000)) // Close logger.Close() files, err = os.ReadDir(tmpDir) if err != nil { t.Fatalf("Error reading logs dir: %v", err) } lastBackup = files[len(files)-1] content, err = os.ReadFile(filepath.Join(tmpDir, lastBackup.Name())) if err != nil { t.Fatalf("Error reading backup file: %v", err) } if !bytes.Contains(content, []byte("on purpose")) || !bytes.Contains(content, []byte("size 2000")) { t.Fatalf("Expected error that file could not rotated and max size bumped to 2000, got %s", content) } } type fileLogFailClose struct { writerAndCloser fail bool } func (l *fileLogFailClose) Close() error { if l.fail { return fmt.Errorf("on purpose") } return l.writerAndCloser.Close() } func expectOutput(t *testing.T, f func(), expected string) { old := os.Stderr // keep backup of the real stderr r, w, _ := os.Pipe() os.Stderr = w f() outC := make(chan string) // copy the output in a separate goroutine so printing can't block indefinitely go func() { var buf bytes.Buffer io.Copy(&buf, r) outC <- buf.String() }() os.Stderr.Close() os.Stderr = old // restoring the real stdout out := <-outC if out != expected { t.Fatalf("Expected '%s', received '%s'\n", expected, out) } } func createFileAtDir(t *testing.T, dir, prefix string) *os.File { t.Helper() f, err := os.CreateTemp(dir, prefix) if err != nil { t.Fatal(err) } return f } nats-server-2.10.27/logger/syslog.go000066400000000000000000000064511477524627100173070ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows // +build !windows package logger import ( "fmt" "log" "log/syslog" "net/url" "os" "strings" ) // SysLogger provides a system logger facility type SysLogger struct { writer *syslog.Writer debug bool trace bool } // SetSyslogName sets the name to use for the syslog. // Currently used only on Windows. func SetSyslogName(name string) {} // GetSysLoggerTag generates the tag name for use in syslog statements. If // the executable is linked, the name of the link will be used as the tag, // otherwise, the name of the executable is used. "nats-server" is the default // for the NATS server. func GetSysLoggerTag() string { procName := os.Args[0] if strings.ContainsRune(procName, os.PathSeparator) { parts := strings.FieldsFunc(procName, func(c rune) bool { return c == os.PathSeparator }) procName = parts[len(parts)-1] } return procName } // NewSysLogger creates a new system logger func NewSysLogger(debug, trace bool) *SysLogger { w, err := syslog.New(syslog.LOG_DAEMON|syslog.LOG_NOTICE, GetSysLoggerTag()) if err != nil { log.Fatalf("error connecting to syslog: %q", err.Error()) } return &SysLogger{ writer: w, debug: debug, trace: trace, } } // NewRemoteSysLogger creates a new remote system logger func NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger { network, addr := getNetworkAndAddr(fqn) w, err := syslog.Dial(network, addr, syslog.LOG_DEBUG, GetSysLoggerTag()) if err != nil { log.Fatalf("error connecting to syslog: %q", err.Error()) } return &SysLogger{ writer: w, debug: debug, trace: trace, } } func getNetworkAndAddr(fqn string) (network, addr string) { u, err := url.Parse(fqn) if err != nil { log.Fatal(err) } network = u.Scheme if network == "udp" || network == "tcp" { addr = u.Host } else if network == "unix" { addr = u.Path } else { log.Fatalf("error invalid network type: %q", u.Scheme) } return } // Noticef logs a notice statement func (l *SysLogger) Noticef(format string, v ...any) { l.writer.Notice(fmt.Sprintf(format, v...)) } // Warnf logs a warning statement func (l *SysLogger) Warnf(format string, v ...any) { l.writer.Warning(fmt.Sprintf(format, v...)) } // Fatalf logs a fatal error func (l *SysLogger) Fatalf(format string, v ...any) { l.writer.Crit(fmt.Sprintf(format, v...)) } // Errorf logs an error statement func (l *SysLogger) Errorf(format string, v ...any) { l.writer.Err(fmt.Sprintf(format, v...)) } // Debugf logs a debug statement func (l *SysLogger) Debugf(format string, v ...any) { if l.debug { l.writer.Debug(fmt.Sprintf(format, v...)) } } // Tracef logs a trace statement func (l *SysLogger) Tracef(format string, v ...any) { if l.trace { l.writer.Notice(fmt.Sprintf(format, v...)) } } nats-server-2.10.27/logger/syslog_test.go000066400000000000000000000130001477524627100203320ustar00rootroot00000000000000// Copyright 2012-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows // +build !windows package logger import ( "fmt" "log" "net" "os" "path/filepath" "strings" "testing" "time" ) var serverFQN string func TestSysLogger(t *testing.T) { logger := NewSysLogger(false, false) if logger.debug { t.Fatalf("Expected %t, received %t\n", false, logger.debug) } if logger.trace { t.Fatalf("Expected %t, received %t\n", false, logger.trace) } } func TestSysLoggerWithDebugAndTrace(t *testing.T) { logger := NewSysLogger(true, true) if !logger.debug { t.Fatalf("Expected %t, received %t\n", true, logger.debug) } if !logger.trace { t.Fatalf("Expected %t, received %t\n", true, logger.trace) } } func testTag(t *testing.T, exePath, expected string) { os.Args[0] = exePath if result := GetSysLoggerTag(); result != expected { t.Fatalf("Expected %s, received %s", expected, result) } } func restoreArg(orig string) { os.Args[0] = orig } func TestSysLoggerTagGen(t *testing.T) { origArg := os.Args[0] defer restoreArg(origArg) testTag(t, "nats-server", "nats-server") testTag(t, filepath.Join(".", "nats-server"), "nats-server") testTag(t, filepath.Join("home", "bin", "nats-server"), "nats-server") testTag(t, filepath.Join("..", "..", "nats-server"), "nats-server") testTag(t, "nats-server.service1", "nats-server.service1") testTag(t, "nats-server_service1", "nats-server_service1") testTag(t, "nats-server-service1", "nats-server-service1") testTag(t, "nats-server service1", "nats-server service1") } func TestSysLoggerTag(t *testing.T) { origArg := os.Args[0] defer restoreArg(origArg) os.Args[0] = "ServerLoggerTag" done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, true) logger.Noticef("foo") line := <-done data := strings.Split(line, "[") if len(data) != 2 { t.Fatalf("Unexpected syslog line %s\n", line) } if !strings.Contains(data[0], os.Args[0]) { t.Fatalf("Expected '%s', received '%s'\n", os.Args[0], data[0]) } } func TestRemoteSysLogger(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, true) if !logger.debug { t.Fatalf("Expected %t, received %t\n", true, logger.debug) } if !logger.trace { t.Fatalf("Expected %t, received %t\n", true, logger.trace) } logger.Noticef("foo") <-done } func TestRemoteSysLoggerNotice(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, true) logger.Noticef("foo %s", "bar") expectSyslogOutput(t, <-done, "foo bar\n") } func TestRemoteSysLoggerDebug(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, true) logger.Debugf("foo %s", "qux") expectSyslogOutput(t, <-done, "foo qux\n") } func TestRemoteSysLoggerDebugDisabled(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, false, false) logger.Debugf("foo %s", "qux") rcvd := <-done if rcvd != "" { t.Fatalf("Unexpected syslog response %s\n", rcvd) } } func TestRemoteSysLoggerTrace(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, true) logger.Tracef("foo %s", "qux") expectSyslogOutput(t, <-done, "foo qux\n") } func TestRemoteSysLoggerTraceDisabled(t *testing.T) { done := make(chan string) startServer(done) logger := NewRemoteSysLogger(serverFQN, true, false) logger.Tracef("foo %s", "qux") rcvd := <-done if rcvd != "" { t.Fatalf("Unexpected syslog response %s\n", rcvd) } } func TestGetNetworkAndAddrUDP(t *testing.T) { n, a := getNetworkAndAddr("udp://foo.com:1000") if n != "udp" { t.Fatalf("Unexpected network %s\n", n) } if a != "foo.com:1000" { t.Fatalf("Unexpected addr %s\n", a) } } func TestGetNetworkAndAddrTCP(t *testing.T) { n, a := getNetworkAndAddr("tcp://foo.com:1000") if n != "tcp" { t.Fatalf("Unexpected network %s\n", n) } if a != "foo.com:1000" { t.Fatalf("Unexpected addr %s\n", a) } } func TestGetNetworkAndAddrUnix(t *testing.T) { n, a := getNetworkAndAddr("unix:///foo.sock") if n != "unix" { t.Fatalf("Unexpected network %s\n", n) } if a != "/foo.sock" { t.Fatalf("Unexpected addr %s\n", a) } } func expectSyslogOutput(t *testing.T, line string, expected string) { data := strings.Split(line, "]: ") if len(data) != 2 { t.Fatalf("Unexpected syslog line %s\n", line) } if data[1] != expected { t.Fatalf("Expected '%s', received '%s'\n", expected, data[1]) } } func runSyslog(c net.PacketConn, done chan<- string) { var buf [4096]byte var rcvd string for { n, _, err := c.ReadFrom(buf[:]) if err != nil || n == 0 { break } rcvd += string(buf[:n]) } done <- rcvd } func startServer(done chan<- string) { c, e := net.ListenPacket("udp", "127.0.0.1:0") if e != nil { log.Fatalf("net.ListenPacket failed udp :0 %v", e) } serverFQN = fmt.Sprintf("udp://%s", c.LocalAddr().String()) c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) go runSyslog(c, done) } nats-server-2.10.27/logger/syslog_windows.go000066400000000000000000000056421477524627100210620ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package logger logs to the windows event log package logger import ( "fmt" "os" "strings" "golang.org/x/sys/windows/svc/eventlog" ) var natsEventSource = "NATS-Server" // SetSyslogName sets the name to use for the system log event source func SetSyslogName(name string) { natsEventSource = name } // SysLogger logs to the windows event logger type SysLogger struct { writer *eventlog.Log debug bool trace bool } // NewSysLogger creates a log using the windows event logger func NewSysLogger(debug, trace bool) *SysLogger { if err := eventlog.InstallAsEventCreate(natsEventSource, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil { if !strings.Contains(err.Error(), "registry key already exists") { panic(fmt.Sprintf("could not access event log: %v", err)) } } w, err := eventlog.Open(natsEventSource) if err != nil { panic(fmt.Sprintf("could not open event log: %v", err)) } return &SysLogger{ writer: w, debug: debug, trace: trace, } } // NewRemoteSysLogger creates a remote event logger func NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger { w, err := eventlog.OpenRemote(fqn, natsEventSource) if err != nil { panic(fmt.Sprintf("could not open event log: %v", err)) } return &SysLogger{ writer: w, debug: debug, trace: trace, } } func formatMsg(tag, format string, v ...any) string { orig := fmt.Sprintf(format, v...) return fmt.Sprintf("pid[%d][%s]: %s", os.Getpid(), tag, orig) } // Noticef logs a notice statement func (l *SysLogger) Noticef(format string, v ...any) { l.writer.Info(1, formatMsg("NOTICE", format, v...)) } // Warnf logs a warning statement func (l *SysLogger) Warnf(format string, v ...any) { l.writer.Info(1, formatMsg("WARN", format, v...)) } // Fatalf logs a fatal error func (l *SysLogger) Fatalf(format string, v ...any) { msg := formatMsg("FATAL", format, v...) l.writer.Error(5, msg) panic(msg) } // Errorf logs an error statement func (l *SysLogger) Errorf(format string, v ...any) { l.writer.Error(2, formatMsg("ERROR", format, v...)) } // Debugf logs a debug statement func (l *SysLogger) Debugf(format string, v ...any) { if l.debug { l.writer.Info(3, formatMsg("DEBUG", format, v...)) } } // Tracef logs a trace statement func (l *SysLogger) Tracef(format string, v ...any) { if l.trace { l.writer.Info(4, formatMsg("TRACE", format, v...)) } } nats-server-2.10.27/logger/syslog_windows_test.go000066400000000000000000000076311477524627100221210ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows // +build windows package logger import ( "os/exec" "strings" "testing" "golang.org/x/sys/windows/svc/eventlog" ) // Skips testing if we do not have privledges to run this test. // This lets us skip the tests for general (non admin/system) users. func checkPrivledges(t *testing.T) { src := "NATS-eventlog-testsource" defer eventlog.Remove(src) if err := eventlog.InstallAsEventCreate(src, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil { if strings.Contains(err.Error(), "Access is denied") { // Skip this test because elevated privileges are required. t.SkipNow() } // let the tests report other types of errors } } // lastLogEntryContains reads the last entry (/c:1 /rd:true) written // to the event log by the NATS-Server source, returning true if the // passed text was found, false otherwise. func lastLogEntryContains(t *testing.T, text string) bool { var output []byte var err error cmd := exec.Command("wevtutil.exe", "qe", "Application", "/q:*[System[Provider[@Name='NATS-Server']]]", "/rd:true", "/c:1") if output, err = cmd.Output(); err != nil { t.Fatalf("Unable to execute command: %v", err) } return strings.Contains(string(output), text) } // TestSysLogger tests event logging on windows func TestSysLogger(t *testing.T) { checkPrivledges(t) logger := NewSysLogger(false, false) if logger.debug { t.Fatalf("Expected %t, received %t\n", false, logger.debug) } if logger.trace { t.Fatalf("Expected %t, received %t\n", false, logger.trace) } logger.Noticef("%s", "Noticef") if !lastLogEntryContains(t, "[NOTICE]: Noticef") { t.Fatalf("missing log entry") } logger.Errorf("%s", "Errorf") if !lastLogEntryContains(t, "[ERROR]: Errorf") { t.Fatalf("missing log entry") } logger.Tracef("%s", "Tracef") if lastLogEntryContains(t, "Tracef") { t.Fatalf("should not contain log entry") } logger.Debugf("%s", "Debugf") if lastLogEntryContains(t, "Debugf") { t.Fatalf("should not contain log entry") } } // TestSysLoggerWithDebugAndTrace tests event logging func TestSysLoggerWithDebugAndTrace(t *testing.T) { checkPrivledges(t) logger := NewSysLogger(true, true) if !logger.debug { t.Fatalf("Expected %t, received %t\n", true, logger.debug) } if !logger.trace { t.Fatalf("Expected %t, received %t\n", true, logger.trace) } logger.Tracef("%s", "Tracef") if !lastLogEntryContains(t, "[TRACE]: Tracef") { t.Fatalf("missing log entry") } logger.Debugf("%s", "Debugf") if !lastLogEntryContains(t, "[DEBUG]: Debugf") { t.Fatalf("missing log entry") } } // TestRemoteSysLoggerWithDebugAndTrace tests remote event logging func TestRemoteSysLoggerWithDebugAndTrace(t *testing.T) { checkPrivledges(t) logger := NewRemoteSysLogger("", true, true) if !logger.debug { t.Fatalf("Expected %t, received %t\n", true, logger.debug) } if !logger.trace { t.Fatalf("Expected %t, received %t\n", true, logger.trace) } logger.Tracef("NATS %s", "[TRACE]: Remote Noticef") if !lastLogEntryContains(t, "Remote Noticef") { t.Fatalf("missing log entry") } } func TestSysLoggerFatalf(t *testing.T) { defer func() { if r := recover(); r != nil { if !lastLogEntryContains(t, "[FATAL]: Fatalf") { t.Fatalf("missing log entry") } } }() checkPrivledges(t) logger := NewSysLogger(true, true) logger.Fatalf("%s", "Fatalf") t.Fatalf("did not panic when expected to") } nats-server-2.10.27/logos/000077500000000000000000000000001477524627100152765ustar00rootroot00000000000000nats-server-2.10.27/logos/nats-horizontal-color.png000066400000000000000000000526601477524627100222650ustar00rootroot00000000000000‰PNG  IHDRýÌöÑd pHYs.#.#x¥?v IDATxÚìÝ”Õõ}àÿ÷ܹsçÞ;¿o.sap¤dˆ«"‚ÆБ°¡¡’bz‚ß ÃÒ¥‰-»Ò&Ç´»Kö»ìn»œ¥+[ͦ›Z»+ûMLÖYSµD%!ÅŠQ¨!ng˜a~3 ß?dâ€h¸÷3Ç999ˆ|^Ÿ{çÇ}Þ÷û]rüøñ¯˜@qý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~Päâãõ]¾õg7…ž2RΧW–_ZRë¸ó߇þ_w€óé… âëÞõ·}éoC-îçѶü忼©þ"›ž»É×=ηÿðÙþöß÷j¥9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢9ÑŠœèENô€"'ú@‘ý È‰~PäD?(r¢¹¸@ô”Ç+C}zÆ»~Lmrj¶¢éàùú;ŒNþCÇ÷sïôû]ƒB÷à7 ÆèfRÅŒ,­ !¼ïk®Ý7ú{µå ±ºTãÈدMNvŽÿ©TaÚù¼–æ)·žÕÇ;~´ÿÈàÁ“BdÛ‘—âƒÇz‡Cx{H<Ø·' ÷xÐ0á‰~6³®!T%R~;Ú÷z0¡±úšBSª®è¬Hd»ËK+â UW‡Bª¬¶º¼´¢îþÕieF¥%e©S#fmrêIóN!±g(ß><28Â[¡°w(_½ÿÈËu!„D›è7ΪËbáòÚ„Aœ¥W…î£#Á¸˜U›UeÅ}dé‘£#a÷á!7xOšsMáÞE·Dáî=nÀ¸ºþ¶/¹êòÉ¡¾bFh¬¹vßäŠËcUåõ‰±Á)ˆzmìjÂLêÒÂÛWxcßh]-8p¬'êÝc€$Ñ‚ÐŽgþæ׺Vm;Xcu¼w¬ÊW'RY“8ñMo¬4½©eu~éCd&ÀY›T1#L®ø@ÿÔê«;ª®öÓ†í{ ø…I3ÂôÚyWLúh_&uéhØË™ gktÛÐSÏ<ÖÛÙ;ô}ûº^níúá´ƒ½{B÷àc\‰~œÑð÷/¾w(µýЀÀY¸ïŸþNGôݬ•€Œ%úQ6Ìͤf¿/‘_ÿb‡ pŠ V屸¯‘gûp¬4½©eu~éCdvåñÊ0¯aE×µ Ë'VõYH@äT&²¹+ëoWÖßBxs%àˇ¾“~ýðöºC½{ hý(Ÿ˜^™­.‹uÜñ\>c𦕳ZšsM¢Õ9Ê¥k³ëç/ÏßõüV3€"V¯ mú×íM™s!„a"ɤ.ú¡K?>té§Ãà±ÞνÏôíéxzꞎg g‚ý(*‹/Ig6ßþ „0³®!¬½dÄ$Þ›eMó²ýô‡aGû^À"sÊÊ>[xâ9QZQwŤ_«»bÒ¯…cÇöÿìðŽü®ƒ7 €ƒèGÑ ë_ìÈtõ:'צ–Õùx¬Ô µq°yáš~ã‹É#Cý†Eâªú›û?<ý÷JJceVöÁi”–”¥¦×]ß8½îzp‚ý(J‹/Ig®Ì$:nyü€ðÀ„´~þò|.]+ø“D,žÜ¸`Uþö'î5S(p“*f„%3þm{6=ÝÊ>8C§À݇ïýÑof-1# X5¤ã™‡Oî¨.ó0`biÎ5…eMóÄ©ñŸkv嬖“€ÂµàÒOwÝvõW‚àç®´¤,ueýÍÙÛ®þJh¬¾Æ@"D-¡¨ L4U‰Tؼp0už¬½ddf]ƒA@)W†Û®þ‹üuSWØÊà(%½†t<ó7¿ÞÐ5«6aDÞÆ«ò‰X,Ÿ^ÙoDÁúùËó¹t­m=/°D,žÜ¸`•m>à[Ütgþ²÷ýjÆ$ÎŽèGdm˜›I »æ\SXÖ4Oð»xóÏ®œÕ2`pa\Usÿ•õ7ûàˆ~Dšð@1«J¤Âæ…k§‹líì%#3ë Î3gø¼7¢‘·an&u×ÜŒ­¹(:¬Ê'bqÕ^dñXizSËjßKÀy4©bFXôþß?nçNôcBøÄôÊìæ²&@±X9«e 9×dK›‘K×f×Ï_.üÀyP¯ ˯øÓ®XI¯ª1 €÷FôcBY|I:óõ›êóÕeú®M-«óñX©w¸ Í × T%1ãenÃÿ30©Â.'ãAù`™7)™}hñäá€B´~þò|.]ëÞ•ˆÅ“¬²Í'ŒƒêòÉáC—þ¶]NƉêÁ„ÔŽg„? Ms®),kš'øþ}Ê®œÕ2`ðÞ,ûÀåã0~&,á€BR•H…Í ×IEbíì%#3ë ÎÑeïûÕ>ÛzŒ/µƒ ­!Ï<óSfÕ& €‹jã‚UùD,ž4‰â•¦7µ¬¶Í'œ£¼ÿΣ¦0¾D?&¼D¬$ùà¢\¿ðÀŲrVË@sÎ;\‹M.]›]?¹ðgéªú›û“ñª“_¢„Êb%)ဋaf]CX;{I$®ÿ£çÿwÿkm‘ŽbËšæe›sMÈpL»}ÈÆŸè'Œ†¿Å—¤ûL€ eSËê|&bñäÆ«ló gàúÆU½¦p~ˆ~pæfRÂçSs®),kšÉsüîxê+É#Co}}hïó©í{ó¿ŸÙ•³Z<²àMª˜ªõÎ38OâF§·an&BèßúzýºWU‰TؼpÍ@!µkûÖÞíùí{ßöCüOoÉ~ç7ÿÝ@"OFõ¾®½ddû׫mäps&<Bý΃ž¡|ûðÈàÛވش+v ÷•³:G¾µë‡Ó‡{Îùïò+µóºâ¥Éçû½òÒŠxCÕ•Ãc-+OV&²9wÞ;ÑÞņ¹™Ô‚ÉÉŽ;žËgL€ñ²qÁª|"ܺí}‡ówÿèáÓ^ב¡þpÇS_IÞ»èöè~c+MojY_úÐyNã²÷ÝTj ïìØñ£ýG !„ÎþÖØáÁ¶‘BÈ÷î­?<ðÆ/&t ݃Ný× &šµvï¬ !œó¹Õå“CMùäBåñÊÐXsí¾N†©²ÚêòÒŠ:8™è¿ÄâKÒ™Í7d…?ÆÅÊY-͹¦HF¡Ïnûjv춞§ÚѾ7|kïö|T·5 !„\º6»~þòü]Ïoþ`ŒI3‚HÂá7ö½Ñuc"^*„0m¢Ï¨û”¨¹§ã™wIcõ5!„j“Sû³MGã „LD¢§õô¾““i“x“ðÀx˜Y×ÖÎ^2ÅkÛòòS]¯v¶ýÒwrÞý£‡³×O™™Ï¥k#Å–5ÍË>öÓ†í{=èà„+&-n´í|;vühÿÁž×òm=/%þ¡ãû¹1aošGÃøjíÞ9úÿ§¦åñÊPŸžñ‹Uƒ£QÐjA¢Hôã´þû+ÝéÇÞ×â\;›áïë7Õçÿå³ùl÷Ñà¬mjYÇJ#»^ëlËiç£gt]G†úÃg·}5ûWK~?Ò÷zóÂ5ýÆ“ï¶ò&’éµ×Gý÷y¹uoç3•¯Þ^w¨wO*„ÐèÎ_|ƒÃ=¿ƒ§[58©bFH–V†÷g>ØžI^:T—j©*¯¯/-)óÚ8EGôãm}½'BþƘ7)™}hñäŽ[?þ8ëç/ä궣#Çú?÷½¯Õu½ÚÙ¶¼üTת+ÖDõ~'bñäÆ«ò·?q¯m>!"þú±íñÒØ€I¯\¶*ù«×NÄ “¾¡|ä{¯ô¸«ÅkøØH2L OB™Ô¥S£x]ÃGº^lûë‘·»np¸Gä+B‡z÷„BhíÞyÒçåÑ‚Sª®è̤¦õ5T]9,RèD?Þ•ð÷v éxFøàl4çšBTϱ[ÿì_ßßÛyÖÿÞ—v>ZóÁ)È_V×ð}Ï®œÕ2pÿîmIÏ(~w?ðŒç‹Üµ³¦†_½vz$®¥£«¿ç¿üå÷lÑ3×¢d`øH×ÿùÉjZ»wÖ¸ÃÑ4ºB°µ{g]áÛ€V—O5å“Ãû3lo¨¼r¨ª¼>áì@ …èÇ/µõõžÔKCáÁE¹þ²X‰ø„?Î\U"6/\3Bˆ\øÙѾ7ÿdë®sŽvŸûÞײßøØôÅc¥‘=Gxíì%#Û¼^ílód`šRuEgMŠÝ o<ÐõôϾ,öMPÝ'Îg¸(×ëí©Ý‡‡ €“¬œÕ2ÐœkŠäö•ŸzKÍ‘¡÷þ^ ‡ö>Ÿúè¯ÌÉGuN!„K×f×ÍYš¿ëù­Î÷`BªK5Fb«¬ïîÝØ¿ëà#^åŒØô+þªË'‡úŠaFfÁ—Ö4Ç­d¼YéÇY ûûŽåMãMe±’Ôƒ‹rýVü0Ö̺†°vö’Hîý­½Ûó;Ú÷ŽÛŸwçÓ[²C#ÃQ~<,kš—ýpãU}žLDUåõõÅ~ ­Ý;ó‚ïU÷à°§ã™ð=ÿiê—w|"wÏ K±°£ÿgo˜ï•èÇYÛ}x(,{|¶­o¸Ã4Þ4þ_’öb!„6µ¬ÎGñ¬ºö¾Ãã¾bíÈP¸ã©¯$£þ˜Øp㊘m>˜ˆJKÊŠþáÿ»çíÜÁ¸î ­Ý;öŸþYîk;?5utKЗ>’ïÊ·›gKôãœt ·<~ #ü½¥,V’Ú|C6½|z¥s&¸õó—çséÚHþ@øÙm_=/×µ£}oøÖÞí‘ÞI ‹'ï[t»Ý˜P&UÌ(úkØäåÖîÁn&Dk÷ÎðøÞÙÑ•€ßþÉúðzçœ Èý8gÂßém˜›I Ws®),kšÉàw÷xµ³í|þùÙö¾Ã‘Žb—Õ5dWÎjðL`¢H–Vý5¼tð«ü¸(‡{žŽgÂ7_ùÃÆ{¶ß\÷—·&<ó³/wÚ ”w"úñžt ~¤-óøÏû„¿16Ìͤ~ëÕ^И`ª©°yášH~þ­³-ÿîmçu Î#Cýçm%a!Y7girf]ƒ' ‰½?±O?áPïž°ý꾶óSSïyaiøîÞý¯wþ õØñ£¡Bý'w<—þNñù«k“›oÈš À²qÁª|"ÜÙtGGŽõî{_» 1îÕζ°å姺¢þXÙÔ²:ï|?&‚)UWý–„‡z÷¸‘œÁáž°ëà#©o¾ò‡wÿà#©Ñm@À‰MôcÜo·ø’tFø˜VÎjhÎ5Er•Úúgÿêøþ xtÀ—v>Z³¿·3Ò_?séÚìº9Kï@äU$²Ý¦çßè6 àÄ&ú1®îx.Ÿùï¯tw™Ä[„?€è›Y×ÖÎ^2Åk{²uWÇ“­»Òú¿û™¿ùóÌðȱ¾(?n–5ÍË~¸ñª>Ï ÆÓØøݽû÷y¹ÕT&Ñq÷_v®ù‹ÞA0ÆâKÒ™‡>29_]æ)E›ZVçã±ÒtÔ®«{¨¿ë®ç·f.Æ{og¸çÇEþ ç†WÄló …mRÅ C hí:øHê¯_úÝÆ{^XžùÙ—;†X´a çÅÖ×{RÂßÉ.¯MdZ<¹Cøˆ–õó—çséÚHnëùù§·Ôºx_Îïß½-¹£}o¤·ÀLÄâÉûÝn›O(`“+>àuNŠÞàpOØþÆuöÂÇj¶þýï…×;`õ_©œ7ÂßÛ5¤ãá :šsMaYÓ¼H¿-/?Õµ£}ïEÿ{Üùô–ìÐÈð@”G—Õ5dWÎjðŒ€Â4#³Àõˆ”Öîᛯüaã=/, ûöH„(œW[_ïI}ü»ÂÑ‘ãâß £áojEÜ0ŠXU"6/\ÉPÓÞw8ÿ¥ÖÂßåÈPø³ŒDýñ´nÎÒä̺O,(@Óë®o,W‘38܇{ "BD?λ݇‡Â­O´§„¿·4¤ã™ï|tJÿ¬Ú„a© Vå±x2j×5l<—®Í®›³Ô¶AP€®˜ôkS¯ª¿Ùk›@Aý¸`FÃ_÷ÐH—i¼Iø(>3ëÂÚÙK"¹Ýäkmùûwo+ÈÕ‹G†úÃï+MGíº†F†>óĽ½zñÕζ°åå§"ÿæ¡ 7®ˆÙ怨È÷î­ÊµÄJâéÍübzÁ¥ŸîrÆPŸ§Œ€ ­ûèH¸åñáï-ñXIzÃÜLJø(lëç/ÏçÒµ‘ÜÖó Ï>0rd¨ð¿ }iç£5û{;#ý=D"OÞ·èvÛ| ‡ÞˆÜ;Ú®›º¢æö¹ßì_p駻ªË'»É@Áý¸(„¿Óþ Ws®),kšÉà÷­½ÛóO¶î*šÕ‹Ÿù›?Ï ‹ô–:—Õ5dWÎjðÌ€ÂTZR–ºnꊚ5×þÏðÛÍæçM]Ñ9©b†Á•èÇE3þÿyŸð7Ɔ¹™Ôo} Ú‹|¤*‘ ›®‰äçæî¡þ®»ôpQÅÌý½áž?ùïc×ÍYšœY×à @QkíÞ9~f¬Ï~èÒO×ÝvõWºë¿Ûÿϯùú §ß±oFæCÁJ@àBŠS÷Ñ‘pÇsùÌ沋/IgLäMŸ¿º6yM&ÑqÇsy3(¬Ê'bñH®òû'î­)†m=OuÿîmÉ5ÌÊ7çš²Q~ìmjYÿ䣛²Åx`"*-)KeR—Nͤ. s&ÿæ/~½g(ß>t¬ox_׋Ã#LJ“ÿÐñý\!ìÛ‡{ ¢Aø{»Å—¤3›oÈ ÙÊY-Q K[^~ªëÕζšbýûßùô–ìw~óß $bñdT¹tmvÝœ¥ù»žßšõl Xxc_mrê´‰<ƒÊD6B™Ô¥!„š§ÜzÒï;~´ÿÈàÁƒ!„ÐÙß;<Ø6Bca! ë ‡z÷xP§%úQ0îx.Ÿ¹kn&ÿ‰é•^Ô:Aø¸¸fÖ5„µ³—ŒDñÚö÷vv|iç£EýõåÈPø³ŒüÉ‚OEúq¸¬i^öûm¯ôÓ¹‹0VÛ‘—âµÉ©ñ.JKÊR£aôÔYOÕ3”o<éHŠþ£]±½¯œöçÙÖ®N{·Õ…aKVˆ*Ñ‚²þÅŽìÿq¨ÃÜLÊ4Þ´ø’tæ¡LÎêof»ŽÀ´©eu>+Ü›Q†GŽõ}æoþ<o(y²uWú[{·ç—5Í‹ô›†6ܸ"öÂ7¾ló @1êèßç lçÑè*±j“SÔª+Nûñc·=W§ §3rüX|_׋Ãïö1ï!H8;¢gëë=©‚ð7Æ嵉ìC‹'wÜòøŒðpa¬Ÿ¿<ŸK×F2$ÝóãÇbû{;#s=wÿèáìÂÆ«ºª©š¨>±xò¾E·çW<ö§vD è¼~x{݇.ý´ADÈéBã;ÝÒôœm„»ê¨SW6žº-j"ƒèGAþÞ®!ÏFs®)DuåØŽö½ùûwo‹Ôµê¿óĽ5µä÷#ý¸¼¬®!»rVËÀý»·%=K(&‡z÷„cÇö—–”y­÷lìV¨£N·²ñݶE<ÖÛÙôpw'ÃÞ¡|ü¼˜ IDATõþ#/×…B×àÐ=xÀÀ)*¢këë=©}=Ãá/~uRY¬Ä7á­ðwÛß̼Ñ;l çAU"6/\3Bˆ\X¸óé-‘Œ™¯v¶…-/?յꊅ5Q~|®›³4¹ýÀkáÕÎ6OVŠÊÁž×òSª®h4 AyiE]yiE]ï¾ê¨Ãoì áÍó)õç{÷Öx#5p¬'êÝc  Ñ‚¶ýÐ@¸õ‰öÔƒ‹rÂß éxæ;Òëí©Ý‡‡ `œm\°*ŸˆÅ#ƾðì#Q>îK;­Y<횎)u™(?F7µ¬ÎòÑMYçûPL^:øHö—…(T£+ k“SOûû£[ŽŽ®‚ûö„w:¯ÎÑ‚·ûððwŠ²XIêÁE9á`œ­œÕ2ÐœkŠdðûÖÞíù'[wEþ<¸ÏüÍŸg¾ñ±?è‹ÇJÓQ½Æ\º6»nÎÒü]Ïou¾Ec_ׯëY£[ŽžnÕàhl;òR¼£_zÿ‘—ëÄ@ÎÑ¢0þ¶´ÔwU'b5&"üŒ·™u aíì%‘<4µ½ïpþî=<þé£m5m}æñ¦²XIjë¢\ßòé•ö÷x6µ¬ÎGuuØg·}uBmyÿîmÉ×:ÛòQ¿Î 7®ˆU%¼Y€âñƒÖ-¦o*/­¨Ë¤.:gòoNûÜÖ^÷pX;ï‘Î_þÇ­ó¦®èœT1Ã8k¢E¥ûèH¸åñáï-ñXIzÃÜLJø8wëç/ÏçÒµ‘|·é–—Ÿêzµ³mÂÝÓÏ‹'£xmòÂ7Kö÷vNø{|×ó[3ÝCý‘C̦–ÕyçûP º„½϶›¼7¥%e©¦Ì¹Û®þJX;ï‘Η~ºË ˆ~=áïí_’Îïn嬖æ\S$WÂíhß›hïó Pxs›ÏÏ?½%ò[œæÒµÙus–:߀¢`µŒ¯òÒŠºë¦®¨Ysíÿ Ÿ¼òÏZ«¯1” Jô#îx.Ÿù߯÷x¡k áàͬkkg/‰äyoC#Ãw>½%ë.¿eGûÞ°åå§"¿ÚoYӼ쇯êsÇ(tVûÁù3¥êŠÆåÿ俆ßn~0UýÍý&2±ˆ~DÆú;²_x±Ã'±1_’Η®äJ¸oíÝžßѾ×M~ŸÝöÕìðȱHÿ “ˆÅ“÷-ºÝî¼Áážðèž #&çW2^Uó±™_LjöWÛ'UÌ0ˆSˆáïí„?€75çšÂ²¦y‘ ~í}‡ówÿèaÛz¾‹W;ÛÂ=?~,ò_ /«kÈ®œÕâÓ¼×þñ{éÖîÞ´@6==wÛÕ_ ‹›îÌ—Ç+ $¢"iëë=©OýíÁptä¸øwBC:žùÖâ)ùYµ Ã&¤ªD*l^¸&²!ä³Û¾šµ­ç/wÿîmÉ×:Û"ÿ¢Âº9K“3ëÜp Þ·²>{dè ðÈ•õ7gÿÅœ¿îj¬¾Æ0"Hô#²¶·>ÑžþÞ2%]š}pQ®_ø&¢ Vå±x2Š×v÷xµ³ÍM>CŸyâÞìÐÈpäWÂmjYw¾…np¸'<ôÊ¿É;~Ôkxp$ãU5ËÿÉ .ýt—iD‹èG¤í><$ü¢,V’þ€‰f嬖æ\S$·¾|­³-ÿîmIwùÌê_xöÈŸ’K×f×ÍYêÓ¼C½{»~'%üÁ…uÝÔ5·]ý¶ûŒÑÈ C#Çms‚ðL$3ëÂÚÙK"xŽŽëÿÜ÷¾æ¿sðdë®ô“­»:¢~Ëšæe?ÜxUŸ;@¡þàâ˜TÑ”½íê¿è¨.Ÿl ú1!ì><>ôí7’m}æñ¦²XIê›™–O¯ôi›ZVçã±Òt¯mý³u|o§›|Žîz~k¦{¨?ò[™l¸qEÌ6Ÿá.Žêò\fõœ¿ìŸT1Ã0ŠœèÇ„Ñ}t$ÜòøŒðw² s3)ሪõó—çséÚH®„ÛѾ7ÿdë®´»|îŽ õ‡Ï?½¥&ê×™ˆÅ“÷-ºÝ6Ÿ…Ñðwdè ïaá*-)K­¸êÏ…¿"'ú1¡§'üQÔœk ËšæE2øuõwÝùôÛzŽƒí{Ö—ŸŠüj¿Ëê²+gµØꀢp¨wOøú+ÛÚ½Søƒ ¨´¤,uËåÿÑELôcÂ Û ø¦aŒ s3©Ï]UÛe@T%RaóÂ5‘ ŸzKÍ‘!ïÕ/_ÚùhM{ßáÈ_°nÎÒä̺7€¢08ܶþýïe¿·ïϽy . ªD}ö¶«ÿ¢™"%ú1!u ŸúÛƒÙÇÞç“׿}yuÍæ²f½ Vå±x2Š×ö­½Ûó;Ú÷ºÉãì³Û¾š9ÖõëÜÔ²:ï|?ŠÉ‹mÿ+ù—·&Øî.œêò\fqÓžsEHôcB»ã¹|Fø;ÙâKÒá(f+gµ 4çš"¹õe{ßáü]Ïoµ­çyðjg[¸çÇEþ{ã\º6»nÎR?¸PTõî ÿ}Ç­ÙïíûócÇÚö.€+ëoÎ6V_cEFôcÂþÞNøŠÕ̺†°vö’‘(^Ûðȱ¾ÏnûªàwÝ¿{[òµÎ¶È±eMó²n¼ªÏ Ø¼Øö¿’÷¾øñÔÞŽgÛMο}`C—óýŠ‹èáÍð÷…;¼KhŒÅ—¤3OÞÜÐQ]æÓP<6µ¬ÎÇc¥é(^Û=?~,öjg››|ž}æ‰{³C#Ñ?3dÃ+b¶ù  ÷„oýäßæ¾òÃÄ?8¿’ñªšy +ºL¢xx5NØúzOJø;YC:žyhñdá( ëç/ÏçÒµ‘\ ÷Zg[þþÝÛ’îòùwd¨?|áÙF¢~‰XÑžþÞ2%]š}pQ®_ø ÍÆ«ò‰X<’[_>Ùº«ãÉÖ]iwùÂ;2Ô>ÿô–š‰p­›ZVçï@”ìéx&üõK¿ÛxÏ KÃw÷nìá½³Ú¯8ˆ~ðv ÂßÉÊb%)á($+gµ 4çš"¹õe÷P×]Ïo͸ËÏŽö½á[{·G~›Ï\º6»nÎRçû9ƒÃ=a×ÁGR ¼wM™såñJƒ(p¢¼‹Ñð74rÜy7'Œ†¿y“’†\T3ëÂÚÙKF¢z}ŸzKm=/¾»žßšmï;ù ¶¬i^öÃWõ¹ãDÕظ鹛·²>¼|èÿ¾Ñ3”·z ÎÐìÜo83³À‰~ðKì><>ôí7’m}æñ¦²XIêë7Õ‡åÓ+½ \4›ZVçã±ÒHn}¹å姺v´ïu“ Äg·}5;®sÃ+b¶ù`¢ØÓñLøΞÿ4õË;>‘»ç…¥" œ¹ ŸÔ” œg ûèH¸åñáïdæfRÂp1¬Ÿ¿<ŸK×F2Ä´÷Îiç£5îráxµ³-Üý£‡#¿ê?‹'ï[t»m>˜p‡{Þ·þýï…ûlßäåÖcÇzý BÉxUͤŠQÀâFgf4ü=´xrGC:6ÌͤBý[_ï±4¸ šsMaYÓ¼H¿á‘c}eUY±¹÷¶ä¯ÿʵùËê"}.«kÈ®œÕ2pÿîmöñ`Âî ­Ý;Ck÷ÎÜè¯U—O5å“Ãû3lo¨¼r¨ª¼>Q™ÈæL‹‰fÎäçß»ÑkJôƒ³0þ¾tc6?oRÒ'¶6Ìͤf¿/‘_ÿb‡™çUU"6/\3Bˆd¸çÇÅ^íls£ Ôç¾÷µìÿ÷±?è/‹•Fú.ëæ,Mn?ðZðX€·t݃N !„0©bF¨)Ÿk®Ý7¹âòXª¬f¤69uš‰U¿R;ÿ˜).ÑÎö üÑ‘ð©¿=˜Ý|C¶cñ%i+þNøÄôÊluY¬ãŽçòfœ7¬Ê'bñH¾Á õH¾gwÇÏ+›sMnt{hÏó±å3?ùëÜÔ²:ÿÉG7e ÙÅÞÍ¡Þ=áPïž°§ã™“B_y¼2Ô§g„ÚäÔþlEÓÁÚò†X]ªq$UV[]^ZQgr«ÊD6W]>9t0Œ$úÁ9ºã¹|Fø;ÙâKÒ™Í7d…?à¼X9«e 9×ÙÅUÙÊ{ÝîF¾ò‰p‘¹tmvÝœ¥ù»žßj?œƒ1[„¦Bo[ù7Cáý™¶ÇJâ£+CÁjA Ù´šæþ]qÜSýà=þÞn4ü­±#Ó}tÄ@€q1³®!¬½Ä'¸€–5ÍË~¿í•¾'[w¥MÆ×h !¼mËбÆÆÁÑUƒ!„P^Zo¨ºrxôãªÊëëKKÊD.ˆ™ù]i4‰Â#úÁ{tÇsùÌòé•ýæf|Q=añ%éÌ•™DÇ-þ€q±©eu>+µâ.° 7®ˆ½ð/Û|ÀÅqJ<íªÁÓi¬¾æ¤žRuEgE"Û=öצÕÌÇJJ‡ÇþšxÈ™˜T1#a …Iôƒq°õõžTAø£!Ï<´x²ð¼gëç/ÏçÒµ‚\‰XÑžþÞR+I À™Z9«e 9×$øAÉ¥k³ëæ,ÍßõüVÏM òÆnizº•…Õå“C}ÅŒÐXsí¾É—Ç2éi•å¥ue>“+.y”ÑÎáïíFÃß¿øÞ¡ÔöCœÖ̺†°vöBZÖ4/ûý¶WúžlÝ•6 `"‚cƒ`uùä0­¦¹ÿÊú›óõ•—e£|¦`ª¬Æk7H‰…óh÷á¡ðÑïìOµõ w˜Æ›Êb%©¯ßT–O¯´7pZ›ZVçã±RA Ô†WĪÞÏpªîÁa×ÁGRýÒï6ÞûâÇSßÝ»±`øH—Ép¡ˆ~pž½Ñ;nyü@Fø;Ù†¹™”ðœjýüåù\ºÖÖPÀ±xò¾E·çMà ÷„]IýÙ «ùîÞýÇŽÔk¡µÉ©Îó+@¢\ÝGG„¿Óþ€±šsMaYÓ<ÁŠÀeu Ù•³ZìÕ pv|$õÀ®ßI:è ”œW¢\ £áï•ÃC>±±an&u×ÜŒ™ÀW•H…Í ×PDÖÍYšœY×`gàPïžðõÿVÖvŸœO¢\@ÝGGÂ-ß=}üç}Vüñ‰é•ÙÍ7dÍ&° Vå±xÒ$ ¸ljYw¾£ªË'‡òx¥AÀ;î [_þý“à|ýà"¸ã¹|Fø;ÙâKÒá&¦•³ZšsM¶õ„"”K×f×ÍYjÅ>!„šnÝ·öº‡Ã²üQû¤Š§q¨wOØÛñl»Ip>ˆ~p‘o7þªË|j‚‰bf]CX;{ɈI@ñZÖ4/ûáÆ«úL€QM™s·]ý•ðÛÍ毪¿¹ßê?8ÙS?ýo9Sà|ðÊ:\Dw<—Ïüç¿;ì «1_’Î<´x²ðĦ–Õùx¬4mPÜ6ܸ"f›ONU•¨Ï~¤éÎÔís¿ÙÿñËÿ¸uFæC†!„îÁaðXo§I0Þ¼ªÙÿøIwò /vô›Ä[Òqá&€õó—çséZÛzB$bñä}‹n·Í'§UZR–š^w}ão|à®°vÞ#‹›îÌÛþ“‰®£o_)0Þ¼¢`ëë=)áïdÂD[s®),kš'øA„\V×]9«Å¼«òÒŠº+ëoÎÞvõW@&´½¯8î„qçÕt(£áoxä¸3qNhHÇ3ÏüÆÔYµ À©J¤Âæ…k„ˆ us–&gÖ5gD„âe{ÒÂ$úAÙúzOjùíé£#Ç­ú;!+I>¸(×/üAtl\°*ŸˆÅ“&Ñ´©euÞù~œ­±pÝõßýÅ€åñJÃÔôp·)Ñ ÌîÃCáÖ'ÚSÂß[Êb%)á¢a嬖æ\“m=!ÂréÚìº9KïÀ9;é Àëÿüš¯¿1oêŠN«‰’Úò}†qçAHø{»Ñð·ø’´íO¡Hͬkkg/±_=LËšæe?Üx•¯ÙŒ‹LêÒ©ºôÓu£Û€~üò?n½ªþæ~«)fu©Æ¢~¤ÿh—¾T€Ü(P»…~gª­o¸Ã4ÞT+Im¾!›^>½R …"´©eu>+M›L n\³Í'ã­¼´¢nzÝõiº3µöº‡Ã§›ÿwûGgüë7lJQ=Žã•¡69uZ1_ÃÞW¼±»Å ×½Ãá–ÇdZ<¹£!Ϙț6ÌͤBý[_ïñJ"‰õó—çséZÛzÂ’ˆÅ“÷-º=¿â±?õÜ༩LdsWLúµpŤ_ !„Ð3”oÿY׎á=OOmíÞ‡{ ‰‚33ÓÒB(ê×6GŽ'ÝÉÂ#úAë>:"ü†ðÅ£9×–5Íó¢?L@—Õ5dWÎj¸÷6? pAœ.êÝ3´§ãéì¾®©îÁ†ÄE·`ÚíC¡È£ß?t|?çNÑŠÀhøûúMõùËk^8?aÃÜLjÁädÇÏåÅP(PU‰TؼpÍ@Á þ0A­›³4¹ýÀkáÕÎ6Ãà‚«Lds•‰l˜^w}!„cÇöìy-ßÖóRâ:¾ŸkíÞiH\P‹›îÌ'ãUEÿïÀ1«h ‘èE¢ûèH¸å»²›oÈv,¾$-r~‘¼$Ù|CVøƒµqÁª|"÷f˜à6µ¬ÎòÑMÙ#CŽåàâ*-)KM©º¢qJÕ¡yÊ­!„·V¾qäï*÷y¹Nä|¹ªþæþ+ëoŽÄë$‡z÷¸¡Hôƒ"sÇsùŒðw2á ÓÊY-͹&Á¹tmvÝœ¥ù»žßêsçÔÕ€!Œ¿—~ºëº©+j¢p-=Cùö‚í= èEHø{»Å—¤3_¿©>ÿ/ŸÍ{1 À̺†°vö’“F-kš—ý~Û+}O¶îJ›…îBà‘ÁƒC£[ƒv Îä—™T1#,™ñoÛ³é鑉d‡z÷ ¹³…Iôƒ"uÇsùÌo} zàóW×:'ë„y“’Ù‡O”˜ \d›ZVçã±R8ɆWÄ^øƃm>(F£!pìÖ !„pxà}ý­±ŽŸ‰üBcõ5anÃ?k^w}cˆØª¸ÖîMr‡ “èEìü¤;ydh¤ÃÜLÊ4ÞÔŽ[ýÙúùËó¹t­à¼M"OÞ·èöüŠÇþÔç"£69uZmrj˜®?m <<Ø6ÒÚõÃi]ƒœƒqÕå“ÃåÙwÎmød,¯ª !4Fñ:÷uí°è¢@‰~P䶾ޓ !@AhÎ5…eM󼘼£Ëê²+gµ Ü¿{›ˆ´ÑBs&ÿæ/~}ðXogÿÑÃÝmG^Š÷wÅÿ¡ãû¹Bpn`ñ©.Ÿ¦Õ4÷ÏÈ,È7T_UY^ZQB¨‹ò5;~´ÿPï¯E(Ñ"@ø AU"6/\3BðB>ð®ÖÍYšÜ~àµðjg›a0á”—VÔ•—VÔÁ±«?ÚdðàÁþ£]±½¯Œôå«÷y¹.Qðb›T1#Ô”O5×î›V37^‘x_úDäK…ˆ®è;ŸÞ‘ŸH×[lD?ˆˆ­¯÷¤^ê .Êõ—ÅJÄ?à‚Û¸`U>‹[圑M-«óŸ|tSÖù~ð–Ò’²Ôè Á)UWœöcz†òíÃ#ƒ£[‡ŽNŽ®8Öc ÑsÔX}M!„ÚäÔþlEÓÁÚò†X]ªq$UV[}"îš6‘ç´ëàÂ_ý Bv ·>Ñžþ€ m嬖æ\“àœ±\º6»nÎÒü]Ïoõ¹ÎBe"› !„Ñ•‚!œ¼ZpÔè6¢!„0Cal$…U„åñÊPŸžñ¶_æƒí±’øÀè?O«™•”‡BUy}}iIÙ©¯£¦Â{ïÆŠÓÂ&úAÄÀ…6³®!¬½dÄ$€³µ¬i^öûm¯ô=Ùº+m0¾F· áä@Âé#á©FW¾ÛÇŒ‰ï&¯96¹òòÒ÷zMïé~™œGÃøx½ó­ƒÃ=Vú0Ñ"h4üýÙ“òSÒ¥Þ=œW›ZVçã±è~®ù?ø_ao§ÍwëÌ;>ÜxU&ê×¹áƱ¾ñÅ`›O(,£+ ßÍ©1‘h³µgáý ¢v Ëߟ}hñ䎆tHÝ8qøÊ¿ãß?ö6g 8ñïçó}<þ‹!>¯·íïç÷ýþ¤6øík=Txæï_õá Åñ®öºê¯éYQQU“æë¬(+¯|ô–{ wüø/ü¬©¡±¾ž/Ö˜Dq+3H¯Þщ¸í¹3uíc¦̵õ ±µaSjߤïèv¯1UßÈ`|ë`SI¼ º¶vMfÛúÆ!«Pœ^mÿk·vIÑRNøæÃòŠªx䦻Sýý}Í ~,ºÃ­Ñôæó=¥p­;6l©\W»Æ¢™ñÉÑÁ#SkÅOôƒÐ;:7ïo¯{î×Â0'¾ñÎBEYyeZ¯o÷ëÏ ïj·Ð…ï·<[Ó1Ð](…kÝÕ¸½°¼¢Ê¢‘×Ú÷Ž õDˆ~PBî}¹ ü—mÛúÆ¡õ ©Ý÷VW{á‰cÍ•Všbr_óã™±‰ñ´_g}õÊÌŽ [ V 8ŒOŒjʽüBôƒ#ü—c]íšøúu·¦ö ÷щñÁû_ØãXOŠÎñ®öø¯G~\ÏÝ·6lÊܼö V`ñxû{“vù%‡è%èÞ— u?Ê÷ö˜0[»·ÊË–T§õúv¾ôääés]š¢ôıæÊ·ºÚKb܃_¹£Ì1Ÿ‹«0ðvÇѳû½8KÑJÔŸí®ùö«ƒ&\¬_Ìê«W¦vÜáŽÖ¶£ÕVšbö~úÃÌÈÄØPÚ¯³Ÿ«jéJIDAT¢¬¼òÑ[îqÌ'À"úñ‰ÿ\o É"úA Ûûv•ð\Œõ ±µaSjƒ_ïÈ`Ï›ëIÑëŒo¿ôÔD)\ëµµk2ÛÖ7Yu€…÷ÂÉ¿z÷Ü ƒHÑJœð|’åUñÈMw§ú÷olªéñ«d8Ðv´ú@ÛÑ’¸?ïŽ [*×Õ®±è ¨­·¥ðjûÿ¨4‰äý€ØûvÕí?9£“Þñ>âáï,T”•§ö‰Þ¾ÖC…íšDy蕽u½#ƒ%qÞ]Û îï°0úFÎþæ—;†”P¢ǺGâ«?í¨þ€™¶­oÚXßÚ'zÝ…‡^Ùë‰, |6ß:ØTS ×Z_½2³cÃ÷÷˜gã“£ƒOçÿcfx¬ß0JôÞ'ü3­«]_¿îÖTß;ì¾æÇ?ëpGkìk=T1lkæÌÍk¿0`ÕæÇøäèàSGÿ°Ê}ü’Mô>`:ü÷‰z(q»·ÊË–T§õúv¿þÌÐñ®v M¢=ôÊÞLÇ@wIzË=>”0‡¿tý€óêˆÛž;S'üAéÙXß[6¥6"ŒNŒÞÿÂÇz’Ç»Úc÷ëÏ •Âµ^[»&³m}ãU¸|Cc}=‚_ºˆ~À Pz–WTÅ#7Ýê7Ôw¾ôääés]›TyâXså[]í%± ndž-•ëj×Xt€ËÐ;ÜÑùW¯­FðKÑøø_þ£qóþöºç~= üA xøÆ; eå©=öòpGká@ÛÑj+MÝÿžÌèÄø`)\ë®Æí÷÷(>çF +LŠßg÷{íßÕ õFʈ~ÀE¹÷å‚ð)·m}ãÐÆú†Ô{9216ôÀÁ&Çz’Z§ÏuÅΗžœ,…k­¯^™Ù±a‹ûû™C§žª}ìµßÖΗ:Æ'GMŠËøÄèÐÞ¿ûF<×ú°÷GRJô.šðéµ®vM|ýº['Rý;ìùÇ*ûF¼æ$Ý´­>ÜÑZ1lkæÌÍk¿0`ÕŠKïð™Ø÷Ë?©ßýóYõ“Ö‡O÷½Ùf*°øZ;_êøááÛ+Ûz[ #ÅD?`Vî}¹P÷£|oI@ºìjÜ^(/[’Úc/÷µ*îhµÐ”„6ezGKâ±úÁ¯ÜQæ˜O€âuôìþª¿~ãÖ>öÚïÇ‹¿úo]}#gÖ lh¬¯gïß}#öýòOêç™~¢0k~´»æÛ¯vÚ.)±ó‹¹B}õÊÔëÐ1Ð]Øýú3Ž­ dô Æ·6ՔµV”•W>zË=Þ8(r½ÃgâЩ§jtø«Æøäèà 'ÿrè¿ø½»ûJG¹—bïÛýU1øà u>^ ¶±¾!¶6lJu»¯ùñŒc=)5‡;Zc_ë¡BÚ¾#"®­]“Ù¶¾qè‰cÍ•V øMÀC§žŠKWÇç37w]·ú¶ñå«|XæÀøäèàkí{Gµ?Ucg_éý€K&üA²-¯¨ŠGnº{("RûFyÓ›Ï÷ïj¯±Ú”¢‡^Ù›ùÒUëR½“wÚŽ [*y+Žwµ[x€™—–_ëêsÕ¿.¬ºòÚÌ’+>åý&˜…Å>??%Jô.ËÞ·û«NöÅ_ýóO~ªì & ßxg¡¢¬<µ1à­®öÂ÷[žõIQJÚ}Ígž¼õ›%q­»·¾öì.;{jx¬?ŽžÝ_uôìþµkW\Ÿ«ûrǺßؼÄ.@¸°¾‘³…Ÿ·5-;zvUDx¶Ä‰~Àe;ôîP|õ§Uÿó–zábÛúÆ¡õ ©}Ñ461>pÿ {¼(¤äïjݯ?3´cÖÔ}Y_½2³cÖÂC¯ìõ³m½-ÑÖÛRßüÎbiù•±vÅõqMݧ²5˯¬ÈÔ›¥îí®Ÿ·½Úþß׶õ¶x ÄûD?`Nëþ !ÖÕ®‰¯_wëDš¯ñÏ~ñ®8}®ËbCD½ìšX½ìŸ ^½â·:×,ÿͱ••WƤHÌk¸¾7ÛÞ8»?s²çp•ÐÇlˆ~ÀœëˆÛž;S'üAqÙXß[6¥ú‰â½Ï?Viwœß¶£Õ‡;Z i¾Ÿç´kk×d¶­ozâXs¥•(Mïž;ïž;Quôìþ«§¿&R¬úG ït¿²äØ»?É´õ¶DD8º“K"úóBøƒâ²¼¢*¹éHíàûZw´úô|Œ6eþï¿ýOCeå©a;6l©¹zÙçË–/]UáÌ·î¡S'ÛûÞ(?Ñyðê¶Þ–ë÷=Çœý€y3þº¡®ówÿQµð‹èáï,T”•§6ˆu tv¿þŒàŸ od0î}þ±ÊÞrOI\ï®Æí…¯=»+c0Ò;|&z‡ÏĉÎ?°ëïÓË®‰š¥«ßUŸª™°3KÑ?Rèx÷܉‘S}ÿïÊÓ}oÖNíäó½Ä¼ý€ù}â4:÷¾\¨{äw2Â,’më‡Ò~œß}Í{S.ÒáŽÖØ×z¨öã~#"ê«WfvlØRx蕽>À¬Lí üH \±tuÔ,]W-ÿ§]Ë*2½Ÿ©¹¡¼bIu¹ÝŒOŽö Ÿ=ÛÞ÷FyçàÉêÏ÷ Fô„ð7;Ãã“îAÄœ¸jYmìØ°%ÕßOMo>ßs¼«½ÆjÃÅÛýú3™M«¯í¼jYmê—·6lʼxêXüì×oXx =¯FǼ^X$Ó;Ûz[j#¢væß--¿2VU_++¯Ì,k8»réš²ÚªµåeK+EÁôè)tŒŒŒìyuìÜHaÅé¾7kÏœˆá±þª°ƒE&úͱCïÅç÷þÊ à<î}¹ ø]ðçóÝsðRÍ<–t® ÷äÿù캃Yý€óšy,)PÜÊŒ’Mô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áÊçð¿õND|×HI¨•±ÃŠJsDüÌ(b¾?(%{<ö0ÏÞñ¸@ ™—Ç™+&''"âó{uWD—íŽ÷vü À¼ý`žÀBý`ÍM¦ÌÑæY>—íÎç²w…ðÌшðÌÑÐTøû¦IsIôƒ–Ïe¿ÛM˜+¢,‚|.»'„?`Žˆ~°H¦ÂßíÑcÀåý`åsÙ§#bsÀeý`‘åsÙ–þ€Ë úA˜þŽ˜0[¢ á¸T¢‘|.ÛÂ0K¢™áoŸiCôƒ"”Ïe»ó¹ìmÑdÀ'ý ˆåsÙ»Bø>èEn*üí6 àBD?H€|.ûˆØnÀùˆ~ù\vOÀyˆ~ 3Â_iÓD?H˜©ð·9„?`Šè ”Ïe[Bø¦ˆ~PÂ0Môƒ› Ÿˆ#¦¥Kôƒ„Ëç²ÝñÞŽ?áJ”è) ü@iý %ò¹lw>—½>"šLJ‹è)“Ïeï áJŠè)$ü@iý ¥¦Âß7MÒOôƒËç²ß‹ˆí&é&úAÊåsÙ=!ü@ª‰~P¦ÂßMÑc>¢”ˆ|.û³ˆØ¤Žè%$ŸË¶„ð©#úA‰™ ×GÄÓ€tý åsÙwâ½Â¤€è%*ŸËv‡ð© úA ›þö™$—è%.ŸËvçsÙÛ"¢É4 ™D? ""ò¹ì]!ü@"‰~Àû¦ÂßwM’Eô> ŸË~'"¶›$‡è|D>—ÝÂ$†èœ×Œð×cPÜD?à‚¦Âßæþ ¨‰~ÀÇÊç²-!ü@Qý€O4#ü4 (>¢pQ¦ÂßõqÄ4 ¸ˆ~ÀEËç²ÝñÞŽ?ኈèÌÊŒð×lPD?`Öò¹lw>—ÝM¦‹Oô.Y>—½+„?Xt¢pY„?X|¢pÙ¦Âßv“€Å!ús"ŸËî á…èÌá‡èÌ©©ð·!"zL†è̹|.Û›Cø€!úóBø€…#úóf*ü]GLæèÌ«|.ûN¼·ãOø€y"úó.ŸËv‡ðóFôÄŒð×d0·D?`ÁäsÙî|.{W0§D?`Á 0·D?`QL…¿ïš\>ÑX4ù\ö;±Ý$àòˆ~À¢Êç²{Bø€Ë"ú‹n*üÝ=¦³'úE!ŸË>›Cø€Yý€¢‘Ïe[Bø€Yý€¢2#üµ˜\œ+&''MÌN?H8ÑNô€„ý áD?H8ÑNô€„ý áD?H8ÑNô€„ý áþ?do (€ÈIEND®B`‚nats-server-2.10.27/logos/nats-server.png000066400000000000000000001350251477524627100202630ustar00rootroot00000000000000‰PNG  IHDR ˜RíÅzsRGB®Îé pHYs%%IR$ð´iTXtXML:com.adobe.xmp 2 2 1 152 544 1 Å ªì@IDATxì½gwGšï™¶²,¼£•(×nzzfîÙ3g÷~û}³g_ÌÎíÛ3m¥–Dï`˧Ýßÿ‰ÌB$[„´@5‚`Vfdd˜ÇÇÎÏ«*ðüŠKéqõíÏ AŒžÊÊ /à§å+æ&\R/þ¡—uõâÞ« +>1ŠãÉ!Ëž@Ž¡ˆa­T¤Ý’ÒRówõuî7?ç `°vpt”íàU vüwà+-•—ÃÍã‚@ä¸ÆD@vpiÔ Bˆ²ìœx[0T¾£ã@ây"ÔqK èÕò§û±ƒðÎ5ìø1K^)bìÎó£&qc)¾îKÖIÜ“‰q7×w@ÀÁ·XFÔ.JwúÄIC‰ÃZ© æü™ø’‰†qìË0ªm$Ê\“3†>èÍõ2 P˜‚1<²‡(CâBŸØ﫪8÷Â¡Ü ¸S“yAähh¼¹^€þ¶gnÍŒ³W0œb¨ä.ä¦áJ øT¼=pq¸$‚HubÌ@XíæúV ÉM‰8è².b)Úñ’€/h;aæ^›.j䤛œ.œqþÔq7?K¨…XD6Pw«£:Áw€U ”P㺄º·d–T W7×K„Š|aws^Uœy ­g¿:ª5ﬓ‹eT…-m7×Ë€€Wee=Ròò(ÂI@"Ä”j1uƒ˜æ§Y`R\kÉ ‘7×wAà<õ·ÐƒævûÍÞ/'2`;6ØIh²*s+/rº¹9ÆGX—@ èì±2*WW&À¦r$/h[X0‹ íìd}ä²±°¯<ß\ÍD*ææúC! Á.ƒäâx&[w‡šÓ«_•K=˜n„Kþ\N¤&OÐ)ç6}ƒ¬Ë‚@XÀ¶l¸ËÉAgŽÁŽ!CHñ+œj§òŒ¨§Ò„› 0†ï=ÇÆ+µŒPyFx53m\(ïZÙÇoùÄ}È•´ —µ²» o@Ào9º‘Bhê†-쵟{R8-=( ÷.˜:w·Ä“aý–õC“òæ÷‡B 7‚À(¬_sH¾#Oë»û6€V§!&àð[ò—ëÉs?Ìì½°ÖÜ8¼Ý\8TŒª@ɦ`üñ-ÛCî!Ûʶ–^{¥MÃEPË:÷ )Ý×dKn\Á"W"o®ï‚À²J²©Gò×34¹s3 ¶Ö[Eä^j|AMªÄ:—ôÝ•5øÈÁÝÞÞ\Þ f&ØéKGÀr5`õ€–‡KøKŒœ)Åû@=|É9¸üdêµÔQžÁì´”›»ïÜËàŽÀ‹}æ!ÝsžI‡°:kI°ÅˆeÍĉ©Hç‡L`þIêÞ¸¨Ç1ƒzšæ÷«ÞÍWç!ãÀLRFµôCo%òV#¯Y?‘´ŽÛŒ¥²*OóìÑô8/r\ô)5@&ð~ øÑ‚**]‹›1€ó ?ó\H• "N5ô6#Éí êx­A÷¢N;ŠADcM“8Èro2›ŸL²é,ã¾XàI8°üÆ;ÃqgJ¾yD±‚•ƒ“`Wš„‘[ÌÁ°(Ã0o'ÑJ?îv’®u`Ø™6÷1¶²qºŸ{Ó¬DPaNëDJs. ™—ÆA·®tÂÕÄïÎ_/  ÁáiIêYÚŒ¢pÒΩiFœ¢æ~ð2÷~÷lôÕÑô ó§E„(.lâ›piHqÒÃK~æ³{ýÖ/öÖô[;¡×ó4½ÖÈÐ 1üͳô›‡[¾zypp2{q„b¡« ÖdÂIÆÕCÏ2é*?,u-jwiþ‰e” ¾MÆóñ?Ê-S•Qá¯Æí϶î>Xߺ³²yadƒd@9/±Ûüá¼zöròÍ£×_}óìõáIµ›±O7¯I@²(¿7NÐxW$Oœ!ìœðP­ —ƒMØ ólÞëD½ûw¶~ù‹/Ú!š±†E¥if}VÎÆó×ÃâÑ“—¦ÏË`†W&0ÎÑ´ÿŒFzW5nâß‘—$Á`½{ÿîöÏZØ`‹ .=Á8ÊÅ/Mjv‹«¬ˆ¹çeÞÃaöûƒôé´fi„‘´YÈR¾¹þpdô%Ù²v1¥ÝA¯¿‘„kIØ]’K€/å$ÍN¦ãß ?ßu4yQˆõíØ G§óÓ hlbú.ÜrÕ¬€›ðn˜¹ )‹?€ðôÊ¢U÷V6ã$éwÚ›ƒUÏë ÑfP£‹à¢4+O¦ŸþñëgOŸ–~Kë–H´k€#Ú¬T ú™2n®o€9¾P³EÐ+jD€º•Y:$wom´’äÎ8¿=¨µ¬‹#«¿qÌö¿}y4{˜zó~(ןßPÿ»iÿû¼i{ƒv´áûÑNq§Š0}Gæä_´¹H½»x|1®º!CŒwÄL²êhîfñqÞ„VûÖåpsýÁ"¿*Ê"ëÞI»˜É‡²:VÝóza°9ÔÌ‹|8ŸûóãrºŸáÂÙ…@K Fè„KMÁÈuVË;Eß„7!àúë±)û‘‚iO“ùVš§RáÄ  ÓX}þ¨ªò¬Ì¦Y6ž¦'³¥T–‚)q’}ƈø}‹`åíM¼ ¤¢U¼j‰sˆV]‹/‹œÞãh2›¦YÁšKb±i.@Pʆ\d…?›W'“r?-G¨œ bòoƒ’üoÂåA MOò|>ëìæKú‘>"äuË ÄÃåÌB¸ ^ršv¢+Jó¢å®ÍIYåŸktÍØñæz€UðÏÀ7eYÍKŠKgT6« Í"˜ ŠqÔIø󪘔¨—M„èã5* š‡'×$茪ªìöæòVh…³›J /Á-E’EkEXI…;8êêþãTCöeYfY†Ô›ÎrFepIZJ׃uYa‰°S·òæúVàÂ…Ì¡nzÛ²Ò”4š„‰,AVÜ’~ž—9ÿm;pu„¯/Ø[ÍîRo:+'™?¶ñ´À ¥Å¨7×wA ¦Ô÷û‰[8EÒ*È| }:„ #g³°òl_àˆ1$ƒ …Iú`˜ñb…Í~‘í³ÙÜ<ý0”y±…IŒÔŠ}ú3üÙü ±YL:f5aÙ9iˆ8à M P%Fö¸—£F=PÉÇ·îþæz@^Qvu" ˆs†b`ð‚"‘-MpÌÁ¼'ðÀhdúQÌ°dÄ`X„u¢Z@Û€¯ôÄ A7×w@À )G¢fÙÃP9€¡g“8ò¢ { |†d" #·°ÈaÍ)} ƒó>‚G°‹3̸V'è­ŠíFå/ ú#Ô|ñF¶O%¥±‘P(ŒJÊÔBgóRäM £RôUbAÓì~H¾¦¼ž^"´Ôoä¿D!ˆ@vÝ\߃’ÜÜéΩþ7®ê`j˜"7:7£@òÆüdÆ$5ý›ÒäXuV’MÚ2Ùf.cP · ~€\<U9GîÕ‹üÔÅAÓÜ\(`%dvV‹È‚°…%ÖÒTŒY'Ô€1PuhÙ?¬gf™Üju,iÅ~†AC‹>Ö»w·7×7  1xõÕÙ7d¬ÃôÊ ÷Œ:tO¸òÙ¶`"RÊ{ö'N1]¿Èä’{|‡‹f{só&€¿¼º-¦çIÉÌ5/ØçÝQÛôA gCÆ¢WnÄMbþ _BŸ€¦‘.’«“lÔï¤syc ÿmH´hžÑ{]ýªø €ÚéÉ™ZÔœE¯:!R0§/AœÁ³É$8D«-Ô1+S®±õ„W‡Ý›ë„@àÇLºÄTó=L3½èïÓ eÉxƒ•¸‡™—R(¨Ü„é]øÏû:¡é½.¢›×7¿K0÷—žåU¬9F Ã6…£Òäb «Xĺ#êËk€ÀœZ#&Ï*œg”57¿ojêÜ~Ó¨#C«ã v~Y”Y¡^")²ÞáAÚÅ*IÈ‘‰@@_É\#¥ðÇbXä èä››ëÛ!@,ðyï+ ›ñ¦î•’Q‚8 uD^‹á‡ý”(—J-K&tóà«7c@”ã HMÊ…… 6Í|Í"æ0—Ø+Èý5«8 哳Qg¾ÖÜv2ƒL"ùÕ.j~d[}ηn¢¡›Œ­‹šÓÀ)kG5ã ƒˆ”^ =ŠxYEÏÌJ?´È(ö5cˆ¬+“‰²5Ô-ˆŠ ¥Ñ_1 r¾–ÌÀõGF.½h‘–"'$#/s&»¦ †EÕ*«Ò›L55Sn­&7l kT¨À`âã®].ïsuð$“E¨©ÆžU&ÔŽDN‹ ÚáY ±®>ZÒ$íÇ}ËÕ ba£^qïÊu¢¢nÅrÖJuÉÁ–«*%:5CÅhL§¶gcWI"TmˆÝgúô ‹#ygJk0¯Û*úü°Ÿˆž›)ãJÐŒisK¼ÞºÆ“›î?ØP¨³WãÑÖ¸AƇ÷ÓgdN.ø”þÜ 2uÍÅ,à«T,æO^, O eˆ¼xÃÒ×–áS¤„tREWÐÁ¨D’CM\KÏÈýžWÉNµÂQ¬#@ØÙb SÖ"£ H€ÞÔ-¸H°BÎ|PS{–€Ñ[eøfž§mdøÝÈ&Ñ‘^à­cD‡uk,N_]´†^0õý9C÷x0ëÚòCo¾6Â(ƒL5íEõDþ«¤‹™5"e*€×¦ÙâMI‰ZÍϹhp$Ø䦯M¥QIOReÑœ‹fü=Ó«™KjÌé<£=-zÂêA;åËt ?- .ªõ>BéiŠÿ®nÍèXfǦQ-šëbÜ·uöŽd¢Àë'ö]›˜F»8©±2–ö6M³‘\—šåhÿ j¹†ueQ1“i<±°iÍ|'‘Ò>'ª,ñ"÷¸y‹lj±/M@,f–-_Ïä«O–À^§xKîõ›+û¹°‚d?ÙxÑc5ɢD‡3>\çãbu—\í·˜¹¨í·,µîÆÄ›’û&]â˹ÒG(ŒñÕ(—%#ºFd2i¢–UĪ)¥Ú*oj!Ü2Ï̹:aÚ$ù ¬t†ÿ¢P®™´Q&‚ÁD÷fý‘rñÉ®Ëâ`²€ÌV‹Ì?´›šväÓˆ*-4‘‚¡±ÎjþÐÚxZ_hÜ=4ˆ[ ×¢i±ø@‘ \š_{ÿ£_½r*[–l|«½Ipk5ÍÇÑq/·BxD68óÖ¼ gZyšìLtýPUìDp†’$¦Àìê±.Õá|j)9.û¶¾\°wµüéûß_XÁ5BE¤¨¾ä@ÒÕ´ ž5JJÞ:Qg óþ5C‡0û¸FžÉ¦úž¹ dø’ÊQ†nJ(Y=º/"¢S$éƒ×¼­Û,˜zx–^ן^pìÑô–OÛ·`nh·ƒ÷5þ,a %»wñ?!½r Š7îÔînÖ`Ç÷ sÔuJ3ËôsýïŧL±húªOq¼x{nµ;­ÊãL‹¬û²ˆq æ45˜¥±¦c–"À-¹™¸µ,\õÎ\íµ½t@vךŸ_+¿ç¹ö¦vDÙ×Wsù F•¼0)*Š¯oˆ¬²ö%S€‰…Ú‰T?}÷C9ÊÞ$µÁFªKyŠpU´ËØ„5Nª+W3Ršêà –•[Y5Ê5/0ðEØ´¶Ñ»–†˜{gÐQ»qX,-Þ(‹ïÉß5…|ËE èo Ë)Ý}tã½7•©¡ÌK™ŸhïøÄub–!àòq1îÞukÞVþ‡çZ#(6ž±ºæZÆlÁò X/W âéüžÒ‹š…çôqJÍ„ˆ7tøGú9åW's¸žÖ¹n†±»k¦ÌNÅ6˜|{­kmTÓõ¹4MöÍï¹×ÊŸO¡êÞ±PnÛÓ¯H õ(½C<ÿ­Êq_1|Ȉ§_^ÅÝ…L3LÔTÆúÖšäZe¯Ðêqˆ¸L˜¾ïÕ¹P¤dé†[J|Pä®·5€ F”QCº©ÏeÿB@N‹P¸©=Æ®­SüfA†lÍ·D#Y-©¾õ{¤Wják_]q•ß¬ÙÆ,·‹bŒÞYÓ S ãøöMþp»·ºþ”€u0Ëf‰Á Á é îÕÔ¤q*ͨ ÍæQ [Øž¢”kIž«Þâ‘·vÏÌO¡éœ(à•ûüt]#Øe²|Ë„î辦~Az tðsÜ¡²êÞ ïšk…í*éªí®Ѥܯû+ V0ÍLIŠf¼¤®Õw-ÀfA’ðV×`®”Kn¢¿}oÝ”¿…'I4ËÌ­Àø®L¶Ž ¨`ES–E£ö¤x,¨&¥íÌËì¡^E$®†êÁ4AØ]¸DšÈò÷m= µ·‰ÕðƒXÓÍ*lÚ~&Ùâ+õl¬N!¦|>ä ÎptàL(¨Âë8ZìÈ#¶¿·¥gè6ÖÝ—º™Í©ö9ûú,C]‡v»ªºš8yvZ+7ÒlÏ6Hs¦ö.ñòç§6wNVF_º6ïë_^‘›®pÊ2+8 í6ùVR¹ &!›ýšsoYÍ,8s9Õ¥ÞŸ‡àwf^ÚŠÎ’iª˜4‰³I鸨ç¡?ly“2.ñû^ääf;Çt’€±T´”ʼ¾ÿÎúþÐÚ†B£J4…&po®u›îùšô'©ˆY éÈ÷A+–Éï´9š¥ú¡U»ß÷çjóVŠ52¨k+Ì5pqñëÙ¦¸ ‘´¸°=Ûª·<©ÝÖØÚ}ʽEÉw ÁWn=ý;ÓèÆÒ2„Ã)öw&Å5x¨+ÙÔÄø–]þsôI+IØ$þŽ_µW§á8gY´ö!"¸«c îþ%Ìô%½&L᳸w±5©Á]—ëÙÜc»?dò¢!KÖ2»¢Ë÷ìÁ˜Œ ®’ªµ„Õ€ƒuÏX#âÚÊZ?%%¬¤Ð#åvÿ7®™ÖEî&±Ýb邲äkâœïÈ‹ ¯"hE0¥2iÄh€u_ZQLíXr¬_ź•Ã66äl5º&6÷^ €JF+ìùTÿí+joït]ÊEÔu½w‚¾nܽV”9;Õ³‡ÑPJ6ŒbU\³ùc­Jmv³säh…(Xøc F>`—¯d•hÛÁrq¥\W¼W½€ÈÞÈUÛ—ˆ(0W8+j6¨0sL‰HšFÄ´ÿ¡^ë-sjRÛœh™~—HH(ìõµ¹~je5Õ-è‰Qm%íNí!½jäºK ž–òÐW\…ÛÓ€¤ÂhB„AævúJ¹/eÌ cñWê*  U/‰;‹iX k[@l̶ex:¹+}†ØuÝ$‘R8KfÙê%ò*Â… µ$äòræ¼$ˆA];ÒAF²8Màå™vp@Ô{sçj0r WÓš«¼dPÐJj£lµÆê*Ï–B¬ö –üB†<–š œ„„D‹Ù®àY"—!š©.Â`I"¼ËÙ,åøܪMjIÈ*Vì lÖWM²MĨòiÙÛ«é´¦šB`Ì€`÷¬FÆAº*¨É’ × ½ GuVÉä굺àj+ò柺#Ìÿ×-ÉCÑ2(ê<œîP§ÈGâ–\'8u˜ê"ô`„aä@^z-ê´4¼‹©åª¨ÆÉ`iÿj/˜¢áB)K2¨e&B³æA«%Ú£C‡¾¸Špa〲T$­*ªæ³åb.Ò¼+zø-ãHÙ¥Äß}7‚Œ¤BQµŒBXÉ‹É`þiKU«"|–S_-CNÈ^ ¶m#i!QƒŠ û4¹ÐÊ^¶j6¥õÊ'ùf<1ª™ Qª©† J5µè‘À“‘ˆ{ºîWãºÓ8‚çšW±O§˜MJÀÀ+ç‹'ÂÑ´8Õšj¾SµÔ5œ<ØU'vIóßà%`òÉUÏr¹j cn±Ñ‰4A]·Rk]²Ùœ¦‹Ò6l)ÀÄÃr°¥ãÈ|ê²!äh¤°ÄvNc”&Rá¤kÕZ ®JQ=Ó›šfY2³ÌüÅœ²ÝH,´!ôV¸ËÂf£›XÂݹ^ô ’‹,9ÕµW_¹° ʃ]‡”  .ÂåîIL]¸šàq@•È°â-ÎSà€6ËV•A@Êø§>þäа/\öÉ•\¾‡‚¡¨hY%Ê`7ŽaÛ È' Îkî'ì)`=í·I1pu÷o½Æ®O)ÿ#Çýr˜ƒÏi4Ã4ŸœæìØ°Ç3Ú | ò.^M^«ª]Ž­ªŽ_% …úK B&¢Srú¥*¸B¨â¬ŠgY>.3í(Y"-Á$–cZhpåæ”ή¦—–«ºØ²Ë¤<¸QC>g=–A¯Ýk·‚v”°  ¡Ý#AØ•¸DÐÐF=ÔRÅ›¸Q¬¸e’ÎGóé<Ï;Ƨê'1S¨í@ui­øQ22ˆQ2 ^+ÛI;ðÛÚ„óž.‹Î’‰Nè!š‘ÎÿP¯ì´/ «¥âsI? 8zÀs6y/*EdŠ‚h¤ÄœÛ|åGÁÌ[ P‹ Bu¢œús>M-žC/†2ÅH½ªê óíöÔÖ‚Ff‹GNsàÑè_ií“(2¥µT°¸ƒG€%з[‰kx”×$ R6cãâa‚º-çQº>‚fÄ”>JE•æ#\=ÈWudìˆUò"Nâó¦P'º!0>VVóêò/,žMRXͬ2N\Š$íý (ûu"o·ÞY_½—d’À\‘H€¶w_Aàh 8,ò£Ù“á|?÷‡y9×€†èÕ‚t¹eÖD\ê¯#³-¯Úé¶v“`«­Ð{¢yLN¤Ø(qš`ôA99o:Ͼœw^γœú<;+¦âHQˆD¥i É×Ìx3è9Ô.ˆ‘8áLDJ­%ÅUè~:šv¿ìïõvwÖÖ׺ƒ.Ç£NãC¶äwŠ]”‡ /"tæFqÈÀ›—Å8›?=Ú|øêåôdóÀ¶(sŸP(Ç]G ï6ödçÂ^¿›ÜÙÝÚÚtZ&)DÝrä®R†l_&˜¨W3›ŽQ:-â†ÎJ0™ÎÇûGÃW‡£é,£ÓÆ|KäÁ‚™ß WœN²š£ö¬€“lX6ØÚ¡‡’Hb?iÇŸq'ºE­8nÅXa' ’8HÂ0áÆ)ø@IÁQ„Ùiô@;Ý×E@ ¨`é ú¨4bŠð^Ë2Î}Ô),Y1KËiÁyƒå¤(ãk6Ew3¤#é9^ŠLÅ«¢;¡žDŽž—ÉÞ©•®È+ V0ÖwqRˆf ŒÄ‘ä½ö¹Ä«Qv‚p·ýlwð›{eê ){u0@ßy¥ÉnŽHm ¾â|<Œ7ãà›áüñ$;Èp7pÀl[€séþj°K¹¡|TÕ ,n'ñgkɃµd‹9¤¼Áù¸êì"tcíKÓØ•Ì•Õd6Ï'+³Éäõ|ªÞĤŒv²Tžv•ôU÷õ ¢ç'I’ûáêgkwîßÚÛê­÷âñc˜bwh·Uõ`lòƒvý¡½„‚ˆ€<¤`ŽÓé×q—³½Ó"›§£)'á„1`±~Ÿ}`z£ºb “$245²è¡X>ùhÛ+½ˆm21>DÌ$'Õ‰ßÈæÉ@8Ò[m„&\/'±&ßýhgww;ÈŒBço@R-¦`¬‰j(ÍG¼bJ1K ikâ«­†4,gûÅdzÂÖo¢öÇ©PV釠w  8ý·#âÌ¿ê&ÞÝ[kŸ}¼½±štØ ´0s5³Œß ³ä;Jþq¢m8kH[žÎ¢xñz8N^½z¥Ù¦ŽÒDÓ°$¼^êƒi!BQtÌ°Jè'x½ßÚ\mßZIv{1N~k­÷#¿+¬ùtFtñ9vS¤îºá¦Thœ]Pר&èsæ%"BÞ`É<ýIôhürOÅsâ*$ÙØeQetYrô ¶Y9§g“åR0ûåÃ4ÏÓá4ÎóãYÎuè•u¤l¸¼UèêÕÌ…©£”ª9cÜM*”˜¼/чēŒ—,ôVÛÞ­®´‹vž¿0“@ Øà ½mr v?a€ßÏÛëi¼2Šò)ŽÜ»µÅ•onŽ3µ…k-\dÞNË‹cˆO V›­’ÔSF‚çu Î;ˆûs¦^1)4ÇŒt`MXçÔ&¼ÿø…„e³é’ ;;;¢»oߺÝÙJ koËÏ¡ȸàèÙ àGß&¼xTG^þÚŸ¿Lǯóé\ú ˆiøŠŒš >Ô_ µUE“4%È+ƒîæÆ`kÝëê`¿¶IÙš0Â\Áœ• Â(„Æ‹ª¿Öï´eK Ý\c¡¦kJýqGðôFZq‡^ËZoµ½·Ù»¿šÜ`# Wc¡'BÅÊŽ¡åZáÞQ¼@hšI{¥Lv(úŒXÄô2A¢äQ+Màw*ñ –—è‰Ä™—–^ÊY¸è˜4—­vÓl6I§³£qz0™½:™…óÌ›f#Cð¡cDr®©Î2¾’Ë…L¡ó¨ŸzõV=-{´ñmý_d•uÃ&@¨äØeºk½UÞÀI“JÎ(+&h*–w+šKSMëL*o§J×’à¶W|ºWüß‚ëS\¿´äN/ð2,rMW¯{>}•­— ÍbͼÕt”µ?™þ÷ׯþsæqñŒžP Æ\6ØY3­¯ô§í¹²’ê68¸5àá¨PˆqT¨gÝIù+’{SçD1Dˆa0DŒOCœfqÕFÇsšN«ù¸Ð‘Œïƒ$· !SÆNõ5rn„ëw©%º ï†^Ê•#>‘·J”ÌN‰¤<6÷FÜùx}ï‹Ýn¯o­´VÛ­´ ÔÂœ¼ñƒC¥ÝK×ëŸR‘N%4Á÷ÈøñN¹ñéí{ÇA:z–M&˜fšG ´MbÇ]šÌ,#BÝ©&‹ú÷¢¢ðÜçWôˆ˜fËK*HBÕÜEÅ”GlŸj@iZd5œ¾óÞª½¤]ô,Vs 2 S„¸Âdœ%ÛEü"µíFƒ•\ÊÞ·&–öͬ•§ª„‡ Ía¬«ìiV§îÞüìG‰hŒÅË™ÄZ¶ ³ÒÚÞYýxsåÓíÁýÕön/Üèx}u&¥\œÄ±šÖ"M-™ÀZá"Jɨ¦g=êî A÷¶Ë™·î#dº})Á¹™•‡Ë~+™ßªÊÖd¬õF+ÇC—¯hÃd-?ÔJ~[—÷wP0âQ·JJ¹â%õ]Û €Ëª"øMç‡ÒƒMöÅLkûÁ^W.…Nwµ÷¢Ãè8v2zÅ6.~Èy‘Ø6|h ÐeÑü­FK£ä)Á8îÒ ÷SÌÂô ëjiͨu$&½o$>#^ý ¹ß^ÿ—½ÿ|û“{+› {JYMÖ®9FüP³ÞðP›öÀ[GÔ Ã¸‘ßp··úË[̓âd6—é~Π%Ó¼E#ÊÍò79)öEñµDT*Pn®Ükƒk˜*Qu›ŸŠIb&j‘°È)µƒJëJÍí‡Ä ïuÕTçåƪù‚®ÁÌîô ¹ÖxdÅÅÒCª]J¾”ÄÑ£þàSE(éEêÃW¹*‚,ùΦåyÀžd(¹ß ÃÈÅezlQéfÉGv0K)ð:½xwkðé­_o­|Ò‹öb¯Ç'pjÿ"×.õHœ[DÕ–u%…d×äG‚0*ÊJžÜŒÆpV”ÈÝœ»7Æ°7– ¸YB·ò•êªÓXz¼”âkztÌœÎV<ˆ‚]FÆðèµX:}H0n®¶ùwP0j@m`¶.j¬`×¼Ãf–ô9¢¹\4k€á˜¾v¢h°3Ç|ÐjWj4®’étš§Ì. µ‘„““j r‡Ÿ¥J2nÂ߆òd8'&Óø;f§yµè«µî÷7¾yû—›w>ÙØøê»HOè¨WHR*é•jÈ«q4¡œ{ð!lØ[xÇ}?¾Ó]¯ïí §^žï?KgCNÅ…oeTR66ÒPÌ}­¬–í_Ê’°lHÉJøñ/Ö3e£šK)@ãüÕ²BbG$ª÷"UÚµâ=®5, uÏ˱d²ÀÇ"™ÝXMHj|¡j¨‰ði÷õ“îߣ&.Zr.XÆšÈ_iˆq\4T Æç>ù1© ÅÊŸ›GÖúöÊ'{k¿ÜêºÝ ¤]ÚøkÕFþð ®Í#÷€Pr1 Š[þä6G¥ÒÙ×xÅO9n» MfäUR:˜`„-ò&‰Š!-?\±­L—¥ØU±– nQç)S6•Ó2réêÔËWtå Ææ‚¡*Mí;øÙ¨¤Ï%<\ #´îEʉ`VfÓ(éË2ƒû¨š¨ÓïÛá8=‰_dÓWi™†œ‹Þb@ˆùiWØ9p¯õpÞŒL|À. Ü3ëÇóqUn'ý_íÜû·½_¬ìm}¶Ä1>À4,L¥cd~űõ'‡xûÌ1cœ|B ¤à+)$þŒ“ öi`éÀG½­t¯`&áh6›ÎçÓ¢œzs–‹p¸9m©–µ*Ža–âj­äëv¡jªéV<—CcZãE”‚›YÐÈÒ-«Ì÷iI£²Í:rW){Ä W-¥6ìeó¥C¦°OJÉïƒ 9!Wç­¼”2"gV¹i½ºlÿ³Âñ‚]br‘“Ñ-Ö+ðã°»Þ¹µ·öùîÚç+­»-o=ðza¢9÷Wƒ®(Cºe¢ffßKs˜˜ƒøsR‡–Òt¥0là_9³S/À]å d”–«aÌ©Jr É”«¡Œ´73OyÀ@lRŽ<”¯¤ òvåÑõ¥ôÒU:ƒþ&þ2¯\Áœ¯lÓ&÷kWy6€¢C HÂ!Ú £Ó2zš×b-slµ’Õrm·ÿšNÿ’ûUuÄšWk51{Õƒ± ¥n7>?_Ÿ›ç³\ÁÙe”9ÃL¿G Ä‘¿Ý^ùtu÷—{±}o«Ýï0ÑËÌ©³ð$Á:f³éøÕðˆ“;ë[[ag •ÒI’~¦ÃÜ·æõ[¾¿w‹ÝQ5ß?9šdóãŽ9>AÃP/º¦|`,«_˜ j‚øÿúÀ)’•³Y,bj•Ù¬‹›Ì²´AWÔ§iñˆjlb2Š½‹Ô)¬ÎÁKšf¡Wj®0s÷UÀ„\Jg¥0õåÂ}'ì3ÉšÞ¯ZWþ6)JÚ*†»}ÎtZz d(åªêP½Šeó¶0ðýò¾ÚT¢HMƈ{ñúJ²·Ñ»·Ö¾x«‘×n)ÚSàjPË»ÂlÖ)š£¬fLpÍË#ŒWxÌx«‘'>B¶á_ajC–)&#–î#af~ÈŠ˜ä«tä5Žúb¹Y\È4(I6a\’’´27Æ×£Jl‚„+ Ú’7¡þÄúUúôJÃÕ+éíX¢UÐ…€<)¼ î>Ž¼-3€´.ˆ)M« »æ—oñ†îFÞ`}À §?›ýpøû“Ùtša2³ý"3”PY²„5í´5çý•BôÏœ±ÁR«Þ(ªGÂ@âåzÜù|ëö¯w>ú|}o¯·Ö£÷-OcäòÑ·¬ë8žd“GÃý?=ÿë‹'qÿ*ðóÁNØ]í†,‘"­YiÂM/ôNYu«x'|2ØÝš1–f_²ê ´i×ã;}!Ó"áSÛ@Xæ"C$:ç^“é5ø•˜††eíšè(Çiº4:8™x~2F(1—aÄ(A’Ô{I}š—™U‹Öbâæ —tñ) #˜ÃÀŒÀŒ /ÁQÚ…õÁ¥Ÿ°Äží˜˜òëˆé4àkÚ¥Ï+’Œ¼ƒ²Þs 4-ÖÁ¼<’? 7ÈS¹¢WeèS„TѨMSºT5ëZzíõ~{5ÓöØ»£š¢UÁϺ(|¡e^RÀó`’Vãiv8ÍŽæű­yd=JÎRGT ë¸PÜhàI‚ô¹Í©Îà@Ml€J:& µª†Ð÷w¹Â–æKû\Ù>€9OIT¶"†²¤l˜†Ö10:¦rõ§qáº2[ñl‰jÏÕ8Ÿä?_½‚©…yý»—ÿëÑWí¸Å挲îúm‘¼ûÖ.Y’DðBÏ ö’A¶ygÆŠÚƒãG“Ã4ËRX–‰¾׆M>7ë¤f¤˜UØHMÙ\£ Ž´*&'0Wb4™¿>þîÏ7IÒbök½hþ1Ú ÐÞ®Eïwut^7ÙÁ]½½>ØÞ^Ù ºíƒK, `@v·vš=º³ÌúêÕ‹ýã“ÑL 5j¡¯É䌔\¨>ØÏ\¢ê²©Å}š2æ…Ç£ù“g‡Ç'cvGå= à¿ìé)ˆ„á-U€˜$ê÷“MtŒ-sÑŒ^sR-$±yûÕulT¿:ðÎæÓéóáüù4=ÈŠÑœ•z9«¦²ÊXW_0) ƒ¹ï)Ðú±Kë–¡d"<ÀA©D¾ß)6!{ÛŠ¦Gì_ǺB% <´ã¨Ã ¾eÛ[-lM¡öq°µJÎf®&"-g nÓ­®–a®\Áȳl‚H´mm©q¥MVlD>òFÓà//žb*uVfñèÓû›¦R´Û—3¹j9fŸ 7Z¢'Åcm“e˜kxÌ:½^¼2Hþ¿§‡_̆^ÌÚ"|÷1¦˜,DmX‚Æ–‚!‹Ç¯&@®Ðy»ë¢¨„¨Åã5¹¡’ª˜Ñ5×E%¬²¸JXÔgé Ý½½²öO»÷ÿiûÞ;wÖƒ.É'Ø@¶™ÉP<H™ÏŽ>SÃ>ùò·¯>š°à#yþuœq7Ùö»­V‹YK‚Â(Qw ªPQ®Â$ƒxœÏóÛãgGóÇ/ž½{m£úD3¨ Ødp´º"Ѐ¾³»™)Ã=Àó@…‘EmŽ"hGÌKùÿýÕÆ rAËb$ d¨ªÇû©)Ts‘‘‹‰r€ÉlcéRÝ¿½þýû?w»¨ó¢KoSY’?@“àoÒKóÌòâÉ˃/¾üÿüý³ýã8î‹2lÁ,žË‹Ô§”PC¬=Nœ‰Ô´…Ö,;ãMòÛIÆÖpkÈƵnŒA­ÿQ4ŒØ5Z6ÊÉ‘Úý¤µÂL"³Giñ®’€Ë LÑ?ûµŒ§Óýà/Ÿ|u8y<+^Mó×¥?ÃE¦Fês AÎ(T” ‹¹dÚâ?<×L÷ ââéHÉô/öÛ—CŒ ‚NäÓƒaA^7‰z­ —´Ø·f 5Óbc~+ŠÁí8L’`@g¨òsëŽÂ”LÃ&„uÈ]iÆ }S0# z…ájs·Š«#– Ž"8wO#MZYyãÜ5ο¥cö#nϧÉʧkñNâ Ðä¨~͘Ê–}›t¼£aß[myΈÓxc?ƒ›¢‡“r?+µBPÚÅ(Ȭ[#&«Ô?üÅà/(œƒ‰QÆÌg³N6“¢­¸÷Åæ­_mÞ»ß[ß {]õÇ… ñ€DUZbm|sP̾>~õ‡×O¾<~ùtz|ìçQQ~sôŠ˜•Ã]¼ë½°#¶µÀ/SŠÈ·_mE»æ‡÷»ëÞæo[#Ä™ÙØc?¿\\FñØ ° ½Ò: å¥,Dº¿>¡¦sÁIú/¯Jü)ÌÀ>8ÚØ6©·ºø/"ÐÉ߬0ZÈÀªÖ6<-–ÿ…áþÑôÖnÑG(9<½} ~š{OŸïÿõÑë‡Ïöƒ¸«],Pa»P}´|OÁ¬†Ú¹„t–eZ6“ µŠ™ù¢hÌÜ öÁßù":#Ðhö‹£n+è2Ô!«_u‹ 8Ô´L+ë»Ð5aÍü~öÍË“¯†_ÏËýÔ?b©"pˆcEÁV ;JìÙ,€ÈF§–K¼ú¤Æ>NÐ3ÑAåOÔ±‘úÑòuÍðf«zÛmo» ´"¶l'­^»Ý¥£³|L'†ml⸃ñ—z£´ÉG§Êc·/Zm£2®ìåÚ\öýÕ+euzYèì^áAðø†Uø|R>ŸŽNâl?8š›¿Ü ¶×S XTkù  [h禨§P:†lzÌÈcúG2H&gÇ_ò—lÊCàE™X¥m•¨+ôü£¸ÖŸã4‰œŒTø¥:a²‘tô7ÿuï“ÇîöÖ»¢òjŽ1x"'îA‹çŸxóÇó“ÿýúáoŸƒšÙOÇ%³ñËòéè°MÿýeÿÖ<©º­;ݘ…M2¦m?Ø…öÁŒz®ÊMŶ“•öf𢗗‡ºú&GìëÇ8¿æž3ÐfÃ/ƃw=mpÓ숦‚×á¡ì¨l4’gÒûz*Ê„¸«¤;Œhƒæ¼çm¢"ô)–”4´zö?žÑÿ´Ž£Ð¼Ó"Û PYŸ ‹ÆØI‘ýÿ¦y0šIykPÙXF•VŸçõ¡[ârW]IÊÊuŠh1‚NŽ7Þá gjsùŽáO¥@Q!µò[.”ôÈÙÚ*æÉC“¦ÙKþ ZÌQ³§çL¼žN_³U4ÛñTë€iK4î…UYuÊ$ïjø,¬À]Ü £~¦ºy^ÀÒNkcôóœý ªé\lªS§X³†ú‰5H£ð<ÿs÷Éz7Y£§ÈÙ‡£t¿`¹#EºS"/Í¢ºòpõ f© ›© M„ݘ‘ ¯oÒ™ûóã4ÛŸæãpjÛ(¯fkÞn×ëE:!ÒY xbô9*E\Ë00ñì¶Q‚Ï’u<ú/¡ß«|pÒúËáøé,y쇥$«ÌhúM75úÇû=¯]`ù¾”¶ØY‹;Ÿ®ßúåæÝ_lÝ»Ý__e€ÑqW˜›i+» }þÐKŸŒ÷ÿ|øì¿^<üËáóÃb†ÐØˇIS)3ÊFÁë'wfI8—l3ê´å)6ñgh]Pµ‚:¬Ÿm÷?míL·ïcΟç9ðI/dJ¤Ê*“±!j@š›®éÔ45ü±ÕÄ”|ðÖ ×atÉ#7ŸÇ™Ñ&%N¨,TL[H¦¾óJvÊ0Ȳæ ‚°9½±‰å!¦@£*\ÏAõqR^BSµ0ª ×HáÈ%jf&þÎ:œ­§+Ô}¥¯ÉG‚Ù ”Ä´DS¬îR‹T«ç|¡B gI¹¹…¢v/-,ÑÃçOC&íK»íV m–£Æ^„FÍ4Þ' ) >uê¾9g g‚ ÏîºHD÷Ox&7’’^}%ˆ†„ð€©(º,qùªvŽÙì¹µÚ‰W¤=aÔ/?Ò™1‰Ód.ëEÆWysõ FpÔoGw’µŽQSU6…Mgsœ¾Ý¨}¯¿þÏ·>þ—íû 6×Ã6[}êƒóÐcô0'¨z=?ùãë'¿{ùퟟ=›²œ…ÙÅè~l§ ŽÆ^þÍáËÿžÇÁjGÃ0ë»Û¡Î[…R ä,Ó¯&‚i4ä´Õjïc6«L&ó<Û/gÇÙ„Eon¶– "Jüe(…&®›ù€¤°ÁZŸÃ¡®à,»¶–þ‚($M¼“ýÒ l¹ C(àWW ‚„ìei³˜$0/°ÎåÍ ÕG৚LmŽ±‚¥¿ÉJ~ú‹|x€ä%µ³%öT(gý¨\e*‹7²ð¹Â¹úýÑ‚kWÚ |¸š4·XàZ ­³†©¾iÏ$J8jooðy9oÑW§¯çå‘&+3ª„¸A•NÞ}7ÙhTKQ”­©`ì d©h 8–¯¨-â½cßñÞ°ìâUgZ‘+UoŠ ­Ú“Ù‹`Ú´)b5+ÇU€7'—Bæ%²Ê(Ÿ+ W¯`‡#kaÓÖ1ífp€Ø"~ä'œÎU"EféœóZ$wW‚-mÓk”*þE,ÀD§(â4ßv[^²Bï±ÃÒ¶|oNé±Î~cœ úB˜Qù•õCÈâ>Gî:t0Ë»­ÞþÚk·~¾uçÓÛk8yÙ|BBGä©àÌ(ùd#î‹ÉÃã×zùø÷/=ãô°’mHlä„6¤MÀK†ÄFóÉ7ÕAøúq·Õév:^«¯M¢¡”l]ÆSŽ4V˜³º…[éàäxÊ^ª'/Æó =R(Æñ BI$6eu»Nä³Á™ prMäÊc I¬{Ó9<’Ç[GÛïuuQùÂøHÍ(Ð4$ÍvU1¡MpÒlomÐrˆå=)DF­sÃ;2Áíƈ0SX5ß«&N0Úi|X Ò1àÝ•E)QŒE/ ®ÌM°RSÓÚü¨w®V¦RuòŠS6F\B¸¡ºMP³8X¬—«áƒr¥Û‰·'l]œq”Iž'8þÝ4eŽc©h}%sX7Z/a®*!]JH¹»{Þ+¥~Øé Ö‡–IY¦ $ú‰AäÜ"½­sɘl°ñnë䪆Ñ"_¢Ì1f¿PÂÅ\n¸z£ýŒnÌ¥Hõ¢_µÅ=1 S¯gVó0:ôƒl–æûé´<9·&»ýj-ÞŠ™—n²Ä~x°Å\<ñYAÔšZ¦âÖ™¸·ÊÆeýµv½=ê¾}y2}9g“^8ÈJ%mÃá— Ð.7àSmÑ17eÉÀþng…ic¿Ù{ðéÚÎV¢êv‘€ô†:gã!6£ÙäáÑË?â{ùøñÑË)ƒ88±mgL˜Ucï‘‘GÑ«rê¼èñ*ú¥ £ö S/Mð’Z¨±C :ÙÑ«4qÝ™ýë;ò°œWÅI>?ò_ÐÑöSgþ«òé÷Z…ºFªbˆ]³=@£X«¨&ñJÚ`Ü+´Oïq5BJuÉ‚<Ð@’Yâ@ 5´ `ùCáÈa`@â|¨+]fó  ºÎ!|„å|ù^5qµ•Þ7qIѦc(G"kÉ…üÌ¡£:·èAsò:­Â>xùT:Æy„Züë5iÚÜUa @bÅnŸw{·wº“¬ÓQÀ:›e3¾eØë8Û˜Ñ-/"½OÐx–­ä¾x–QäÔ Ó™Œ=ÒëѺòŠÑf¥¡<8CŒmù/cñ”eËFýEN…u¶˜&$ÃjeÆbÒê!,äsØ5Aæ4Ò3ÒRIšTƬ n¯.\½‚qu#)<šJQÍsoA§…Ú_Éð-Ì <;£y1{ã`Påݬõ™‰‡MlßhLC?â—WñIˆ–Þýö¦+âþ$îN¢át4AY1su±ÿÀ:ÆéWwÀÄ ®3eouã“Û÷>ݽ»Ý]épŠ<ü±+Sè§Bž²¸µ<:9yôìé·¯ž<=xu4•]Nôãx-a˜ $Æ¢Vk<É^œø )l@IDATöªðNwõ–ß[_‹VúLzÂÄ5.J¡<³ùY}Œ„ãh.Núé“ÉñÓ|8™ÌŠi¨cwEE‘jcte•ºFÖÇIRhÿø\m¤Â4Í¡î—;Ýú.4‚f8ð~WTòÃð>`ïW!S.q…[Ñ-M¢^…ð¡’TA[¥.´;¬3‹‘JðòbY:~³ ÕÇZ$dƒ}è$ŒÄ3…À6&ðw¹Òþ}ÃËÅ&æ3½Å­¼Ò ÕTÀ}Q£Úe°åe»ZeCI1É«)ÓñÛÜC -ègà4™ ˆrŒqþ€éÔJTOÎFT€‹Ô Ç!OœÅ›B!5›>„¨+$#Û È· uÂ~Å<åøøbŽ²áJŸ MôYl¸ã`Æꯘ0ʇf1¦Ö^OœÌ¨ÔüEý!'4E™W®\ÁDLXàdŒ#³âÄB—ÚFc!¨ƒ(i9]º"b[ä# ç¯ßJNªÎ´*öóÖ—/¼ŸåÁ¿'É/ ï~àíhÛ9u¿áƒ9>2¤¸ÞÕɧ,`†YŒ{úgA~gÍûÜŸÿwþ¯jøÕË£ÿ§Ø䃌£ªCŽ÷…󘀑àGÂõ…f<1LÀØ¡šß‘ ¥ÔÍõ ˆu²k UNnE1°¶a øÝçÐIšþ*_ýÅæGÿ¶þÉ¿÷ïîtç,áÒwØ™B…¢ÀJ¥'â­ôñôàwÙ“ÿ{ô—߶ÒY%—9Ëž‡î±R™¨YvH^yÓ.sçßúÃòàËQ«ÊÖ[²¼w:²æðÈ }TejÁqà”T?ì1?9êç|ëÓvý¿ßüñÑtž²œ&æÜ9d©v¼ÃrCÒ’u¤‹ 겶»›Å[g¨y¥A‰×|Ý”d¢C vT]Wó&Éûþr>¹ © p@ûb Æì3N+†ˆ¹L©,t¼qŠ ÷óR¦ƒ¨–gâ¡œc»¹5+Îâ “EUß·B5•)¹9 Ž0Œ4 Ä¥»Zéfu^"> À¡á?D.iÐ{Nò—Kpt8y¸Ö߇·Z¬€Ö4ô(¸èO_I®h0R½Fé6ó}Åêð„ìW×ú%è¥1ða, bg‰ qA=ÈÚ¢T)¾s÷ºš#Œ#^ãäTJ3%d ä¨.;™Óªfå$ÍF“l”eóyñt\à >œÇÓü¨ðÆU”ⵚXÕ 3‹RlÃŽ`¢TAàê•+˜‹V=ÄL¤Á+W(åüd8ç<¶Vöªy´Õ 7ºñÀ_ œ'KÏ6ÿ_õV‚>{û¶Æí8,Í=üdS°¨³ü  p'5´¦$O팓 Uþz&x–«§1®±U» v’þgÛwÕ»÷éÚÞVK[ñÃ,ôñÞ®úú4v¢ÿ?,æO_ýaüä÷/}{ôjÌäm3Êœ•šù(}N¾k•þ3mÒœ¤“?¿|ÂâÍ~‡µÓñíÎj/WfšFVMq?,Û è¹¶ïõÖ‡[wæe:fwÓãTÛ-kzŒ­/išéÚ CRšóÂÔÕÀcG4Âå» ‘›û<H!ãÓtÝÓæLçr:ž¿|yüóÄDúÆt7*¯#W˜Òˆ€”Q¡ [¢ì^äJhÈÐ%P ¯GìBÍèSQÜÕùÔqâA£î79,xY±0J“\ö1ä.»q­J`Ål%9ê'ÛGÍΰ›„íãi‹í8Ó*µEIµëN2ÖJººËµS0IŒÐKîÙÁ¤®ÁpnÕé×c?-™Àœâtœeû[Éça±öÖz­‚¶º¾rE[@ŽAy‡qà©t[ÿ£ãmÅÛ¯G¿}~â§/çÓ!ç¤Íô$æÿ›6Ãøq¡ï{u ÿû䬆˜Ù5ÅÑ©L§Pkne—öVåo²¤q}icÿÔýx¯½ÚEWØú Ée«h-‘K˜€ÉÆ-ÏÓÑ_=ùíÉ·_ _íg“ S ¥©±*¸@ÂV¾˜Ãì?œÉÚ…ŠQ2Ž{y4:Hüh¥ÓëlÐÁîø> Œ\£4‰Gc?¢0*ÖèxþNÜ«¶îd^>šN&eúpt0d5m»¥ S°>Û”‘toF ŽŽ”Íò«¿nJ¹†°^õ¹zA´"W?e cüWX‹–¹[kízÑvXÑÁ·*7º²,¦M%ˆÏÎf¢=É6‘´¸«Kv>½b•„{•ÐQ­é5w«Üˆ5„Ív˜À&#„í¤3$,:[é·7ŸîÿáõèÛ|ö²Šç:V˦õ©ovÅìp팆c™h…­ÌÑÑf.0§±Å@ï(G ¤Ú¢!OçÅü$ÚíwÊAÏëÛK#¹&­lÀ¶q<Ýû9d:ãÈÛc=æ&Û õý,gGì/‡ó§ÓjTc&û…ìúËrLåd4Fû+ü$‚£Ì…Ø…º‰Éè £ Šru>^ÛþÅæŸmܺŸl%a‹}%kãHð4JÆ[)¢VfX¦Ïf'zõø·O¿þrþìå|4aˆÚö†SçO 4ŠéA‰9 R:ÚéÅ—ô¬Œaêøãñáàå#¦›·£V¸n°Â_~MÃ[ãü6´šáWqªft+Y™¬î ·Ç£bÎpNŽ¹ ´˜™¯JFÒNÜZ¡®ùz¶p¾ùý…@3ã(ã&`ÎñùÀì—TrÄÀdØ=Ø<àl˜VÈ4V¶.Æ}‹-…²aàOhMŸGòMœ‘¢hQLåt‚ŽÓî#ê AÇu*ÒÂ<ð)Ë“–V}'»]|y&Z!´È›z¶ü¢[/÷™·œq$¡<̽ óäB¸þiÍ/ùîÚ)À "ÔÁZà¿ööæ™ñ,­Z1Ë–ö±º‡Ù8/žÇáj~o+aªÛŸö…m ‹¥ LG1£CB§#~…±ë%¸×Zë[ƒÞê‹£è`„ÒÊÂÓn2m¸¡…̤׼ݫÿ%có-Ù9a H¡%IË }"Ùc”™‰p¢ÎG+Û¿¹õɯ÷>ú¤»Ñó;²±/èx˜Â`4…5.¯&'xþí½øæ¯?Ìž§Ìì$¦eÒ]Ä—ÅæìÌДsM'M¹ P3°"ÄXž6€´l<›¿š«§ß2ºÚëtáíOÖv8ÙŒQæ´¤"‹«ë#†ƒ`žðv{m¶u-½f3–T øŸäú²ÌU «Êä~5ù-ð8¥Œ󟉾yø‰BÀ9„j«ñ­¶£(Ô¦v³™&ËÇ“Izȼ²×“¯úñv¿½Óom°‹Ä¶Ç³Y9šJa‘Ýb|ÆÅë ÍS4y›\«ãÏ^MâDºøæê&ߺ<ê¬ì{ '2”ÚµÇ>]¥ñHàæÛvý]/IŠi—¼ÎÔÛÌA$í‚ÃëO.ýçÚ)f_H—3.ÂŒ‰`W0lLÇw÷ãÌcî7Ó&Ž«i”Gky ‰£-éú&Uø|gõKßA’®l/ –·W™ØÚª Nòe"îä!“ûØN9$š§©®Y¢œKþß)CGE\„»)o§`8½¬úQ¼Û’e•ŸoìmG}ù ]Ó<ðw´oߧîÝr”Î^ŸŽ‡ŒÕkƒ²N_k)]° “ ò¢{æ*;Æ#ÅGÐÏ`‘9¥ËS€úébKUá¼8És¿½¶×_Sõ´ÀfSÝ]5Ä¥R’B$ÔAÔ¾»²™úÅÑ ffù 3i¦ðö\ïõ¹ÓQ49p=®ž¹ÎxóxM!`=°|±Z[%êÁAË9.ãìp–±ÞÁÑôY/Ù^íî®´w8*&‰´ÓWwÚÞ&­Rrmª¯žñݼOºÕØ DPM2›EÊgƒÒP¼4†~Ýû:•èÔ½!F5×ËK™ÊëÕØq°›ÿxQ7Ö;óqÿ$}4Î_²E Š“ ­õÕ•…k§`Pɬ=Æ•NËm´Ÿµ)<# ÙK(jî6žþJozÄÎeÞ ïàå0ú×[½Ÿ­³v5вœ4O«d“€3É×Û˜¹ßóÚ÷ûkýþjëhåpþÇýãWØÔa ±År`Ç4¼ÖÇjúhƒNKdn^È D´z,ŽääÂÛˆÛ¿¹ûéÿqëÓúÛÛíU†úÕJë¬,žèWL'ÒeO°étêålC>ع½«­Îq(PG™ÕÞ5½¥O£eÅ•7cÀGd­ $Ã(ª•°Ãþb™úð)Æcì³ÈMlg|ä4…*PôY‘±“ßíÎÊÿ¸ÿ¹×Š&_¦£Ùt2+ÒvÙf{œN(M5FŸ©3*5U³c“‹J¿éÁ, ñ½>ªm/5^çIªA„©ˆÜX±ÀÐ$êA6 1:ϲ'i:f¯÷'ßânB»tÃu®Ä2ðw™5©ÁbNvã˜CvbæèIÖ?0 ɤvÕщdÈ I¸É›lURe¤O¬ü˜‚15CU ×šþ•lÁeîCÓX2¸pâ 31»éѤwÚ&“Ú ýöÚ`w+ß;ž|;a—@¶Aa½oNsV1—®Ÿ‚1Ý,HËÚF:ª_É“ ‡Ph­ÈcP—¹L_'c¦ƒ±tR›‘w>é·³¶Ç[­4‡ ß8Àky­©t²®ÌïcMFâ5v<_éÄ#ŽÄürœ>åX$<yi“Y%ÒÍ$•èâƒÈYê)QË#óLØw0-ºA¼Ý鱶ÇA/ì6¶×[ís°Áßh”Q<ÄNcnñ½µ­¤³¼ÁÏ@Q}5ÜÙ† ñRt>}#¸vŠ‚‘Ê—Ú—ˆx”xF»¬Én×&’iU `v9‚YDáø‰ Cs@Av³~y´>ÙßaŸÁâùÉá°œP–&•¡Èìd¯a!WiÛ‚_Os¿¹ûG‚€‘ó‚$¤K$&ÌÞQ¬f“jüWž Ñj9ÉO¤üà eV*›¶¯$ÁJ¶;þCH1Ž˜žÏîI ís.nAÇp…c˜)I¢aÍ©G!Úµ•eèŠíÌ{äz1êóØUwÚ‚‘©ÄÆÅèÓ‚ä¦ü9}ƹ5½}¾‘ÈR;$.M;2öìX¶µ£Î¼°Å2”E~‹‹wÖu»Z¬_;#]"ƒTFê¶8•ìŒ„) 2 ‰4i?O‹ùëÙñhôú¤{pkó»oLH€˜Ò–J–T¦‚áBC cðõ·{göZÝAÛk¿¯²G“bœ¡c{È©:`› µL„Â#˜ƒQ-A®3òÁx†C¦˜6ö³Áοî}ü«­»­n¼D¾cióÓv-¤°t”¸¥unü•v÷v ÃZC|‹tU—Ýv "­-䯿jŒÌÂ)lÅPè…Å­—Ã’¯8p”c•Yìɼ2FÂÀ¯«€+]È´üÜUìî‚d«~²QÍåG‹Lògù¤Ò!µ˜mš:(Ì)®® u#Ž¶žF,ÞÜÜü´!° "…# ' ¬ÙR3î0™¶°ÌÄ>">l%ÝÂq{³0¹¬%K~£I}iõ78õP&©:.ò’px\f½Mrf±73ŠÕgâ?5@±˜:¡yU õ?ÁQ0@Õá~ ŒX“XPZvàÇnýNÂh z!’LsšDÛ2©¨!zgÖ&ãFíÖf³ScÏ/ÙÕ†„ëtW‡õë§`ÌŒ€ù‘8H< $cB \‚lccP,,¾˜k‡1Î×ìï"˜f»c´²–NSN„BGU’(P½_‡Àmá4ëzÞNË‹6¢¸Õ…(°°“ýÙ“IqœlŽ‰Ø¢G‹B”Ñå’°R®u^ECͪ~^`­%½{½ Ž@þ§­îö×Ø7$t "ok³â€…I:ó]D… r®Å~ý 1‚—‚RFÀ þÜUB__ÕôRäMÅE ¦µt×D¸h]"HÏFœÛíþg›·ÒÙ4;~›NF£ƒChLô™ yÝI©‰–]]?Eß„Ÿ6}¹6"d 5úÅ"‰‘)‚ŒC¦áíú%³ ¿Ý~†¡E(  °føK9u§*æ—¦Kr/e…“ ËVº„ mHF¯”F‚‹ )˜»*#{Ž©‘Ž(Sñò¡‰<¹Aߺ;ø¸8>¦Ï¤^QÀµÓŽVZíì®FÛ½p-ò{ªÆ‚µh™µ´A-y&œ²ÌŸÉ¼! Fµ:Ë¿MâKû½v W˜95[-Ê3`‹$ø5˜(¢‰q}f ±5u븈¾Wã''GcÎÊ^ý§Ý6CY;àíäÔŠð 8! 2¶.°PÆ×v2Á ä5X “Õhô_¯NXÿz¨n Ì(€aûjñYóùõþ¥æTž>œ¡-Ó¼µîèçÝa`ÿ“]¦k8R‰Œ)¬9&¬õÌ[¹¦à/î8alAÅ°v™}ÂÅ}医wß:¥#è?ç6Lè­ æS¼«g…eE0µØÊßð®ÅX»ˆ¿=ßßëoä›ójgúd–3{ôz2”“ÍyÜyöáB»(ƒEíÕÍå' eírÚDS¢G£&Yµ,nµ’~v›añ¼ÏÒcí°RÌÕqa7PHóVÛ½Iü#BDÂŽ»‡Í4"S¢ôO܆½Vpf±´—ÙCÆ(s€àŒö~1W˜Æ^¬ïrzmz3‹óeRs–WÙÃteP]Œô°ûYo¥}k§ÿ€3ivW>YåÎ?ëfh×B°/‡réQ±\ cˆ¡O­ÁSÀ\òÝ¢—œï÷ÏN~Ca9Æš~-© È÷<è¯êêDEð„±Á#)S?ÄO‚ˆÐÏ[ñŒ¤•ß©˜Ž†:±É”ŒM©†d&±I›Ãä›aèm¶"6%QŸµüöà˜‰kìðà ¿i:}ÄM-ÜôtÝüì0Å÷f¯¼NÅ;ý/6oý|÷ÞÇë;[û †iš„&‰ö–ƒ“ÅÆšàE†ØÙ|\g®-çMB’Š3-ÛŠR,Æ~\¼ûqyghp†˜UöŒlø»÷¾Mæ'œ%4æM}ŠñZˆŒ¨Ac•¤Â"Íynò½ùýƒ€‘”$U€ Š7ªt7⻽NU±yäþ“1UrÄNɬcm¯f?jÓwueøú±94Ê‚5šmò˜XÀF¼ê˜Íù9H«RıêY‚®&öšä•ªÈØÞŸÙ0˜4ŠYpž—·[¯æLž¬Zýöz¯½ÉªD¹) M'ŽâtYj(JÍÅ¥¦ö\vMòKÿ½v †qTÐÐ0»`r8uÚÆ¥œ p7¨"Œ\úµÌd²n¬±Áÿ£I6vtß¾»Þh©•(Ìq‚q]³Î‡ŽáZhsÁqjãÒ£³–ô£­[_ôÆ““áÌ/'ù ©±i@ÐgËì-®§‹J×ùÞüüC@@Úö¯ U;Û"—“¤³¶¹¹DñxjKÂ’öÞÆÖÇ;·>Ù½³7ØêjŠoj C„ó÷rÓ§ŽXÅE<‘ÌÅq}ûž±ö….Kßòä>äJ0†«qÊ£!‚ªÀ#M˜àaPó¦e¦·D»lÝ Äú:Œ¦Æý•·OFÃÃ*=ªR¤9‹=m³l•JÐpNµ4Lïâo®?uHšÎåZ7)ïÈH²ž)<Ä0á˜éaƒÎîÀÛ꯱ºjšæ'¬™å¯»YåýÓœS"˜Ä‰d“²˜h#vÂh‚@£!ÜÄ(»†]M—Ôåª:â ]­>‹G‹s•„Vaa1)¾2b¯F~S7þçÆgÿ¶ñà³`c³`Cì)'çEÅVÍð„µR¼@î« š/ု4ÄrådI½<­H¶þy[¼ËΡÀ’)¢¡tP áYQ,UÐ4 š™fƪrb»nI)œÝ˜æiÐÑÞ­í¬|±>㣴ô燓Ù<˜Åìš¡rE~ì[G… Ž²äý᛾Oõm”èj~ÐHÎtÈUîx׃T70„ä ýÕp-ðC‰LÆ)2Eˆykÿ*ÀÿÏ I.p¨¹¯cr¡æÓ#5†8l]ÒDg²{T$QÖçx$ïµ¢µ¤ì 8/×Ïoµ~^ÆóyÌÆø,‚碱íM4æNú: ¿Å…>g/¾ÌcÏwoF_-ô9ƒ’s-å:0Š&ôH.,‚³ŒÑ,Z‚nñž›ÒC³´æ¦#S®Ee»¬%þúzëînçÁ^÷óÕpˆ|jZl Ýãr1eC–LL ï’iA[">bŽGƒL.xÉ÷V0”¶­k/ÊUД:@¡«¶ð¬A Ñ7ïucñ‹TÒòˆ2'¿Üë÷¾Êå¥Ä’€ÒDät6¯²*Ê&íyØKÇÝb%ÚêÑ5I8sG§À(½SôÜ q†ÍÿëTU^é÷[¸Ê:)÷¸É SJû³o©ê©¥`ùÕÐÀ wM¢''­U1º@ZŽüÎ{iÖ7B¡C„Õ×soh!´ ±²F"_iµïö7ìÜùhç˜^›‰ÙQ]zCÐ.W±0‡²r²w>Çäá´½E¶uÑUuhô}ó|É¿(–‰¶ý°µOA%ÒB`Yãñ“ªð·“UåV¹ýéôÞKKŸ¿šUš5ªé£h,¾ã£º5—\å1;ƒM]¾#Ik£´N1™$XøpjXbñ/$-ŠýÉšvJ³´Îdýi+r_¶ „«0ã ÙdzÞ÷ú¹Çþ1ìå5㌯ŒÝ‹1º£WuP3)*&Óù’¸ªØ¾Œ©œÎ‚-/Ù)•_ .4ÊÂöN…趮Æ)¸Ý«EeHàd-1†‰C1³È*öë[ëÅ›kÝÛ›ý{[íxÀ®06 ÀÉ(-473L˜åŽ9¶ø÷ØúÚ2=–íÖ$"Ä1§¥Ÿ‚ãòî.¬`$ÃDˆ§ÕÒºyñ¸¨áŽTÅ<"ŽVñ'‡—³7EêD›P^|0Ç\" ê´Â¦w[‡^¨}¢›…¤T¥.ôvÛ,&Â79}ˆÆf0†#xfÕƒÝþJ×c'z¬8N:R&sÀtbBU[ÆÊ„6Œë´ã.c2U¬½â×ÌCåªh­‚²lÀeI"@æWãx*j‰™ù=á¾·¯ß÷Òd¢ôT’–=‚â5¿Dà%r´}ÌXŸuò·ãþ?oÝû—´LwƒîÝW’É&p,Õ‡þ{Z¤û³ÑÁl|<ó9 R„ew¥× dfÅFÜÙé¬ltû+ ;ÚbÞ`‰hüÇAqi&†ž@Ý éÝ]ßÅ…Ál¼=œNdIz)cu¤àr`…üøÀ¿Ò–~¿ÌÖÆRsÜBbI¤qPÉ ±"e¸#qnÜß÷«ÑõÿJdN-ñº€Ìا×ç³D¨É×A°~¿ìbÅ JféWœ}3£‘Í5ÿžÆýÙ‘þ [ëÙaÆÜçtet¶1Òƒ7†0ïAµ7‹,Ì%Åîaò9Y×æ¡/Áÿµby¡MO¶½8W?›].Ž=k¶xûñ£m“=”1Ñ]î¹a;”æÓh.}.F^ƒr„Á /ô»É—¡o)™"ÖèÀ *«t`#–µðµðn÷É€UQê™êAC1þj·6ÕAI gŽîrÞîð™Ë‹^}Ç7Üvðèûñ©<¶ÜωtŒ×lV‰àR¾5×\y|ñÌWN>þàô‰™ÚG["½SMGÇAh Q" (ƒó›”Ýh*Ž^;~€w²•|a²À–Ûɯvh ɤ¢Û²W{,S¼²<Ùœl.]U_é4›gž^b׺AØPÀ½\ÃZ&•q‰:›–b "'š 6|Q{4‡\ˆƒtk Iv,ÞåÓ’÷ffœªj˜‰÷^Ì^ô C‹6À„ 󬑲0ÃÎDo¨.c$R "'oŒ6q«àQ÷{è³9ŽI‚Ìè=4èÍðÈñ(»¢™…Èy¼) JƒHƒbèr$ŸkõTÕ‰¤$Ÿ‚ìÆ&Í-Äpp`pP;šòÉqµš™[jžž]yjvå™Fg^oÆj:>©ÔÓwÿ‚ Œý…¶(8‘Ï8€'â› Ó«¨b˜8OÀÙÅßô`œK)Ý8¡åõ°sG+¢F¯‡©*Vµ±é3P$œ°Ë„([@fÛç‹õ•ùÂy.Ùjr¹Ã˜öŒqîcIÛg¤ÖH†P?Ð`ñåñ3n†bäjò(ÚqG’Ðøzi€xS½@ôVë3`Tàªsx‰x…Ùª€µi6™+ç‹GË“/9rÍ‹¯¸æê}‡öñM%â Çdí=€'ÙÒ†Œëb¦ùìùn{àôÓÍ<7ßaa“sÞtoB¨Òk#7-!~:z4Õ™b…ÛAmrr’û•GÆómTu D^~Ö}¥e’Ó*\rÃåÊç——Vj+çZÕZ…MŽôÓ"F¯cß °B™&`Î#ú71q{›stc¤¥ÔÐ5Þ”ÜÊ´è¥^X&QÀ² „¦zeg ‹¶Á•L˜s%ˆfÊ´91rfwj¡ÑÉém‚§Fe-^“kj‹údž#ý‰\RU*¼•ór†ué¥MÚÇô° t|4ç‚D0ûÏ–êùùÆsÓ‹OL/ç¦Î:}H–_‘h ÍiX¤º@ÁíÖ7Q~pö_DsˆêŽÀ¯Þ¢`™YwÄÉ]Å!ÓðÃ42ㇸˆ(õÖ"`\Ý¿‰áKMGÑ£ RÍ;5¿¸ÜZ>r°ydôú}™±ó`²1ȃØ(£d-M¶Q‡f &+Y!gz.U§h*QÕfçðy„Á±.Ô™­ÊHú^¸£hQR‚bÈ”úªµÒj+‡&¦n¸öeWÝxí+ÇøÀ§'… rñAZ>‰;ü¯7ë'jÓ|ê‰çŸ9½yº>O“®¦>²qÚ ð!•ëq‹àUJöèDÀ¡ÕŠÀE }Æd1@™â³ÓŒÖªá ,–ðãÈ»fTäe_z¾ccª8¢Àjs©(V*–ßù5³Mýt_½*Æ "‰½G¦"ôœ˜¶¼a¯¦½—@‘XJbµÕ{NEŠö–R”,éÔÇS¡]€m÷˜²GY˜I³#-Ï`ÝCåT+Ï~H׊×sççj'NŸøÔÜŒ`VZ³MPƒç  “iKÓÝÎ ƒà‘îEf!Øf#‘ƒFHuÂ/¢b9µL¹‰*d‘0@U q¢cíŠOÐMË¡Ç.bI&›kÔçåê s¤ËÅ\…s’—3åk'9”“/›~–4L(-g™T¸œáçöôÔÝ[ç );õŽ°žF jN"LŸÍüuù·ˆ 'õ«¼„#’.ˆ$ÀLc—b;w¤<~ã+n›ºææ©+—'ËêÛ`D“MÌÞDn“q±Y]\Y:¾4ýÈó'ŽÍœbr¬ÊæN«,Ð 6a ½j†#šÉ» hú¤jz’ìü_nUsK³£Ó'Æs…#ûöOVÆ´µTób‚¯r`âN§”ËOåF®Û¿xèèJ£z~aa±¶\câT³—Jf=€^Lì+ - v¨ùÈIé#äj½hLv\À™<¯¬‘ïR1c HàîÒ¦OŒEøëxõBJtkÍUÏì«Ot8€¿Ã5¹ŽšÒ'þÚåÂþJå2ábÃÊÑC!"¿Õ@ÚBÂP W#‘€FM7€Í D[!1ûi<^­›æ†`&B0R¦+©`üÚÙgµµÂÅÏÕæ c—çYzj¡1ÓÌJIètgl§¬Zàr\A¿ÿîØÀ …Nh#±à}Ñ£¬3ã(×:UDžý:¶ RGR.$LóZÊ.îÄœâ(™¥·çøô\!ŸF~漮 žHÑ‹Q{“I¡Gæó‘DªL\¥ÄŽ¤Ñv¾}²Zk™]h6¦kcwÔø¸=”{Ú>\^ª.!6–¸R~½6щ£ùËÉvzek°i˜o¶H¥ÝNâ¥k|µ¡òÛÆž^:÷Àô3ßœ~êØÒ‰•f•û/\û5µ F€ÂQé)³²šâ)H³Sðwœ¾®):“(Ì é™ö™úbþì `+•Jl¼æ|R#ÓèÆè|h£ÜæÌ‚ý£·\ym³˜]¬­ÔlÁ}b,<;Fz`ŠË¦ztêÞÈÑ%àaVÍø±`]ƒ]6\á¢BŽD‘äzEE>ô"R ã)6aÒ”™²òö’çÊŒ¨Mˆ|˜ ]Çvf¡ó\Õ\=9)ç9Ÿ;TƸÅr¤|°¤»,Ç+Å‘R~¤”á{v òc˜aô‘Ia¼¯û£*¨F½:w4}¤)O]ë'^9Ÿ’‰­Ÿ¨M;& ‘Yir¦k{e¥±¨“›µ™Æc+5_\™®69Íb‰ÑŒY š—f„¹%_ÒY @?Ÿv,@& ,V”Ò=8PÕß]Édg›gÏgo3YTðïúІPŒ2CºüXg±7²¹g3§«-®@nÐg¶%,“áÙ3àŠ›œ4&‘MéNã&lj$ TòµffºÆ¾ðêJ»Uít;SÓõü¾×X©÷«ýêjaÊm[ ULBÉ¡qQ¾`Ã6e[zãûÝ¥•ÆéÅò2‚ª}ë6® v5z%Ô±†MQ9VØö=Y—ž*ÖÂ.¸0x´ÄfùßáOÏÏÖ²+,À;².F[I›Ì'ýdfêËOž?ûÀ̉'§ç«Kì³SÛlGP¢'5/Ò]Î’|sH.ç ž9;|´2;^Bò)†‰ØBQ’i‚ˆ¦¯çj'ÏÌÍÌrð‰ð23f˜±V!¼Ë¸½Àê5¤p¤˜2ˆ „Þé’N¢gL'-[­sW{¯L¦²¥GHdÇ+å©Rë"Ã,H)Ï¥–œôÁ½,üŠ8].©ïþås,ž¦>¤0ÖU‹¦©7¡þNh¶fÆ“$V3“¦×JÃã~3K(O êvW ßÓ´dZ˜‡Y¬2åÐX\aV¬Q;ß:^kT—u4‡@WéÂT„BýC¬ š*;8é´º˜x2¦z4û ìÕ¢ g=es ÍÎ3lÕ:]<ßaU•>‚3údä€LòƒÚbͪ™-Î׳O/ÔO°¿«Ã„¿ÂE¶rÚ‘)ÂÑ1ƒ(ZÏ ex0‘’"æt,Í,Ç¡qÓíVm¥1Ûœ?Uk\])Mf¹à’õ¤÷Î#*M¦ŠÔ#.s7ØB·MAd9ÂIÛ`Ÿlg¦«UpC%rĶ€¡)ÑG)Q!.ì-ˆ{ X·:Œ`j'—æN¬Ì=ß`ÌÉt|¢ò‘¦j‘hJ@±¡FX›iQIª7`¤çÁ8ÝÙÀž}îAý˜&ïXð¯Ÿ\™Ïžyúìüìᑉ‰B™A"˜F Ûì¢CdÂ& Äür²Ã™åó5°çØ&㟮½¼´Ì`fŒ‡æxÏœ{àá'OŸ>=ZÔv(ÞBTU>§/ìIvÂæó+õÆÜùê¹ùÚ.)Z =+í´“òw\ö%üפ6‚?jtEªH#t¨Á;ˆ¤y/탑Ì7êeÚA>7w~ñ Ɔ™ ®$Ô’U¶RÊq€1#´£"‡drŸgåUHÉçǘ•’Éá”#´W kPý)^Æ|I,]sU/§±£œFT°c…^J¦ˆš2-|ϵÄWœ5¤iÓ‹à” æùñù¢³ÎÛVf‘N#‡°â¢I6ëkÑC¶ŽV&šÔ¡FL ”çõOD¿mÿÙ±A,ÍÖZÿWø✓á•h²úÉ/u:§kµÅ™Ùi­ÓkMMšÑÁ²©¡¦ ™ËbB|ïÚ¹*Êͽmø•ÝÊu‰|PÎ,€Mãx9‘4ñ`ÓÔ YÔk³Ì2g5çu\ëD>7‘Í—¹ Ý®¡œKVëc|eH@ÄlªŠxÛi²šÊþ ¦Bg3Ì·i&[èZ“‡«¦ãa5IÉÈOPÖ3è„m#LÆÈÆ×a‡Ð¢CéàdÚçØ6Ý8ŽLM³–PœjØ8¡£½ˆhfa©£­¥õ˜óÒeLÒ=Ú°Àª´£²o'Àì–(%9E¥Ô‘é,¶êÏ,Ì``&K##ÅæÐQãÒ/6š¬ÈjǺæ*Í^i5ª›Ú+§Å€¯Èb‚L̆¬ÔÚ'ÎÌNÏÎ2gîF«ý ï°À|1¸¸Í*ƒƒ8v8W«VËÕ,›r¬bémšÖáí0êÒIc#5sj ³Ê¨©_É?Ì@@4}ŠÏ±•ÌHÇ DíÄôÕ˜aû †$ƒAÑKÛl–co6†ušR‰»>TX<¦Ñ.V³Øvþ¸Õ¬Ç˜jRôž{ä-gÏÀ&lçÐhZŒh°%=Óá›Ì5[U ú›ÒJP2 áÛL©cR2£V—u¥xÔ¾ù?àæ«ï?væ ²¤àòA SFuYBBŽ»¿ØyÔÉœ¯VZœ¦,½&=AÏLÁ–ºÔ-SÌ~ÚÕ¤™<ß8ÁL@‚î[Ã&“"‡\D[F°C,ƒO›SP‚•)·¹<†Qlljó?ÍÁ°ÚY<Ïð¸‘61¢å5èkÔèÂQ¦ a‚[ëõ$#AI->Û¡ƒ§‚±jŒ”ɸ¬ úÜÙlo$Ž8²Ð]!¾‘Žx%§\(ß{Þ2xö£ ÐIâ{ÑåN“¾ÍLsÅKvCA”ÍÓA%b˜FCH‘kÒl¹ -›­2ª”ÂÑ{`™BLT]-ojgpŽq(ìø¹˜cÖ‚ÞÝJ§¾ÔÂ4Ö¸ÐthÍ‚+O:Ñ„vfø‚ž–‘$(ê·SšÆ:mæ2.ŒƒCöJe)*# l¼Ð>LµÖ¬Öù±aH=ô+ùܵ„Á`¤ê¢_zDRq–Á¬ZUFW»Xd³ÔP}íÿàºx²˜†èµ™ ”’¾L0ßÀ¨½óZI@  4ÂvðÀrºdËN˜f|C¥•YAóu ¥`B¤Â¥mñ¢T)yÉ¢•†ßahH˜b¥ ‚#™RÓHÕ=‹4Œ^oÍÝLVõ© Y#ݶ©NH}¶Ë°h4—ÏŽ\µo$CPY?Ó$IJ«!‘Gd€nÇÆ`Ñì“im°2bÁëKTT˜TÆ_L4q$/BÛ¬%Mkœ«¸ˆT)gÀ¢:ÚŒ|’óc#¨Qi\Â{ÓŸÒ÷¢µÕSKä3‹&Õd¾¸BKÒ~$ˆ7 ®Þæxw€Õf0lDé´c3  ª„•óÂï @iÑ°iŽÈÈ1ã'-ÇvZwAë’M &xƒB8[@Já™y¾æR‡ ®|²ûO‚ꘜä2q,;²Ï1{&£E¤FÌ e SL#•|;Oÿ °,¡=1U`Õ žNvd„Û1¹M6¶ “Á;¦òØù®L1Š™„0¥6CùÕ ™Îñã†Ã4µ£L¯ÝŠÈvÂDqEòÔ£à ¹`â\CmçÒàÑV ´ —;)ž&'_Aõ€%€:'«S,Óà Ã¶#FË>ÓÝåº Ä˜Ü4I/EiIF;@÷ óöiº±‚4Iâ:Ll ñD)Æ”dPWˆƒnVCG§›!ÐÜ éFž¨ÒXÃðiIìªghxá¾­ÿ¥²‚¢¶%^&üÈüXqÁ¼$ßâ5¨B¤[l¨š0óa÷ZP;ÒjÇxà!}E_£o#0ú÷'úJ(I„Ï• €ŽfŠ#£Ç-‘MR3V6È#+=Œe·‚ØN%¨9Ÿƒ<é{qJ$çørä¢?ft±¡ìÏQEz§œœþª—ÏÌÊ,á¬kAÐñqßÞ™gGà ¾F¤ÃÑIz¦X }"HÑ7®1œq©Ñœ þ¨9ª<>V"›å¶2î‘P…L¡Áã’wAAL©km_›8ÓÛlÑßJÆ6(À³Ì!F1ÇâA›ó-ŒÒ€:ê….K)œpAHUp¬£c![†KJ‘vb+7î»/,M-ß©ñÍd``i½‘Ø&?O¾ê~þ˜®¾ï[H; MÀL¢¼X™SpgÝÅŽ‹è#Cð%RkmÏ#œ_¢ª÷RD¤ˆþ¼Š-71 ;—,.-kÊ&J}§CLY¥PïÃ{×$ï—Rô“|®˜â£&Ér€×åÌGð¹Æ]nU¢½1èÙè«:úND±!Ê'1Rî8Hçµ$ièá5‚K2Øa|ê\ëà…øÏÁ†o$²’£¶L&² 2\ÓÞ^Œ´´uÅxŒ¢1±j9ˆk9k‰ó»ÐžÖu¼JêËK7_¯ÏXZü¥cšH£˜Œê ›i\¶0,“‹¦.ôNç(ÇÈHF|öXúHNÚ"I^í³©Ûž¥®Ç»g²Í#×bóô4t*¸oÓß¼´í¿áLÈG"¸ýr^HÊHyÅÍÜëw?–Kïa%N@˜T|ö*z‡¥ÙHÞcÚQ–]I¬žæNÜvðr&)¹I }i›”ÿ_ ‚>/¤md×üô—ê{¡þô¡”-¡ˆõÆ– /Ó›˜.’hnFñƒmY¨C$lû~èuvA»ýÇÍK4¾²êŒETŬ‚;¾5ŽÈ6ˆfW L—úóžÙéR£•…:÷H@ô΋.*bÜWÝæsv5ýB] oY뜶LÝ¿‘÷¡@M~ø/¶[H.÷ÀÖ%;ÞÅœ±1dp˜zÉà+ðÎrˆqëX7~*j$6âIDAT #fHèÊ;:V®`˜ã¶õ7η­Ä»˜¨xà ICx‡¨% ý³&yé3“¥B½Í¤”–⿧—ñŒ[Ëj"ÊÜÙ07dMý¢@000ÃITr‚]âfxÔä•™œùúÎdûµtý‚é₾JŠíCra)Ùp-zˆ®«õ®† %x°#ÝGbw—ŠÁØ)ãú…W TOàBDö˜žŒ%7Ö…]L³ÕÔM’÷¬²O‘ë;%[0Ü*aŸß¯#Z\> ªË³ê€7ù¸ú‚xmðQ„Æ4©ÛM $¹ÛÍ 8HÇa½s6_<þzû3hLÚ¸}GïfF/”6IÃ’ƒ»^ùÒ¸¾QÀHÏfÒ·:.—‚’ºhuÖäh«NCjZ.±a{jg[íÖð;$b2gÀ®w½ýªÔ{¬ë¥vMçÈzŽé ñ¥Þ$ xìi6V©N×LwhB%òiA¬'½@pó¾y¿èy©—³9¡ ïzQÙeI‘­¢%ÒôäOâý`‚t/vTp¿F„;ªtËIJ[&êNгu'JŸ‡CÒoÖ¥»ö$›]s]<~7¬ý~4¦¨0oA% §Ž‰îñ]š.ù¸¾IŘPd¿‰²‡ÊsúG,ˆyq©ã‡ÁØ‘»ð]/Å;jÐ}ÁA“¦6 „5˜nzØ'9|ª-Nh+„•Mñö ˆö½" Cöù²OU»¾ŒÂç[(û° ð@ L ùc7J<«×iyý=y]R½«1P|Í–P¥¾h±~’–Ã0ŽìŒÞúçÕú:ØñÅg.À¾Ó7WÚó´Ä’dtÓè5p2EIJ?xN<&òÝ锉ž^£Ï%bòÚ#¯0äÒ|Ô™(}ßÁ$ªÛAW-éýë¥@íXjâ,à#vqÑ>ß›Š®§ÀpbØ<‚"–¿½!ò††)oi¨*.ª\zL_ªÚ¶$æ ÑGZlþÁ>O—Þ5 Æ/Æ–dîvº‹ÆÙC^™1+M%èTûhr€˜êK{^ã<3ƒ¦Ãw4×bmÄTa rÀ< EÆ8Á@¼eÄOÝ6)€2rrED“ÈD£ÝÜ©-`Cìµ$$Y®šµ‰ÕýJ‘ŠL¦JÛR@¶y­ã Ιµx(Ì)\œi¤#*"­M¬§¨ùÈÆûúf’“fY_H³%ø¾Š³Öi4„-“`CÃaM|õ"‹âÓžXGÓâ\—]qº$«—Ó§Y}rÒQ÷”Ã"d^y¬÷­øÞPrD”F‰Æ-Ž´£!5›œñaѬÍa‡iœÂçÄæd`hrœHM‡MÍjíì¿:q©»P Èlh”L‡FZ_Ô h |¶mGæÑÌÔÞt™4ÇÐøgÒÁºô× ç²Ê'ÒA|Ùm±”¥w_§¸ˆk¨œëçMBl"½”‡AYg`Õò¯É–>\(¬eÈ£üÍ M0Én™Â:¯‹mälJvîJù꼞f›cÎa;êlP>R¤‰Ÿµ¾.se“ˆ·±ýá‰T9ÂŽsó¸MšhŠãXRM½©L$#‡‡)%s\&WÒf ¹<ÇëˆgòlPj0q›"Ì µ6œÞFí\ŸEFE–G%‘øÔß’FîèÀéLѯQ˨鉌¦Ô8+ÐèXÓÁÖÄ>×j$·¤\ÊÍ)`DÒ™Ýr´èC5m¬…‹{$ÒJc)»<5s0„¼šÓÔ²º4IR‡¤áR@d¶éb¸"^ln^6ÁX£25ʼnºš€SêÏéP0°¹ Áïˆi“¹Š[_4zSbÊ躔H¡ð!«RxŒÊM]  ÄŒèzµJ*Îög-z#nÂMŒž¢.Ãg9)>º>Üì`]Ûäì‘W§1eê6¦@¹HQ âÛ,µÚ57:6põjò,è¤îBéGà×xm´)ˆn\wúf§0-Ñ—~ÅÉÏÚGÂlÃËÐÈâ4´âJ¥Ìèèèädî—¡³Q,UèÖ Ö­Ū3L(CK¨Éjpõ‰œ›b:‡€j[ƒ.Á`-¤T*•K\ƒše†p®ŽmàbæL¶ÒXš+ŒUFò»®.<Ýòûj¸"ìðÁƒÜÕ¬—¹ô—±NÀצ+ɢƉÊäKïA‚É—í–ÙûÞÙ…^åff¢41>2ŠhJxmó¡aÀˆ‘‘‘}“‡ö¨µJ¹|¥iÂ[9à0¡WÓºäiÖW¸»^½$É­SÏ;|¬.J¶Y n7ÇFòŒNLL ‹6ªvðvS¨tFÔ.X.^o6Ö•É­Œv«‹7!õÍdWX›a÷šu"×¹{ûŠÜÊR}©Ê]!m®ÛpVm]á±ÚŸ\—”5cK4=ÄÝ‹ÕêÒÒÒ£ùÉg¦§§Û¹åbe‘ûreä’> ·÷¡ýr08fÒ“¦‚$I²ªÓT·-µ ËÉX¯éÜ>~õÕG¸ˆXÔwõš¹V¶5V›ª¾è–Ñ} {tK87¶0æ)4hh¬8Cú·¨´æs¥ÉÊÁ‘ÎUv]=3ûšn‰ùG[¡ƒecVyhqÃ{¾ŠQBuÁ,Uf¼œ=<µ¯P*ÖÚÙ†]ëmYyé]äþûÌñÙ¢z·Ï$¢`\ªÄôPjjme²S/W¸{¡vªÁÕ†Xæǵ´§ÕW3»´dHl9G}+£ bíÀT€Ô¥£j ·#ùüÁ±‘ë&ǯ/Ìñ¡Öñu“m5 íÖD1{`ld¬Èµg‚RlÔ¨!e¸w’fvÓÔÑúJµÚ¨³*¦ÍÔ¬ %Œ¤aÔË·bR¯7º 3RØ°”ÅíÁ“•Ñ‰Ñ±"seÖ„,?‚ƒ„d‹ùìäÄHçêÃãc#U®AU¯KìR—17Ì—JhS·5Ö5ÔMd¦ÅÅ™b)_)•'''¦ÆmH¸†¢è3z†#ûÆök¾ad¼“ãh­ÆP£f/£f´uý—gŠô³îÒëôI®UA9UÊ“¥BÙt©M#¯áˆ5Q”-u $åÀ KæIÚª˜ÍìËeÊ£ùCåqºñ¾KFÚ˜œž>EöDÌÅÏ»óè\~Íö8»®*õì‘R§rv¥3³Ü®e—éÀ0R¡‘k³"ë}vç¡ÙJ¦K”34¤´òa󺣓A:w)£'ý´Ç󙣣•O®Û—9`Wß _Àa\X (¼\È”˜Ò·ûL5­ ¬bX:5±¯œ«´Ë->žÑ" ÈLº)UXÊoy¶œmbíF!á@ ¦¥¤[!5g©—ˆ½úl :/4²ÂxqtdêÐñ–väÙ±¬$ýùÄ,Qp\GÊZÓB ç†ÆmÝLÌ' H;S2e›#Ìi‘8Ã7/#è±B©R*L´F›¶s£åX4 ]·(uú§Ö›¡óÚSg²t¸™Y)•²=êChGjËÄìŒk·ö@?u²eôZ!3ÊX•dвMš#ÖEÔn%[kf&sãRË|ù|ýôJs®Õ¬ùh„dZŒE€]sb,ùµtgš½“BDD'kX‡¶æÇtZ§’ËŒå3û ú]±z¿¬×ÝgŸ„<¢¹€™‰HjI=VRùb‘‹õÉ…"y­D„…hö˜à÷Ü=ZœËU¬¸"$ò‘œDJ-æÅâ [öGJº[m’W©ë`sEÈÓ.£¬Iûk1„Ê°ÓÊé–p%€êš©§Ú)F—Â{òÔï¦@—Ìw¿^÷ÜÊ,›ºbx©™á ¤.D ,ˆþ^IHk°¹ÅY!Kævi]½Ãˆ@'at¨¯,dÆ'2•li¬05U^zäìÂÓíÎl«S/jtc\}³ëT©‘>ÄÐç@ôù$Qh6 ê›OÙéðmJ]GpbÅ€1Ëk,˜¹¸&ÐTX‹H²êˆÉ'!õ­->8áaWØcÜ·¬©·1LŒ¢yz踹Cž">iIo¬Á­ÏM;¯ëž¯OyYÇØõšAŒq]a PÔ€ùÁo§D1éíÍ òv4öö'…jµf‹,ãeMß-ßRä»òg5耼Ì"›^r¢o\J!â)BÛñ"ñ¡xK©>SâûÐý[á_Buz»–à˜è`ÙV¢i¥q†Y¹ÑÜx©Ý(å[Ϭԟo²™7‡•0:hˆÄÒËg­6g_7à¥T ­×hæ\[²ÑZ–ab]I»È ÔýuÖ犋ĚçzD*-‚:²3Ru¸-ýd‰¡è4Л êÇÍGQ2#Þ„ @Ö² ½„ËLÓÆí‹Ì6”úQ€‡ N¾5.h¹W÷ êš±p²Ã¥X³H)M†H¶É³Æ©Ó¿ë(€‚ˆ'vÕõaÔÔƒÎÝ>}n©¦UÅâ ÙSJ §ÁIäœgJ¤2u±Ñ¦£¢v±˜wƒttZ€Ô*5ÑrC©~Œw%4ZJ1Ø#†¥Lö@.SÉ”ögGóûö–FÎ-f–§™ó >OàŒ<ÍcèƒF,ŒŸ–™µ)s#kåD3~y1ŸŽöÊ5øår „4ˆò`°n©{Îj °$ðP©Y$=­± ¥qV/·ç¶p{ÅíµTIqîA*{ ŒÕ—F-Æã¢hòi2¹ž2šÚ1EI[!KýÞH«õ%K.Å©ªÈO2Ìbc6ÂÏ­‹þR•šK§Iÿ®£€è­º}6FA;ŸýMúX5“Àqn]™‰ˆŸªGœ‹ÓYÞhÚÓÂTÎ#¯á_˜ÿW86Sý 1Æ~\‹Ô+;%6òb0z Ñç³×ŒgF*…òÈ8§F¶ÛõJ-s:ÓœkòÐli·"ó†œûe;Ê,¿Æ,1¥˜QǨÊ!‘î³jÒúf@˜ªdŒu ‚IÖÆœch˜®‚¨>3ÀHiñG°“Ì{€^ïÇå¥{S€ !NjÓa ú£RË,¬1Å¥4qž(`-x+DY݉¤HÿmFkÔÐËÓ¬N!¦‰Q=FÆÈܬ‰õÄþ†vqΙ@&s«ìŒ"Ò?ÝHêÛ8lÜåAͬñ­¹¬éQ¹ÒY-Ôé5…‚Ýœ …n«ï<äÊ5ntŠÓì‘Ü ühæÚê'ù0%,1òJÃ;kç26k¡u×ÌT6sãXfôú©ÃS£_}ö\‰¥öt½™/–Uv=ò¥ ùÙ0†yÖ*‡¾šobxøv^†4WÈbXšì®ËZù¢ŒÚ 0ut8*F˜mä@_:LH:dY\¿Yß™¼š€ŽÃ–ÖsD>ôOPÙÄÙl6ùB0Tî„]‡†HÛæÎ(‚¯ŽxìJ'à %*Ð 1ë ¯ºP^°…‰wE$?ÉŽdØ3[È¢#auù e[Šø)Œ~8³gÀškü–¿-3Ñö’µ¹Jgùì¬ÅÑèõF½T,‘2™7N6¼ ¶ÇK,œlª×¡MÖh6Šm’ ‘nhÔP€°Ñ!ftæç±ñ;;a$jÎ6OMEV”5p•lL2}hõôòÖB♺Dš|⺸’ 3x8ç…ƒÊc°€£<=ÆiÖÞ–KÃÄeM(² «¥9W n÷mg¬ÒGók Ê…u ±738?’0¯>ò褿úšõ{=Äó#¹ÌáJ¦‚m—²WNŒ”GÏ­”³µÙFc™Ù(Ž‘°¯ l*Ó‚Å|fÎÐIÚ¹Å*Œ—‡ÍÉð•Ðà0õ’ÒkBRz+ÔÀ][‰óÓÆ¢\žxSßu19h£X´ÖÛé‡ØIH-Þö†ÊÆxzô8I¶TÓq1ù œ]#X°ì`5y$11n#ÉÅ+“˜vèREdD‹$ý»’F)¢¼ŒóºÈxj:ܱ&Œ²³Q–¯3u<„˜©ÿª‹¿m€·S´-ŽE?E¶€ªÓ )À&/ÖÅÓ'ófr8Œb½„aúRO¢¬`rvu ]#µŸ2U¯ÕJå21ê¤iË«Þ‚9XsÉ…‘·Ó‰7¿-ƒ¿RTDgmüÔìM̲šäªÆ\PS/Gó'Ú•¬k^7n2‰ ™<ÆߺÏ+wwm ]$„.¾Ð Â–GÀ£-Сѣ–ŸÕSñVÌ#Ž¼Üêâ8FÒe2¹eسlê¯a¬viy#Wi»ZŽé"ñÕçà<å üµ ¦Øõ€M#ƒ¼šXä*XBæ¦qNûÛ?RžÈeFs'çVN5ZÍ|¹šåËLŠQÒXʧ•[SAüÙcÖ¦ãÈô˜¾•cAfðøºåÝ®Qý¼UÜ°°_Q’-™\HX¡Xlšœ©aÓ;¶&Yóí/aÕ©Ï 2¤Ç‡¤Dæ ‘]Û²º¾'æ³Ái9À¯Fb@‚5ƒ-Ñ=Höè„sDåzÜ•^Þ*9ÉÐÓÅ)ü½ÙH)»ž9’2Ì ~¨WÀå9>XAiüMFÇ\Ë`‰³‡¦Ö¾)sêH ÓýY!¤hyá5F -?Y¢ø­§ŽÏdú«Ùhä |u*ÍÎ}øê ]­V¹T–F >gц xHß…x)8‡ÜqŽE;¦mD e±W®y4½Û$Ç".ùñÞ ¥ü 8%''>Ä ‰ÑÄhV`Õ×)ä8C¤Áuøôo$pôU%o›JÂj!ý±KžBùTHíBðyL’N‡·€*È Hoô?)Ùn …z½Ê§u–c»ßÒÃIÙ‚^¸¯rÍKæ›q9g¡BH¯MQp QÝ·H,#Ö‘dx½æÇ4‹×x½â )Ú™¾>U?ˆ‹ÙýùÌèH¦œ-”:c#Å\I›kóÍÖt†[½Èa䛕a’h`"–ãv"Ê$n.¨ƒÁÔADÛ$µÁ `8†`”*PÀ_lê{KQãgô“Í}õþû?ðœ;wŽÃfhÿ8¤ …"m qöïßã7þê¯þêáÇQä’¢[Ï•˜üƒþKsNÑZþöo?õWõW§NªÕjDâ 7E4{N¦—«¯¾zjjê]ïz× 7Ü‘n=¸Aº^Y†ð24í,”s¿+‹¥HpèQÂhX 6Ç3c )øŸþéŸ>ö±ÍÎÎrÖŒ€Î@^¯×à“““•Jå%/yɯüʯLNMYÿ>Ñ$™Æ¯kè>Öåý÷Ý÷÷ÿ÷+++ˆÊG?úÑ}““h/WÍŽµ÷ ÿò“ÿçþàÀŒî»ï¾»î¾›Áœ ¦Z*Ä0J&(¦ü ߆ )C€¤^ ò$Š Lfüéïþîï>ñ‰OÀ‹åååÐ"à¥Ìèn;öÔ%êСCW^y%uÝu×ñ–4>VN<¼ ìžë£ƒE» $Ê›ƒË•âwàÀk®¹æ}ï{áR©"X“”ßì²ÞÜOæéAê¸kš¬&ÉÉØ´„÷zé ㇊à«xXš;õÛˆfŽ4¥Ð&NAºÜ4åkÊ™ÊÁÒh%?Z*Œ—ž™9¿Òl¡™êH·fÉP´ g4&§r¾`]têŒSÌjÖñ–ƒ¸*ÁDŒ÷ù,"4'ÐN|º™h7áÖlýïÿù¿>ññÓþ ÜÛøm‡RÏ”Ñ7÷kF#ƒ R2ZªJ·ò¨ïªyï»ßóøãÓ~è½»æ!@ç¥ìóÖè4èñ9@»ûλn¸îzD¤7˜D; dôG•Ô}=K\M¿F>Õçåƒ@ºüXšü/þâ/?qì ^kÀD‡É–c„O<?ñ<~ß÷ý×;î˜ô>ªJäý0ëíÝ?ø¡ãß:îÄÿÚ×¾ñš×¼:ŒÀ€‚£ ò…þŸÿùÇ>û¹Ï:ßñŸyÅ+îŠpa&‡yõJ ¯§aŒ—¦±{Ø 5Û8 ‘Û¸Ü ŠÒ€ÊÜßÿxðÁ¡³ó"âLáÑ9â¼ð··ßöòù‘ ²Aù¡œAdQ‚6~ÍdßõÎÿöôÓO* Ñ‚üóÉ!¡Aýð[ßvÇ+^®]ÒŽeÉ鶖Ú#™¥?¯·¡šî-1#¼TÜ`œM°®)Ú×B”ƒ < a(ˇ (NbÔoèœ5‘;ÓÌ<½”yä\퉓Ïq±<·´4ߨղÅ:W2²·SÙõá £È|f¤±|ëþ‘×ÞpÅ·(qZÌUÑ 2Ôßç@kMÃPá­#¼â'cà ¯{iqñá‡>yòd²¿Ý ‚†Òù<ÝLú8wÞu—×å3æôL“ºã…ƒ±ý±VÓA¹ZµúÐCÑß\XX SlÑ:žÞåôî$½3ÊçlÝ}ûö1 »óÎ;73A’b˜¥“m(~˹Bºð®g€n Ç—ÐÙ‘¢2£˜Í¬,U|øÙ™¹ÙùsõjƒiªËœhÖ,æKtnð ¥<«‡£ã#/}ñ­×\wTVÖk4_åP¢ ‰øa†Q[Çò+÷ø÷œzËyK€Gš‹^ùH÷ÉÓ'ÿï¿ý¿ù…¹ý“î}ãÆFÇ'ÄÝ~ȘÄ+P5`b¢€3Ê}ÏÉ o8æ'K‹Ãè5º\úðò‘Ç9;½°´X¯Öªu+çzÚ%§A¿­rÈŸœØWª”¯¹úèíw¼l „q™Ã¤„'b…¾ËeêÕú×¾ñõ¥…Åss³øê¡p¨‹N1â­Â#cô±‹ãû&n¾ñ¦o¾Itáÿ&\Ô^ݲ;£¨Äg]$· þF­@/¢—ŠŒf…©N4Å)4TçÕReTsx6(¬ÉlêÔJ izÉ’jŠK1ùåLf®•9³’yúÌÌ&Ÿ=;óìÒâ\;3ÛÌ®°ÉPGý"P—m,¿lrä;n<üm*/ÚŸ»’Wêè€ .Ö­GÙÓl"J lé[ýšÄ žød9¼õÇž*Û“)‡Æ~ø01$XÁ€¯xtD5Ä{ <¾P€try#""V¢`O"c<ð¹n¼ceKóÌæÀÖr±â¾äÏzýœ¢Gï˜4Äøˆ‡ž> —0S»Z&”¬‘Rß*`/à±)ƒÚ‰c lfX')[6c¸7V‚QÕ¦ÈÄ ;h)ÂèmȤ|4â'ÊÓˆŸçêé{·É}jëÃx‡ƒçôCJWÖ,顬‰ <åýy€„-!>«¥G4¡°R· 0J†ŽÛ‡Ü©Nú.GQk Œük»•’ê}” 9ubÑz¥w^êj”Å÷Ý[{¬÷¬¦uo£úÏ(VrÁ'ûPˆÆGO‹ºÂT>SÏ”;SÓ“œ\t¨>“éÔZK5¦ËHI‰¾–ôYëc”IYî÷»õ&1ZG[^J¸ÝYE²3g›)Éúèã}éK_‚@têßô¦7¹Rv‚y4$Ï>ûì—¿üå¹¹9¦kï½÷^–èÑòùôUݧԚY²í €ñÄOÞw÷wcx`0¦…ù1†•XT—ºEÚÅ ëh4ØÀmAN£yWVëõ†„Öš"_IèúbK ÑªCC´-ý ‡yðþûïÇX~üÚªÚ&™oõaKÒ+_ùÊŸþéŸöÍc¼R’–À={–žõ­·Þ /`ŠWá¯ÜïYé"•¡H=zôî»ï —( JI2˜Â€ŒÕ`Ù;Þñ†;$pš ÎMª@œÐ³ ­h×^{-ku‹Mr]$¯\B(€/hHÈ=÷Üó“?ù“ÜþIdg >éáÅ‘#G^úÒ—þÔOý”ób IúZÁnîxq`,ñÉa&³ÔÈœZÌ<:]{`z᡹•§–›Ü‰©»¿¸K§ºtûTéµ7úÏGJ7O®YýL(”q)\]yÑÔ!m~“æÍ+\ÀÄ(4`ˆ~ÉÛðð˜DW j$>lÛ¬Ña&qœáѱpfNÅóºïɼRÆ1ÐÁ{ßÛ£ïÉ€5Íýž¡d$&G‡Ÿ˜$Ê C‚w˜RãƒÇ `Ãã…Ô%zŽEè*0°ƒxr9Ùɘҕ«—¸úܘëiíiQBüøÔ¿2ÎT¹È&2vOÍÕ²KîÄÔ‘Ë:Ü–`6.ýÒyÜyB¶pÄl„'æ­ÛÄë²y–ŠêKüú6t–«*‡ í°‹@n‰)°yà¤Í»^ ¾tô²Ç“2àèHáû+Oæ‘!ï–µ"¨Üºø#Ì‹.˜ÝðÔ<¥û›Èá `î*Õ#ÒAêJsÑ>zsvðÜŠãr"Ö î¾£Ff9;Ș´=ƒCswºƒÃgË’9^¬ÒÉð‰Ëþ7Žgî¾¢øª«&ï92~ÛdåÊbf´Ó(d9IYbÇêú ¢æ­ˆé×±±1Ž€cÇŽmD(ÄÔ_y?twU-àzùð™Šaë£û'NAÊdx#ì†ïP¹]€°?ú£?ÊŠ7ì ¶-x$Z€€ûpÄý×Ö¤g¢òoþæo<¾;O? û¯ >Sù¬Ó aŒÏý€”q üÖ·¾N?ߟzê)-ì"o/¹T@HÔ.ð3{üä“O›ŽZŸ[£´¾Õÿ¡ú!Ä Dø”}:¼E<\Šðݨðö7~ã7˜=ƒøLϲàä¹P¶•o¨ÐmáÅÓ(Í|§>QÈ\=š¹í@ö?ÃÆÜ>Y¾f¤3™oY4n4ØÄÞ²KÝ…ÆüÌ3ÏÐ…o}ë[çÏŸß/Wè!²ˆî¢:ÐH€=ÅŒ9 ±PÄ&+ÞÒ@gwîFô„t à½E'ãS&͉d‰… {D‚’Dk8E2^ÍÌÌ€/GÓÙAš*BfÿDø¾ àyåbü¬BóÜsÏ=ÿüóÄ“Òa#Ìypn^Ô¦³ð Ba‘róôÛ[§3Àc¡-qRƒ¨%åÄ…‡æƒD!‡HøÂ…Ac´g×`zºûöÎŒîƒa+(ôÍ×[™ÙzæøRæ¡éŇf—ž›™ã¾ä»yåÁ±›”®ÒG{—¶kÄ íì4ºÃ¬ò¹ð­GŒ”AìP ˆéú4ÃŒqx$(, >Kßô¼$ÌÄm§u,XX 8¹#¾tŽpóãi`|Ô7‰Ù¦ÌX‡žŒxlÚ]Ä)ÂN?~íÆ'ºtŸŠx€Z ~€&ürripÔbã) 6ÌØ" ® -…›£l'_ûÚ×þñÿ‘)>à'Þá§]`–gè쳩—ÏAÜ3:ú›W1è·ƒÃIxàfÀ™¡ã‰d(…y Hàóé,ÀÐÍä{RPóWÁ4À[–œÒOÎW¿úÕ¿ø‹¿`°ÂJÌ/üÂ/°ß×õ¯þú¯ÿúóŸÿ<´ ½‡9 âí%È/|á žk,سSdNß¾¯Œj¦Œc¢p j8«‡Sh ¶]leêvÈ+÷ÊsÚov©;ôš‹rüøqŽXg‰A â…CF ê¯áè5ßpà ?þã?r]<è?ýešÇ’¿4„íËr»ð ÉJÀ·û·¿úÕ¯¦™škŠÝE!IIãßøâ¿Èº7Mš € 3ÊÇ8†Ñg+ì.ü=kGr¼wÌi1ùÈG`оóïdøAŒpf)±4ï~÷»=½—+w5.œ°‚PqcbÀ<¼, ñÑ1#oD$x¹]0`ƒE» ]3‚~€G¨h ‘}àñ 0cˆÉÆ9²$ŒǹËÏÀH™&·‹ñWß¼pØ¥ŽÒÉý,Æ°“”߉´BÃg1—ö,"m)Ä…VÖÞ0’1A^“‘žADR)Çá»Pd¡ƒF_˜†ä˜ãè|O¿‹*Àiàwhyt_<™·öÐæ“h†,žòâñF@èFßåıÄ÷>À»]ñ˜$šCÆË!t²;ä<â„à‚#?€gIvß:¨œ’áÑá Ih“ôwIó¼µA.kÃ2 —tHæQOÑÈ&º“µ Œ6šeW§2Á€A—领OEÞN‚T%#h F9ôƒXŒ¹þúëIC{ Š/$ZÀ¡EÑ~ºùœNHíôÅØ l!1$$ó\IL“:bhÀ‡ŠªÐªécêöŽN‡#:ÈÞ5ö1“{þ}½~F“ ˆ å\<À—`ïÙ¦ìc2DjÓefÆŸ 0‡4 LÆ0 ڼܼ!êœ1wصY․p»êF•¨¼]ðH»` 5´par-‹”Àô2KJøI47*¿ñTsy9¦ôµä< 0¬Äè‚K-SðŸU…ì=§¦’¦uéŸv :v„">¨Ñøñ‘?áÙ˽ãï@ª|®æÿðщ›$îU@Ÿãl/H8Ï*´¦€ôþ˜„`wwH<ìŽÎ]wÝåfŸùTt´«pD;³ €,èŽ$FUØQÃg=±A£‚AÊ1þ‹_üb˜âV’þâá ÔhàEà”SÈðÉGW8ât¨ ð+^ñ ·âHÇ;HÄc8™ºt‘(628Ìž± ©A r1vŽ¿ÐÆ0z!ºÿ…K ’è72íšfÊ8¾Ÿq‹Òq±ë¥=?DoóHÞ :šH„,)g¬ÒqF4Ùùãóé$ö¼ƒ`Ç–eºª RÉm·Ý†ýã«ZŽ¿E£y°sŒ¼X"Sܲ¢% =PùlžÃ ‘yÄ1c¤èU“Œ«ÒXsöôáÜh’í"ü›¨TÒÜxã@κYÈ.œV€âsŽà;³œ $ f“‡ð à1Þ/{ÙËx$‡Å NBåíÂ!qàyë xÄQÂàìY… v{ÖꀇHÄYxR@k E Ñ`1°€Ä¤q¿gùýŠ¼ì¦ÈXx‘‰‰e›Eb˜&ËfjºüK”±+¢¡0³h× øŒ.s´+Dv˜|KYD:7JÜ•w@^;x£òÇÐÎ{†¤EÉp»¨€!8ÎQp€=Ò)Æ«$žÌ™°m_x±· ~G²;^‹§$,–X.2ú«áû°¨H‚À bü1É—. AÁ3vÅíÑa öG'¬û æx9²ÉW›àÕ_ø/;Ó_ò]ü¥QsíœÔ¼]Í›”8ÚURRAâ»q¯Ý!O6Œ•G†WÞÔêJ‚œNÌ@Ò2>‰Åv%öŠBšê\|’¼Iì¼F´žË›ƒ½‹pnI@ü``ÈFž±'Ù_2ò6$ز¢¾'H’ݘd$5ʃJd‡·}‡-˜˜@Š½p·ßú­ßâÀD†#Ìüñÿ±£‰¢éøò“Ÿüýßÿ}¦ø,ëøCo4E²í ŸL@ˆs¸±êSŸúëL‘ýÑýX$ ɼÁ[ŽÈXzÌn)‚$xÞ°ñ!,[xÙHÊ„Æ?øAf#äN^NäKLö˜‚àŸþ韲øìyCw•qš Ÿ#^c 2lÿ¹Ÿû9ŽVCrè f,òã¾ó;¿ó=ïyÏnA¸y½ÀOÚŧ?ýiÂlpçdf,׋ŠsÇ}g¨‡7/Ðo0hž.Q´ bXäÿð‡?Ì1.0Éî»ï>¾O@l¹?ù“?A Ix¨;u{›.ˆÌé‰ÿÜç>G¤Ç'q'†cYi`.|¥¿]Ÿ2™kaµÐf˜Gö®1¸pù«Ç;¼N´[€‡»?P9 X¿#Â]¼õ-<)Yp5‡¦þÌg>Ó¿^IôÖà ¨Žð‡>ô¡dGÄç'M罪ÕU! ëaÞ.ùpÓ+ñ G ß¹C$µâwTi¿wÑß?`s¨4,é,}bx’õ ªõå¤#—®=ëÃrTâøðúA©¢¿ÌYAM;æ$#@J¶Ï~éK_bó #\AS°0ˆ "¸»B# TÀÆ<°`“ €w;`ø8Ô¾ÃéXÔ\Yï ü^©cá>°=øàƒ_ùÊW0@È6ÉŽ;æ{èVï÷~o@:€æ."’¬Ú©í>¢¶¸-Ñ÷¼a/ù¬ÅóÛo¿dþdö‹$̱=ðª2vA¢6•.ëÝe ƒÿè£"0¨7½éMØBZ®ét~ä‘Gø¶—Ä4DÎE1™`@ìH Ì€{ëmÃ…‰0.ha ìjQ!ñ¤ þ®ã0Z,èkG!´ö d=ån¡àðx›ï‚!´sà f†4<†ÆßÅ Þ¢¾™}"ÍúW]åô1 €ãHŒ×`;!f Àì´ð $žÑ!lÈîT ´ ¯B¤!¼[].#ˆG€*°O§ød Ô,ÈápEj`ƒölYD˜¼Güÿý÷ßÏM*()D çRˆð‘µÅœó3–åäð·]­qÈ”x³7xæúéròíðó¹"è {—Ÿ”<ú×$ôæ诅9!ƒª ºb‚ Mû`è5æ›8Î qÔ„ÕèóiÈüÇð¹%+^¨fD€yå`#NîÀˆ1=rÅ×Ç>u‘à…@[(´kÀö‘=(÷ŽôgãõÚ×¾–/cÀn#úô'2¥îr €Ï]pÞ-r“lí(,TrF$aaŽ&ô6æ¹v‘>´jÇ÷–ÿªW½ "/;Ë#çX``üÙqù½ßû½‹~ŒŸ“\°yÐ øþáB|œo~ó›ÁLAÄÏ‘tRà;_œ8]¹†öè/jtŸå"Z|fÆÔ»¡AµÍŠ*Œ '×9‘áÎ$Kk]Î…Š ~÷ww›µ 4Yè ÂïEAyÀ£ñR¯#è Bind to host address (default: 0.0.0.0) -p, --port Use port for clients (default: 4222) -n, --name --server_name Server name (default: auto) -P, --pid File to store PID -m, --http_port Use port for http monitoring -ms,--https_port Use port for https monitoring -c, --config Configuration file -t Test configuration and exit -sl,--signal [=] Send signal to nats-server process (ldm, stop, quit, term, reopen, reload) can be either a PID (e.g. 1) or the path to a PID file (e.g. /var/run/nats-server.pid) --client_advertise Client URL to advertise to other servers --ports_file_dir Creates a ports file in the specified directory (_.ports). Logging Options: -l, --log File to redirect log output -T, --logtime Timestamp log entries (default: true) -s, --syslog Log to syslog or windows event log -r, --remote_syslog Syslog server addr (udp://localhost:514) -D, --debug Enable debugging output -V, --trace Trace the raw protocol -VV Verbose trace (traces system account as well) -DV Debug and trace -DVV Debug and verbose trace (traces system account as well) --log_size_limit Logfile size limit (default: auto) --max_traced_msg_len Maximum printable length for traced messages (default: unlimited) JetStream Options: -js, --jetstream Enable JetStream functionality -sd, --store_dir Set the storage directory Authorization Options: --user User required for connections --pass Password required for connections --auth Authorization token required for connections TLS Options: --tls Enable TLS, do not verify clients (default: false) --tlscert Server certificate file --tlskey Private key for server certificate --tlsverify Enable TLS, verify client certificates --tlscacert Client certificate CA for verification Cluster Options: --routes Routes to solicit and connect --cluster Cluster URL for solicited routes --cluster_name Cluster Name, if not set one will be dynamically generated --no_advertise Do not advertise known cluster information to clients --cluster_advertise Cluster URL to advertise to other servers --connect_retries For implicit routes, number of connect retries --cluster_listen Cluster url from which members can solicit routes Profiling Options: --profile Profiling HTTP port Common Options: -h, --help Show this message -v, --version Show version --help_tls TLS help ` // usage will print out the flag options for the server. func usage() { fmt.Printf("%s\n", usageStr) os.Exit(0) } func main() { exe := "nats-server" // Create a FlagSet and sets the usage fs := flag.NewFlagSet(exe, flag.ExitOnError) fs.Usage = usage // Configure the options from the flags/config file opts, err := server.ConfigureOptions(fs, os.Args[1:], server.PrintServerAndExit, fs.Usage, server.PrintTLSHelpAndDie) if err != nil { server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err)) } else if opts.CheckConfig { fmt.Fprintf(os.Stderr, "%s: configuration file %s is valid\n", exe, opts.ConfigFile) os.Exit(0) } // Create the server with appropriate options. s, err := server.NewServer(opts) if err != nil { server.PrintAndDie(fmt.Sprintf("%s: %s", exe, err)) } // Configure the logger based on the flags s.ConfigureLogger() // Start things up. Block here until done. if err := server.Run(s); err != nil { server.PrintAndDie(err.Error()) } // Adjust MAXPROCS if running under linux/cgroups quotas. undo, err := maxprocs.Set(maxprocs.Logger(s.Debugf)) if err != nil { s.Warnf("Failed to set GOMAXPROCS: %v", err) } else { defer undo() } s.WaitForShutdown() } nats-server-2.10.27/scripts/000077500000000000000000000000001477524627100156425ustar00rootroot00000000000000nats-server-2.10.27/scripts/cov.sh000077500000000000000000000046451477524627100170010ustar00rootroot00000000000000#!/bin/bash # Run from directory above via ./scripts/cov.sh check_file () { # If the content of the file is simply "mode: atomic", then it means that the # code coverage did not complete due to a panic in one of the tests. if [[ $(cat ./cov/$2) == "mode: atomic" ]]; then echo "#############################################" echo "## Code coverage for $1 package failed ##" echo "#############################################" exit 1 fi } # Do not globally set the -e flag because we don't a flapper to prevent the push to coverall. export GO111MODULE="on" go install github.com/wadey/gocovmerge@latest # Fail fast by checking if we can run gocovmerge gocovmerge if [[ $? != 0 ]]; then echo "Unable to run gocovmerge" exit 1 fi rm -rf ./cov mkdir cov # # Since it is difficult to get a full run without a flapper, do not use `-failfast`. # It is better to have one flapper or two and still get the report than have # to re-run the whole code coverage. One or two failed tests should not affect # so much the code coverage. # # However, we need to take into account that if there is a panic in one test, all # other tests in that package will not run, which then would cause the code coverage # to drastically be lowered. In that case, we don't want the code coverage to be # uploaded. # go test -v -covermode=atomic -coverprofile=./cov/conf.out ./conf -timeout=1h -tags=skip_no_race_tests check_file "conf" "conf.out" go test -v -covermode=atomic -coverprofile=./cov/internal.out ./internal/ldap -timeout=1h -tags=skip_no_race_tests check_file "internal" "internal.out" go test -v -covermode=atomic -coverprofile=./cov/log.out ./logger -timeout=1h -tags=skip_no_race_tests check_file "logger" "log.out" go test -v -covermode=atomic -coverprofile=./cov/server_avl.out ./server/avl -timeout=1h -tags=skip_no_race_tests check_file "server_avl" "server_avl.out" go test -v -covermode=atomic -coverprofile=./cov/server.out ./server -timeout=1h -tags=skip_no_race_tests check_file "server" "server.out" go test -v -covermode=atomic -coverprofile=./cov/test.out -coverpkg=./server ./test -timeout=1h -tags=skip_no_race_tests check_file "test" "test.out" # At this point, if that fails, we want the caller to know about the failure. set -e gocovmerge ./cov/*.out > acc.out rm -rf ./cov # If no argument passed, launch a browser to see the results. if [[ $1 == "" ]]; then go tool cover -html=acc.out fi set +e nats-server-2.10.27/scripts/runTestsOnTravis.sh000077500000000000000000000112271477524627100215210ustar00rootroot00000000000000#!/bin/sh set -ex if [ "$1" = "compile" ]; then # First check that NATS builds. go build -v; # Now run the linters. go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.4; golangci-lint run; if [ "$TRAVIS_TAG" != "" ]; then go test -v -run=TestVersionMatchesTag ./server -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -count=1 -vet=off fi elif [ "$1" = "build_only" ]; then go build -v; elif [ "$1" = "no_race_tests" ]; then # Run tests without the `-race` flag. By convention, those tests start # with `TestNoRace`. go test -v -p=1 -run=TestNoRace ./... -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "store_tests" ]; then # Run store tests. By convention, all file store tests start with `TestFileStore`, # and memory store tests start with `TestMemStore`. go test $RACE -v -p=1 -run=Test\(Store\|FileStore\|MemStore\) ./server -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_tests" ]; then # Run JetStream non-clustered tests. By convention, all JS tests start # with `TestJetStream`. We exclude the clustered and super-clustered # tests by using the appropriate tags. go test $RACE -v -p=1 -run=TestJetStream ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4,skip_js_super_cluster_tests -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_cluster_tests_1" ]; then # Run JetStream clustered tests. By convention, all JS cluster tests # start with `TestJetStreamCluster`. Will run the first batch of tests, # excluding others with use of proper tags. go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_cluster_tests_2" ]; then # Run JetStream clustered tests. By convention, all JS cluster tests # start with `TestJetStreamCluster`. Will run the second batch of tests, # excluding others with use of proper tags. go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_cluster_tests_3" ]; then # Run JetStream clustered tests. By convention, all JS cluster tests # start with `TestJetStreamCluster`. Will run the third batch of tests, # excluding others with use of proper tags. # go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_cluster_tests_4" ]; then # Run JetStream clustered tests. By convention, all JS cluster tests # start with `TestJetStreamCluster`. Will run the third batch of tests, # excluding others with use of proper tags. # go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3 -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "js_super_cluster_tests" ]; then # Run JetStream super clustered tests. By convention, all JS super cluster # tests start with `TestJetStreamSuperCluster`. go test $RACE -v -p=1 -run=TestJetStreamSuperCluster ./server -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "mqtt_tests" ]; then # Run MQTT tests. By convention, all MQTT tests start with `TestMQTT`. go test $RACE -v -p=1 -run=TestMQTT ./server -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "msgtrace_tests" ]; then # Run Message Tracing tests. By convention, all message tracing tests start with `TestMsgTrace`. go test $RACE -v -p=1 -run=TestMsgTrace ./server -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "srv_pkg_non_js_tests" ]; then # Run all non JetStream tests in the server package. We exclude the # store tests by using the `skip_store_tests` build tag, the JS tests # by using `skip_js_tests`, MQTT tests by using `skip_mqtt_tests` and # message tracing tests by using `skip_msgtrace_tests`. # Also including the ldflag with the version since this includes the `TestVersionMatchesTag`. go test $RACE -v -p=1 ./server/... -ldflags="-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG" -tags=skip_store_tests,skip_js_tests,skip_mqtt_tests,skip_msgtrace_tests,skip_no_race_tests -count=1 -vet=off -timeout=30m -failfast elif [ "$1" = "non_srv_pkg_tests" ]; then # Run all tests of all non server package. go test $RACE -v -p=1 $(go list ./... | grep -v "/server") -count=1 -vet=off -timeout=30m -failfast fi nats-server-2.10.27/scripts/updateCopyrights.sh000077500000000000000000000030641477524627100215420ustar00rootroot00000000000000#!/bin/bash # Ensure script is run at the root of a git repository git rev-parse --is-inside-work-tree &>/dev/null || { echo "Not inside a git repository"; exit 1; } # Find all .go files tracked by git git ls-files "*.go" | while read -r file; do # Skip files that don't have a copyright belonging to "The NATS Authors" current_copyright=$(grep -oE "^// Copyright [0-9]{4}(-[0-9]{4})? The NATS Authors" "$file" || echo "") [[ -z "$current_copyright" ]] && continue # Get the last commit year for the file last_year=$(git log --follow --format="%ad" --date=format:%Y -- "$file" | head -1) existing_years=$(echo "$current_copyright" | grep -oE "[0-9]{4}(-[0-9]{4})?") # Determine the new copyright range if [[ "$existing_years" =~ ^([0-9]{4})-([0-9]{4})$ ]]; then first_year=${BASH_REMATCH[1]} new_copyright="// Copyright $first_year-$last_year The NATS Authors" elif [[ "$existing_years" =~ ^([0-9]{4})$ ]]; then first_year=${BASH_REMATCH[1]} if [[ "$first_year" == "$last_year" ]]; then new_copyright="// Copyright $first_year The NATS Authors" else new_copyright="// Copyright $first_year-$last_year The NATS Authors" fi else continue # If the format is somehow incorrect, skip the file fi # Update the first line if sed --version &>/dev/null; then # Linux sed sed -i "1s|^// Copyright.*|$new_copyright|" "$file" else # BSD/macOS sed, needs -i '' sed -i '' "1s|^// Copyright.*|$new_copyright|" "$file" fi done nats-server-2.10.27/server/000077500000000000000000000000001477524627100154615ustar00rootroot00000000000000nats-server-2.10.27/server/README-MQTT.md000066400000000000000000000710711477524627100175310ustar00rootroot00000000000000**MQTT Implementation Overview** Revision 1.1 Authors: Ivan Kozlovic, Lev Brouk NATS Server currently supports most of MQTT 3.1.1. This document describes how it is implementated. It is strongly recommended to review the [MQTT v3.1.1 specifications](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) and get a detailed understanding before proceeding with this document. # Contents 1. [Concepts](#1-concepts) - [Server, client](#server-client) - [Connection, client ID, session](#connection-client-id-session) - [Packets, messages, and subscriptions](#packets-messages-and-subscriptions) - [Quality of Service (QoS), publish identifier (PI)](#quality-of-service-qos-publish-identifier-pi) - [Retained message](#retained-message) - [Will message](#will-message) 2. [Use of JetStream](#2-use-of-jetstream) - [JetStream API](#jetstream-api) - [Streams](#streams) - [Consumers and Internal NATS Subscriptions](#consumers-and-internal-nats-subscriptions) 3. [Lifecycles](#3-lifecycles) - [Connection, Session](#connection-session) - [Subscription](#subscription) - [Message](#message) - [Retained messages](#retained-messages) 4. [Implementation Notes](#4-implementation-notes) - [Hooking into NATS I/O](#hooking-into-nats-io) - [Session Management](#session-management) - [Processing QoS acks: PUBACK, PUBREC, PUBCOMP](#processing-qos-acks-puback-pubrec-pubcomp) - [Subject Wildcards](#subject-wildcards) 5. [Known issues](#5-known-issues) # 1. Concepts ## Server, client In the MQTT specification there are concepts of **Client** and **Server**, used somewhat interchangeably with those of **Sender** and **Receiver**. A **Server** acts as a **Receiver** when it gets `PUBLISH` messages from a **Sender** **Client**, and acts as a **Sender** when it delivers them to subscribed **Clients**. In the NATS server implementation there are also concepts (types) `server` and `client`. `client` is an internal representation of a (connected) client and runs its own read and write loops. Both of these have an `mqtt` field that if set makes them behave as MQTT-compliant. The code and comments may sometimes be confusing as they refer to `server` and `client` sometimes ambiguously between MQTT and NATS. ## Connection, client ID, session When an MQTT client connects to a server, it must send a `CONNECT` packet to create an **MQTT Connection**. The packet must include a **Client Identifier**. The server will then create or load a previously saved **Session** for the (hash of) the client ID. ## Packets, messages, and subscriptions The low level unit of transmission in MQTT is a **Packet**. Examples of packets are: `CONNECT`, `SUBSCRIBE`, `SUBACK`, `PUBLISH`, `PUBCOMP`, etc. An **MQTT Message** starts with a `PUBLISH` packet that a client sends to the server. It is then matched against the current **MQTT Subscriptions** and is delivered to them as appropriate. During the message delivery the server acts as an MQTT client, and the receiver acts as an MQTT server. Internally we use **NATS Messages** and **NATS Subscriptions** to facilitate message delivery. This may be somewhat confusing as the code refers to `msg` and `sub`. What may be even more confusing is that some MQTT packets (specifically, `PUBREL`) are represented as NATS messages, and that the original MQTT packet "metadata" may be encoded as NATS message headers. ## Quality of Service (QoS), publish identifier (PI) MQTT specifies 3 levels of quality of service (**QoS**): - `0` for at most once. A single delivery attempt. - `1` for at least once. Will try to redeliver until acknowledged by the receiver. - `2` for exactly once. See the [SPEC REF] for the acknowledgement flow. QoS 1 and 2 messages need to be identified with publish identifiers (**PI**s). A PI is a 16-bit integer that must uniquely identify a message for the duration of the required exchange of acknowledgment packets. Note that the QoS applies separately to the transmission of a message from a sender client to the server, and from the server to the receiving client. There is no protocol-level acknowledgements between the receiver and the original sender. The sender passes the ownership of messages to the server, and the server then delivers them at maximum possible QoS to the receivers (subscribers). The PIs for in-flight outgoing messages are issued and stored per session. ## Retained message A **Retained Message** is not part of any MQTT session and is not removed when the session that produced it goes away. Instead, the server needs to persist a _single_ retained message per topic. When a subscription is started, the server needs to send the “matching†retained messages, that is, messages that would have been delivered to the new subscription should that subscription had been running prior to the publication of this message. Retained messages are removed when the server receives a retained message with an empty body. Still, this retained message that serves as a “delete†of a retained message will be processed as a normal published message. Retained messages can have QoS. ## Will message The `CONNECT` packet can contain information about a **Will Message** that needs to be sent to any client subscribing on the Will topic/subject in the event that the client is disconnected implicitly, that is, not as a result as the client sending the `DISCONNECT` packet. Will messages can have the retain flag and QoS. # 2. Use of JetStream The MQTT implementation relies heavily on JetStream. We use it to: - Persist (and restore) the [Session](#connection-client-id-session) state. - Store and retrieve [Retained messages](#retained-message). - Persist incoming [QoS 1 and 2](#quality-of-service-qos-publish-identifier-pi) messages, and re-deliver if needed. - Store and de-duplicate incoming [QoS 2](#quality-of-service-qos-publish-identifier-pi) messages. - Persist and re-deliver outgoing [QoS 2](#quality-of-service-qos-publish-identifier-pi) `PUBREL` packets. Here is the overview of how we set up and use JetStream **streams**, **consumers**, and **internal NATS subscriptions**. ## JetStream API All interactions with JetStream are performed via `mqttJSA` that sends NATS requests to JetStream. Most are processed synchronously and await a response, some (e.g. `jsa.sendAck()`) are sent asynchronously. JetStream API is usually referred to as `jsa` in the code. No special locking is required to use `jsa`, however the asynchronous use of JetStream may create race conditions with delivery callbacks. ## Streams We create the following streams unless they already exist. Failing to ensure the streams would prevent the client from connecting. Each stream is created with a replica value that is determined by the size of the cluster but limited to 3. It can also be overwritten by the stream_replicas option in the MQTT configuration block. The streams are created the first time an Account Session Manager is initialized and are used by all sessions in it. Note that to avoid race conditions, some subscriptions are created first. The streams are never deleted. See `mqttCreateAccountSessionManager()` for details. 1. `$MQTT_sess` stores persisted **Session** records. It filters on `"$MQTT.sess.>` subject and has a “limits†policy with `MaxMsgsPer` setting of 1. 2. `$MQTT_msgs` is used for **QoS 1 and 2 message delivery**. It filters on `$MQTT.msgs.>` subject and has an “interest†policy. 3. `$MQTT_rmsgs` stores **Retained Messages**. They are all stored (and filtered) on a single subject `$MQTT.rmsg`. This stream has a limits policy. 4. `$MQTT_qos2in` stores and deduplicates **Incoming QoS 2 Messages**. It filters on `$MQTT.qos2.in.>` and has a "limits" policy with `MaxMsgsPer` of 1. 5. `$MQTT_out` stores **Outgoing QoS 2** `PUBREL` packets. It filters on `$MQTT.out.>` and has a "interest" retention policy. ## Consumers and Internal NATS Subscriptions ### Account Scope - A durable consumer for [Retained Messages](#retained-message) - `$MQTT_rmsgs_` - A subscription to handle all [jsa](#jetstream-api) replies for the account. - A subscription to replies to "session persist" requests, so that we can detect the use of a session with the same client ID anywhere in the cluster. - 2 subscriptions to support [retained messages](#retained-message): `$MQTT.sub.` for the messages themselves, and one to receive replies to "delete retained message" JS API (on the JS reply subject var). ### Session Scope When a new QoS 2 MQTT subscription is detected in a session, we ensure that there is a durable consumer for [QoS 2](#quality-of-service-qos-publish-identifier-pi) `PUBREL`s out for delivery - `$MQTT_PUBREL_` ### Subscription Scope For all MQTT subscriptions, regardless of their QoS, we create internal NATS subscriptions to - `subject` (directly encoded from `topic`). This subscription is used to deliver QoS 0 messages, and messages originating from NATS. - if needed, `subject fwc` complements `subject` for topics like `topic.#` to include `topic` itself, see [top-level wildcards](#subject-wildcards) For QoS 1 or 2 MQTT subscriptions we ensure: - A durable consumer for messages out for delivery - `_` - An internal subscription to `$MQTT.sub.` to deliver the messages to the receiving client. ### (Old) Notes As indicated before, for a QoS1 or QoS2 subscription, the server will create a JetStream consumer with the appropriate subject filter. If the subscription already existed, then only the NATS subscription is created for the JetStream consumer’s delivery subject. Note that JS consumers can be created with an “Replicas†override, which from recent discussion is problematic with “Interest†policy streams, which “$MQTT_msgs†is. We do handle situations where a subscription on the same subject filter is sent with a different QoS as per MQTT specifications. If the existing was on QoS 1 or 2, and the “new†is for QoS 0, then we delete the existing JS consumer. Subscriptions that are QoS 0 have a NATS subscription with the callback function being `mqttDeliverMsgCbQos0()`; while QoS 1 and 2 have a NATS subscription with callback `mqttDeliverMsgCbQos12()`. Both those functions have comments that describe the reason for their existence and what they are doing. For instance the `mqttDeliverMsgCbQos0()` callback will reject any producing client that is of type JETSTREAM, so that it handles only non JetStream (QoS 1 and 2) messages. Both these functions end-up calling mqttDeliver() which will first enqueue the possible retained messages buffer before delivering any new message. The message itself being delivered is serialized in MQTT format and enqueued to the client’s outbound buffer and call to addToPCD is made so that it is flushed out of the readloop. # 3. Lifecycles ## Connection, Session An MQTT connection is created when a listening MQTT server receives a `CONNECT` packet. See `mqttProcessConnect()`. A connection is associated with a session. Steps: 1. Ensure that we have an `AccountSessionManager` so we can have an `mqttSession`. Lazily initialize JetStream streams, and internal consumers and subscriptions. See `getOrCreateMQTTAccountSessionManager()`. 2. Find and disconnect any previous session/client for the same ID. See `mqttProcessConnect()`. 3. Ensure we have an `mqttSession` - create a new or load a previously persisted one. If the clean flag is set in `CONNECT`, clean the session. see `mqttSession.clear()` 4. Initialize session's subscriptions, if any. 5. Always send back a `CONNACK` packet. If there were errors in previous steps, include the error. An MQTT connection can be closed for a number of reasons, including receiving a `DISCONNECT` from the client, explicit internal errors processing MQTT packets, or the server receiving another `CONNECT` packet with the same client ID. See `mqttHandleClosedClient()` and `mqttHandleWill()`. Steps: 1. Send out the Will Message if applicable (if not caused by a `DISCONNECT` packet) 2. Delete the JetStream consumers for to QoS 1 and 2 packet delivery through JS API calls (if "clean" session flag is set) 3. Delete the session record from the “$MQTT_sess†stream, based on recorded stream sequence. (if "clean" session flag is set) 4. Close the client connection. On an explicit disconnect, that is, the client sends the DISCONNECT packet, the server will NOT send the Will, as per specifications. For sessions that had the “clean†flag, the JS consumers corresponding to QoS 1 subscriptions are deleted through JS API calls, the session record is then deleted (based on recorded stream sequence) from the “$MQTT_sess†stream. Finally, the client connection is closed Sessions are persisted on disconnect, and on subscriptions changes. ## Subscription Receiving an MQTT `SUBSCRIBE` packet creates new subscriptions, or updates existing subscriptions in a session. Each `SUBSCRIBE` packet may contain several specific subscriptions (`topic` + QoS in each). We always respond with a `SUBACK`, which may indicate which subscriptions errored out. For each subscription in the packet, we: 1. Ignore it if `topic` starts with `$MQTT.sub.`. 2. Set up QoS 0 message delivery - an internal NATS subscription on `topic`. 3. Replay any retained messages for `topic`, once as QoS 0. 4. If we already have a subscription on `topic`, update its QoS 5. If this is a QoS 2 subscription in the session, ensure we have the [PUBREL consumer](#session-scope) for the session. 6. If this is a QoS 1 or 2 subscription, ensure we have the [Message consumer](#subscription-scope) for this subscription (or delete one if it exists and this is now a QoS 0 sub). 7. Add an extra subscription for the [top-level wildcard](#subject-wildcards) case. 8. Update the session, persist it if changed. When a session is restored (no clean flag), we go through the same steps to re-subscribe to its stored subscription, except step #8 which would have been redundant. When we get an `UNSUBSCRIBE` packet, it can contain multiple subscriptions to unsubscribe. The parsing will generate a slice of mqttFilter objects that contain the “filter†(the topic with possibly wildcard of the subscription) and the QoS value. The server goes through the list and deletes the JS consumer (if QoS 1 or 2) and unsubscribes the NATS subscription for the delivery subject (if it was a QoS 1 or 2) or on the actual topic/subject. In case of the “#†wildcard, the server will handle the “level up†subscriptions that NATS had to create. Again, we update the session and persist it as needed in the `$MQTT_sess` stream. ## Message 1. Detect an incoming PUBLISH packet, parse and check the message QoS. Fill out the session's `mqttPublish` struct that contains information about the published message. (see `mqttParse()`, `mqttParsePub()`) 2. Process the message according to its QoS (see `mqttProcessPub()`) - QoS 0: - Initiate message delivery - QoS 1: - Initiate message delivery - Send back a `PUBACK` - QoS 2: - Store the message in `$MQTT_qos2in` stream, using a PI-specific subject. Since `MaxMsgsPer` is set to 1, we will ignore duplicates on the PI. - Send back a `PUBREC` - "Wait" for a `PUBREL`, then initiate message delivery - Remove the previously stored QoS2 message - Send back a `PUBCOMP` 3. Initiate message delivery (see `mqttInitiateMsgDelivery()`) - Convert the MQTT `topic` into a NATS `subject` using `mqttTopicToNATSPubSubject()` function. If there is a known subject mapping, then we select the new subject using `selectMappedSubject()` function and then convert back this subject into an MQTT topic using `natsSubjectToMQTTTopic()` function. - Re-serialize the `PUBLISH` packet received as a NATS message. Use NATS headers for the metadata, and the deliverable MQTT `PUBLISH` packet as the contents. - Publish the messages as `subject` (and `subject fwc` if applicable, see [subject wildcards](#subject-wildcards)). Use the "standard" NATS `c.processInboundClientMsg()` to do that. `processInboundClientMsg()` will distribute the message to any NATS subscriptions (including routes, gateways, leafnodes) and the relevant MQTT subscriptions. - Check for retained messages, process as needed. See `c.processInboundClientMsg()` calling `c.mqttHandlePubRetain()` For MQTT clients. - If the message QoS is 1 or 2, store it in `$MQTT_msgs` stream as `$MQTT.msgs.` for "at least once" delivery with retries. 4. Let NATS and JetStream deliver to the internal subscriptions, and to the receiving clients. See `mqttDeliverMsgCb...()` - The NATS message posted to `subject` (and `subject fwc`) will be delivered to each relevant internal subscription by calling `mqttDeliverMsgCbQoS0()`. The function has access to both the publishing and the receiving clients. - Ignore all irrelevant invocations. Specifically, do nothing if the message needs to be delivered with a higher QoS - that will be handled by the other, `...QoS12` callback. Note that if the original message was publuished with a QoS 1 or 2, but the subscription has its maximum QoS set to 0, the message will be delivered by this callback. - Ignore "reserved" subscriptions, as per MQTT spec. - Decode delivery `topic` from the NATS `subject`. - Write (enqueue) outgoing `PUBLISH` packet. - **DONE for QoS 0** - The NATS message posted to JetStream as `$MQTT.msgs.subject` will be consumed by subscription-specific consumers. Note that MQTT subscriptions with max QoS 0 do not have JetStream consumers. They are handled by the QoS0 callback. The consumers will deliver it to the `$MQTT.sub.` subject for their respective NATS subscriptions by calling `mqttDeliverMsgCbQoS12()`. This callback too has access to both the publishing and the receiving clients. - Ignore "reserved" subscriptions, as per MQTT spec. - See if this is a re-delivery from JetStream by checking `sess.cpending` for the JS reply subject. If so, use the existing PI and treat this as a duplicate redelivery. - Otherwise, assign the message a new PI (see `trackPublish()` and `bumpPI()`) and store it in `sess.cpending` and `sess.pendingPublish`, along with the JS reply subject that can be used to remove this pending message from the consumer once it's delivered to the receipient. - Decode delivery `topic` from the NATS `subject`. - Write (enqueue) outgoing `PUBLISH` packet. 5. QoS 1: "Wait" for a `PUBACK`. See `mqttProcessPubAck()`. - When received, remove the PI from the tracking maps, send an ACK to consumer to remove the message. - **DONE for QoS 1** 6. QoS 2: "Wait" for a `PUBREC`. When received, we need to do all the same things as in the QoS 1 `PUBACK` case, but we need to send out a `PUBREL`, and continue using the same PI until the delivery flow is complete and we get back a `PUBCOMP`. For that, we add the PI to `sess.pendingPubRel`, and to `sess.cpending` with the PubRel consumer durable name. We also compose and store a headers-only NATS message signifying a `PUBREL` out for delivery, and store it in the `$MQTT_qos2out` stream, as `$MQTT.qos2.out.`. 7. QoS 2: Deliver `PUBREL`. The PubRel session-specific consumer will publish to internal subscription on `$MQTT.qos2.delivery`, calling `mqttDeliverPubRelCb()`. We store the ACK reply subject in `cpending` to remove the JS message on `PUBCOMP`, compose and send out a `PUBREL` packet. 8. QoS 2: "Wait" for a `PUBCOMP`. See `mqttProcessPubComp()`. - When received, remove the PI from the tracking maps, send an ACK to consumer to remove the `PUBREL` message. - **DONE for QoS 2** ## Retained messages When we process an inbound `PUBLISH` and submit it to `processInboundClientMsg()` function, for MQTT clients it will invoke `mqttHandlePubRetain()` which checks if the published message is “retained†or not. If it is, then we construct a record representing the retained message and store it in the `$MQTT_rmsg` stream, under the single `$MQTT.rmsg` subject. The stored record (in JSON) contains information about the subject, topic, MQTT flags, user that produced this message and the message content itself. It is stored and the stream sequence is remembered in the memory structure that contains retained messages. Note that when creating an account session manager, the retained messages stream is read from scratch to load all the messages through the use of a JS consumer. The associated subscription will process the recovered retained messages or any new that comes from the network. A retained message is added to a map and a subscription is created and inserted into a sublist that will be used to perform a ReverseMatch() when a subscription is started and we want to find all retained messages that the subscription would have received if it had been running prior to the message being published. If a retained message on topic “foo†already exists, then the server has to delete the old message at the stream sequence we saved when storing it. This could have been done with having retained messages stored under `$MQTT.rmsg.` as opposed to all under a single subject, and make use of the `MaxMsgsPer` field set to 1. The `MaxMsgsPer` option was introduced well into the availability of MQTT and changes to the sessions was made in [PR #2501](https://github.com/nats-io/nats-server/pull/2501), with a conversion of existing streams such as `$MQTT*sess*` into a single stream with unique subjects, but the changes were not made to the retained messages stream. There are also subscriptions for the handling of retained messages which are messages that are asked by the publisher to be retained by the MQTT server to be delivered to matching subscriptions when they start. There is a single message per topic. Retained messages are deleted when the user sends a retained message (there is a flag in the PUBLISH protocol) on a given topic with an empty body. The difficulty with retained messages is to handle them in a cluster since all servers need to be aware of their presence so that they can deliver them to subscriptions that those servers may become the leader for. - `$MQTT_rmsgs` which has a “limits†policy and holds retained messages, all under `$MQTT.rmsg` single subject. Not sure why I did not use MaxMsgsPer for this stream and not filter `$MQTT.rmsg.>`. The first step when processing a new subscription is to gather the retained messages that would be a match for this subscription. To do so, the server will serialize into a buffer all messages for the account session manager’s sublist’s ReverseMatch result. We use the returned subscriptions’ subject to find from a map appropriate retained message (see `serializeRetainedMsgsForSub()` for details). # 4. Implementation Notes ## Hooking into NATS I/O ### Starting the accept loop The MQTT accept loop is started when the server detects that an MQTT port has been defined in the configuration file. It works similarly to all other accept loops. Note that for MQTT over websocket, the websocket port has to be defined and MQTT clients will connect to that port instead of the MQTT port and need to provide `/mqtt` as part of the URL to redirect the creation of the client to an MQTT client (with websocket support) instead of a regular NATS with websocket. See the branching done in `startWebsocketServer()`. See `startMQTT()`. ### Starting the read/write loops When a TCP connection is accepted, the internal go routine will invoke `createMQTTClient()`. This function will set a `c.mqtt` object that will make it become an MQTT client (through the `isMqtt()` helper function). The `readLoop()` and `writeLoop()` are started similarly to other clients. However, the read loop will branch out to `mqttParse()` instead when detecting that this is an MQTT client. ## Session Management ### Account Session Manager `mqttAccountSessionManager` is an object that holds the state of all sessions in an account. It also manages the lifecycle of JetStream streams and internal subscriptions for processing JS API replies, session updates, etc. See `mqttCreateAccountSessionManager()`. It is lazily initialized upon the first MQTT `CONNECT` packet received. Account session manager is referred to as `asm` in the code. Note that creating the account session manager (and attempting to create the streams) is done only once per account on a given server, since once created the account session manager for a given account would be found in the sessions map of the mqttSessionManager object. ### Find and disconnect previous session/client Once all that is done, we now go to the creation of the session object itself. For that, we first need to make sure that it does not already exist, meaning that it is registered on the server - or anywhere in the cluster. Note that MQTT dictates that if a session with the same ID connects, the OLD session needs to be closed, not the new one being created. NATS Server complies with this requirement. Once a session is detected to already exists, the old one (as described above) is closed and the new one accepted, however, the session ID is maintained in a flappers map so that we detect situations where sessions with the same ID are started multiple times causing the previous one to be closed. When that detection occurs, the newly created session is put in “jail†for a second to avoid a very rapid succession of connect/disconnect. This has already been seen by users since there was some issue there where we would schedule the connection closed instead of waiting in place which was causing a panic. We also protect from multiple clients on a given server trying to connect with the same ID at the “same time†while the processing of a CONNECT of a session is not yet finished. This is done with the use of a sessLocked map, keyed by the session ID. ### Create or restore the session If everything is good up to that point, the server will either create or restore a session from the stream. This is done in the `createOrRestoreSession()` function. The client/session ID is hashed and added to the session’s stream subject along with the JS domain to prevent clients connecting from different domains to “pollute†the session stream of a given domain. Since each session constitutes a subject and the stream has a maximum of 1 message per subject, we attempt to load the last message on the formed subject. If we don’t find it, then the session object is created “emptyâ€, while if we find a record, we create the session object based on the record persisted on the stream. If the session was restored from the JS stream, we keep track of the stream sequence where the record was located. When we save the session (even if it already exists) we will use this sequence number to set the `JSExpectedLastSubjSeq` header so that we handle possibly different servers in a (super)cluster to detect the race of clients trying to use the same session ID, since only one of the write should succeed. On success, the session’s new sequence is remembered by the server that did the write. When created or restored, the CONNACK can now be sent back to the client, and if there were any recovered subscriptions, they are now processed. ## Processing QoS acks: PUBACK, PUBREC, PUBCOMP When the server delivers a message with QoS 1 or 2 (also a `PUBREL` for QoS 2) to a subscribed client, the client will send back an acknowledgement. See `mqttProcessPubAck()`, `mqttProcessPubRec()`, and `mqttProcessPubComp()` While the specific logic for each packet differs, these handlers all update the session's PI mappings (`cpending`, `pendingPublish`, `pendingPubRel`), and if needed send an ACK to JetStream to remove the message from its consumer and stop the re-delivery attempts. ## Subject Wildcards Note that MQTT subscriptions have wildcards too, the `“+â€` wildcard is equivalent to NATS’s `“*â€` wildcard, however, MQTT’s wildcard `“#â€` is similar to `“>â€`, except that it also includes the level above. That is, a subscription on `“foo/#â€` would receive messages on `“foo/bar/bazâ€`, but also on `“fooâ€`. So, for MQTT subscriptions enging with a `'#'` we are forced to create 2 internal NATS subscriptions, one on `“fooâ€` and one on `“foo.>â€`. # 5. Known issues - "active" redelivery for QoS from JetStream (compliant, just a note) - JetStream QoS redelivery happens out of (original) order - finish delivery of in-flight messages after UNSUB - finish delivery of in-flight messages after a reconnect - consider replacing `$MQTT_msgs` with `$MQTT_out`. - consider using unique `$MQTT.rmsg.>` and `MaxMsgsPer` for retained messages. - add a cli command to list/clean old sessions nats-server-2.10.27/server/README.md000066400000000000000000000015551477524627100167460ustar00rootroot00000000000000# Tests Tests that run on Travis have been split into jobs that run in their own VM in parallel. This reduces the overall running time but also is allowing recycling of a job when we get a flapper as opposed to have to recycle the whole test suite. ## JetStream Tests For JetStream tests, we need to observe a naming convention so that no tests are omitted when running on Travis. The script `runTestsOnTravis.sh` will run a given job based on the definition found in "`.travis.yml`". As for the naming convention: - All JetStream tests name should start with `TestJetStream` - Cluster tests should go into `jetstream_cluster_test.go` and start with `TestJetStreamCluster` - Super-cluster tests should go into `jetstream_super_cluster_test.go` and start with `TestJetStreamSuperCluster` Not following this convention means that some tests may not be executed on Travis. nats-server-2.10.27/server/accounts.go000066400000000000000000003751561477524627100176500ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" "encoding/hex" "errors" "fmt" "io" "io/fs" "math" "math/rand" "net/http" "net/textproto" "reflect" "slices" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/internal/fastrand" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) // For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an // account will be grouped in the default global account. const globalAccountName = DEFAULT_GLOBAL_ACCOUNT const defaultMaxSubLimitReportThreshold = int64(2 * time.Second) var maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold // Account are subject namespace definitions. By default no messages are shared between accounts. // You can share via Exports and Imports of Streams and Services. type Account struct { stats gwReplyMapping Name string Nkey string Issuer string claimJWT string updated time.Time mu sync.RWMutex sqmu sync.Mutex sl *Sublist ic *client isid uint64 etmr *time.Timer ctmr *time.Timer strack map[string]sconns nrclients int32 sysclients int32 nleafs int32 nrleafs int32 clients map[*client]struct{} rm map[string]int32 lqws map[string]int32 usersRevoked map[string]int64 mappings []*mapping hasMapped atomic.Bool lmu sync.RWMutex lleafs []*client leafClusters map[string]uint64 imports importMap exports exportMap js *jsAccount jsLimits map[string]JetStreamAccountLimits limits expired atomic.Bool incomplete bool signingKeys map[string]jwt.Scope extAuth *jwt.ExternalAuthorization srv *Server // server this account is registered with (possibly nil) lds string // loop detection subject for leaf nodes siReply []byte // service reply prefix, will form wildcard subscription. eventIds *nuid.NUID eventIdsMu sync.Mutex defaultPerms *Permissions tags jwt.TagList nameTag string lastLimErr int64 routePoolIdx int // Guarantee that only one goroutine can be running either checkJetStreamMigrate // or clearObserverState at a given time for this account to prevent interleaving. jscmMu sync.Mutex } const ( accDedicatedRoute = -1 accTransitioningToDedicatedRoute = -2 ) // Account based limits. type limits struct { mpay int32 msubs int32 mconns int32 mleafs int32 disallowBearer bool } // Used to track remote clients and leafnodes per remote server. type sconns struct { conns int32 leafs int32 } // Import stream mapping struct type streamImport struct { acc *Account from string to string tr *subjectTransform rtr *subjectTransform claim *jwt.Import usePub bool invalid bool } const ClientInfoHdr = "Nats-Request-Info" // Import service mapping struct type serviceImport struct { acc *Account claim *jwt.Import se *serviceExport sid []byte from string to string tr *subjectTransform ts int64 rt ServiceRespType latency *serviceLatency m1 *ServiceLatency rc *client usePub bool response bool invalid bool share bool tracking bool didDeliver bool trackingHdr http.Header // header from request } // This is used to record when we create a mapping for implicit service // imports. We use this to clean up entries that are not singletons when // we detect that interest is no longer present. The key to the map will // be the actual interest. We record the mapped subject and the account. type serviceRespEntry struct { acc *Account msub string } // ServiceRespType represents the types of service request response types. type ServiceRespType uint8 // Service response types. Defaults to a singleton. const ( Singleton ServiceRespType = iota Streamed Chunked ) // String helper. func (rt ServiceRespType) String() string { switch rt { case Singleton: return "Singleton" case Streamed: return "Streamed" case Chunked: return "Chunked" } return "Unknown ServiceResType" } // exportAuth holds configured approvals or boolean indicating an // auth token is required for import. type exportAuth struct { tokenReq bool accountPos uint approved map[string]*Account actsRevoked map[string]int64 } // streamExport type streamExport struct { exportAuth } // serviceExport holds additional information for exported services. type serviceExport struct { exportAuth acc *Account respType ServiceRespType latency *serviceLatency rtmr *time.Timer respThresh time.Duration } // Used to track service latency. type serviceLatency struct { sampling int8 // percentage from 1-100 or 0 to indicate triggered by header subject string } // exportMap tracks the exported streams and services. type exportMap struct { streams map[string]*streamExport services map[string]*serviceExport responses map[string]*serviceImport } // importMap tracks the imported streams and services. // For services we will also track the response mappings as well. type importMap struct { streams []*streamImport services map[string]*serviceImport rrMap map[string][]*serviceRespEntry } // NewAccount creates a new unlimited account with the given name. func NewAccount(name string) *Account { a := &Account{ Name: name, limits: limits{-1, -1, -1, -1, false}, eventIds: nuid.New(), } return a } func (a *Account) String() string { return a.Name } // Used to create shallow copies of accounts for transfer // from opts to real accounts in server struct. func (a *Account) shallowCopy(na *Account) { na.Nkey = a.Nkey na.Issuer = a.Issuer if a.imports.streams != nil { na.imports.streams = make([]*streamImport, 0, len(a.imports.streams)) for _, v := range a.imports.streams { si := *v na.imports.streams = append(na.imports.streams, &si) } } if a.imports.services != nil { na.imports.services = make(map[string]*serviceImport) for k, v := range a.imports.services { si := *v na.imports.services[k] = &si } } if a.exports.streams != nil { na.exports.streams = make(map[string]*streamExport) for k, v := range a.exports.streams { if v != nil { se := *v na.exports.streams[k] = &se } else { na.exports.streams[k] = nil } } } if a.exports.services != nil { na.exports.services = make(map[string]*serviceExport) for k, v := range a.exports.services { if v != nil { se := *v na.exports.services[k] = &se } else { na.exports.services[k] = nil } } } na.mappings = a.mappings na.hasMapped.Store(len(na.mappings) > 0) // JetStream na.jsLimits = a.jsLimits // Server config account limits. na.limits = a.limits } // nextEventID uses its own lock for better concurrency. func (a *Account) nextEventID() string { a.eventIdsMu.Lock() id := a.eventIds.Next() a.eventIdsMu.Unlock() return id } // Returns a slice of clients stored in the account, or nil if none is present. // Lock is held on entry. func (a *Account) getClientsLocked() []*client { if len(a.clients) == 0 { return nil } clients := make([]*client, 0, len(a.clients)) for c := range a.clients { clients = append(clients, c) } return clients } // Returns a slice of clients stored in the account, or nil if none is present. func (a *Account) getClients() []*client { a.mu.RLock() clients := a.getClientsLocked() a.mu.RUnlock() return clients } // Called to track a remote server and connections and leafnodes it // has for this account. func (a *Account) updateRemoteServer(m *AccountNumConns) []*client { a.mu.Lock() if a.strack == nil { a.strack = make(map[string]sconns) } // This does not depend on receiving all updates since each one is idempotent. // FIXME(dlc) - We should cleanup when these both go to zero. prev := a.strack[m.Server.ID] a.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)} a.nrclients += int32(m.Conns) - prev.conns a.nrleafs += int32(m.LeafNodes) - prev.leafs mtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns)) // If we are over here some have snuck in and we need to rebalance. // All others will probably be doing the same thing but better to be // conservative and bit harsh here. Clients will reconnect if we over compensate. var clients []*client if mtce { clients = a.getClientsLocked() slices.SortFunc(clients, func(i, j *client) int { return -i.start.Compare(j.start) }) // reserve over := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns) if over < len(clients) { clients = clients[:over] } } // Now check leafnodes. mtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs) if mtlce { // Take ones from the end. a.lmu.RLock() leafs := a.lleafs over := int(a.nleafs + a.nrleafs - a.mleafs) if over < len(leafs) { leafs = leafs[len(leafs)-over:] } clients = append(clients, leafs...) a.lmu.RUnlock() } a.mu.Unlock() // If we have exceeded our max clients this will be populated. return clients } // Removes tracking for a remote server that has shutdown. func (a *Account) removeRemoteServer(sid string) { a.mu.Lock() if a.strack != nil { prev := a.strack[sid] delete(a.strack, sid) a.nrclients -= prev.conns a.nrleafs -= prev.leafs } a.mu.Unlock() } // When querying for subject interest this is the number of // expected responses. We need to actually check that the entry // has active connections. func (a *Account) expectedRemoteResponses() (expected int32) { a.mu.RLock() for _, sc := range a.strack { if sc.conns > 0 || sc.leafs > 0 { expected++ } } a.mu.RUnlock() return } // Clears eventing and tracking for this account. func (a *Account) clearEventing() { a.mu.Lock() a.nrclients = 0 // Now clear state clearTimer(&a.etmr) clearTimer(&a.ctmr) a.clients = nil a.strack = nil a.mu.Unlock() } // GetName will return the accounts name. func (a *Account) GetName() string { if a == nil { return "n/a" } a.mu.RLock() name := a.Name a.mu.RUnlock() return name } // NumConnections returns active number of clients for this account for // all known servers. func (a *Account) NumConnections() int { a.mu.RLock() nc := len(a.clients) - int(a.sysclients) + int(a.nrclients) a.mu.RUnlock() return nc } // NumRemoteConnections returns the number of client or leaf connections that // are not on this server. func (a *Account) NumRemoteConnections() int { a.mu.RLock() nc := int(a.nrclients + a.nrleafs) a.mu.RUnlock() return nc } // NumLocalConnections returns active number of clients for this account // on this server. func (a *Account) NumLocalConnections() int { a.mu.RLock() nlc := a.numLocalConnections() a.mu.RUnlock() return nlc } // Do not account for the system accounts. func (a *Account) numLocalConnections() int { return len(a.clients) - int(a.sysclients) - int(a.nleafs) } // This is for extended local interest. // Lock should not be held. func (a *Account) numLocalAndLeafConnections() int { a.mu.RLock() nlc := len(a.clients) - int(a.sysclients) a.mu.RUnlock() return nlc } func (a *Account) numLocalLeafNodes() int { return int(a.nleafs) } // MaxTotalConnectionsReached returns if we have reached our limit for number of connections. func (a *Account) MaxTotalConnectionsReached() bool { var mtce bool a.mu.RLock() if a.mconns != jwt.NoLimit { mtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns) } a.mu.RUnlock() return mtce } // MaxActiveConnections return the set limit for the account system // wide for total number of active connections. func (a *Account) MaxActiveConnections() int { a.mu.RLock() mconns := int(a.mconns) a.mu.RUnlock() return mconns } // MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes. func (a *Account) MaxTotalLeafNodesReached() bool { a.mu.RLock() mtc := a.maxTotalLeafNodesReached() a.mu.RUnlock() return mtc } func (a *Account) maxTotalLeafNodesReached() bool { if a.mleafs != jwt.NoLimit { return a.nleafs+a.nrleafs >= a.mleafs } return false } // NumLeafNodes returns the active number of local and remote // leaf node connections. func (a *Account) NumLeafNodes() int { a.mu.RLock() nln := int(a.nleafs + a.nrleafs) a.mu.RUnlock() return nln } // NumRemoteLeafNodes returns the active number of remote // leaf node connections. func (a *Account) NumRemoteLeafNodes() int { a.mu.RLock() nrn := int(a.nrleafs) a.mu.RUnlock() return nrn } // MaxActiveLeafNodes return the set limit for the account system // wide for total number of leavenode connections. // NOTE: these are tracked separately. func (a *Account) MaxActiveLeafNodes() int { a.mu.RLock() mleafs := int(a.mleafs) a.mu.RUnlock() return mleafs } // RoutedSubs returns how many subjects we would send across a route when first // connected or expressing interest. Local client subs. func (a *Account) RoutedSubs() int { a.mu.RLock() defer a.mu.RUnlock() return len(a.rm) } // TotalSubs returns total number of Subscriptions for this account. func (a *Account) TotalSubs() int { a.mu.RLock() defer a.mu.RUnlock() if a.sl == nil { return 0 } return int(a.sl.Count()) } func (a *Account) shouldLogMaxSubErr() bool { if a == nil { return true } a.mu.RLock() last := a.lastLimErr a.mu.RUnlock() if now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold { a.mu.Lock() a.lastLimErr = now a.mu.Unlock() return true } return false } // MapDest is for mapping published subjects for clients. type MapDest struct { Subject string `json:"subject"` Weight uint8 `json:"weight"` Cluster string `json:"cluster,omitempty"` } func NewMapDest(subject string, weight uint8) *MapDest { return &MapDest{subject, weight, _EMPTY_} } // destination is for internal representation for a weighted mapped destination. type destination struct { tr *subjectTransform weight uint8 } // mapping is an internal entry for mapping subjects. type mapping struct { src string wc bool dests []*destination cdests map[string][]*destination } // AddMapping adds in a simple route mapping from src subject to dest subject // for inbound client messages. func (a *Account) AddMapping(src, dest string) error { return a.AddWeightedMappings(src, NewMapDest(dest, 100)) } // AddWeightedMappings will add in a weighted mappings for the destinations. func (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error { a.mu.Lock() defer a.mu.Unlock() if !IsValidSubject(src) { return ErrBadSubject } m := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)} seen := make(map[string]struct{}) var tw = make(map[string]uint8) for _, d := range dests { if _, ok := seen[d.Subject]; ok { return fmt.Errorf("duplicate entry for %q", d.Subject) } seen[d.Subject] = struct{}{} if d.Weight > 100 { return fmt.Errorf("individual weights need to be <= 100") } tw[d.Cluster] += d.Weight if tw[d.Cluster] > 100 { return fmt.Errorf("total weight needs to be <= 100") } err := ValidateMappingDestination(d.Subject) if err != nil { return err } tr, err := NewSubjectTransform(src, d.Subject) if err != nil { return err } if d.Cluster == _EMPTY_ { m.dests = append(m.dests, &destination{tr, d.Weight}) } else { // We have a cluster scoped filter. if m.cdests == nil { m.cdests = make(map[string][]*destination) } ad := m.cdests[d.Cluster] ad = append(ad, &destination{tr, d.Weight}) m.cdests[d.Cluster] = ad } } processDestinations := func(dests []*destination) ([]*destination, error) { var ltw uint8 for _, d := range dests { ltw += d.weight } // Auto add in original at weight difference if all entries weight does not total to 100. // Iff the src was not already added in explicitly, meaning they want loss. _, haveSrc := seen[src] if ltw != 100 && !haveSrc { dest := src if m.wc { // We need to make the appropriate markers for the wildcards etc. dest = transformTokenize(dest) } tr, err := NewSubjectTransform(src, dest) if err != nil { return nil, err } aw := 100 - ltw if len(dests) == 0 { aw = 100 } dests = append(dests, &destination{tr, aw}) } slices.SortFunc(dests, func(i, j *destination) int { return cmp.Compare(i.weight, j.weight) }) var lw uint8 for _, d := range dests { d.weight += lw lw = d.weight } return dests, nil } var err error if m.dests, err = processDestinations(m.dests); err != nil { return err } // Option cluster scoped destinations for cluster, dests := range m.cdests { if dests, err = processDestinations(dests); err != nil { return err } m.cdests[cluster] = dests } // Replace an old one if it exists. for i, em := range a.mappings { if em.src == src { a.mappings[i] = m return nil } } // If we did not replace add to the end. a.mappings = append(a.mappings, m) a.hasMapped.Store(len(a.mappings) > 0) // If we have connected leafnodes make sure to update. if a.nleafs > 0 { // Need to release because lock ordering is client -> account a.mu.Unlock() // Now grab the leaf list lock. We can hold client lock under this one. a.lmu.RLock() for _, lc := range a.lleafs { lc.forceAddToSmap(src) } a.lmu.RUnlock() a.mu.Lock() } return nil } // RemoveMapping will remove an existing mapping. func (a *Account) RemoveMapping(src string) bool { a.mu.Lock() defer a.mu.Unlock() for i, m := range a.mappings { if m.src == src { // Swap last one into this spot. Its ok to change order. a.mappings[i] = a.mappings[len(a.mappings)-1] a.mappings[len(a.mappings)-1] = nil // gc a.mappings = a.mappings[:len(a.mappings)-1] a.hasMapped.Store(len(a.mappings) > 0) // If we have connected leafnodes make sure to update. if a.nleafs > 0 { // Need to release because lock ordering is client -> account a.mu.Unlock() // Now grab the leaf list lock. We can hold client lock under this one. a.lmu.RLock() for _, lc := range a.lleafs { lc.forceRemoveFromSmap(src) } a.lmu.RUnlock() a.mu.Lock() } return true } } return false } // Indicates we have mapping entries. func (a *Account) hasMappings() bool { if a == nil { return false } return a.hasMapped.Load() } // This performs the logic to map to a new dest subject based on mappings. // Should only be called from processInboundClientMsg or service import processing. func (a *Account) selectMappedSubject(dest string) (string, bool) { if !a.hasMappings() { return dest, false } a.mu.RLock() // In case we have to tokenize for subset matching. tsa := [32]string{} tts := tsa[:0] var m *mapping for _, rm := range a.mappings { if !rm.wc && rm.src == dest { m = rm break } else { // tokenize and reuse for subset matching. if len(tts) == 0 { start := 0 subject := dest for i := 0; i < len(subject); i++ { if subject[i] == btsep { tts = append(tts, subject[start:i]) start = i + 1 } } tts = append(tts, subject[start:]) } if isSubsetMatch(tts, rm.src) { m = rm break } } } if m == nil { a.mu.RUnlock() return dest, false } // The selected destination for the mapping. var d *destination var ndest string dests := m.dests if len(m.cdests) > 0 { cn := a.srv.cachedClusterName() dests = m.cdests[cn] if dests == nil { // Fallback to main if we do not match the cluster. dests = m.dests } } // Optimize for single entry case. if len(dests) == 1 && dests[0].weight == 100 { d = dests[0] } else { w := uint8(fastrand.Uint32n(100)) for _, rm := range dests { if w < rm.weight { d = rm break } } } if d != nil { if len(d.tr.dtokmftokindexesargs) == 0 { ndest = d.tr.dest } else { ndest = d.tr.TransformTokenizedSubject(tts) } } a.mu.RUnlock() return ndest, true } // SubscriptionInterest returns true if this account has a matching subscription // for the given `subject`. func (a *Account) SubscriptionInterest(subject string) bool { return a.Interest(subject) > 0 } // Interest returns the number of subscriptions for a given subject that match. func (a *Account) Interest(subject string) int { var nms int a.mu.RLock() if a.sl != nil { np, nq := a.sl.NumInterest(subject) nms = np + nq } a.mu.RUnlock() return nms } // addClient keeps our accounting of local active clients or leafnodes updated. // Returns previous total. func (a *Account) addClient(c *client) int { a.mu.Lock() n := len(a.clients) // Could come here earlier than the account is registered with the server. // Make sure we can still track clients. if a.clients == nil { a.clients = make(map[*client]struct{}) } a.clients[c] = struct{}{} // If we did not add it, we are done if n == len(a.clients) { a.mu.Unlock() return n } if c.kind != CLIENT && c.kind != LEAF { a.sysclients++ } else if c.kind == LEAF { a.nleafs++ } a.mu.Unlock() // If we added a new leaf use the list lock and add it to the list. if c.kind == LEAF { a.lmu.Lock() a.lleafs = append(a.lleafs, c) a.lmu.Unlock() } if c != nil && c.srv != nil { c.srv.accConnsUpdate(a) } return n } // For registering clusters for remote leafnodes. // We only register as the hub. func (a *Account) registerLeafNodeCluster(cluster string) { a.mu.Lock() defer a.mu.Unlock() if a.leafClusters == nil { a.leafClusters = make(map[string]uint64) } a.leafClusters[cluster]++ } // Check to see if we already have this cluster registered. func (a *Account) hasLeafNodeCluster(cluster string) bool { a.mu.RLock() defer a.mu.RUnlock() return a.leafClusters[cluster] > 0 } // Check to see if this cluster is isolated, meaning the only one. // Read Lock should be held. func (a *Account) isLeafNodeClusterIsolated(cluster string) bool { if cluster == _EMPTY_ { return false } if len(a.leafClusters) > 1 { return false } return a.leafClusters[cluster] == uint64(a.nleafs) } // Helper function to remove leaf nodes. If number of leafnodes gets large // this may need to be optimized out of linear search but believe number // of active leafnodes per account scope to be small and therefore cache friendly. // Lock should not be held on general account lock. func (a *Account) removeLeafNode(c *client) { // Make sure we hold the list lock as well. a.lmu.Lock() defer a.lmu.Unlock() ll := len(a.lleafs) for i, l := range a.lleafs { if l == c { a.lleafs[i] = a.lleafs[ll-1] if ll == 1 { a.lleafs = nil } else { a.lleafs = a.lleafs[:ll-1] } return } } } // removeClient keeps our accounting of local active clients updated. func (a *Account) removeClient(c *client) int { a.mu.Lock() n := len(a.clients) delete(a.clients, c) // If we did not actually remove it, we are done. if n == len(a.clients) { a.mu.Unlock() return n } if c.kind != CLIENT && c.kind != LEAF { a.sysclients-- } else if c.kind == LEAF { a.nleafs-- // Need to do cluster accounting here. // Do cluster accounting if we are a hub. if c.isHubLeafNode() { cluster := c.remoteCluster() if count := a.leafClusters[cluster]; count > 1 { a.leafClusters[cluster]-- } else if count == 1 { delete(a.leafClusters, cluster) } } } a.mu.Unlock() if c.kind == LEAF { a.removeLeafNode(c) } if c != nil && c.srv != nil { c.srv.accConnsUpdate(a) } return n } func setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error { if accountPos > 0 { token := strings.Split(subject, tsep) if len(token) < int(accountPos) || token[accountPos-1] != "*" { return ErrInvalidSubject } } ea.accountPos = accountPos // empty means auth required but will be import token. if accounts == nil { return nil } if len(accounts) == 0 { ea.tokenReq = true return nil } if ea.approved == nil { ea.approved = make(map[string]*Account, len(accounts)) } for _, acc := range accounts { ea.approved[acc.Name] = acc } return nil } // AddServiceExport will configure the account with the defined export. func (a *Account) AddServiceExport(subject string, accounts []*Account) error { return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0) } // AddServiceExport will configure the account with the defined export. func (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { return a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos) } // AddServiceExportWithResponse will configure the account with the defined export and response type. func (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error { return a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0) } // AddServiceExportWithresponse will configure the account with the defined export and response type. func (a *Account) addServiceExportWithResponseAndAccountPos( subject string, respType ServiceRespType, accounts []*Account, accountPos uint) error { if a == nil { return ErrMissingAccount } a.mu.Lock() if a.exports.services == nil { a.exports.services = make(map[string]*serviceExport) } se := a.exports.services[subject] // Always create a service export if se == nil { se = &serviceExport{} } if respType != Singleton { se.respType = respType } if accounts != nil || accountPos > 0 { if err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil { a.mu.Unlock() return err } } lrt := a.lowestServiceExportResponseTime() se.acc = a se.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD a.exports.services[subject] = se var clients []*client nlrt := a.lowestServiceExportResponseTime() if nlrt != lrt && len(a.clients) > 0 { clients = a.getClientsLocked() } // Need to release because lock ordering is client -> Account a.mu.Unlock() if len(clients) > 0 { updateAllClientsServiceExportResponseTime(clients, nlrt) } return nil } // TrackServiceExport will enable latency tracking of the named service. // Results will be published in this account to the given results subject. func (a *Account) TrackServiceExport(service, results string) error { return a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING) } // TrackServiceExportWithSampling will enable latency tracking of the named service for the given // sampling rate (1-100). Results will be published in this account to the given results subject. func (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error { if a == nil { return ErrMissingAccount } if sampling != 0 { // 0 means triggered by header if sampling < 1 || sampling > 100 { return ErrBadSampling } } if !IsValidPublishSubject(results) { return ErrBadPublishSubject } // Don't loop back on outselves. if a.IsExportService(results) { return ErrBadPublishSubject } if a.srv != nil && !a.srv.EventsEnabled() { return ErrNoSysAccount } a.mu.Lock() if a.exports.services == nil { a.mu.Unlock() return ErrMissingService } ea, ok := a.exports.services[service] if !ok { a.mu.Unlock() return ErrMissingService } if ea == nil { ea = &serviceExport{} a.exports.services[service] = ea } else if ea.respType != Singleton { a.mu.Unlock() return ErrBadServiceType } ea.latency = &serviceLatency{ sampling: int8(sampling), subject: results, } s := a.srv a.mu.Unlock() if s == nil { return nil } // Now track down the imports and add in latency as needed to enable. s.accounts.Range(func(k, v any) bool { acc := v.(*Account) acc.mu.Lock() for _, im := range acc.imports.services { if im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) { im.latency = ea.latency } } acc.mu.Unlock() return true }) return nil } // UnTrackServiceExport will disable latency tracking of the named service. func (a *Account) UnTrackServiceExport(service string) { if a == nil || (a.srv != nil && !a.srv.EventsEnabled()) { return } a.mu.Lock() if a.exports.services == nil { a.mu.Unlock() return } ea, ok := a.exports.services[service] if !ok || ea == nil || ea.latency == nil { a.mu.Unlock() return } // We have latency here. ea.latency = nil s := a.srv a.mu.Unlock() if s == nil { return } // Now track down the imports and clean them up. s.accounts.Range(func(k, v any) bool { acc := v.(*Account) acc.mu.Lock() for _, im := range acc.imports.services { if im != nil && im.acc.Name == a.Name { if subjectIsSubsetMatch(im.to, service) { im.latency, im.m1 = nil, nil } } } acc.mu.Unlock() return true }) } // IsExportService will indicate if this service exists. Will check wildcard scenarios. func (a *Account) IsExportService(service string) bool { a.mu.RLock() defer a.mu.RUnlock() _, ok := a.exports.services[service] if ok { return true } tokens := strings.Split(service, tsep) for subj := range a.exports.services { if isSubsetMatch(tokens, subj) { return true } } return false } // IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled. func (a *Account) IsExportServiceTracking(service string) bool { a.mu.RLock() ea, ok := a.exports.services[service] if ok && ea == nil { a.mu.RUnlock() return false } if ok && ea != nil && ea.latency != nil { a.mu.RUnlock() return true } // FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking. tokens := strings.Split(service, tsep) for subj, ea := range a.exports.services { if isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil { a.mu.RUnlock() return true } } a.mu.RUnlock() return false } // ServiceLatency is the JSON message sent out in response to latency tracking for // an accounts exported services. Additional client info is available in requestor // and responder. Note that for a requestor, the only information shared by default // is the RTT used to calculate the total latency. The requestor's account can // designate to share the additional information in the service import. type ServiceLatency struct { TypedEvent Status int `json:"status"` Error string `json:"description,omitempty"` Requestor *ClientInfo `json:"requestor,omitempty"` Responder *ClientInfo `json:"responder,omitempty"` RequestHeader http.Header `json:"header,omitempty"` // only contains header(s) triggering the measurement RequestStart time.Time `json:"start"` ServiceLatency time.Duration `json:"service"` SystemLatency time.Duration `json:"system"` TotalLatency time.Duration `json:"total"` } // ServiceLatencyType is the NATS Event Type for ServiceLatency const ServiceLatencyType = "io.nats.server.metric.v1.service_latency" // NATSTotalTime is a helper function that totals the NATS latencies. func (m1 *ServiceLatency) NATSTotalTime() time.Duration { return m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency } // Merge function to merge m1 and m2 (requestor and responder) measurements // when there are two samples. This happens when the requestor and responder // are on different servers. // // m2 ServiceLatency is correct, so use that. // m1 TotalLatency is correct, so use that. // Will use those to back into NATS latency. func (m1 *ServiceLatency) merge(m2 *ServiceLatency) { rtt := time.Duration(0) if m2.Responder != nil { rtt = m2.Responder.RTT } m1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt) m1.ServiceLatency = m2.ServiceLatency m1.Responder = m2.Responder sanitizeLatencyMetric(m1) } // sanitizeLatencyMetric adjusts latency metric values that could go // negative in some edge conditions since we estimate client RTT // for both requestor and responder. // These numbers are never meant to be negative, it just could be // how we back into the values based on estimated RTT. func sanitizeLatencyMetric(sl *ServiceLatency) { if sl.ServiceLatency < 0 { sl.ServiceLatency = 0 } if sl.SystemLatency < 0 { sl.SystemLatency = 0 } } // Used for transporting remote latency measurements. type remoteLatency struct { Account string `json:"account"` ReqId string `json:"req_id"` M2 ServiceLatency `json:"m2"` respThresh time.Duration } // sendLatencyResult will send a latency result and clear the si of the requestor(rc). func (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) { sl.Type = ServiceLatencyType sl.ID = a.nextEventID() sl.Time = time.Now().UTC() a.mu.Lock() lsubj := si.latency.subject si.rc = nil a.mu.Unlock() a.srv.sendInternalAccountMsg(a, lsubj, sl) } // Used to send a bad request metric when we do not have a reply subject func (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) { sl := &ServiceLatency{ Status: 400, Error: "Bad Request", Requestor: requestor.getClientInfo(si.share), } sl.RequestHeader = header sl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC() a.sendLatencyResult(si, sl) } // Used to send a latency result when the requestor interest was lost before the // response could be delivered. func (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) { sl := &ServiceLatency{ Status: 408, Error: "Request Timeout", } a.mu.RLock() rc, share, ts := si.rc, si.share, si.ts sl.RequestHeader = si.trackingHdr a.mu.RUnlock() if rc != nil { sl.Requestor = rc.getClientInfo(share) } sl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC() a.sendLatencyResult(si, sl) } func (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) { sl := &ServiceLatency{} a.mu.RLock() rc, share, ts := si.rc, si.share, si.ts sl.RequestHeader = si.trackingHdr a.mu.RUnlock() if rc != nil { sl.Requestor = rc.getClientInfo(share) } var reqRTT time.Duration if sl.Requestor != nil { reqRTT = sl.Requestor.RTT } sl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC() if reason == rsiNoDelivery { sl.Status = 503 sl.Error = "Service Unavailable" } else if reason == rsiTimeout { sl.Status = 504 sl.Error = "Service Timeout" } a.sendLatencyResult(si, sl) } // sendTrackingLatency will send out the appropriate tracking information for the // service request/response latency. This is called when the requestor's server has // received the response. // TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit. func (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool { a.mu.RLock() rc := si.rc a.mu.RUnlock() if rc == nil { return true } ts := time.Now() serviceRTT := time.Duration(ts.UnixNano() - si.ts) requestor := si.rc sl := &ServiceLatency{ Status: 200, Requestor: requestor.getClientInfo(si.share), Responder: responder.getClientInfo(true), } var respRTT, reqRTT time.Duration if sl.Responder != nil { respRTT = sl.Responder.RTT } if sl.Requestor != nil { reqRTT = sl.Requestor.RTT } sl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC() sl.ServiceLatency = serviceRTT - respRTT sl.TotalLatency = reqRTT + serviceRTT if respRTT > 0 { sl.SystemLatency = time.Since(ts) sl.TotalLatency += sl.SystemLatency } sl.RequestHeader = si.trackingHdr sanitizeLatencyMetric(sl) sl.Type = ServiceLatencyType sl.ID = a.nextEventID() sl.Time = time.Now().UTC() // If we are expecting a remote measurement, store our sl here. // We need to account for the race between this and us receiving the // remote measurement. // FIXME(dlc) - We need to clean these up but this should happen // already with the auto-expire logic. if responder != nil && responder.kind != CLIENT { si.acc.mu.Lock() if si.m1 != nil { m1, m2 := sl, si.m1 m1.merge(m2) si.acc.mu.Unlock() a.srv.sendInternalAccountMsg(a, si.latency.subject, m1) a.mu.Lock() si.rc = nil a.mu.Unlock() return true } si.m1 = sl si.acc.mu.Unlock() return false } else { a.srv.sendInternalAccountMsg(a, si.latency.subject, sl) a.mu.Lock() si.rc = nil a.mu.Unlock() } return true } // This will check to make sure our response lower threshold is set // properly in any clients doing rrTracking. func updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) { for _, c := range clients { c.mu.Lock() if c.rrTracking != nil && lrt != c.rrTracking.lrt { c.rrTracking.lrt = lrt if c.rrTracking.ptmr.Stop() { c.rrTracking.ptmr.Reset(lrt) } } c.mu.Unlock() } } // Will select the lowest respThresh from all service exports. // Read lock should be held. func (a *Account) lowestServiceExportResponseTime() time.Duration { // Lowest we will allow is 5 minutes. Its an upper bound for this function. lrt := 5 * time.Minute for _, se := range a.exports.services { if se.respThresh < lrt { lrt = se.respThresh } } return lrt } // AddServiceImportWithClaim will add in the service import via the jwt claim. func (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error { return a.addServiceImportWithClaim(destination, from, to, imClaim, false) } // addServiceImportWithClaim will add in the service import via the jwt claim. // It will also skip the authorization check in cases where internal is true func (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error { if destination == nil { return ErrMissingAccount } // Empty means use from. if to == _EMPTY_ { to = from } if !IsValidSubject(from) || !IsValidSubject(to) { return ErrInvalidSubject } // First check to see if the account has authorized us to route to the "to" subject. if !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) { return ErrServiceImportAuthorization } // Check if this introduces a cycle before proceeding. // From will be the mapped subject. // If the 'to' has a wildcard make sure we pre-transform the 'from' before we check for cycles, e.g. '$1' fromT := from if subjectHasWildcard(to) { fromT, _ = transformUntokenize(from) } if err := a.serviceImportFormsCycle(destination, fromT); err != nil { return err } _, err := a.addServiceImport(destination, from, to, imClaim) return err } const MaxAccountCycleSearchDepth = 1024 func (a *Account) serviceImportFormsCycle(dest *Account, from string) error { return dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true}) } func (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error { if len(visited) >= MaxAccountCycleSearchDepth { return ErrCycleSearchDepth } a.mu.RLock() for _, si := range a.imports.services { if SubjectsCollide(from, si.to) { a.mu.RUnlock() if visited[si.acc.Name] { return ErrImportFormsCycle } // Push ourselves and check si.acc visited[a.Name] = true if subjectIsSubsetMatch(si.from, from) { from = si.from } if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil { return err } a.mu.RLock() } } a.mu.RUnlock() return nil } func (a *Account) streamImportFormsCycle(dest *Account, to string) error { return dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true}) } // Lock should be held. func (a *Account) hasServiceExportMatching(to string) bool { for subj := range a.exports.services { if subjectIsSubsetMatch(to, subj) { return true } } return false } // Lock should be held. func (a *Account) hasStreamExportMatching(to string) bool { for subj := range a.exports.streams { if subjectIsSubsetMatch(to, subj) { return true } } return false } func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error { if len(visited) >= MaxAccountCycleSearchDepth { return ErrCycleSearchDepth } a.mu.RLock() if !a.hasStreamExportMatching(to) { a.mu.RUnlock() return nil } for _, si := range a.imports.streams { if SubjectsCollide(to, si.to) { a.mu.RUnlock() if visited[si.acc.Name] { return ErrImportFormsCycle } // Push ourselves and check si.acc visited[a.Name] = true if subjectIsSubsetMatch(si.to, to) { to = si.to } if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil { return err } a.mu.RLock() } } a.mu.RUnlock() return nil } // SetServiceImportSharing will allow sharing of information about requests with the export account. // Used for service latency tracking at the moment. func (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error { return a.setServiceImportSharing(destination, to, true, allow) } // setServiceImportSharing will allow sharing of information about requests with the export account. func (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error { a.mu.Lock() defer a.mu.Unlock() if check && a.isClaimAccount() { return fmt.Errorf("claim based accounts can not be updated directly") } for _, si := range a.imports.services { if si.acc == destination && si.to == to { si.share = allow return nil } } return fmt.Errorf("service import not found") } // AddServiceImport will add a route to an account to send published messages / requests // to the destination account. From is the local subject to map, To is the // subject that will appear on the destination account. Destination will need // to have an import rule to allow access via addService. func (a *Account) AddServiceImport(destination *Account, from, to string) error { return a.AddServiceImportWithClaim(destination, from, to, nil) } // NumPendingReverseResponses returns the number of response mappings we have for all outstanding // requests for service imports. func (a *Account) NumPendingReverseResponses() int { a.mu.RLock() defer a.mu.RUnlock() return len(a.imports.rrMap) } // NumPendingAllResponses return the number of all responses outstanding for service exports. func (a *Account) NumPendingAllResponses() int { return a.NumPendingResponses(_EMPTY_) } // NumPendingResponses returns the number of responses outstanding for service exports // on this account. An empty filter string returns all responses regardless of which export. // If you specify the filter we will only return ones that are for that export. // NOTE this is only for what this server is tracking. func (a *Account) NumPendingResponses(filter string) int { a.mu.RLock() defer a.mu.RUnlock() if filter == _EMPTY_ { return len(a.exports.responses) } se := a.getServiceExport(filter) if se == nil { return 0 } var nre int for _, si := range a.exports.responses { if si.se == se { nre++ } } return nre } // NumServiceImports returns the number of service imports we have configured. func (a *Account) NumServiceImports() int { a.mu.RLock() defer a.mu.RUnlock() return len(a.imports.services) } // Reason why we are removing this response serviceImport. type rsiReason int const ( rsiOk = rsiReason(iota) rsiNoDelivery rsiTimeout ) // removeRespServiceImport removes a response si mapping and the reverse entries for interest detection. func (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) { if si == nil { return } a.mu.Lock() c := a.ic delete(a.exports.responses, si.from) dest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver a.mu.Unlock() // If we have a sid make sure to unsub. if len(si.sid) > 0 && c != nil { c.processUnsub(si.sid) } if tracking && rc != nil && !didDeliver { a.sendBackendErrorTrackingLatency(si, reason) } dest.checkForReverseEntry(to, si, false) } // removeServiceImport will remove the route by subject. func (a *Account) removeServiceImport(subject string) { a.mu.Lock() si, ok := a.imports.services[subject] delete(a.imports.services, subject) var sid []byte c := a.ic if ok && si != nil { if a.ic != nil && si.sid != nil { sid = si.sid } } a.mu.Unlock() if sid != nil { c.processUnsub(sid) } } // This tracks responses to service requests mappings. This is used for cleanup. func (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) { a.mu.Lock() if a.imports.rrMap == nil { a.imports.rrMap = make(map[string][]*serviceRespEntry) } sre := &serviceRespEntry{acc, from} sra := a.imports.rrMap[reply] a.imports.rrMap[reply] = append(sra, sre) a.mu.Unlock() } // checkForReverseEntries is for when we are trying to match reverse entries to a wildcard. // This will be called from checkForReverseEntry when the reply arg is a wildcard subject. // This will usually be called in a go routine since we need to walk all the entries. func (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) { if subjectIsLiteral(reply) { a._checkForReverseEntry(reply, nil, checkInterest, recursed) return } a.mu.RLock() if len(a.imports.rrMap) == 0 { a.mu.RUnlock() return } var _rs [64]string rs := _rs[:0] if n := len(a.imports.rrMap); n > cap(rs) { rs = make([]string, 0, n) } for k := range a.imports.rrMap { rs = append(rs, k) } a.mu.RUnlock() tsa := [32]string{} tts := tokenizeSubjectIntoSlice(tsa[:0], reply) rsa := [32]string{} for _, r := range rs { rts := tokenizeSubjectIntoSlice(rsa[:0], r) // isSubsetMatchTokenized is heavy so make sure we do this without the lock. if isSubsetMatchTokenized(rts, tts) { a._checkForReverseEntry(r, nil, checkInterest, recursed) } } } // This checks for any response map entries. If you specify an si we will only match and // clean up for that one, otherwise we remove them all. func (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) { a._checkForReverseEntry(reply, si, checkInterest, false) } // Callers should use checkForReverseEntry instead. This function exists to help prevent // infinite recursion. func (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) { a.mu.RLock() if len(a.imports.rrMap) == 0 { a.mu.RUnlock() return } if subjectHasWildcard(reply) { if recursed { // If we have reached this condition then it is because the reverse entries also // contain wildcards (that shouldn't happen but a client *could* provide an inbox // prefix that is illegal because it ends in a wildcard character), at which point // we will end up with infinite recursion between this func and checkForReverseEntries. // To avoid a stack overflow panic, we'll give up instead. a.mu.RUnlock() return } doInline := len(a.imports.rrMap) <= 64 a.mu.RUnlock() if doInline { a.checkForReverseEntries(reply, checkInterest, true) } else { go a.checkForReverseEntries(reply, checkInterest, true) } return } if sres := a.imports.rrMap[reply]; sres == nil { a.mu.RUnlock() return } // If we are here we have an entry we should check. // If requested we will first check if there is any // interest for this subject for the entire account. // If there is we can not delete any entries yet. // Note that if we are here reply has to be a literal subject. if checkInterest { // If interest still exists we can not clean these up yet. if a.sl.HasInterest(reply) { a.mu.RUnlock() return } } a.mu.RUnlock() // Delete the appropriate entries here based on optional si. a.mu.Lock() // We need a new lookup here because we have released the lock. sres := a.imports.rrMap[reply] if si == nil { delete(a.imports.rrMap, reply) } else if sres != nil { // Find the one we are looking for.. for i, sre := range sres { if sre.msub == si.from { sres = append(sres[:i], sres[i+1:]...) break } } if len(sres) > 0 { a.imports.rrMap[si.to] = sres } else { delete(a.imports.rrMap, si.to) } } a.mu.Unlock() // If we are here we no longer have interest and we have // response entries that we should clean up. if si == nil { // sres is now known to have been removed from a.imports.rrMap, so we // can safely (data race wise) iterate through. for _, sre := range sres { acc := sre.acc var trackingCleanup bool var rsi *serviceImport acc.mu.Lock() c := acc.ic if rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver { delete(acc.exports.responses, rsi.from) trackingCleanup = rsi.tracking && rsi.rc != nil } acc.mu.Unlock() // If we are doing explicit subs for all responses (e.g. bound to leafnode) // we will have a non-empty sid here. if rsi != nil && len(rsi.sid) > 0 && c != nil { c.processUnsub(rsi.sid) } if trackingCleanup { acc.sendReplyInterestLostTrackLatency(rsi) } } } } // Checks to see if a potential service import subject is already overshadowed. func (a *Account) serviceImportShadowed(from string) bool { a.mu.RLock() defer a.mu.RUnlock() if a.imports.services[from] != nil { return true } // We did not find a direct match, so check individually. for subj := range a.imports.services { if subjectIsSubsetMatch(from, subj) { return true } } return false } // Internal check to see if a service import exists. func (a *Account) serviceImportExists(from string) bool { a.mu.RLock() dup := a.imports.services[from] a.mu.RUnlock() return dup != nil } // Add a service import. // This does no checks and should only be called by the msg processing code. // Use AddServiceImport from above if responding to user input or config changes, etc. func (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) { rt := Singleton var lat *serviceLatency if dest == nil { return nil, ErrMissingAccount } dest.mu.RLock() se := dest.getServiceExport(to) if se != nil { rt = se.respType lat = se.latency } dest.mu.RUnlock() a.mu.Lock() if a.imports.services == nil { a.imports.services = make(map[string]*serviceImport) } else if dup := a.imports.services[from]; dup != nil { a.mu.Unlock() return nil, fmt.Errorf("duplicate service import subject %q, previously used in import for account %q, subject %q", from, dup.acc.Name, dup.to) } if to == _EMPTY_ { to = from } // Check to see if we have a wildcard var ( usePub bool tr *subjectTransform err error ) if subjectHasWildcard(to) { // If to and from match, then we use the published subject. if to == from { usePub = true } else { to, _ = transformUntokenize(to) // Create a transform. Do so in reverse such that $ symbols only exist in to if tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil { a.mu.Unlock() return nil, fmt.Errorf("failed to create mapping transform for service import subject from %q to %q: %v", from, to, err) } else { // un-tokenize and reverse transform so we get the transform needed from, _ = transformUntokenize(from) tr = tr.reverse() } } } var share bool if claim != nil { share = claim.Share } si := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, nil} a.imports.services[from] = si a.mu.Unlock() if err := a.addServiceImportSub(si); err != nil { a.removeServiceImport(si.from) return nil, err } return si, nil } // Returns the internal client, will create one if not present. // Lock should be held. func (a *Account) internalClient() *client { if a.ic == nil && a.srv != nil { a.ic = a.srv.createInternalAccountClient() a.ic.acc = a } return a.ic } // Internal account scoped subscriptions. func (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) { return a.subscribeInternalEx(subject, cb, false) } // Unsubscribe from an internal account subscription. func (a *Account) unsubscribeInternal(sub *subscription) { if ic := a.internalClient(); ic != nil { ic.processUnsub(sub.sid) } } // Creates internal subscription for service import responses. func (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) { return a.subscribeInternalEx(subject, a.processServiceImportResponse, true) } func (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) { a.mu.Lock() a.isid++ c, sid := a.internalClient(), strconv.FormatUint(a.isid, 10) a.mu.Unlock() // This will happen in parsing when the account has not been properly setup. if c == nil { return nil, fmt.Errorf("no internal account client") } return c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri) } // This will add an account subscription that matches the "from" from a service import entry. func (a *Account) addServiceImportSub(si *serviceImport) error { a.mu.Lock() c := a.internalClient() // This will happen in parsing when the account has not been properly setup. if c == nil { a.mu.Unlock() return nil } if si.sid != nil { a.mu.Unlock() return fmt.Errorf("duplicate call to create subscription for service import") } a.isid++ sid := strconv.FormatUint(a.isid, 10) si.sid = []byte(sid) subject := si.from a.mu.Unlock() cb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) { c.pa.delivered = c.processServiceImport(si, acc, msg) } sub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false) if err != nil { return err } // Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription // This is similar to what initLeafNodeSmapAndSendSubs does // TODO we need to consider performing this update as we get client subscriptions. // This behavior would result in subscription propagation only where actually used. a.updateLeafNodes(sub, 1) return nil } // Remove all the subscriptions associated with service imports. func (a *Account) removeAllServiceImportSubs() { a.mu.RLock() var sids [][]byte for _, si := range a.imports.services { if si.sid != nil { sids = append(sids, si.sid) si.sid = nil } } c := a.ic a.ic = nil a.mu.RUnlock() if c == nil { return } for _, sid := range sids { c.processUnsub(sid) } c.closeConnection(InternalClient) } // Add in subscriptions for all registered service imports. func (a *Account) addAllServiceImportSubs() { var sis [32]*serviceImport serviceImports := sis[:0] a.mu.RLock() for _, si := range a.imports.services { serviceImports = append(serviceImports, si) } a.mu.RUnlock() for _, si := range serviceImports { a.addServiceImportSub(si) } } var ( // header where all information is encoded in one value. trcUber = textproto.CanonicalMIMEHeaderKey("Uber-Trace-Id") trcCtx = textproto.CanonicalMIMEHeaderKey("Traceparent") trcB3 = textproto.CanonicalMIMEHeaderKey("B3") // openzipkin header to check trcB3Sm = textproto.CanonicalMIMEHeaderKey("X-B3-Sampled") trcB3Id = textproto.CanonicalMIMEHeaderKey("X-B3-TraceId") // additional header needed to include when present trcB3PSId = textproto.CanonicalMIMEHeaderKey("X-B3-ParentSpanId") trcB3SId = textproto.CanonicalMIMEHeaderKey("X-B3-SpanId") trcCtxSt = textproto.CanonicalMIMEHeaderKey("Tracestate") trcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey("Uberctx-") ) func newB3Header(h http.Header) http.Header { retHdr := http.Header{} if v, ok := h[trcB3Sm]; ok { retHdr[trcB3Sm] = v } if v, ok := h[trcB3Id]; ok { retHdr[trcB3Id] = v } if v, ok := h[trcB3PSId]; ok { retHdr[trcB3PSId] = v } if v, ok := h[trcB3SId]; ok { retHdr[trcB3SId] = v } return retHdr } func newUberHeader(h http.Header, tId []string) http.Header { retHdr := http.Header{trcUber: tId} for k, v := range h { if strings.HasPrefix(k, trcUberCtxPrefix) { retHdr[k] = v } } return retHdr } func newTraceCtxHeader(h http.Header, tId []string) http.Header { retHdr := http.Header{trcCtx: tId} if v, ok := h[trcCtxSt]; ok { retHdr[trcCtxSt] = v } return retHdr } // Helper to determine when to sample. When header has a value, sampling is driven by header func shouldSample(l *serviceLatency, c *client) (bool, http.Header) { if l == nil { return false, nil } if l.sampling < 0 { return false, nil } if l.sampling >= 100 { return true, nil } if l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) { return true, nil } h := c.parseState.getHeader() if len(h) == 0 { return false, nil } if tId := h[trcUber]; len(tId) != 0 { // sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1 tk := strings.Split(tId[0], ":") if len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 { dst := [2]byte{} src := [2]byte{'0', tk[3][0]} if len(tk[3]) == 2 { src[1] = tk[3][1] } if _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 { return true, newUberHeader(h, tId) } } return false, nil } else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == "1" { return true, newB3Header(h) // allowed } else if len(sampled) != 0 && sampled[0] == "0" { return false, nil // denied } else if _, ok := h[trcB3Id]; ok { // sample 80f198ee56343ba864fe8b2a57d3eff7 // presence (with X-B3-Sampled not being 0) means sampling left to recipient return true, newB3Header(h) } else if b3 := h[trcB3]; len(b3) != 0 { // sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90 // sample 0 tk := strings.Split(b3[0], "-") if len(tk) > 2 && tk[2] == "0" { return false, nil // denied } else if len(tk) == 1 && tk[0] == "0" { return false, nil // denied } return true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header } else if tId := h[trcCtx]; len(tId) != 0 { // sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 tk := strings.Split(tId[0], "-") if len(tk) == 4 && len([]byte(tk[3])) == 2 && tk[3] == "01" { return true, newTraceCtxHeader(h, tId) } else { return false, nil } } return false, nil } // Used to mimic client like replies. const ( replyPrefix = "_R_." replyPrefixLen = len(replyPrefix) baseServerLen = 10 replyLen = 6 minReplyLen = 15 digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" base = 62 ) // This is where all service export responses are handled. func (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { a.mu.RLock() if a.expired.Load() || len(a.exports.responses) == 0 { a.mu.RUnlock() return } si := a.exports.responses[subject] if si == nil || si.invalid { a.mu.RUnlock() return } a.mu.RUnlock() // Send for normal processing. c.processServiceImport(si, a, msg) } // Will create the response prefix for fast generation of responses. // A wildcard subscription may be used handle interest graph propagation // for all service replies, unless we are bound to a leafnode. // Lock should be held. func (a *Account) createRespWildcard() { var b = [baseServerLen]byte{'_', 'R', '_', '.'} rn := fastrand.Uint64() for i, l := replyPrefixLen, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } a.siReply = append(b[:], '.') } // Test whether this is a tracked reply. func isTrackedReply(reply []byte) bool { lreply := len(reply) - 1 return lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T' } // Generate a new service reply from the wildcard prefix. // FIXME(dlc) - probably do not have to use rand here. about 25ns per. func (a *Account) newServiceReply(tracking bool) []byte { a.mu.Lock() s := a.srv rn := fastrand.Uint64() // Check if we need to create the reply here. var createdSiReply bool if a.siReply == nil { a.createRespWildcard() createdSiReply = true } replyPre := a.siReply a.mu.Unlock() // If we created the siReply and we are not bound to a leafnode // we need to do the wildcard subscription. if createdSiReply { a.subscribeServiceImportResponse(string(append(replyPre, '>'))) } var b [replyLen]byte for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } // Make sure to copy. reply := make([]byte, 0, len(replyPre)+len(b)) reply = append(reply, replyPre...) reply = append(reply, b[:]...) if tracking && s.sys != nil { // Add in our tracking identifier. This allows the metrics to get back to only // this server without needless SUBS/UNSUBS. reply = append(reply, '.') reply = append(reply, s.sys.shash...) reply = append(reply, '.', 'T') } return reply } // Checks if a serviceImport was created to map responses. func (si *serviceImport) isRespServiceImport() bool { return si != nil && si.response } // Sets the response threshold timer for a service export. // Account lock should be held func (se *serviceExport) setResponseThresholdTimer() { if se.rtmr != nil { return // Already set } se.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses) } // Account lock should be held func (se *serviceExport) clearResponseThresholdTimer() bool { if se.rtmr == nil { return true } stopped := se.rtmr.Stop() se.rtmr = nil return stopped } // checkExpiredResponses will check for any pending responses that need to // be cleaned up. func (se *serviceExport) checkExpiredResponses() { acc := se.acc if acc == nil { se.clearResponseThresholdTimer() return } var expired []*serviceImport mints := time.Now().UnixNano() - int64(se.respThresh) // TODO(dlc) - Should we release lock while doing this? Or only do these in batches? // Should we break this up for responses only from this service export? // Responses live on acc directly for fast inbound processsing for the _R_ wildcard. // We could do another indirection at this level but just to get to the service export? var totalResponses int acc.mu.RLock() for _, si := range acc.exports.responses { if si.se == se { totalResponses++ if si.ts <= mints { expired = append(expired, si) } } } acc.mu.RUnlock() for _, si := range expired { acc.removeRespServiceImport(si, rsiTimeout) } // Pull out expired to determine if we have any left for timer. totalResponses -= len(expired) // Redo timer as needed. acc.mu.Lock() if totalResponses > 0 && se.rtmr != nil { se.rtmr.Stop() se.rtmr.Reset(se.respThresh) } else { se.clearResponseThresholdTimer() } acc.mu.Unlock() } // ServiceExportResponseThreshold returns the current threshold. func (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) { a.mu.Lock() defer a.mu.Unlock() se := a.getServiceExport(export) if se == nil { return 0, fmt.Errorf("no export defined for %q", export) } return se.respThresh, nil } // SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered // from a service export responder. func (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error { a.mu.Lock() if a.isClaimAccount() { a.mu.Unlock() return fmt.Errorf("claim based accounts can not be updated directly") } lrt := a.lowestServiceExportResponseTime() se := a.getServiceExport(export) if se == nil { a.mu.Unlock() return fmt.Errorf("no export defined for %q", export) } se.respThresh = maxTime var clients []*client nlrt := a.lowestServiceExportResponseTime() if nlrt != lrt && len(a.clients) > 0 { clients = a.getClientsLocked() } // Need to release because lock ordering is client -> Account a.mu.Unlock() if len(clients) > 0 { updateAllClientsServiceExportResponseTime(clients, nlrt) } return nil } // This is for internal service import responses. func (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport { nrr := string(osi.acc.newServiceReply(tracking)) a.mu.Lock() rt := osi.rt // dest is the requestor's account. a is the service responder with the export. // Marked as internal here, that is how we distinguish. si := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, nil} if a.exports.responses == nil { a.exports.responses = make(map[string]*serviceImport) } a.exports.responses[nrr] = si // Always grab time and make sure response threshold timer is running. si.ts = time.Now().UnixNano() if osi.se != nil { osi.se.setResponseThresholdTimer() } if rt == Singleton && tracking { si.latency = osi.latency si.tracking = true si.trackingHdr = header } a.mu.Unlock() // We do add in the reverse map such that we can detect loss of interest and do proper // cleanup of this si as interest goes away. dest.addReverseRespMapEntry(a, to, nrr) return si } // AddStreamImportWithClaim will add in the stream import from a specific account with optional token. func (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error { if account == nil { return ErrMissingAccount } // First check to see if the account has authorized export of the subject. if !account.checkStreamImportAuthorized(a, from, imClaim) { return ErrStreamImportAuthorization } // Check prefix if it exists and make sure its a literal. // Append token separator if not already present. if prefix != _EMPTY_ { // Make sure there are no wildcards here, this prefix needs to be a literal // since it will be prepended to a publish subject. if !subjectIsLiteral(prefix) { return ErrStreamImportBadPrefix } if prefix[len(prefix)-1] != btsep { prefix = prefix + string(btsep) } } return a.AddMappedStreamImportWithClaim(account, from, prefix+from, imClaim) } // AddMappedStreamImport helper for AddMappedStreamImportWithClaim func (a *Account) AddMappedStreamImport(account *Account, from, to string) error { return a.AddMappedStreamImportWithClaim(account, from, to, nil) } // AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token. func (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error { if account == nil { return ErrMissingAccount } // First check to see if the account has authorized export of the subject. if !account.checkStreamImportAuthorized(a, from, imClaim) { return ErrStreamImportAuthorization } if to == _EMPTY_ { to = from } // Check if this forms a cycle. if err := a.streamImportFormsCycle(account, to); err != nil { return err } if err := a.streamImportFormsCycle(account, from); err != nil { return err } var ( usePub bool tr *subjectTransform err error ) if subjectHasWildcard(from) { if to == from { usePub = true } else { // Create a transform if tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil { return fmt.Errorf("failed to create mapping transform for stream import subject from %q to %q: %v", from, to, err) } to, _ = transformUntokenize(to) } } a.mu.Lock() if a.isStreamImportDuplicate(account, from) { a.mu.Unlock() return ErrStreamImportDuplicate } a.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false}) a.mu.Unlock() return nil } // isStreamImportDuplicate checks for duplicate. // Lock should be held. func (a *Account) isStreamImportDuplicate(acc *Account, from string) bool { for _, si := range a.imports.streams { if si.acc == acc && si.from == from { return true } } return false } // AddStreamImport will add in the stream import from a specific account. func (a *Account) AddStreamImport(account *Account, from, prefix string) error { return a.AddStreamImportWithClaim(account, from, prefix, nil) } // IsPublicExport is a placeholder to denote a public export. var IsPublicExport = []*Account(nil) // AddStreamExport will add an export to the account. If accounts is nil // it will signify a public export, meaning anyone can import. func (a *Account) AddStreamExport(subject string, accounts []*Account) error { return a.addStreamExportWithAccountPos(subject, accounts, 0) } // AddStreamExport will add an export to the account. If accounts is nil // it will signify a public export, meaning anyone can import. // if accountPos is > 0, all imports will be granted where the following holds: // strings.Split(subject, tsep)[accountPos] == account id will be granted. func (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error { if a == nil { return ErrMissingAccount } a.mu.Lock() defer a.mu.Unlock() if a.exports.streams == nil { a.exports.streams = make(map[string]*streamExport) } ea := a.exports.streams[subject] if accounts != nil || accountPos > 0 { if ea == nil { ea = &streamExport{} } if err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil { return err } } a.exports.streams[subject] = ea return nil } // Check if another account is authorized to import from us. func (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { // Find the subject in the exports list. a.mu.RLock() auth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim) a.mu.RUnlock() return auth } func (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { if a.exports.streams == nil || !IsValidSubject(subject) { return false } return a.checkStreamExportApproved(account, subject, imClaim) } func (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool { // if ea is nil or ea.approved is nil, that denotes a public export if ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) { return true } // Check if the export is protected and enforces presence of importing account identity if ea.accountPos > 0 { return ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name } // Check if token required if ea.tokenReq { return a.checkActivation(account, imClaim, ea, true) } if ea.approved == nil { return false } // If we have a matching account we are authorized _, ok := ea.approved[account.Name] return ok } func (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { // Check direct match of subject first ea, ok := a.exports.streams[subject] if ok { // if ea is nil or eq.approved is nil, that denotes a public export if ea == nil { return true } return a.checkAuth(&ea.exportAuth, account, imClaim, nil) } // ok if we are here we did not match directly so we need to test each one. // The import subject arg has to take precedence, meaning the export // has to be a true subset of the import claim. We already checked for // exact matches above. tokens := strings.Split(subject, tsep) for subj, ea := range a.exports.streams { if isSubsetMatch(tokens, subj) { if ea == nil { return true } return a.checkAuth(&ea.exportAuth, account, imClaim, tokens) } } return false } func (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool { // Check direct match of subject first se, ok := a.exports.services[subject] if ok { // if se is nil or eq.approved is nil, that denotes a public export if se == nil { return true } return a.checkAuth(&se.exportAuth, account, imClaim, nil) } // ok if we are here we did not match directly so we need to test each one. // The import subject arg has to take precedence, meaning the export // has to be a true subset of the import claim. We already checked for // exact matches above. tokens := strings.Split(subject, tsep) for subj, se := range a.exports.services { if isSubsetMatch(tokens, subj) { if se == nil { return true } return a.checkAuth(&se.exportAuth, account, imClaim, tokens) } } return false } // Helper function to get a serviceExport. // Lock should be held on entry. func (a *Account) getServiceExport(subj string) *serviceExport { se, ok := a.exports.services[subj] // The export probably has a wildcard, so lookup that up. if !ok { se = a.getWildcardServiceExport(subj) } return se } // This helper is used when trying to match a serviceExport record that is // represented by a wildcard. // Lock should be held on entry. func (a *Account) getWildcardServiceExport(from string) *serviceExport { tokens := strings.Split(from, tsep) for subj, se := range a.exports.services { if isSubsetMatch(tokens, subj) { return se } } return nil } // These are import stream specific versions for when an activation expires. func (a *Account) streamActivationExpired(exportAcc *Account, subject string) { a.mu.RLock() if a.expired.Load() || a.imports.streams == nil { a.mu.RUnlock() return } var si *streamImport for _, si = range a.imports.streams { if si.acc == exportAcc && si.from == subject { break } } if si == nil || si.invalid { a.mu.RUnlock() return } a.mu.RUnlock() if si.acc.checkActivation(a, si.claim, nil, false) { // The token has been updated most likely and we are good to go. return } a.mu.Lock() si.invalid = true clients := a.getClientsLocked() awcsti := map[string]struct{}{a.Name: {}} a.mu.Unlock() for _, c := range clients { c.processSubsOnConfigReload(awcsti) } } // These are import service specific versions for when an activation expires. func (a *Account) serviceActivationExpired(subject string) { a.mu.RLock() if a.expired.Load() || a.imports.services == nil { a.mu.RUnlock() return } si := a.imports.services[subject] if si == nil || si.invalid { a.mu.RUnlock() return } a.mu.RUnlock() if si.acc.checkActivation(a, si.claim, nil, false) { // The token has been updated most likely and we are good to go. return } a.mu.Lock() si.invalid = true a.mu.Unlock() } // Fires for expired activation tokens. We could track this with timers etc. // Instead we just re-analyze where we are and if we need to act. func (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) { switch kind { case jwt.Stream: a.streamActivationExpired(exportAcc, subject) case jwt.Service: a.serviceActivationExpired(subject) } } func isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool { if len(revocations) == 0 { return false } if t, ok := revocations[subject]; !ok || t < issuedAt { if t, ok := revocations[jwt.All]; !ok || t < issuedAt { return false } } return true } // checkActivation will check the activation token for validity. // ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer. func (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool { if claim == nil || claim.Token == _EMPTY_ { return false } // Create a quick clone so we can inline Token JWT. clone := *claim vr := jwt.CreateValidationResults() clone.Validate(importAcc.Name, vr) if vr.IsBlocking(true) { return false } act, err := jwt.DecodeActivationClaims(clone.Token) if err != nil { return false } if !a.isIssuerClaimTrusted(act) { return false } vr = jwt.CreateValidationResults() act.Validate(vr) if vr.IsBlocking(true) { return false } if act.Expires != 0 { tn := time.Now().Unix() if act.Expires <= tn { return false } if expTimer { expiresAt := time.Duration(act.Expires - tn) time.AfterFunc(expiresAt*time.Second, func() { importAcc.activationExpired(a, string(act.ImportSubject), claim.Type) }) } } if ea == nil { return true } // Check for token revocation.. return !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt) } // Returns true if the activation claim is trusted. That is the issuer matches // the account or is an entry in the signing keys. func (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool { // if no issuer account, issuer is the account if claims.IssuerAccount == _EMPTY_ { return true } // If the IssuerAccount is not us, then this is considered an error. if a.Name != claims.IssuerAccount { if a.srv != nil { a.srv.Errorf("Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q", claims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name) } return false } _, ok := a.hasIssuerNoLock(claims.Issuer) return ok } // Returns true if `a` and `b` stream imports are the same. Note that the // check is done with the account's name, not the pointer. This is used // during config reload where we are comparing current and new config // in which pointers are different. // Acquires `a` read lock, but `b` is assumed to not be accessed // by anyone but the caller (`b` is not registered anywhere). func (a *Account) checkStreamImportsEqual(b *Account) bool { a.mu.RLock() defer a.mu.RUnlock() if len(a.imports.streams) != len(b.imports.streams) { return false } // Load the b imports into a map index by what we are looking for. bm := make(map[string]*streamImport, len(b.imports.streams)) for _, bim := range b.imports.streams { bm[bim.acc.Name+bim.from+bim.to] = bim } for _, aim := range a.imports.streams { if _, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok { return false } } return true } // Returns true if `a` and `b` stream exports are the same. // Acquires `a` read lock, but `b` is assumed to not be accessed // by anyone but the caller (`b` is not registered anywhere). func (a *Account) checkStreamExportsEqual(b *Account) bool { a.mu.RLock() defer a.mu.RUnlock() if len(a.exports.streams) != len(b.exports.streams) { return false } for subj, aea := range a.exports.streams { bea, ok := b.exports.streams[subj] if !ok { return false } if !isStreamExportEqual(aea, bea) { return false } } return true } func isStreamExportEqual(a, b *streamExport) bool { if a == nil && b == nil { return true } if (a == nil && b != nil) || (a != nil && b == nil) { return false } return isExportAuthEqual(&a.exportAuth, &b.exportAuth) } // Returns true if `a` and `b` service exports are the same. // Acquires `a` read lock, but `b` is assumed to not be accessed // by anyone but the caller (`b` is not registered anywhere). func (a *Account) checkServiceExportsEqual(b *Account) bool { a.mu.RLock() defer a.mu.RUnlock() if len(a.exports.services) != len(b.exports.services) { return false } for subj, aea := range a.exports.services { bea, ok := b.exports.services[subj] if !ok { return false } if !isServiceExportEqual(aea, bea) { return false } } return true } func isServiceExportEqual(a, b *serviceExport) bool { if a == nil && b == nil { return true } if (a == nil && b != nil) || (a != nil && b == nil) { return false } if !isExportAuthEqual(&a.exportAuth, &b.exportAuth) { return false } if a.acc.Name != b.acc.Name { return false } if a.respType != b.respType { return false } if a.latency != nil || b.latency != nil { if (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) { return false } if a.latency.sampling != b.latency.sampling { return false } if a.latency.subject != b.latency.subject { return false } } return true } // Returns true if `a` and `b` exportAuth structures are equal. // Both `a` and `b` are guaranteed to be non-nil. // Locking is handled by the caller. func isExportAuthEqual(a, b *exportAuth) bool { if a.tokenReq != b.tokenReq { return false } if a.accountPos != b.accountPos { return false } if len(a.approved) != len(b.approved) { return false } for ak, av := range a.approved { if bv, ok := b.approved[ak]; !ok || av.Name != bv.Name { return false } } if len(a.actsRevoked) != len(b.actsRevoked) { return false } for ak, av := range a.actsRevoked { if bv, ok := b.actsRevoked[ak]; !ok || av != bv { return false } } return true } // Check if another account is authorized to route requests to this service. func (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool { a.mu.RLock() authorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim) a.mu.RUnlock() return authorized } // Check if another account is authorized to route requests to this service. func (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool { // Find the subject in the services list. if a.exports.services == nil { return false } return a.checkServiceExportApproved(account, subject, imClaim) } // IsExpired returns expiration status. func (a *Account) IsExpired() bool { return a.expired.Load() } // Called when an account has expired. func (a *Account) expiredTimeout() { // Mark expired first. a.expired.Store(true) // Collect the clients and expire them. cs := a.getClients() for _, c := range cs { c.accountAuthExpired() } } // Sets the expiration timer for an account JWT that has it set. func (a *Account) setExpirationTimer(d time.Duration) { a.etmr = time.AfterFunc(d, a.expiredTimeout) } // Lock should be held func (a *Account) clearExpirationTimer() bool { if a.etmr == nil { return true } stopped := a.etmr.Stop() a.etmr = nil return stopped } // checkUserRevoked will check if a user has been revoked. func (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool { a.mu.RLock() defer a.mu.RUnlock() return isRevoked(a.usersRevoked, nkey, issuedAt) } // failBearer will return if bearer token are allowed (false) or disallowed (true) func (a *Account) failBearer() bool { a.mu.RLock() defer a.mu.RUnlock() return a.disallowBearer } // Check expiration and set the proper state as needed. func (a *Account) checkExpiration(claims *jwt.ClaimsData) { a.mu.Lock() defer a.mu.Unlock() a.clearExpirationTimer() if claims.Expires == 0 { a.expired.Store(false) return } tn := time.Now().Unix() if claims.Expires <= tn { a.expired.Store(true) return } expiresAt := time.Duration(claims.Expires - tn) a.setExpirationTimer(expiresAt * time.Second) a.expired.Store(false) } // hasIssuer returns true if the issuer matches the account // If the issuer is a scoped signing key, the scope will be returned as well // issuer or it is a signing key for the account. func (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) { a.mu.RLock() scope, ok := a.hasIssuerNoLock(issuer) a.mu.RUnlock() return scope, ok } // hasIssuerNoLock is the unlocked version of hasIssuer func (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) { scope, ok := a.signingKeys[issuer] return scope, ok } // Returns the loop detection subject used for leafnodes func (a *Account) getLDSubject() string { a.mu.RLock() lds := a.lds a.mu.RUnlock() return lds } // Placeholder for signaling token auth required. var tokenAuthReq = []*Account{} func authAccounts(tokenReq bool) []*Account { if tokenReq { return tokenAuthReq } return nil } // SetAccountResolver will assign the account resolver. func (s *Server) SetAccountResolver(ar AccountResolver) { s.mu.Lock() s.accResolver = ar s.mu.Unlock() } // AccountResolver returns the registered account resolver. func (s *Server) AccountResolver() AccountResolver { s.mu.RLock() ar := s.accResolver s.mu.RUnlock() return ar } // isClaimAccount returns if this account is backed by a JWT claim. // Lock should be held. func (a *Account) isClaimAccount() bool { return a.claimJWT != _EMPTY_ } // UpdateAccountClaims will update an existing account with new claims. // This will replace any exports or imports previously defined. // Lock MUST NOT be held upon entry. func (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) { s.updateAccountClaimsWithRefresh(a, ac, true) } func (a *Account) traceLabel() string { if a == nil { return _EMPTY_ } if a.nameTag != _EMPTY_ { return fmt.Sprintf("%s/%s", a.Name, a.nameTag) } return a.Name } // Check if an account has external auth set. // Operator/Account Resolver only. func (a *Account) hasExternalAuth() bool { if a == nil { return false } a.mu.RLock() defer a.mu.RUnlock() return a.extAuth != nil } // Deterimine if this is an external auth user. func (a *Account) isExternalAuthUser(userID string) bool { if a == nil { return false } a.mu.RLock() defer a.mu.RUnlock() if a.extAuth != nil { for _, u := range a.extAuth.AuthUsers { if userID == u { return true } } } return false } // Return the external authorization xkey if external authorization is enabled and the xkey is set. // Operator/Account Resolver only. func (a *Account) externalAuthXKey() string { if a == nil { return _EMPTY_ } a.mu.RLock() defer a.mu.RUnlock() if a.extAuth != nil && a.extAuth.XKey != _EMPTY_ { return a.extAuth.XKey } return _EMPTY_ } // Check if an account switch for external authorization is allowed. func (a *Account) isAllowedAcount(acc string) bool { if a == nil { return false } a.mu.RLock() defer a.mu.RUnlock() if a.extAuth != nil { // if we have a single allowed account, and we have a wildcard // we accept it if len(a.extAuth.AllowedAccounts) == 1 && a.extAuth.AllowedAccounts[0] == jwt.AnyAccount { return true } // otherwise must match exactly for _, a := range a.extAuth.AllowedAccounts { if a == acc { return true } } } return false } // updateAccountClaimsWithRefresh will update an existing account with new claims. // If refreshImportingAccounts is true it will also update incomplete dependent accounts // This will replace any exports or imports previously defined. // Lock MUST NOT be held upon entry. func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) { if a == nil { return } s.Debugf("Updating account claims: %s/%s", a.Name, ac.Name) a.checkExpiration(ac.Claims()) a.mu.Lock() // Clone to update, only select certain fields. old := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys} // overwrite claim meta data a.nameTag = ac.Name a.tags = ac.Tags // Grab trace label under lock. tl := a.traceLabel() // Check for external authorization. if ac.HasExternalAuthorization() { a.extAuth = &jwt.ExternalAuthorization{} a.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...) a.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...) a.extAuth.XKey = ac.Authorization.XKey } // Reset exports and imports here. // Exports is creating a whole new map. a.exports = exportMap{} // Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline. if a.imports.streams != nil { old.imports.streams = a.imports.streams a.imports.streams = nil } if a.imports.services != nil { old.imports.services = make(map[string]*serviceImport, len(a.imports.services)) for k, v := range a.imports.services { old.imports.services[k] = v delete(a.imports.services, k) } } alteredScope := map[string]struct{}{} // update account signing keys a.signingKeys = nil _, strict := s.strictSigningKeyUsage[a.Issuer] if len(ac.SigningKeys) > 0 || !strict { a.signingKeys = make(map[string]jwt.Scope) } signersChanged := false for k, scope := range ac.SigningKeys { a.signingKeys[k] = scope } if !strict { a.signingKeys[a.Name] = nil } if len(a.signingKeys) != len(old.signingKeys) { signersChanged = true } for k, scope := range a.signingKeys { if oldScope, ok := old.signingKeys[k]; !ok { signersChanged = true } else if !reflect.DeepEqual(scope, oldScope) { signersChanged = true alteredScope[k] = struct{}{} } } // collect mappings that need to be removed removeList := []string{} for _, m := range a.mappings { if _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok { removeList = append(removeList, m.src) } } a.mu.Unlock() for sub, wm := range ac.Mappings { mappings := make([]*MapDest, len(wm)) for i, m := range wm { mappings[i] = &MapDest{ Subject: string(m.Subject), Weight: m.GetWeight(), Cluster: m.Cluster, } } // This will overwrite existing entries a.AddWeightedMappings(string(sub), mappings...) } // remove mappings for _, rmMapping := range removeList { a.RemoveMapping(rmMapping) } // Re-register system exports/imports. if a == s.SystemAccount() { s.addSystemAccountExports(a) } else { s.registerSystemImports(a) } jsEnabled := s.JetStreamEnabled() streamTokenExpirationChanged := false serviceTokenExpirationChanged := false for _, e := range ac.Exports { switch e.Type { case jwt.Stream: s.Debugf("Adding stream export %q for %s", e.Subject, tl) if err := a.addStreamExportWithAccountPos( string(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { s.Debugf("Error adding stream export to account [%s]: %v", tl, err.Error()) } case jwt.Service: s.Debugf("Adding service export %q for %s", e.Subject, tl) rt := Singleton switch e.ResponseType { case jwt.ResponseTypeStream: rt = Streamed case jwt.ResponseTypeChunked: rt = Chunked } if err := a.addServiceExportWithResponseAndAccountPos( string(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil { s.Debugf("Error adding service export to account [%s]: %v", tl, err) continue } sub := string(e.Subject) if e.Latency != nil { if err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil { hdrNote := _EMPTY_ if e.Latency.Sampling == jwt.Headers { hdrNote = " (using headers)" } s.Debugf("Error adding latency tracking%s for service export to account [%s]: %v", hdrNote, tl, err) } } if e.ResponseThreshold != 0 { // Response threshold was set in options. if err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil { s.Debugf("Error adding service export response threshold for [%s]: %v", tl, err) } } } var revocationChanged *bool var ea *exportAuth a.mu.Lock() switch e.Type { case jwt.Stream: revocationChanged = &streamTokenExpirationChanged if se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil { ea = &se.exportAuth } case jwt.Service: revocationChanged = &serviceTokenExpirationChanged if se, ok := a.exports.services[string(e.Subject)]; ok && se != nil { ea = &se.exportAuth } } if ea != nil { oldRevocations := ea.actsRevoked if len(e.Revocations) == 0 { // remove all, no need to evaluate existing imports ea.actsRevoked = nil } else if len(oldRevocations) == 0 { // add all, existing imports need to be re evaluated ea.actsRevoked = e.Revocations *revocationChanged = true } else { ea.actsRevoked = e.Revocations // diff, existing imports need to be conditionally re evaluated, depending on: // if a key was added, or it's timestamp increased for k, t := range e.Revocations { if tOld, ok := oldRevocations[k]; !ok || tOld < t { *revocationChanged = true } } } } a.mu.Unlock() } var incompleteImports []*jwt.Import for _, i := range ac.Imports { acc, err := s.lookupAccount(i.Account) if acc == nil || err != nil { s.Errorf("Can't locate account [%s] for import of [%v] %s (err=%v)", i.Account, i.Subject, i.Type, err) incompleteImports = append(incompleteImports, i) continue } // Capture trace labels. acc.mu.RLock() atl := acc.traceLabel() acc.mu.RUnlock() // Grab from and to from, to := string(i.Subject), i.GetTo() switch i.Type { case jwt.Stream: if i.LocalSubject != _EMPTY_ { // set local subject implies to is empty to = string(i.LocalSubject) s.Debugf("Adding stream import %s:%q for %s:%q", atl, from, tl, to) err = a.AddMappedStreamImportWithClaim(acc, from, to, i) } else { s.Debugf("Adding stream import %s:%q for %s:%q", atl, from, tl, to) err = a.AddStreamImportWithClaim(acc, from, to, i) } if err != nil { s.Debugf("Error adding stream import to account [%s]: %v", tl, err.Error()) incompleteImports = append(incompleteImports, i) } case jwt.Service: if i.LocalSubject != _EMPTY_ { from = string(i.LocalSubject) to = string(i.Subject) } s.Debugf("Adding service import %s:%q for %s:%q", atl, from, tl, to) if err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil { s.Debugf("Error adding service import to account [%s]: %v", tl, err.Error()) incompleteImports = append(incompleteImports, i) } } } // Now let's apply any needed changes from import/export changes. if !a.checkStreamImportsEqual(old) { awcsti := map[string]struct{}{a.Name: {}} for _, c := range a.getClients() { c.processSubsOnConfigReload(awcsti) } } // Now check if stream exports have changed. if !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged { clients := map[*client]struct{}{} // We need to check all accounts that have an import claim from this account. awcsti := map[string]struct{}{} s.accounts.Range(func(k, v any) bool { acc := v.(*Account) // Move to the next if this account is actually account "a". if acc.Name == a.Name { return true } // TODO: checkStreamImportAuthorized() stack should not be trying // to lock "acc". If we find that to be needed, we will need to // rework this to ensure we don't lock acc. acc.mu.Lock() for _, im := range acc.imports.streams { if im != nil && im.acc.Name == a.Name { // Check for if we are still authorized for an import. im.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim) awcsti[acc.Name] = struct{}{} for c := range acc.clients { clients[c] = struct{}{} } } } acc.mu.Unlock() return true }) // Now walk clients. for c := range clients { c.processSubsOnConfigReload(awcsti) } } // Now check if service exports have changed. if !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged { s.accounts.Range(func(k, v any) bool { acc := v.(*Account) // Move to the next if this account is actually account "a". if acc.Name == a.Name { return true } // TODO: checkServiceImportAuthorized() stack should not be trying // to lock "acc". If we find that to be needed, we will need to // rework this to ensure we don't lock acc. acc.mu.Lock() for _, si := range acc.imports.services { if si != nil && si.acc.Name == a.Name { // Check for if we are still authorized for an import. si.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim) if si.latency != nil && !si.response { // Make sure we should still be tracking latency. if se := a.getServiceExport(si.to); se != nil { si.latency = se.latency } } } } acc.mu.Unlock() return true }) } // Now make sure we shutdown the old service import subscriptions. var sids [][]byte a.mu.RLock() c := a.ic for _, si := range old.imports.services { if c != nil && si.sid != nil { sids = append(sids, si.sid) } } a.mu.RUnlock() for _, sid := range sids { c.processUnsub(sid) } // Now do limits if they are present. a.mu.Lock() a.msubs = int32(ac.Limits.Subs) a.mpay = int32(ac.Limits.Payload) a.mconns = int32(ac.Limits.Conn) a.mleafs = int32(ac.Limits.LeafNodeConn) a.disallowBearer = ac.Limits.DisallowBearer // Check for any revocations if len(ac.Revocations) > 0 { // We will always replace whatever we had with most current, so no // need to look at what we have. a.usersRevoked = make(map[string]int64, len(ac.Revocations)) for pk, t := range ac.Revocations { a.usersRevoked[pk] = t } } else { a.usersRevoked = nil } a.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions) a.incomplete = len(incompleteImports) != 0 for _, i := range incompleteImports { s.incompleteAccExporterMap.Store(i.Account, struct{}{}) } if a.srv == nil { a.srv = s } if ac.Limits.IsJSEnabled() { toUnlimited := func(value int64) int64 { if value > 0 { return value } return -1 } if ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 { // JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited a.jsLimits = map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: ac.Limits.JetStreamLimits.MemoryStorage, MaxStore: ac.Limits.JetStreamLimits.DiskStorage, MaxStreams: int(ac.Limits.JetStreamLimits.Streams), MaxConsumers: int(ac.Limits.JetStreamLimits.Consumer), MemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes), StoreMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes), MaxBytesRequired: ac.Limits.JetStreamLimits.MaxBytesRequired, MaxAckPending: int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)), }, } } else { a.jsLimits = map[string]JetStreamAccountLimits{} for t, l := range ac.Limits.JetStreamTieredLimits { a.jsLimits[t] = JetStreamAccountLimits{ MaxMemory: l.MemoryStorage, MaxStore: l.DiskStorage, MaxStreams: int(l.Streams), MaxConsumers: int(l.Consumer), MemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes), StoreMaxStreamBytes: toUnlimited(l.DiskMaxStreamBytes), MaxBytesRequired: l.MaxBytesRequired, MaxAckPending: int(toUnlimited(l.MaxAckPending)), } } } } else if a.jsLimits != nil { // covers failed update followed by disable a.jsLimits = nil } a.updated = time.Now() clients := a.getClientsLocked() a.mu.Unlock() // Sort if we are over the limit. if a.MaxTotalConnectionsReached() { slices.SortFunc(clients, func(i, j *client) int { return -i.start.Compare(j.start) }) // sort in reverse order } // If JetStream is enabled for this server we will call into configJetStream for the account // regardless of enabled or disabled. It handles both cases. if jsEnabled { if err := s.configJetStream(a); err != nil { s.Errorf("Error configuring jetstream for account [%s]: %v", tl, err.Error()) a.mu.Lock() // Absent reload of js server cfg, this is going to be broken until js is disabled a.incomplete = true a.mu.Unlock() } } else if a.jsLimits != nil { // We do not have JS enabled for this server, but the account has it enabled so setup // our imports properly. This allows this server to proxy JS traffic correctly. s.checkJetStreamExports() a.enableAllJetStreamServiceImportsAndMappings() } for i, c := range clients { a.mu.RLock() exceeded := a.mconns != jwt.NoLimit && i >= int(a.mconns) a.mu.RUnlock() if exceeded { c.maxAccountConnExceeded() continue } c.mu.Lock() c.applyAccountLimits() // if we have an nkey user we are a callout user - save // the issuedAt, and nkey user id to honor revocations var nkeyUserID string var issuedAt int64 if c.user != nil { issuedAt = c.user.Issued nkeyUserID = c.user.Nkey } theJWT := c.opts.JWT c.mu.Unlock() // Check for being revoked here. We use ac one to avoid the account lock. if (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ { if juc, err := jwt.DecodeUserClaims(theJWT); err != nil { c.Debugf("User JWT not valid: %v", err) c.authViolation() continue } else if juc.BearerToken && ac.Limits.DisallowBearer { c.Debugf("Bearer User JWT not allowed") c.authViolation() continue } else if ok := ac.IsClaimRevoked(juc); ok { c.sendErrAndDebug("User Authentication Revoked") c.closeConnection(Revocation) continue } } // if we extracted nkeyUserID and issuedAt we are a callout type // calloutIAT should only be set if we are in callout scenario as // the user JWT is _NOT_ associated with the client for callouts, // so we rely on the calloutIAT to know when the JWT was issued // revocations simply state that JWT issued before or by that date // are not valid if ac.Revocations != nil && nkeyUserID != _EMPTY_ && issuedAt > 0 { seconds, ok := ac.Revocations[jwt.All] if ok && seconds >= issuedAt { c.sendErrAndDebug("User Authentication Revoked") c.closeConnection(Revocation) continue } seconds, ok = ac.Revocations[nkeyUserID] if ok && seconds >= issuedAt { c.sendErrAndDebug("User Authentication Revoked") c.closeConnection(Revocation) continue } } } // Check if the signing keys changed, might have to evict if signersChanged { for _, c := range clients { c.mu.Lock() if c.user == nil { c.mu.Unlock() continue } sk := c.user.SigningKey c.mu.Unlock() if sk == _EMPTY_ { continue } if _, ok := alteredScope[sk]; ok { c.closeConnection(AuthenticationViolation) } else if _, ok := a.hasIssuer(sk); !ok { c.closeConnection(AuthenticationViolation) } } } if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts { s.incompleteAccExporterMap.Delete(old.Name) s.accounts.Range(func(key, value any) bool { acc := value.(*Account) acc.mu.RLock() incomplete := acc.incomplete name := acc.Name label := acc.traceLabel() // Must use jwt in account or risk failing on fetch // This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap claimJWT := acc.claimJWT acc.mu.RUnlock() if incomplete && name != old.Name { if accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil { // Since claimJWT has not changed, acc can become complete // but it won't alter incomplete for it's dependents accounts. s.updateAccountClaimsWithRefresh(acc, accClaims, false) // old.Name was deleted before ranging over accounts // If it exists again, UpdateAccountClaims set it for failed imports of acc. // So there was one import of acc that imported this account and failed again. // Since this account just got updated, the import itself may be in error. So trace that. if _, ok := s.incompleteAccExporterMap.Load(old.Name); ok { s.incompleteAccExporterMap.Delete(old.Name) s.Errorf("Account %s has issues importing account %s", label, old.Name) } } } return true }) } } // Helper to build an internal account structure from a jwt.AccountClaims. // Lock MUST NOT be held upon entry. func (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account { acc := NewAccount(ac.Subject) acc.Issuer = ac.Issuer // Set this here since we are placing in s.tmpAccounts below and may be // referenced by an route RS+, etc. s.setAccountSublist(acc) // We don't want to register an account that is in the process of // being built, however, to solve circular import dependencies, we // need to store it here. if v, loaded := s.tmpAccounts.LoadOrStore(ac.Subject, acc); loaded { return v.(*Account) } // Update based on claims. s.UpdateAccountClaims(acc, ac) return acc } // Helper to build Permissions from jwt.Permissions // or return nil if none were specified func buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions { if uc == nil { return nil } var p *Permissions if len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 { p = &Permissions{} p.Publish = &SubjectPermission{} p.Publish.Allow = uc.Pub.Allow p.Publish.Deny = uc.Pub.Deny } if len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 { if p == nil { p = &Permissions{} } p.Subscribe = &SubjectPermission{} p.Subscribe.Allow = uc.Sub.Allow p.Subscribe.Deny = uc.Sub.Deny } if uc.Resp != nil { if p == nil { p = &Permissions{} } p.Response = &ResponsePermission{ MaxMsgs: uc.Resp.MaxMsgs, Expires: uc.Resp.Expires, } validateResponsePermissions(p) } return p } // Helper to build internal NKeyUser. func buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser { nu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts, Issued: uc.IssuedAt} if uc.IssuerAccount != _EMPTY_ { nu.SigningKey = uc.Issuer } // Now check for permissions. var p = buildPermissionsFromJwt(&uc.Permissions) if p == nil && acc.defaultPerms != nil { p = acc.defaultPerms.clone() } nu.Permissions = p return nu } func fetchAccount(res AccountResolver, name string) (string, error) { if !nkeys.IsValidPublicAccountKey(name) { return _EMPTY_, fmt.Errorf("will only fetch valid account keys") } return res.Fetch(copyString(name)) } // AccountResolver interface. This is to fetch Account JWTs by public nkeys type AccountResolver interface { Fetch(name string) (string, error) Store(name, jwt string) error IsReadOnly() bool Start(server *Server) error IsTrackingUpdate() bool Reload() error Close() } // Default implementations of IsReadOnly/Start so only need to be written when changed type resolverDefaultsOpsImpl struct{} func (*resolverDefaultsOpsImpl) IsReadOnly() bool { return true } func (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool { return false } func (*resolverDefaultsOpsImpl) Start(*Server) error { return nil } func (*resolverDefaultsOpsImpl) Reload() error { return nil } func (*resolverDefaultsOpsImpl) Close() { } func (*resolverDefaultsOpsImpl) Store(_, _ string) error { return fmt.Errorf("store operation not supported for URL Resolver") } // MemAccResolver is a memory only resolver. // Mostly for testing. type MemAccResolver struct { sm sync.Map resolverDefaultsOpsImpl } // Fetch will fetch the account jwt claims from the internal sync.Map. func (m *MemAccResolver) Fetch(name string) (string, error) { if j, ok := m.sm.Load(name); ok { return j.(string), nil } return _EMPTY_, ErrMissingAccount } // Store will store the account jwt claims in the internal sync.Map. func (m *MemAccResolver) Store(name, jwt string) error { m.sm.Store(name, jwt) return nil } func (m *MemAccResolver) IsReadOnly() bool { return false } // URLAccResolver implements an http fetcher. type URLAccResolver struct { url string c *http.Client resolverDefaultsOpsImpl } // NewURLAccResolver returns a new resolver for the given base URL. func NewURLAccResolver(url string) (*URLAccResolver, error) { if !strings.HasSuffix(url, "/") { url += "/" } // FIXME(dlc) - Make timeout and others configurable. // We create our own transport to amortize TLS. tr := &http.Transport{ MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, } ur := &URLAccResolver{ url: url, c: &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr}, } return ur, nil } // Fetch will fetch the account jwt claims from the base url, appending the // account name onto the end. func (ur *URLAccResolver) Fetch(name string) (string, error) { url := ur.url + name resp, err := ur.c.Get(url) if err != nil { return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), err) } else if resp == nil { return _EMPTY_, fmt.Errorf("could not fetch <%q>: no response", redactURLString(url)) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return _EMPTY_, fmt.Errorf("could not fetch <%q>: %v", redactURLString(url), resp.Status) } body, err := io.ReadAll(resp.Body) if err != nil { return _EMPTY_, err } return string(body), nil } // Resolver based on nats for synchronization and backing directory for storage. type DirAccResolver struct { *DirJWTStore *Server syncInterval time.Duration fetchTimeout time.Duration } func (dr *DirAccResolver) IsTrackingUpdate() bool { return true } func (dr *DirAccResolver) Reload() error { return dr.DirJWTStore.Reload() } func respondToUpdate(s *Server, respSubj string, acc string, message string, err error) { if err == nil { if acc == _EMPTY_ { s.Debugf("%s", message) } else { s.Debugf("%s - %s", message, acc) } } else { if acc == _EMPTY_ { s.Errorf("%s - %s", message, err) } else { s.Errorf("%s - %s - %s", message, acc, err) } } if respSubj == _EMPTY_ { return } server := &ServerInfo{} response := map[string]interface{}{"server": server} m := map[string]interface{}{} if acc != _EMPTY_ { m["account"] = acc } if err == nil { m["code"] = http.StatusOK m["message"] = message response["data"] = m } else { m["code"] = http.StatusInternalServerError m["description"] = fmt.Sprintf("%s - %v", message, err) response["error"] = m } s.sendInternalMsgLocked(respSubj, _EMPTY_, server, response) } func handleListRequest(store *DirJWTStore, s *Server, reply string) { if reply == _EMPTY_ { return } accIds := make([]string, 0, 1024) if err := store.PackWalk(1, func(partialPackMsg string) { if tk := strings.Split(partialPackMsg, "|"); len(tk) == 2 { accIds = append(accIds, tk[0]) } }); err != nil { // let them timeout s.Errorf("list request error: %v", err) } else { s.Debugf("list request responded with %d account ids", len(accIds)) server := &ServerInfo{} response := map[string]any{"server": server, "data": accIds} s.sendInternalMsgLocked(reply, _EMPTY_, server, response) } } func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) { var accIds []any var subj, sysAccName string if sysAcc := s.SystemAccount(); sysAcc != nil { sysAccName = sysAcc.GetName() } // Only operator and operator signing key are allowed to delete gk, err := jwt.DecodeGeneric(string(msg)) if err == nil { subj = gk.Subject if store.deleteType == NoDelete { err = fmt.Errorf("delete must be enabled in server config") } else if subj != gk.Issuer { err = fmt.Errorf("not self signed") } else if _, ok := store.operator[gk.Issuer]; !ok { err = fmt.Errorf("not trusted") } else if list, ok := gk.Data["accounts"]; !ok { err = fmt.Errorf("malformed request") } else if accIds, ok = list.([]any); !ok { err = fmt.Errorf("malformed request") } else { for _, entry := range accIds { if acc, ok := entry.(string); !ok || acc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) { err = fmt.Errorf("malformed request") break } else if acc == sysAccName { err = fmt.Errorf("not allowed to delete system account") break } } } } if err != nil { respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("delete accounts request by %s failed", subj), err) return } errs := []string{} passCnt := 0 for _, acc := range accIds { if err := store.delete(acc.(string)); err != nil { errs = append(errs, err.Error()) } else { passCnt++ } } if len(errs) == 0 { respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts", passCnt), nil) } else { respondToUpdate(s, reply, _EMPTY_, fmt.Sprintf("deleted %d accounts, failed for %d", passCnt, len(errs)), errors.New(strings.Join(errs, "\n"))) } } func getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) { var op string var strict bool keys := make(map[string]struct{}) if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 { op = opts.TrustedOperators[0].Subject strict = opts.TrustedOperators[0].StrictSigningKeyUsage if !strict { keys[opts.TrustedOperators[0].Subject] = struct{}{} } for _, key := range opts.TrustedOperators[0].SigningKeys { keys[key] = struct{}{} } } if len(keys) == 0 { return _EMPTY_, nil, false, fmt.Errorf("no operator key found") } return op, keys, strict, nil } func claimValidate(claim *jwt.AccountClaims) error { vr := &jwt.ValidationResults{} claim.Validate(vr) if vr.IsBlocking(false) { return fmt.Errorf("validation errors: %v", vr.Errors()) } return nil } func removeCb(s *Server, pubKey string) { v, ok := s.accounts.Load(pubKey) if !ok { return } a := v.(*Account) s.Debugf("Disable account %s due to remove", pubKey) a.mu.Lock() // lock out new clients a.msubs = 0 a.mpay = 0 a.mconns = 0 a.mleafs = 0 a.updated = time.Now() jsa := a.js a.mu.Unlock() // set the account to be expired and disconnect clients a.expiredTimeout() // For JS, we need also to disable it. if js := s.getJetStream(); js != nil && jsa != nil { js.disableJetStream(jsa) // Remove JetStream state in memory, this will be reset // on the changed callback from the account in case it is // enabled again. a.js = nil } // We also need to remove all ServerImport subscriptions a.removeAllServiceImportSubs() a.mu.Lock() a.clearExpirationTimer() a.mu.Unlock() } func (dr *DirAccResolver) Start(s *Server) error { op, opKeys, strict, err := getOperatorKeys(s) if err != nil { return err } dr.Lock() defer dr.Unlock() dr.Server = s dr.operator = opKeys dr.DirJWTStore.changed = func(pubKey string) { if v, ok := s.accounts.Load(pubKey); ok { if theJwt, err := dr.LoadAcc(pubKey); err != nil { s.Errorf("DirResolver - Update got error on load: %v", err) } else { acc := v.(*Account) if err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil { s.Errorf("DirResolver - Update for account %q resulted in error %v", pubKey, err) } else { if _, jsa, err := acc.checkForJetStream(); err != nil { if !IsNatsErr(err, JSNotEnabledForAccountErr) { s.Warnf("DirResolver - Error checking for JetStream support for account %q: %v", pubKey, err) } } else if jsa == nil { if err = s.configJetStream(acc); err != nil { s.Errorf("DirResolver - Error configuring JetStream for account %q: %v", pubKey, err) } } } } } } dr.DirJWTStore.deleted = func(pubKey string) { removeCb(s, pubKey) } packRespIb := s.newRespInbox() for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { // subscribe to account jwt update requests if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { var pubKey string tk := strings.Split(subj, tsep) if len(tk) == accUpdateTokensNew { pubKey = tk[accReqAccIndex] } else if len(tk) == accUpdateTokensOld { pubKey = tk[accUpdateAccIdxOld] } else { s.Debugf("DirResolver - jwt update skipped due to bad subject %q", subj) return } if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) } else if err := claimValidate(claim); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) } else if claim.Subject != pubKey { err := errors.New("subject does not match jwt content") respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else if claim.Issuer == op && strict { err := errors.New("operator requires issuer to be a signing key") respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else if err := dr.save(pubKey, string(msg)); err != nil { respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else { respondToUpdate(s, resp, pubKey, "jwt updated", nil) } }); err != nil { return fmt.Errorf("error setting up update handling: %v", err) } } if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { // As this is a raw message, we need to extract payload and only decode claims from it, // in case request is sent with headers. _, msg = c.msgParts(msg) if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { respondToUpdate(s, resp, "n/a", "jwt update resulted in error", err) } else if claim.Issuer == op && strict { err := errors.New("operator requires issuer to be a signing key") respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) } else if err := claimValidate(claim); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt validation failed", err) } else if err := dr.save(claim.Subject, string(msg)); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt update resulted in error", err) } else { respondToUpdate(s, resp, claim.Subject, "jwt updated", nil) } }); err != nil { return fmt.Errorf("error setting up update handling: %v", err) } // respond to lookups with our version if _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, "*"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) { if reply == _EMPTY_ { return } tk := strings.Split(subj, tsep) if len(tk) != accLookupReqTokens { return } accName := tk[accReqAccIndex] if theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil { if errors.Is(err, fs.ErrNotExist) { s.Debugf("DirResolver - Could not find account %q", accName) // Reply with empty response to signal absence of JWT to others. s.sendInternalMsgLocked(reply, _EMPTY_, nil, nil) } else { s.Errorf("DirResolver - Error looking up account %q: %v", accName, err) } } else { s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT)) } }); err != nil { return fmt.Errorf("error setting up lookup request handling: %v", err) } // respond to pack requests with one or more pack messages // an empty message signifies the end of the response responder. if _, err := s.sysSubscribeQ(accPackReqSubj, "responder", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) { if reply == _EMPTY_ { return } ourHash := dr.DirJWTStore.Hash() if bytes.Equal(theirHash, ourHash[:]) { s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) s.Debugf("DirResolver - Pack request matches hash %x", ourHash[:]) } else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) { s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg)) }); err != nil { // let them timeout s.Errorf("DirResolver - Pack request error: %v", err) } else { s.Debugf("DirResolver - Pack request hash %x - finished responding with hash %x", theirHash, ourHash) s.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{}) } }); err != nil { return fmt.Errorf("error setting up pack request handling: %v", err) } // respond to list requests with one message containing all account ids if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { handleListRequest(dr.DirJWTStore, s, reply) }); err != nil { return fmt.Errorf("error setting up list request handling: %v", err) } if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { handleDeleteRequest(dr.DirJWTStore, s, msg, reply) }); err != nil { return fmt.Errorf("error setting up delete request handling: %v", err) } // embed pack responses into store if _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) { hash := dr.DirJWTStore.Hash() if len(msg) == 0 { // end of response stream s.Debugf("DirResolver - Merging finished and resulting in: %x", dr.DirJWTStore.Hash()) return } else if err := dr.DirJWTStore.Merge(string(msg)); err != nil { s.Errorf("DirResolver - Merging resulted in error: %v", err) } else { s.Debugf("DirResolver - Merging succeeded and changed %x to %x", hash, dr.DirJWTStore.Hash()) } }); err != nil { return fmt.Errorf("error setting up pack response handling: %v", err) } // periodically send out pack message quit := s.quitCh s.startGoRoutine(func() { defer s.grWG.Done() ticker := time.NewTicker(dr.syncInterval) for { select { case <-quit: ticker.Stop() return case <-ticker.C: } ourHash := dr.DirJWTStore.Hash() s.Debugf("DirResolver - Checking store state: %x", ourHash) s.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:]) } }) s.Noticef("Managing all jwt in exclusive directory %s", dr.directory) return nil } func (dr *DirAccResolver) Fetch(name string) (string, error) { if theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ { return theJWT, nil } else { dr.Lock() srv := dr.Server to := dr.fetchTimeout dr.Unlock() if srv == nil { return _EMPTY_, err } return srv.fetch(dr, name, to) // lookup from other server } } func (dr *DirAccResolver) Store(name, jwt string) error { return dr.saveIfNewer(name, jwt) } type DirResOption func(s *DirAccResolver) error // limits the amount of time spent waiting for an account fetch to complete func FetchTimeout(to time.Duration) DirResOption { return func(r *DirAccResolver) error { if to <= time.Duration(0) { return fmt.Errorf("Fetch timeout %v is too smal", to) } r.fetchTimeout = to return nil } } func (dr *DirAccResolver) apply(opts ...DirResOption) error { for _, o := range opts { if err := o(dr); err != nil { return err } } return nil } func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) { if limit == 0 { limit = math.MaxInt64 } if syncInterval <= 0 { syncInterval = time.Minute } store, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil) if err != nil { return nil, err } res := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT} if err := res.apply(opts...); err != nil { return nil, err } return res, nil } // Caching resolver using nats for lookups and making use of a directory for storage type CacheDirAccResolver struct { DirAccResolver ttl time.Duration } func (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) { if s == nil { return _EMPTY_, ErrNoAccountResolver } respC := make(chan []byte, 1) accountLookupRequest := fmt.Sprintf(accLookupReqSubj, name) s.mu.Lock() if s.sys == nil || s.sys.replies == nil { s.mu.Unlock() return _EMPTY_, fmt.Errorf("eventing shut down") } // Resolver will wait for detected active servers to reply // before serving an error in case there weren't any found. expectedServers := len(s.sys.servers) replySubj := s.newRespInbox() replies := s.sys.replies // Store our handler. replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { var clone []byte isEmpty := len(msg) == 0 if !isEmpty { clone = make([]byte, len(msg)) copy(clone, msg) } s.mu.Lock() defer s.mu.Unlock() expectedServers-- // Skip empty responses until getting all the available servers. if isEmpty && expectedServers > 0 { return } // Use the first valid response if there is still interest or // one of the empty responses to signal that it was not found. if _, ok := replies[replySubj]; ok { select { case respC <- clone: default: } } } s.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{}) quit := s.quitCh s.mu.Unlock() var err error var theJWT string select { case <-quit: err = errors.New("fetching jwt failed due to shutdown") case <-time.After(timeout): err = errors.New("fetching jwt timed out") case m := <-respC: if len(m) == 0 { err = errors.New("account jwt not found") } else if err = res.Store(name, string(m)); err == nil { theJWT = string(m) } } s.mu.Lock() delete(replies, replySubj) s.mu.Unlock() close(respC) return theJWT, err } func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) { if limit <= 0 { limit = 1_000 } store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil) if err != nil { return nil, err } res := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl} if err := res.apply(opts...); err != nil { return nil, err } return res, nil } func (dr *CacheDirAccResolver) Start(s *Server) error { op, opKeys, strict, err := getOperatorKeys(s) if err != nil { return err } dr.Lock() defer dr.Unlock() dr.Server = s dr.operator = opKeys dr.DirJWTStore.changed = func(pubKey string) { if v, ok := s.accounts.Load(pubKey); !ok { } else if theJwt, err := dr.LoadAcc(pubKey); err != nil { s.Errorf("DirResolver - Update got error on load: %v", err) } else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil { s.Errorf("DirResolver - Update resulted in error %v", err) } } dr.DirJWTStore.deleted = func(pubKey string) { removeCb(s, pubKey) } for _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { // subscribe to account jwt update requests if _, err := s.sysSubscribe(fmt.Sprintf(reqSub, "*"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) { var pubKey string tk := strings.Split(subj, tsep) if len(tk) == accUpdateTokensNew { pubKey = tk[accReqAccIndex] } else if len(tk) == accUpdateTokensOld { pubKey = tk[accUpdateAccIdxOld] } else { s.Debugf("DirResolver - jwt update cache skipped due to bad subject %q", subj) return } if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) } else if claim.Subject != pubKey { err := errors.New("subject does not match jwt content") respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) } else if claim.Issuer == op && strict { err := errors.New("operator requires issuer to be a signing key") respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) } else if _, ok := s.accounts.Load(pubKey); !ok { respondToUpdate(s, resp, pubKey, "jwt update cache skipped", nil) } else if err := claimValidate(claim); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) } else if err := dr.save(pubKey, string(msg)); err != nil { respondToUpdate(s, resp, pubKey, "jwt update cache resulted in error", err) } else { respondToUpdate(s, resp, pubKey, "jwt updated cache", nil) } }); err != nil { return fmt.Errorf("error setting up update handling: %v", err) } } if _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) { // As this is a raw message, we need to extract payload and only decode claims from it, // in case request is sent with headers. _, msg = c.msgParts(msg) if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { respondToUpdate(s, resp, "n/a", "jwt update cache resulted in error", err) } else if claim.Issuer == op && strict { err := errors.New("operator requires issuer to be a signing key") respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) } else if _, ok := s.accounts.Load(claim.Subject); !ok { respondToUpdate(s, resp, claim.Subject, "jwt update cache skipped", nil) } else if err := claimValidate(claim); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt update cache validation failed", err) } else if err := dr.save(claim.Subject, string(msg)); err != nil { respondToUpdate(s, resp, claim.Subject, "jwt update cache resulted in error", err) } else { respondToUpdate(s, resp, claim.Subject, "jwt updated cache", nil) } }); err != nil { return fmt.Errorf("error setting up update handling: %v", err) } // respond to list requests with one message containing all account ids if _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) { handleListRequest(dr.DirJWTStore, s, reply) }); err != nil { return fmt.Errorf("error setting up list request handling: %v", err) } if _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { handleDeleteRequest(dr.DirJWTStore, s, msg, reply) }); err != nil { return fmt.Errorf("error setting up list request handling: %v", err) } s.Noticef("Managing some jwt in exclusive directory %s", dr.directory) return nil } func (dr *CacheDirAccResolver) Reload() error { return dr.DirAccResolver.Reload() } nats-server-2.10.27/server/accounts_test.go000066400000000000000000003267001477524627100206760ustar00rootroot00000000000000// Copyright 2018-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "encoding/base64" "encoding/json" "fmt" "net/http" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func simpleAccountServer(t *testing.T) (*Server, *Account, *Account) { opts := defaultServerOptions s := New(&opts) // Now create two accounts. f, err := s.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account 'foo': %v", err) } b, err := s.RegisterAccount("$bar") if err != nil { t.Fatalf("Error creating account 'bar': %v", err) } return s, f, b } func TestRegisterDuplicateAccounts(t *testing.T) { s, _, _ := simpleAccountServer(t) if _, err := s.RegisterAccount("$foo"); err == nil { t.Fatal("Expected an error registering 'foo' twice") } } func TestAccountIsolation(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error register client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error register client with 'bar' account: %v", err) } // Make sure they are different accounts/sl. if cfoo.acc == cbar.acc { t.Fatalf("Error, accounts the same for both clients") } // Now do quick test that makes sure messages do not cross over. // setup bar as a foo subscriber. cbar.parseAsync("SUB foo 1\r\nPING\r\nPING\r\n") l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response incorrect: %q", l) } cfoo.parseAsync("SUB foo 1\r\nPUB foo 5\r\nhello\r\nPING\r\n") l, err = crFoo.ReadString('\n') if err != nil { t.Fatalf("Error for client 'foo' from server: %v", err) } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crFoo, []byte("hello\r\n"), t) // Now make sure nothing shows up on bar. l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response incorrect: %q", l) } } func TestAccountIsolationExportImport(t *testing.T) { checkIsolation := func(t *testing.T, pubSubj string, ncExp, ncImp *nats.Conn) { // We keep track of 2 subjects. // One subject (pubSubj) is based off the stream import. // The other subject "fizz" is not imported and should be isolated. gotSubjs := map[string]int{ pubSubj: 0, "fizz": 0, } count := int32(0) ch := make(chan struct{}, 1) if _, err := ncImp.Subscribe(">", func(m *nats.Msg) { gotSubjs[m.Subject] += 1 if n := atomic.AddInt32(&count, 1); n == 3 { ch <- struct{}{} } }); err != nil { t.Fatalf("Error on subscribe: %v", err) } // Since both prod and cons use same server, flushing here will ensure // that the interest is registered and known at the time we publish. ncImp.Flush() if err := ncExp.Publish(pubSubj, []byte(fmt.Sprintf("ncExp pub %s", pubSubj))); err != nil { t.Fatal(err) } if err := ncImp.Publish(pubSubj, []byte(fmt.Sprintf("ncImp pub %s", pubSubj))); err != nil { t.Fatal(err) } if err := ncExp.Publish("fizz", []byte("ncExp pub fizz")); err != nil { t.Fatal(err) } if err := ncImp.Publish("fizz", []byte("ncImp pub fizz")); err != nil { t.Fatal(err) } wantSubjs := map[string]int{ // Subscriber ncImp should receive publishes from ncExp and ncImp. pubSubj: 2, // Subscriber ncImp should only receive the publish from ncImp. "fizz": 1, } // Wait for at least the 3 expected messages select { case <-ch: case <-time.After(time.Second): t.Fatalf("Expected 3 messages, got %v", atomic.LoadInt32(&count)) } // But now wait a bit to see if subscription receives more than expected. time.Sleep(50 * time.Millisecond) if got, want := len(gotSubjs), len(wantSubjs); got != want { t.Fatalf("unexpected subjs len, got=%d; want=%d", got, want) } for key, gotCnt := range gotSubjs { if wantCnt := wantSubjs[key]; gotCnt != wantCnt { t.Errorf("unexpected receive count for subject %q, got=%d, want=%d", key, gotCnt, wantCnt) } } } cases := []struct { name string exp, imp string pubSubj string }{ { name: "export literal, import literal", exp: "foo", imp: "foo", pubSubj: "foo", }, { name: "export full wildcard, import literal", exp: "foo.>", imp: "foo.bar", pubSubj: "foo.bar", }, { name: "export full wildcard, import sublevel full wildcard", exp: "foo.>", imp: "foo.bar.>", pubSubj: "foo.bar.whizz", }, { name: "export full wildcard, import full wildcard", exp: "foo.>", imp: "foo.>", pubSubj: "foo.bar", }, { name: "export partial wildcard, import partial wildcard", exp: "foo.*", imp: "foo.*", pubSubj: "foo.bar", }, { name: "export mid partial wildcard, import mid partial wildcard", exp: "foo.*.bar", imp: "foo.*.bar", pubSubj: "foo.whizz.bar", }, } for _, c := range cases { t.Run(fmt.Sprintf("%s jwt", c.name), func(t *testing.T) { // Setup NATS server. s := opTrustBasicSetup() defer s.Shutdown() s.Start() if err := s.readyForConnections(5 * time.Second); err != nil { t.Fatal(err) } buildMemAccResolver(s) // Setup exporter account. accExpPair, accExpPub := createKey(t) accExpClaims := jwt.NewAccountClaims(accExpPub) if c.exp != "" { accExpClaims.Limits.WildcardExports = true accExpClaims.Exports.Add(&jwt.Export{ Name: fmt.Sprintf("%s-stream-export", c.exp), Subject: jwt.Subject(c.exp), Type: jwt.Stream, }) } accExpJWT, err := accExpClaims.Encode(oKp) require_NoError(t, err) addAccountToMemResolver(s, accExpPub, accExpJWT) // Setup importer account. accImpPair, accImpPub := createKey(t) accImpClaims := jwt.NewAccountClaims(accImpPub) if c.imp != "" { accImpClaims.Imports.Add(&jwt.Import{ Name: fmt.Sprintf("%s-stream-import", c.imp), Subject: jwt.Subject(c.imp), Account: accExpPub, Type: jwt.Stream, }) } accImpJWT, err := accImpClaims.Encode(oKp) require_NoError(t, err) addAccountToMemResolver(s, accImpPub, accImpJWT) // Connect with different accounts. ncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, accExpPair), nats.Name(fmt.Sprintf("nc-exporter-%s", c.exp))) defer ncExp.Close() ncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, accImpPair), nats.Name(fmt.Sprintf("nc-importer-%s", c.imp))) defer ncImp.Close() checkIsolation(t, c.pubSubj, ncExp, ncImp) if t.Failed() { t.Logf("exported=%q; imported=%q", c.exp, c.imp) } }) t.Run(fmt.Sprintf("%s conf", c.name), func(t *testing.T) { // Setup NATS server. cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 accounts: { accExp: { users: [{user: accExp, password: accExp}] exports: [{stream: %q}] } accImp: { users: [{user: accImp, password: accImp}] imports: [{stream: {account: accExp, subject: %q}}] } } `, c.exp, c.imp, ))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() // Connect with different accounts. ncExp := natsConnect(t, s.ClientURL(), nats.UserInfo("accExp", "accExp"), nats.Name(fmt.Sprintf("nc-exporter-%s", c.exp))) defer ncExp.Close() ncImp := natsConnect(t, s.ClientURL(), nats.UserInfo("accImp", "accImp"), nats.Name(fmt.Sprintf("nc-importer-%s", c.imp))) defer ncImp.Close() checkIsolation(t, c.pubSubj, ncExp, ncImp) if t.Failed() { t.Logf("exported=%q; imported=%q", c.exp, c.imp) } }) } } func TestMultiAccountsIsolation(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { PUBLIC: { users:[{user: public, password: public}] exports: [ { stream: orders.client.stream.> } { stream: orders.client2.stream.> } ] } CLIENT: { users:[{user: client, password: client}] imports: [ { stream: { account: PUBLIC, subject: orders.client.stream.> }} ] } CLIENT2: { users:[{user: client2, password: client2}] imports: [ { stream: { account: PUBLIC, subject: orders.client2.stream.> }} ] } }`)) s, _ := RunServerWithConfig(conf) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() // Create a connection for CLIENT and subscribe on orders.> clientnc := natsConnect(t, s.ClientURL(), nats.UserInfo("client", "client")) defer clientnc.Close() clientsub := natsSubSync(t, clientnc, "orders.>") natsFlush(t, clientnc) // Now same for CLIENT2. client2nc := natsConnect(t, s.ClientURL(), nats.UserInfo("client2", "client2")) defer client2nc.Close() client2sub := natsSubSync(t, client2nc, "orders.>") natsFlush(t, client2nc) // Now create a connection for PUBLIC publicnc := natsConnect(t, s.ClientURL(), nats.UserInfo("public", "public")) defer publicnc.Close() // Publish on 'orders.client.stream.entry', so only CLIENT should receive it. natsPub(t, publicnc, "orders.client.stream.entry", []byte("test1")) // Verify that clientsub gets it. msg := natsNexMsg(t, clientsub, time.Second) require_Equal(t, string(msg.Data), "test1") // And also verify that client2sub does NOT get it. _, err := client2sub.NextMsg(100 * time.Microsecond) require_Error(t, err, nats.ErrTimeout) clientsub.Unsubscribe() natsFlush(t, clientnc) client2sub.Unsubscribe() natsFlush(t, client2nc) // Now have both accounts subscribe to "orders.*.stream.entry" clientsub = natsSubSync(t, clientnc, "orders.*.stream.entry") natsFlush(t, clientnc) client2sub = natsSubSync(t, client2nc, "orders.*.stream.entry") natsFlush(t, client2nc) // Using the PUBLIC account, publish on the "CLIENT" subject natsPub(t, publicnc, "orders.client.stream.entry", []byte("test2")) natsFlush(t, publicnc) msg = natsNexMsg(t, clientsub, time.Second) require_Equal(t, string(msg.Data), "test2") _, err = client2sub.NextMsg(100 * time.Microsecond) require_Error(t, err, nats.ErrTimeout) } func TestAccountFromOptions(t *testing.T) { opts := defaultServerOptions opts.Accounts = []*Account{NewAccount("foo"), NewAccount("bar")} s := New(&opts) defer s.Shutdown() ta := s.numReservedAccounts() + 2 if la := s.numAccounts(); la != ta { t.Fatalf("Expected to have a server with %d active accounts, got %v", ta, la) } // Check that sl is filled in. fooAcc, _ := s.LookupAccount("foo") barAcc, _ := s.LookupAccount("bar") if fooAcc == nil || barAcc == nil { t.Fatalf("Error retrieving accounts for 'foo' and 'bar'") } if fooAcc.sl == nil || barAcc.sl == nil { t.Fatal("Expected Sublists to be filled in on Opts.Accounts") } } // Clients used to be able to ask that the account be forced to be new. // This was for dynamic sandboxes for demo environments but was never really used. // Make sure it always errors if set. func TestNewAccountAndRequireNewAlwaysError(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: ua, password: pa} ] }, B: { users: [ {user: ub, password: pb} ] }, } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Success case c, _, _ := newClientForServer(s) connectOp := "CONNECT {\"user\":\"ua\", \"pass\":\"pa\"}\r\n" err := c.parse([]byte(connectOp)) require_NoError(t, err) c.close() // Simple cases, any setting of account or new_account always errors. // Even with proper auth. c, cr, _ := newClientForServer(s) connectOp = "CONNECT {\"user\":\"ua\", \"pass\":\"pa\", \"account\":\"ANY\"}\r\n" c.parseAsync(connectOp) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR 'Authorization Violation'") { t.Fatalf("Expected an error, got %q", l) } c.close() // new_account with proper credentials. c, cr, _ = newClientForServer(s) connectOp = "CONNECT {\"user\":\"ua\", \"pass\":\"pa\", \"new_account\":true}\r\n" c.parseAsync(connectOp) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR 'Authorization Violation'") { t.Fatalf("Expected an error, got %q", l) } c.close() // switch acccounts with proper credentials. c, cr, _ = newClientForServer(s) connectOp = "CONNECT {\"user\":\"ua\", \"pass\":\"pa\", \"account\":\"B\"}\r\n" c.parseAsync(connectOp) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR 'Authorization Violation'") { t.Fatalf("Expected an error, got %q", l) } c.close() // Even if correct account designation, still make sure we error. c, cr, _ = newClientForServer(s) connectOp = "CONNECT {\"user\":\"ua\", \"pass\":\"pa\", \"account\":\"A\"}\r\n" c.parseAsync(connectOp) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR 'Authorization Violation'") { t.Fatalf("Expected an error, got %q", l) } c.close() } func accountNameExists(name string, accounts []*Account) bool { for _, acc := range accounts { if strings.Compare(acc.Name, name) == 0 { return true } } return false } func TestAccountSimpleConfig(t *testing.T) { cfg1 := ` accounts = [foo, bar] ` confFileName := createConfFile(t, []byte(cfg1)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error processing config file: %v", err) } if la := len(opts.Accounts); la != 2 { t.Fatalf("Expected to see 2 accounts in opts, got %d", la) } if !accountNameExists("foo", opts.Accounts) { t.Fatal("Expected a 'foo' account") } if !accountNameExists("bar", opts.Accounts) { t.Fatal("Expected a 'bar' account") } cfg2 := ` accounts = [foo, foo] ` // Make sure double entries is an error. confFileName = createConfFile(t, []byte(cfg2)) _, err = ProcessConfigFile(confFileName) if err == nil { t.Fatalf("Expected an error with double account entries") } } func TestAccountParseConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` accounts { synadia { users = [ {user: alice, password: foo} {user: bob, password: bar} ] } nats.io { users = [ {user: derek, password: foo} {user: ivan, password: bar} ] } } `)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error processing config file: %v", err) } if la := len(opts.Accounts); la != 2 { t.Fatalf("Expected to see 2 accounts in opts, got %d", la) } if lu := len(opts.Users); lu != 4 { t.Fatalf("Expected 4 total Users, got %d", lu) } var natsAcc *Account for _, acc := range opts.Accounts { if acc.Name == "nats.io" { natsAcc = acc break } } if natsAcc == nil { t.Fatalf("Error retrieving account for 'nats.io'") } for _, u := range opts.Users { if u.Username == "derek" { if u.Account != natsAcc { t.Fatalf("Expected to see the 'nats.io' account, but received %+v", u.Account) } } } } func TestAccountParseConfigDuplicateUsers(t *testing.T) { confFileName := createConfFile(t, []byte(` accounts { synadia { users = [ {user: alice, password: foo} {user: bob, password: bar} ] } nats.io { users = [ {user: alice, password: bar} ] } } `)) _, err := ProcessConfigFile(confFileName) if err == nil { t.Fatalf("Expected an error with double user entries") } } func TestAccountParseConfigImportsExports(t *testing.T) { opts, err := ProcessConfigFile("./configs/accounts.conf") if err != nil { t.Fatal("parsing failed: ", err) } if la := len(opts.Accounts); la != 3 { t.Fatalf("Expected to see 3 accounts in opts, got %d", la) } if lu := len(opts.Nkeys); lu != 4 { t.Fatalf("Expected 4 total Nkey users, got %d", lu) } if lu := len(opts.Users); lu != 0 { t.Fatalf("Expected no Users, got %d", lu) } var natsAcc, synAcc *Account for _, acc := range opts.Accounts { if acc.Name == "nats.io" { natsAcc = acc } else if acc.Name == "synadia" { synAcc = acc } } if natsAcc == nil { t.Fatalf("Error retrieving account for 'nats.io'") } if natsAcc.Nkey != "AB5UKNPVHDWBP5WODG742274I3OGY5FM3CBIFCYI4OFEH7Y23GNZPXFE" { t.Fatalf("Expected nats account to have an nkey, got %q\n", natsAcc.Nkey) } // Check user assigned to the correct account. for _, nk := range opts.Nkeys { if nk.Nkey == "UBRYMDSRTC6AVJL6USKKS3FIOE466GMEU67PZDGOWYSYHWA7GSKO42VW" { if nk.Account != natsAcc { t.Fatalf("Expected user to be associated with natsAcc, got %q\n", nk.Account.Name) } break } } // Now check for the imports and exports of streams and services. if lis := len(natsAcc.imports.streams); lis != 2 { t.Fatalf("Expected 2 imported streams, got %d\n", lis) } if lis := len(natsAcc.imports.services); lis != 1 { t.Fatalf("Expected 1 imported service, got %d\n", lis) } if les := len(natsAcc.exports.services); les != 4 { t.Fatalf("Expected 4 exported services, got %d\n", les) } if les := len(natsAcc.exports.streams); les != 0 { t.Fatalf("Expected no exported streams, got %d\n", les) } ea := natsAcc.exports.services["nats.time"] if ea == nil { t.Fatalf("Expected to get a non-nil exportAuth for service") } if ea.respType != Streamed { t.Fatalf("Expected to get a Streamed response type, got %q", ea.respType) } ea = natsAcc.exports.services["nats.photo"] if ea == nil { t.Fatalf("Expected to get a non-nil exportAuth for service") } if ea.respType != Chunked { t.Fatalf("Expected to get a Chunked response type, got %q", ea.respType) } ea = natsAcc.exports.services["nats.add"] if ea == nil { t.Fatalf("Expected to get a non-nil exportAuth for service") } if ea.respType != Singleton { t.Fatalf("Expected to get a Singleton response type, got %q", ea.respType) } if synAcc == nil { t.Fatalf("Error retrieving account for 'synadia'") } if lis := len(synAcc.imports.streams); lis != 0 { t.Fatalf("Expected no imported streams, got %d\n", lis) } if lis := len(synAcc.imports.services); lis != 1 { t.Fatalf("Expected 1 imported service, got %d\n", lis) } if les := len(synAcc.exports.services); les != 2 { t.Fatalf("Expected 2 exported service, got %d\n", les) } if les := len(synAcc.exports.streams); les != 2 { t.Fatalf("Expected 2 exported streams, got %d\n", les) } } func TestImportExportConfigFailures(t *testing.T) { // Import from unknow account cf := createConfFile(t, []byte(` accounts { nats.io { imports = [{stream: {account: "synadia", subject:"foo"}}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with import from unknown account") } // Import a service with no account. cf = createConfFile(t, []byte(` accounts { nats.io { imports = [{service: subject:"foo.*"}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with import of a service with no account") } // Import a service with a wildcard subject. cf = createConfFile(t, []byte(` accounts { nats.io { imports = [{service: {account: "nats.io", subject:"foo.*"}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with import of a service with wildcard subject") } // Export with unknown keyword. cf = createConfFile(t, []byte(` accounts { nats.io { exports = [{service: "foo.*", wat:true}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with export with unknown keyword") } // Import with unknown keyword. cf = createConfFile(t, []byte(` accounts { nats.io { imports = [{stream: {account: nats.io, subject: "foo.*"}, wat:true}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with import with unknown keyword") } // Export with an account. cf = createConfFile(t, []byte(` accounts { nats.io { exports = [{service: {account: nats.io, subject:"foo.*"}}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with export with account") } } func TestImportAuthorized(t *testing.T) { _, foo, bar := simpleAccountServer(t) checkBool(foo.checkStreamImportAuthorized(bar, "foo", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, ">", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.*", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.>", nil), false, t) foo.AddStreamExport("foo", IsPublicExport) checkBool(foo.checkStreamImportAuthorized(bar, "foo", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*", nil), false, t) foo.AddStreamExport("*", []*Account{bar}) checkBool(foo.checkStreamImportAuthorized(bar, "foo", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "baz", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, ">", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.*", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*.*", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*.>", nil), false, t) // Reset and test '>' public export _, foo, bar = simpleAccountServer(t) foo.AddStreamExport(">", nil) // Everything should work. checkBool(foo.checkStreamImportAuthorized(bar, "foo", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "baz", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, ">", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "*", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.*", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "*.*", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "*.>", nil), true, t) _, foo, bar = simpleAccountServer(t) foo.addStreamExportWithAccountPos("foo.*", []*Account{}, 2) foo.addStreamExportWithAccountPos("bar.*.foo", []*Account{}, 2) if err := foo.addStreamExportWithAccountPos("baz.*.>", []*Account{}, 3); err == nil { t.Fatal("expected error") } checkBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf("foo.%s", bar.Name), nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf("bar.%s.foo", bar.Name), nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf("baz.foo.%s", bar.Name), nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.X", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar.X.foo", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "baz.foo.X", nil), false, t) foo.addServiceExportWithAccountPos("a.*", []*Account{}, 2) foo.addServiceExportWithAccountPos("b.*.a", []*Account{}, 2) if err := foo.addServiceExportWithAccountPos("c.*.>", []*Account{}, 3); err == nil { t.Fatal("expected error") } checkBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf("a.%s", bar.Name), nil), true, t) checkBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf("b.%s.a", bar.Name), nil), true, t) checkBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf("c.a.%s", bar.Name), nil), false, t) checkBool(foo.checkServiceImportAuthorized(bar, "a.X", nil), false, t) checkBool(foo.checkServiceImportAuthorized(bar, "b.X.a", nil), false, t) checkBool(foo.checkServiceImportAuthorized(bar, "c.a.X", nil), false, t) // Reset and test pwc and fwc s, foo, bar := simpleAccountServer(t) foo.AddStreamExport("foo.*.baz.>", []*Account{bar}) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.baz.1", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.baz.*", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.*.baz.1.1", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.22.baz.22", nil), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.baz", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.*.*", nil), false, t) // Make sure we match the account as well fb, _ := s.RegisterAccount("foobar") bz, _ := s.RegisterAccount("baz") checkBool(foo.checkStreamImportAuthorized(fb, "foo.bar.baz.1", nil), false, t) checkBool(foo.checkStreamImportAuthorized(bz, "foo.bar.baz.1", nil), false, t) } func TestSimpleMapping(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Test first that trying to import with no matching export permission returns an error. if err := cbar.acc.AddStreamImport(fooAcc, "foo", "import"); err != ErrStreamImportAuthorization { t.Fatalf("Expected error of ErrAccountImportAuthorization but got %v", err) } // Now map the subject space between foo and bar. // Need to do export first. if err := cfoo.acc.AddStreamExport("foo", nil); err != nil { // Public with no accounts defined. t.Fatalf("Error adding account export to client foo: %v", err) } if err := cbar.acc.AddStreamImport(fooAcc, "foo", "import"); err != nil { t.Fatalf("Error adding account import to client bar: %v", err) } // Normal and Queue Subscription on bar client. if err := cbar.parse([]byte("SUB import.foo 1\r\nSUB import.foo bar 2\r\n")); err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // Now publish our message. cfoo.parseAsync("PUB foo 5\r\nhello\r\n") checkMsg := func(l, sid string) { t.Helper() mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "import.foo" { t.Fatalf("Did not get correct subject: wanted %q, got %q", "import.foo", matches[SUB_INDEX]) } if matches[SID_INDEX] != sid { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } } // Now check we got the message from normal subscription. l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } checkMsg(l, "1") checkPayload(crBar, []byte("hello\r\n"), t) l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } checkMsg(l, "2") checkPayload(crBar, []byte("hello\r\n"), t) // We should have 2 subscriptions in both. Normal and Queue Subscriber // for barAcc which are local, and 2 that are shadowed in fooAcc. // Now make sure that when we unsubscribe we clean up properly for both. if bslc := barAcc.sl.Count(); bslc != 2 { t.Fatalf("Expected 2 normal subscriptions on barAcc, got %d", bslc) } if fslc := fooAcc.sl.Count(); fslc != 2 { t.Fatalf("Expected 2 shadowed subscriptions on fooAcc, got %d", fslc) } // Now unsubscribe. if err := cbar.parse([]byte("UNSUB 1\r\nUNSUB 2\r\n")); err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // We should have zero on both. if bslc := barAcc.sl.Count(); bslc != 0 { t.Fatalf("Expected no normal subscriptions on barAcc, got %d", bslc) } if fslc := fooAcc.sl.Count(); fslc != 0 { t.Fatalf("Expected no shadowed subscriptions on fooAcc, got %d", fslc) } } // https://github.com/nats-io/nats-server/issues/1159 func TestStreamImportLengthBug(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, _, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } if err := cfoo.acc.AddStreamExport("client.>", nil); err != nil { t.Fatalf("Error adding account export to client foo: %v", err) } if err := cbar.acc.AddStreamImport(fooAcc, "client.>", "events.>"); err == nil { t.Fatalf("Expected an error when using a stream import prefix with a wildcard") } if err := cbar.acc.AddStreamImport(fooAcc, "client.>", "events"); err != nil { t.Fatalf("Error adding account import to client bar: %v", err) } if err := cbar.parse([]byte("SUB events.> 1\r\n")); err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // Also make sure that we will get an error from a config version. // JWT will be updated separately. cf := createConfFile(t, []byte(` accounts { foo { exports = [{stream: "client.>"}] } bar { imports = [{stream: {account: "foo", subject:"client.>"}, prefix:"events.>"}] } } `)) if _, err := ProcessConfigFile(cf); err == nil { t.Fatalf("Expected an error with import with wildcard prefix") } } func TestShadowSubsCleanupOnClientClose(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() // Now map the subject space between foo and bar. // Need to do export first. if err := fooAcc.AddStreamExport("foo", nil); err != nil { // Public with no accounts defined. t.Fatalf("Error adding account export to client foo: %v", err) } if err := barAcc.AddStreamImport(fooAcc, "foo", "import"); err != nil { t.Fatalf("Error adding account import to client bar: %v", err) } cbar, _, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Normal and Queue Subscription on bar client. if err := cbar.parse([]byte("SUB import.foo 1\r\nSUB import.foo bar 2\r\n")); err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } if fslc := fooAcc.sl.Count(); fslc != 2 { t.Fatalf("Expected 2 shadowed subscriptions on fooAcc, got %d", fslc) } // Now close cbar and make sure we remove shadows. cbar.closeConnection(ClientClosed) checkFor(t, time.Second, 10*time.Millisecond, func() error { if fslc := fooAcc.sl.Count(); fslc != 0 { return fmt.Errorf("Number of shadow subscriptions is %d", fslc) } return nil }) } func TestNoPrefixWildcardMapping(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } if err := cfoo.acc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } if err := cbar.acc.AddStreamImport(fooAcc, "*", ""); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } // Normal Subscription on bar client for literal "foo". cbar.parseAsync("SUB foo 1\r\nPING\r\n") _, err := crBar.ReadString('\n') // Make sure subscriptions were processed. if err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // Now publish our message. cfoo.parseAsync("PUB foo 5\r\nhello\r\n") // Now check we got the message from normal subscription. l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("hello\r\n"), t) } func TestPrefixWildcardMapping(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } if err := cfoo.acc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } // Checking that trailing '.' is accepted, tested that it is auto added above. if err := cbar.acc.AddStreamImport(fooAcc, "*", "pub.imports."); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } // Normal Subscription on bar client for wildcard. cbar.parseAsync("SUB pub.imports.* 1\r\nPING\r\n") _, err := crBar.ReadString('\n') // Make sure subscriptions were processed. if err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // Now publish our message. cfoo.parseAsync("PUB foo 5\r\nhello\r\n") // Now check we got the messages from wildcard subscription. l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "pub.imports.foo" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("hello\r\n"), t) } func TestPrefixWildcardMappingWithLiteralSub(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } if err := fooAcc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } if err := barAcc.AddStreamImport(fooAcc, "*", "pub.imports."); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } // Normal Subscription on bar client for wildcard. cbar.parseAsync("SUB pub.imports.foo 1\r\nPING\r\n") _, err := crBar.ReadString('\n') // Make sure subscriptions were processed. if err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // Now publish our message. cfoo.parseAsync("PUB foo 5\r\nhello\r\n") // Now check we got the messages from wildcard subscription. l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "pub.imports.foo" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("hello\r\n"), t) } func TestMultipleImportsAndSingleWCSub(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } if err := fooAcc.AddStreamExport("foo", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to account foo: %v", err) } if err := fooAcc.AddStreamExport("bar", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to account foo: %v", err) } if err := barAcc.AddStreamImport(fooAcc, "foo", "pub."); err != nil { t.Fatalf("Error adding stream import to account bar: %v", err) } if err := barAcc.AddStreamImport(fooAcc, "bar", "pub."); err != nil { t.Fatalf("Error adding stream import to account bar: %v", err) } // Wildcard Subscription on bar client for both imports. cbar.parse([]byte("SUB pub.* 1\r\n")) // Now publish a message on 'foo' and 'bar' cfoo.parseAsync("PUB foo 5\r\nhello\r\nPUB bar 5\r\nworld\r\n") // Now check we got the messages from the wildcard subscription. l, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "pub.foo" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("hello\r\n"), t) l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw = msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches = mraw[0] if matches[SUB_INDEX] != "pub.bar" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("world\r\n"), t) // Check subscription count. if fslc := fooAcc.sl.Count(); fslc != 2 { t.Fatalf("Expected 2 shadowed subscriptions on fooAcc, got %d", fslc) } if bslc := barAcc.sl.Count(); bslc != 1 { t.Fatalf("Expected 1 normal subscriptions on barAcc, got %d", bslc) } // Now unsubscribe. if err := cbar.parse([]byte("UNSUB 1\r\n")); err != nil { t.Fatalf("Error for client 'bar' from server: %v", err) } // We should have zero on both. if bslc := barAcc.sl.Count(); bslc != 0 { t.Fatalf("Expected no normal subscriptions on barAcc, got %d", bslc) } if fslc := fooAcc.sl.Count(); fslc != 0 { t.Fatalf("Expected no shadowed subscriptions on fooAcc, got %d", fslc) } } // Make sure the AddServiceExport function is additive if called multiple times. func TestAddServiceExport(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) bazAcc, err := s.RegisterAccount("$baz") if err != nil { t.Fatalf("Error creating account 'baz': %v", err) } defer s.Shutdown() if err := fooAcc.AddServiceExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr := fooAcc.exports.services["test.request"] if len(tr.approved) != 0 { t.Fatalf("Expected no authorized accounts, got %d", len(tr.approved)) } if err := fooAcc.AddServiceExport("test.request", []*Account{barAcc}); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr = fooAcc.exports.services["test.request"] if tr == nil { t.Fatalf("Expected authorized accounts, got nil") } if ls := len(tr.approved); ls != 1 { t.Fatalf("Expected 1 authorized accounts, got %d", ls) } if err := fooAcc.AddServiceExport("test.request", []*Account{bazAcc}); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr = fooAcc.exports.services["test.request"] if tr == nil { t.Fatalf("Expected authorized accounts, got nil") } if ls := len(tr.approved); ls != 2 { t.Fatalf("Expected 2 authorized accounts, got %d", ls) } } func TestServiceExportWithWildcards(t *testing.T) { for _, test := range []struct { name string public bool }{ { name: "public", public: true, }, { name: "private", public: false, }, } { t.Run(test.name, func(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() var accs []*Account if !test.public { accs = []*Account{barAcc} } // Add service export with a wildcard if err := fooAcc.AddServiceExport("ngs.update.*", accs); err != nil { t.Fatalf("Error adding account service export: %v", err) } // Import on bar account if err := barAcc.AddServiceImport(fooAcc, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding account service import: %v", err) } cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Now setup the responder under cfoo cfoo.parse([]byte("SUB ngs.update.* 1\r\n")) // Now send the request. Remember we expect the request on our local ngs.update. // We added the route with that "from" and will map it to "ngs.update.$bar" cbar.parseAsync("SUB reply 11\r\nPUB ngs.update reply 4\r\nhelp\r\n") // Now read the request from crFoo l, err := crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "ngs.update.$bar" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } // Make sure this looks like _INBOX if !strings.HasPrefix(matches[REPLY_INDEX], "_R_.") { t.Fatalf("Expected an _R_.* like reply, got '%s'", matches[REPLY_INDEX]) } checkPayload(crFoo, []byte("help\r\n"), t) replyOp := fmt.Sprintf("PUB %s 2\r\n22\r\n", matches[REPLY_INDEX]) cfoo.parseAsync(replyOp) // Now read the response from crBar l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw = msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches = mraw[0] if matches[SUB_INDEX] != "reply" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "11" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } if matches[REPLY_INDEX] != "" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("22\r\n"), t) if nr := barAcc.NumPendingAllResponses(); nr != 0 { t.Fatalf("Expected no responses on barAcc, got %d", nr) } }) } } func TestAccountAddServiceImportRace(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() if err := fooAcc.AddServiceExport("foo.*", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } total := 100 errCh := make(chan error, total) for i := 0; i < 100; i++ { go func(i int) { err := barAcc.AddServiceImport(fooAcc, fmt.Sprintf("foo.%d", i), "") errCh <- err // nil is a valid value. }(i) } for i := 0; i < 100; i++ { err := <-errCh if err != nil { t.Fatalf("Error adding account service import: %v", err) } } barAcc.mu.Lock() lens := len(barAcc.imports.services) c := barAcc.internalClient() barAcc.mu.Unlock() if lens != total { t.Fatalf("Expected %d imported services, got %d", total, lens) } c.mu.Lock() lens = len(c.subs) c.mu.Unlock() if lens != total { t.Fatalf("Expected %d subscriptions in internal client, got %d", total, lens) } } func TestServiceImportWithWildcards(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() if err := fooAcc.AddServiceExport("test.*", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // We can not map wildcards atm, so if we supply a to mapping and a wildcard we should fail. if err := barAcc.AddServiceImport(fooAcc, "test.*", "foo"); err == nil { t.Fatalf("Expected error adding account service import with wildcard and mapping, got none") } if err := barAcc.AddServiceImport(fooAcc, "test.>", ""); err == nil { t.Fatalf("Expected error adding account service import with broader wildcard, got none") } // This should work. if err := barAcc.AddServiceImport(fooAcc, "test.*", ""); err != nil { t.Fatalf("Error adding account service import: %v", err) } // Make sure we can send and receive. cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } // Now setup the resonder under cfoo cfoo.parse([]byte("SUB test.* 1\r\n")) cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Now send the request. go cbar.parse([]byte("SUB bar 11\r\nPUB test.22 bar 4\r\nhelp\r\n")) // Now read the request from crFoo l, err := crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "test.22" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } // Make sure this looks like _INBOX if !strings.HasPrefix(matches[REPLY_INDEX], "_R_.") { t.Fatalf("Expected an _R_.* like reply, got '%s'", matches[REPLY_INDEX]) } checkPayload(crFoo, []byte("help\r\n"), t) replyOp := fmt.Sprintf("PUB %s 2\r\n22\r\n", matches[REPLY_INDEX]) go cfoo.parse([]byte(replyOp)) // Now read the response from crBar l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw = msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches = mraw[0] if matches[SUB_INDEX] != "bar" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "11" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } if matches[REPLY_INDEX] != "" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("22\r\n"), t) // Remove the service import with the wildcard and make sure hasWC is cleared. barAcc.removeServiceImport("test.*") barAcc.mu.Lock() defer barAcc.mu.Unlock() if len(barAcc.imports.services) != 0 { t.Fatalf("Expected no imported services, got %d", len(barAcc.imports.services)) } } // Make sure the AddStreamExport function is additive if called multiple times. func TestAddStreamExport(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) bazAcc, err := s.RegisterAccount("$baz") if err != nil { t.Fatalf("Error creating account 'baz': %v", err) } defer s.Shutdown() if err := fooAcc.AddStreamExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr := fooAcc.exports.streams["test.request"] if tr != nil { t.Fatalf("Expected no authorized accounts, got %d", len(tr.approved)) } if err := fooAcc.AddStreamExport("test.request", []*Account{barAcc}); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr = fooAcc.exports.streams["test.request"] if tr == nil { t.Fatalf("Expected authorized accounts, got nil") } if ls := len(tr.approved); ls != 1 { t.Fatalf("Expected 1 authorized accounts, got %d", ls) } if err := fooAcc.AddStreamExport("test.request", []*Account{bazAcc}); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } tr = fooAcc.exports.streams["test.request"] if tr == nil { t.Fatalf("Expected authorized accounts, got nil") } if ls := len(tr.approved); ls != 2 { t.Fatalf("Expected 2 authorized accounts, got %d", ls) } } func TestCrossAccountRequestReply(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Add in the service export for the requests. Make it public. if err := cfoo.acc.AddServiceExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // Test addServiceImport to make sure it requires accounts. if err := cbar.acc.AddServiceImport(nil, "foo", "test.request"); err != ErrMissingAccount { t.Fatalf("Expected ErrMissingAccount but received %v.", err) } if err := cbar.acc.AddServiceImport(fooAcc, "foo", "test..request."); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject but received %v.", err) } // Now add in the route mapping for request to be routed to the foo account. if err := cbar.acc.AddServiceImport(fooAcc, "foo", "test.request"); err != nil { t.Fatalf("Error adding account service import to client bar: %v", err) } // Now setup the resonder under cfoo cfoo.parse([]byte("SUB test.request 1\r\n")) // Now send the request. Remember we expect the request on our local foo. We added the route // with that "from" and will map it to "test.request" cbar.parseAsync("SUB bar 11\r\nPUB foo bar 4\r\nhelp\r\n") // Now read the request from crFoo l, err := crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] if matches[SUB_INDEX] != "test.request" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } // Make sure this looks like _INBOX if !strings.HasPrefix(matches[REPLY_INDEX], "_R_.") { t.Fatalf("Expected an _R_.* like reply, got '%s'", matches[REPLY_INDEX]) } checkPayload(crFoo, []byte("help\r\n"), t) replyOp := fmt.Sprintf("PUB %s 2\r\n22\r\n", matches[REPLY_INDEX]) cfoo.parseAsync(replyOp) // Now read the response from crBar l, err = crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw = msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches = mraw[0] if matches[SUB_INDEX] != "bar" { t.Fatalf("Did not get correct subject: '%s'", matches[SUB_INDEX]) } if matches[SID_INDEX] != "11" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } if matches[REPLY_INDEX] != "" { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } checkPayload(crBar, []byte("22\r\n"), t) if nr := barAcc.NumPendingAllResponses(); nr != 0 { t.Fatalf("Expected no responses on barAcc, got %d", nr) } } func TestAccountRequestReplyTrackLatency(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() // Run server in Go routine. We need this one running for internal sending of msgs. s.Start() // Wait for accept loop(s) to be started if err := s.readyForConnections(10 * time.Second); err != nil { t.Fatal(err) } cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Add in the service export for the requests. Make it public. if err := fooAcc.AddServiceExport("track.service", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // Now let's add in tracking // First check we get an error if service does not exist. if err := fooAcc.TrackServiceExport("track.wrong", "results"); err != ErrMissingService { t.Fatalf("Expected error enabling tracking latency for wrong service") } // Check results should be a valid subject if err := fooAcc.TrackServiceExport("track.service", "results.*"); err != ErrBadPublishSubject { t.Fatalf("Expected error enabling tracking latency for bad results subject") } // Make sure we can not loop around on ourselves.. if err := fooAcc.TrackServiceExport("track.service", "track.service"); err != ErrBadPublishSubject { t.Fatalf("Expected error enabling tracking latency for same subject") } // Check bad sampling if err := fooAcc.TrackServiceExportWithSampling("track.service", "results", -1); err != ErrBadSampling { t.Fatalf("Expected error enabling tracking latency for bad sampling") } if err := fooAcc.TrackServiceExportWithSampling("track.service", "results", 101); err != ErrBadSampling { t.Fatalf("Expected error enabling tracking latency for bad sampling") } // Now let's add in tracking for real. This will be 100% if err := fooAcc.TrackServiceExport("track.service", "results"); err != nil { t.Fatalf("Error enabling tracking latency: %v", err) } // Now add in the route mapping for request to be routed to the foo account. if err := barAcc.AddServiceImport(fooAcc, "req", "track.service"); err != nil { t.Fatalf("Error adding account service import to client bar: %v", err) } // Now setup the responder under cfoo and the listener for the results cfoo.parse([]byte("SUB track.service 1\r\nSUB results 2\r\n")) readFooMsg := func() ([]byte, string) { t.Helper() l, err := crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'foo': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } msg := mraw[0] msgSize, _ := strconv.Atoi(msg[LEN_INDEX]) return grabPayload(crFoo, msgSize), msg[REPLY_INDEX] } start := time.Now() // Now send the request. Remember we expect the request on our local foo. We added the route // with that "from" and will map it to "test.request" cbar.parseAsync("SUB resp 11\r\nPUB req resp 4\r\nhelp\r\n") // Now read the request from crFoo _, reply := readFooMsg() replyOp := fmt.Sprintf("PUB %s 2\r\n22\r\n", reply) serviceTime := 25 * time.Millisecond // We will wait a bit to check latency results go func() { time.Sleep(serviceTime) cfoo.parseAsync(replyOp) }() // Now read the response from crBar _, err := crBar.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } // Now let's check that we got the sampling results rMsg, _ := readFooMsg() // Unmarshal and check it. var sl ServiceLatency err = json.Unmarshal(rMsg, &sl) if err != nil { t.Fatalf("Could not parse latency json: %v\n", err) } startDelta := sl.RequestStart.Sub(start) if startDelta > 5*time.Millisecond { t.Fatalf("Bad start delta %v", startDelta) } if sl.ServiceLatency < serviceTime { t.Fatalf("Bad service latency: %v", sl.ServiceLatency) } if sl.TotalLatency < sl.ServiceLatency { t.Fatalf("Bad total latency: %v", sl.ServiceLatency) } } // This will test for leaks in the remote latency tracking via client.rrTracking func TestAccountTrackLatencyRemoteLeaks(t *testing.T) { optsA, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true optsA.ServerName = "A" srvA := RunServer(optsA) defer srvA.Shutdown() optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) optsB.ServerName = "B" srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) srvs := []*Server{srvA, srvB} // Now add in the accounts and setup tracking. for _, s := range srvs { s.SetSystemAccount(globalAccountName) fooAcc, _ := s.RegisterAccount("$foo") fooAcc.AddServiceExport("track.service", nil) fooAcc.TrackServiceExport("track.service", "results") barAcc, _ := s.RegisterAccount("$bar") if err := barAcc.AddServiceImport(fooAcc, "req", "track.service"); err != nil { t.Fatalf("Failed to import: %v", err) } } getClient := func(s *Server, name string) *client { t.Helper() s.mu.Lock() defer s.mu.Unlock() for _, c := range s.clients { c.mu.Lock() n := c.opts.Name c.mu.Unlock() if n == name { return c } } t.Fatalf("Did not find client %q on server %q", name, s.info.ID) return nil } // Test with a responder on second server, srvB. but they will not respond. cfooNC := natsConnect(t, srvB.ClientURL(), nats.Name("foo")) defer cfooNC.Close() cfoo := getClient(srvB, "foo") fooAcc, _ := srvB.LookupAccount("$foo") if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } // Set new limits for _, srv := range srvs { fooAcc, _ := srv.LookupAccount("$foo") err := fooAcc.SetServiceExportResponseThreshold("track.service", 5*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } } // Now setup the responder under cfoo and the listener for the results time.Sleep(50 * time.Millisecond) baseSubs := int(srvA.NumSubscriptions()) fooSub := natsSubSync(t, cfooNC, "track.service") natsFlush(t, cfooNC) // Wait for it to propagate. checkExpectedSubs(t, baseSubs+1, srvA) cbarNC := natsConnect(t, srvA.ClientURL(), nats.Name("bar")) defer cbarNC.Close() cbar := getClient(srvA, "bar") barAcc, _ := srvA.LookupAccount("$bar") if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } readFooMsg := func() { t.Helper() if _, err := fooSub.NextMsg(time.Second); err != nil { t.Fatalf("Did not receive foo msg: %v", err) } } // Send 2 requests natsSubSync(t, cbarNC, "resp") natsPubReq(t, cbarNC, "req", "resp", []byte("help")) natsPubReq(t, cbarNC, "req", "resp", []byte("help")) readFooMsg() readFooMsg() var rc *client // Pull out first client srvB.mu.Lock() for _, rc = range srvB.clients { if rc != nil { break } } srvB.mu.Unlock() tracking := func() int { rc.mu.Lock() var nt int if rc.rrTracking != nil { nt = len(rc.rrTracking.rmap) } rc.mu.Unlock() return nt } expectTracking := func(expected int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { if numTracking := tracking(); numTracking != expected { return fmt.Errorf("Expected to have %d tracking replies, got %d", expected, numTracking) } return nil }) } expectTracking(2) // Make sure these remote tracking replies honor the current respThresh for a service export. time.Sleep(10 * time.Millisecond) expectTracking(0) // Also make sure tracking is removed rc.mu.Lock() removed := rc.rrTracking == nil rc.mu.Unlock() if !removed { t.Fatalf("Expected the rrTracking to be removed") } // Now let's test that a lower response threshold is picked up. fSub := natsSubSync(t, cfooNC, "foo") natsFlush(t, cfooNC) // Wait for it to propagate. checkExpectedSubs(t, baseSubs+4, srvA) // queue up some first. We want to test changing when rrTracking exists. natsPubReq(t, cbarNC, "req", "resp", []byte("help")) readFooMsg() expectTracking(1) for _, s := range srvs { fooAcc, _ := s.LookupAccount("$foo") barAcc, _ := s.LookupAccount("$bar") fooAcc.AddServiceExport("foo", nil) fooAcc.TrackServiceExport("foo", "foo.results") fooAcc.SetServiceExportResponseThreshold("foo", time.Millisecond) barAcc.AddServiceImport(fooAcc, "foo", "foo") } natsSubSync(t, cbarNC, "reply") natsPubReq(t, cbarNC, "foo", "reply", []byte("help")) if _, err := fSub.NextMsg(time.Second); err != nil { t.Fatalf("Did not receive foo msg: %v", err) } expectTracking(2) rc.mu.Lock() lrt := rc.rrTracking.lrt rc.mu.Unlock() if lrt != time.Millisecond { t.Fatalf("Expected lrt of %v, got %v", time.Millisecond, lrt) } // Now make sure we clear on close. rc.closeConnection(ClientClosed) // Actual tear down will be not inline. checkFor(t, time.Second, 5*time.Millisecond, func() error { rc.mu.Lock() removed = rc.rrTracking == nil rc.mu.Unlock() if !removed { return fmt.Errorf("Expected the rrTracking to be removed after client close") } return nil }) } func TestCrossAccountServiceResponseTypes(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() cfoo, crFoo, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, crBar, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Add in the service export for the requests. Make it public. if err := fooAcc.AddServiceExportWithResponse("test.request", Streamed, nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // Now add in the route mapping for request to be routed to the foo account. if err := barAcc.AddServiceImport(fooAcc, "foo", "test.request"); err != nil { t.Fatalf("Error adding account service import to client bar: %v", err) } // Now setup the resonder under cfoo cfoo.parse([]byte("SUB test.request 1\r\n")) // Now send the request. Remember we expect the request on our local foo. We added the route // with that "from" and will map it to "test.request" cbar.parseAsync("SUB bar 11\r\nPUB foo bar 4\r\nhelp\r\n") // Now read the request from crFoo l, err := crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches := mraw[0] reply := matches[REPLY_INDEX] if !strings.HasPrefix(reply, "_R_.") { t.Fatalf("Expected an _R_.* like reply, got '%s'", reply) } crFoo.ReadString('\n') replyOp := fmt.Sprintf("PUB %s 2\r\n22\r\n", matches[REPLY_INDEX]) var mReply []byte for i := 0; i < 10; i++ { mReply = append(mReply, replyOp...) } cfoo.parseAsync(string(mReply)) var buf []byte for i := 0; i < 20; i++ { b, err := crBar.ReadBytes('\n') if err != nil { t.Fatalf("Error reading response: %v", err) } buf = append(buf[:], b...) if mraw = msgPat.FindAllStringSubmatch(string(buf), -1); len(mraw) == 10 { break } } if len(mraw) != 10 { t.Fatalf("Expected a response but got %d", len(mraw)) } // Also make sure the response map gets cleaned up when interest goes away. cbar.closeConnection(ClientClosed) checkFor(t, time.Second, 10*time.Millisecond, func() error { if nr := barAcc.NumPendingAllResponses(); nr != 0 { return fmt.Errorf("Number of responses is %d", nr) } return nil }) // Now test bogus reply subjects are handled and do not accumulate the response maps. cbar, _, _ = newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } // Do not create any interest in the reply subject 'bar'. Just send a request. cbar.parseAsync("PUB foo bar 4\r\nhelp\r\n") // Now read the request from crFoo l, err = crFoo.ReadString('\n') if err != nil { t.Fatalf("Error reading from client 'bar': %v", err) } mraw = msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } matches = mraw[0] reply = matches[REPLY_INDEX] if !strings.HasPrefix(reply, "_R_.") { t.Fatalf("Expected an _R_.* like reply, got '%s'", reply) } crFoo.ReadString('\n') replyOp = fmt.Sprintf("PUB %s 2\r\n22\r\n", matches[REPLY_INDEX]) cfoo.parseAsync(replyOp) // Now wait for a bit, the reply should trip a no interest condition // which should clean this up. checkFor(t, time.Second, 10*time.Millisecond, func() error { if nr := fooAcc.NumPendingAllResponses(); nr != 0 { return fmt.Errorf("Number of responses is %d", nr) } return nil }) // Also make sure the response map entry is gone as well. fooAcc.mu.RLock() lrm := len(fooAcc.exports.responses) fooAcc.mu.RUnlock() if lrm != 0 { t.Fatalf("Expected the responses to be cleared, got %d entries", lrm) } } func TestAccountMapsUsers(t *testing.T) { // Used for the nkey users to properly sign. seed1 := "SUAPM67TC4RHQLKBX55NIQXSMATZDOZK6FNEOSS36CAYA7F7TY66LP4BOM" seed2 := "SUAIS5JPX4X4GJ7EIIJEQ56DH2GWPYJRPWN5XJEDENJOZHCBLI7SEPUQDE" confFileName := createConfFile(t, []byte(` accounts { synadia { users = [ {user: derek, password: foo}, {nkey: UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR} ] } nats { users = [ {user: ivan, password: bar}, {nkey: UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH} ] } } `)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Unexpected error parsing config file: %v", err) } opts.NoSigs = true s := New(opts) defer s.Shutdown() synadia, _ := s.LookupAccount("synadia") nats, _ := s.LookupAccount("nats") if synadia == nil || nats == nil { t.Fatalf("Expected non nil accounts during lookup") } // Make sure a normal log in maps the accounts correctly. c, _, _ := newClientForServer(s) defer c.close() connectOp := []byte("CONNECT {\"user\":\"derek\",\"pass\":\"foo\"}\r\n") c.parse(connectOp) if c.acc != synadia { t.Fatalf("Expected the client's account to match 'synadia', got %v", c.acc) } c, _, _ = newClientForServer(s) defer c.close() connectOp = []byte("CONNECT {\"user\":\"ivan\",\"pass\":\"bar\"}\r\n") c.parse(connectOp) if c.acc != nats { t.Fatalf("Expected the client's account to match 'nats', got %v", c.acc) } // Now test nkeys as well. kp, _ := nkeys.FromSeed([]byte(seed1)) pubKey, _ := kp.PublicKey() c, cr, l := newClientForServer(s) defer c.close() // Check for Nonce var info nonceInfo err = json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err := kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs := fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } if c.acc != synadia { t.Fatalf("Expected the nkey client's account to match 'synadia', got %v", c.acc) } // Now nats account nkey user. kp, _ = nkeys.FromSeed([]byte(seed2)) pubKey, _ = kp.PublicKey() c, cr, l = newClientForServer(s) defer c.close() // Check for Nonce err = json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err = kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig = base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } if c.acc != nats { t.Fatalf("Expected the nkey client's account to match 'nats', got %v", c.acc) } } func TestAccountGlobalDefault(t *testing.T) { opts := defaultServerOptions s := New(&opts) if acc, _ := s.LookupAccount(globalAccountName); acc == nil { t.Fatalf("Expected a global default account on a new server, got none.") } // Make sure we can not create one with same name.. if _, err := s.RegisterAccount(globalAccountName); err == nil { t.Fatalf("Expected error trying to create a new reserved account") } // Make sure we can not define one in a config file either. confFileName := createConfFile(t, []byte(`accounts { $G {} }`)) if _, err := ProcessConfigFile(confFileName); err == nil { t.Fatalf("Expected an error parsing config file with reserved account") } } func TestAccountCheckStreamImportsEqual(t *testing.T) { // Create bare accounts for this test fooAcc := NewAccount("foo") if err := fooAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } barAcc := NewAccount("bar") if err := barAcc.AddStreamImport(fooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } bazAcc := NewAccount("baz") if err := bazAcc.AddStreamImport(fooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !barAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be the same") } if err := bazAcc.AddStreamImport(fooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if barAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be different") } if err := barAcc.AddStreamImport(fooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !barAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be the same") } // Create another account that is named "foo". We want to make sure // that the comparison still works (based on account name, not pointer) newFooAcc := NewAccount("foo") if err := newFooAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } batAcc := NewAccount("bat") if err := batAcc.AddStreamImport(newFooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if err := batAcc.AddStreamImport(newFooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !batAcc.checkStreamImportsEqual(barAcc) { t.Fatal("Expected stream imports to be the same") } if !batAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be the same") } // Test with account with different "from" expAcc := NewAccount("new_acc") if err := expAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } aAcc := NewAccount("a") if err := aAcc.AddStreamImport(expAcc, "bar", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } bAcc := NewAccount("b") if err := bAcc.AddStreamImport(expAcc, "baz", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { t.Fatal("Expected stream imports to be different") } // Test with account with different "prefix" aAcc = NewAccount("a") if err := aAcc.AddStreamImport(expAcc, "bar", "prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } bAcc = NewAccount("b") if err := bAcc.AddStreamImport(expAcc, "bar", "diff_prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { t.Fatal("Expected stream imports to be different") } // Test with account with different "name" expAcc = NewAccount("diff_name") if err := expAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } bAcc = NewAccount("b") if err := bAcc.AddStreamImport(expAcc, "bar", "prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { t.Fatal("Expected stream imports to be different") } } func TestAccountNoDeadlockOnQueueSubRouteMapUpdate(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.QueueSubscribeSync("foo", "bar") var accs []*Account for i := 0; i < 10; i++ { acc, _ := s.RegisterAccount(fmt.Sprintf("acc%d", i)) acc.mu.Lock() accs = append(accs, acc) } opts2 := DefaultOptions() opts2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, opts.Cluster.Port)) s2 := RunServer(opts2) defer s2.Shutdown() wg := sync.WaitGroup{} wg.Add(1) go func() { time.Sleep(100 * time.Millisecond) for _, acc := range accs { acc.mu.Unlock() } wg.Done() }() nc.QueueSubscribeSync("foo", "bar") nc.Flush() wg.Wait() } func TestAccountDuplicateServiceImportSubject(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() fooAcc, _ := s.RegisterAccount("foo") fooAcc.AddServiceExport("remote1", nil) fooAcc.AddServiceExport("remote2", nil) barAcc, _ := s.RegisterAccount("bar") if err := barAcc.AddServiceImport(fooAcc, "foo", "remote1"); err != nil { t.Fatalf("Error adding service import: %v", err) } if err := barAcc.AddServiceImport(fooAcc, "foo", "remote2"); err == nil || !strings.Contains(err.Error(), "duplicate") { t.Fatalf("Expected an error about duplicate service import subject, got %q", err) } } func TestMultipleStreamImportsWithSameSubjectDifferentPrefix(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() fooAcc, _ := s.RegisterAccount("foo") fooAcc.AddStreamExport("test", nil) barAcc, _ := s.RegisterAccount("bar") barAcc.AddStreamExport("test", nil) importAcc, _ := s.RegisterAccount("import") if err := importAcc.AddStreamImport(fooAcc, "test", "foo"); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := importAcc.AddStreamImport(barAcc, "test", "bar"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now make sure we can see messages from both. cimport, crImport, _ := newClientForServer(s) defer cimport.close() if err := cimport.registerWithAccount(importAcc); err != nil { t.Fatalf("Error registering client with 'import' account: %v", err) } if err := cimport.parse([]byte("SUB *.test 1\r\n")); err != nil { t.Fatalf("Error for client 'import' from server: %v", err) } cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, _, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } readMsg := func() { t.Helper() l, err := crImport.ReadString('\n') if err != nil { t.Fatalf("Error reading msg header from client 'import': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } // Consume msg body too. if _, err = crImport.ReadString('\n'); err != nil { t.Fatalf("Error reading msg body from client 'import': %v", err) } } cbar.parseAsync("PUB test 9\r\nhello-bar\r\n") readMsg() cfoo.parseAsync("PUB test 9\r\nhello-foo\r\n") readMsg() } // This should work with prefixes that are different but we also want it to just work with same subject // being imported from multiple accounts. func TestMultipleStreamImportsWithSameSubject(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() fooAcc, _ := s.RegisterAccount("foo") fooAcc.AddStreamExport("test", nil) barAcc, _ := s.RegisterAccount("bar") barAcc.AddStreamExport("test", nil) importAcc, _ := s.RegisterAccount("import") if err := importAcc.AddStreamImport(fooAcc, "test", ""); err != nil { t.Fatalf("Unexpected error: %v", err) } // Since we allow this now, make sure we do detect a duplicate import from same account etc. // That should be not allowed. if err := importAcc.AddStreamImport(fooAcc, "test", ""); err != ErrStreamImportDuplicate { t.Fatalf("Expected ErrStreamImportDuplicate but got %v", err) } if err := importAcc.AddStreamImport(barAcc, "test", ""); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now make sure we can see messages from both. cimport, crImport, _ := newClientForServer(s) defer cimport.close() if err := cimport.registerWithAccount(importAcc); err != nil { t.Fatalf("Error registering client with 'import' account: %v", err) } if err := cimport.parse([]byte("SUB test 1\r\n")); err != nil { t.Fatalf("Error for client 'import' from server: %v", err) } cfoo, _, _ := newClientForServer(s) defer cfoo.close() if err := cfoo.registerWithAccount(fooAcc); err != nil { t.Fatalf("Error registering client with 'foo' account: %v", err) } cbar, _, _ := newClientForServer(s) defer cbar.close() if err := cbar.registerWithAccount(barAcc); err != nil { t.Fatalf("Error registering client with 'bar' account: %v", err) } readMsg := func() { t.Helper() l, err := crImport.ReadString('\n') if err != nil { t.Fatalf("Error reading msg header from client 'import': %v", err) } mraw := msgPat.FindAllStringSubmatch(l, -1) if len(mraw) == 0 { t.Fatalf("No message received") } // Consume msg body too. if _, err = crImport.ReadString('\n'); err != nil { t.Fatalf("Error reading msg body from client 'import': %v", err) } } cbar.parseAsync("PUB test 9\r\nhello-bar\r\n") readMsg() cfoo.parseAsync("PUB test 9\r\nhello-foo\r\n") readMsg() } func TestAccountBasicRouteMapping(t *testing.T) { opts := DefaultOptions() opts.Port = -1 s := RunServer(opts) defer s.Shutdown() acc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT) acc.AddMapping("foo", "bar") nc := natsConnect(t, s.ClientURL()) defer nc.Close() fsub, _ := nc.SubscribeSync("foo") bsub, _ := nc.SubscribeSync("bar") nc.Publish("foo", nil) nc.Flush() checkPending := func(sub *nats.Subscription, expected int) { t.Helper() if n, _, _ := sub.Pending(); n != expected { t.Fatalf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } } checkPending(fsub, 0) checkPending(bsub, 1) acc.RemoveMapping("foo") nc.Publish("foo", nil) nc.Flush() checkPending(fsub, 1) checkPending(bsub, 1) } func TestAccountWildcardRouteMapping(t *testing.T) { opts := DefaultOptions() opts.Port = -1 s := RunServer(opts) defer s.Shutdown() acc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT) addMap := func(src, dest string) { t.Helper() if err := acc.AddMapping(src, dest); err != nil { t.Fatalf("Error adding mapping: %v", err) } } addMap("foo.*.*", "bar.$2.$1") addMap("bar.*.>", "baz.$1.>") nc := natsConnect(t, s.ClientURL()) defer nc.Close() pub := func(subj string) { t.Helper() err := nc.Publish(subj, nil) if err == nil { err = nc.Flush() } if err != nil { t.Fatalf("Error publishing: %v", err) } } fsub, _ := nc.SubscribeSync("foo.>") bsub, _ := nc.SubscribeSync("bar.>") zsub, _ := nc.SubscribeSync("baz.>") checkPending := func(sub *nats.Subscription, expected int) { t.Helper() if n, _, _ := sub.Pending(); n != expected { t.Fatalf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } } pub("foo.1.2") checkPending(fsub, 0) checkPending(bsub, 1) checkPending(zsub, 0) } func TestAccountRouteMappingChangesAfterClientStart(t *testing.T) { opts := DefaultOptions() opts.Port = -1 s := RunServer(opts) defer s.Shutdown() // Create the client first then add in mapping. nc := natsConnect(t, s.ClientURL()) defer nc.Close() nc.Flush() acc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT) acc.AddMapping("foo", "bar") fsub, _ := nc.SubscribeSync("foo") bsub, _ := nc.SubscribeSync("bar") nc.Publish("foo", nil) nc.Flush() checkPending := func(sub *nats.Subscription, expected int) { t.Helper() if n, _, _ := sub.Pending(); n != expected { t.Fatalf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } } checkPending(fsub, 0) checkPending(bsub, 1) acc.RemoveMapping("foo") nc.Publish("foo", nil) nc.Flush() checkPending(fsub, 1) checkPending(bsub, 1) } func TestAccountSimpleWeightedRouteMapping(t *testing.T) { opts := DefaultOptions() opts.Port = -1 s := RunServer(opts) defer s.Shutdown() acc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT) acc.AddWeightedMappings("foo", NewMapDest("bar", 50)) nc := natsConnect(t, s.ClientURL()) defer nc.Close() fsub, _ := nc.SubscribeSync("foo") bsub, _ := nc.SubscribeSync("bar") total := 500 for i := 0; i < total; i++ { nc.Publish("foo", nil) } nc.Flush() fpending, _, _ := fsub.Pending() bpending, _, _ := bsub.Pending() h := total / 2 tp := h / 5 min, max := h-tp, h+tp if fpending < min || fpending > max { t.Fatalf("Expected about %d msgs, got %d and %d", h, fpending, bpending) } } func TestAccountMultiWeightedRouteMappings(t *testing.T) { opts := DefaultOptions() opts.Port = -1 s := RunServer(opts) defer s.Shutdown() acc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT) // Check failures for bad weights. shouldErr := func(rds ...*MapDest) { t.Helper() if acc.AddWeightedMappings("foo", rds...) == nil { t.Fatalf("Expected an error, got none") } } shouldNotErr := func(rds ...*MapDest) { t.Helper() if err := acc.AddWeightedMappings("foo", rds...); err != nil { t.Fatalf("Unexpected error: %v", err) } } shouldErr(NewMapDest("bar", 150)) shouldNotErr(NewMapDest("bar", 50)) shouldNotErr(NewMapDest("bar", 50), NewMapDest("baz", 50)) // Same dest duplicated should error. shouldErr(NewMapDest("bar", 50), NewMapDest("bar", 50)) // total over 100 shouldErr(NewMapDest("bar", 50), NewMapDest("baz", 60)) acc.RemoveMapping("foo") // 20 for original, you can leave it off will be auto-added. shouldNotErr(NewMapDest("bar", 50), NewMapDest("baz", 30)) nc := natsConnect(t, s.ClientURL()) defer nc.Close() fsub, _ := nc.SubscribeSync("foo") bsub, _ := nc.SubscribeSync("bar") zsub, _ := nc.SubscribeSync("baz") // For checking later. rds := []struct { sub *nats.Subscription w uint8 }{ {fsub, 20}, {bsub, 50}, {zsub, 30}, } total := 5000 for i := 0; i < total; i++ { nc.Publish("foo", nil) } nc.Flush() for _, rd := range rds { pending, _, _ := rd.sub.Pending() expected := total / int(100/rd.w) tp := expected / 5 // 20% min, max := expected-tp, expected+tp if pending < min || pending > max { t.Fatalf("Expected about %d msgs for %q, got %d", expected, rd.sub.Subject, pending) } } } func TestGlobalAccountRouteMappingsConfiguration(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 mappings = { foo: bar foo.*: [ { dest: bar.v1.$1, weight: 40% }, { destination: baz.v2.$1, weight: 20 } ] bar.*.*: RAB.$2.$1 } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL()) defer nc.Close() bsub, _ := nc.SubscribeSync("bar") fsub1, _ := nc.SubscribeSync("bar.v1.>") fsub2, _ := nc.SubscribeSync("baz.v2.>") zsub, _ := nc.SubscribeSync("RAB.>") f22sub, _ := nc.SubscribeSync("foo.*") checkPending := func(sub *nats.Subscription, expected int) { t.Helper() if n, _, _ := sub.Pending(); n != expected { t.Fatalf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } } nc.Publish("foo", nil) nc.Publish("bar.11.22", nil) total := 500 for i := 0; i < total; i++ { nc.Publish("foo.22", nil) } nc.Flush() checkPending(bsub, 1) checkPending(zsub, 1) fpending, _, _ := f22sub.Pending() fpending1, _, _ := fsub1.Pending() fpending2, _, _ := fsub2.Pending() if fpending1 < fpending2 || fpending < fpending2 { t.Fatalf("Loadbalancing seems off for the foo.* mappings: %d and %d and %d", fpending, fpending1, fpending2) } } func TestAccountRouteMappingsConfiguration(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 accounts { synadia { users = [{user: derek, password: foo}] mappings = { foo: bar foo.*: [ { dest: bar.v1.$1, weight: 40% }, { destination: baz.v2.$1, weight: 20 } ] bar.*.*: RAB.$2.$1 } } } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() // We test functionality above, so for this one just make sure we have mappings for the account. acc, _ := s.LookupAccount("synadia") if !acc.hasMappings() { t.Fatalf("Account %q does not have mappings", "synadia") } az, err := s.Accountz(&AccountzOptions{"synadia"}) if err != nil { t.Fatalf("Error getting Accountz: %v", err) } if az.Account == nil { t.Fatalf("Expected an Account") } if len(az.Account.Mappings) != 3 { t.Fatalf("Expected %d mappings, saw %d", 3, len(az.Account.Mappings)) } } func TestAccountRouteMappingsWithLossInjection(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 mappings = { foo: { dest: foo, weight: 80% } bar: { dest: bar, weight: 0% } } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub, _ := nc.SubscribeSync("foo") total := 1000 for i := 0; i < total; i++ { nc.Publish("foo", nil) } nc.Flush() if pending, _, _ := sub.Pending(); pending == total { t.Fatalf("Expected some loss and pending to not be same as sent") } sub, _ = nc.SubscribeSync("bar") for i := 0; i < total; i++ { nc.Publish("bar", nil) } nc.Flush() if pending, _, _ := sub.Pending(); pending != 0 { t.Fatalf("Expected all messages to be dropped and pending to be 0, got %d", pending) } } func TestAccountRouteMappingsWithOriginClusterFilter(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 mappings = { foo: { dest: bar, cluster: SYN, weight: 100% } } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub, _ := nc.SubscribeSync("foo") total := 1000 for i := 0; i < total; i++ { nc.Publish("foo", nil) } nc.Flush() if pending, _, _ := sub.Pending(); pending != total { t.Fatalf("Expected pending to be %d, got %d", total, pending) } s.setClusterName("SYN") sub, _ = nc.SubscribeSync("bar") for i := 0; i < total; i++ { nc.Publish("foo", nil) } nc.Flush() if pending, _, _ := sub.Pending(); pending != total { t.Fatalf("Expected pending to be %d, got %d", total, pending) } } func TestAccountServiceImportWithRouteMappings(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 accounts { foo { users = [{user: derek, password: foo}] exports = [{service: "request"}] } bar { users = [{user: ivan, password: bar}] imports = [{service: {account: "foo", subject:"request"}}] } } `)) s, opts := RunServerWithConfig(cf) defer s.Shutdown() acc, _ := s.LookupAccount("foo") acc.AddMapping("request", "request.v2") // Create the service client first. ncFoo := natsConnect(t, fmt.Sprintf("nats://derek:foo@%s:%d", opts.Host, opts.Port)) defer ncFoo.Close() fooSub := natsSubSync(t, ncFoo, "request.v2") ncFoo.Flush() // Requestor ncBar := natsConnect(t, fmt.Sprintf("nats://ivan:bar@%s:%d", opts.Host, opts.Port)) defer ncBar.Close() ncBar.Publish("request", nil) ncBar.Flush() checkFor(t, time.Second, 10*time.Millisecond, func() error { if n, _, _ := fooSub.Pending(); n != 1 { return fmt.Errorf("Expected a request for %q, but got %d", fooSub.Subject, n) } return nil }) } func TestAccountImportsWithWildcardSupport(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 accounts { foo { users = [{user: derek, password: foo}] exports = [ { service: "request.*" } { stream: "events.>" } { stream: "info.*.*.>" } ] } bar { users = [{user: ivan, password: bar}] imports = [ { service: {account: "foo", subject:"request.*"}, to:"my.request.*"} { stream: {account: "foo", subject:"events.>"}, to:"foo.events.>"} { stream: {account: "foo", subject:"info.*.*.>"}, to:"foo.info.$2.$1.>"} ] } } `)) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncFoo := natsConnect(t, fmt.Sprintf("nats://derek:foo@%s:%d", opts.Host, opts.Port)) defer ncFoo.Close() ncBar := natsConnect(t, fmt.Sprintf("nats://ivan:bar@%s:%d", opts.Host, opts.Port)) defer ncBar.Close() // Create subscriber for the service endpoint in foo. _, err := ncFoo.QueueSubscribe("request.*", "t22", func(m *nats.Msg) { if m.Subject != "request.22" { t.Fatalf("Expected literal subject for request, got %q", m.Subject) } m.Respond([]byte("yes!")) }) if err != nil { t.Fatalf("Error on subscribe: %v", err) } ncFoo.Flush() // Now test service import. resp, err := ncBar.Request("my.request.22", []byte("yes?"), time.Second) if err != nil { t.Fatalf("Expected a response") } if string(resp.Data) != "yes!" { t.Fatalf("Expected a response of %q, got %q", "yes!", resp.Data) } // Now test stream imports. esub, _ := ncBar.SubscribeSync("foo.events.*") // subset isub, _ := ncBar.SubscribeSync("foo.info.>") ncBar.Flush() // Now publish some stream events. ncFoo.Publish("events.22", nil) ncFoo.Publish("info.11.22.bar", nil) ncFoo.Flush() checkPending := func(sub *nats.Subscription, expected int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { if n, _, _ := sub.Pending(); n != expected { return fmt.Errorf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } return nil }) } checkPending(esub, 1) checkPending(isub, 1) // Now check to make sure the subjects are correct etc. m, err := esub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if m.Subject != "foo.events.22" { t.Fatalf("Incorrect subject for stream import, expected %q, got %q", "foo.events.22", m.Subject) } m, err = isub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if m.Subject != "foo.info.22.11.bar" { t.Fatalf("Incorrect subject for stream import, expected %q, got %q", "foo.info.22.11.bar", m.Subject) } } // duplicates TestJWTAccountImportsWithWildcardSupport (jwt_test.go) in config func TestAccountImportsWithWildcardSupportStreamAndService(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 accounts { foo { users = [{user: derek, password: foo}] exports = [ { service: "$request.*.$in.*.>" } { stream: "$events.*.$in.*.>" } ] } bar { users = [{user: ivan, password: bar}] imports = [ { service: {account: "foo", subject:"$request.*.$in.*.>"}, to:"my.request.$2.$1.>"} { stream: {account: "foo", subject:"$events.*.$in.*.>"}, to:"my.events.$2.$1.>"} ] } } `)) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncFoo := natsConnect(t, fmt.Sprintf("nats://derek:foo@%s:%d", opts.Host, opts.Port)) defer ncFoo.Close() ncBar := natsConnect(t, fmt.Sprintf("nats://ivan:bar@%s:%d", opts.Host, opts.Port)) defer ncBar.Close() // Create subscriber for the service endpoint in foo. _, err := ncFoo.Subscribe("$request.>", func(m *nats.Msg) { if m.Subject != "$request.2.$in.1.bar" { t.Fatalf("Expected literal subject for request, got %q", m.Subject) } m.Respond([]byte("yes!")) }) if err != nil { t.Fatalf("Error on subscribe: %v", err) } ncFoo.Flush() // Now test service import. if resp, err := ncBar.Request("my.request.1.2.bar", []byte("yes?"), time.Second); err != nil { t.Fatalf("Expected a response") } else if string(resp.Data) != "yes!" { t.Fatalf("Expected a response of %q, got %q", "yes!", resp.Data) } subBar, err := ncBar.SubscribeSync("my.events.>") if err != nil { t.Fatalf("Expected a response") } ncBar.Flush() ncFoo.Publish("$events.1.$in.2.bar", nil) m, err := subBar.NextMsg(time.Second) if err != nil { t.Fatalf("Expected a response") } if m.Subject != "my.events.2.1.bar" { t.Fatalf("Expected literal subject for request, got %q", m.Subject) } } func BenchmarkNewRouteReply(b *testing.B) { opts := defaultServerOptions s := New(&opts) g := s.globalAccount() b.ResetTimer() for i := 0; i < b.N; i++ { g.newServiceReply(false) } } func TestSamplingHeader(t *testing.T) { test := func(expectSampling bool, h http.Header) { t.Helper() b := strings.Builder{} b.WriteString("\r\n") // simulate status line h.Write(&b) b.WriteString("\r\n") hdrString := b.String() c := &client{parseState: parseState{msgBuf: []byte(hdrString), pa: pubArg{hdr: len(hdrString)}}} sample, hdr := shouldSample(&serviceLatency{0, "foo"}, c) if expectSampling { if !sample { t.Fatal("Expected to sample") } else if hdr == nil { t.Fatal("Expected a header") } for k, v := range h { if hdr.Get(k) != v[0] { t.Fatal("Expect header to match") } } } else { if sample { t.Fatal("Expected not to sample") } else if hdr != nil { t.Fatal("Expected no header") } } } test(false, http.Header{"Uber-Trace-Id": []string{"0:0:0:0"}}) test(false, http.Header{"Uber-Trace-Id": []string{"0:0:0:00"}}) // one byte encoded as two hex digits test(true, http.Header{"Uber-Trace-Id": []string{"0:0:0:1"}}) test(true, http.Header{"Uber-Trace-Id": []string{"0:0:0:01"}}) test(true, http.Header{"Uber-Trace-Id": []string{"0:0:0:5"}}) // debug and sample test(true, http.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1"}}) test(true, http.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}}) test(false, http.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:0"}}) test(false, http.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:0"}}) test(true, http.Header{"X-B3-Sampled": []string{"1"}}) test(false, http.Header{"X-B3-Sampled": []string{"0"}}) test(true, http.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}}) // decision left to recipient test(false, http.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, "X-B3-Sampled": []string{"0"}}) test(true, http.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, "X-B3-Sampled": []string{"1"}}) test(false, http.Header{"B3": []string{"0"}}) // deny only test(false, http.Header{"B3": []string{"0-0-0-0"}}) test(false, http.Header{"B3": []string{"0-0-0"}}) test(true, http.Header{"B3": []string{"0-0-1-0"}}) test(true, http.Header{"B3": []string{"0-0-1"}}) test(true, http.Header{"B3": []string{"0-0-d"}}) // debug is not a deny test(true, http.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1"}}) test(true, http.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90"}}) test(false, http.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-0-05e3ac9a4f6e3b90"}}) test(true, http.Header{"traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}) test(false, http.Header{"traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"}}) } func TestAccountSystemPermsWithGlobalAccess(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Make sure we can connect with no auth to global account as normal. nc, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Failed to create client: %v", err) } defer nc.Close() // Make sure we can connect to the system account with correct credentials. sc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer sc.Close() } const importSubscriptionOverlapTemplate = ` listen: 127.0.0.1:-1 accounts: { ACCOUNT_X: { users: [ {user: publisher} ] exports: [ {stream: %s} ] }, ACCOUNT_Y: { users: [ {user: subscriber} ] imports: [ {stream: {account: ACCOUNT_X, subject: %s }, %s} ] } }` func TestImportSubscriptionPartialOverlapWithPrefix(t *testing.T) { cf := createConfFile(t, []byte(fmt.Sprintf(importSubscriptionOverlapTemplate, ">", ">", "prefix: myprefix"))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncX := natsConnect(t, fmt.Sprintf("nats://%s:%s@127.0.0.1:%d", "publisher", "", opts.Port)) defer ncX.Close() ncY := natsConnect(t, fmt.Sprintf("nats://%s:%s@127.0.0.1:%d", "subscriber", "", opts.Port)) defer ncY.Close() for _, subj := range []string{">", "myprefix.*", "myprefix.>", "myprefix.test", "*.>", "*.*", "*.test"} { t.Run(subj, func(t *testing.T) { sub, err := ncY.SubscribeSync(subj) sub.AutoUnsubscribe(1) require_NoError(t, err) require_NoError(t, ncY.Flush()) ncX.Publish("test", []byte("hello")) m, err := sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, string(m.Data) == "hello") }) } } func TestImportSubscriptionPartialOverlapWithTransform(t *testing.T) { cf := createConfFile(t, []byte(fmt.Sprintf(importSubscriptionOverlapTemplate, "*.*.>", "*.*.>", "to: myprefix.$2.$1.>"))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncX := natsConnect(t, fmt.Sprintf("nats://%s:%s@127.0.0.1:%d", "publisher", "", opts.Port)) defer ncX.Close() ncY := natsConnect(t, fmt.Sprintf("nats://%s:%s@127.0.0.1:%d", "subscriber", "", opts.Port)) defer ncY.Close() for _, subj := range []string{">", "*.*.*.>", "*.2.*.>", "*.*.1.>", "*.2.1.>", "*.*.*.*", "*.2.1.*", "*.*.*.test", "*.*.1.test", "*.2.*.test", "*.2.1.test", "myprefix.*.*.*", "myprefix.>", "myprefix.*.>", "myprefix.*.*.>", "myprefix.2.>", "myprefix.2.1.>", "myprefix.*.1.>", "myprefix.2.*.>", "myprefix.2.1.*", "myprefix.*.*.test", "myprefix.2.1.test"} { t.Run(subj, func(t *testing.T) { sub, err := ncY.SubscribeSync(subj) sub.AutoUnsubscribe(1) require_NoError(t, err) require_NoError(t, ncY.Flush()) ncX.Publish("1.2.test", []byte("hello")) m, err := sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, string(m.Data) == "hello") require_Equal(t, m.Subject, "myprefix.2.1.test") }) } } func TestAccountLimitsServerConfig(t *testing.T) { cf := createConfFile(t, []byte(` port: -1 max_connections: 10 accounts { MAXC { users = [{user: derek, password: foo}] limits { max_connections: 5 max_subs: 10 max_payload: 32k max_leafnodes: 1 } } } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() acc, err := s.lookupAccount("MAXC") require_NoError(t, err) if mc := acc.MaxActiveConnections(); mc != 5 { t.Fatalf("Did not set MaxActiveConnections properly, expected 5, got %d", mc) } if mlc := acc.MaxActiveLeafNodes(); mlc != 1 { t.Fatalf("Did not set MaxActiveLeafNodes properly, expected 1, got %d", mlc) } // Do quick test on connections, but if they are registered above should be good. for i := 0; i < 5; i++ { c, err := nats.Connect(s.ClientURL(), nats.UserInfo("derek", "foo")) require_NoError(t, err) defer c.Close() } // Should fail. _, err = nats.Connect(s.ClientURL(), nats.UserInfo("derek", "foo")) require_Error(t, err) } func TestAccountUserSubPermsWithQueueGroups(t *testing.T) { cf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 authorization { users = [ { user: user, password: "pass", permissions: { publish: "foo.restricted" subscribe: { allow: "foo.>", deny: "foo.restricted" } allow_responses: { max: 1, ttl: 0s } } } ]} `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("user", "pass")) require_NoError(t, err) defer nc.Close() // qsub solo. qsub, err := nc.QueueSubscribeSync("foo.>", "qg") require_NoError(t, err) err = nc.Publish("foo.restricted", []byte("RESTRICTED")) require_NoError(t, err) nc.Flush() // Expect no msgs. checkSubsPending(t, qsub, 0) } func TestAccountImportCycle(t *testing.T) { tmpl := ` port: -1 accounts: { CP: { users: [ {user: cp, password: cp}, ], exports: [ {service: "q1.>", response_type: Singleton}, {service: "q2.>", response_type: Singleton}, %s ], }, A: { users: [ {user: a, password: a}, ], imports: [ {service: {account: CP, subject: "q1.>"}}, {service: {account: CP, subject: "q2.>"}}, %s ] }, } ` cf := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, _EMPTY_))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() ncCp, err := nats.Connect(s.ClientURL(), nats.UserInfo("cp", "cp")) require_NoError(t, err) defer ncCp.Close() ncA, err := nats.Connect(s.ClientURL(), nats.UserInfo("a", "a")) require_NoError(t, err) defer ncA.Close() // setup responder natsSub(t, ncCp, "q1.>", func(m *nats.Msg) { m.Respond([]byte("reply")) }) // setup requestor ib := "q2.inbox" subAResp, err := ncA.SubscribeSync(ib) ncA.Flush() require_NoError(t, err) req := func() { t.Helper() // send request err = ncA.PublishRequest("q1.a", ib, []byte("test")) ncA.Flush() require_NoError(t, err) mRep, err := subAResp.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, string(mRep.Data), "reply") } req() // Update the config and do a config reload and make sure it all still work changeCurrentConfigContentWithNewContent(t, cf, []byte( fmt.Sprintf(tmpl, `{service: "q3.>", response_type: Singleton},`, `{service: {account: CP, subject: "q3.>"}},`))) err = s.Reload() require_NoError(t, err) req() } func TestAccountImportOwnExport(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts: { A: { exports: [ { service: echo, accounts: [A], latency: { subject: "latency.echo" } } ], imports: [ { service: { account: A, subject: echo } } ] users: [ { user: user, pass: pass } ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL(), nats.UserInfo("user", "pass")) defer nc.Close() natsSub(t, nc, "echo", func(m *nats.Msg) { m.Respond(nil) }) _, err := nc.Request("echo", []byte("request"), time.Second) require_NoError(t, err) } // Test for a bug that would cause duplicate deliveries in certain situations when // service export/imports and leafnodes involved. // https://github.com/nats-io/nats-server/issues/3191 func TestAccountImportDuplicateResponseDeliveryWithLeafnodes(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts: { A: { users = [{user: A, password: P}] exports: [ { service: "foo", response_type: stream } ] } B: { users = [{user: B, password: P}] imports: [ { service: {account: "A", subject:"foo"} } ] } } leaf { listen: "127.0.0.1:17222" } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Requestors will connect to account B. nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("B", "P")) require_NoError(t, err) defer nc.Close() // By sending a request (regardless of no responders), this will trigger a wildcard _R_ subscription since // we do not have a leafnode connected. nc.PublishRequest("foo", "reply", nil) nc.Flush() // Now connect the LN. This will be where the service responder lives. conf = createConfFile(t, []byte(` port: -1 leaf { remotes [ { url: "nats://A:P@127.0.0.1:17222" } ] } `)) ln, _ := RunServerWithConfig(conf) defer ln.Shutdown() checkLeafNodeConnected(t, s) // Now attach a responder to the LN. lnc, err := nats.Connect(ln.ClientURL()) require_NoError(t, err) defer lnc.Close() lnc.Subscribe("foo", func(m *nats.Msg) { m.Respond([]byte("bar")) }) lnc.Flush() checkSubInterest(t, s, "A", "foo", time.Second) // Make sure it works, but request only wants one, so need second test to show failure, but // want to make sure we are wired up correctly. _, err = nc.Request("foo", nil, time.Second) require_NoError(t, err) // Now setup inbox reply so we can check if we get multiple responses. reply := nats.NewInbox() sub, err := nc.SubscribeSync(reply) require_NoError(t, err) nc.PublishRequest("foo", reply, nil) // Do another to make sure we know the other request will have been processed too. _, err = nc.Request("foo", nil, time.Second) require_NoError(t, err) if n, _, _ := sub.Pending(); n > 1 { t.Fatalf("Expected only 1 response, got %d", n) } } func TestAccountReloadServiceImportPanic(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts { A { users = [ { user: "a", pass: "p" } ] exports [ { service: "HELP" } ] } B { users = [ { user: "b", pass: "p" } ] imports [ { service: { account: A, subject: "HELP"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Now connect up the subscriber for HELP. No-op for this test. nc, _ := jsClientConnect(t, s, nats.UserInfo("a", "p")) defer nc.Close() _, err := nc.Subscribe("HELP", func(m *nats.Msg) { m.Respond([]byte("OK")) }) require_NoError(t, err) // Now create connection to account b where we will publish to HELP. nc, _ = jsClientConnect(t, s, nats.UserInfo("b", "p")) defer nc.Close() // We want to continually be publishing messages that will trigger the service import while calling reload. done := make(chan bool) var wg sync.WaitGroup wg.Add(1) var requests, responses atomic.Uint64 reply := nats.NewInbox() _, err = nc.Subscribe(reply, func(m *nats.Msg) { responses.Add(1) }) require_NoError(t, err) go func() { defer wg.Done() for { select { case <-done: return default: nc.PublishRequest("HELP", reply, []byte("HELP")) requests.Add(1) } } }() // Perform a bunch of reloads. for i := 0; i < 1000; i++ { require_NoError(t, s.Reload()) } close(done) wg.Wait() totalRequests := requests.Load() checkFor(t, 20*time.Second, 250*time.Millisecond, func() error { resp := responses.Load() if resp == totalRequests { return nil } return fmt.Errorf("Have not received all responses, want %d got %d", totalRequests, resp) }) } // https://github.com/nats-io/nats-server/issues/4674 func TestAccountServiceAndStreamExportDoubleDelivery(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { tenant1: { jetstream: enabled users: [ { user: "one", password: "one" } ] exports: [ { stream: "DW.>" } { service: "DW.>" } ] } global: { jetstream: enabled users: [ { user: "global", password: "global" } ] imports: [ { stream: { account: tenant1, subject: "DW.>" }, prefix: tenant1 } { service: { account: tenant1, subject: "DW.>" }, to: "tenant1.DW.>" } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Now connect up the subscriber for HELP. No-op for this test. nc, _ := jsClientConnect(t, s, nats.UserInfo("one", "one")) defer nc.Close() var msgs atomic.Int32 _, err := nc.Subscribe(">", func(m *nats.Msg) { msgs.Add(1) }) require_NoError(t, err) nc.Publish("DW.test.123", []byte("test")) time.Sleep(200 * time.Millisecond) require_Equal(t, msgs.Load(), 1) } func TestAccountServiceImportNoResponders(t *testing.T) { // Setup NATS server. cf := createConfFile(t, []byte(` port: -1 accounts: { accExp: { users: [{user: accExp, password: accExp}] exports: [{service: "foo"}] } accImp: { users: [{user: accImp, password: accImp}] imports: [{service: {account: accExp, subject: "foo"}}] } } `)) s, _ := RunServerWithConfig(cf) defer s.Shutdown() // Connect to the import account. We will not setup any responders, so a request should // error out with ErrNoResponders. nc := natsConnect(t, s.ClientURL(), nats.UserInfo("accImp", "accImp")) defer nc.Close() _, err := nc.Request("foo", []byte("request"), 250*time.Millisecond) require_Error(t, err, nats.ErrNoResponders) } nats-server-2.10.27/server/auth.go000066400000000000000000001272141477524627100167600ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "crypto/sha256" "crypto/subtle" "crypto/tls" "crypto/x509/pkix" "encoding/asn1" "encoding/base64" "encoding/hex" "fmt" "net" "net/url" "regexp" "strings" "sync/atomic" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/internal/ldap" "github.com/nats-io/nkeys" "golang.org/x/crypto/bcrypt" ) // Authentication is an interface for implementing authentication type Authentication interface { // Check if a client is authorized to connect Check(c ClientAuthentication) bool } // ClientAuthentication is an interface for client authentication type ClientAuthentication interface { // GetOpts gets options associated with a client GetOpts() *ClientOpts // GetTLSConnectionState if TLS is enabled, TLS ConnectionState, nil otherwise GetTLSConnectionState() *tls.ConnectionState // RegisterUser optionally map a user after auth. RegisterUser(*User) // RemoteAddress expose the connection information of the client RemoteAddress() net.Addr // GetNonce is the nonce presented to the user in the INFO line GetNonce() []byte // Kind indicates what type of connection this is matching defined constants like CLIENT, ROUTER, GATEWAY, LEAF etc Kind() int } // NkeyUser is for multiple nkey based users type NkeyUser struct { Nkey string `json:"user"` Issued int64 `json:"issued,omitempty"` // this is a copy of the issued at (iat) field in the jwt Permissions *Permissions `json:"permissions,omitempty"` Account *Account `json:"account,omitempty"` SigningKey string `json:"signing_key,omitempty"` AllowedConnectionTypes map[string]struct{} `json:"connection_types,omitempty"` } // User is for multiple accounts/users. type User struct { Username string `json:"user"` Password string `json:"password"` Permissions *Permissions `json:"permissions,omitempty"` Account *Account `json:"account,omitempty"` ConnectionDeadline time.Time `json:"connection_deadline,omitempty"` AllowedConnectionTypes map[string]struct{} `json:"connection_types,omitempty"` } // clone performs a deep copy of the User struct, returning a new clone with // all values copied. func (u *User) clone() *User { if u == nil { return nil } clone := &User{} *clone = *u // Account is not cloned because it is always by reference to an existing struct. clone.Permissions = u.Permissions.clone() if u.AllowedConnectionTypes != nil { clone.AllowedConnectionTypes = make(map[string]struct{}) for k, v := range u.AllowedConnectionTypes { clone.AllowedConnectionTypes[k] = v } } return clone } // clone performs a deep copy of the NkeyUser struct, returning a new clone with // all values copied. func (n *NkeyUser) clone() *NkeyUser { if n == nil { return nil } clone := &NkeyUser{} *clone = *n // Account is not cloned because it is always by reference to an existing struct. clone.Permissions = n.Permissions.clone() if n.AllowedConnectionTypes != nil { clone.AllowedConnectionTypes = make(map[string]struct{}) for k, v := range n.AllowedConnectionTypes { clone.AllowedConnectionTypes[k] = v } } return clone } // SubjectPermission is an individual allow and deny struct for publish // and subscribe authorizations. type SubjectPermission struct { Allow []string `json:"allow,omitempty"` Deny []string `json:"deny,omitempty"` } // ResponsePermission can be used to allow responses to any reply subject // that is received on a valid subscription. type ResponsePermission struct { MaxMsgs int `json:"max"` Expires time.Duration `json:"ttl"` } // Permissions are the allowed subjects on a per // publish or subscribe basis. type Permissions struct { Publish *SubjectPermission `json:"publish"` Subscribe *SubjectPermission `json:"subscribe"` Response *ResponsePermission `json:"responses,omitempty"` } // RoutePermissions are similar to user permissions // but describe what a server can import/export from and to // another server. type RoutePermissions struct { Import *SubjectPermission `json:"import"` Export *SubjectPermission `json:"export"` } // clone will clone an individual subject permission. func (p *SubjectPermission) clone() *SubjectPermission { if p == nil { return nil } clone := &SubjectPermission{} if p.Allow != nil { clone.Allow = make([]string, len(p.Allow)) copy(clone.Allow, p.Allow) } if p.Deny != nil { clone.Deny = make([]string, len(p.Deny)) copy(clone.Deny, p.Deny) } return clone } // clone performs a deep copy of the Permissions struct, returning a new clone // with all values copied. func (p *Permissions) clone() *Permissions { if p == nil { return nil } clone := &Permissions{} if p.Publish != nil { clone.Publish = p.Publish.clone() } if p.Subscribe != nil { clone.Subscribe = p.Subscribe.clone() } if p.Response != nil { clone.Response = &ResponsePermission{ MaxMsgs: p.Response.MaxMsgs, Expires: p.Response.Expires, } } return clone } // checkAuthforWarnings will look for insecure settings and log concerns. // Lock is assumed held. func (s *Server) checkAuthforWarnings() { warn := false opts := s.getOpts() if opts.Password != _EMPTY_ && !isBcrypt(opts.Password) { warn = true } for _, u := range s.users { // Skip warn if using TLS certs based auth // unless a password has been left in the config. if u.Password == _EMPTY_ && opts.TLSMap { continue } // Check if this is our internal sys client created on the fly. if s.sysAccOnlyNoAuthUser != _EMPTY_ && u.Username == s.sysAccOnlyNoAuthUser { continue } if !isBcrypt(u.Password) { warn = true break } } if warn { // Warning about using plaintext passwords. s.Warnf("Plaintext passwords detected, use nkeys or bcrypt") } } // If Users or Nkeys options have definitions without an account defined, // assign them to the default global account. // Lock should be held. func (s *Server) assignGlobalAccountToOrphanUsers(nkeys map[string]*NkeyUser, users map[string]*User) { for _, u := range users { if u.Account == nil { u.Account = s.gacc } } for _, u := range nkeys { if u.Account == nil { u.Account = s.gacc } } } // If the given permissions has a ResponsePermission // set, ensure that defaults are set (if values are 0) // and that a Publish permission is set, and Allow // is disabled if not explicitly set. func validateResponsePermissions(p *Permissions) { if p == nil || p.Response == nil { return } if p.Publish == nil { p.Publish = &SubjectPermission{} } if p.Publish.Allow == nil { // We turn off the blanket allow statement. p.Publish.Allow = []string{} } // If there is a response permission, ensure // that if value is 0, we set the default value. if p.Response.MaxMsgs == 0 { p.Response.MaxMsgs = DEFAULT_ALLOW_RESPONSE_MAX_MSGS } if p.Response.Expires == 0 { p.Response.Expires = DEFAULT_ALLOW_RESPONSE_EXPIRATION } } // configureAuthorization will do any setup needed for authorization. // Lock is assumed held. func (s *Server) configureAuthorization() { opts := s.getOpts() if opts == nil { return } // Check for multiple users first // This just checks and sets up the user map if we have multiple users. if opts.CustomClientAuthentication != nil { s.info.AuthRequired = true } else if s.trustedKeys != nil { s.info.AuthRequired = true } else if opts.Nkeys != nil || opts.Users != nil { s.nkeys, s.users = s.buildNkeysAndUsersFromOptions(opts.Nkeys, opts.Users) s.info.AuthRequired = true } else if opts.Username != _EMPTY_ || opts.Authorization != _EMPTY_ { s.info.AuthRequired = true } else { s.users = nil s.nkeys = nil s.info.AuthRequired = false } // Do similar for websocket config s.wsConfigAuth(&opts.Websocket) // And for mqtt config s.mqttConfigAuth(&opts.MQTT) // Check for server configured auth callouts. if opts.AuthCallout != nil { s.mu.Unlock() // Give operator log entries if not valid account and auth_users. _, err := s.lookupAccount(opts.AuthCallout.Account) s.mu.Lock() if err != nil { s.Errorf("Authorization callout account %q not valid", opts.AuthCallout.Account) } for _, u := range opts.AuthCallout.AuthUsers { // Check for user in users and nkeys since this is server config. var found bool if len(s.users) > 0 { _, found = s.users[u] } if !found && len(s.nkeys) > 0 { _, found = s.nkeys[u] } if !found { s.Errorf("Authorization callout user %q not valid: %v", u, err) } } } } // Takes the given slices of NkeyUser and User options and build // corresponding maps used by the server. The users are cloned // so that server does not reference options. // The global account is assigned to users that don't have an // existing account. // Server lock is held on entry. func (s *Server) buildNkeysAndUsersFromOptions(nko []*NkeyUser, uo []*User) (map[string]*NkeyUser, map[string]*User) { var nkeys map[string]*NkeyUser var users map[string]*User if nko != nil { nkeys = make(map[string]*NkeyUser, len(nko)) for _, u := range nko { copy := u.clone() if u.Account != nil { if v, ok := s.accounts.Load(u.Account.Name); ok { copy.Account = v.(*Account) } } if copy.Permissions != nil { validateResponsePermissions(copy.Permissions) } nkeys[u.Nkey] = copy } } if uo != nil { users = make(map[string]*User, len(uo)) for _, u := range uo { copy := u.clone() if u.Account != nil { if v, ok := s.accounts.Load(u.Account.Name); ok { copy.Account = v.(*Account) } } if copy.Permissions != nil { validateResponsePermissions(copy.Permissions) } users[u.Username] = copy } } s.assignGlobalAccountToOrphanUsers(nkeys, users) return nkeys, users } // checkAuthentication will check based on client type and // return boolean indicating if client is authorized. func (s *Server) checkAuthentication(c *client) bool { switch c.kind { case CLIENT: return s.isClientAuthorized(c) case ROUTER: return s.isRouterAuthorized(c) case GATEWAY: return s.isGatewayAuthorized(c) case LEAF: return s.isLeafNodeAuthorized(c) default: return false } } // isClientAuthorized will check the client against the proper authorization method and data. // This could be nkey, token, or username/password based. func (s *Server) isClientAuthorized(c *client) bool { opts := s.getOpts() // Check custom auth first, then jwts, then nkeys, then // multiple users with TLS map if enabled, then token, // then single user/pass. if opts.CustomClientAuthentication != nil && !opts.CustomClientAuthentication.Check(c) { return false } if opts.CustomClientAuthentication == nil && !s.processClientOrLeafAuthentication(c, opts) { return false } if c.kind == CLIENT || c.kind == LEAF { // Generate an event if we have a system account. s.accountConnectEvent(c) } return true } // returns false if the client needs to be disconnected func (c *client) matchesPinnedCert(tlsPinnedCerts PinnedCertSet) bool { if tlsPinnedCerts == nil { return true } tlsState := c.GetTLSConnectionState() if tlsState == nil || len(tlsState.PeerCertificates) == 0 || tlsState.PeerCertificates[0] == nil { c.Debugf("Failed pinned cert test as client did not provide a certificate") return false } sha := sha256.Sum256(tlsState.PeerCertificates[0].RawSubjectPublicKeyInfo) keyId := hex.EncodeToString(sha[:]) if _, ok := tlsPinnedCerts[keyId]; !ok { c.Debugf("Failed pinned cert test for key id: %s", keyId) return false } return true } func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.UserClaims, acc *Account) (jwt.UserPermissionLimits, error) { nArrayCartesianProduct := func(a ...[]string) [][]string { c := 1 for _, a := range a { c *= len(a) } if c == 0 { return nil } p := make([][]string, c) b := make([]string, c*len(a)) n := make([]int, len(a)) s := 0 for i := range p { e := s + len(a) pi := b[s:e] p[i] = pi s = e for j, n := range n { pi[j] = a[j][n] } for j := len(n) - 1; j >= 0; j-- { n[j]++ if n[j] < len(a[j]) { break } n[j] = 0 } } return p } applyTemplate := func(list jwt.StringList, failOnBadSubject bool) (jwt.StringList, error) { found := false FOR_FIND: for i := 0; i < len(list); i++ { // check if templates are present for _, tk := range strings.Split(list[i], tsep) { if strings.HasPrefix(tk, "{{") && strings.HasSuffix(tk, "}}") { found = true break FOR_FIND } } } if !found { return list, nil } // process the templates emittedList := make([]string, 0, len(list)) for i := 0; i < len(list); i++ { tokens := strings.Split(list[i], tsep) newTokens := make([]string, len(tokens)) tagValues := [][]string{} for tokenNum, tk := range tokens { if strings.HasPrefix(tk, "{{") && strings.HasSuffix(tk, "}}") { op := strings.ToLower(strings.TrimSuffix(strings.TrimPrefix(tk, "{{"), "}}")) switch { case op == "name()": tk = ujwt.Name case op == "subject()": tk = ujwt.Subject case op == "account-name()": acc.mu.RLock() name := acc.nameTag acc.mu.RUnlock() tk = name case op == "account-subject()": tk = ujwt.IssuerAccount case (strings.HasPrefix(op, "tag(") || strings.HasPrefix(op, "account-tag(")) && strings.HasSuffix(op, ")"): // insert dummy tav value that will throw of subject validation (in case nothing is found) tk = _EMPTY_ // collect list of matching tag values var tags jwt.TagList var tagPrefix string if strings.HasPrefix(op, "account-tag(") { acc.mu.RLock() tags = acc.tags acc.mu.RUnlock() tagPrefix = fmt.Sprintf("%s:", strings.ToLower( strings.TrimSuffix(strings.TrimPrefix(op, "account-tag("), ")"))) } else { tags = ujwt.Tags tagPrefix = fmt.Sprintf("%s:", strings.ToLower( strings.TrimSuffix(strings.TrimPrefix(op, "tag("), ")"))) } valueList := []string{} for _, tag := range tags { if strings.HasPrefix(tag, tagPrefix) { tagValue := strings.TrimPrefix(tag, tagPrefix) valueList = append(valueList, tagValue) } } if len(valueList) != 0 { tagValues = append(tagValues, valueList) } default: // if macro is not recognized, throw off subject check on purpose tk = " " } } newTokens[tokenNum] = tk } // fill in tag value placeholders if len(tagValues) == 0 { emitSubj := strings.Join(newTokens, tsep) if IsValidSubject(emitSubj) { emittedList = append(emittedList, emitSubj) } else if failOnBadSubject { return nil, fmt.Errorf("generated invalid subject") } // else skip emitting } else { // compute the cartesian product and compute subject to emit for each combination for _, valueList := range nArrayCartesianProduct(tagValues...) { b := strings.Builder{} for i, token := range newTokens { if token == _EMPTY_ && len(valueList) > 0 { b.WriteString(valueList[0]) valueList = valueList[1:] } else { b.WriteString(token) } if i != len(newTokens)-1 { b.WriteString(tsep) } } emitSubj := b.String() if IsValidSubject(emitSubj) { emittedList = append(emittedList, emitSubj) } else if failOnBadSubject { return nil, fmt.Errorf("generated invalid subject") } // else skip emitting } } } return emittedList, nil } subAllowWasNotEmpty := len(lim.Permissions.Sub.Allow) > 0 pubAllowWasNotEmpty := len(lim.Permissions.Pub.Allow) > 0 var err error if lim.Permissions.Sub.Allow, err = applyTemplate(lim.Permissions.Sub.Allow, false); err != nil { return jwt.UserPermissionLimits{}, err } else if lim.Permissions.Sub.Deny, err = applyTemplate(lim.Permissions.Sub.Deny, true); err != nil { return jwt.UserPermissionLimits{}, err } else if lim.Permissions.Pub.Allow, err = applyTemplate(lim.Permissions.Pub.Allow, false); err != nil { return jwt.UserPermissionLimits{}, err } else if lim.Permissions.Pub.Deny, err = applyTemplate(lim.Permissions.Pub.Deny, true); err != nil { return jwt.UserPermissionLimits{}, err } // if pub/sub allow were not empty, but are empty post template processing, add in a "deny >" to compensate if subAllowWasNotEmpty && len(lim.Permissions.Sub.Allow) == 0 { lim.Permissions.Sub.Deny.Add(">") } if pubAllowWasNotEmpty && len(lim.Permissions.Pub.Allow) == 0 { lim.Permissions.Pub.Deny.Add(">") } return lim, nil } func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (authorized bool) { var ( nkey *NkeyUser juc *jwt.UserClaims acc *Account user *User ok bool err error ao bool // auth override ) // Check if we have auth callouts enabled at the server level or in the bound account. defer func() { // Default reason reason := AuthenticationViolation.String() // No-op if juc == nil && opts.AuthCallout == nil { if !authorized { s.sendAccountAuthErrorEvent(c, c.acc, reason) } return } // We have a juc defined here, check account. if juc != nil && !acc.hasExternalAuth() { if !authorized { s.sendAccountAuthErrorEvent(c, c.acc, reason) } return } // We have auth callout set here. var skip bool // Check if we are on the list of auth_users. userID := c.getRawAuthUser() if juc != nil { skip = acc.isExternalAuthUser(userID) } else { for _, u := range opts.AuthCallout.AuthUsers { if userID == u { skip = true break } } } // If we are here we have an auth callout defined and we have failed auth so far // so we will callout to our auth backend for processing. if !skip { authorized, reason = s.processClientOrLeafCallout(c, opts) } // Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject. if authorized { var authAccountName string if juc == nil && opts.AuthCallout != nil { authAccountName = opts.AuthCallout.Account } else if juc != nil { authAccountName = acc.Name } c.mu.Lock() if c.acc != nil && c.acc.Name == authAccountName { c.mergeDenyPermissions(pub, []string{AuthCalloutSubject}) } c.mu.Unlock() } else { // If we are here we failed external authorization. // Send an account scoped event. Server config mode acc will be nil, // so lookup the auth callout assigned account, that is where this will be sent. if acc == nil { acc, _ = s.lookupAccount(opts.AuthCallout.Account) } s.sendAccountAuthErrorEvent(c, acc, reason) } }() s.mu.Lock() authRequired := s.info.AuthRequired if !authRequired { // If no auth required for regular clients, then check if // we have an override for MQTT or Websocket clients. switch c.clientType() { case MQTT: authRequired = s.mqtt.authOverride case WS: authRequired = s.websocket.authOverride } } if !authRequired { // TODO(dlc) - If they send us credentials should we fail? s.mu.Unlock() return true } var ( username string password string token string noAuthUser string pinnedAcounts map[string]struct{} ) tlsMap := opts.TLSMap if c.kind == CLIENT { switch c.clientType() { case MQTT: mo := &opts.MQTT // Always override TLSMap. tlsMap = mo.TLSMap // The rest depends on if there was any auth override in // the mqtt's config. if s.mqtt.authOverride { noAuthUser = mo.NoAuthUser username = mo.Username password = mo.Password token = mo.Token ao = true } case WS: wo := &opts.Websocket // Always override TLSMap. tlsMap = wo.TLSMap // The rest depends on if there was any auth override in // the websocket's config. if s.websocket.authOverride { noAuthUser = wo.NoAuthUser username = wo.Username password = wo.Password token = wo.Token ao = true } } } else { tlsMap = opts.LeafNode.TLSMap } if !ao { noAuthUser = opts.NoAuthUser // If a leaf connects using websocket, and websocket{} block has a no_auth_user // use that one instead. if c.kind == LEAF && c.isWebsocket() && opts.Websocket.NoAuthUser != _EMPTY_ { noAuthUser = opts.Websocket.NoAuthUser } username = opts.Username password = opts.Password token = opts.Authorization } // Check if we have trustedKeys defined in the server. If so we require a user jwt. if s.trustedKeys != nil { if c.opts.JWT == _EMPTY_ { s.mu.Unlock() c.Debugf("Authentication requires a user JWT") return false } // So we have a valid user jwt here. juc, err = jwt.DecodeUserClaims(c.opts.JWT) if err != nil { s.mu.Unlock() c.Debugf("User JWT not valid: %v", err) return false } vr := jwt.CreateValidationResults() juc.Validate(vr) if vr.IsBlocking(true) { s.mu.Unlock() c.Debugf("User JWT no longer valid: %+v", vr) return false } pinnedAcounts = opts.resolverPinnedAccounts } // Check if we have nkeys or users for client. hasNkeys := len(s.nkeys) > 0 hasUsers := len(s.users) > 0 if hasNkeys { if (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ && c.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ && c.opts.Nkey == _EMPTY_ { if _, exists := s.nkeys[noAuthUser]; exists { c.mu.Lock() c.opts.Nkey = noAuthUser c.mu.Unlock() } } if c.opts.Nkey != _EMPTY_ { nkey, ok = s.nkeys[c.opts.Nkey] if !ok || !c.connectionTypeAllowed(nkey.AllowedConnectionTypes) { s.mu.Unlock() return false } } } if hasUsers && nkey == nil { // Check if we are tls verify and are mapping users from the client_certificate. if tlsMap { authorized := checkClientTLSCertSubject(c, func(u string, certDN *ldap.DN, _ bool) (string, bool) { // First do literal lookup using the resulting string representation // of RDNSequence as implemented by the pkix package from Go. if u != _EMPTY_ { usr, ok := s.users[u] if !ok || !c.connectionTypeAllowed(usr.AllowedConnectionTypes) { return _EMPTY_, false } user = usr return usr.Username, true } if certDN == nil { return _EMPTY_, false } // Look through the accounts for a DN that is equal to the one // presented by the certificate. dns := make(map[*User]*ldap.DN) for _, usr := range s.users { if !c.connectionTypeAllowed(usr.AllowedConnectionTypes) { continue } // TODO: Use this utility to make a full validation pass // on start in case tlsmap feature is being used. inputDN, err := ldap.ParseDN(usr.Username) if err != nil { continue } if inputDN.Equal(certDN) { user = usr return usr.Username, true } // In case it did not match exactly, then collect the DNs // and try to match later in case the DN was reordered. dns[usr] = inputDN } // Check in case the DN was reordered. for usr, inputDN := range dns { if inputDN.RDNsMatch(certDN) { user = usr return usr.Username, true } } return _EMPTY_, false }) if !authorized { s.mu.Unlock() return false } if c.opts.Username != _EMPTY_ { s.Warnf("User %q found in connect proto, but user required from cert", c.opts.Username) } // Already checked that the client didn't send a user in connect // but we set it here to be able to identify it in the logs. c.opts.Username = user.Username } else { if (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ && c.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ { if u, exists := s.users[noAuthUser]; exists { c.mu.Lock() c.opts.Username = u.Username c.opts.Password = u.Password c.mu.Unlock() } } if c.opts.Username != _EMPTY_ { user, ok = s.users[c.opts.Username] if !ok || !c.connectionTypeAllowed(user.AllowedConnectionTypes) { s.mu.Unlock() return false } } } } s.mu.Unlock() // If we have a jwt and a userClaim, make sure we have the Account, etc associated. // We need to look up the account. This will use an account resolver if one is present. if juc != nil { issuer := juc.Issuer if juc.IssuerAccount != _EMPTY_ { issuer = juc.IssuerAccount } if pinnedAcounts != nil { if _, ok := pinnedAcounts[issuer]; !ok { c.Debugf("Account %s not listed as operator pinned account", issuer) atomic.AddUint64(&s.pinnedAccFail, 1) return false } } if acc, err = s.LookupAccount(issuer); acc == nil { c.Debugf("Account JWT lookup error: %v", err) return false } acc.mu.RLock() aissuer := acc.Issuer acc.mu.RUnlock() if !s.isTrustedIssuer(aissuer) { c.Debugf("Account JWT not signed by trusted operator") return false } if scope, ok := acc.hasIssuer(juc.Issuer); !ok { c.Debugf("User JWT issuer is not known") return false } else if scope != nil { if err := scope.ValidateScopedSigner(juc); err != nil { c.Debugf("User JWT is not valid: %v", err) return false } else if uSc, ok := scope.(*jwt.UserScope); !ok { c.Debugf("User JWT is not valid") return false } else if juc.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, juc, acc); err != nil { c.Debugf("User JWT generated invalid permissions") return false } } if acc.IsExpired() { c.Debugf("Account JWT has expired") return false } if juc.BearerToken && acc.failBearer() { c.Debugf("Account does not allow bearer tokens") return false } // We check the allowed connection types, but only after processing // of scoped signer (so that it updates `juc` with what is defined // in the account. allowedConnTypes, err := convertAllowedConnectionTypes(juc.AllowedConnectionTypes) if err != nil { // We got an error, which means some connection types were unknown. As long as // a valid one is returned, we proceed with auth. If not, we have to reject. // In other words, suppose that JWT allows "WEBSOCKET" in the array. No error // is returned and allowedConnTypes will contain "WEBSOCKET" only. // Client will be rejected if not a websocket client, or proceed with rest of // auth if it is. // Now suppose JWT allows "WEBSOCKET, MQTT" and say MQTT is not known by this // server. In this case, allowedConnTypes would contain "WEBSOCKET" and we // would get `err` indicating that "MQTT" is an unknown connection type. // If a websocket client connects, it should still be allowed, since after all // the admin wanted to allow websocket and mqtt connection types. // However, say that the JWT only allows "MQTT" (and again suppose this server // does not know about MQTT connection type), then since the allowedConnTypes // map would be empty (no valid types found), and since empty means allow-all, // then we should reject because the intent was to allow connections for this // user only as an MQTT client. c.Debugf("%v", err) if len(allowedConnTypes) == 0 { return false } } if !c.connectionTypeAllowed(allowedConnTypes) { c.Debugf("Connection type not allowed") return false } // skip validation of nonce when presented with a bearer token // FIXME: if BearerToken is only for WSS, need check for server with that port enabled if !juc.BearerToken { // Verify the signature against the nonce. if c.opts.Sig == _EMPTY_ { c.Debugf("Signature missing") return false } sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) if err != nil { // Allow fallback to normal base64. sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) if err != nil { c.Debugf("Signature not valid base64") return false } } pub, err := nkeys.FromPublicKey(juc.Subject) if err != nil { c.Debugf("User nkey not valid: %v", err) return false } if err := pub.Verify(c.nonce, sig); err != nil { c.Debugf("Signature not verified") return false } } if acc.checkUserRevoked(juc.Subject, juc.IssuedAt) { c.Debugf("User authentication revoked") return false } if !validateSrc(juc, c.host) { c.Errorf("Bad src Ip %s", c.host) return false } allowNow, validFor := validateTimes(juc) if !allowNow { c.Errorf("Outside connect times") return false } nkey = buildInternalNkeyUser(juc, allowedConnTypes, acc) if err := c.RegisterNkeyUser(nkey); err != nil { return false } // Warn about JetStream restrictions if c.perms != nil { deniedPub := []string{} deniedSub := []string{} for _, sub := range denyAllJs { if c.perms.pub.deny != nil { if c.perms.pub.deny.HasInterest(sub) { deniedPub = append(deniedPub, sub) } } if c.perms.sub.deny != nil { if c.perms.sub.deny.HasInterest(sub) { deniedSub = append(deniedSub, sub) } } } if len(deniedPub) > 0 || len(deniedSub) > 0 { c.Noticef("Connected %s has JetStream denied on pub: %v sub: %v", c.kindString(), deniedPub, deniedSub) } } // Hold onto the user's public key. c.mu.Lock() c.pubKey = juc.Subject c.tags = juc.Tags c.nameTag = juc.Name c.mu.Unlock() // Check if we need to set an auth timer if the user jwt expires. c.setExpiration(juc.Claims(), validFor) acc.mu.RLock() c.Debugf("Authenticated JWT: %s %q (claim-name: %q, claim-tags: %q) "+ "signed with %q by Account %q (claim-name: %q, claim-tags: %q) signed with %q has mappings %t accused %p", c.kindString(), juc.Subject, juc.Name, juc.Tags, juc.Issuer, issuer, acc.nameTag, acc.tags, acc.Issuer, acc.hasMappings(), acc) acc.mu.RUnlock() return true } if nkey != nil { // If we did not match noAuthUser check signature which is required. if nkey.Nkey != noAuthUser { if c.opts.Sig == _EMPTY_ { c.Debugf("Signature missing") return false } sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) if err != nil { // Allow fallback to normal base64. sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) if err != nil { c.Debugf("Signature not valid base64") return false } } pub, err := nkeys.FromPublicKey(c.opts.Nkey) if err != nil { c.Debugf("User nkey not valid: %v", err) return false } if err := pub.Verify(c.nonce, sig); err != nil { c.Debugf("Signature not verified") return false } } if err := c.RegisterNkeyUser(nkey); err != nil { return false } return true } if user != nil { ok = comparePasswords(user.Password, c.opts.Password) // If we are authorized, register the user which will properly setup any permissions // for pub/sub authorizations. if ok { c.RegisterUser(user) } return ok } if c.kind == CLIENT { if token != _EMPTY_ { return comparePasswords(token, c.opts.Token) } else if username != _EMPTY_ { if username != c.opts.Username { return false } return comparePasswords(password, c.opts.Password) } } else if c.kind == LEAF { // There is no required username/password to connect and // there was no u/p in the CONNECT or none that matches the // know users. Register the leaf connection with global account // or the one specified in config (if provided). return s.registerLeafWithAccount(c, opts.LeafNode.Account) } return false } func getTLSAuthDCs(rdns *pkix.RDNSequence) string { dcOID := asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25} dcs := []string{} for _, rdn := range *rdns { if len(rdn) == 0 { continue } for _, atv := range rdn { value, ok := atv.Value.(string) if !ok { continue } if atv.Type.Equal(dcOID) { dcs = append(dcs, "DC="+value) } } } return strings.Join(dcs, ",") } type tlsMapAuthFn func(string, *ldap.DN, bool) (string, bool) func checkClientTLSCertSubject(c *client, fn tlsMapAuthFn) bool { tlsState := c.GetTLSConnectionState() if tlsState == nil { c.Debugf("User required in cert, no TLS connection state") return false } if len(tlsState.PeerCertificates) == 0 { c.Debugf("User required in cert, no peer certificates found") return false } cert := tlsState.PeerCertificates[0] if len(tlsState.PeerCertificates) > 1 { c.Debugf("Multiple peer certificates found, selecting first") } hasSANs := len(cert.DNSNames) > 0 hasEmailAddresses := len(cert.EmailAddresses) > 0 hasSubject := len(cert.Subject.String()) > 0 hasURIs := len(cert.URIs) > 0 if !hasEmailAddresses && !hasSubject && !hasURIs { c.Debugf("User required in cert, none found") return false } switch { case hasEmailAddresses: for _, u := range cert.EmailAddresses { if match, ok := fn(u, nil, false); ok { c.Debugf("Using email found in cert for auth [%q]", match) return true } } fallthrough case hasSANs: for _, u := range cert.DNSNames { if match, ok := fn(u, nil, true); ok { c.Debugf("Using SAN found in cert for auth [%q]", match) return true } } fallthrough case hasURIs: for _, u := range cert.URIs { if match, ok := fn(u.String(), nil, false); ok { c.Debugf("Using URI found in cert for auth [%q]", match) return true } } } // Use the string representation of the full RDN Sequence including // the domain components in case there are any. rdn := cert.Subject.ToRDNSequence().String() // Match using the raw subject to avoid ignoring attributes. // https://github.com/golang/go/issues/12342 dn, err := ldap.FromRawCertSubject(cert.RawSubject) if err == nil { if match, ok := fn(_EMPTY_, dn, false); ok { c.Debugf("Using DistinguishedNameMatch for auth [%q]", match) return true } c.Debugf("DistinguishedNameMatch could not be used for auth [%q]", rdn) } var rdns pkix.RDNSequence if _, err := asn1.Unmarshal(cert.RawSubject, &rdns); err == nil { // If found domain components then include roughly following // the order from https://tools.ietf.org/html/rfc2253 // // NOTE: The original sequence from string representation by ToRDNSequence does not follow // the correct ordering, so this addition ofdomainComponents would likely be deprecated in // another release in favor of using the correct ordered as parsed by the go-ldap library. // dcs := getTLSAuthDCs(&rdns) if len(dcs) > 0 { u := strings.Join([]string{rdn, dcs}, ",") if match, ok := fn(u, nil, false); ok { c.Debugf("Using RDNSequence for auth [%q]", match) return true } c.Debugf("RDNSequence could not be used for auth [%q]", u) } } // If no match, then use the string representation of the RDNSequence // from the subject without the domainComponents. if match, ok := fn(rdn, nil, false); ok { c.Debugf("Using certificate subject for auth [%q]", match) return true } c.Debugf("User in cert [%q], not found", rdn) return false } func dnsAltNameLabels(dnsAltName string) []string { return strings.Split(strings.ToLower(dnsAltName), ".") } // Check DNS name according to https://tools.ietf.org/html/rfc6125#section-6.4.1 func dnsAltNameMatches(dnsAltNameLabels []string, urls []*url.URL) bool { URLS: for _, url := range urls { if url == nil { continue URLS } hostLabels := strings.Split(strings.ToLower(url.Hostname()), ".") // Following https://tools.ietf.org/html/rfc6125#section-6.4.3, should not => will not, may => will not // The wildcard * never matches multiple label and only matches the left most label. if len(hostLabels) != len(dnsAltNameLabels) { continue URLS } i := 0 // only match wildcard on left most label if dnsAltNameLabels[0] == "*" { i++ } for ; i < len(dnsAltNameLabels); i++ { if dnsAltNameLabels[i] != hostLabels[i] { continue URLS } } return true } return false } // checkRouterAuth checks optional router authorization which can be nil or username/password. func (s *Server) isRouterAuthorized(c *client) bool { // Snapshot server options. opts := s.getOpts() // Check custom auth first, then TLS map if enabled // then single user/pass. if opts.CustomRouterAuthentication != nil { return opts.CustomRouterAuthentication.Check(c) } if opts.Cluster.TLSMap || opts.Cluster.TLSCheckKnownURLs { return checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) { if user == _EMPTY_ { return _EMPTY_, false } if opts.Cluster.TLSCheckKnownURLs && isDNSAltName { if dnsAltNameMatches(dnsAltNameLabels(user), opts.Routes) { return _EMPTY_, true } } if opts.Cluster.TLSMap && opts.Cluster.Username == user { return _EMPTY_, true } return _EMPTY_, false }) } if opts.Cluster.Username == _EMPTY_ { return true } if opts.Cluster.Username != c.opts.Username { return false } if !comparePasswords(opts.Cluster.Password, c.opts.Password) { return false } return true } // isGatewayAuthorized checks optional gateway authorization which can be nil or username/password. func (s *Server) isGatewayAuthorized(c *client) bool { // Snapshot server options. opts := s.getOpts() // Check whether TLS map is enabled, otherwise use single user/pass. if opts.Gateway.TLSMap || opts.Gateway.TLSCheckKnownURLs { return checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) { if user == _EMPTY_ { return _EMPTY_, false } if opts.Gateway.TLSCheckKnownURLs && isDNSAltName { labels := dnsAltNameLabels(user) for _, gw := range opts.Gateway.Gateways { if gw != nil && dnsAltNameMatches(labels, gw.URLs) { return _EMPTY_, true } } } if opts.Gateway.TLSMap && opts.Gateway.Username == user { return _EMPTY_, true } return _EMPTY_, false }) } if opts.Gateway.Username == _EMPTY_ { return true } if opts.Gateway.Username != c.opts.Username { return false } return comparePasswords(opts.Gateway.Password, c.opts.Password) } func (s *Server) registerLeafWithAccount(c *client, account string) bool { var err error acc := s.globalAccount() if account != _EMPTY_ { acc, err = s.lookupAccount(account) if err != nil { s.Errorf("authentication of user %q failed, unable to lookup account %q: %v", c.opts.Username, account, err) return false } } if err = c.registerWithAccount(acc); err != nil { return false } return true } // isLeafNodeAuthorized will check for auth for an inbound leaf node connection. func (s *Server) isLeafNodeAuthorized(c *client) bool { opts := s.getOpts() isAuthorized := func(username, password, account string) bool { if username != c.opts.Username { return false } if !comparePasswords(password, c.opts.Password) { return false } return s.registerLeafWithAccount(c, account) } // If leafnodes config has an authorization{} stanza, this takes precedence. // The user in CONNECT must match. We will bind to the account associated // with that user (from the leafnode's authorization{} config). if opts.LeafNode.Username != _EMPTY_ { return isAuthorized(opts.LeafNode.Username, opts.LeafNode.Password, opts.LeafNode.Account) } else if opts.LeafNode.Nkey != _EMPTY_ { if c.opts.Nkey != opts.LeafNode.Nkey { return false } if c.opts.Sig == _EMPTY_ { c.Debugf("Signature missing") return false } sig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig) if err != nil { // Allow fallback to normal base64. sig, err = base64.StdEncoding.DecodeString(c.opts.Sig) if err != nil { c.Debugf("Signature not valid base64") return false } } pub, err := nkeys.FromPublicKey(c.opts.Nkey) if err != nil { c.Debugf("User nkey not valid: %v", err) return false } if err := pub.Verify(c.nonce, sig); err != nil { c.Debugf("Signature not verified") return false } return s.registerLeafWithAccount(c, opts.LeafNode.Account) } else if len(opts.LeafNode.Users) > 0 { if opts.LeafNode.TLSMap { var user *User found := checkClientTLSCertSubject(c, func(u string, _ *ldap.DN, _ bool) (string, bool) { // This is expected to be a very small array. for _, usr := range opts.LeafNode.Users { if u == usr.Username { user = usr return u, true } } return _EMPTY_, false }) if !found { return false } if c.opts.Username != _EMPTY_ { s.Warnf("User %q found in connect proto, but user required from cert", c.opts.Username) } c.opts.Username = user.Username // EMPTY will result in $G accName := _EMPTY_ if user.Account != nil { accName = user.Account.GetName() } // This will authorize since are using an existing user, // but it will also register with proper account. return isAuthorized(user.Username, user.Password, accName) } // This is expected to be a very small array. for _, u := range opts.LeafNode.Users { if u.Username == c.opts.Username { var accName string if u.Account != nil { accName = u.Account.Name } return isAuthorized(u.Username, u.Password, accName) } } return false } // We are here if we accept leafnode connections without any credentials. // Still, if the CONNECT has some user info, we will bind to the // user's account or to the specified default account (if provided) // or to the global account. return s.isClientAuthorized(c) } // Support for bcrypt stored passwords and tokens. var validBcryptPrefix = regexp.MustCompile(`^\$2[abxy]\$\d{2}\$.*`) // isBcrypt checks whether the given password or token is bcrypted. func isBcrypt(password string) bool { if strings.HasPrefix(password, "$") { return validBcryptPrefix.MatchString(password) } return false } func comparePasswords(serverPassword, clientPassword string) bool { // Check to see if the server password is a bcrypt hash if isBcrypt(serverPassword) { if err := bcrypt.CompareHashAndPassword([]byte(serverPassword), []byte(clientPassword)); err != nil { return false } } else { // stringToBytes should be constant-time near enough compared to // turning a string into []byte normally. spass := stringToBytes(serverPassword) cpass := stringToBytes(clientPassword) if subtle.ConstantTimeCompare(spass, cpass) == 0 { return false } } return true } func validateAuth(o *Options) error { if err := validatePinnedCerts(o.TLSPinnedCerts); err != nil { return err } for _, u := range o.Users { if err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil { return err } } for _, u := range o.Nkeys { if err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil { return err } } return validateNoAuthUser(o, o.NoAuthUser) } func validateAllowedConnectionTypes(m map[string]struct{}) error { for ct := range m { ctuc := strings.ToUpper(ct) switch ctuc { case jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket, jwt.ConnectionTypeLeafnode, jwt.ConnectionTypeLeafnodeWS, jwt.ConnectionTypeMqtt, jwt.ConnectionTypeMqttWS: default: return fmt.Errorf("unknown connection type %q", ct) } if ctuc != ct { delete(m, ct) m[ctuc] = struct{}{} } } return nil } func validateNoAuthUser(o *Options, noAuthUser string) error { if noAuthUser == _EMPTY_ { return nil } if len(o.TrustedOperators) > 0 { return fmt.Errorf("no_auth_user not compatible with Trusted Operator") } if o.Nkeys == nil && o.Users == nil { return fmt.Errorf(`no_auth_user: "%s" present, but users/nkeys are not defined`, noAuthUser) } for _, u := range o.Users { if u.Username == noAuthUser { return nil } } for _, u := range o.Nkeys { if u.Nkey == noAuthUser { return nil } } return fmt.Errorf( `no_auth_user: "%s" not present as user or nkey in authorization block or account configuration`, noAuthUser) } nats-server-2.10.27/server/auth_callout.go000066400000000000000000000354761477524627100205130ustar00rootroot00000000000000// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "encoding/pem" "errors" "fmt" "time" "unicode" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" ) const ( AuthCalloutSubject = "$SYS.REQ.USER.AUTH" AuthRequestSubject = "nats-authorization-request" AuthRequestXKeyHeader = "Nats-Server-Xkey" ) // Process a callout on this client's behalf. func (s *Server) processClientOrLeafCallout(c *client, opts *Options) (authorized bool, errStr string) { isOperatorMode := len(opts.TrustedKeys) > 0 // this is the account the user connected in, or the one running the callout var acc *Account if !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.Account != _EMPTY_ { aname := opts.AuthCallout.Account var err error acc, err = s.LookupAccount(aname) if err != nil { errStr = fmt.Sprintf("No valid account %q for auth callout request: %v", aname, err) s.Warnf(errStr) return false, errStr } } else { acc = c.acc } // Check if we have been requested to encrypt. var xkp nkeys.KeyPair var xkey string var pubAccXKey string if !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.XKey != _EMPTY_ { pubAccXKey = opts.AuthCallout.XKey } else if isOperatorMode { pubAccXKey = acc.externalAuthXKey() } // If set grab server's xkey keypair and public key. if pubAccXKey != _EMPTY_ { // These are only set on creation, so lock not needed. xkp, xkey = s.xkp, s.info.XKey } // FIXME: so things like the server ID that get assigned, are used as a sort of nonce - but // reality is that the keypair here, is generated, so the response generated a JWT has to be // this user - no replay possible // Create a keypair for the user. We will expect this public user to be in the signed response. // This prevents replay attacks. ukp, _ := nkeys.CreateUser() pub, _ := ukp.PublicKey() reply := s.newRespInbox() respCh := make(chan string, 1) decodeResponse := func(rc *client, rmsg []byte, acc *Account) (*jwt.UserClaims, error) { account := acc.Name _, msg := rc.msgParts(rmsg) // This signals not authorized. // Since this is an account subscription will always have "\r\n". if len(msg) <= LEN_CR_LF { return nil, fmt.Errorf("auth callout violation: %q on account %q", "no reason supplied", account) } // Strip trailing CRLF. msg = msg[:len(msg)-LEN_CR_LF] encrypted := false // If we sent an encrypted request the response could be encrypted as well. // we are expecting the input to be `eyJ` if it is a JWT if xkp != nil && len(msg) > 0 && !bytes.HasPrefix(msg, []byte(jwtPrefix)) { var err error msg, err = xkp.Open(msg, pubAccXKey) if err != nil { return nil, fmt.Errorf("error decrypting auth callout response on account %q: %v", account, err) } encrypted = true } cr, err := jwt.DecodeAuthorizationResponseClaims(string(msg)) if err != nil { return nil, err } vr := jwt.CreateValidationResults() cr.Validate(vr) if len(vr.Issues) > 0 { return nil, fmt.Errorf("authorization response had validation errors: %v", vr.Issues[0]) } // the subject is the user id if cr.Subject != pub { return nil, errors.New("auth callout violation: auth callout response is not for expected user") } // check the audience to be the server ID if cr.Audience != s.info.ID { return nil, errors.New("auth callout violation: auth callout response is not for server") } // check if had an error message from the auth account if cr.Error != _EMPTY_ { return nil, fmt.Errorf("auth callout service returned an error: %v", cr.Error) } // if response is encrypted none of this is needed if isOperatorMode && !encrypted { pkStr := cr.Issuer if cr.IssuerAccount != _EMPTY_ { pkStr = cr.IssuerAccount } if pkStr != account { if _, ok := acc.signingKeys[pkStr]; !ok { return nil, errors.New("auth callout signing key is unknown") } } } return jwt.DecodeUserClaims(cr.Jwt) } // getIssuerAccount returns the issuer (as per JWT) - it also asserts that // only in operator mode we expect to receive `issuer_account`. getIssuerAccount := func(arc *jwt.UserClaims, account string) (string, error) { // Make sure correct issuer. var issuer string if opts.AuthCallout != nil { issuer = opts.AuthCallout.Issuer } else { // Operator mode is who we send the request on unless switching accounts. issuer = acc.Name } // the jwt issuer can be a signing key jwtIssuer := arc.Issuer if arc.IssuerAccount != _EMPTY_ { if !isOperatorMode { // this should be invalid - effectively it would allow the auth callout // to issue on another account which may be allowed given the configuration // where the auth callout account can handle multiple different ones.. return _EMPTY_, fmt.Errorf("error non operator mode account %q: attempted to use issuer_account", account) } jwtIssuer = arc.IssuerAccount } if jwtIssuer != issuer { if !isOperatorMode { return _EMPTY_, fmt.Errorf("wrong issuer for auth callout response on account %q, expected %q got %q", account, issuer, jwtIssuer) } else if !acc.isAllowedAcount(jwtIssuer) { return _EMPTY_, fmt.Errorf("account %q not permitted as valid account option for auth callout for account %q", arc.Issuer, account) } } return jwtIssuer, nil } getExpirationAndAllowedConnections := func(arc *jwt.UserClaims, account string) (time.Duration, map[string]struct{}, error) { allowNow, expiration := validateTimes(arc) if !allowNow { c.Errorf("Outside connect times") return 0, nil, fmt.Errorf("authorized user on account %q outside of valid connect times", account) } allowedConnTypes, err := convertAllowedConnectionTypes(arc.User.AllowedConnectionTypes) if err != nil { c.Debugf("%v", err) if len(allowedConnTypes) == 0 { return 0, nil, fmt.Errorf("authorized user on account %q using invalid connection type", account) } } return expiration, allowedConnTypes, nil } assignAccountAndPermissions := func(arc *jwt.UserClaims, account string) (*Account, error) { // Apply to this client. var err error issuerAccount, err := getIssuerAccount(arc, account) if err != nil { return nil, err } // if we are not in operator mode, they can specify placement as a tag var placement string if !isOperatorMode { // only allow placement if we are not in operator mode placement = arc.Audience } else { placement = issuerAccount } targetAcc, err := s.LookupAccount(placement) if err != nil { return nil, fmt.Errorf("no valid account %q for auth callout response on account %q: %v", placement, account, err) } if isOperatorMode { // this will validate the signing key that emitted the user, and if it is a signing // key it assigns the permissions from the target account if scope, ok := targetAcc.hasIssuer(arc.Issuer); !ok { return nil, fmt.Errorf("user JWT issuer %q is not known", arc.Issuer) } else if scope != nil { // this possibly has to be different because it could just be a plain issued by a non-scoped signing key if err := scope.ValidateScopedSigner(arc); err != nil { return nil, fmt.Errorf("user JWT is not valid: %v", err) } else if uSc, ok := scope.(*jwt.UserScope); !ok { return nil, fmt.Errorf("user JWT is not a valid scoped user") } else if arc.User.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, arc, targetAcc); err != nil { return nil, fmt.Errorf("user JWT generated invalid permissions: %v", err) } } } return targetAcc, nil } processReply := func(_ *subscription, rc *client, racc *Account, subject, reply string, rmsg []byte) { titleCase := func(m string) string { r := []rune(m) return string(append([]rune{unicode.ToUpper(r[0])}, r[1:]...)) } arc, err := decodeResponse(rc, rmsg, racc) if err != nil { c.authViolation() respCh <- titleCase(err.Error()) return } vr := jwt.CreateValidationResults() arc.Validate(vr) if len(vr.Issues) > 0 { c.authViolation() respCh <- fmt.Sprintf("Error validating user JWT: %v", vr.Issues[0]) return } // Make sure that the user is what we requested. if arc.Subject != pub { c.authViolation() respCh <- fmt.Sprintf("Expected authorized user of %q but got %q on account %q", pub, arc.Subject, racc.Name) return } expiration, allowedConnTypes, err := getExpirationAndAllowedConnections(arc, racc.Name) if err != nil { c.authViolation() respCh <- titleCase(err.Error()) return } targetAcc, err := assignAccountAndPermissions(arc, racc.Name) if err != nil { c.authViolation() respCh <- titleCase(err.Error()) return } // the JWT is cleared, because if in operator mode it may hold the JWT // for the bearer token that connected to the callout if in operator mode // the permissions are already set on the client, this prevents a decode // on c.RegisterNKeyUser which would have wrong values c.mu.Lock() c.opts.JWT = _EMPTY_ c.mu.Unlock() // Build internal user and bind to the targeted account. nkuser := buildInternalNkeyUser(arc, allowedConnTypes, targetAcc) if err := c.RegisterNkeyUser(nkuser); err != nil { c.authViolation() respCh <- fmt.Sprintf("Could not register auth callout user: %v", err) return } // See if the response wants to override the username. if arc.Name != _EMPTY_ { c.mu.Lock() c.opts.Username = arc.Name // Clear any others. c.opts.Nkey = _EMPTY_ c.pubKey = _EMPTY_ c.opts.Token = _EMPTY_ c.mu.Unlock() } // Check if we need to set an auth timer if the user jwt expires. c.setExpiration(arc.Claims(), expiration) respCh <- _EMPTY_ } // create a subscription to receive a response from the authcallout sub, err := acc.subscribeInternal(reply, processReply) if err != nil { errStr = fmt.Sprintf("Error setting up reply subscription for auth request: %v", err) s.Warnf(errStr) return false, errStr } defer acc.unsubscribeInternal(sub) // Build our request claims - jwt subject should be nkey jwtSub := acc.Name if opts.AuthCallout != nil { jwtSub = opts.AuthCallout.Issuer } // The public key of the server, if set is available on Varz.Key // This means that when a service connects, it can now peer // authenticate if it wants to - but that also means that it needs to be // listening to cluster changes claim := jwt.NewAuthorizationRequestClaims(jwtSub) claim.Audience = AuthRequestSubject // Set expected public user nkey. claim.UserNkey = pub s.mu.RLock() claim.Server = jwt.ServerID{ Name: s.info.Name, Host: s.info.Host, ID: s.info.ID, Version: s.info.Version, Cluster: s.info.Cluster, } s.mu.RUnlock() // Tags claim.Server.Tags = s.getOpts().Tags // Check if we have been requested to encrypt. // FIXME: possibly this public key also needs to be on the // Varz, because then it can be peer verified? if xkp != nil { claim.Server.XKey = xkey } authTimeout := secondsToDuration(s.getOpts().AuthTimeout) claim.Expires = time.Now().Add(time.Duration(authTimeout)).UTC().Unix() // Grab client info for the request. c.mu.Lock() c.fillClientInfo(&claim.ClientInformation) c.fillConnectOpts(&claim.ConnectOptions) // If we have a sig in the client opts, fill in nonce. if claim.ConnectOptions.SignedNonce != _EMPTY_ { claim.ClientInformation.Nonce = string(c.nonce) } // TLS if c.flags.isSet(handshakeComplete) && c.nc != nil { var ct jwt.ClientTLS conn := c.nc.(*tls.Conn) cs := conn.ConnectionState() ct.Version = tlsVersion(cs.Version) ct.Cipher = tlsCipher(cs.CipherSuite) // Check verified chains. for _, vs := range cs.VerifiedChains { var certs []string for _, c := range vs { blk := &pem.Block{ Type: "CERTIFICATE", Bytes: c.Raw, } certs = append(certs, string(pem.EncodeToMemory(blk))) } ct.VerifiedChains = append(ct.VerifiedChains, certs) } // If we do not have verified chains put in peer certs. if len(ct.VerifiedChains) == 0 { for _, c := range cs.PeerCertificates { blk := &pem.Block{ Type: "CERTIFICATE", Bytes: c.Raw, } ct.Certs = append(ct.Certs, string(pem.EncodeToMemory(blk))) } } claim.TLS = &ct } c.mu.Unlock() b, err := claim.Encode(s.kp) if err != nil { errStr = fmt.Sprintf("Error encoding auth request claim on account %q: %v", acc.Name, err) s.Warnf(errStr) return false, errStr } req := []byte(b) var hdr map[string]string // Check if we have been asked to encrypt. if xkp != nil { req, err = xkp.Seal([]byte(req), pubAccXKey) if err != nil { errStr = fmt.Sprintf("Error encrypting auth request claim on account %q: %v", acc.Name, err) s.Warnf(errStr) return false, errStr } hdr = map[string]string{AuthRequestXKeyHeader: xkey} } // Send out our request. if err := s.sendInternalAccountMsgWithReply(acc, AuthCalloutSubject, reply, hdr, req, false); err != nil { errStr = fmt.Sprintf("Error sending authorization request: %v", err) s.Debugf(errStr) return false, errStr } select { case errStr = <-respCh: if authorized = errStr == _EMPTY_; !authorized { s.Warnf(errStr) } case <-time.After(authTimeout): s.Debugf(fmt.Sprintf("Authorization callout response not received in time on account %q", acc.Name)) } return authorized, errStr } // Fill in client information for the request. // Lock should be held. func (c *client) fillClientInfo(ci *jwt.ClientInformation) { if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) { return } // Do it this way to fail to compile if fields are added to jwt.ClientInformation. *ci = jwt.ClientInformation{ Host: c.host, ID: c.cid, User: c.getRawAuthUser(), Name: c.opts.Name, Tags: c.tags, NameTag: c.nameTag, Kind: c.kindString(), Type: c.clientTypeString(), MQTT: c.getMQTTClientID(), } } // Fill in client options. // Lock should be held. func (c *client) fillConnectOpts(opts *jwt.ConnectOptions) { if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) { return } o := c.opts // Do it this way to fail to compile if fields are added to jwt.ClientInformation. *opts = jwt.ConnectOptions{ JWT: o.JWT, Nkey: o.Nkey, SignedNonce: o.Sig, Token: o.Token, Username: o.Username, Password: o.Password, Name: o.Name, Lang: o.Lang, Version: o.Version, Protocol: o.Protocol, } } nats-server-2.10.27/server/auth_callout_test.go000066400000000000000000002015571477524627100215450ustar00rootroot00000000000000// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "os" "reflect" "slices" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) // Helper function to decode an auth request. func decodeAuthRequest(t *testing.T, ejwt []byte) (string, *jwt.ServerID, *jwt.ClientInformation, *jwt.ConnectOptions, *jwt.ClientTLS) { t.Helper() ac, err := jwt.DecodeAuthorizationRequestClaims(string(ejwt)) require_NoError(t, err) return ac.UserNkey, &ac.Server, &ac.ClientInformation, &ac.ConnectOptions, ac.TLS } const ( authCalloutPub = "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" authCalloutSeed = "SUAP277QP7U4JMFFPVZHLJYEQJ2UHOTYVEIZJYAWRJXQLP4FRSEHYZJJOU" authCalloutIssuer = "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" authCalloutIssuerSeed = "SAANDLKMXL6CUS3CP52WIXBEDN6YJ545GDKC65U5JZPPV6WH6ESWUA6YAI" authCalloutIssuerSK = "SAAE46BB675HKZKSVJEUZAKKWIV6BJJO6XYE46Z3ZHO7TCI647M3V42IJE" ) func serviceResponse(t *testing.T, userID string, serverID string, uJwt string, errMsg string, expires time.Duration) []byte { cr := jwt.NewAuthorizationResponseClaims(userID) cr.Audience = serverID cr.Error = errMsg cr.Jwt = uJwt if expires != 0 { cr.Expires = time.Now().Add(expires).Unix() } aa, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) token, err := cr.Encode(aa) require_NoError(t, err) return []byte(token) } func newScopedRole(t *testing.T, role string, pub []string, sub []string, allowResponses bool) (*jwt.UserScope, nkeys.KeyPair) { akp, pk := createKey(t) r := jwt.NewUserScope() r.Key = pk r.Template.Sub.Allow.Add(sub...) r.Template.Pub.Allow.Add(pub...) if allowResponses { r.Template.Resp = &jwt.ResponsePermission{ MaxMsgs: 1, Expires: time.Second * 3, } } r.Role = role return r, akp } // Will create a signed user jwt as an authorized user. func createAuthUser(t *testing.T, user, name, account, issuerAccount string, akp nkeys.KeyPair, expires time.Duration, limits *jwt.UserPermissionLimits) string { t.Helper() if akp == nil { var err error akp, err = nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) } uc := jwt.NewUserClaims(user) if issuerAccount != "" { if _, err := nkeys.FromPublicKey(issuerAccount); err != nil { t.Fatalf("issuer account is not a public key: %v", err) } uc.IssuerAccount = issuerAccount } // The callout uses the audience as the target account // only if in non-operator mode, otherwise the user JWT has // correct attribution - issuer or issuer_account if _, err := nkeys.FromPublicKey(account); err != nil { // if it is not a public key, set the audience uc.Audience = account } if name != _EMPTY_ { uc.Name = name } if expires != 0 { uc.Expires = time.Now().Add(expires).Unix() } if limits != nil { uc.UserPermissionLimits = *limits } vr := jwt.CreateValidationResults() uc.Validate(vr) require_Len(t, len(vr.Errors()), 0) tok, err := uc.Encode(akp) require_NoError(t, err) return tok } type authTest struct { t *testing.T srv *Server conf string authClient *nats.Conn clients []*nats.Conn } func NewAuthTest(t *testing.T, config string, authHandler nats.MsgHandler, clientOptions ...nats.Option) *authTest { a := &authTest{t: t} a.conf = createConfFile(t, []byte(config)) a.srv, _ = RunServerWithConfig(a.conf) var err error a.authClient = a.ConnectCallout(clientOptions...) _, err = a.authClient.Subscribe(AuthCalloutSubject, authHandler) require_NoError(t, err) return a } func (at *authTest) NewClient(clientOptions ...nats.Option) (*nats.Conn, error) { conn, err := nats.Connect(at.srv.ClientURL(), clientOptions...) if err != nil { return nil, err } at.clients = append(at.clients, conn) return conn, nil } func (at *authTest) ConnectCallout(clientOptions ...nats.Option) *nats.Conn { conn, err := at.NewClient(clientOptions...) if err != nil { err = fmt.Errorf("callout client failed: %w", err) } require_NoError(at.t, err) return conn } func (at *authTest) Connect(clientOptions ...nats.Option) *nats.Conn { conn, err := at.NewClient(clientOptions...) require_NoError(at.t, err) return conn } func (at *authTest) WSNewClient(clientOptions ...nats.Option) (*nats.Conn, error) { pi := at.srv.PortsInfo(10 * time.Millisecond) require_False(at.t, pi == nil) // test cert is SAN to DNS localhost, not local IPs returned by server in test environments wssUrl := strings.Replace(pi.WebSocket[0], "127.0.0.1", "localhost", 1) // Seeing 127.0.1.1 in some test environments... wssUrl = strings.Replace(wssUrl, "127.0.1.1", "localhost", 1) conn, err := nats.Connect(wssUrl, clientOptions...) if err != nil { return nil, err } at.clients = append(at.clients, conn) return conn, nil } func (at *authTest) WSConnect(clientOptions ...nats.Option) *nats.Conn { conn, err := at.WSNewClient(clientOptions...) require_NoError(at.t, err) return conn } func (at *authTest) RequireConnectError(clientOptions ...nats.Option) { _, err := at.NewClient(clientOptions...) require_Error(at.t, err) } func (at *authTest) Cleanup() { if at.authClient != nil { at.authClient.Close() } if at.srv != nil { at.srv.Shutdown() removeFile(at.t, at.conf) } } func TestAuthCalloutBasics(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A authorization { timeout: 1s users: [ { user: "auth", password: "pwd" } ] auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" # users that will power the auth callout service. auth_users: [ auth ] } } ` callouts := uint32(0) handler := func(m *nats.Msg) { atomic.AddUint32(&callouts, 1) user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) require_True(t, si.Name == "A") require_True(t, ci.Host == "127.0.0.1") // Allow dlc user. if opts.Username == "dlc" && opts.Password == "zzz" { var j jwt.UserPermissionLimits j.Pub.Allow.Add("$SYS.>") j.Payload = 1024 ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, &j) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { // Nil response signals no authentication. m.Respond(nil) } } at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer at.Cleanup() // This one should fail since bad password. at.RequireConnectError(nats.UserInfo("dlc", "xxx")) // This one will use callout since not defined in server config. nc := at.Connect(nats.UserInfo("dlc", "zzz")) defer nc.Close() resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) dlc := &UserInfo{ UserID: "dlc", Account: globalAccountName, Permissions: &Permissions{ Publish: &SubjectPermission{ Allow: []string{"$SYS.>"}, Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. }, Subscribe: &SubjectPermission{}, }, } expires := userInfo.Expires userInfo.Expires = 0 if !reflect.DeepEqual(dlc, userInfo) { t.Fatalf("User info for %q did not match", "dlc") } if expires > 10*time.Minute || expires < (10*time.Minute-5*time.Second) { t.Fatalf("Expected expires of ~%v, got %v", 10*time.Minute, expires) } } func TestAuthCalloutMultiAccounts(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: ZZ accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} BAR {} BAZ {} } authorization { timeout: 1s auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) require_True(t, si.Name == "ZZ") require_True(t, ci.Host == "127.0.0.1") // Allow dlc user and map to the BAZ account. if opts.Username == "dlc" && opts.Password == "zzz" { ujwt := createAuthUser(t, user, _EMPTY_, "BAZ", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { // Nil response signals no authentication. m.Respond(nil) } } at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer at.Cleanup() // This one will use callout since not defined in server config. nc := at.Connect(nats.UserInfo("dlc", "zzz")) defer nc.Close() resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_True(t, userInfo.UserID == "dlc") require_True(t, userInfo.Account == "BAZ") } func TestAuthCalloutClientTLSCerts(t *testing.T) { conf := ` listen: "localhost:-1" server_name: T tls { cert_file = "../test/configs/certs/tlsauth/server.pem" key_file = "../test/configs/certs/tlsauth/server-key.pem" ca_file = "../test/configs/certs/tlsauth/ca.pem" verify = true } accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { timeout: 1s auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, ci, _, ctls := decodeAuthRequest(t, m.Data) require_True(t, si.Name == "T") require_True(t, ci.Host == "127.0.0.1") require_True(t, ctls != nil) // Zero since we are verified and will be under verified chains. require_True(t, len(ctls.Certs) == 0) require_True(t, len(ctls.VerifiedChains) == 1) // Since we have a CA. require_True(t, len(ctls.VerifiedChains[0]) == 2) blk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0])) cert, err := x509.ParseCertificate(blk.Bytes) require_NoError(t, err) if strings.HasPrefix(cert.Subject.String(), "CN=example.com") { // Override blank name here, server will substitute. ujwt := createAuthUser(t, user, "dlc", "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd"), nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), nats.RootCAs("../test/configs/certs/tlsauth/ca.pem")) defer ac.Cleanup() // Will use client cert to determine user. nc := ac.Connect( nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), nats.RootCAs("../test/configs/certs/tlsauth/ca.pem"), ) defer nc.Close() resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_True(t, userInfo.UserID == "dlc") require_True(t, userInfo.Account == "FOO") } func TestAuthCalloutVerifiedUserCalloutsWithSig(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A authorization { timeout: 1s users: [ { user: "auth", password: "pwd" } { nkey: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" } ] auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" # users that will power the auth callout service. auth_users: [ auth ] } } ` callouts := uint32(0) handler := func(m *nats.Msg) { atomic.AddUint32(&callouts, 1) user, si, ci, opts, _ := decodeAuthRequest(t, m.Data) require_True(t, si.Name == "A") require_True(t, ci.Host == "127.0.0.1") require_True(t, opts.SignedNonce != _EMPTY_) require_True(t, ci.Nonce != _EMPTY_) ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() seedFile := createTempFile(t, _EMPTY_) defer removeFile(t, seedFile.Name()) seedFile.WriteString(authCalloutSeed) nkeyOpt, err := nats.NkeyOptionFromSeed(seedFile.Name()) require_NoError(t, err) nc := ac.Connect(nkeyOpt) defer nc.Close() // Make sure that the callout was called. if atomic.LoadUint32(&callouts) != 1 { t.Fatalf("Expected callout to be called") } resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) dlc := &UserInfo{ UserID: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON", Account: globalAccountName, Permissions: &Permissions{ Publish: &SubjectPermission{ Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. }, Subscribe: &SubjectPermission{}, }, } if !reflect.DeepEqual(dlc, userInfo) { t.Fatalf("User info for %q did not match", "dlc") } } // For creating the authorized users in operator mode. func createAuthServiceUser(t *testing.T, accKp nkeys.KeyPair) (pub, creds string) { t.Helper() ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Name = "auth-service" uclaim.Subject = upub vr := jwt.ValidationResults{} uclaim.Validate(&vr) require_Len(t, len(vr.Errors()), 0) ujwt, err := uclaim.Encode(accKp) require_NoError(t, err) return upub, genCredsFile(t, ujwt, seed) } func createBasicAccountUser(t *testing.T, accKp nkeys.KeyPair) (creds string) { return createBasicAccount(t, "auth-client", accKp, true) } func createBasicAccountLeaf(t *testing.T, accKp nkeys.KeyPair) (creds string) { return createBasicAccount(t, "auth-leaf", accKp, false) } func createBasicAccount(t *testing.T, name string, accKp nkeys.KeyPair, addDeny bool) (creds string) { t.Helper() ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub uclaim.Name = name if addDeny { // For these deny all permission uclaim.Permissions.Pub.Deny.Add(">") uclaim.Permissions.Sub.Deny.Add(">") } vr := jwt.ValidationResults{} uclaim.Validate(&vr) require_Len(t, len(vr.Errors()), 0) ujwt, err := uclaim.Encode(accKp) require_NoError(t, err) return genCredsFile(t, ujwt, seed) } func createScopedUser(t *testing.T, accKp nkeys.KeyPair, sk nkeys.KeyPair) (creds string) { t.Helper() ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() apk, _ := accKp.PublicKey() uclaim.IssuerAccount = apk uclaim.Subject = upub uclaim.Name = "scoped-user" uclaim.SetScoped(true) // Uncomment this to set the sub limits // uclaim.Limits.Subs = 0 vr := jwt.ValidationResults{} uclaim.Validate(&vr) require_Len(t, len(vr.Errors()), 0) ujwt, err := uclaim.Encode(sk) require_NoError(t, err) return genCredsFile(t, ujwt, seed) } func TestAuthCalloutOperatorNoServerConfigCalloutAllowed(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: MEM authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" auth_users: [ auth ] } } `, ojwt))) defer removeFile(t, conf) opts := LoadConfig(conf) _, err := NewServer(opts) require_Error(t, err, errors.New("operators do not allow authorization callouts to be configured directly")) } func TestAuthCalloutOperatorModeBasics(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // TEST account. tkp, tpub := createKey(t) tSigningKp, tSigningPub := createKey(t) accClaim := jwt.NewAccountClaims(tpub) accClaim.Name = "TEST" accClaim.SigningKeys.Add(tSigningPub) scope, scopedKp := newScopedRole(t, "foo", []string{"foo.>", "$SYS.REQ.USER.INFO"}, []string{"foo.>", "_INBOX.>"}, false) accClaim.SigningKeys.AddScopedSigner(scope) accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) // AUTH service account. akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) apub, err := akp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, akp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(apub) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add(tpub) authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s %s: %s } `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt) const secretToken = "--XX--" const dummyToken = "--ZZ--" const skKeyToken = "--SK--" const scopedToken = "--Scoped--" const badScopedToken = "--BADScoped--" dkp, notAllowAccountPub := createKey(t) handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if opts.Token == secretToken { ujwt := createAuthUser(t, user, "dlc", tpub, "", tkp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == dummyToken { ujwt := createAuthUser(t, user, "dummy", notAllowAccountPub, "", dkp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == skKeyToken { ujwt := createAuthUser(t, user, "sk", tpub, tpub, tSigningKp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == scopedToken { // must have no limits set ujwt := createAuthUser(t, user, "scoped", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{}) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == badScopedToken { // limits are nil - here which result in a default user - this will fail scoped ujwt := createAuthUser(t, user, "bad-scoped", tpub, tpub, scopedKp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) expected := &UserInfo{ UserID: upub, Account: apub, Permissions: &Permissions{ Publish: &SubjectPermission{ Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. }, Subscribe: &SubjectPermission{}, }, } if !reflect.DeepEqual(expected, userInfo) { t.Fatalf("User info did not match expected, expected auto-deny permissions on callout subject") } // Bearer token etc.. // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createBasicAccountUser(t, akp) defer removeFile(t, creds) // We require a token. ac.RequireConnectError(nats.UserCredentials(creds)) // Send correct token. This should switch us to the test account. nc := ac.Connect(nats.UserCredentials(creds), nats.Token(secretToken)) require_NoError(t, err) defer nc.Close() resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) // Make sure we switch accounts. if userInfo.Account != tpub { t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) } // Now make sure that if the authorization service switches to an account that is not allowed, we reject. ac.RequireConnectError(nats.UserCredentials(creds), nats.Token(dummyToken)) // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key nc = ac.Connect(nats.UserCredentials(creds), nats.Token(skKeyToken)) defer nc.Close() resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) if userInfo.Account != tpub { t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) } // bad scoped user ac.RequireConnectError(nats.UserCredentials(creds), nats.Token(badScopedToken)) // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key nc = ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) require_NoError(t, err) defer nc.Close() resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) if userInfo.Account != tpub { t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) } require_True(t, len(userInfo.Permissions.Publish.Allow) == 2) slices.Sort(userInfo.Permissions.Publish.Allow) require_Equal(t, "foo.>", userInfo.Permissions.Publish.Allow[1]) slices.Sort(userInfo.Permissions.Subscribe.Allow) require_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2) require_Equal(t, "foo.>", userInfo.Permissions.Subscribe.Allow[1]) } func testAuthCalloutScopedUser(t *testing.T, allowAnyAccount bool) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // TEST account. _, tpub := createKey(t) _, tSigningPub := createKey(t) accClaim := jwt.NewAccountClaims(tpub) accClaim.Name = "TEST" accClaim.SigningKeys.Add(tSigningPub) scope, scopedKp := newScopedRole(t, "foo", []string{"foo.>", "$SYS.REQ.USER.INFO"}, []string{"foo.>", "_INBOX.>"}, true) scope.Template.Limits.Subs = 10 scope.Template.Limits.Payload = 512 accClaim.SigningKeys.AddScopedSigner(scope) accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) // AUTH service account. akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) apub, err := akp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, akp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(apub) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) if allowAnyAccount { authClaim.Authorization.AllowedAccounts.Add("*") } else { authClaim.Authorization.AllowedAccounts.Add(tpub) } // the scope for the bearer token which has no permissions sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) sentinelScope.Template.Sub.Deny.Add(">") sentinelScope.Template.Pub.Deny.Add(">") sentinelScope.Template.Limits.Subs = 0 sentinelScope.Template.Payload = 0 authClaim.SigningKeys.AddScopedSigner(sentinelScope) authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s %s: %s } `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt) const scopedToken = "--Scoped--" handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if opts.Token == scopedToken { // must have no limits set ujwt := createAuthUser(t, user, "scoped", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{}) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) expected := &UserInfo{ UserID: upub, Account: apub, Permissions: &Permissions{ Publish: &SubjectPermission{ Deny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account. }, Subscribe: &SubjectPermission{}, }, } if !reflect.DeepEqual(expected, userInfo) { t.Fatalf("User info did not match expected, expected auto-deny permissions on callout subject") } // Bearer token - this has no permissions see sentinelScope // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createScopedUser(t, akp, authKP) defer removeFile(t, creds) // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key nc := ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken)) resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) if userInfo.Account != tpub { t.Fatalf("Expected to be switched to %q, but got %q", tpub, userInfo.Account) } require_True(t, len(userInfo.Permissions.Publish.Allow) == 2) slices.Sort(userInfo.Permissions.Publish.Allow) require_Equal(t, "foo.>", userInfo.Permissions.Publish.Allow[1]) slices.Sort(userInfo.Permissions.Subscribe.Allow) require_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2) require_Equal(t, "foo.>", userInfo.Permissions.Subscribe.Allow[1]) _, err = nc.Subscribe("foo.>", func(msg *nats.Msg) { t.Log("got request on foo.>") require_NoError(t, msg.Respond(nil)) }) require_NoError(t, err) m, err := nc.Request("foo.bar", nil, time.Second) require_NoError(t, err) require_NotNil(t, m) t.Log("go response from foo.bar") nc.Close() } func TestAuthCalloutScopedUserAssignedAccount(t *testing.T) { testAuthCalloutScopedUser(t, false) } func TestAuthCalloutScopedUserAllAccount(t *testing.T) { testAuthCalloutScopedUser(t, true) } const ( curveSeed = "SXAAXMRAEP6JWWHNB6IKFL554IE6LZVT6EY5MBRICPILTLOPHAG73I3YX4" curvePublic = "XAB3NANV3M6N7AHSQP2U5FRWKKUT7EG2ZXXABV4XVXYQRJGM4S2CZGHT" ) func TestAuthCalloutServerConfigEncryption(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" server_name: A authorization { timeout: 1s users: [ { user: "auth", password: "pwd" } ] auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" # users that will power the auth callout service. auth_users: [ auth ] # This is a public xkey (x25519). The auth service has the private key. xkey: "%s" } } ` conf := fmt.Sprintf(tmpl, curvePublic) rkp, err := nkeys.FromCurveSeed([]byte(curveSeed)) require_NoError(t, err) handler := func(m *nats.Msg) { // This will be encrypted. _, err := jwt.DecodeAuthorizationRequestClaims(string(m.Data)) require_Error(t, err) xkey := m.Header.Get(AuthRequestXKeyHeader) require_True(t, xkey != _EMPTY_) decrypted, err := rkp.Open(m.Data, xkey) require_NoError(t, err) user, si, ci, opts, _ := decodeAuthRequest(t, decrypted) // The header xkey must match the signed xkey in server info. require_True(t, si.XKey == xkey) require_True(t, si.Name == "A") require_True(t, ci.Host == "127.0.0.1") // Allow dlc user. if opts.Username == "dlc" && opts.Password == "zzz" { ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Username == "dlc" && opts.Password == "xxx" { ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) // Encrypt this response. data, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, "", 0), si.XKey) // Server's public xkey. require_NoError(t, err) m.Respond(data) } else { // Nil response signals no authentication. m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() nc := ac.Connect(nats.UserInfo("dlc", "zzz")) defer nc.Close() // Authorization services can optionally encrypt the responses using the server's public xkey. nc = ac.Connect(nats.UserInfo("dlc", "xxx")) defer nc.Close() } func TestAuthCalloutOperatorModeEncryption(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // TEST account. tkp, tpub := createKey(t) accClaim := jwt.NewAccountClaims(tpub) accClaim.Name = "TEST" accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) // AUTH service account. akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) apub, err := akp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, akp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(apub) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add(tpub) authClaim.Authorization.XKey = curvePublic authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s %s: %s } `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt) rkp, err := nkeys.FromCurveSeed([]byte(curveSeed)) require_NoError(t, err) const tokenA = "--XX--" const tokenB = "--ZZ--" handler := func(m *nats.Msg) { // Make sure this is an encrypted request. if bytes.HasPrefix(m.Data, []byte(jwtPrefix)) { t.Fatalf("Request not encrypted") } xkey := m.Header.Get(AuthRequestXKeyHeader) require_True(t, xkey != _EMPTY_) decrypted, err := rkp.Open(m.Data, xkey) require_NoError(t, err) user, si, ci, opts, _ := decodeAuthRequest(t, decrypted) // The header xkey must match the signed xkey in server info. require_True(t, si.XKey == xkey) require_True(t, ci.Host == "127.0.0.1") if opts.Token == tokenA { ujwt := createAuthUser(t, user, "dlc", tpub, "", tkp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == tokenB { ujwt := createAuthUser(t, user, "rip", tpub, "", tkp, 0, nil) // Encrypt this response. data, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, "", 0), si.XKey) // Server's public xkey. require_NoError(t, err) m.Respond(data) } else { m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() // Bearer token etc.. // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createBasicAccountUser(t, akp) defer removeFile(t, creds) // This will receive an encrypted request to the auth service but send plaintext response. nc := ac.Connect(nats.UserCredentials(creds), nats.Token(tokenA)) defer nc.Close() // This will receive an encrypted request to the auth service and send an encrypted response. nc = ac.Connect(nats.UserCredentials(creds), nats.Token(tokenB)) defer nc.Close() } func TestAuthCalloutServerTags(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A server_tags: ["foo", "bar"] authorization { users: [ { user: "auth", password: "pwd" } ] auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" auth_users: [ auth ] } } ` tch := make(chan jwt.TagList, 1) handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) tch <- si.Tags ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() nc := ac.Connect() defer nc.Close() tags := <-tch require_True(t, len(tags) == 2) require_True(t, tags.Contains("foo")) require_True(t, tags.Contains("bar")) } func TestAuthCalloutServerClusterAndVersion(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A authorization { users: [ { user: "auth", password: "pwd" } ] auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" auth_users: [ auth ] } } cluster { name: HUB } ` ch := make(chan string, 2) handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) ch <- si.Cluster ch <- si.Version ujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, "", nil, 10*time.Minute, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() nc := ac.Connect() defer nc.Close() cluster := <-ch require_True(t, cluster == "HUB") version := <-ch require_True(t, len(version) > 0) ok, err := versionAtLeastCheckError(version, 2, 10, 0) require_NoError(t, err) require_True(t, ok) } func TestAuthCalloutErrorResponse(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A authorization { users: [ { user: "auth", password: "pwd" } ] auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) m.Respond(serviceResponse(t, user, si.ID, "", "BAD AUTH", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() ac.RequireConnectError(nats.UserInfo("dlc", "zzz")) } func TestAuthCalloutAuthUserFailDoesNotInvokeCallout(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A authorization { users: [ { user: "auth", password: "pwd" } ] auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" auth_users: [ auth ] } } ` callouts := uint32(0) handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) atomic.AddUint32(&callouts, 1) m.Respond(serviceResponse(t, user, si.ID, "", "WRONG PASSWORD", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() ac.RequireConnectError(nats.UserInfo("auth", "zzz")) if atomic.LoadUint32(&callouts) != 0 { t.Fatalf("Expected callout to not be called") } } func TestAuthCalloutAuthErrEvents(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} BAR {} } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) // Allow dlc user and map to the BAZ account. if opts.Username == "dlc" && opts.Password == "zzz" { ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Username == "dlc" { m.Respond(serviceResponse(t, user, si.ID, "", "WRONG PASSWORD", 0)) } else { m.Respond(serviceResponse(t, user, si.ID, "", "BAD CREDS", 0)) } } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // This is where the event fires, in this account. sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) require_NoError(t, err) // This one will use callout since not defined in server config. nc := ac.Connect(nats.UserInfo("dlc", "zzz")) defer nc.Close() checkSubsPending(t, sub, 0) checkAuthErrEvent := func(user, pass, reason string) { ac.RequireConnectError(nats.UserInfo(user, pass)) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) if !strings.Contains(dm.Reason, reason) { t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) } } checkAuthErrEvent("dlc", "xxx", "WRONG PASSWORD") checkAuthErrEvent("rip", "abc", "BAD CREDS") } func TestAuthCalloutConnectEvents(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} BAR {} $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth, admin ] } } ` handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) // Allow dlc user and map to the BAZ account. if opts.Username == "dlc" && opts.Password == "zzz" { ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Username == "rip" && opts.Password == "xxx" { ujwt := createAuthUser(t, user, _EMPTY_, "BAR", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(serviceResponse(t, user, si.ID, "", "BAD CREDS", 0)) } } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // Setup system user. snc := ac.Connect(nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() // Allow this connect event to pass us by.. time.Sleep(250 * time.Millisecond) // Watch for connect events. csub, err := snc.SubscribeSync(fmt.Sprintf(connectEventSubj, "*")) require_NoError(t, err) // Watch for disconnect events. dsub, err := snc.SubscribeSync(fmt.Sprintf(disconnectEventSubj, "*")) require_NoError(t, err) // Connections updates. Old acOldSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjOld, "*")) require_NoError(t, err) // Connections updates. New acNewSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjNew, "*")) require_NoError(t, err) snc.Flush() checkConnectEvents := func(user, pass, acc string) { nc := ac.Connect(nats.UserInfo(user, pass)) require_NoError(t, err) m, err := csub.NextMsg(time.Second) require_NoError(t, err) var cm ConnectEventMsg err = json.Unmarshal(m.Data, &cm) require_NoError(t, err) require_True(t, cm.Client.User == user) require_True(t, cm.Client.Account == acc) // Check that we have updates, 1 each, for the connections updates. m, err = acOldSub.NextMsg(time.Second) require_NoError(t, err) var anc AccountNumConns err = json.Unmarshal(m.Data, &anc) require_NoError(t, err) require_True(t, anc.AccountStat.Account == acc) require_True(t, anc.AccountStat.Conns == 1) m, err = acNewSub.NextMsg(time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &anc) require_NoError(t, err) require_True(t, anc.AccountStat.Account == acc) require_True(t, anc.AccountStat.Conns == 1) // Force the disconnect. nc.Close() m, err = dsub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) m, err = acOldSub.NextMsg(time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &anc) require_NoError(t, err) require_True(t, anc.AccountStat.Account == acc) require_True(t, anc.AccountStat.Conns == 0) m, err = acNewSub.NextMsg(time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &anc) require_NoError(t, err) require_True(t, anc.AccountStat.Account == acc) require_True(t, anc.AccountStat.Conns == 0) // Make sure no double events sent. time.Sleep(200 * time.Millisecond) checkSubsPending(t, csub, 0) checkSubsPending(t, dsub, 0) checkSubsPending(t, acOldSub, 0) checkSubsPending(t, acNewSub, 0) } checkConnectEvents("dlc", "zzz", "FOO") checkConnectEvents("rip", "xxx", "BAR") } func TestAuthCalloutBadServer(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, _, _, _, _ := decodeAuthRequest(t, m.Data) skp, err := nkeys.CreateServer() require_NoError(t, err) spk, err := skp.PublicKey() require_NoError(t, err) ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, spk, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // This is where the event fires, in this account. sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) require_NoError(t, err) checkAuthErrEvent := func(user, pass, reason string) { ac.RequireConnectError(nats.UserInfo(user, pass)) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) if !strings.Contains(dm.Reason, reason) { t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) } } checkAuthErrEvent("hello", "world", "response is not for server") } func TestAuthCalloutBadUser(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { _, si, _, _, _ := decodeAuthRequest(t, m.Data) kp, err := nkeys.CreateUser() require_NoError(t, err) upk, err := kp.PublicKey() require_NoError(t, err) ujwt := createAuthUser(t, upk, _EMPTY_, "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, upk, si.ID, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // This is where the event fires, in this account. sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) require_NoError(t, err) checkAuthErrEvent := func(user, pass, reason string) { ac.RequireConnectError(nats.UserInfo(user, pass)) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) if !strings.Contains(dm.Reason, reason) { t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) } } checkAuthErrEvent("hello", "world", "auth callout response is not for expected user") } func TestAuthCalloutExpiredUser(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, time.Second*-5, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // This is where the event fires, in this account. sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) require_NoError(t, err) checkAuthErrEvent := func(user, pass, reason string) { ac.RequireConnectError(nats.UserInfo(user, pass)) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) if !strings.Contains(dm.Reason, reason) { t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) } } checkAuthErrEvent("hello", "world", "claim is expired") } func TestAuthCalloutExpiredResponse(t *testing.T) { conf := ` listen: "127.0.0.1:-1" server_name: A accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { auth_callout { issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) ujwt := createAuthUser(t, user, _EMPTY_, "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", time.Second*-5)) } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer ac.Cleanup() // This is where the event fires, in this account. sub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj) require_NoError(t, err) checkAuthErrEvent := func(user, pass, reason string) { ac.RequireConnectError(nats.UserInfo(user, pass)) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var dm DisconnectEventMsg err = json.Unmarshal(m.Data, &dm) require_NoError(t, err) if !strings.Contains(dm.Reason, reason) { t.Fatalf("Expected %q reason, but got %q", reason, dm.Reason) } } checkAuthErrEvent("hello", "world", "claim is expired") } func TestAuthCalloutOperator_AnyAccount(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // A account. akp, apk := createKey(t) aClaim := jwt.NewAccountClaims(apk) aClaim.Name = "A" aJwt, err := aClaim.Encode(oKp) require_NoError(t, err) // B account. bkp, bpk := createKey(t) bClaim := jwt.NewAccountClaims(bpk) bClaim.Name = "B" bJwt, err := bClaim.Encode(oKp) require_NoError(t, err) // AUTH callout service account. ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) cpk, err := ckp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, ckp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(cpk) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add("*") authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s %s: %s %s: %s } `, ojwt, spub, cpk, authJwt, apk, aJwt, bpk, bJwt, spub, sysJwt) handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if opts.Token == "PutMeInA" { ujwt := createAuthUser(t, user, "user_a", apk, "", akp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else if opts.Token == "PutMeInB" { ujwt := createAuthUser(t, user, "user_b", bpk, "", bkp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() resp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) // Bearer token etc.. // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createBasicAccountUser(t, ckp) defer removeFile(t, creds) // We require a token. ac.RequireConnectError(nats.UserCredentials(creds)) // Send correct token. This should switch us to the A account. nc := ac.Connect(nats.UserCredentials(creds), nats.Token("PutMeInA")) require_NoError(t, err) defer nc.Close() resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_Equal(t, userInfo.Account, apk) nc = ac.Connect(nats.UserCredentials(creds), nats.Token("PutMeInB")) require_NoError(t, err) defer nc.Close() resp, err = nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) require_Equal(t, userInfo.Account, bpk) } func TestAuthCalloutWSClientTLSCerts(t *testing.T) { conf := ` server_name: T listen: "localhost:-1" tls { cert_file = "../test/configs/certs/tlsauth/server.pem" key_file = "../test/configs/certs/tlsauth/server-key.pem" ca_file = "../test/configs/certs/tlsauth/ca.pem" verify = true } websocket: { listen: "localhost:-1" tls { cert_file = "../test/configs/certs/tlsauth/server.pem" key_file = "../test/configs/certs/tlsauth/server-key.pem" ca_file = "../test/configs/certs/tlsauth/ca.pem" verify = true } } accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } FOO {} } authorization { timeout: 1s auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, ci, _, ctls := decodeAuthRequest(t, m.Data) require_Equal(t, si.Name, "T") require_Equal(t, ci.Host, "127.0.0.1") require_NotEqual(t, ctls, nil) // Zero since we are verified and will be under verified chains. require_Equal(t, len(ctls.Certs), 0) require_Equal(t, len(ctls.VerifiedChains), 1) // Since we have a CA. require_Equal(t, len(ctls.VerifiedChains[0]), 2) blk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0])) cert, err := x509.ParseCertificate(blk.Bytes) require_NoError(t, err) if strings.HasPrefix(cert.Subject.String(), "CN=example.com") { // Override blank name here, server will substitute. ujwt := createAuthUser(t, user, "dlc", "FOO", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } } ac := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd"), nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), nats.RootCAs("../test/configs/certs/tlsauth/ca.pem")) defer ac.Cleanup() // Will use client cert to determine user. nc := ac.WSConnect( nats.ClientCert("../test/configs/certs/tlsauth/client2.pem", "../test/configs/certs/tlsauth/client2-key.pem"), nats.RootCAs("../test/configs/certs/tlsauth/ca.pem"), ) defer nc.Close() resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_Equal(t, userInfo.UserID, "dlc") require_Equal(t, userInfo.Account, "FOO") } func testConfClientClose(t *testing.T, respondNil bool) { conf := ` listen: "127.0.0.1:-1" server_name: ZZ accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } } authorization { timeout: 1s auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } ` handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) if respondNil { m.Respond(nil) } else { m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) } } at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer at.Cleanup() // This one will use callout since not defined in server config. _, err := at.NewClient(nats.UserInfo("a", "x")) require_Error(t, err) require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) } func TestAuthCallout_ClientAuthErrorConf(t *testing.T) { testConfClientClose(t, true) testConfClientClose(t, false) } func testAuthCall_ClientAuthErrorOperatorMode(t *testing.T, respondNil bool) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // AUTH service account. akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) apub, err := akp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, akp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(apub) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add("*") // the scope for the bearer token which has no permissions sentinelScope, authKP := newScopedRole(t, "sentinel", nil, nil, false) sentinelScope.Template.Sub.Deny.Add(">") sentinelScope.Template.Pub.Deny.Add(">") sentinelScope.Template.Limits.Subs = 0 sentinelScope.Template.Payload = 0 authClaim.SigningKeys.AddScopedSigner(sentinelScope) authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, spub, apub, authJwt, spub, sysJwt) handler := func(m *nats.Msg) { user, si, _, _, _ := decodeAuthRequest(t, m.Data) if respondNil { m.Respond(nil) } else { m.Respond(serviceResponse(t, user, si.ID, "", "not today", 0)) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() // Bearer token - this has no permissions see sentinelScope // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createScopedUser(t, akp, authKP) defer removeFile(t, creds) // Send the signing key token. This should switch us to the test account, but the user // is signed with the account signing key _, err = ac.NewClient(nats.UserCredentials(creds)) require_Error(t, err) require_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR)) } func TestAuthCallout_ClientAuthErrorOperatorMode(t *testing.T) { testAuthCall_ClientAuthErrorOperatorMode(t, true) testAuthCall_ClientAuthErrorOperatorMode(t, false) } func TestOperatorModeUserRevocation(t *testing.T) { skp, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // TEST account. tkp, tpub := createKey(t) accClaim := jwt.NewAccountClaims(tpub) accClaim.Name = "TEST" accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) // AUTH service account. akp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) apub, err := akp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, akp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(apub) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add(tpub) authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) jwtDir, err := os.MkdirTemp("", "") require_NoError(t, err) defer func() { _ = os.RemoveAll(jwtDir) }() conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: "full" dir: %s allow_delete: false interval: "2m" timeout: "1.9s" } resolver_preload: { %s: %s %s: %s %s: %s } `, ojwt, spub, jwtDir, apub, authJwt, tpub, accJwt, spub, sysJwt) const token = "--secret--" users := make(map[string]string) handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if opts.Token == token { // must have no limits set ujwt := createAuthUser(t, user, "user", tpub, tpub, tkp, 0, &jwt.UserPermissionLimits{}) users[opts.Name] = ujwt m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } ac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer ac.Cleanup() // create a system user _, sysCreds := createAuthServiceUser(t, skp) defer removeFile(t, sysCreds) // connect the system user sysNC, err := ac.NewClient(nats.UserCredentials(sysCreds)) require_NoError(t, err) defer sysNC.Close() // Bearer token etc.. // This is used by all users, and the customization will be in other connect args. // This needs to also be bound to the authorization account. creds = createBasicAccountUser(t, akp) defer removeFile(t, creds) var fwg sync.WaitGroup // connect three clients nc := ac.Connect(nats.UserCredentials(creds), nats.Name("first"), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil && strings.Contains(err.Error(), "authentication revoked") { fwg.Done() } })) fwg.Add(1) var swg sync.WaitGroup // connect another user ncA := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.Name("second"), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil && strings.Contains(err.Error(), "authentication revoked") { swg.Done() } })) swg.Add(1) ncB := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil && strings.Contains(err.Error(), "authentication revoked") { swg.Done() } })) swg.Add(1) require_NoError(t, err) // revoke the user first - look at the JWT we issued uc, err := jwt.DecodeUserClaims(users["first"]) require_NoError(t, err) // revoke the user in account accClaim.Revocations = make(map[string]int64) accClaim.Revocations.Revoke(uc.Subject, time.Now().Add(time.Minute)) accJwt, err = accClaim.Encode(oKp) require_NoError(t, err) // send the update request updateAccount(t, sysNC, accJwt) // wait for the user to be disconnected with the error we expect fwg.Wait() require_Equal(t, nc.IsConnected(), false) // update the account to remove any revocations accClaim.Revocations = make(map[string]int64) accJwt, err = accClaim.Encode(oKp) require_NoError(t, err) updateAccount(t, sysNC, accJwt) // we should still be connected on the other 2 clients require_Equal(t, ncA.IsConnected(), true) require_Equal(t, ncB.IsConnected(), true) // update the jwt and revoke all users accClaim.Revocations.Revoke(jwt.All, time.Now().Add(time.Minute)) accJwt, err = accClaim.Encode(oKp) require_NoError(t, err) updateAccount(t, sysNC, accJwt) swg.Wait() require_Equal(t, ncA.IsConnected(), false) require_Equal(t, ncB.IsConnected(), false) } func updateAccount(t *testing.T, sys *nats.Conn, jwtToken string) { ac, err := jwt.DecodeAccountClaims(jwtToken) require_NoError(t, err) r, err := sys.Request(fmt.Sprintf(`$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE`, ac.Subject), []byte(jwtToken), time.Second*2) require_NoError(t, err) type data struct { Account string `json:"account"` Code int `json:"code"` } type serverResponse struct { Data data `json:"data"` } var response serverResponse err = json.Unmarshal(r.Data, &response) require_NoError(t, err) require_NotNil(t, response.Data) require_Equal(t, response.Data.Code, int(200)) } func TestAuthCalloutLeafNodeAndOperatorMode(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // A account. akp, apk := createKey(t) aClaim := jwt.NewAccountClaims(apk) aClaim.Name = "A" aJwt, err := aClaim.Encode(oKp) require_NoError(t, err) // AUTH callout service account. ckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed)) require_NoError(t, err) cpk, err := ckp.PublicKey() require_NoError(t, err) // The authorized user for the service. upub, creds := createAuthServiceUser(t, ckp) defer removeFile(t, creds) authClaim := jwt.NewAccountClaims(cpk) authClaim.Name = "AUTH" authClaim.EnableExternalAuthorization(upub) authClaim.Authorization.AllowedAccounts.Add("*") authJwt, err := authClaim.Encode(oKp) require_NoError(t, err) conf := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s %s: %s } leafnodes { listen: "127.0.0.1:-1" } `, ojwt, spub, cpk, authJwt, apk, aJwt, spub, sysJwt) handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if (opts.Username == "leaf" && opts.Password == "pwd") || (opts.Token == "token") { ujwt := createAuthUser(t, user, "user_a", apk, "", akp, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } at := NewAuthTest(t, conf, handler, nats.UserCredentials(creds)) defer at.Cleanup() ucreds := createBasicAccountUser(t, ckp) defer removeFile(t, ucreds) // This should switch us to the A account. nc := at.Connect(nats.UserCredentials(ucreds), nats.Token("token")) defer nc.Close() natsSub(t, nc, "foo", func(m *nats.Msg) { m.Respond([]byte("here")) }) natsFlush(t, nc) // Create creds for the leaf account. lcreds := createBasicAccountLeaf(t, ckp) defer removeFile(t, lcreds) hopts := at.srv.getOpts() for _, test := range []struct { name string up string ok bool }{ {"bad token", "tokenx", false}, {"bad username and password", "leaf:pwdx", false}, {"token", "token", true}, {"username and password", "leaf:pwd", true}, } { t.Run(test.name, func(t *testing.T) { lconf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "LEAF" leafnodes { remotes [ { url: "nats://%s@127.0.0.1:%d" credentials: "%s" } ] } `, test.up, hopts.LeafNode.Port, lcreds))) leaf, _ := RunServerWithConfig(lconf) defer leaf.Shutdown() if !test.ok { // Expect failure to connect. Wait a bit before checking. time.Sleep(50 * time.Millisecond) checkLeafNodeConnectedCount(t, leaf, 0) return } checkLeafNodeConnected(t, leaf) checkSubInterest(t, leaf, globalAccountName, "foo", time.Second) ncl := natsConnect(t, leaf.ClientURL()) defer ncl.Close() resp, err := ncl.Request("foo", []byte("hello"), time.Second) require_NoError(t, err) require_Equal(t, string(resp.Data), "here") }) } } func TestAuthCalloutLeafNodeAndConfigMode(t *testing.T) { conf := ` listen: "127.0.0.1:-1" accounts { AUTH { users [ {user: "auth", password: "pwd"} ] } A {} } authorization { timeout: 1s auth_callout { # Needs to be a public account nkey, will work for both server config and operator mode. issuer: "ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA" account: AUTH auth_users: [ auth ] } } leafnodes { listen: "127.0.0.1:-1" } ` handler := func(m *nats.Msg) { user, si, _, opts, _ := decodeAuthRequest(t, m.Data) if (opts.Username == "leaf" && opts.Password == "pwd") || (opts.Token == "token") { ujwt := createAuthUser(t, user, _EMPTY_, "A", "", nil, 0, nil) m.Respond(serviceResponse(t, user, si.ID, ujwt, "", 0)) } else { m.Respond(nil) } } at := NewAuthTest(t, conf, handler, nats.UserInfo("auth", "pwd")) defer at.Cleanup() // This should switch us to the A account. nc := at.Connect(nats.Token("token")) defer nc.Close() natsSub(t, nc, "foo", func(m *nats.Msg) { m.Respond([]byte("here")) }) natsFlush(t, nc) hopts := at.srv.getOpts() for _, test := range []struct { name string up string ok bool }{ {"bad token", "tokenx", false}, {"bad username and password", "leaf:pwdx", false}, {"token", "token", true}, {"username and password", "leaf:pwd", true}, } { t.Run(test.name, func(t *testing.T) { lconf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "LEAF" leafnodes { remotes [{url: "nats://%s@127.0.0.1:%d"}] } `, test.up, hopts.LeafNode.Port))) leaf, _ := RunServerWithConfig(lconf) defer leaf.Shutdown() if !test.ok { // Expect failure to connect. Wait a bit before checking. time.Sleep(50 * time.Millisecond) checkLeafNodeConnectedCount(t, leaf, 0) return } checkLeafNodeConnected(t, leaf) checkSubInterest(t, leaf, globalAccountName, "foo", time.Second) ncl := natsConnect(t, leaf.ClientURL()) defer ncl.Close() resp, err := ncl.Request("foo", []byte("hello"), time.Second) require_NoError(t, err) require_Equal(t, string(resp.Data), "here") }) } } nats-server-2.10.27/server/auth_test.go000066400000000000000000000251571477524627100200220ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "encoding/json" "fmt" "net" "net/url" "os" "reflect" "strings" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) func TestUserCloneNilPermissions(t *testing.T) { user := &User{ Username: "foo", Password: "bar", } clone := user.clone() if !reflect.DeepEqual(user, clone) { t.Fatalf("Cloned Users are incorrect.\nexpected: %+v\ngot: %+v", user, clone) } clone.Password = "baz" if reflect.DeepEqual(user, clone) { t.Fatal("Expected Users to be different") } } func TestUserClone(t *testing.T) { user := &User{ Username: "foo", Password: "bar", Permissions: &Permissions{ Publish: &SubjectPermission{ Allow: []string{"foo"}, }, Subscribe: &SubjectPermission{ Allow: []string{"bar"}, }, }, } clone := user.clone() if !reflect.DeepEqual(user, clone) { t.Fatalf("Cloned Users are incorrect.\nexpected: %+v\ngot: %+v", user, clone) } clone.Permissions.Subscribe.Allow = []string{"baz"} if reflect.DeepEqual(user, clone) { t.Fatal("Expected Users to be different") } } func TestUserClonePermissionsNoLists(t *testing.T) { user := &User{ Username: "foo", Password: "bar", Permissions: &Permissions{}, } clone := user.clone() if clone.Permissions.Publish != nil { t.Fatalf("Expected Publish to be nil, got: %v", clone.Permissions.Publish) } if clone.Permissions.Subscribe != nil { t.Fatalf("Expected Subscribe to be nil, got: %v", clone.Permissions.Subscribe) } } func TestUserCloneNoPermissions(t *testing.T) { user := &User{ Username: "foo", Password: "bar", } clone := user.clone() if clone.Permissions != nil { t.Fatalf("Expected Permissions to be nil, got: %v", clone.Permissions) } } func TestUserCloneNil(t *testing.T) { user := (*User)(nil) clone := user.clone() if clone != nil { t.Fatalf("Expected nil, got: %+v", clone) } } func TestUserUnknownAllowedConnectionType(t *testing.T) { o := DefaultOptions() o.Users = []*User{{ Username: "user", Password: "pwd", AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, "someNewType"}), }} _, err := NewServer(o) if err == nil || !strings.Contains(err.Error(), "connection type") { t.Fatalf("Expected error about unknown connection type, got %v", err) } o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{"websocket"}) s, err := NewServer(o) if err != nil { t.Fatalf("Unexpected error: %v", err) } s.mu.Lock() user := s.opts.Users[0] s.mu.Unlock() for act := range user.AllowedConnectionTypes { if act != jwt.ConnectionTypeWebsocket { t.Fatalf("Expected map to have been updated with proper case, got %v", act) } } // Same with NKey user now. o.Users = nil o.Nkeys = []*NkeyUser{{ Nkey: "somekey", AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, "someNewType"}), }} _, err = NewServer(o) if err == nil || !strings.Contains(err.Error(), "connection type") { t.Fatalf("Expected error about unknown connection type, got %v", err) } o.Nkeys[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{"websocket"}) s, err = NewServer(o) if err != nil { t.Fatalf("Unexpected error: %v", err) } s.mu.Lock() nkey := s.opts.Nkeys[0] s.mu.Unlock() for act := range nkey.AllowedConnectionTypes { if act != jwt.ConnectionTypeWebsocket { t.Fatalf("Expected map to have been updated with proper case, got %v", act) } } } func TestDNSAltNameMatching(t *testing.T) { for idx, test := range []struct { altName string urls []string match bool }{ {"foo", []string{"FOO"}, true}, {"foo", []string{".."}, false}, {"foo", []string{"."}, false}, {"Foo", []string{"foO"}, true}, {"FOO", []string{"foo"}, true}, {"foo1", []string{"bar"}, false}, {"multi", []string{"m", "mu", "mul", "multi"}, true}, {"multi", []string{"multi", "m", "mu", "mul"}, true}, {"foo.bar", []string{"foo", "foo.bar.bar", "foo.baz"}, false}, {"foo.Bar", []string{"foo", "bar.foo", "Foo.Bar"}, true}, {"foo.*", []string{"foo", "bar.foo", "Foo.Bar"}, false}, // only match left most {"f*.bar", []string{"foo", "bar.foo", "Foo.Bar"}, false}, {"*.bar", []string{"foo.bar"}, true}, {"*", []string{"baz.bar", "bar", "z.y"}, true}, {"*", []string{"bar"}, true}, {"*", []string{"."}, false}, {"*", []string{""}, true}, {"*", []string{"*"}, true}, {"bar.*", []string{"bar.*"}, true}, {"*.Y-X-red-mgmt.default.svc", []string{"A.Y-X-red-mgmt.default.svc"}, true}, {"*.Y-X-green-mgmt.default.svc", []string{"A.Y-X-green-mgmt.default.svc"}, true}, {"*.Y-X-blue-mgmt.default.svc", []string{"A.Y-X-blue-mgmt.default.svc"}, true}, {"Y-X-red-mgmt", []string{"Y-X-red-mgmt"}, true}, {"Y-X-red-mgmt", []string{"X-X-red-mgmt"}, false}, {"Y-X-red-mgmt", []string{"Y-X-green-mgmt"}, false}, {"Y-X-red-mgmt", []string{"Y"}, false}, {"Y-X-red-mgmt", []string{"Y-X"}, false}, {"Y-X-red-mgmt", []string{"Y-X-red"}, false}, {"Y-X-red-mgmt", []string{"X-red-mgmt"}, false}, {"Y-X-green-mgmt", []string{"Y-X-green-mgmt"}, true}, {"Y-X-blue-mgmt", []string{"Y-X-blue-mgmt"}, true}, {"connect.Y.local", []string{"connect.Y.local"}, true}, {"connect.Y.local", []string{".Y.local"}, false}, {"connect.Y.local", []string{"..local"}, false}, {"gcp.Y.local", []string{"gcp.Y.local"}, true}, {"uswest1.gcp.Y.local", []string{"uswest1.gcp.Y.local"}, true}, } { urlSet := make([]*url.URL, len(test.urls)) for i, u := range test.urls { var err error urlSet[i], err = url.Parse("nats://" + u) if err != nil { t.Fatal(err) } } if dnsAltNameMatches(dnsAltNameLabels(test.altName), urlSet) != test.match { t.Fatal("Test", idx, "Match miss match, expected:", test.match) } } } func TestNoAuthUser(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { FOO { users [{user: "foo", password: "pwd1"}] } BAR { users [{user: "bar", password: "pwd2"}] } } no_auth_user: "foo" `)) defer os.Remove(conf) s, o := RunServerWithConfig(conf) defer s.Shutdown() for _, test := range []struct { name string usrInfo string ok bool account string }{ {"valid user/pwd", "bar:pwd2@", true, "BAR"}, {"invalid pwd", "bar:wrong@", false, _EMPTY_}, {"some token", "sometoken@", false, _EMPTY_}, {"user used without pwd", "bar@", false, _EMPTY_}, // will be treated as a token {"user with empty password", "bar:@", false, _EMPTY_}, {"no user", _EMPTY_, true, "FOO"}, } { t.Run(test.name, func(t *testing.T) { url := fmt.Sprintf("nats://%s127.0.0.1:%d", test.usrInfo, o.Port) nc, err := nats.Connect(url) if err != nil { if test.ok { t.Fatalf("Unexpected error: %v", err) } return } else if !test.ok { nc.Close() t.Fatalf("Should have failed, did not") } var accName string s.mu.Lock() for _, c := range s.clients { c.mu.Lock() if c.acc != nil { accName = c.acc.Name } c.mu.Unlock() break } s.mu.Unlock() nc.Close() checkClientsCount(t, s, 0) if accName != test.account { t.Fatalf("The account should have been %q, got %q", test.account, accName) } }) } } func TestNoAuthUserNkey(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { FOO { users [{user: "foo", password: "pwd1"}] } BAR { users [{nkey: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON"}] } } no_auth_user: "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON" `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Make sure we connect ok and to the correct account. nc := natsConnect(t, s.ClientURL()) defer nc.Close() resp, err := nc.Request(userDirectInfoSubj, nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_Equal(t, userInfo.UserID, "UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON") require_Equal(t, userInfo.Account, "BAR") } func TestUserConnectionDeadline(t *testing.T) { clientAuth := &DummyAuth{ t: t, register: true, deadline: time.Now().Add(50 * time.Millisecond), } opts := DefaultOptions() opts.CustomClientAuthentication = clientAuth s := RunServer(opts) defer s.Shutdown() var dcerr error ctx, cancel := context.WithTimeout(context.Background(), time.Second) nc, err := nats.Connect( s.ClientURL(), nats.UserInfo("valid", _EMPTY_), nats.NoReconnect(), nats.ErrorHandler(func(nc *nats.Conn, _ *nats.Subscription, err error) { dcerr = err cancel() })) if err != nil { t.Fatalf("Expected client to connect, got: %s", err) } <-ctx.Done() checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if nc.IsConnected() { return fmt.Errorf("Expected to be disconnected") } return nil }) if dcerr == nil || dcerr.Error() != "nats: authentication expired" { t.Fatalf("Expected a auth expired error: got: %v", dcerr) } } func TestNoAuthUserNoConnectProto(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { A { users [{user: "foo", password: "pwd"}] } } authorization { timeout: 1 } no_auth_user: "foo" `)) defer os.Remove(conf) s, o := RunServerWithConfig(conf) defer s.Shutdown() checkClients := func(n int) { t.Helper() time.Sleep(100 * time.Millisecond) if nc := s.NumClients(); nc != n { t.Fatalf("Expected %d clients, got %d", n, nc) } } conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.Host, o.Port)) require_NoError(t, err) defer conn.Close() checkClientsCount(t, s, 1) // With no auth user we should not require a CONNECT. // Make sure we are good on not sending CONN first. _, err = conn.Write([]byte("PUB foo 2\r\nok\r\n")) require_NoError(t, err) checkClients(1) conn.Close() // Now make sure we still do get timed out though. conn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", o.Host, o.Port)) require_NoError(t, err) defer conn.Close() checkClientsCount(t, s, 1) time.Sleep(1200 * time.Millisecond) checkClientsCount(t, s, 0) } nats-server-2.10.27/server/avl/000077500000000000000000000000001477524627100162435ustar00rootroot00000000000000nats-server-2.10.27/server/avl/norace_test.go000066400000000000000000000126641477524627100211110ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !race && !skip_no_race_tests // +build !race,!skip_no_race_tests package avl import ( "flag" "fmt" "math" "math/rand" "runtime" "runtime/debug" "testing" "time" ) // Print Results: go test -v --args --results var printResults = flag.Bool("results", false, "Enable Results Logging") // SequenceSet memory tests vs dmaps. func TestNoRaceSeqSetSizeComparison(t *testing.T) { // Create 5M random entries (dupes possible but ok for this test) out of 8M range. num := 5_000_000 max := 7_000_000 seqs := make([]uint64, 0, num) for i := 0; i < num; i++ { n := uint64(rand.Int63n(int64(max + 1))) seqs = append(seqs, n) } runtime.GC() // Disable to get stable results. gcp := debug.SetGCPercent(-1) defer debug.SetGCPercent(gcp) mem := runtime.MemStats{} runtime.ReadMemStats(&mem) inUseBefore := mem.HeapInuse dmap := make(map[uint64]struct{}, num) for _, n := range seqs { dmap[n] = struct{}{} } runtime.ReadMemStats(&mem) dmapUse := mem.HeapInuse - inUseBefore inUseBefore = mem.HeapInuse // Now do SequenceSet on same dataset. var sset SequenceSet for _, n := range seqs { sset.Insert(n) } runtime.ReadMemStats(&mem) seqSetUse := mem.HeapInuse - inUseBefore if seqSetUse > 2*1024*1024 { t.Fatalf("Expected SequenceSet size to be < 2M, got %v", friendlyBytes(seqSetUse)) } if seqSetUse*50 > dmapUse { t.Fatalf("Expected SequenceSet to be at least 50x better then dmap approach: %v vs %v", friendlyBytes(seqSetUse), friendlyBytes(dmapUse), ) } } func TestNoRaceSeqSetEncodeLarge(t *testing.T) { num := 2_500_000 max := 5_000_000 dmap := make(map[uint64]struct{}, num) var ss SequenceSet for i := 0; i < num; i++ { n := uint64(rand.Int63n(int64(max + 1))) ss.Insert(n) dmap[n] = struct{}{} } // Disable to get stable results. gcp := debug.SetGCPercent(-1) defer debug.SetGCPercent(gcp) // In general should be about the same, but can see some variability. expected := time.Millisecond start := time.Now() b, err := ss.Encode(nil) require_NoError(t, err) if elapsed := time.Since(start); elapsed > expected { t.Fatalf("Expected encode of %d items with encoded size %v to take less than %v, got %v", num, friendlyBytes(len(b)), expected, elapsed) } else { logResults("Encode time for %d items was %v, encoded size is %v\n", num, elapsed, friendlyBytes(len(b))) } start = time.Now() ss2, _, err := Decode(b) require_NoError(t, err) if elapsed := time.Since(start); elapsed > expected { t.Fatalf("Expected decode to take less than %v, got %v", expected, elapsed) } else { logResults("Decode time is %v\n", elapsed) } require_True(t, ss.Nodes() == ss2.Nodes()) require_True(t, ss.Size() == ss2.Size()) } func TestNoRaceSeqSetRelativeSpeed(t *testing.T) { // Create 1M random entries (dupes possible but ok for this test) out of 3M range. num := 1_000_000 max := 3_000_000 seqs := make([]uint64, 0, num) for i := 0; i < num; i++ { n := uint64(rand.Int63n(int64(max + 1))) seqs = append(seqs, n) } start := time.Now() // Now do SequenceSet on same dataset. var sset SequenceSet for _, n := range seqs { sset.Insert(n) } ssInsertElapsed := time.Since(start) logResults("Inserts SequenceSet: %v for %d items\n", ssInsertElapsed, num) start = time.Now() for _, n := range seqs { if ok := sset.Exists(n); !ok { t.Fatalf("Should exist") } } ssLookupElapsed := time.Since(start) logResults("Lookups: %v\n", ssLookupElapsed) // Now do a map. dmap := make(map[uint64]struct{}) start = time.Now() for _, n := range seqs { dmap[n] = struct{}{} } mapInsertElapsed := time.Since(start) logResults("Inserts Map[uint64]: %v for %d items\n", mapInsertElapsed, num) start = time.Now() for _, n := range seqs { if _, ok := dmap[n]; !ok { t.Fatalf("Should exist") } } mapLookupElapsed := time.Since(start) logResults("Lookups: %v\n", mapLookupElapsed) // In general we are between 1.5 and 1.75 times slower atm then a straight map. // Let's test an upper bound of 2x for now. if mapInsertElapsed*2 <= ssInsertElapsed { t.Fatalf("Expected SequenceSet insert to be no more than 2x slower (%v vs %v)", mapInsertElapsed, ssInsertElapsed) } if mapLookupElapsed*3 <= ssLookupElapsed { t.Fatalf("Expected SequenceSet lookups to be no more than 3x slower (%v vs %v)", mapLookupElapsed, ssLookupElapsed) } } // friendlyBytes returns a string with the given bytes int64 // represented as a size, such as 1KB, 10MB, etc... func friendlyBytes[T int | uint64 | int64](bytes T) string { fbytes := float64(bytes) base := 1024 pre := []string{"K", "M", "G", "T", "P", "E"} if fbytes < float64(base) { return fmt.Sprintf("%v B", fbytes) } exp := int(math.Log(fbytes) / math.Log(float64(base))) index := exp - 1 return fmt.Sprintf("%.2f %sB", fbytes/math.Pow(float64(base), float64(exp)), pre[index]) } func logResults(format string, args ...any) { if *printResults { fmt.Printf(format, args...) } } nats-server-2.10.27/server/avl/seqset.go000066400000000000000000000337311477524627100201050ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package avl import ( "cmp" "encoding/binary" "errors" "math/bits" "slices" ) // SequenceSet is a memory and encoding optimized set for storing unsigned ints. // // SequenceSet is ~80-100 times more efficient memory wise than a map[uint64]struct{}. // SequenceSet is ~1.75 times slower at inserts than the same map. // SequenceSet is not thread safe. // // We use an AVL tree with nodes that hold bitmasks for set membership. // // Encoding will convert to a space optimized encoding using bitmasks. type SequenceSet struct { root *node // root node size int // number of items nodes int // number of nodes // Having this here vs on the stack in Insert/Delete // makes a difference in memory usage. changed bool } // Insert will insert the sequence into the set. // The tree will be balanced inline. func (ss *SequenceSet) Insert(seq uint64) { if ss.root = ss.root.insert(seq, &ss.changed, &ss.nodes); ss.changed { ss.changed = false ss.size++ } } // Exists will return true iff the sequence is a member of this set. func (ss *SequenceSet) Exists(seq uint64) bool { for n := ss.root; n != nil; { if seq < n.base { n = n.l continue } else if seq >= n.base+numEntries { n = n.r continue } return n.exists(seq) } return false } // SetInitialMin should be used to set the initial minimum sequence when known. // This will more effectively utilize space versus self selecting. // The set should be empty. func (ss *SequenceSet) SetInitialMin(min uint64) error { if !ss.IsEmpty() { return ErrSetNotEmpty } ss.root, ss.nodes = &node{base: min, h: 1}, 1 return nil } // Delete will remove the sequence from the set. // Will optionally remove nodes and rebalance. // Returns where the sequence was set. func (ss *SequenceSet) Delete(seq uint64) bool { if ss == nil || ss.root == nil { return false } ss.root = ss.root.delete(seq, &ss.changed, &ss.nodes) if ss.changed { ss.changed = false ss.size-- if ss.size == 0 { ss.Empty() } return true } return false } // Size returns the number of items in the set. func (ss *SequenceSet) Size() int { return ss.size } // Nodes returns the number of nodes in the tree. func (ss *SequenceSet) Nodes() int { return ss.nodes } // Empty will clear all items from a set. func (ss *SequenceSet) Empty() { ss.root = nil ss.size = 0 ss.nodes = 0 } // IsEmpty is a fast check of the set being empty. func (ss *SequenceSet) IsEmpty() bool { if ss == nil || ss.root == nil { return true } return false } // Range will invoke the given function for each item in the set. // They will range over the set in ascending order. // If the callback returns false we terminate the iteration. func (ss *SequenceSet) Range(f func(uint64) bool) { ss.root.iter(f) } // Heights returns the left and right heights of the tree. func (ss *SequenceSet) Heights() (l, r int) { if ss.root == nil { return 0, 0 } if ss.root.l != nil { l = ss.root.l.h } if ss.root.r != nil { r = ss.root.r.h } return l, r } // Returns min, max and number of set items. func (ss *SequenceSet) State() (min, max, num uint64) { if ss == nil || ss.root == nil { return 0, 0, 0 } min, max = ss.MinMax() return min, max, uint64(ss.Size()) } // MinMax will return the minunum and maximum values in the set. func (ss *SequenceSet) MinMax() (min, max uint64) { if ss.root == nil { return 0, 0 } for l := ss.root; l != nil; l = l.l { if l.l == nil { min = l.min() } } for r := ss.root; r != nil; r = r.r { if r.r == nil { max = r.max() } } return min, max } func clone(src *node, target **node) { if src == nil { return } n := &node{base: src.base, bits: src.bits, h: src.h} *target = n clone(src.l, &n.l) clone(src.r, &n.r) } // Clone will return a clone of the given SequenceSet. func (ss *SequenceSet) Clone() *SequenceSet { if ss == nil { return nil } css := &SequenceSet{nodes: ss.nodes, size: ss.size} clone(ss.root, &css.root) return css } // Union will union this SequenceSet with ssa. func (ss *SequenceSet) Union(ssa ...*SequenceSet) { for _, sa := range ssa { sa.root.nodeIter(func(n *node) { for nb, b := range n.bits { for pos := uint64(0); b != 0; pos++ { if b&1 == 1 { seq := n.base + (uint64(nb) * uint64(bitsPerBucket)) + pos ss.Insert(seq) } b >>= 1 } } }) } } // Union will return a union of all sets. func Union(ssa ...*SequenceSet) *SequenceSet { if len(ssa) == 0 { return nil } // Sort so we can clone largest. slices.SortFunc(ssa, func(i, j *SequenceSet) int { return -cmp.Compare(i.Size(), j.Size()) }) // reverse order ss := ssa[0].Clone() // Insert the rest through range call. for i := 1; i < len(ssa); i++ { ssa[i].Range(func(n uint64) bool { ss.Insert(n) return true }) } return ss } const ( // Magic is used to identify the encode binary state.. magic = uint8(22) // Version version = uint8(2) // hdrLen hdrLen = 2 // minimum length of an encoded SequenceSet. minLen = 2 + 8 // magic + version + num nodes + num entries. ) // EncodeLen returns the bytes needed for encoding. func (ss SequenceSet) EncodeLen() int { return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2)) } func (ss SequenceSet) Encode(buf []byte) ([]byte, error) { nn, encLen := ss.Nodes(), ss.EncodeLen() if cap(buf) < encLen { buf = make([]byte, encLen) } else { buf = buf[:encLen] } // TODO(dlc) - Go 1.19 introduced Append to not have to keep track. // Once 1.20 is out we could change this over. // Also binary.Write() is way slower, do not use. var le = binary.LittleEndian buf[0], buf[1] = magic, version i := hdrLen le.PutUint32(buf[i:], uint32(nn)) le.PutUint32(buf[i+4:], uint32(ss.size)) i += 8 ss.root.nodeIter(func(n *node) { le.PutUint64(buf[i:], n.base) i += 8 for _, b := range n.bits { le.PutUint64(buf[i:], b) i += 8 } le.PutUint16(buf[i:], uint16(n.h)) i += 2 }) return buf[:i], nil } // ErrBadEncoding is returned when we can not decode properly. var ( ErrBadEncoding = errors.New("ss: bad encoding") ErrBadVersion = errors.New("ss: bad version") ErrSetNotEmpty = errors.New("ss: set not empty") ) // Decode returns the sequence set and number of bytes read from the buffer on success. func Decode(buf []byte) (*SequenceSet, int, error) { if len(buf) < minLen || buf[0] != magic { return nil, -1, ErrBadEncoding } switch v := buf[1]; v { case 1: return decodev1(buf) case 2: return decodev2(buf) default: return nil, -1, ErrBadVersion } } // Helper to decode v2. func decodev2(buf []byte) (*SequenceSet, int, error) { var le = binary.LittleEndian index := 2 nn := int(le.Uint32(buf[index:])) sz := int(le.Uint32(buf[index+4:])) index += 8 expectedLen := minLen + (nn * ((numBuckets+1)*8 + 2)) if len(buf) < expectedLen { return nil, -1, ErrBadEncoding } ss, nodes := SequenceSet{size: sz}, make([]node, nn) for i := 0; i < nn; i++ { n := &nodes[i] n.base = le.Uint64(buf[index:]) index += 8 for bi := range n.bits { n.bits[bi] = le.Uint64(buf[index:]) index += 8 } n.h = int(le.Uint16(buf[index:])) index += 2 ss.insertNode(n) } return &ss, index, nil } // Helper to decode v1 into v2 which has fixed buckets of 32 vs 64 originally. func decodev1(buf []byte) (*SequenceSet, int, error) { var le = binary.LittleEndian index := 2 nn := int(le.Uint32(buf[index:])) sz := int(le.Uint32(buf[index+4:])) index += 8 const v1NumBuckets = 64 expectedLen := minLen + (nn * ((v1NumBuckets+1)*8 + 2)) if len(buf) < expectedLen { return nil, -1, ErrBadEncoding } var ss SequenceSet for i := 0; i < nn; i++ { base := le.Uint64(buf[index:]) index += 8 for nb := uint64(0); nb < v1NumBuckets; nb++ { n := le.Uint64(buf[index:]) // Walk all set bits and insert sequences manually for this decode from v1. for pos := uint64(0); n != 0; pos++ { if n&1 == 1 { seq := base + (nb * uint64(bitsPerBucket)) + pos ss.Insert(seq) } n >>= 1 } index += 8 } // Skip over encoded height. index += 2 } // Sanity check. if ss.Size() != sz { return nil, -1, ErrBadEncoding } return &ss, index, nil } // insertNode places a decoded node into the tree. // These should be done in tree order as defined by Encode() // This allows us to not have to calculate height or do rebalancing. // So much better performance this way. func (ss *SequenceSet) insertNode(n *node) { ss.nodes++ if ss.root == nil { ss.root = n return } // Walk our way to the insertion point. for p := ss.root; p != nil; { if n.base < p.base { if p.l == nil { p.l = n return } p = p.l } else { if p.r == nil { p.r = n return } p = p.r } } } const ( bitsPerBucket = 64 // bits in uint64 numBuckets = 32 numEntries = numBuckets * bitsPerBucket ) type node struct { //v dvalue base uint64 bits [numBuckets]uint64 l *node r *node h int } // Set the proper bit. // seq should have already been qualified and inserted should be non nil. func (n *node) set(seq uint64, inserted *bool) { seq -= n.base i := seq / bitsPerBucket mask := uint64(1) << (seq % bitsPerBucket) if (n.bits[i] & mask) == 0 { n.bits[i] |= mask *inserted = true } } func (n *node) insert(seq uint64, inserted *bool, nodes *int) *node { if n == nil { base := (seq / numEntries) * numEntries n := &node{base: base, h: 1} n.set(seq, inserted) *nodes++ return n } if seq < n.base { n.l = n.l.insert(seq, inserted, nodes) } else if seq >= n.base+numEntries { n.r = n.r.insert(seq, inserted, nodes) } else { n.set(seq, inserted) } n.h = maxH(n) + 1 // Don't make a function, impacts performance. if bf := balanceF(n); bf > 1 { // Left unbalanced. if balanceF(n.l) < 0 { n.l = n.l.rotateL() } return n.rotateR() } else if bf < -1 { // Right unbalanced. if balanceF(n.r) > 0 { n.r = n.r.rotateR() } return n.rotateL() } return n } func (n *node) rotateL() *node { r := n.r if r != nil { n.r = r.l r.l = n n.h = maxH(n) + 1 r.h = maxH(r) + 1 } else { n.r = nil n.h = maxH(n) + 1 } return r } func (n *node) rotateR() *node { l := n.l if l != nil { n.l = l.r l.r = n n.h = maxH(n) + 1 l.h = maxH(l) + 1 } else { n.l = nil n.h = maxH(n) + 1 } return l } func balanceF(n *node) int { if n == nil { return 0 } var lh, rh int if n.l != nil { lh = n.l.h } if n.r != nil { rh = n.r.h } return lh - rh } func maxH(n *node) int { if n == nil { return 0 } var lh, rh int if n.l != nil { lh = n.l.h } if n.r != nil { rh = n.r.h } if lh > rh { return lh } return rh } // Clear the proper bit. // seq should have already been qualified and deleted should be non nil. // Will return true if this node is now empty. func (n *node) clear(seq uint64, deleted *bool) bool { seq -= n.base i := seq / bitsPerBucket mask := uint64(1) << (seq % bitsPerBucket) if (n.bits[i] & mask) != 0 { n.bits[i] &^= mask *deleted = true } for _, b := range n.bits { if b != 0 { return false } } return true } func (n *node) delete(seq uint64, deleted *bool, nodes *int) *node { if n == nil { return nil } if seq < n.base { n.l = n.l.delete(seq, deleted, nodes) } else if seq >= n.base+numEntries { n.r = n.r.delete(seq, deleted, nodes) } else if empty := n.clear(seq, deleted); empty { *nodes-- if n.l == nil { n = n.r } else if n.r == nil { n = n.l } else { // We have both children. n.r = n.r.insertNodePrev(n.l) n = n.r } } if n != nil { n.h = maxH(n) + 1 } // Check balance. if bf := balanceF(n); bf > 1 { // Left unbalanced. if balanceF(n.l) < 0 { n.l = n.l.rotateL() } return n.rotateR() } else if bf < -1 { // right unbalanced. if balanceF(n.r) > 0 { n.r = n.r.rotateR() } return n.rotateL() } return n } // Will insert nn into the node assuming it is less than all other nodes in n. // Will re-calculate height and balance. func (n *node) insertNodePrev(nn *node) *node { if n.l == nil { n.l = nn } else { n.l = n.l.insertNodePrev(nn) } n.h = maxH(n) + 1 // Check balance. if bf := balanceF(n); bf > 1 { // Left unbalanced. if balanceF(n.l) < 0 { n.l = n.l.rotateL() } return n.rotateR() } else if bf < -1 { // right unbalanced. if balanceF(n.r) > 0 { n.r = n.r.rotateR() } return n.rotateL() } return n } func (n *node) exists(seq uint64) bool { seq -= n.base i := seq / bitsPerBucket mask := uint64(1) << (seq % bitsPerBucket) return n.bits[i]&mask != 0 } // Return minimum sequence in the set. // This node can not be empty. func (n *node) min() uint64 { for i, b := range n.bits { if b != 0 { return n.base + uint64(i*bitsPerBucket) + uint64(bits.TrailingZeros64(b)) } } return 0 } // Return maximum sequence in the set. // This node can not be empty. func (n *node) max() uint64 { for i := numBuckets - 1; i >= 0; i-- { if b := n.bits[i]; b != 0 { return n.base + uint64(i*bitsPerBucket) + uint64(bitsPerBucket-bits.LeadingZeros64(b>>1)) } } return 0 } // This is done in tree order. func (n *node) nodeIter(f func(n *node)) { if n == nil { return } f(n) n.l.nodeIter(f) n.r.nodeIter(f) } // iter will iterate through the set's items in this node. // If the supplied function returns false we terminate the iteration. func (n *node) iter(f func(uint64) bool) bool { if n == nil { return true } if ok := n.l.iter(f); !ok { return false } for num := n.base; num < n.base+numEntries; num++ { if n.exists(num) { if ok := f(num); !ok { return false } } } if ok := n.r.iter(f); !ok { return false } return true } nats-server-2.10.27/server/avl/seqset_test.go000066400000000000000000000214521477524627100211410ustar00rootroot00000000000000// Copyright 2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package avl import ( "encoding/base64" "math/rand" "testing" ) func TestSeqSetBasics(t *testing.T) { var ss SequenceSet seqs := []uint64{22, 222, 2000, 2, 2, 4} for _, seq := range seqs { ss.Insert(seq) require_True(t, ss.Exists(seq)) } require_True(t, ss.Nodes() == 1) require_True(t, ss.Size() == len(seqs)-1) // We have one dup in there. lh, rh := ss.Heights() require_True(t, lh == 0) require_True(t, rh == 0) } func TestSeqSetLeftLean(t *testing.T) { var ss SequenceSet for i := uint64(4 * numEntries); i > 0; i-- { ss.Insert(i) } require_True(t, ss.Nodes() == 5) require_True(t, ss.Size() == 4*numEntries) lh, rh := ss.Heights() require_True(t, lh == 2) require_True(t, rh == 1) } func TestSeqSetRightLean(t *testing.T) { var ss SequenceSet for i := uint64(0); i < uint64(4*numEntries); i++ { ss.Insert(i) } require_True(t, ss.Nodes() == 4) require_True(t, ss.Size() == 4*numEntries) lh, rh := ss.Heights() require_True(t, lh == 1) require_True(t, rh == 2) } func TestSeqSetCorrectness(t *testing.T) { // Generate 100k sequences across 500k range. num := 100_000 max := 500_000 set := make(map[uint64]struct{}, num) var ss SequenceSet for i := 0; i < num; i++ { n := uint64(rand.Int63n(int64(max + 1))) ss.Insert(n) set[n] = struct{}{} } for i := uint64(0); i <= uint64(max); i++ { _, exists := set[i] require_True(t, ss.Exists(i) == exists) } } func TestSeqSetRange(t *testing.T) { num := 2*numEntries + 22 nums := make([]uint64, 0, num) for i := 0; i < num; i++ { nums = append(nums, uint64(i)) } rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) var ss SequenceSet for _, n := range nums { ss.Insert(n) } nums = nums[:0] ss.Range(func(n uint64) bool { nums = append(nums, n) return true }) require_True(t, len(nums) == num) for i := uint64(0); i < uint64(num); i++ { require_True(t, nums[i] == i) } // Test truncating the range call. nums = nums[:0] ss.Range(func(n uint64) bool { if n >= 10 { return false } nums = append(nums, n) return true }) require_True(t, len(nums) == 10) for i := uint64(0); i < 10; i++ { require_True(t, nums[i] == i) } } func TestSeqSetDelete(t *testing.T) { var ss SequenceSet // Simple single node. seqs := []uint64{22, 222, 2222, 2, 2, 4} for _, seq := range seqs { ss.Insert(seq) } for _, seq := range seqs { ss.Delete(seq) require_True(t, !ss.Exists(seq)) } require_True(t, ss.root == nil) } func TestSeqSetInsertAndDeletePedantic(t *testing.T) { var ss SequenceSet num := 50*numEntries + 22 nums := make([]uint64, 0, num) for i := 0; i < num; i++ { nums = append(nums, uint64(i)) } rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) // Make sure always balanced. testBalanced := func() { t.Helper() // Check heights. ss.root.nodeIter(func(n *node) { if n != nil && n.h != maxH(n)+1 { t.Fatalf("Node height is wrong: %+v", n) } }) // Check balance factor. if bf := balanceF(ss.root); bf > 1 || bf < -1 { t.Fatalf("Unbalanced tree") } } for _, n := range nums { ss.Insert(n) testBalanced() } require_True(t, ss.root != nil) for _, n := range nums { ss.Delete(n) testBalanced() require_True(t, !ss.Exists(n)) if ss.Size() > 0 { require_True(t, ss.root != nil) } } require_True(t, ss.root == nil) } func TestSeqSetMinMax(t *testing.T) { var ss SequenceSet // Simple single node. seqs := []uint64{22, 222, 2222, 2, 2, 4} for _, seq := range seqs { ss.Insert(seq) } min, max := ss.MinMax() require_True(t, min == 2 && max == 2222) // Multi-node ss.Empty() num := 22*numEntries + 22 nums := make([]uint64, 0, num) for i := 0; i < num; i++ { nums = append(nums, uint64(i)) } rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) for _, n := range nums { ss.Insert(n) } min, max = ss.MinMax() require_True(t, min == 0 && max == uint64(num-1)) } func TestSeqSetClone(t *testing.T) { // Generate 100k sequences across 500k range. num := 100_000 max := 500_000 var ss SequenceSet for i := 0; i < num; i++ { ss.Insert(uint64(rand.Int63n(int64(max + 1)))) } ssc := ss.Clone() require_True(t, ss.Size() == ssc.Size()) require_True(t, ss.Nodes() == ssc.Nodes()) } func TestSeqSetUnion(t *testing.T) { var ss1, ss2 SequenceSet seqs1 := []uint64{22, 222, 2222, 2, 2, 4} for _, seq := range seqs1 { ss1.Insert(seq) } seqs2 := []uint64{33, 333, 3333, 3, 33_333, 333_333} for _, seq := range seqs2 { ss2.Insert(seq) } ss := Union(&ss1, &ss2) require_True(t, ss.Size() == 11) seqs := append(seqs1, seqs2...) for _, n := range seqs { require_True(t, ss.Exists(n)) } } func TestSeqSetFirst(t *testing.T) { var ss SequenceSet seqs := []uint64{22, 222, 2222, 222_222} for _, seq := range seqs { // Normal case where we pick first/base. ss.Insert(seq) require_True(t, ss.root.base == (seq/numEntries)*numEntries) ss.Empty() // Where we set the minimum start value. ss.SetInitialMin(seq) ss.Insert(seq) require_True(t, ss.root.base == seq) ss.Empty() } } // Test that we can union with nodes vs individual sequence insertion. func TestSeqSetDistinctUnion(t *testing.T) { // Distinct sets. var ss1 SequenceSet seqs1 := []uint64{1, 10, 100, 200} for _, seq := range seqs1 { ss1.Insert(seq) } var ss2 SequenceSet seqs2 := []uint64{5000, 6100, 6200, 6222} for _, seq := range seqs2 { ss2.Insert(seq) } ss := ss1.Clone() allSeqs := append(seqs1, seqs2...) ss.Union(&ss2) require_True(t, ss.Size() == len(allSeqs)) for _, seq := range allSeqs { require_True(t, ss.Exists(seq)) } } func TestSeqSetDecodeV1(t *testing.T) { // Encoding from v1 which was 64 buckets. seqs := []uint64{22, 222, 2222, 222_222, 2_222_222} encStr := ` FgEDAAAABQAAAABgAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAADgIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAA== ` enc, err := base64.StdEncoding.DecodeString(encStr) require_NoError(t, err) ss, _, err := Decode(enc) require_NoError(t, err) require_True(t, ss.Size() == len(seqs)) for _, seq := range seqs { require_True(t, ss.Exists(seq)) } } func require_NoError(t *testing.T, err error) { t.Helper() if err != nil { t.Fatalf("require no error, but got: %v", err) } } func require_True(t *testing.T, b bool) { t.Helper() if !b { t.Fatalf("require true") } } nats-server-2.10.27/server/benchmark_publish_test.go000066400000000000000000000103401477524627100225250ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "math/rand" "sync/atomic" "testing" "github.com/nats-io/nats.go" ) func BenchmarkPublish(b *testing.B) { const ( verbose = false seed = 12345 minMessages = 10_000 subject = "S" queue = "Q" ) const ( KB = 1024 MB = KB * KB ) type SubscriberType string const ( Async SubscriberType = "Async" QueueAsync SubscriberType = "AsyncQueue" None SubscriberType = "None" ) benchmarksCases := []struct { messageSize int }{ {0}, {1}, {32}, {128}, {512}, {4 * KB}, {32 * KB}, {128 * KB}, {512 * KB}, {1 * MB}, } // All the cases above are run for each of the subscriber cases below subscribersCases := []struct { numSubs int subType SubscriberType }{ {0, None}, {1, Async}, {1, QueueAsync}, {10, Async}, {10, QueueAsync}, } for _, bc := range benchmarksCases { bcName := fmt.Sprintf( "MsgSz=%db", bc.messageSize, ) b.Run( bcName, func(b *testing.B) { for _, sc := range subscribersCases { scName := fmt.Sprintf("Subs=%dx%v", sc.numSubs, sc.subType) if sc.subType == None { scName = fmt.Sprintf("Subs=%v", sc.subType) } b.Run( scName, func(b *testing.B) { // Skip short runs, benchmark gets re-executed with a larger N if b.N < minMessages { b.ResetTimer() return } if verbose { b.Logf("Running %s/%s with %d ops", bcName, scName, b.N) } subErrors := uint64(0) handleSubError := func(_ *nats.Conn, _ *nats.Subscription, _ error) { atomic.AddUint64(&subErrors, 1) } // Start single server (no JS) opts := DefaultTestOptions opts.Port = -1 s := RunServer(&opts) defer s.Shutdown() // Create subscribers for i := 0; i < sc.numSubs; i++ { subConn, connErr := nats.Connect(s.ClientURL(), nats.ErrorHandler(handleSubError)) if connErr != nil { b.Fatalf("Failed to connect: %v", connErr) } defer subConn.Close() var sub *nats.Subscription var subErr error switch sc.subType { case None: // No subscription case Async: sub, subErr = subConn.Subscribe(subject, func(*nats.Msg) {}) case QueueAsync: sub, subErr = subConn.QueueSubscribe(subject, queue, func(*nats.Msg) {}) default: b.Fatalf("Unknow subscribers type: %v", sc.subType) } if subErr != nil { b.Fatalf("Failed to subscribe: %v", subErr) } defer sub.Unsubscribe() // Do not drop messages due to slow subscribers: sub.SetPendingLimits(-1, -1) } // Create publisher connection nc, err := nats.Connect(s.ClientURL()) if err != nil { b.Fatalf("Failed to connect: %v", err) } defer nc.Close() rng := rand.New(rand.NewSource(int64(seed))) message := make([]byte, bc.messageSize) var published, errors int // Benchmark starts here b.ResetTimer() for i := 0; i < b.N; i++ { rng.Read(message) pubErr := nc.Publish(subject, message) if pubErr != nil { errors++ } else { published++ b.SetBytes(int64(bc.messageSize)) } } // Benchmark ends here b.StopTimer() if published+errors != b.N { b.Fatalf("Something doesn't add up: %d + %d != %d", published, errors, b.N) } else if subErrors > 0 { b.Fatalf("Subscribers errors: %d", subErrors) } b.ReportMetric(float64(errors)*100/float64(b.N), "%error") }, ) } }, ) } } nats-server-2.10.27/server/certidp/000077500000000000000000000000001477524627100171135ustar00rootroot00000000000000nats-server-2.10.27/server/certidp/certidp.go000066400000000000000000000237601477524627100211040ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certidp import ( "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/json" "fmt" "net/url" "strings" "time" "golang.org/x/crypto/ocsp" ) const ( DefaultAllowedClockSkew = 30 * time.Second DefaultOCSPResponderTimeout = 2 * time.Second DefaultTTLUnsetNextUpdate = 1 * time.Hour ) type StatusAssertion int var ( StatusAssertionStrToVal = map[string]StatusAssertion{ "good": ocsp.Good, "revoked": ocsp.Revoked, "unknown": ocsp.Unknown, } StatusAssertionValToStr = map[StatusAssertion]string{ ocsp.Good: "good", ocsp.Revoked: "revoked", ocsp.Unknown: "unknown", } StatusAssertionIntToVal = map[int]StatusAssertion{ 0: ocsp.Good, 1: ocsp.Revoked, 2: ocsp.Unknown, } ) // GetStatusAssertionStr returns the corresponding string representation of the StatusAssertion. func GetStatusAssertionStr(sa int) string { // If the provided status assertion value is not found in the map (StatusAssertionIntToVal), // the function defaults to "unknown" to avoid defaulting to "good," which is the default iota value // for the ocsp.StatusAssertion enumeration (https://pkg.go.dev/golang.org/x/crypto/ocsp#pkg-constants). // This ensures that we don't unintentionally default to "good" when there's no map entry. v, ok := StatusAssertionIntToVal[sa] if !ok { // set unknown as fallback v = ocsp.Unknown } return StatusAssertionValToStr[v] } func (sa StatusAssertion) MarshalJSON() ([]byte, error) { // This ensures that we don't unintentionally default to "good" when there's no map entry. // (see more details in the GetStatusAssertionStr() comment) str, ok := StatusAssertionValToStr[sa] if !ok { // set unknown as fallback str = StatusAssertionValToStr[ocsp.Unknown] } return json.Marshal(str) } func (sa *StatusAssertion) UnmarshalJSON(in []byte) error { // This ensures that we don't unintentionally default to "good" when there's no map entry. // (see more details in the GetStatusAssertionStr() comment) v, ok := StatusAssertionStrToVal[strings.ReplaceAll(string(in), "\"", "")] if !ok { // set unknown as fallback v = StatusAssertionStrToVal["unknown"] } *sa = v return nil } type ChainLink struct { Leaf *x509.Certificate Issuer *x509.Certificate OCSPWebEndpoints *[]*url.URL } // OCSPPeerConfig holds the parsed OCSP peer configuration section of TLS configuration type OCSPPeerConfig struct { Verify bool Timeout float64 ClockSkew float64 WarnOnly bool UnknownIsGood bool AllowWhenCAUnreachable bool TTLUnsetNextUpdate float64 } func NewOCSPPeerConfig() *OCSPPeerConfig { return &OCSPPeerConfig{ Verify: false, Timeout: DefaultOCSPResponderTimeout.Seconds(), ClockSkew: DefaultAllowedClockSkew.Seconds(), WarnOnly: false, UnknownIsGood: false, AllowWhenCAUnreachable: false, TTLUnsetNextUpdate: DefaultTTLUnsetNextUpdate.Seconds(), } } // Log is a neutral method of passing server loggers to plugins type Log struct { Debugf func(format string, v ...any) Noticef func(format string, v ...any) Warnf func(format string, v ...any) Errorf func(format string, v ...any) Tracef func(format string, v ...any) } type CertInfo struct { Subject string `json:"subject,omitempty"` Issuer string `json:"issuer,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Raw []byte `json:"raw,omitempty"` } var OCSPPeerUsage = ` For client, leaf spoke (remotes), and leaf hub connections, you may enable OCSP peer validation: tls { ... # mTLS must be enabled (with exception of Leaf remotes) verify: true ... # short form enables peer verify and takes option defaults ocsp_peer: true # long form includes settable options ocsp_peer { # Enable OCSP peer validation (default false) verify: true # OCSP responder timeout in seconds (may be fractional, default 2 seconds) ca_timeout: 2 # Allowed skew between server and OCSP responder time in seconds (may be fractional, default 30 seconds) allowed_clockskew: 30 # Warn-only and never reject connections (default false) warn_only: false # Treat response Unknown status as valid certificate (default false) unknown_is_good: false # Warn-only if no CA response can be obtained and no cached revocation exists (default false) allow_when_ca_unreachable: false # If response NextUpdate unset by CA, set a default cache TTL in seconds from ThisUpdate (default 1 hour) cache_ttl_when_next_update_unset: 3600 } ... } Note: OCSP validation for route and gateway connections is enabled using the 'ocsp' configuration option. ` // GenerateFingerprint returns a base64-encoded SHA256 hash of the raw certificate func GenerateFingerprint(cert *x509.Certificate) string { data := sha256.Sum256(cert.Raw) return base64.StdEncoding.EncodeToString(data[:]) } func getWebEndpoints(uris []string) []*url.URL { var urls []*url.URL for _, uri := range uris { endpoint, err := url.ParseRequestURI(uri) if err != nil { // skip invalid URLs continue } if endpoint.Scheme != "http" && endpoint.Scheme != "https" { // skip non-web URLs continue } urls = append(urls, endpoint) } return urls } // GetSubjectDNForm returns RDN sequence concatenation of the certificate's subject to be // used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes. func GetSubjectDNForm(cert *x509.Certificate) string { if cert == nil { return "" } return strings.TrimSuffix(fmt.Sprintf("%s+", cert.Subject.ToRDNSequence()), "+") } // GetIssuerDNForm returns RDN sequence concatenation of the certificate's issuer to be // used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes. func GetIssuerDNForm(cert *x509.Certificate) string { if cert == nil { return "" } return strings.TrimSuffix(fmt.Sprintf("%s+", cert.Issuer.ToRDNSequence()), "+") } // CertOCSPEligible checks if the certificate's issuer has populated AIA with OCSP responder endpoint(s) // and is thus eligible for OCSP validation func CertOCSPEligible(link *ChainLink) bool { if link == nil || link.Leaf.Raw == nil || len(link.Leaf.Raw) == 0 { return false } if len(link.Leaf.OCSPServer) == 0 { return false } urls := getWebEndpoints(link.Leaf.OCSPServer) if len(urls) == 0 { return false } link.OCSPWebEndpoints = &urls return true } // GetLeafIssuerCert returns the issuer certificate of the leaf (positional) certificate in the chain func GetLeafIssuerCert(chain []*x509.Certificate, leafPos int) *x509.Certificate { if len(chain) == 0 || leafPos < 0 { return nil } // self-signed certificate or too-big leafPos if leafPos >= len(chain)-1 { return nil } // returns pointer to issuer cert or nil return (chain)[leafPos+1] } // OCSPResponseCurrent checks if the OCSP response is current (i.e. not expired and not future effective) func OCSPResponseCurrent(ocspr *ocsp.Response, opts *OCSPPeerConfig, log *Log) bool { skew := time.Duration(opts.ClockSkew * float64(time.Second)) if skew < 0*time.Second { skew = DefaultAllowedClockSkew } now := time.Now().UTC() // Typical effectivity check based on CA response ThisUpdate and NextUpdate semantics if !ocspr.NextUpdate.IsZero() && ocspr.NextUpdate.Before(now.Add(-1*skew)) { t := ocspr.NextUpdate.Format(time.RFC3339Nano) nt := now.Format(time.RFC3339Nano) log.Debugf(DbgResponseExpired, t, nt, skew) return false } // CA responder can assert NextUpdate unset, in which case use config option to set a default cache TTL if ocspr.NextUpdate.IsZero() { ttl := time.Duration(opts.TTLUnsetNextUpdate * float64(time.Second)) if ttl < 0*time.Second { ttl = DefaultTTLUnsetNextUpdate } expiryTime := ocspr.ThisUpdate.Add(ttl) if expiryTime.Before(now.Add(-1 * skew)) { t := expiryTime.Format(time.RFC3339Nano) nt := now.Format(time.RFC3339Nano) log.Debugf(DbgResponseTTLExpired, t, nt, skew) return false } } if ocspr.ThisUpdate.After(now.Add(skew)) { t := ocspr.ThisUpdate.Format(time.RFC3339Nano) nt := now.Format(time.RFC3339Nano) log.Debugf(DbgResponseFutureDated, t, nt, skew) return false } return true } // ValidDelegationCheck checks if the CA OCSP Response was signed by a valid CA Issuer delegate as per (RFC 6960, section 4.2.2.2) // If a valid delegate or direct-signed by CA Issuer, true returned. func ValidDelegationCheck(iss *x509.Certificate, ocspr *ocsp.Response) bool { // This call assumes prior successful parse and signature validation of the OCSP response // The Go OCSP library (as of x/crypto/ocsp v0.9) will detect and perform a 1-level delegate signature check but does not // implement the additional criteria for delegation specified in RFC 6960, section 4.2.2.2. if iss == nil || ocspr == nil { return false } // not a delegation, no-op if ocspr.Certificate == nil { return true } // delegate is self-same with CA Issuer, not a delegation although response issued in that form if ocspr.Certificate.Equal(iss) { return true } // we need to verify CA Issuer stamped id-kp-OCSPSigning on delegate delegatedSigner := false for _, keyUseExt := range ocspr.Certificate.ExtKeyUsage { if keyUseExt == x509.ExtKeyUsageOCSPSigning { delegatedSigner = true break } } return delegatedSigner } nats-server-2.10.27/server/certidp/certidp_test.go000066400000000000000000000025561477524627100221430ustar00rootroot00000000000000// Copyright 2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certidp import "testing" // Checks the return values of the function GetStatusAssertionStr func TestGetStatusAssertionStr(t *testing.T) { tests := []struct { name string input int expected string }{ { name: "GoodStatus", input: 0, expected: "good", }, { name: "RevokedStatus", input: 1, expected: "revoked", }, { name: "UnknownStatus", input: 2, expected: "unknown", }, // Invalid status assertion value. { name: "InvalidStatus", input: 42, expected: "unknown", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := GetStatusAssertionStr(tt.input) if got != tt.expected { t.Errorf("Expected GetStatusAssertionStr: %v, got %v", tt.expected, got) } }) } } nats-server-2.10.27/server/certidp/messages.go000066400000000000000000000153161477524627100212570ustar00rootroot00000000000000// Copyright 2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certidp var ( // Returned errors ErrIllegalPeerOptsConfig = "expected map to define OCSP peer options, got [%T]" ErrIllegalCacheOptsConfig = "expected map to define OCSP peer cache options, got [%T]" ErrParsingPeerOptFieldGeneric = "error parsing tls peer config, unknown field [%q]" ErrParsingPeerOptFieldTypeConversion = "error parsing tls peer config, conversion error: %s" ErrParsingCacheOptFieldTypeConversion = "error parsing OCSP peer cache config, conversion error: %s" ErrUnableToPlugTLSEmptyConfig = "unable to plug TLS verify connection, config is nil" ErrMTLSRequired = "OCSP peer verification for client connections requires TLS verify (mTLS) to be enabled" ErrUnableToPlugTLSClient = "unable to register client OCSP verification" ErrUnableToPlugTLSServer = "unable to register server OCSP verification" ErrCannotWriteCompressed = "error writing to compression writer: %w" ErrCannotReadCompressed = "error reading compression reader: %w" ErrTruncatedWrite = "short write on body (%d != %d)" ErrCannotCloseWriter = "error closing compression writer: %w" ErrParsingCacheOptFieldGeneric = "error parsing OCSP peer cache config, unknown field [%q]" ErrUnknownCacheType = "error parsing OCSP peer cache config, unknown type [%s]" ErrInvalidChainlink = "invalid chain link" ErrBadResponderHTTPStatus = "bad OCSP responder http status: [%d]" ErrNoAvailOCSPServers = "no available OCSP servers" ErrFailedWithAllRequests = "exhausted OCSP responders: %w" // Direct logged errors ErrLoadCacheFail = "Unable to load OCSP peer cache: %s" ErrSaveCacheFail = "Unable to save OCSP peer cache: %s" ErrBadCacheTypeConfig = "Unimplemented OCSP peer cache type [%v]" ErrResponseCompressFail = "Unable to compress OCSP response for key [%s]: %s" ErrResponseDecompressFail = "Unable to decompress OCSP response for key [%s]: %s" ErrPeerEmptyNoEvent = "Peer certificate is nil, cannot send OCSP peer reject event" ErrPeerEmptyAutoReject = "Peer certificate is nil, rejecting OCSP peer" // Debug information DbgPlugTLSForKind = "Plugging TLS OCSP peer for [%s]" DbgNumServerChains = "Peer OCSP enabled: %d TLS server chain(s) will be evaluated" DbgNumClientChains = "Peer OCSP enabled: %d TLS client chain(s) will be evaluated" DbgLinksInChain = "Chain [%d]: %d total link(s)" DbgSelfSignedValid = "Chain [%d] is self-signed, thus peer is valid" DbgValidNonOCSPChain = "Chain [%d] has no OCSP eligible links, thus peer is valid" DbgChainIsOCSPEligible = "Chain [%d] has %d OCSP eligible link(s)" DbgChainIsOCSPValid = "Chain [%d] is OCSP valid for all eligible links, thus peer is valid" DbgNoOCSPValidChains = "No OCSP valid chains, thus peer is invalid" DbgCheckingCacheForCert = "Checking OCSP peer cache for [%s], key [%s]" DbgCurrentResponseCached = "Cached OCSP response is current, status [%s]" DbgExpiredResponseCached = "Cached OCSP response is expired, status [%s]" DbgOCSPValidPeerLink = "OCSP verify pass for [%s]" DbgCachingResponse = "Caching OCSP response for [%s], key [%s]" DbgAchievedCompression = "OCSP response compression ratio: [%f]" DbgCacheHit = "OCSP peer cache hit for key [%s]" DbgCacheMiss = "OCSP peer cache miss for key [%s]" DbgPreservedRevocation = "Revoked OCSP response for key [%s] preserved by cache policy" DbgDeletingCacheResponse = "Deleting OCSP peer cached response for key [%s]" DbgStartingCache = "Starting OCSP peer cache" DbgStoppingCache = "Stopping OCSP peer cache" DbgLoadingCache = "Loading OCSP peer cache [%s]" DbgNoCacheFound = "No OCSP peer cache found, starting with empty cache" DbgSavingCache = "Saving OCSP peer cache [%s]" DbgCacheSaved = "Saved OCSP peer cache successfully (%d bytes)" DbgMakingCARequest = "Trying OCSP responder url [%s]" DbgResponseExpired = "OCSP response NextUpdate [%s] is before now [%s] with clockskew [%s]" DbgResponseTTLExpired = "OCSP response cache expiry [%s] is before now [%s] with clockskew [%s]" DbgResponseFutureDated = "OCSP response ThisUpdate [%s] is before now [%s] with clockskew [%s]" DbgCacheSaveTimerExpired = "OCSP peer cache save timer expired" DbgCacheDirtySave = "OCSP peer cache is dirty, saving" // Returned to peer as TLS reject reason MsgTLSClientRejectConnection = "client not OCSP valid" MsgTLSServerRejectConnection = "server not OCSP valid" // Expected runtime errors (direct logged) ErrCAResponderCalloutFail = "Attempt to obtain OCSP response from CA responder for [%s] failed: %s" ErrNewCAResponseNotCurrent = "New OCSP CA response obtained for [%s] but not current" ErrCAResponseParseFailed = "Could not parse OCSP CA response for [%s]: %s" ErrOCSPInvalidPeerLink = "OCSP verify fail for [%s] with CA status [%s]" // Policy override warnings (direct logged) MsgAllowWhenCAUnreachableOccurred = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; no cached revocation so allowing" MsgAllowWhenCAUnreachableOccurredCachedRevoke = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; cached revocation exists so rejecting" MsgAllowWarnOnlyOccurred = "OCSP verify fail for [%s] but WarnOnly is true so allowing" // Info (direct logged) MsgCacheOnline = "OCSP peer cache online, type [%s]" MsgCacheOffline = "OCSP peer cache offline, type [%s]" // OCSP cert invalid reasons (debug and event reasons) MsgFailedOCSPResponseFetch = "Failed OCSP response fetch" MsgOCSPResponseNotEffective = "OCSP response not in effectivity window" MsgFailedOCSPResponseParse = "Failed OCSP response parse" MsgOCSPResponseInvalidStatus = "Invalid OCSP response status: %s" MsgOCSPResponseDelegationInvalid = "Invalid OCSP response delegation: %s" MsgCachedOCSPResponseInvalid = "Invalid cached OCSP response for [%s] with fingerprint [%s]" ) nats-server-2.10.27/server/certidp/ocsp_responder.go000066400000000000000000000041361477524627100224730ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certidp import ( "encoding/base64" "errors" "fmt" "io" "net/http" "strings" "time" "golang.org/x/crypto/ocsp" ) func FetchOCSPResponse(link *ChainLink, opts *OCSPPeerConfig, log *Log) ([]byte, error) { if link == nil || link.Leaf == nil || link.Issuer == nil || opts == nil || log == nil { return nil, errors.New(ErrInvalidChainlink) } timeout := time.Duration(opts.Timeout * float64(time.Second)) if timeout <= 0*time.Second { timeout = DefaultOCSPResponderTimeout } getRequestBytes := func(u string, hc *http.Client) ([]byte, error) { resp, err := hc.Get(u) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf(ErrBadResponderHTTPStatus, resp.StatusCode) } return io.ReadAll(resp.Body) } // Request documentation: // https://tools.ietf.org/html/rfc6960#appendix-A.1 reqDER, err := ocsp.CreateRequest(link.Leaf, link.Issuer, nil) if err != nil { return nil, err } reqEnc := base64.StdEncoding.EncodeToString(reqDER) responders := *link.OCSPWebEndpoints if len(responders) == 0 { return nil, errors.New(ErrNoAvailOCSPServers) } var raw []byte hc := &http.Client{ Timeout: timeout, } for _, u := range responders { url := u.String() log.Debugf(DbgMakingCARequest, url) url = strings.TrimSuffix(url, "/") raw, err = getRequestBytes(fmt.Sprintf("%s/%s", url, reqEnc), hc) if err == nil { break } } if err != nil { return nil, fmt.Errorf(ErrFailedWithAllRequests, err) } return raw, nil } nats-server-2.10.27/server/certstore/000077500000000000000000000000001477524627100174735ustar00rootroot00000000000000nats-server-2.10.27/server/certstore/certstore.go000066400000000000000000000051051477524627100220350ustar00rootroot00000000000000// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package certstore import ( "crypto" "crypto/x509" "io" "runtime" "strings" ) type StoreType int const MATCHBYEMPTY = 0 const STOREEMPTY = 0 const ( windowsCurrentUser StoreType = iota + 1 windowsLocalMachine ) var StoreMap = map[string]StoreType{ "windowscurrentuser": windowsCurrentUser, "windowslocalmachine": windowsLocalMachine, } var StoreOSMap = map[StoreType]string{ windowsCurrentUser: "windows", windowsLocalMachine: "windows", } type MatchByType int const ( matchByIssuer MatchByType = iota + 1 matchBySubject matchByThumbprint ) var MatchByMap = map[string]MatchByType{ "issuer": matchByIssuer, "subject": matchBySubject, "thumbprint": matchByThumbprint, } var Usage = ` In place of cert_file and key_file you may use the windows certificate store: tls { cert_store: "WindowsCurrentUser" cert_match_by: "Subject" cert_match: "MyServer123" } ` func ParseCertStore(certStore string) (StoreType, error) { certStoreType, exists := StoreMap[strings.ToLower(certStore)] if !exists { return 0, ErrBadCertStore } validOS, exists := StoreOSMap[certStoreType] if !exists || validOS != runtime.GOOS { return 0, ErrOSNotCompatCertStore } return certStoreType, nil } func ParseCertMatchBy(certMatchBy string) (MatchByType, error) { certMatchByType, exists := MatchByMap[strings.ToLower(certMatchBy)] if !exists { return 0, ErrBadMatchByType } return certMatchByType, nil } func GetLeafIssuer(leaf *x509.Certificate, vOpts x509.VerifyOptions) (issuer *x509.Certificate) { chains, err := leaf.Verify(vOpts) if err != nil || len(chains) == 0 { issuer = nil } else { issuer = chains[0][1] } return } // credential provides access to a public key and is a crypto.Signer. type credential interface { // Public returns the public key corresponding to the leaf certificate. Public() crypto.PublicKey // Sign signs digest with the private key. Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) } nats-server-2.10.27/server/certstore/certstore_other.go000066400000000000000000000026251477524627100232420ustar00rootroot00000000000000// Copyright 2022-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows package certstore import ( "crypto" "crypto/tls" "io" ) var _ = MATCHBYEMPTY // otherKey implements crypto.Signer and crypto.Decrypter to satisfy linter on platforms that don't implement certstore type otherKey struct{} func TLSConfig(_ StoreType, _ MatchByType, _ string, _ []string, _ bool, _ *tls.Config) error { return ErrOSNotCompatCertStore } // Public always returns nil public key since this is a stub on non-supported platform func (k otherKey) Public() crypto.PublicKey { return nil } // Sign always returns a nil signature since this is a stub on non-supported platform func (k otherKey) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) { _, _, _ = rand, digest, opts return nil, nil } // Verify interface conformance. var _ credential = &otherKey{} nats-server-2.10.27/server/certstore/certstore_windows.go000066400000000000000000000736111477524627100236160ustar00rootroot00000000000000// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // Adapted, updated, and enhanced from CertToStore, https://github.com/google/certtostore/releases/tag/v1.0.2 // Apache License, Version 2.0, Copyright 2017 Google Inc. package certstore import ( "bytes" "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rsa" "crypto/tls" "crypto/x509" "encoding/binary" "fmt" "io" "math/big" "reflect" "sync" "syscall" "unicode/utf16" "unsafe" "golang.org/x/crypto/cryptobyte" "golang.org/x/crypto/cryptobyte/asn1" "golang.org/x/sys/windows" ) const ( // wincrypt.h constants winAcquireCached = windows.CRYPT_ACQUIRE_CACHE_FLAG winAcquireSilent = windows.CRYPT_ACQUIRE_SILENT_FLAG winAcquireOnlyNCryptKey = windows.CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG winEncodingX509ASN = windows.X509_ASN_ENCODING winEncodingPKCS7 = windows.PKCS_7_ASN_ENCODING winCertStoreProvSystem = windows.CERT_STORE_PROV_SYSTEM winCertStoreCurrentUser = windows.CERT_SYSTEM_STORE_CURRENT_USER winCertStoreLocalMachine = windows.CERT_SYSTEM_STORE_LOCAL_MACHINE winCertStoreReadOnly = windows.CERT_STORE_READONLY_FLAG winInfoIssuerFlag = windows.CERT_INFO_ISSUER_FLAG winInfoSubjectFlag = windows.CERT_INFO_SUBJECT_FLAG winCompareNameStrW = windows.CERT_COMPARE_NAME_STR_W winCompareShift = windows.CERT_COMPARE_SHIFT // Reference https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore winFindIssuerStr = windows.CERT_FIND_ISSUER_STR_W winFindSubjectStr = windows.CERT_FIND_SUBJECT_STR_W winFindHashStr = windows.CERT_FIND_HASH_STR winNcryptKeySpec = windows.CERT_NCRYPT_KEY_SPEC winBCryptPadPKCS1 uintptr = 0x2 winBCryptPadPSS uintptr = 0x8 // Modern TLS 1.2+ winBCryptPadPSSSalt uint32 = 32 // default 20, 32 optimal for typical SHA256 hash winRSA1Magic = 0x31415352 // "RSA1" BCRYPT_RSAPUBLIC_MAGIC winECS1Magic = 0x31534345 // "ECS1" BCRYPT_ECDSA_PUBLIC_P256_MAGIC winECS3Magic = 0x33534345 // "ECS3" BCRYPT_ECDSA_PUBLIC_P384_MAGIC winECS5Magic = 0x35534345 // "ECS5" BCRYPT_ECDSA_PUBLIC_P521_MAGIC winECK1Magic = 0x314B4345 // "ECK1" BCRYPT_ECDH_PUBLIC_P256_MAGIC winECK3Magic = 0x334B4345 // "ECK3" BCRYPT_ECDH_PUBLIC_P384_MAGIC winECK5Magic = 0x354B4345 // "ECK5" BCRYPT_ECDH_PUBLIC_P521_MAGIC winCryptENotFound = windows.CRYPT_E_NOT_FOUND providerMSSoftware = "Microsoft Software Key Storage Provider" ) var ( winBCryptRSAPublicBlob = winWide("RSAPUBLICBLOB") winBCryptECCPublicBlob = winWide("ECCPUBLICBLOB") winNCryptAlgorithmGroupProperty = winWide("Algorithm Group") // NCRYPT_ALGORITHM_GROUP_PROPERTY winNCryptUniqueNameProperty = winWide("Unique Name") // NCRYPT_UNIQUE_NAME_PROPERTY winNCryptECCCurveNameProperty = winWide("ECCCurveName") // NCRYPT_ECC_CURVE_NAME_PROPERTY winCurveIDs = map[uint32]elliptic.Curve{ winECS1Magic: elliptic.P256(), // BCRYPT_ECDSA_PUBLIC_P256_MAGIC winECS3Magic: elliptic.P384(), // BCRYPT_ECDSA_PUBLIC_P384_MAGIC winECS5Magic: elliptic.P521(), // BCRYPT_ECDSA_PUBLIC_P521_MAGIC winECK1Magic: elliptic.P256(), // BCRYPT_ECDH_PUBLIC_P256_MAGIC winECK3Magic: elliptic.P384(), // BCRYPT_ECDH_PUBLIC_P384_MAGIC winECK5Magic: elliptic.P521(), // BCRYPT_ECDH_PUBLIC_P521_MAGIC } winCurveNames = map[string]elliptic.Curve{ "nistP256": elliptic.P256(), // BCRYPT_ECC_CURVE_NISTP256 "nistP384": elliptic.P384(), // BCRYPT_ECC_CURVE_NISTP384 "nistP521": elliptic.P521(), // BCRYPT_ECC_CURVE_NISTP521 } winAlgIDs = map[crypto.Hash]*uint16{ crypto.SHA1: winWide("SHA1"), // BCRYPT_SHA1_ALGORITHM crypto.SHA256: winWide("SHA256"), // BCRYPT_SHA256_ALGORITHM crypto.SHA384: winWide("SHA384"), // BCRYPT_SHA384_ALGORITHM crypto.SHA512: winWide("SHA512"), // BCRYPT_SHA512_ALGORITHM } // MY is well-known system store on Windows that holds personal certificates. Read // More about the CA locations here: // https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/wcf/certificate-of-clientcertificate-element?redirectedfrom=MSDN // https://superuser.com/questions/217719/what-are-the-windows-system-certificate-stores // https://docs.microsoft.com/en-us/windows/win32/seccrypto/certificate-stores // https://learn.microsoft.com/en-us/windows/win32/seccrypto/system-store-locations // https://stackoverflow.com/questions/63286085/which-x509-storename-refers-to-the-certificates-stored-beneath-trusted-root-cert#:~:text=4-,StoreName.,is%20%22Intermediate%20Certification%20Authorities%22. winMyStore = winWide("MY") winIntermediateCAStore = winWide("CA") winRootStore = winWide("Root") winAuthRootStore = winWide("AuthRoot") // These DLLs must be available on all Windows hosts winCrypt32 = windows.NewLazySystemDLL("crypt32.dll") winNCrypt = windows.NewLazySystemDLL("ncrypt.dll") winCertFindCertificateInStore = winCrypt32.NewProc("CertFindCertificateInStore") winCertVerifyTimeValidity = winCrypt32.NewProc("CertVerifyTimeValidity") winCryptAcquireCertificatePrivateKey = winCrypt32.NewProc("CryptAcquireCertificatePrivateKey") winNCryptExportKey = winNCrypt.NewProc("NCryptExportKey") winNCryptOpenStorageProvider = winNCrypt.NewProc("NCryptOpenStorageProvider") winNCryptGetProperty = winNCrypt.NewProc("NCryptGetProperty") winNCryptSignHash = winNCrypt.NewProc("NCryptSignHash") winFnGetProperty = winGetProperty ) func init() { for _, d := range []*windows.LazyDLL{ winCrypt32, winNCrypt, } { if err := d.Load(); err != nil { panic(err) } } for _, p := range []*windows.LazyProc{ winCertFindCertificateInStore, winCryptAcquireCertificatePrivateKey, winNCryptExportKey, winNCryptOpenStorageProvider, winNCryptGetProperty, winNCryptSignHash, } { if err := p.Find(); err != nil { panic(err) } } } type winPKCS1PaddingInfo struct { pszAlgID *uint16 } type winPSSPaddingInfo struct { pszAlgID *uint16 cbSalt uint32 } // createCACertsPool generates a CertPool from the Windows certificate store, // adding all matching certificates from the caCertsMatch array to the pool. // All matching certificates (vs first) are added to the pool based on a user // request. If no certificates are found an error is returned. func createCACertsPool(cs *winCertStore, storeType uint32, caCertsMatch []string, skipInvalid bool) (*x509.CertPool, error) { var errs []error caPool := x509.NewCertPool() for _, s := range caCertsMatch { lfs, err := cs.caCertsBySubjectMatch(s, storeType, skipInvalid) if err != nil { errs = append(errs, err) } else { for _, lf := range lfs { caPool.AddCert(lf) } } } // If every lookup failed return the errors. if len(errs) == len(caCertsMatch) { return nil, fmt.Errorf("unable to match any CA certificate: %v", errs) } return caPool, nil } // TLSConfig fulfills the same function as reading cert and key pair from // pem files but sources the Windows certificate store instead. The // certMatchBy and certMatch fields search the "MY" certificate location // for the first certificate that matches the certMatch field. The // caCertsMatch field is used to search the Trusted Root, Third Party Root, // and Intermediate Certificate Authority locations for certificates with // Subjects matching the provided strings. If a match is found, the // certificate is added to the pool that is used to verify the certificate // chain. func TLSConfig(certStore StoreType, certMatchBy MatchByType, certMatch string, caCertsMatch []string, skipInvalid bool, config *tls.Config) error { var ( leaf *x509.Certificate leafCtx *windows.CertContext pk *winKey vOpts = x509.VerifyOptions{} chains [][]*x509.Certificate chain []*x509.Certificate rawChain [][]byte ) // By StoreType, open a store if certStore == windowsCurrentUser || certStore == windowsLocalMachine { var scope uint32 cs, err := winOpenCertStore(providerMSSoftware) if err != nil || cs == nil { return err } if certStore == windowsCurrentUser { scope = winCertStoreCurrentUser } if certStore == windowsLocalMachine { scope = winCertStoreLocalMachine } // certByIssuer or certBySubject if certMatchBy == matchBySubject || certMatchBy == MATCHBYEMPTY { leaf, leafCtx, err = cs.certBySubject(certMatch, scope, skipInvalid) } else if certMatchBy == matchByIssuer { leaf, leafCtx, err = cs.certByIssuer(certMatch, scope, skipInvalid) } else if certMatchBy == matchByThumbprint { leaf, leafCtx, err = cs.certByThumbprint(certMatch, scope, skipInvalid) } else { return ErrBadMatchByType } if err != nil { // pass through error from cert search return err } if leaf == nil || leafCtx == nil { return ErrFailedCertSearch } pk, err = cs.certKey(leafCtx) if err != nil { return err } if pk == nil { return ErrNoPrivateKeyStoreRef } // Look for CA Certificates if len(caCertsMatch) != 0 { caPool, err := createCACertsPool(cs, scope, caCertsMatch, skipInvalid) if err != nil { return err } config.ClientCAs = caPool } } else { return ErrBadCertStore } // Get intermediates in the cert store for the found leaf IFF there is a full chain of trust in the store // otherwise just use leaf as the final chain. // // Using std lib Verify as a reliable way to get valid chains out of the win store for the leaf; however, // using empty options since server TLS stanza could be TLS role as server identity or client identity. chains, err := leaf.Verify(vOpts) if err != nil || len(chains) == 0 { chains = append(chains, []*x509.Certificate{leaf}) } // We have at least one verified chain so pop the first chain and remove the self-signed CA cert (if present) // from the end of the chain chain = chains[0] if len(chain) > 1 { chain = chain[:len(chain)-1] } // For tls.Certificate.Certificate need a [][]byte from []*x509.Certificate // Approximate capacity for efficiency rawChain = make([][]byte, 0, len(chain)) for _, link := range chain { rawChain = append(rawChain, link.Raw) } tlsCert := tls.Certificate{ Certificate: rawChain, PrivateKey: pk, Leaf: leaf, } config.Certificates = []tls.Certificate{tlsCert} // note: pk is a windows pointer (not freed by Go) but needs to live the life of the server for Signing. // The cert context (leafCtx) windows pointer must not be freed underneath the pk so also life of the server. return nil } // winWide returns a pointer to uint16 representing the equivalent // to a Windows LPCWSTR. func winWide(s string) *uint16 { w := utf16.Encode([]rune(s)) w = append(w, 0) return &w[0] } // winOpenProvider gets a provider handle for subsequent calls func winOpenProvider(provider string) (uintptr, error) { var hProv uintptr pname := winWide(provider) // Open the provider, the last parameter is not used r, _, err := winNCryptOpenStorageProvider.Call(uintptr(unsafe.Pointer(&hProv)), uintptr(unsafe.Pointer(pname)), 0) if r == 0 { return hProv, nil } return hProv, fmt.Errorf("NCryptOpenStorageProvider returned %X: %v", r, err) } // winFindCert wraps the CertFindCertificateInStore library call. Note that any cert context passed // into prev will be freed. If no certificate was found, nil will be returned. func winFindCert(store windows.Handle, enc, findFlags, findType uint32, para *uint16, prev *windows.CertContext) (*windows.CertContext, error) { h, _, err := winCertFindCertificateInStore.Call( uintptr(store), uintptr(enc), uintptr(findFlags), uintptr(findType), uintptr(unsafe.Pointer(para)), uintptr(unsafe.Pointer(prev)), ) if h == 0 { // Actual error, or simply not found? if errno, ok := err.(syscall.Errno); ok && errno == syscall.Errno(winCryptENotFound) { return nil, ErrFailedCertSearch } return nil, ErrFailedCertSearch } // nolint:govet return (*windows.CertContext)(unsafe.Pointer(h)), nil } // winVerifyCertValid wraps the CertVerifyTimeValidity and simply returns true if the certificate is valid func winVerifyCertValid(timeToVerify *windows.Filetime, certInfo *windows.CertInfo) bool { // this function does not document returning errors / setting lasterror r, _, _ := winCertVerifyTimeValidity.Call( uintptr(unsafe.Pointer(timeToVerify)), uintptr(unsafe.Pointer(certInfo)), ) return r == 0 } // winCertStore is a store implementation for the Windows Certificate Store type winCertStore struct { Prov uintptr ProvName string stores map[string]*winStoreHandle mu sync.Mutex } // winOpenCertStore creates a winCertStore func winOpenCertStore(provider string) (*winCertStore, error) { cngProv, err := winOpenProvider(provider) if err != nil { // pass through error from winOpenProvider return nil, err } wcs := &winCertStore{ Prov: cngProv, ProvName: provider, stores: make(map[string]*winStoreHandle), } return wcs, nil } // winCertContextToX509 creates an x509.Certificate from a Windows cert context. func winCertContextToX509(ctx *windows.CertContext) (*x509.Certificate, error) { var der []byte slice := (*reflect.SliceHeader)(unsafe.Pointer(&der)) slice.Data = uintptr(unsafe.Pointer(ctx.EncodedCert)) slice.Len = int(ctx.Length) slice.Cap = int(ctx.Length) return x509.ParseCertificate(der) } // certByIssuer matches and returns the first certificate found by passed issuer. // CertContext pointer returned allows subsequent key operations like Sign. Caller specifies // current user's personal certs or local machine's personal certs using storeType. // See CERT_FIND_ISSUER_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore func (w *winCertStore) certByIssuer(issuer string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { return w.certSearch(winFindIssuerStr, issuer, winMyStore, storeType, skipInvalid) } // certBySubject matches and returns the first certificate found by passed subject field. // CertContext pointer returned allows subsequent key operations like Sign. Caller specifies // current user's personal certs or local machine's personal certs using storeType. // See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore func (w *winCertStore) certBySubject(subject string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { return w.certSearch(winFindSubjectStr, subject, winMyStore, storeType, skipInvalid) } // certByThumbprint matches and returns the first certificate found by passed SHA1 thumbprint. // CertContext pointer returned allows subsequent key operations like Sign. Caller specifies // current user's personal certs or local machine's personal certs using storeType. // See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore func (w *winCertStore) certByThumbprint(hash string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { return w.certSearch(winFindHashStr, hash, winMyStore, storeType, skipInvalid) } // caCertsBySubjectMatch matches and returns all matching certificates of the subject field. // // The following locations are searched: // 1) Root (Trusted Root Certification Authorities) // 2) AuthRoot (Third-Party Root Certification Authorities) // 3) CA (Intermediate Certification Authorities) // // Caller specifies current user's personal certs or local machine's personal certs using storeType. // See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore func (w *winCertStore) caCertsBySubjectMatch(subject string, storeType uint32, skipInvalid bool) ([]*x509.Certificate, error) { var ( leaf *x509.Certificate searchLocations = [3]*uint16{winRootStore, winAuthRootStore, winIntermediateCAStore} rv []*x509.Certificate ) // surprisingly, an empty string returns a result. We'll treat this as an error. if subject == "" { return nil, ErrBadCaCertMatchField } for _, sr := range searchLocations { var err error if leaf, _, err = w.certSearch(winFindSubjectStr, subject, sr, storeType, skipInvalid); err == nil { rv = append(rv, leaf) } else { // Ignore the failed search from a single location. Errors we catch include // ErrFailedX509Extract (resulting from a malformed certificate) and errors // around invalid attributes, unsupported algorithms, etc. These are corner // cases as certificates with these errors shouldn't have been allowed // to be added to the store in the first place. if err != ErrFailedCertSearch { return nil, err } } } // Not found anywhere if len(rv) == 0 { return nil, ErrFailedCertSearch } return rv, nil } // certSearch is a helper function to lookup certificates based on search type and match value. // store is used to specify which store to perform the lookup in (system or user). func (w *winCertStore) certSearch(searchType uint32, matchValue string, searchRoot *uint16, store uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) { // store handle to "MY" store h, err := w.storeHandle(store, searchRoot) if err != nil { return nil, nil, err } var prev *windows.CertContext var cert *x509.Certificate i, err := windows.UTF16PtrFromString(matchValue) if err != nil { return nil, nil, ErrFailedCertSearch } // pass 0 as the third parameter because it is not used // https://msdn.microsoft.com/en-us/library/windows/desktop/aa376064(v=vs.85).aspx for { nc, err := winFindCert(h, winEncodingX509ASN|winEncodingPKCS7, 0, searchType, i, prev) if err != nil { return nil, nil, err } if nc != nil { // certificate found prev = nc var now *windows.Filetime if skipInvalid && !winVerifyCertValid(now, nc.CertInfo) { continue } // Extract the DER-encoded certificate from the cert context xc, err := winCertContextToX509(nc) if err == nil { cert = xc break } else { return nil, nil, ErrFailedX509Extract } } else { return nil, nil, ErrFailedCertSearch } } if cert == nil { return nil, nil, ErrFailedX509Extract } return cert, prev, nil } type winStoreHandle struct { handle *windows.Handle } func winNewStoreHandle(provider uint32, store *uint16) (*winStoreHandle, error) { var s winStoreHandle if s.handle != nil { return &s, nil } st, err := windows.CertOpenStore( winCertStoreProvSystem, 0, 0, provider|winCertStoreReadOnly, uintptr(unsafe.Pointer(store))) if err != nil { return nil, ErrBadCryptoStoreProvider } s.handle = &st return &s, nil } // winKey implements crypto.Signer and crypto.Decrypter for key based operations. type winKey struct { handle uintptr pub crypto.PublicKey Container string AlgorithmGroup string } // Public exports a public key to implement crypto.Signer func (k winKey) Public() crypto.PublicKey { return k.pub } // Sign returns the signature of a hash to implement crypto.Signer func (k winKey) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { switch k.AlgorithmGroup { case "ECDSA", "ECDH": return winSignECDSA(k.handle, digest) case "RSA": hf := opts.HashFunc() algID, ok := winAlgIDs[hf] if !ok { return nil, ErrBadRSAHashAlgorithm } switch opts.(type) { case *rsa.PSSOptions: return winSignRSAPSSPadding(k.handle, digest, algID) default: return winSignRSAPKCS1Padding(k.handle, digest, algID) } default: return nil, ErrBadSigningAlgorithm } } func winSignECDSA(kh uintptr, digest []byte) ([]byte, error) { var size uint32 // Obtain the size of the signature r, _, _ := winNCryptSignHash.Call( kh, 0, uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), 0, 0, uintptr(unsafe.Pointer(&size)), 0) if r != 0 { return nil, ErrStoreECDSASigningError } // Obtain the signature data buf := make([]byte, size) r, _, _ = winNCryptSignHash.Call( kh, 0, uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), uintptr(unsafe.Pointer(&buf[0])), uintptr(size), uintptr(unsafe.Pointer(&size)), 0) if r != 0 { return nil, ErrStoreECDSASigningError } if len(buf) != int(size) { return nil, ErrStoreECDSASigningError } return winPackECDSASigValue(bytes.NewReader(buf[:size]), len(digest)) } func winPackECDSASigValue(r io.Reader, digestLength int) ([]byte, error) { sigR := make([]byte, digestLength) if _, err := io.ReadFull(r, sigR); err != nil { return nil, ErrStoreECDSASigningError } sigS := make([]byte, digestLength) if _, err := io.ReadFull(r, sigS); err != nil { return nil, ErrStoreECDSASigningError } var b cryptobyte.Builder b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) { b.AddASN1BigInt(new(big.Int).SetBytes(sigR)) b.AddASN1BigInt(new(big.Int).SetBytes(sigS)) }) return b.Bytes() } func winSignRSAPKCS1Padding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) { // PKCS#1 v1.5 padding for some TLS 1.2 padInfo := winPKCS1PaddingInfo{pszAlgID: algID} var size uint32 // Obtain the size of the signature r, _, _ := winNCryptSignHash.Call( kh, uintptr(unsafe.Pointer(&padInfo)), uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), 0, 0, uintptr(unsafe.Pointer(&size)), winBCryptPadPKCS1) if r != 0 { return nil, ErrStoreRSASigningError } // Obtain the signature data sig := make([]byte, size) r, _, _ = winNCryptSignHash.Call( kh, uintptr(unsafe.Pointer(&padInfo)), uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), uintptr(unsafe.Pointer(&sig[0])), uintptr(size), uintptr(unsafe.Pointer(&size)), winBCryptPadPKCS1) if r != 0 { return nil, ErrStoreRSASigningError } return sig[:size], nil } func winSignRSAPSSPadding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) { // PSS padding for TLS 1.3 and some TLS 1.2 padInfo := winPSSPaddingInfo{pszAlgID: algID, cbSalt: winBCryptPadPSSSalt} var size uint32 // Obtain the size of the signature r, _, _ := winNCryptSignHash.Call( kh, uintptr(unsafe.Pointer(&padInfo)), uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), 0, 0, uintptr(unsafe.Pointer(&size)), winBCryptPadPSS) if r != 0 { return nil, ErrStoreRSASigningError } // Obtain the signature data sig := make([]byte, size) r, _, _ = winNCryptSignHash.Call( kh, uintptr(unsafe.Pointer(&padInfo)), uintptr(unsafe.Pointer(&digest[0])), uintptr(len(digest)), uintptr(unsafe.Pointer(&sig[0])), uintptr(size), uintptr(unsafe.Pointer(&size)), winBCryptPadPSS) if r != 0 { return nil, ErrStoreRSASigningError } return sig[:size], nil } // certKey wraps CryptAcquireCertificatePrivateKey. It obtains the CNG private // key of a known certificate and returns a pointer to a winKey which implements // both crypto.Signer. When a nil cert context is passed // a nil key is intentionally returned, to model the expected behavior of a // non-existent cert having no private key. // https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-cryptacquirecertificateprivatekey func (w *winCertStore) certKey(cert *windows.CertContext) (*winKey, error) { // Return early if a nil cert was passed. if cert == nil { return nil, nil } var ( kh uintptr spec uint32 mustFree int ) r, _, _ := winCryptAcquireCertificatePrivateKey.Call( uintptr(unsafe.Pointer(cert)), winAcquireCached|winAcquireSilent|winAcquireOnlyNCryptKey, 0, // Reserved, must be null. uintptr(unsafe.Pointer(&kh)), uintptr(unsafe.Pointer(&spec)), uintptr(unsafe.Pointer(&mustFree)), ) // If the function succeeds, the return value is nonzero (TRUE). if r == 0 { return nil, ErrNoPrivateKeyStoreRef } if mustFree != 0 { return nil, ErrNoPrivateKeyStoreRef } if spec != winNcryptKeySpec { return nil, ErrNoPrivateKeyStoreRef } return winKeyMetadata(kh) } func winKeyMetadata(kh uintptr) (*winKey, error) { // uc is used to populate the unique container name attribute of the private key uc, err := winGetPropertyStr(kh, winNCryptUniqueNameProperty) if err != nil { // unable to determine key unique name return nil, ErrExtractingPrivateKeyMetadata } alg, err := winGetPropertyStr(kh, winNCryptAlgorithmGroupProperty) if err != nil { // unable to determine key algorithm return nil, ErrExtractingPrivateKeyMetadata } var pub crypto.PublicKey switch alg { case "ECDSA", "ECDH": buf, err := winExport(kh, winBCryptECCPublicBlob) if err != nil { // failed to export ECC public key return nil, ErrExtractingECCPublicKey } pub, err = unmarshalECC(buf, kh) if err != nil { return nil, ErrExtractingECCPublicKey } case "RSA": buf, err := winExport(kh, winBCryptRSAPublicBlob) if err != nil { return nil, ErrExtractingRSAPublicKey } pub, err = winUnmarshalRSA(buf) if err != nil { return nil, ErrExtractingRSAPublicKey } default: return nil, ErrBadPublicKeyAlgorithm } return &winKey{handle: kh, pub: pub, Container: uc, AlgorithmGroup: alg}, nil } func winGetProperty(kh uintptr, property *uint16) ([]byte, error) { var strSize uint32 r, _, _ := winNCryptGetProperty.Call( kh, uintptr(unsafe.Pointer(property)), 0, 0, uintptr(unsafe.Pointer(&strSize)), 0, 0) if r != 0 { return nil, ErrExtractPropertyFromKey } buf := make([]byte, strSize) r, _, _ = winNCryptGetProperty.Call( kh, uintptr(unsafe.Pointer(property)), uintptr(unsafe.Pointer(&buf[0])), uintptr(strSize), uintptr(unsafe.Pointer(&strSize)), 0, 0) if r != 0 { return nil, ErrExtractPropertyFromKey } return buf, nil } func winGetPropertyStr(kh uintptr, property *uint16) (string, error) { buf, err := winFnGetProperty(kh, property) if err != nil { return "", ErrExtractPropertyFromKey } uc := bytes.ReplaceAll(buf, []byte{0x00}, []byte("")) return string(uc), nil } func winExport(kh uintptr, blobType *uint16) ([]byte, error) { var size uint32 // When obtaining the size of a public key, most parameters are not required r, _, _ := winNCryptExportKey.Call( kh, 0, uintptr(unsafe.Pointer(blobType)), 0, 0, 0, uintptr(unsafe.Pointer(&size)), 0) if r != 0 { return nil, ErrExtractingPublicKey } // Place the exported key in buf now that we know the size required buf := make([]byte, size) r, _, _ = winNCryptExportKey.Call( kh, 0, uintptr(unsafe.Pointer(blobType)), 0, uintptr(unsafe.Pointer(&buf[0])), uintptr(size), uintptr(unsafe.Pointer(&size)), 0) if r != 0 { return nil, ErrExtractingPublicKey } return buf, nil } func unmarshalECC(buf []byte, kh uintptr) (*ecdsa.PublicKey, error) { // BCRYPT_ECCKEY_BLOB from bcrypt.h header := struct { Magic uint32 Key uint32 }{} r := bytes.NewReader(buf) if err := binary.Read(r, binary.LittleEndian, &header); err != nil { return nil, ErrExtractingECCPublicKey } curve, ok := winCurveIDs[header.Magic] if !ok { // Fix for b/185945636, where despite specifying the curve, nCrypt returns // an incorrect response with BCRYPT_ECDSA_PUBLIC_GENERIC_MAGIC. var err error curve, err = winCurveName(kh) if err != nil { // unsupported header magic or cannot match the curve by name return nil, err } } keyX := make([]byte, header.Key) if n, err := r.Read(keyX); n != int(header.Key) || err != nil { // failed to read key X return nil, ErrExtractingECCPublicKey } keyY := make([]byte, header.Key) if n, err := r.Read(keyY); n != int(header.Key) || err != nil { // failed to read key Y return nil, ErrExtractingECCPublicKey } pub := &ecdsa.PublicKey{ Curve: curve, X: new(big.Int).SetBytes(keyX), Y: new(big.Int).SetBytes(keyY), } return pub, nil } // winCurveName reads the curve name property and returns the corresponding curve. func winCurveName(kh uintptr) (elliptic.Curve, error) { cn, err := winGetPropertyStr(kh, winNCryptECCCurveNameProperty) if err != nil { // unable to determine the curve property name return nil, ErrExtractPropertyFromKey } curve, ok := winCurveNames[cn] if !ok { // unknown curve name return nil, ErrBadECCCurveName } return curve, nil } func winUnmarshalRSA(buf []byte) (*rsa.PublicKey, error) { // BCRYPT_RSA_BLOB from bcrypt.h header := struct { Magic uint32 BitLength uint32 PublicExpSize uint32 ModulusSize uint32 UnusedPrime1 uint32 UnusedPrime2 uint32 }{} r := bytes.NewReader(buf) if err := binary.Read(r, binary.LittleEndian, &header); err != nil { return nil, ErrExtractingRSAPublicKey } if header.Magic != winRSA1Magic { // invalid header magic return nil, ErrExtractingRSAPublicKey } if header.PublicExpSize > 8 { // unsupported public exponent size return nil, ErrExtractingRSAPublicKey } exp := make([]byte, 8) if n, err := r.Read(exp[8-header.PublicExpSize:]); n != int(header.PublicExpSize) || err != nil { // failed to read public exponent return nil, ErrExtractingRSAPublicKey } mod := make([]byte, header.ModulusSize) if n, err := r.Read(mod); n != int(header.ModulusSize) || err != nil { // failed to read modulus return nil, ErrExtractingRSAPublicKey } pub := &rsa.PublicKey{ N: new(big.Int).SetBytes(mod), E: int(binary.BigEndian.Uint64(exp)), } return pub, nil } // storeHandle returns a handle to a given cert store, opening the handle as needed. func (w *winCertStore) storeHandle(provider uint32, store *uint16) (windows.Handle, error) { w.mu.Lock() defer w.mu.Unlock() key := fmt.Sprintf("%d%s", provider, windows.UTF16PtrToString(store)) var err error if w.stores[key] == nil { w.stores[key], err = winNewStoreHandle(provider, store) if err != nil { return 0, ErrBadCryptoStoreProvider } } return *w.stores[key].handle, nil } // Verify interface conformance. var _ credential = &winKey{} nats-server-2.10.27/server/certstore/errors.go000066400000000000000000000100101477524627100213260ustar00rootroot00000000000000package certstore import ( "errors" ) var ( // ErrBadCryptoStoreProvider represents inablity to establish link with a certificate store ErrBadCryptoStoreProvider = errors.New("unable to open certificate store or store not available") // ErrBadRSAHashAlgorithm represents a bad or unsupported RSA hash algorithm ErrBadRSAHashAlgorithm = errors.New("unsupported RSA hash algorithm") // ErrBadSigningAlgorithm represents a bad or unsupported signing algorithm ErrBadSigningAlgorithm = errors.New("unsupported signing algorithm") // ErrStoreRSASigningError represents an error returned from store during RSA signature ErrStoreRSASigningError = errors.New("unable to obtain RSA signature from store") // ErrStoreECDSASigningError represents an error returned from store during ECDSA signature ErrStoreECDSASigningError = errors.New("unable to obtain ECDSA signature from store") // ErrNoPrivateKeyStoreRef represents an error getting a handle to a private key in store ErrNoPrivateKeyStoreRef = errors.New("unable to obtain private key handle from store") // ErrExtractingPrivateKeyMetadata represents a family of errors extracting metadata about the private key in store ErrExtractingPrivateKeyMetadata = errors.New("unable to extract private key metadata") // ErrExtractingECCPublicKey represents an error exporting ECC-type public key from store ErrExtractingECCPublicKey = errors.New("unable to extract ECC public key from store") // ErrExtractingRSAPublicKey represents an error exporting RSA-type public key from store ErrExtractingRSAPublicKey = errors.New("unable to extract RSA public key from store") // ErrExtractingPublicKey represents a general error exporting public key from store ErrExtractingPublicKey = errors.New("unable to extract public key from store") // ErrBadPublicKeyAlgorithm represents a bad or unsupported public key algorithm ErrBadPublicKeyAlgorithm = errors.New("unsupported public key algorithm") // ErrExtractPropertyFromKey represents a general failure to extract a metadata property field ErrExtractPropertyFromKey = errors.New("unable to extract property from key") // ErrBadECCCurveName represents an ECC signature curve name that is bad or unsupported ErrBadECCCurveName = errors.New("unsupported ECC curve name") // ErrFailedCertSearch represents not able to find certificate in store ErrFailedCertSearch = errors.New("unable to find certificate in store") // ErrFailedX509Extract represents not being able to extract x509 certificate from found cert in store ErrFailedX509Extract = errors.New("unable to extract x509 from certificate") // ErrBadMatchByType represents unknown CERT_MATCH_BY passed ErrBadMatchByType = errors.New("cert match by type not implemented") // ErrBadCertStore represents unknown CERT_STORE passed ErrBadCertStore = errors.New("cert store type not implemented") // ErrConflictCertFileAndStore represents ambiguous configuration of both file and store ErrConflictCertFileAndStore = errors.New("'cert_file' and 'cert_store' may not both be configured") // ErrBadCertStoreField represents malformed cert_store option ErrBadCertStoreField = errors.New("expected 'cert_store' to be a valid non-empty string") // ErrBadCertMatchByField represents malformed cert_match_by option ErrBadCertMatchByField = errors.New("expected 'cert_match_by' to be a valid non-empty string") // ErrBadCertMatchField represents malformed cert_match option ErrBadCertMatchField = errors.New("expected 'cert_match' to be a valid non-empty string") // ErrBadCaCertMatchField represents malformed cert_match option ErrBadCaCertMatchField = errors.New("expected 'ca_certs_match' to be a valid non-empty string array") // ErrBadCertMatchSkipInvalidField represents malformed cert_match_skip_invalid option ErrBadCertMatchSkipInvalidField = errors.New("expected 'cert_match_skip_invalid' to be a boolean") // ErrOSNotCompatCertStore represents cert_store passed that exists but is not valid on current OS ErrOSNotCompatCertStore = errors.New("cert_store not compatible with current operating system") ) nats-server-2.10.27/server/certstore_windows_test.go000066400000000000000000000237401477524627100226410ustar00rootroot00000000000000// Copyright 2022-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package server import ( "fmt" "net/url" "os" "os/exec" "strings" "testing" "time" "github.com/nats-io/nats.go" ) func runPowershellScript(scriptFile string, args []string) error { psExec, _ := exec.LookPath("powershell.exe") execArgs := []string{psExec, "-command", fmt.Sprintf("& '%s'", scriptFile)} if len(args) > 0 { execArgs = append(execArgs, args...) } cmdImport := &exec.Cmd{ Path: psExec, Args: execArgs, Stdout: os.Stdout, Stderr: os.Stderr, } return cmdImport.Run() } func runConfiguredLeaf(t *testing.T, hubPort int, certStore string, matchBy string, match string, caMatch string, expectedLeafCount int) { // Fire up the leaf u, err := url.Parse(fmt.Sprintf("nats://localhost:%d", hubPort)) if err != nil { t.Fatalf("Error parsing url: %v", err) } configStr := fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { cert_store: "%s" cert_match_by: "%s" cert_match: "%s" ca_certs_match: %s # Test settings that succeed should be equivalent to: # cert_file: "../test/configs/certs/tlsauth/client.pem" # key_file: "../test/configs/certs/tlsauth/client-key.pem" # ca_file: "../test/configs/certs/tlsauth/ca.pem" timeout: 5 } } ] } `, u.String(), certStore, matchBy, match, caMatch) leafConfig := createConfFile(t, []byte(configStr)) defer removeFile(t, leafConfig) leafServer, _ := RunServerWithConfig(leafConfig) defer leafServer.Shutdown() // After client verify, hub will match by SAN email, SAN dns, and Subject (in that order) // Our test client specifies Subject only so we should match on that... // A little settle time time.Sleep(1 * time.Second) checkLeafNodeConnectedCount(t, leafServer, expectedLeafCount) } // TestLeafTLSWindowsCertStore tests the topology of two NATS Servers connected as leaf and hub with authentication of // leaf to hub via mTLS with leaf's certificate and signing key provisioned in the Windows certificate store. func TestLeafTLSWindowsCertStore(t *testing.T) { // Client Identity (client.pem) // Issuer: O = NATS CA, OU = NATS.io, CN = localhost // Subject: OU = NATS.io, CN = example.com // Make sure windows cert store is reset to avoid conflict with other tests err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) if err != nil { t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) } // Provision Windows cert store with client cert and secret err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-client.ps1", nil) if err != nil { t.Fatalf("expected powershell provision to succeed: %s", err.Error()) } err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1", nil) if err != nil { t.Fatalf("expected powershell provision CA to succeed: %s", err.Error()) } // Fire up the hub hubConfig := createConfFile(t, []byte(` port: -1 leaf { listen: "127.0.0.1:-1" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/server.pem" key_file: "../test/configs/certs/tlsauth/server-key.pem" timeout: 5 verify_and_map: true } } accounts: { AcctA: { users: [ {user: "OU = NATS.io, CN = example.com"} ] }, AcctB: { users: [ {user: UserB1} ] }, SYS: { users: [ {user: System} ] } } system_account: "SYS" `)) defer removeFile(t, hubConfig) hubServer, hubOptions := RunServerWithConfig(hubConfig) defer hubServer.Shutdown() testCases := []struct { certStore string certMatchBy string certMatch string caCertsMatch string expectedLeafCount int }{ // Test subject and issuer {"WindowsCurrentUser", "Subject", "example.com", "\"NATS CA\"", 1}, {"WindowsCurrentUser", "Issuer", "NATS CA", "\"NATS CA\"", 1}, {"WindowsCurrentUser", "Issuer", "Frodo Baggins, Inc.", "\"NATS CA\"", 0}, {"WindowsCurrentUser", "Thumbprint", "7e44f478114a2e29b98b00beb1b3687d8dc0e481", "\"NATS CA\"", 0}, // Test CAs, NATS CA is valid, others are missing {"WindowsCurrentUser", "Subject", "example.com", "[\"NATS CA\"]", 1}, {"WindowsCurrentUser", "Subject", "example.com", "[\"GlobalSign\"]", 0}, {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing NATS Cert\"]", 0}, {"WindowsCurrentUser", "Subject", "example.com", "[\"NATS CA\", \"Missing NATS Cert1\"]", 1}, {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing Cert2\",\"NATS CA\"]", 1}, {"WindowsCurrentUser", "Subject", "example.com", "[\"Missing, Cert3\",\"Missing NATS Cert4\"]", 0}, } for _, tc := range testCases { testName := fmt.Sprintf("%s by %s match %s", tc.certStore, tc.certMatchBy, tc.certMatch) t.Run(fmt.Sprintf(testName, tc.certStore, tc.certMatchBy, tc.certMatch, tc.caCertsMatch), func(t *testing.T) { defer func() { if r := recover(); r != nil { if tc.expectedLeafCount != 0 { t.Fatalf("did not expect panic: %s", testName) } else { if !strings.Contains(fmt.Sprintf("%v", r), "Error processing configuration file") { t.Fatalf("did not expect unknown panic: %s", testName) } } } }() runConfiguredLeaf(t, hubOptions.LeafNode.Port, tc.certStore, tc.certMatchBy, tc.certMatch, tc.caCertsMatch, tc.expectedLeafCount) }) } } // TestServerTLSWindowsCertStore tests the topology of a NATS server requiring TLS and gettings it own server // cert identiy (as used when accepting NATS client connections and negotiating TLS) from Windows certificate store. func TestServerTLSWindowsCertStore(t *testing.T) { // Server Identity (server.pem) // Issuer: O = NATS CA, OU = NATS.io, CN = localhost // Subject: OU = NATS.io Operators, CN = localhost // Make sure windows cert store is reset to avoid conflict with other tests err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) if err != nil { t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) } // Provision Windows cert store with server cert and secret err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", nil) if err != nil { t.Fatalf("expected powershell provision to succeed: %s", err.Error()) } err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1", nil) if err != nil { t.Fatalf("expected powershell provision CA to succeed: %s", err.Error()) } // Fire up the server srvConfig := createConfFile(t, []byte(` listen: "localhost:-1" tls { cert_store: "WindowsCurrentUser" cert_match_by: "Subject" cert_match: "NATS.io Operators" ca_certs_match: ["NATS CA"] timeout: 5 } `)) defer removeFile(t, srvConfig) srvServer, _ := RunServerWithConfig(srvConfig) if srvServer == nil { t.Fatalf("expected to be able start server with cert store configuration") } defer srvServer.Shutdown() testCases := []struct { clientCA string expect bool }{ {"../test/configs/certs/tlsauth/ca.pem", true}, {"../test/configs/certs/tlsauth/client.pem", false}, } for _, tc := range testCases { t.Run(fmt.Sprintf("Client CA: %s", tc.clientCA), func(t *testing.T) { nc, _ := nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA)) err := nc.Publish("foo", []byte("hello TLS server-authenticated server")) if (err != nil) == tc.expect { t.Fatalf("expected publish result %v to TLS authenticated server", tc.expect) } nc.Close() for i := 0; i < 5; i++ { nc, _ = nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA)) err = nc.Publish("foo", []byte("hello TLS server-authenticated server")) if (err != nil) == tc.expect { t.Fatalf("expected repeated connection result %v to TLS authenticated server", tc.expect) } nc.Close() } }) } } // TestServerIgnoreExpiredCerts tests if the server skips expired certificates in configuration, and finds non-expired ones func TestServerIgnoreExpiredCerts(t *testing.T) { // Server Identities: expired.pem; not-expired.pem // Issuer: OU = NATS.io, CN = localhost // Subject: OU = NATS.io Operators, CN = localhost testCases := []struct { certFile string expect bool }{ {"expired.p12", false}, {"not-expired.p12", true}, } for _, tc := range testCases { t.Run(fmt.Sprintf("Server certificate: %s", tc.certFile), func(t *testing.T) { // Make sure windows cert store is reset to avoid conflict with other tests err := runPowershellScript("../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", nil) if err != nil { t.Fatalf("expected powershell cert delete to succeed: %s", err.Error()) } // Provision Windows cert store with server cert and secret err = runPowershellScript("../test/configs/certs/tlsauth/certstore/import-p12-server.ps1", []string{tc.certFile}) if err != nil { t.Fatalf("expected powershell provision to succeed: %s", err.Error()) } // Fire up the server srvConfig := createConfFile(t, []byte(` listen: "localhost:-1" tls { cert_store: "WindowsCurrentUser" cert_match_by: "Subject" cert_match: "NATS.io Operators" cert_match_skip_invalid: true timeout: 5 } `)) defer removeFile(t, srvConfig) cfg, _ := ProcessConfigFile(srvConfig) if (cfg != nil) == tc.expect { return } if tc.expect == false { t.Fatalf("expected server start to fail with expired certificate") } else { t.Fatalf("expected server to start with non expired certificate") } }) } } nats-server-2.10.27/server/ciphersuites.go000066400000000000000000000127301477524627100205220ustar00rootroot00000000000000// Copyright 2016-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "crypto/tls" ) // Where we maintain all of the available ciphers var cipherMap = map[string]uint16{ "TLS_RSA_WITH_RC4_128_SHA": tls.TLS_RSA_WITH_RC4_128_SHA, "TLS_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, "TLS_RSA_WITH_AES_128_CBC_SHA": tls.TLS_RSA_WITH_AES_128_CBC_SHA, "TLS_RSA_WITH_AES_128_CBC_SHA256": tls.TLS_RSA_WITH_AES_128_CBC_SHA256, "TLS_RSA_WITH_AES_256_CBC_SHA": tls.TLS_RSA_WITH_AES_256_CBC_SHA, "TLS_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_RSA_WITH_AES_256_GCM_SHA384, "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA": tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, "TLS_ECDHE_RSA_WITH_RC4_128_SHA": tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, "TLS_AES_128_GCM_SHA256": tls.TLS_AES_128_GCM_SHA256, "TLS_AES_256_GCM_SHA384": tls.TLS_AES_256_GCM_SHA384, "TLS_CHACHA20_POLY1305_SHA256": tls.TLS_CHACHA20_POLY1305_SHA256, } var cipherMapByID = map[uint16]string{ tls.TLS_RSA_WITH_RC4_128_SHA: "TLS_RSA_WITH_RC4_128_SHA", tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", tls.TLS_RSA_WITH_AES_128_CBC_SHA: "TLS_RSA_WITH_AES_128_CBC_SHA", tls.TLS_RSA_WITH_AES_128_CBC_SHA256: "TLS_RSA_WITH_AES_128_CBC_SHA256", tls.TLS_RSA_WITH_AES_256_CBC_SHA: "TLS_RSA_WITH_AES_256_CBC_SHA", tls.TLS_RSA_WITH_AES_256_GCM_SHA384: "TLS_RSA_WITH_AES_256_GCM_SHA384", tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA: "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA: "TLS_ECDHE_RSA_WITH_RC4_128_SHA", tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", tls.TLS_AES_128_GCM_SHA256: "TLS_AES_128_GCM_SHA256", tls.TLS_AES_256_GCM_SHA384: "TLS_AES_256_GCM_SHA384", tls.TLS_CHACHA20_POLY1305_SHA256: "TLS_CHACHA20_POLY1305_SHA256", } func defaultCipherSuites() []uint16 { return []uint16{ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, } } // Where we maintain available curve preferences var curvePreferenceMap = map[string]tls.CurveID{ "X25519": tls.X25519, "CurveP256": tls.CurveP256, "CurveP384": tls.CurveP384, "CurveP521": tls.CurveP521, } // reorder to default to the highest level of security. See: // https://blog.bracebin.com/achieving-perfect-ssl-labs-score-with-go func defaultCurvePreferences() []tls.CurveID { return []tls.CurveID{ tls.X25519, // faster than P256, arguably more secure tls.CurveP256, tls.CurveP384, tls.CurveP521, } } nats-server-2.10.27/server/client.go000066400000000000000000005220051477524627100172720ustar00rootroot00000000000000// Copyright 2012-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "io" "math/rand" "net" "net/http" "net/url" "regexp" "runtime" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/internal/fastrand" ) // Type of client connection. const ( // CLIENT is an end user. CLIENT = iota // ROUTER represents another server in the cluster. ROUTER // GATEWAY is a link between 2 clusters. GATEWAY // SYSTEM is an internal system client. SYSTEM // LEAF is for leaf node connections. LEAF // JETSTREAM is an internal jetstream client. JETSTREAM // ACCOUNT is for the internal client for accounts. ACCOUNT ) // Extended type of a CLIENT connection. This is returned by c.clientType() // and indicate what type of client connection we are dealing with. // If invoked on a non CLIENT connection, NON_CLIENT type is returned. const ( // If the connection is not a CLIENT connection. NON_CLIENT = iota // Regular NATS client. NATS // MQTT client. MQTT // Websocket client. WS ) const ( // ClientProtoZero is the original Client protocol from 2009. // http://nats.io/documentation/internals/nats-protocol/ ClientProtoZero = iota // ClientProtoInfo signals a client can receive more then the original INFO block. // This can be used to update clients on other cluster members, etc. ClientProtoInfo ) const ( pingProto = "PING" + _CRLF_ pongProto = "PONG" + _CRLF_ errProto = "-ERR '%s'" + _CRLF_ okProto = "+OK" + _CRLF_ ) // TLS Hanshake client types const ( tlsHandshakeLeaf = "leafnode" tlsHandshakeMQTT = "mqtt" ) const ( // Scratch buffer size for the processMsg() calls. msgScratchSize = 1024 msgHeadProto = "RMSG " msgHeadProtoLen = len(msgHeadProto) // For controlling dynamic buffer sizes. startBufSize = 512 // For INFO/CONNECT block minBufSize = 64 // Smallest to shrink to for PING/PONG maxBufSize = 65536 // 64k shortsToShrink = 2 // Trigger to shrink dynamic buffers maxFlushPending = 10 // Max fsps to have in order to wait for writeLoop readLoopReport = 2 * time.Second // Server should not send a PING (for RTT) before the first PONG has // been sent to the client. However, in case some client libs don't // send CONNECT+PING, cap the maximum time before server can send // the RTT PING. maxNoRTTPingBeforeFirstPong = 2 * time.Second // For stalling fast producers stallClientMinDuration = 2 * time.Millisecond stallClientMaxDuration = 5 * time.Millisecond stallTotalAllowed = 10 * time.Millisecond ) var readLoopReportThreshold = readLoopReport // Represent client booleans with a bitmask type clientFlag uint16 const ( hdrLine = "NATS/1.0\r\n" emptyHdrLine = "NATS/1.0\r\n\r\n" ) // Some client state represented as flags const ( connectReceived clientFlag = 1 << iota // The CONNECT proto has been received infoReceived // The INFO protocol has been received firstPongSent // The first PONG has been sent handshakeComplete // For TLS clients, indicate that the handshake is complete flushOutbound // Marks client as having a flushOutbound call in progress. noReconnect // Indicate that on close, this connection should not attempt a reconnect closeConnection // Marks that closeConnection has already been called. connMarkedClosed // Marks that markConnAsClosed has already been called. writeLoopStarted // Marks that the writeLoop has been started. skipFlushOnClose // Marks that flushOutbound() should not be called on connection close. expectConnect // Marks if this connection is expected to send a CONNECT connectProcessFinished // Marks if this connection has finished the connect process. compressionNegotiated // Marks if this connection has negotiated compression level with remote. didTLSFirst // Marks if this connection requested and was accepted doing the TLS handshake first (prior to INFO). ) // set the flag (would be equivalent to set the boolean to true) func (cf *clientFlag) set(c clientFlag) { *cf |= c } // clear the flag (would be equivalent to set the boolean to false) func (cf *clientFlag) clear(c clientFlag) { *cf &= ^c } // isSet returns true if the flag is set, false otherwise func (cf clientFlag) isSet(c clientFlag) bool { return cf&c != 0 } // setIfNotSet will set the flag `c` only if that flag was not already // set and return true to indicate that the flag has been set. Returns // false otherwise. func (cf *clientFlag) setIfNotSet(c clientFlag) bool { if *cf&c == 0 { *cf |= c return true } return false } // ClosedState is the reason client was closed. This will // be passed into calls to clearConnection, but will only // be stored in ConnInfo for monitoring. type ClosedState int const ( ClientClosed = ClosedState(iota + 1) AuthenticationTimeout AuthenticationViolation TLSHandshakeError SlowConsumerPendingBytes SlowConsumerWriteDeadline WriteError ReadError ParseError StaleConnection ProtocolViolation BadClientProtocolVersion WrongPort MaxAccountConnectionsExceeded MaxConnectionsExceeded MaxPayloadExceeded MaxControlLineExceeded MaxSubscriptionsExceeded DuplicateRoute RouteRemoved ServerShutdown AuthenticationExpired WrongGateway MissingAccount Revocation InternalClient MsgHeaderViolation NoRespondersRequiresHeaders ClusterNameConflict DuplicateRemoteLeafnodeConnection DuplicateClientID DuplicateServerName MinimumVersionRequired ClusterNamesIdentical Kicked ) // Some flags passed to processMsgResults const pmrNoFlag int = 0 const ( pmrCollectQueueNames int = 1 << iota pmrIgnoreEmptyQueueFilter pmrAllowSendFromRouteToRoute pmrMsgImportedFromService ) type client struct { // Here first because of use of atomics, and memory alignment. stats gwReplyMapping kind int srv *Server acc *Account perms *permissions in readCache parseState opts ClientOpts rrTracking *rrTracking mpay int32 msubs int32 mcl int32 mu sync.Mutex cid uint64 start time.Time nonce []byte pubKey string nc net.Conn ncs atomic.Value out outbound user *NkeyUser host string port uint16 subs map[string]*subscription replies map[string]*resp mperms *msgDeny darray []string pcd map[*client]struct{} atmr *time.Timer expires time.Time ping pinfo msgb [msgScratchSize]byte last time.Time lastIn time.Time repliesSincePrune uint16 lastReplyPrune time.Time headers bool rtt time.Duration rttStart time.Time route *route gw *gateway leaf *leaf ws *websocket mqtt *mqtt flags clientFlag // Compact booleans into a single field. Size will be increased when needed. rref byte trace bool echo bool noIcb bool tags jwt.TagList nameTag string tlsTo *time.Timer } type rrTracking struct { rmap map[string]*remoteLatency ptmr *time.Timer lrt time.Duration } // Struct for PING initiation from the server. type pinfo struct { tmr *time.Timer out int } // outbound holds pending data for a socket. type outbound struct { nb net.Buffers // Pending buffers for send, each has fixed capacity as per nbPool below. wnb net.Buffers // Working copy of "nb", reused on each flushOutbound call, partial writes may leave entries here for next iteration. pb int64 // Total pending/queued bytes. fsp int32 // Flush signals that are pending per producer from readLoop's pcd. sg *sync.Cond // To signal writeLoop that there is data to flush. wdl time.Duration // Snapshot of write deadline. mp int64 // Snapshot of max pending for client. lft time.Duration // Last flush time for Write. stc chan struct{} // Stall chan we create to slow down producers on overrun, e.g. fan-in. cw *s2.Writer } const nbMaxVectorSize = 1024 // == IOV_MAX on Linux/Darwin and most other Unices (except Solaris/AIX) const nbPoolSizeSmall = 512 // Underlying array size of small buffer const nbPoolSizeMedium = 4096 // Underlying array size of medium buffer const nbPoolSizeLarge = 65536 // Underlying array size of large buffer var nbPoolSmall = &sync.Pool{ New: func() any { b := [nbPoolSizeSmall]byte{} return &b }, } var nbPoolMedium = &sync.Pool{ New: func() any { b := [nbPoolSizeMedium]byte{} return &b }, } var nbPoolLarge = &sync.Pool{ New: func() any { b := [nbPoolSizeLarge]byte{} return &b }, } // nbPoolGet returns a frame that is a best-effort match for the given size. // Once a pooled frame is no longer needed, it should be recycled by passing // it to nbPoolPut. func nbPoolGet(sz int) []byte { switch { case sz <= nbPoolSizeSmall: return nbPoolSmall.Get().(*[nbPoolSizeSmall]byte)[:0] case sz <= nbPoolSizeMedium: return nbPoolMedium.Get().(*[nbPoolSizeMedium]byte)[:0] default: return nbPoolLarge.Get().(*[nbPoolSizeLarge]byte)[:0] } } // nbPoolPut recycles a frame that was retrieved from nbPoolGet. It is not // safe to return multiple slices referring to chunks of the same underlying // array as this may create overlaps when the buffers are returned to their // original size, resulting in race conditions. func nbPoolPut(b []byte) { switch cap(b) { case nbPoolSizeSmall: b := (*[nbPoolSizeSmall]byte)(b[0:nbPoolSizeSmall]) nbPoolSmall.Put(b) case nbPoolSizeMedium: b := (*[nbPoolSizeMedium]byte)(b[0:nbPoolSizeMedium]) nbPoolMedium.Put(b) case nbPoolSizeLarge: b := (*[nbPoolSizeLarge]byte)(b[0:nbPoolSizeLarge]) nbPoolLarge.Put(b) default: // Ignore frames that are the wrong size, this might happen // with WebSocket/MQTT messages as they are framed } } type perm struct { allow *Sublist deny *Sublist } type permissions struct { // Have these 2 first for memory alignment due to the use of atomic. pcsz int32 prun int32 sub perm pub perm resp *ResponsePermission pcache sync.Map } // This is used to dynamically track responses and reply subjects // for dynamic permissioning. type resp struct { t time.Time n int } // msgDeny is used when a user permission for subscriptions has a deny // clause but a subscription could be made that is of broader scope. // e.g. deny = "foo", but user subscribes to "*". That subscription should // succeed but no message sent on foo should be delivered. type msgDeny struct { deny *Sublist dcache map[string]bool } // routeTarget collects information regarding routes and queue groups for // sending information to a remote. type routeTarget struct { sub *subscription qs []byte _qs [32]byte } const ( maxResultCacheSize = 512 maxDenyPermCacheSize = 256 maxPermCacheSize = 128 pruneSize = 32 routeTargetInit = 8 replyPermLimit = 4096 replyPruneTime = time.Second ) // Represent read cache booleans with a bitmask type readCacheFlag uint16 const ( hasMappings readCacheFlag = 1 << iota // For account subject mappings. switchToCompression readCacheFlag = 1 << 1 ) const sysGroup = "_sys_" // Used in readloop to cache hot subject lookups and group statistics. type readCache struct { // These are for clients who are bound to a single account. genid uint64 results map[string]*SublistResult // This is for routes and gateways to have their own L1 as well that is account aware. pacache map[string]*perAccountCache // This is for when we deliver messages across a route. We use this structure // to make sure to only send one message and properly scope to queues as needed. rts []routeTarget // These are all temporary totals for an invocation of a read in readloop. msgs int32 bytes int32 subs int32 rsz int32 // Read buffer size srs int32 // Short reads, used for dynamic buffer resizing. // These are for readcache flags to avoid locks. flags readCacheFlag // Capture the time we started processing our readLoop. start time.Time // Total time stalled so far for readLoop processing. tst time.Duration } // set the flag (would be equivalent to set the boolean to true) func (rcf *readCacheFlag) set(c readCacheFlag) { *rcf |= c } // clear the flag (would be equivalent to set the boolean to false) func (rcf *readCacheFlag) clear(c readCacheFlag) { *rcf &= ^c } // isSet returns true if the flag is set, false otherwise func (rcf readCacheFlag) isSet(c readCacheFlag) bool { return rcf&c != 0 } const ( defaultMaxPerAccountCacheSize = 8192 defaultPrunePerAccountCacheSize = 1024 defaultClosedSubsCheckInterval = 5 * time.Minute ) var ( maxPerAccountCacheSize = defaultMaxPerAccountCacheSize prunePerAccountCacheSize = defaultPrunePerAccountCacheSize closedSubsCheckInterval = defaultClosedSubsCheckInterval ) // perAccountCache is for L1 semantics for inbound messages from a route or gateway to mimic the performance of clients. type perAccountCache struct { acc *Account results *SublistResult genid uint64 } func (c *client) String() (id string) { loaded := c.ncs.Load() if loaded != nil { return loaded.(string) } return _EMPTY_ } // GetNonce returns the nonce that was presented to the user on connection func (c *client) GetNonce() []byte { c.mu.Lock() defer c.mu.Unlock() return c.nonce } // GetName returns the application supplied name for the connection. func (c *client) GetName() string { c.mu.Lock() name := c.opts.Name c.mu.Unlock() return name } // GetOpts returns the client options provided by the application. func (c *client) GetOpts() *ClientOpts { return &c.opts } // GetTLSConnectionState returns the TLS ConnectionState if TLS is enabled, nil // otherwise. Implements the ClientAuth interface. func (c *client) GetTLSConnectionState() *tls.ConnectionState { c.mu.Lock() defer c.mu.Unlock() if c.nc == nil { return nil } tc, ok := c.nc.(*tls.Conn) if !ok { return nil } state := tc.ConnectionState() return &state } // For CLIENT connections, this function returns the client type, that is, // NATS (for regular clients), MQTT or WS for websocket. // If this is invoked for a non CLIENT connection, NON_CLIENT is returned. // // This function does not lock the client and accesses fields that are supposed // to be immutable and therefore it can be invoked outside of the client's lock. func (c *client) clientType() int { switch c.kind { case CLIENT: if c.isMqtt() { return MQTT } else if c.isWebsocket() { return WS } return NATS default: return NON_CLIENT } } var clientTypeStringMap = map[int]string{ NON_CLIENT: _EMPTY_, NATS: "nats", WS: "websocket", MQTT: "mqtt", } func (c *client) clientTypeString() string { if typeStringVal, ok := clientTypeStringMap[c.clientType()]; ok { return typeStringVal } return _EMPTY_ } // This is the main subscription struct that indicates // interest in published messages. // FIXME(dlc) - This is getting bloated for normal subs, need // to optionally have an opts section for non-normal stuff. type subscription struct { client *client im *streamImport // This is for import stream support. rsi bool si bool shadow []*subscription // This is to track shadowed accounts. icb msgHandler subject []byte queue []byte sid []byte origin []byte nm int64 max int64 qw int32 closed int32 mqtt *mqttSub } // Indicate that this subscription is closed. // This is used in pruning of route and gateway cache items. func (s *subscription) close() { atomic.StoreInt32(&s.closed, 1) } // Return true if this subscription was unsubscribed // or its connection has been closed. func (s *subscription) isClosed() bool { return atomic.LoadInt32(&s.closed) == 1 } type ClientOpts struct { Echo bool `json:"echo"` Verbose bool `json:"verbose"` Pedantic bool `json:"pedantic"` TLSRequired bool `json:"tls_required"` Nkey string `json:"nkey,omitempty"` JWT string `json:"jwt,omitempty"` Sig string `json:"sig,omitempty"` Token string `json:"auth_token,omitempty"` Username string `json:"user,omitempty"` Password string `json:"pass,omitempty"` Name string `json:"name"` Lang string `json:"lang"` Version string `json:"version"` Protocol int `json:"protocol"` Account string `json:"account,omitempty"` AccountNew bool `json:"new_account,omitempty"` Headers bool `json:"headers,omitempty"` NoResponders bool `json:"no_responders,omitempty"` // Routes and Leafnodes only Import *SubjectPermission `json:"import,omitempty"` Export *SubjectPermission `json:"export,omitempty"` // Leafnodes RemoteAccount string `json:"remote_account,omitempty"` } var defaultOpts = ClientOpts{Verbose: true, Pedantic: true, Echo: true} var internalOpts = ClientOpts{Verbose: false, Pedantic: false, Echo: false} func (c *client) setTraceLevel() { if c.kind == SYSTEM && !(atomic.LoadInt32(&c.srv.logging.traceSysAcc) != 0) { c.trace = false } else { c.trace = (atomic.LoadInt32(&c.srv.logging.trace) != 0) } } // Lock should be held func (c *client) initClient() { s := c.srv c.cid = atomic.AddUint64(&s.gcid, 1) // Outbound data structure setup c.out.sg = sync.NewCond(&(c.mu)) opts := s.getOpts() // Snapshots to avoid mutex access in fast paths. c.out.wdl = opts.WriteDeadline c.out.mp = opts.MaxPending // Snapshot max control line since currently can not be changed on reload and we // were checking it on each call to parse. If this changes and we allow MaxControlLine // to be reloaded without restart, this code will need to change. c.mcl = int32(opts.MaxControlLine) if c.mcl == 0 { c.mcl = MAX_CONTROL_LINE_SIZE } c.subs = make(map[string]*subscription) c.echo = true c.setTraceLevel() // This is a scratch buffer used for processMsg() // The msg header starts with "RMSG ", which can be used // for both local and routes. // in bytes that is [82 77 83 71 32]. c.msgb = [msgScratchSize]byte{82, 77, 83, 71, 32} // This is to track pending clients that have data to be flushed // after we process inbound msgs from our own connection. c.pcd = make(map[*client]struct{}) // snapshot the string version of the connection var conn string if c.nc != nil { if addr := c.nc.RemoteAddr(); addr != nil { if conn = addr.String(); conn != _EMPTY_ { host, port, _ := net.SplitHostPort(conn) iPort, _ := strconv.Atoi(port) c.host, c.port = host, uint16(iPort) if c.isWebsocket() && c.ws.clientIP != _EMPTY_ { cip := c.ws.clientIP // Surround IPv6 addresses with square brackets, as // net.JoinHostPort would do... if strings.Contains(cip, ":") { cip = "[" + cip + "]" } conn = fmt.Sprintf("%s/%s", cip, conn) } // Now that we have extracted host and port, escape // the string because it is going to be used in Sprintf conn = strings.ReplaceAll(conn, "%", "%%") } } } switch c.kind { case CLIENT: switch c.clientType() { case NATS: c.ncs.Store(fmt.Sprintf("%s - cid:%d", conn, c.cid)) case WS: c.ncs.Store(fmt.Sprintf("%s - wid:%d", conn, c.cid)) case MQTT: var ws string if c.isWebsocket() { ws = "_ws" } c.ncs.Store(fmt.Sprintf("%s - mid%s:%d", conn, ws, c.cid)) } case ROUTER: c.ncs.Store(fmt.Sprintf("%s - rid:%d", conn, c.cid)) case GATEWAY: c.ncs.Store(fmt.Sprintf("%s - gid:%d", conn, c.cid)) case LEAF: var ws string if c.isWebsocket() { ws = "_ws" } c.ncs.Store(fmt.Sprintf("%s - lid%s:%d", conn, ws, c.cid)) case SYSTEM: c.ncs.Store("SYSTEM") case JETSTREAM: c.ncs.Store("JETSTREAM") case ACCOUNT: c.ncs.Store("ACCOUNT") } } // RemoteAddress expose the Address of the client connection, // nil when not connected or unknown func (c *client) RemoteAddress() net.Addr { c.mu.Lock() defer c.mu.Unlock() if c.nc == nil { return nil } return c.nc.RemoteAddr() } // Helper function to report errors. func (c *client) reportErrRegisterAccount(acc *Account, err error) { if err == ErrTooManyAccountConnections { c.maxAccountConnExceeded() return } c.Errorf("Problem registering with account %q: %s", acc.Name, err) c.sendErr("Failed Account Registration") } // Kind returns the client kind and will be one of the defined constants like CLIENT, ROUTER, GATEWAY, LEAF func (c *client) Kind() int { c.mu.Lock() kind := c.kind c.mu.Unlock() return kind } // registerWithAccount will register the given user with a specific // account. This will change the subject namespace. func (c *client) registerWithAccount(acc *Account) error { if acc == nil { return ErrBadAccount } acc.mu.RLock() bad := acc.sl == nil acc.mu.RUnlock() if bad { return ErrBadAccount } // If we were previously registered, usually to $G, do accounting here to remove. if c.acc != nil { if prev := c.acc.removeClient(c); prev == 1 && c.srv != nil { c.srv.decActiveAccounts() } } c.mu.Lock() kind := c.kind srv := c.srv c.acc = acc c.applyAccountLimits() c.mu.Unlock() // Check if we have a max connections violation if kind == CLIENT && acc.MaxTotalConnectionsReached() { return ErrTooManyAccountConnections } else if kind == LEAF { // Check if we are already connected to this cluster. if rc := c.remoteCluster(); rc != _EMPTY_ && acc.hasLeafNodeCluster(rc) { return ErrLeafNodeLoop } if acc.MaxTotalLeafNodesReached() { return ErrTooManyAccountConnections } } // Add in new one. if prev := acc.addClient(c); prev == 0 && srv != nil { srv.incActiveAccounts() } return nil } // Helper to determine if we have met or exceeded max subs. func (c *client) subsAtLimit() bool { return c.msubs != jwt.NoLimit && len(c.subs) >= int(c.msubs) } func minLimit(value *int32, limit int32) bool { v := atomic.LoadInt32(value) if v != jwt.NoLimit { if limit != jwt.NoLimit { if limit < v { atomic.StoreInt32(value, limit) return true } } } else if limit != jwt.NoLimit { atomic.StoreInt32(value, limit) return true } return false } // Apply account limits // Lock is held on entry. // FIXME(dlc) - Should server be able to override here? func (c *client) applyAccountLimits() { if c.acc == nil || (c.kind != CLIENT && c.kind != LEAF) { return } atomic.StoreInt32(&c.mpay, jwt.NoLimit) c.msubs = jwt.NoLimit if c.opts.JWT != _EMPTY_ { // user jwt implies account if uc, _ := jwt.DecodeUserClaims(c.opts.JWT); uc != nil { atomic.StoreInt32(&c.mpay, int32(uc.Limits.Payload)) c.msubs = int32(uc.Limits.Subs) if uc.IssuerAccount != _EMPTY_ && uc.IssuerAccount != uc.Issuer { if scope, ok := c.acc.signingKeys[uc.Issuer]; ok { if userScope, ok := scope.(*jwt.UserScope); ok { // if signing key disappeared or changed and we don't get here, the client will be disconnected c.mpay = int32(userScope.Template.Limits.Payload) c.msubs = int32(userScope.Template.Limits.Subs) } } } } } c.acc.mu.RLock() minLimit(&c.mpay, c.acc.mpay) minLimit(&c.msubs, c.acc.msubs) c.acc.mu.RUnlock() s := c.srv opts := s.getOpts() mPay := opts.MaxPayload // options encode unlimited differently if mPay == 0 { mPay = jwt.NoLimit } mSubs := int32(opts.MaxSubs) if mSubs == 0 { mSubs = jwt.NoLimit } wasUnlimited := c.mpay == jwt.NoLimit if minLimit(&c.mpay, mPay) && !wasUnlimited { c.Errorf("Max Payload set to %d from server overrides account or user config", opts.MaxPayload) } wasUnlimited = c.msubs == jwt.NoLimit if minLimit(&c.msubs, mSubs) && !wasUnlimited { c.Errorf("Max Subscriptions set to %d from server overrides account or user config", opts.MaxSubs) } if c.subsAtLimit() { go func() { c.maxSubsExceeded() time.Sleep(20 * time.Millisecond) c.closeConnection(MaxSubscriptionsExceeded) }() } } // RegisterUser allows auth to call back into a new client // with the authenticated user. This is used to map // any permissions into the client and setup accounts. func (c *client) RegisterUser(user *User) { // Register with proper account and sublist. if user.Account != nil { if err := c.registerWithAccount(user.Account); err != nil { c.reportErrRegisterAccount(user.Account, err) return } } c.mu.Lock() // Assign permissions. if user.Permissions == nil { // Reset perms to nil in case client previously had them. c.perms = nil c.mperms = nil } else { c.setPermissions(user.Permissions) } // allows custom authenticators to set a username to be reported in // server events and more if user.Username != _EMPTY_ { c.opts.Username = user.Username } // if a deadline time stamp is set we start a timer to disconnect the user at that time if !user.ConnectionDeadline.IsZero() { c.setExpirationTimerUnlocked(time.Until(user.ConnectionDeadline)) } c.mu.Unlock() } // RegisterNkeyUser allows auth to call back into a new nkey // client with the authenticated user. This is used to map // any permissions into the client and setup accounts. func (c *client) RegisterNkeyUser(user *NkeyUser) error { // Register with proper account and sublist. if user.Account != nil { if err := c.registerWithAccount(user.Account); err != nil { c.reportErrRegisterAccount(user.Account, err) return err } } c.mu.Lock() c.user = user // Assign permissions. if user.Permissions == nil { // Reset perms to nil in case client previously had them. c.perms = nil c.mperms = nil } else { c.setPermissions(user.Permissions) } c.mu.Unlock() return nil } func splitSubjectQueue(sq string) ([]byte, []byte, error) { vals := strings.Fields(strings.TrimSpace(sq)) s := []byte(vals[0]) var q []byte if len(vals) == 2 { q = []byte(vals[1]) } else if len(vals) > 2 { return nil, nil, fmt.Errorf("invalid subject-queue %q", sq) } return s, q, nil } // Initializes client.perms structure. // Lock is held on entry. func (c *client) setPermissions(perms *Permissions) { if perms == nil { return } c.perms = &permissions{} // Loop over publish permissions if perms.Publish != nil { if perms.Publish.Allow != nil { c.perms.pub.allow = NewSublistWithCache() } for _, pubSubject := range perms.Publish.Allow { sub := &subscription{subject: []byte(pubSubject)} c.perms.pub.allow.Insert(sub) } if len(perms.Publish.Deny) > 0 { c.perms.pub.deny = NewSublistWithCache() } for _, pubSubject := range perms.Publish.Deny { sub := &subscription{subject: []byte(pubSubject)} c.perms.pub.deny.Insert(sub) } } // Check if we are allowed to send responses. if perms.Response != nil { rp := *perms.Response c.perms.resp = &rp c.replies = make(map[string]*resp) } // Loop over subscribe permissions if perms.Subscribe != nil { var err error if len(perms.Subscribe.Allow) > 0 { c.perms.sub.allow = NewSublistWithCache() } for _, subSubject := range perms.Subscribe.Allow { sub := &subscription{} sub.subject, sub.queue, err = splitSubjectQueue(subSubject) if err != nil { c.Errorf("%s", err.Error()) continue } c.perms.sub.allow.Insert(sub) } if len(perms.Subscribe.Deny) > 0 { c.perms.sub.deny = NewSublistWithCache() // Also hold onto this array for later. c.darray = perms.Subscribe.Deny } for _, subSubject := range perms.Subscribe.Deny { sub := &subscription{} sub.subject, sub.queue, err = splitSubjectQueue(subSubject) if err != nil { c.Errorf("%s", err.Error()) continue } c.perms.sub.deny.Insert(sub) } } // If we are a leafnode and we are the hub copy the extracted perms // to resend back to soliciting server. These are reversed from the // way routes interpret them since this is how the soliciting server // will receive these back in an update INFO. if c.isHubLeafNode() { c.opts.Import = perms.Subscribe c.opts.Export = perms.Publish } } // Build public permissions from internal ones. // Used for user info requests. func (c *client) publicPermissions() *Permissions { c.mu.Lock() defer c.mu.Unlock() if c.perms == nil { return nil } perms := &Permissions{ Publish: &SubjectPermission{}, Subscribe: &SubjectPermission{}, } _subs := [32]*subscription{} // Publish if c.perms.pub.allow != nil { subs := _subs[:0] c.perms.pub.allow.All(&subs) for _, sub := range subs { perms.Publish.Allow = append(perms.Publish.Allow, string(sub.subject)) } } if c.perms.pub.deny != nil { subs := _subs[:0] c.perms.pub.deny.All(&subs) for _, sub := range subs { perms.Publish.Deny = append(perms.Publish.Deny, string(sub.subject)) } } // Subsribe if c.perms.sub.allow != nil { subs := _subs[:0] c.perms.sub.allow.All(&subs) for _, sub := range subs { perms.Subscribe.Allow = append(perms.Subscribe.Allow, string(sub.subject)) } } if c.perms.sub.deny != nil { subs := _subs[:0] c.perms.sub.deny.All(&subs) for _, sub := range subs { perms.Subscribe.Deny = append(perms.Subscribe.Deny, string(sub.subject)) } } // Responses. if c.perms.resp != nil { rp := *c.perms.resp perms.Response = &rp } return perms } type denyType int const ( pub = denyType(iota + 1) sub both ) // Merge client.perms structure with additional pub deny permissions // Lock is held on entry. func (c *client) mergeDenyPermissions(what denyType, denyPubs []string) { if len(denyPubs) == 0 { return } if c.perms == nil { c.perms = &permissions{} } var perms []*perm switch what { case pub: perms = []*perm{&c.perms.pub} case sub: perms = []*perm{&c.perms.sub} case both: perms = []*perm{&c.perms.pub, &c.perms.sub} } for _, p := range perms { if p.deny == nil { p.deny = NewSublistWithCache() } FOR_DENY: for _, subj := range denyPubs { r := p.deny.Match(subj) for _, v := range r.qsubs { for _, s := range v { if string(s.subject) == subj { continue FOR_DENY } } } for _, s := range r.psubs { if string(s.subject) == subj { continue FOR_DENY } } sub := &subscription{subject: []byte(subj)} p.deny.Insert(sub) } } } // Merge client.perms structure with additional pub deny permissions // Client lock must not be held on entry func (c *client) mergeDenyPermissionsLocked(what denyType, denyPubs []string) { c.mu.Lock() c.mergeDenyPermissions(what, denyPubs) c.mu.Unlock() } // Check to see if we have an expiration for the user JWT via base claims. // FIXME(dlc) - Clear on connect with new JWT. func (c *client) setExpiration(claims *jwt.ClaimsData, validFor time.Duration) { if claims.Expires == 0 { if validFor != 0 { c.setExpirationTimer(validFor) } return } expiresAt := time.Duration(0) tn := time.Now().Unix() if claims.Expires > tn { expiresAt = time.Duration(claims.Expires-tn) * time.Second } if validFor != 0 && validFor < expiresAt { c.setExpirationTimer(validFor) } else { c.setExpirationTimer(expiresAt) } } // This will load up the deny structure used for filtering delivered // messages based on a deny clause for subscriptions. // Lock should be held. func (c *client) loadMsgDenyFilter() { c.mperms = &msgDeny{NewSublistWithCache(), make(map[string]bool)} for _, sub := range c.darray { c.mperms.deny.Insert(&subscription{subject: []byte(sub)}) } } // writeLoop is the main socket write functionality. // Runs in its own Go routine. func (c *client) writeLoop() { defer c.srv.grWG.Done() c.mu.Lock() if c.isClosed() { c.mu.Unlock() return } c.flags.set(writeLoopStarted) c.mu.Unlock() // Used to check that we did flush from last wake up. waitOk := true var closed bool // Main loop. Will wait to be signaled and then will use // buffered outbound structure for efficient writev to the underlying socket. for { c.mu.Lock() if closed = c.isClosed(); !closed { owtf := c.out.fsp > 0 && c.out.pb < maxBufSize && c.out.fsp < maxFlushPending if waitOk && (c.out.pb == 0 || owtf) { c.out.sg.Wait() // Check that connection has not been closed while lock was released // in the conditional wait. closed = c.isClosed() } } if closed { c.flushAndClose(false) c.mu.Unlock() // We should always call closeConnection() to ensure that state is // properly cleaned-up. It will be a no-op if already done. c.closeConnection(WriteError) // Now explicitly call reconnect(). Thanks to ref counting, we know // that the reconnect will execute only after connection has been // removed from the server state. c.reconnect() return } // Flush data waitOk = c.flushOutbound() c.mu.Unlock() } } // flushClients will make sure to flush any clients we may have // sent to during processing. We pass in a budget as a time.Duration // for how much time to spend in place flushing for this client. func (c *client) flushClients(budget time.Duration) time.Time { last := time.Now() // Check pending clients for flush. for cp := range c.pcd { // TODO(dlc) - Wonder if it makes more sense to create a new map? delete(c.pcd, cp) // Queue up a flush for those in the set cp.mu.Lock() // Update last activity for message delivery cp.last = last // Remove ourselves from the pending list. cp.out.fsp-- // Just ignore if this was closed. if cp.isClosed() { cp.mu.Unlock() continue } if budget > 0 && cp.out.lft < 2*budget && cp.flushOutbound() { budget -= cp.out.lft } else { cp.flushSignal() } cp.mu.Unlock() } return last } // readLoop is the main socket read functionality. // Runs in its own Go routine. func (c *client) readLoop(pre []byte) { // Grab the connection off the client, it will be cleared on a close. // We check for that after the loop, but want to avoid a nil dereference c.mu.Lock() s := c.srv defer s.grWG.Done() if c.isClosed() { c.mu.Unlock() return } nc := c.nc ws := c.isWebsocket() if c.isMqtt() { c.mqtt.r = &mqttReader{reader: nc} } c.in.rsz = startBufSize // Check the per-account-cache for closed subscriptions cpacc := c.kind == ROUTER || c.kind == GATEWAY // Last per-account-cache check for closed subscriptions lpacc := time.Now() acc := c.acc var masking bool if ws { masking = c.ws.maskread } checkCompress := c.kind == ROUTER || c.kind == LEAF c.mu.Unlock() defer func() { if c.isMqtt() { s.mqttHandleClosedClient(c) } // These are used only in the readloop, so we can set them to nil // on exit of the readLoop. c.in.results, c.in.pacache = nil, nil }() // Start read buffer. b := make([]byte, c.in.rsz) // Websocket clients will return several slices if there are multiple // websocket frames in the blind read. For non WS clients though, we // will always have 1 slice per loop iteration. So we define this here // so non WS clients will use bufs[0] = b[:n]. var _bufs [1][]byte bufs := _bufs[:1] var wsr *wsReadInfo if ws { wsr = &wsReadInfo{mask: masking} wsr.init() } var decompress bool var reader io.Reader reader = nc for { var n int var err error // If we have a pre buffer parse that first. if len(pre) > 0 { b = pre n = len(pre) pre = nil } else { n, err = reader.Read(b) // If we have any data we will try to parse and exit at the end. if n == 0 && err != nil { c.closeConnection(closedStateForErr(err)) return } } if ws { bufs, err = c.wsRead(wsr, reader, b[:n]) if bufs == nil && err != nil { if err != io.EOF { c.Errorf("read error: %v", err) } c.closeConnection(closedStateForErr(err)) return } else if bufs == nil { continue } } else { bufs[0] = b[:n] } // Check if the account has mappings and if so set the local readcache flag. // We check here to make sure any changes such as config reload are reflected here. if c.kind == CLIENT || c.kind == LEAF { if acc.hasMappings() { c.in.flags.set(hasMappings) } else { c.in.flags.clear(hasMappings) } } c.in.start = time.Now() // Clear inbound stats cache c.in.msgs = 0 c.in.bytes = 0 c.in.subs = 0 // Main call into parser for inbound data. This will generate callouts // to process messages, etc. for i := 0; i < len(bufs); i++ { if err := c.parse(bufs[i]); err != nil { if err == ErrMinimumVersionRequired { // Special case here, currently only for leaf node connections. // When process the CONNECT protocol, if the minimum version // required was not met, an error was printed and sent back to // the remote, and connection was closed after a certain delay // (to avoid "rapid" reconnection from the remote). // We don't need to do any of the things below, simply return. return } if dur := time.Since(c.in.start); dur >= readLoopReportThreshold { c.Warnf("Readloop processing time: %v", dur) } // Need to call flushClients because some of the clients have been // assigned messages and their "fsp" incremented, and need now to be // decremented and their writeLoop signaled. c.flushClients(0) // handled inline if err != ErrMaxPayload && err != ErrAuthentication { c.Error(err) c.closeConnection(ProtocolViolation) } return } // Clear total stalled time here. if c.in.tst >= stallClientMaxDuration { c.rateLimitFormatWarnf("Producer was stalled for a total of %v", c.in.tst.Round(time.Millisecond)) } c.in.tst = 0 } // If we are a ROUTER/LEAF and have processed an INFO, it is possible that // we are asked to switch to compression now. if checkCompress && c.in.flags.isSet(switchToCompression) { c.in.flags.clear(switchToCompression) // For now we support only s2 compression... reader = s2.NewReader(nc) decompress = true } // Updates stats for client and server that were collected // from parsing through the buffer. if c.in.msgs > 0 { atomic.AddInt64(&c.inMsgs, int64(c.in.msgs)) atomic.AddInt64(&c.inBytes, int64(c.in.bytes)) if acc != nil { atomic.AddInt64(&acc.inMsgs, int64(c.in.msgs)) atomic.AddInt64(&acc.inBytes, int64(c.in.bytes)) } atomic.AddInt64(&s.inMsgs, int64(c.in.msgs)) atomic.AddInt64(&s.inBytes, int64(c.in.bytes)) } // Signal to writeLoop to flush to socket. last := c.flushClients(0) // Update activity, check read buffer size. c.mu.Lock() // Activity based on interest changes or data/msgs. // Also update last receive activity for ping sender if c.in.msgs > 0 || c.in.subs > 0 { c.last = last c.lastIn = last } if n >= cap(b) { c.in.srs = 0 } else if n < cap(b)/2 { // divide by 2 b/c we want less than what we would shrink to. c.in.srs++ } // Update read buffer size as/if needed. if n >= cap(b) && cap(b) < maxBufSize { // Grow c.in.rsz = int32(cap(b) * 2) b = make([]byte, c.in.rsz) } else if n < cap(b) && cap(b) > minBufSize && c.in.srs > shortsToShrink { // Shrink, for now don't accelerate, ping/pong will eventually sort it out. c.in.rsz = int32(cap(b) / 2) b = make([]byte, c.in.rsz) } // re-snapshot the account since it can change during reload, etc. acc = c.acc // Refresh nc because in some cases, we have upgraded c.nc to TLS. if nc != c.nc { nc = c.nc if decompress && nc != nil { // For now we support only s2 compression... reader.(*s2.Reader).Reset(nc) } else if !decompress { reader = nc } } c.mu.Unlock() // Connection was closed if nc == nil { return } if dur := time.Since(c.in.start); dur >= readLoopReportThreshold { c.Warnf("Readloop processing time: %v", dur) } // We could have had a read error from above but still read some data. // If so do the close here unconditionally. if err != nil { c.closeConnection(closedStateForErr(err)) return } if cpacc && (c.in.start.Sub(lpacc)) >= closedSubsCheckInterval { c.pruneClosedSubFromPerAccountCache() lpacc = time.Now() } } } // Returns the appropriate closed state for a given read error. func closedStateForErr(err error) ClosedState { if err == io.EOF { return ClientClosed } return ReadError } // collapsePtoNB will either returned framed WebSocket buffers or it will // return a reference to c.out.nb. func (c *client) collapsePtoNB() (net.Buffers, int64) { if c.isWebsocket() { return c.wsCollapsePtoNB() } return c.out.nb, c.out.pb } // flushOutbound will flush outbound buffer to a client. // Will return true if data was attempted to be written. // Lock must be held func (c *client) flushOutbound() bool { if c.flags.isSet(flushOutbound) { // For CLIENT connections, it is possible that the readLoop calls // flushOutbound(). If writeLoop and readLoop compete and we are // here we should release the lock to reduce the risk of spinning. c.mu.Unlock() runtime.Gosched() c.mu.Lock() return false } c.flags.set(flushOutbound) defer func() { // Check flushAndClose() for explanation on why we do this. if c.isClosed() { for i := range c.out.wnb { nbPoolPut(c.out.wnb[i]) } c.out.wnb = nil } c.flags.clear(flushOutbound) }() // Check for nothing to do. if c.nc == nil || c.srv == nil || c.out.pb == 0 { return true // true because no need to queue a signal. } // In the case of a normal socket connection, "collapsed" is just a ref // to "nb". In the case of WebSockets, additional framing is added to // anything that is waiting in "nb". Also keep a note of how many bytes // were queued before we release the mutex. collapsed, attempted := c.collapsePtoNB() // Frustratingly, (net.Buffers).WriteTo() modifies the receiver so we // can't work on "nb" directly — while the mutex is unlocked during IO, // something else might call queueOutbound and modify it. So instead we // need a working copy — we'll operate on "wnb" instead. Note that in // the case of a partial write, "wnb" may have remaining data from the // previous write, and in the case of WebSockets, that data may already // be framed, so we are careful not to re-frame "wnb" here. Instead we // will just frame up "nb" and append it onto whatever is left on "wnb". // "nb" will be set to nil so that we can manipulate "collapsed" outside // of the client's lock, which is interesting in case of compression. c.out.nb = nil // In case it goes away after releasing the lock. nc := c.nc // Capture this (we change the value in some tests) wdl := c.out.wdl // Check for compression cw := c.out.cw if cw != nil { // We will have to adjust once we have compressed, so remove for now. c.out.pb -= attempted if c.isWebsocket() { c.ws.fs -= attempted } } // Do NOT hold lock during actual IO. c.mu.Unlock() // Compress outside of the lock if cw != nil { var err error bb := bytes.Buffer{} cw.Reset(&bb) for _, buf := range collapsed { if _, err = cw.Write(buf); err != nil { break } } if err == nil { err = cw.Close() } if err != nil { c.Errorf("Error compressing data: %v", err) // We need to grab the lock now before marking as closed and exiting c.mu.Lock() c.markConnAsClosed(WriteError) return false } collapsed = append(net.Buffers(nil), bb.Bytes()) attempted = int64(len(collapsed[0])) } // This is safe to do outside of the lock since "collapsed" is no longer // referenced in c.out.nb (which can be modified in queueOutboud() while // the lock is released). c.out.wnb = append(c.out.wnb, collapsed...) var _orig [nbMaxVectorSize][]byte orig := append(_orig[:0], c.out.wnb...) // Since WriteTo is lopping things off the beginning, we need to remember // the start position of the underlying array so that we can get back to it. // Otherwise we'll always "slide forward" and that will result in reallocs. startOfWnb := c.out.wnb[0:] // flush here start := time.Now() var n int64 // Total bytes written var wn int64 // Bytes written per loop var err error // Error from last write, if any for len(c.out.wnb) > 0 { // Limit the number of vectors to no more than nbMaxVectorSize, // which if 1024, will mean a maximum of 64MB in one go. wnb := c.out.wnb if len(wnb) > nbMaxVectorSize { wnb = wnb[:nbMaxVectorSize] } consumed := len(wnb) // Actual write to the socket. The deadline applies to each batch // rather than the total write, such that the configured deadline // can be tuned to a known maximum quantity (64MB). nc.SetWriteDeadline(time.Now().Add(wdl)) wn, err = wnb.WriteTo(nc) nc.SetWriteDeadline(time.Time{}) // Update accounting, move wnb slice onwards if needed, or stop // if a write error was reported that wasn't a short write. n += wn c.out.wnb = c.out.wnb[consumed-len(wnb):] if err != nil && err != io.ErrShortWrite { break } } lft := time.Since(start) // Re-acquire client lock. c.mu.Lock() // Adjust if we were compressing. if cw != nil { c.out.pb += attempted if c.isWebsocket() { c.ws.fs += attempted } } // At this point, "wnb" has been mutated by WriteTo and any consumed // buffers have been lopped off the beginning, so in order to return // them to the pool, we need to look at the difference between "orig" // and "wnb". for i := 0; i < len(orig)-len(c.out.wnb); i++ { nbPoolPut(orig[i]) } // At this point it's possible that "nb" has been modified by another // call to queueOutbound while the lock was released, so we'll leave // those for the next iteration. Meanwhile it's possible that we only // managed a partial write of "wnb", so we'll shift anything that // remains up to the beginning of the array to prevent reallocating. // Anything left in "wnb" has already been framed for WebSocket conns // so leave them alone for the next call to flushOutbound. c.out.wnb = append(startOfWnb[:0], c.out.wnb...) // If we've written everything but the underlying array of our working // buffer has grown excessively then free it — the GC will tidy it up // and we can allocate a new one next time. if len(c.out.wnb) == 0 && cap(c.out.wnb) > nbPoolSizeLarge*8 { c.out.wnb = nil } // Ignore ErrShortWrite errors, they will be handled as partials. if err != nil && err != io.ErrShortWrite { // Handle timeout error (slow consumer) differently if ne, ok := err.(net.Error); ok && ne.Timeout() { if closed := c.handleWriteTimeout(n, attempted, len(orig)); closed { return true } } else { // Other errors will cause connection to be closed. // For clients, report as debug but for others report as error. report := c.Debugf if c.kind != CLIENT { report = c.Errorf } report("Error flushing: %v", err) c.markConnAsClosed(WriteError) return true } } // Update flush time statistics. c.out.lft = lft // Subtract from pending bytes and messages. c.out.pb -= n if c.isWebsocket() { c.ws.fs -= n } // Check that if there is still data to send and writeLoop is in wait, // then we need to signal. if c.out.pb > 0 { c.flushSignal() } // Check if we have a stalled gate and if so and we are recovering release // any stalled producers. Only kind==CLIENT will stall. if c.out.stc != nil && (n == attempted || c.out.pb < c.out.mp/4*3) { close(c.out.stc) c.out.stc = nil } return true } // This is invoked from flushOutbound() for io/timeout error (slow consumer). // Returns a boolean to indicate if the connection has been closed or not. // Lock is held on entry. func (c *client) handleWriteTimeout(written, attempted int64, numChunks int) bool { if tlsConn, ok := c.nc.(*tls.Conn); ok { if !tlsConn.ConnectionState().HandshakeComplete { // Likely a TLSTimeout error instead... c.markConnAsClosed(TLSHandshakeError) // Would need to coordinate with tlstimeout() // to avoid double logging, so skip logging // here, and don't report a slow consumer error. return true } } else if c.flags.isSet(expectConnect) && !c.flags.isSet(connectReceived) { // Under some conditions, a connection may hit a slow consumer write deadline // before the authorization timeout. If that is the case, then we handle // as slow consumer though we do not increase the counter as that can be // misleading. c.markConnAsClosed(SlowConsumerWriteDeadline) return true } // Aggregate slow consumers. atomic.AddInt64(&c.srv.slowConsumers, 1) switch c.kind { case CLIENT: c.srv.scStats.clients.Add(1) case ROUTER: c.srv.scStats.routes.Add(1) case GATEWAY: c.srv.scStats.gateways.Add(1) case LEAF: c.srv.scStats.leafs.Add(1) } if c.acc != nil { atomic.AddInt64(&c.acc.slowConsumers, 1) } c.Noticef("Slow Consumer Detected: WriteDeadline of %v exceeded with %d chunks of %d total bytes.", c.out.wdl, numChunks, attempted) // We always close CLIENT connections, or when nothing was written at all... if c.kind == CLIENT || written == 0 { c.markConnAsClosed(SlowConsumerWriteDeadline) return true } return false } // Marks this connection has closed with the given reason. // Sets the connMarkedClosed flag and skipFlushOnClose depending on the reason. // Depending on the kind of connection, the connection will be saved. // If a writeLoop has been started, the final flush will be done there, otherwise // flush and close of TCP connection is done here in place. // Returns true if closed in place, flase otherwise. // Lock is held on entry. func (c *client) markConnAsClosed(reason ClosedState) { // Possibly set skipFlushOnClose flag even if connection has already been // mark as closed. The rationale is that a connection may be closed with // a reason that justifies a flush (say after sending an -ERR), but then // the flushOutbound() gets a write error. If that happens, connection // being lost, there is no reason to attempt to flush again during the // teardown when the writeLoop exits. var skipFlush bool switch reason { case ReadError, WriteError, SlowConsumerPendingBytes, SlowConsumerWriteDeadline, TLSHandshakeError: c.flags.set(skipFlushOnClose) skipFlush = true } if c.flags.isSet(connMarkedClosed) { return } c.flags.set(connMarkedClosed) // For a websocket client, unless we are told not to flush, enqueue // a websocket CloseMessage based on the reason. if !skipFlush && c.isWebsocket() && !c.ws.closeSent { c.wsEnqueueCloseMessage(reason) } // Be consistent with the creation: for routes, gateways and leaf, // we use Noticef on create, so use that too for delete. if c.srv != nil { if c.kind == LEAF { if c.acc != nil { c.Noticef("%s connection closed: %s - Account: %s", c.kindString(), reason, c.acc.traceLabel()) } else { c.Noticef("%s connection closed: %s", c.kindString(), reason) } } else if c.kind == ROUTER || c.kind == GATEWAY { c.Noticef("%s connection closed: %s", c.kindString(), reason) } else { // Client, System, Jetstream, and Account connections. c.Debugf("%s connection closed: %s", c.kindString(), reason) } } // Save off the connection if its a client or leafnode. if c.kind == CLIENT || c.kind == LEAF { if nc := c.nc; nc != nil && c.srv != nil { // TODO: May want to send events to single go routine instead // of creating a new go routine for each save. // Pass the c.subs as a reference. It may be set to nil in // closeConnection. go c.srv.saveClosedClient(c, nc, c.subs, reason) } } // If writeLoop exists, let it do the final flush, close and teardown. if c.flags.isSet(writeLoopStarted) { // Since we want the writeLoop to do the final flush and tcp close, // we want the reconnect to be done there too. However, it should'nt // happen before the connection has been removed from the server // state (end of closeConnection()). This ref count allows us to // guarantee that. c.rref++ c.flushSignal() return } // Flush (if skipFlushOnClose is not set) and close in place. If flushing, // use a small WriteDeadline. c.flushAndClose(true) } // flushSignal will use server to queue the flush IO operation to a pool of flushers. // Lock must be held. func (c *client) flushSignal() { // Check that sg is not nil, which will happen if the connection is closed. if c.out.sg != nil { c.out.sg.Signal() } } // Traces a message. // Will NOT check if tracing is enabled, does NOT need the client lock. func (c *client) traceMsg(msg []byte) { maxTrace := c.srv.getOpts().MaxTracedMsgLen if maxTrace > 0 && (len(msg)-LEN_CR_LF) > maxTrace { tm := fmt.Sprintf("%q", msg[:maxTrace]) c.Tracef("<<- MSG_PAYLOAD: [\"%s...\"]", tm[1:maxTrace+1]) } else { c.Tracef("<<- MSG_PAYLOAD: [%q]", msg[:len(msg)-LEN_CR_LF]) } } // Traces an incoming operation. // Will NOT check if tracing is enabled, does NOT need the client lock. func (c *client) traceInOp(op string, arg []byte) { c.traceOp("<<- %s", op, arg) } // Traces an outgoing operation. // Will NOT check if tracing is enabled, does NOT need the client lock. func (c *client) traceOutOp(op string, arg []byte) { c.traceOp("->> %s", op, arg) } func (c *client) traceOp(format, op string, arg []byte) { opa := []any{} if op != _EMPTY_ { opa = append(opa, op) } if arg != nil { opa = append(opa, bytesToString(arg)) } c.Tracef(format, opa) } // Process the information messages from Clients and other Routes. func (c *client) processInfo(arg []byte) error { info := Info{} if err := json.Unmarshal(arg, &info); err != nil { return err } switch c.kind { case ROUTER: c.processRouteInfo(&info) case GATEWAY: c.processGatewayInfo(&info) case LEAF: c.processLeafnodeInfo(&info) } return nil } func (c *client) processErr(errStr string) { close := true switch c.kind { case CLIENT: c.Errorf("Client Error %s", errStr) case ROUTER: c.Errorf("Route Error %s", errStr) case GATEWAY: c.Errorf("Gateway Error %s", errStr) case LEAF: c.Errorf("Leafnode Error %s", errStr) c.leafProcessErr(errStr) close = false case JETSTREAM: c.Errorf("JetStream Error %s", errStr) } if close { c.closeConnection(ParseError) } } // Password pattern matcher. var passPat = regexp.MustCompile(`"?\s*pass\S*?"?\s*[:=]\s*"?(([^",\r\n}])*)`) // removePassFromTrace removes any notion of passwords from trace // messages for logging. func removePassFromTrace(arg []byte) []byte { if !bytes.Contains(arg, []byte(`pass`)) { return arg } // Take a copy of the connect proto just for the trace message. var _arg [4096]byte buf := append(_arg[:0], arg...) m := passPat.FindAllSubmatchIndex(buf, -1) if len(m) == 0 { return arg } redactedPass := []byte("[REDACTED]") for _, i := range m { if len(i) < 4 { continue } start := i[2] end := i[3] // Replace password substring. buf = append(buf[:start], append(redactedPass, buf[end:]...)...) break } return buf } // Returns the RTT by computing the elapsed time since now and `start`. // On Windows VM where I (IK) run tests, time.Since() will return 0 // (I suspect some time granularity issues). So return at minimum 1ns. func computeRTT(start time.Time) time.Duration { rtt := time.Since(start) if rtt <= 0 { rtt = time.Nanosecond } return rtt } // processConnect will process a client connect op. func (c *client) processConnect(arg []byte) error { supportsHeaders := c.srv.supportsHeaders() c.mu.Lock() // If we can't stop the timer because the callback is in progress... if !c.clearAuthTimer() { // wait for it to finish and handle sending the failure back to // the client. for !c.isClosed() { c.mu.Unlock() time.Sleep(25 * time.Millisecond) c.mu.Lock() } c.mu.Unlock() return nil } c.last = time.Now().UTC() // Estimate RTT to start. if c.kind == CLIENT { c.rtt = computeRTT(c.start) if c.srv != nil { c.clearPingTimer() c.setFirstPingTimer() } } kind := c.kind srv := c.srv // Moved unmarshalling of clients' Options under the lock. // The client has already been added to the server map, so it is possible // that other routines lookup the client, and access its options under // the client's lock, so unmarshalling the options outside of the lock // would cause data RACEs. if err := json.Unmarshal(arg, &c.opts); err != nil { c.mu.Unlock() return err } // Indicate that the CONNECT protocol has been received, and that the // server now knows which protocol this client supports. c.flags.set(connectReceived) // Capture these under lock c.echo = c.opts.Echo proto := c.opts.Protocol verbose := c.opts.Verbose lang := c.opts.Lang account := c.opts.Account accountNew := c.opts.AccountNew if c.kind == CLIENT { var ncs string if c.opts.Version != _EMPTY_ { ncs = fmt.Sprintf("v%s", c.opts.Version) } if c.opts.Lang != _EMPTY_ { if c.opts.Version == _EMPTY_ { ncs = c.opts.Lang } else { ncs = fmt.Sprintf("%s:%s", ncs, c.opts.Lang) } } if c.opts.Name != _EMPTY_ { if c.opts.Version == _EMPTY_ && c.opts.Lang == _EMPTY_ { ncs = c.opts.Name } else { ncs = fmt.Sprintf("%s:%s", ncs, c.opts.Name) } } if ncs != _EMPTY_ { c.ncs.CompareAndSwap(nil, fmt.Sprintf("%s - %q", c, ncs)) } } // If websocket client and JWT not in the CONNECT, use the cookie JWT (possibly empty). if ws := c.ws; ws != nil && c.opts.JWT == "" { c.opts.JWT = ws.cookieJwt } // when not in operator mode, discard the jwt if srv != nil && srv.trustedKeys == nil { c.opts.JWT = _EMPTY_ } ujwt := c.opts.JWT // For headers both client and server need to support. c.headers = supportsHeaders && c.opts.Headers c.mu.Unlock() if srv != nil { // Applicable to clients only: // As soon as c.opts is unmarshalled and if the proto is at // least ClientProtoInfo, we need to increment the following counter. // This is decremented when client is removed from the server's // clients map. if kind == CLIENT && proto >= ClientProtoInfo { srv.mu.Lock() srv.cproto++ srv.mu.Unlock() } // Check for Auth if ok := srv.checkAuthentication(c); !ok { // We may fail here because we reached max limits on an account. if ujwt != _EMPTY_ { c.mu.Lock() acc := c.acc c.mu.Unlock() srv.mu.Lock() tooManyAccCons := acc != nil && acc != srv.gacc srv.mu.Unlock() if tooManyAccCons { return ErrTooManyAccountConnections } } c.authViolation() return ErrAuthentication } // Check for Account designation, we used to have this as an optional feature for dynamic // sandbox environments. Now its considered an error. if accountNew || account != _EMPTY_ { c.authViolation() return ErrAuthentication } // If no account designation. // Do this only for CLIENT and LEAF connections. if c.acc == nil && (c.kind == CLIENT || c.kind == LEAF) { // By default register with the global account. c.registerWithAccount(srv.globalAccount()) } } switch kind { case CLIENT: // Check client protocol request if it exists. if proto < ClientProtoZero || proto > ClientProtoInfo { c.sendErr(ErrBadClientProtocol.Error()) c.closeConnection(BadClientProtocolVersion) return ErrBadClientProtocol } // Check to see that if no_responders is requested // they have header support on as well. c.mu.Lock() misMatch := c.opts.NoResponders && !c.headers c.mu.Unlock() if misMatch { c.sendErr(ErrNoRespondersRequiresHeaders.Error()) c.closeConnection(NoRespondersRequiresHeaders) return ErrNoRespondersRequiresHeaders } if verbose { c.sendOK() } case ROUTER: // Delegate the rest of processing to the route return c.processRouteConnect(srv, arg, lang) case GATEWAY: // Delegate the rest of processing to the gateway return c.processGatewayConnect(arg) case LEAF: // Delegate the rest of processing to the leaf node return c.processLeafNodeConnect(srv, arg, lang) } return nil } func (c *client) sendErrAndErr(err string) { c.sendErr(err) c.Errorf(err) } func (c *client) sendErrAndDebug(err string) { c.sendErr(err) c.Debugf(err) } func (c *client) authTimeout() { c.sendErrAndDebug("Authentication Timeout") c.closeConnection(AuthenticationTimeout) } func (c *client) authExpired() { c.sendErrAndDebug("User Authentication Expired") c.closeConnection(AuthenticationExpired) } func (c *client) accountAuthExpired() { c.sendErrAndDebug("Account Authentication Expired") c.closeConnection(AuthenticationExpired) } func (c *client) authViolation() { var s *Server var hasTrustedNkeys, hasNkeys, hasUsers bool if s = c.srv; s != nil { s.mu.RLock() hasTrustedNkeys = s.trustedKeys != nil hasNkeys = s.nkeys != nil hasUsers = s.users != nil s.mu.RUnlock() defer s.sendAuthErrorEvent(c) } if hasTrustedNkeys { c.Errorf("%v", ErrAuthentication) } else if hasNkeys { c.Errorf("%s - Nkey %q", ErrAuthentication.Error(), c.opts.Nkey) } else if hasUsers { c.Errorf("%s - User %q", ErrAuthentication.Error(), c.opts.Username) } else { if c.srv != nil { c.Errorf(ErrAuthentication.Error()) } } if c.isMqtt() { c.mqttEnqueueConnAck(mqttConnAckRCNotAuthorized, false) } else { c.sendErr("Authorization Violation") } c.closeConnection(AuthenticationViolation) } func (c *client) maxAccountConnExceeded() { c.sendErrAndErr(ErrTooManyAccountConnections.Error()) c.closeConnection(MaxAccountConnectionsExceeded) } func (c *client) maxConnExceeded() { c.sendErrAndErr(ErrTooManyConnections.Error()) c.closeConnection(MaxConnectionsExceeded) } func (c *client) maxSubsExceeded() { if c.acc.shouldLogMaxSubErr() { c.Errorf(ErrTooManySubs.Error()) } c.sendErr(ErrTooManySubs.Error()) } func (c *client) maxPayloadViolation(sz int, max int32) { c.Errorf("%s: %d vs %d", ErrMaxPayload.Error(), sz, max) c.sendErr("Maximum Payload Violation") c.closeConnection(MaxPayloadExceeded) } // queueOutbound queues data for a clientconnection. // Lock should be held. func (c *client) queueOutbound(data []byte) { // Do not keep going if closed if c.isClosed() { return } // Add to pending bytes total. c.out.pb += int64(len(data)) // Take a copy of the slice ref so that we can chop bits off the beginning // without affecting the original "data" slice. toBuffer := data // All of the queued []byte have a fixed capacity, so if there's a []byte // at the tail of the buffer list that isn't full yet, we should top that // up first. This helps to ensure we aren't pulling more []bytes from the // pool than we need to. if len(c.out.nb) > 0 { last := &c.out.nb[len(c.out.nb)-1] if free := cap(*last) - len(*last); free > 0 { if l := len(toBuffer); l < free { free = l } *last = append(*last, toBuffer[:free]...) toBuffer = toBuffer[free:] } } // Now we can push the rest of the data into new []bytes from the pool // in fixed size chunks. This ensures we don't go over the capacity of any // of the buffers and end up reallocating. for len(toBuffer) > 0 { new := nbPoolGet(len(toBuffer)) n := copy(new[:cap(new)], toBuffer) c.out.nb = append(c.out.nb, new[:n]) toBuffer = toBuffer[n:] } // Check for slow consumer via pending bytes limit. // ok to return here, client is going away. if c.kind == CLIENT && c.out.pb > c.out.mp { // Perf wise, it looks like it is faster to optimistically add than // checking current pb+len(data) and then add to pb. c.out.pb -= int64(len(data)) // Increment the total and client's slow consumer counters. atomic.AddInt64(&c.srv.slowConsumers, 1) c.srv.scStats.clients.Add(1) if c.acc != nil { atomic.AddInt64(&c.acc.slowConsumers, 1) } c.Noticef("Slow Consumer Detected: MaxPending of %d Exceeded", c.out.mp) c.markConnAsClosed(SlowConsumerPendingBytes) return } // Check here if we should create a stall channel if we are falling behind. // We do this here since if we wait for consumer's writeLoop it could be // too late with large number of fan in producers. // If the outbound connection is > 75% of maximum pending allowed, create a stall gate. if c.out.pb > c.out.mp/4*3 && c.out.stc == nil { c.out.stc = make(chan struct{}) } } // Assume the lock is held upon entry. func (c *client) enqueueProtoAndFlush(proto []byte, doFlush bool) { if c.isClosed() { return } c.queueOutbound(proto) if !(doFlush && c.flushOutbound()) { c.flushSignal() } } // Queues and then flushes the connection. This should only be called when // the writeLoop cannot be started yet. Use enqueueProto() otherwise. // Lock is held on entry. func (c *client) sendProtoNow(proto []byte) { c.enqueueProtoAndFlush(proto, true) } // Enqueues the given protocol and signal the writeLoop if necessary. // Lock is held on entry. func (c *client) enqueueProto(proto []byte) { c.enqueueProtoAndFlush(proto, false) } // Assume the lock is held upon entry. func (c *client) sendPong() { if c.trace { c.traceOutOp("PONG", nil) } c.enqueueProto([]byte(pongProto)) } // Used to kick off a RTT measurement for latency tracking. func (c *client) sendRTTPing() bool { c.mu.Lock() sent := c.sendRTTPingLocked() c.mu.Unlock() return sent } // Used to kick off a RTT measurement for latency tracking. // This is normally called only when the caller has checked that // the c.rtt is 0 and wants to force an update by sending a PING. // Client lock held on entry. func (c *client) sendRTTPingLocked() bool { if c.isMqtt() { return false } // Most client libs send a CONNECT+PING and wait for a PONG from the // server. So if firstPongSent flag is set, it is ok for server to // send the PING. But in case we have client libs that don't do that, // allow the send of the PING if more than 2 secs have elapsed since // the client TCP connection was accepted. if !c.isClosed() && (c.flags.isSet(firstPongSent) || time.Since(c.start) > maxNoRTTPingBeforeFirstPong) { c.sendPing() return true } return false } // Assume the lock is held upon entry. func (c *client) sendPing() { c.rttStart = time.Now().UTC() c.ping.out++ if c.trace { c.traceOutOp("PING", nil) } c.enqueueProto([]byte(pingProto)) } // Generates the INFO to be sent to the client with the client ID included. // info arg will be copied since passed by value. // Assume lock is held. func (c *client) generateClientInfoJSON(info Info) []byte { info.CID = c.cid info.ClientIP = c.host info.MaxPayload = c.mpay if c.isWebsocket() { info.ClientConnectURLs = info.WSConnectURLs // Otherwise lame duck info can panic if c.srv != nil { ws := &c.srv.websocket info.TLSAvailable, info.TLSRequired = ws.tls, ws.tls info.Host, info.Port = ws.host, ws.port } } info.WSConnectURLs = nil return generateInfoJSON(&info) } func (c *client) sendErr(err string) { c.mu.Lock() if c.trace { c.traceOutOp("-ERR", []byte(err)) } if !c.isMqtt() { c.enqueueProto([]byte(fmt.Sprintf(errProto, err))) } c.mu.Unlock() } func (c *client) sendOK() { c.mu.Lock() if c.trace { c.traceOutOp("OK", nil) } c.enqueueProto([]byte(okProto)) c.mu.Unlock() } func (c *client) processPing() { c.mu.Lock() if c.isClosed() { c.mu.Unlock() return } c.sendPong() // Record this to suppress us sending one if this // is within a given time interval for activity. c.lastIn = time.Now() // If not a CLIENT, we are done. Also the CONNECT should // have been received, but make sure it is so before proceeding if c.kind != CLIENT || !c.flags.isSet(connectReceived) { c.mu.Unlock() return } // If we are here, the CONNECT has been received so we know // if this client supports async INFO or not. var ( checkInfoChange bool srv = c.srv ) // For older clients, just flip the firstPongSent flag if not already // set and we are done. if c.opts.Protocol < ClientProtoInfo || srv == nil { c.flags.setIfNotSet(firstPongSent) } else { // This is a client that supports async INFO protocols. // If this is the first PING (so firstPongSent is not set yet), // we will need to check if there was a change in cluster topology // or we have a different max payload. We will send this first before // pong since most clients do flush after connect call. checkInfoChange = !c.flags.isSet(firstPongSent) } c.mu.Unlock() if checkInfoChange { opts := srv.getOpts() srv.mu.Lock() c.mu.Lock() // Now that we are under both locks, we can flip the flag. // This prevents sendAsyncInfoToClients() and code here to // send a double INFO protocol. c.flags.set(firstPongSent) // If there was a cluster update since this client was created, // send an updated INFO protocol now. if srv.lastCURLsUpdate >= c.start.UnixNano() || c.mpay != int32(opts.MaxPayload) { c.enqueueProto(c.generateClientInfoJSON(srv.copyInfo())) } c.mu.Unlock() srv.mu.Unlock() } } func (c *client) processPong() { c.mu.Lock() c.ping.out = 0 c.rtt = computeRTT(c.rttStart) srv := c.srv reorderGWs := c.kind == GATEWAY && c.gw.outbound // If compression is currently active for a route/leaf connection, if the // compression configuration is s2_auto, check if we should change // the compression level. if c.kind == ROUTER && needsCompression(c.route.compression) { c.updateS2AutoCompressionLevel(&srv.getOpts().Cluster.Compression, &c.route.compression) } else if c.kind == LEAF && needsCompression(c.leaf.compression) { var co *CompressionOpts if r := c.leaf.remote; r != nil { co = &r.Compression } else { co = &srv.getOpts().LeafNode.Compression } c.updateS2AutoCompressionLevel(co, &c.leaf.compression) } c.mu.Unlock() if reorderGWs { srv.gateway.orderOutboundConnections() } } // Select the s2 compression level based on the client's current RTT and the configured // RTT thresholds slice. If current level is different than selected one, save the // new compression level string and create a new s2 writer. // Lock held on entry. func (c *client) updateS2AutoCompressionLevel(co *CompressionOpts, compression *string) { if co.Mode != CompressionS2Auto { return } if cm := selectS2AutoModeBasedOnRTT(c.rtt, co.RTTThresholds); cm != *compression { *compression = cm c.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...) } } // Will return the parts from the raw wire msg. func (c *client) msgParts(data []byte) (hdr []byte, msg []byte) { if c != nil && c.pa.hdr > 0 { return data[:c.pa.hdr], data[c.pa.hdr:] } return nil, data } // Header pubs take form HPUB [reply] \r\n func (c *client) processHeaderPub(arg []byte) error { if !c.headers { return ErrMsgHeadersNotSupported } // Unroll splitArgs to avoid runtime/heap issues a := [MAX_HPUB_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 3: c.pa.subject = args[0] c.pa.reply = nil c.pa.hdr = parseSize(args[1]) c.pa.size = parseSize(args[2]) c.pa.hdb = args[1] c.pa.szb = args[2] case 4: c.pa.subject = args[0] c.pa.reply = args[1] c.pa.hdr = parseSize(args[2]) c.pa.size = parseSize(args[3]) c.pa.hdb = args[2] c.pa.szb = args[3] default: return fmt.Errorf("processHeaderPub Parse Error: %q", arg) } if c.pa.hdr < 0 { return fmt.Errorf("processHeaderPub Bad or Missing Header Size: %q", arg) } // If number overruns an int64, parseSize() will have returned a negative value if c.pa.size < 0 { return fmt.Errorf("processHeaderPub Bad or Missing Total Size: %q", arg) } if c.pa.hdr > c.pa.size { return fmt.Errorf("processHeaderPub Header Size larger then TotalSize: %q", arg) } maxPayload := atomic.LoadInt32(&c.mpay) // Use int64() to avoid int32 overrun... if maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) { c.maxPayloadViolation(c.pa.size, maxPayload) return ErrMaxPayload } if c.opts.Pedantic && !IsValidLiteralSubject(bytesToString(c.pa.subject)) { c.sendErr("Invalid Publish Subject") } return nil } func (c *client) processPub(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_PUB_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 2: c.pa.subject = args[0] c.pa.reply = nil c.pa.size = parseSize(args[1]) c.pa.szb = args[1] case 3: c.pa.subject = args[0] c.pa.reply = args[1] c.pa.size = parseSize(args[2]) c.pa.szb = args[2] default: return fmt.Errorf("processPub Parse Error: %q", arg) } // If number overruns an int64, parseSize() will have returned a negative value if c.pa.size < 0 { return fmt.Errorf("processPub Bad or Missing Size: %q", arg) } maxPayload := atomic.LoadInt32(&c.mpay) // Use int64() to avoid int32 overrun... if maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) { c.maxPayloadViolation(c.pa.size, maxPayload) return ErrMaxPayload } if c.opts.Pedantic && !IsValidLiteralSubject(bytesToString(c.pa.subject)) { c.sendErr("Invalid Publish Subject") } return nil } func splitArg(arg []byte) [][]byte { a := [MAX_MSG_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } return args } func (c *client) parseSub(argo []byte, noForward bool) error { // Copy so we do not reference a potentially large buffer // FIXME(dlc) - make more efficient. arg := make([]byte, len(argo)) copy(arg, argo) args := splitArg(arg) var ( subject []byte queue []byte sid []byte ) switch len(args) { case 2: subject = args[0] queue = nil sid = args[1] case 3: subject = args[0] queue = args[1] sid = args[2] default: return fmt.Errorf("processSub Parse Error: %q", arg) } // If there was an error, it has been sent to the client. We don't return an // error here to not close the connection as a parsing error. c.processSub(subject, queue, sid, nil, noForward) return nil } func (c *client) processSub(subject, queue, bsid []byte, cb msgHandler, noForward bool) (*subscription, error) { return c.processSubEx(subject, queue, bsid, cb, noForward, false, false) } func (c *client) processSubEx(subject, queue, bsid []byte, cb msgHandler, noForward, si, rsi bool) (*subscription, error) { // Create the subscription sub := &subscription{client: c, subject: subject, queue: queue, sid: bsid, icb: cb, si: si, rsi: rsi} c.mu.Lock() // Indicate activity. c.in.subs++ // Grab connection type, account and server info. kind := c.kind acc := c.acc srv := c.srv sid := bytesToString(sub.sid) // This check does not apply to SYSTEM or JETSTREAM or ACCOUNT clients (because they don't have a `nc`...) // When a connection is closed though, we set c.subs to nil. So check for the map to not be nil. if (c.isClosed() && (kind != SYSTEM && kind != JETSTREAM && kind != ACCOUNT)) || (c.subs == nil) { c.mu.Unlock() return nil, ErrConnectionClosed } // Check permissions if applicable. if kind == CLIENT { // First do a pass whether queue subscription is valid. This does not necessarily // mean that it will not be able to plain subscribe. // // allow = ["foo"] -> can subscribe or queue subscribe to foo using any queue // allow = ["foo v1"] -> can only queue subscribe to 'foo v1', no plain subs allowed. // allow = ["foo", "foo v1"] -> can subscribe to 'foo' but can only queue subscribe to 'foo v1' // if sub.queue != nil { if !c.canSubscribe(string(sub.subject), string(sub.queue)) || string(sub.queue) == sysGroup { c.mu.Unlock() c.subPermissionViolation(sub) return nil, ErrSubscribePermissionViolation } } else if !c.canSubscribe(string(sub.subject)) { c.mu.Unlock() c.subPermissionViolation(sub) return nil, ErrSubscribePermissionViolation } if opts := srv.getOpts(); opts != nil && opts.MaxSubTokens > 0 { if len(bytes.Split(sub.subject, []byte(tsep))) > int(opts.MaxSubTokens) { c.mu.Unlock() c.maxTokensViolation(sub) return nil, ErrTooManySubTokens } } } // Check if we have a maximum on the number of subscriptions. if c.subsAtLimit() { c.mu.Unlock() c.maxSubsExceeded() return nil, ErrTooManySubs } var updateGWs bool var err error // Subscribe here. es := c.subs[sid] if es == nil { c.subs[sid] = sub if acc != nil && acc.sl != nil { err = acc.sl.Insert(sub) if err != nil { delete(c.subs, sid) } else { updateGWs = c.srv.gateway.enabled } } } // Unlocked from here onward c.mu.Unlock() if err != nil { c.sendErr("Invalid Subject") return nil, ErrMalformedSubject } else if c.opts.Verbose && kind != SYSTEM { c.sendOK() } // If it was already registered, return it. if es != nil { return es, nil } // No account just return. if acc == nil { return sub, nil } if err := c.addShadowSubscriptions(acc, sub, true); err != nil { c.Errorf(err.Error()) } if noForward { return sub, nil } // If we are routing and this is a local sub, add to the route map for the associated account. if kind == CLIENT || kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT { srv.updateRouteSubscriptionMap(acc, sub, 1) if updateGWs { srv.gatewayUpdateSubInterest(acc.Name, sub, 1) } } // Now check on leafnode updates. acc.updateLeafNodes(sub, 1) return sub, nil } // Used to pass stream import matches to addShadowSub type ime struct { im *streamImport overlapSubj string dyn bool } // If the client's account has stream imports and there are matches for this // subscription's subject, then add shadow subscriptions in the other accounts // that export this subject. // // enact=false allows MQTT clients to get the list of shadow subscriptions // without enacting them, in order to first obtain matching "retained" messages. func (c *client) addShadowSubscriptions(acc *Account, sub *subscription, enact bool) error { if acc == nil { return ErrMissingAccount } var ( _ims [16]ime ims = _ims[:0] imTsa [32]string tokens []string tsa [32]string hasWC bool tokensModified bool ) acc.mu.RLock() // If this is from a service import, ignore. if sub.si { acc.mu.RUnlock() return nil } subj := bytesToString(sub.subject) if len(acc.imports.streams) > 0 { tokens = tokenizeSubjectIntoSlice(tsa[:0], subj) for _, tk := range tokens { if tk == pwcs { hasWC = true break } } if !hasWC && tokens[len(tokens)-1] == fwcs { hasWC = true } } // Loop over the import subjects. We have 4 scenarios. If we have an // exact match or a superset match we should use the from field from // the import. If we are a subset or overlap, we have to dynamically calculate // the subject. On overlap, ime requires the overlap subject. for _, im := range acc.imports.streams { if im.invalid { continue } if subj == im.to { ims = append(ims, ime{im, _EMPTY_, false}) continue } if tokensModified { // re-tokenize subj to overwrite modifications from a previous iteration tokens = tokenizeSubjectIntoSlice(tsa[:0], subj) tokensModified = false } imTokens := tokenizeSubjectIntoSlice(imTsa[:0], im.to) if isSubsetMatchTokenized(tokens, imTokens) { ims = append(ims, ime{im, _EMPTY_, true}) } else if hasWC { if isSubsetMatchTokenized(imTokens, tokens) { ims = append(ims, ime{im, _EMPTY_, false}) } else { imTokensLen := len(imTokens) for i, t := range tokens { if i >= imTokensLen { break } if t == pwcs && imTokens[i] != fwcs { tokens[i] = imTokens[i] tokensModified = true } } tokensLen := len(tokens) lastIdx := tokensLen - 1 if tokens[lastIdx] == fwcs { if imTokensLen >= tokensLen { // rewrite ">" in tokens to be more specific tokens[lastIdx] = imTokens[lastIdx] tokensModified = true if imTokensLen > tokensLen { // copy even more specific parts from import tokens = append(tokens, imTokens[tokensLen:]...) } } } if isSubsetMatchTokenized(tokens, imTokens) { // As isSubsetMatchTokenized was already called with tokens and imTokens, // we wouldn't be here if it where not for tokens being modified. // Hence, Join to re compute the subject string ims = append(ims, ime{im, strings.Join(tokens, tsep), true}) } } } } acc.mu.RUnlock() var shadow []*subscription if len(ims) > 0 { shadow = make([]*subscription, 0, len(ims)) } // Now walk through collected stream imports that matched. for i := 0; i < len(ims); i++ { ime := &ims[i] // We will create a shadow subscription. nsub, err := c.addShadowSub(sub, ime, enact) if err != nil { return err } shadow = append(shadow, nsub) } if shadow != nil { c.mu.Lock() sub.shadow = shadow c.mu.Unlock() } return nil } // Add in the shadow subscription. func (c *client) addShadowSub(sub *subscription, ime *ime, enact bool) (*subscription, error) { c.mu.Lock() nsub := *sub // copy c.mu.Unlock() im := ime.im nsub.im = im if !im.usePub && ime.dyn && im.tr != nil { if im.rtr == nil { im.rtr = im.tr.reverse() } s := bytesToString(nsub.subject) if ime.overlapSubj != _EMPTY_ { s = ime.overlapSubj } subj := im.rtr.TransformSubject(s) nsub.subject = []byte(subj) } else if !im.usePub || (im.usePub && ime.overlapSubj != _EMPTY_) || !ime.dyn { if ime.overlapSubj != _EMPTY_ { nsub.subject = []byte(ime.overlapSubj) } else { nsub.subject = []byte(im.from) } } // Else use original subject if !enact { return &nsub, nil } c.Debugf("Creating import subscription on %q from account %q", nsub.subject, im.acc.Name) if err := im.acc.sl.Insert(&nsub); err != nil { errs := fmt.Sprintf("Could not add shadow import subscription for account %q", im.acc.Name) c.Debugf(errs) return nil, errors.New(errs) } // Update our route map here. But only if we are not a leaf node or a hub leafnode. if c.kind != LEAF || c.isHubLeafNode() { c.srv.updateRemoteSubscription(im.acc, &nsub, 1) } return &nsub, nil } // canSubscribe determines if the client is authorized to subscribe to the // given subject. Assumes caller is holding lock. func (c *client) canSubscribe(subject string, optQueue ...string) bool { if c.perms == nil { return true } allowed := true // Optional queue group. var queue string if len(optQueue) > 0 { queue = optQueue[0] } // Check allow list. If no allow list that means all are allowed. Deny can overrule. if c.perms.sub.allow != nil { r := c.perms.sub.allow.Match(subject) allowed = len(r.psubs) > 0 if queue != _EMPTY_ && len(r.qsubs) > 0 { // If the queue appears in the allow list, then DO allow. allowed = queueMatches(queue, r.qsubs) } // Leafnodes operate slightly differently in that they allow broader scoped subjects. // They will prune based on publish perms before sending to a leafnode client. if !allowed && c.kind == LEAF && subjectHasWildcard(subject) { r := c.perms.sub.allow.ReverseMatch(subject) allowed = len(r.psubs) != 0 } } // If we have a deny list and we think we are allowed, check that as well. if allowed && c.perms.sub.deny != nil { r := c.perms.sub.deny.Match(subject) allowed = len(r.psubs) == 0 if queue != _EMPTY_ && len(r.qsubs) > 0 { // If the queue appears in the deny list, then DO NOT allow. allowed = !queueMatches(queue, r.qsubs) } // We use the actual subscription to signal us to spin up the deny mperms // and cache. We check if the subject is a wildcard that contains any of // the deny clauses. // FIXME(dlc) - We could be smarter and track when these go away and remove. if allowed && c.mperms == nil && subjectHasWildcard(subject) { // Whip through the deny array and check if this wildcard subject is within scope. for _, sub := range c.darray { if subjectIsSubsetMatch(sub, subject) { c.loadMsgDenyFilter() break } } } } return allowed } func queueMatches(queue string, qsubs [][]*subscription) bool { if len(qsubs) == 0 { return true } for _, qsub := range qsubs { qs := qsub[0] qname := bytesToString(qs.queue) // NOTE: '*' and '>' tokens can also be valid // queue names so we first check against the // literal name. e.g. v1.* == v1.* if queue == qname || (subjectHasWildcard(qname) && subjectIsSubsetMatch(queue, qname)) { return true } } return false } // Low level unsubscribe for a given client. func (c *client) unsubscribe(acc *Account, sub *subscription, force, remove bool) { if s := c.srv; s != nil && s.isShuttingDown() { return } c.mu.Lock() if !force && sub.max > 0 && sub.nm < sub.max { c.Debugf("Deferring actual UNSUB(%s): %d max, %d received", sub.subject, sub.max, sub.nm) c.mu.Unlock() return } if c.trace { c.traceOp("<-> %s", "DELSUB", sub.sid) } // Remove accounting if requested. This will be false when we close a connection // with open subscriptions. if remove { delete(c.subs, bytesToString(sub.sid)) if acc != nil { acc.sl.Remove(sub) } } // Check to see if we have shadow subscriptions. var updateRoute bool var updateGWs bool shadowSubs := sub.shadow sub.shadow = nil if len(shadowSubs) > 0 { updateRoute = (c.kind == CLIENT || c.kind == SYSTEM || c.kind == LEAF) && c.srv != nil if updateRoute { updateGWs = c.srv.gateway.enabled } } sub.close() c.mu.Unlock() // Process shadow subs if we have them. for _, nsub := range shadowSubs { if err := nsub.im.acc.sl.Remove(nsub); err != nil { c.Debugf("Could not remove shadow import subscription for account %q", nsub.im.acc.Name) } else { if updateRoute { c.srv.updateRouteSubscriptionMap(nsub.im.acc, nsub, -1) } if updateGWs { c.srv.gatewayUpdateSubInterest(nsub.im.acc.Name, nsub, -1) } } // Now check on leafnode updates. nsub.im.acc.updateLeafNodes(nsub, -1) } // Now check to see if this was part of a respMap entry for service imports. // We can skip subscriptions on reserved replies. if acc != nil && !isReservedReply(sub.subject) { acc.checkForReverseEntry(string(sub.subject), nil, true) } } func (c *client) processUnsub(arg []byte) error { args := splitArg(arg) var sid []byte max := int64(-1) switch len(args) { case 1: sid = args[0] case 2: sid = args[0] max = int64(parseSize(args[1])) default: return fmt.Errorf("processUnsub Parse Error: %q", arg) } var sub *subscription var ok, unsub bool c.mu.Lock() // Indicate activity. c.in.subs++ // Grab connection type. kind := c.kind srv := c.srv var acc *Account updateGWs := false if sub, ok = c.subs[string(sid)]; ok { acc = c.acc if max > 0 && max > sub.nm { sub.max = max } else { // Clear it here to override sub.max = 0 unsub = true } updateGWs = srv.gateway.enabled } c.mu.Unlock() if c.opts.Verbose { c.sendOK() } if unsub { c.unsubscribe(acc, sub, false, true) if acc != nil && (kind == CLIENT || kind == SYSTEM || kind == ACCOUNT || kind == JETSTREAM) { srv.updateRouteSubscriptionMap(acc, sub, -1) if updateGWs { srv.gatewayUpdateSubInterest(acc.Name, sub, -1) } } // Now check on leafnode updates. acc.updateLeafNodes(sub, -1) } return nil } // checkDenySub will check if we are allowed to deliver this message in the // presence of deny clauses for subscriptions. Deny clauses will not prevent // larger scoped wildcard subscriptions, so we need to check at delivery time. // Lock should be held. func (c *client) checkDenySub(subject string) bool { if denied, ok := c.mperms.dcache[subject]; ok { return denied } else if np, _ := c.mperms.deny.NumInterest(subject); np != 0 { c.mperms.dcache[subject] = true return true } else { c.mperms.dcache[subject] = false } if len(c.mperms.dcache) > maxDenyPermCacheSize { c.pruneDenyCache() } return false } // Create a message header for routes or leafnodes. Header and origin cluster aware. func (c *client) msgHeaderForRouteOrLeaf(subj, reply []byte, rt *routeTarget, acc *Account) []byte { hasHeader := c.pa.hdr > 0 subclient := rt.sub.client canReceiveHeader := subclient.headers mh := c.msgb[:msgHeadProtoLen] kind := subclient.kind var lnoc bool if kind == ROUTER { // If we are coming from a leaf with an origin cluster we need to handle differently // if we can. We will send a route based LMSG which has origin cluster and headers // by default. if c.kind == LEAF && c.remoteCluster() != _EMPTY_ { subclient.mu.Lock() lnoc = subclient.route.lnoc subclient.mu.Unlock() } if lnoc { mh[0] = 'L' mh = append(mh, c.remoteCluster()...) mh = append(mh, ' ') } else { // Router (and Gateway) nodes are RMSG. Set here since leafnodes may rewrite. mh[0] = 'R' } if len(subclient.route.accName) == 0 { mh = append(mh, acc.Name...) mh = append(mh, ' ') } } else { // Leaf nodes are LMSG mh[0] = 'L' // Remap subject if its a shadow subscription, treat like a normal client. if rt.sub.im != nil { if rt.sub.im.tr != nil { to := rt.sub.im.tr.TransformSubject(bytesToString(subj)) subj = []byte(to) } else if !rt.sub.im.usePub { subj = []byte(rt.sub.im.to) } } } mh = append(mh, subj...) mh = append(mh, ' ') if len(rt.qs) > 0 { if len(reply) > 0 { mh = append(mh, "+ "...) // Signal that there is a reply. mh = append(mh, reply...) mh = append(mh, ' ') } else { mh = append(mh, "| "...) // Only queues } mh = append(mh, rt.qs...) } else if len(reply) > 0 { mh = append(mh, reply...) mh = append(mh, ' ') } if lnoc { // leafnode origin LMSG always have a header entry even if zero. if c.pa.hdr <= 0 { mh = append(mh, '0') } else { mh = append(mh, c.pa.hdb...) } mh = append(mh, ' ') mh = append(mh, c.pa.szb...) } else if hasHeader { if canReceiveHeader { mh[0] = 'H' mh = append(mh, c.pa.hdb...) mh = append(mh, ' ') mh = append(mh, c.pa.szb...) } else { // If we are here we need to truncate the payload size nsz := strconv.Itoa(c.pa.size - c.pa.hdr) mh = append(mh, nsz...) } } else { mh = append(mh, c.pa.szb...) } return append(mh, _CRLF_...) } // Create a message header for clients. Header aware. func (c *client) msgHeader(subj, reply []byte, sub *subscription) []byte { // See if we should do headers. We have to have a headers msg and // the client we are going to deliver to needs to support headers as well. hasHeader := c.pa.hdr > 0 canReceiveHeader := sub.client != nil && sub.client.headers var mh []byte if hasHeader && canReceiveHeader { mh = c.msgb[:msgHeadProtoLen] mh[0] = 'H' } else { mh = c.msgb[1:msgHeadProtoLen] } mh = append(mh, subj...) mh = append(mh, ' ') if len(sub.sid) > 0 { mh = append(mh, sub.sid...) mh = append(mh, ' ') } if reply != nil { mh = append(mh, reply...) mh = append(mh, ' ') } if hasHeader { if canReceiveHeader { mh = append(mh, c.pa.hdb...) mh = append(mh, ' ') mh = append(mh, c.pa.szb...) } else { // If we are here we need to truncate the payload size nsz := strconv.Itoa(c.pa.size - c.pa.hdr) mh = append(mh, nsz...) } } else { mh = append(mh, c.pa.szb...) } mh = append(mh, _CRLF_...) return mh } func (c *client) stalledWait(producer *client) { // Check to see if we have exceeded our total wait time per readLoop invocation. if producer.in.tst > stallTotalAllowed { return } // Grab stall channel which the slow consumer will close when caught up. stall := c.out.stc // Calculate stall time. ttl := stallClientMinDuration if c.out.pb >= c.out.mp { ttl = stallClientMaxDuration } c.mu.Unlock() defer c.mu.Lock() // Now check if we are close to total allowed. if producer.in.tst+ttl > stallTotalAllowed { ttl = stallTotalAllowed - producer.in.tst } delay := time.NewTimer(ttl) defer delay.Stop() start := time.Now() select { case <-stall: case <-delay.C: producer.Debugf("Timed out of fast producer stall (%v)", ttl) } producer.in.tst += time.Since(start) } // Used to treat maps as efficient set var needFlush = struct{}{} // deliverMsg will deliver a message to a matching subscription and its underlying client. // We process all connection/client types. mh is the part that will be protocol/client specific. func (c *client) deliverMsg(prodIsMQTT bool, sub *subscription, acc *Account, subject, reply, mh, msg []byte, gwrply bool) bool { // Check sub client and check echo. Only do this if not a service import. if sub.client == nil || (c == sub.client && !sub.client.echo && !sub.si) { return false } client := sub.client client.mu.Lock() // Check if we have a subscribe deny clause. This will trigger us to check the subject // for a match against the denied subjects. if client.mperms != nil && client.checkDenySub(string(subject)) { client.mu.Unlock() return false } // New race detector forces this now. if sub.isClosed() { client.mu.Unlock() return false } // Check if we are a leafnode and have perms to check. if client.kind == LEAF && client.perms != nil { if !client.pubAllowedFullCheck(string(subject), true, true) { client.mu.Unlock() client.Debugf("Not permitted to deliver to %q", subject) return false } } srv := client.srv sub.nm++ // Check if we should auto-unsubscribe. if sub.max > 0 { if client.kind == ROUTER && sub.nm >= sub.max { // The only router based messages that we will see here are remoteReplies. // We handle these slightly differently. defer client.removeReplySub(sub) } else { // For routing.. shouldForward := client.kind == CLIENT || client.kind == SYSTEM && client.srv != nil // If we are at the exact number, unsubscribe but // still process the message in hand, otherwise // unsubscribe and drop message on the floor. if sub.nm == sub.max { client.Debugf("Auto-unsubscribe limit of %d reached for sid '%s'", sub.max, sub.sid) // Due to defer, reverse the code order so that execution // is consistent with other cases where we unsubscribe. if shouldForward { defer srv.updateRemoteSubscription(client.acc, sub, -1) } defer client.unsubscribe(client.acc, sub, true, true) } else if sub.nm > sub.max { client.Debugf("Auto-unsubscribe limit [%d] exceeded", sub.max) client.mu.Unlock() client.unsubscribe(client.acc, sub, true, true) if shouldForward { srv.updateRemoteSubscription(client.acc, sub, -1) } return false } } } // Check here if we have a header with our message. If this client can not // support we need to strip the headers from the payload. // The actual header would have been processed correctly for us, so just // need to update payload. if c.pa.hdr > 0 && !sub.client.headers { msg = msg[c.pa.hdr:] } // Update statistics // The msg includes the CR_LF, so pull back out for accounting. msgSize := int64(len(msg)) // MQTT producers send messages without CR_LF, so don't remove it for them. if !prodIsMQTT { msgSize -= int64(LEN_CR_LF) } // We do not update the outbound stats if we are doing trace only since // this message will not be sent out. // Also do not update on internal callbacks. if sub.icb == nil { // No atomic needed since accessed under client lock. // Monitor is reading those also under client's lock. client.outMsgs++ client.outBytes += msgSize } // Check for internal subscriptions. if sub.icb != nil && !c.noIcb { if gwrply { // We will store in the account, not the client since it will likely // be a different client that will send the reply. srv.trackGWReply(nil, client.acc, reply, c.pa.reply) } client.mu.Unlock() // For service imports, track if we delivered. didDeliver := true // Internal account clients are for service imports and need the '\r\n'. start := time.Now() if client.kind == ACCOUNT { sub.icb(sub, c, acc, string(subject), string(reply), msg) // If we are a service import check to make sure we delivered the message somewhere. if sub.si { didDeliver = c.pa.delivered } } else { sub.icb(sub, c, acc, string(subject), string(reply), msg[:msgSize]) } if dur := time.Since(start); dur >= readLoopReportThreshold { srv.Warnf("Internal subscription on %q took too long: %v", subject, dur) } return didDeliver } // If we are a client and we detect that the consumer we are // sending to is in a stalled state, go ahead and wait here // with a limit. if c.kind == CLIENT && client.out.stc != nil { if srv.getOpts().NoFastProducerStall { client.mu.Unlock() return false } client.stalledWait(c) } // Check for closed connection if client.isClosed() { client.mu.Unlock() return false } // Do a fast check here to see if we should be tracking this from a latency // perspective. This will be for a request being received for an exported service. // This needs to be from a non-client (otherwise tracking happens at requestor). // // Also this check captures if the original reply (c.pa.reply) is a GW routed // reply (since it is known to be > minReplyLen). If that is the case, we need to // track the binding between the routed reply and the reply set in the message // header (which is c.pa.reply without the GNR routing prefix). if client.kind == CLIENT && len(c.pa.reply) > minReplyLen { if gwrply { // Note that we keep track of the GW routed reply in the destination // connection (`client`). The routed reply subject is in `c.pa.reply`, // should that change, we would have to pass the GW routed reply as // a parameter of deliverMsg(). srv.trackGWReply(client, nil, reply, c.pa.reply) } // If we do not have a registered RTT queue that up now. if client.rtt == 0 { client.sendRTTPingLocked() } // FIXME(dlc) - We may need to optimize this. // We will have tagged this with a suffix ('.T') if we are tracking. This is // needed from sampling. Not all will be tracked. if c.kind != CLIENT && isTrackedReply(c.pa.reply) { client.trackRemoteReply(string(subject), string(c.pa.reply)) } } // Queue to outbound buffer client.queueOutbound(mh) client.queueOutbound(msg) if prodIsMQTT { // Need to add CR_LF since MQTT producers don't send CR_LF client.queueOutbound([]byte(CR_LF)) } // If we are tracking dynamic publish permissions that track reply subjects, // do that accounting here. We only look at client.replies which will be non-nil. // Only reply subject permissions if the client is not already allowed to publish to the reply subject. if client.replies != nil && len(reply) > 0 && !client.pubAllowedFullCheck(string(reply), true, true) { client.replies[string(reply)] = &resp{time.Now(), 0} client.repliesSincePrune++ if client.repliesSincePrune > replyPermLimit || time.Since(client.lastReplyPrune) > replyPruneTime { client.pruneReplyPerms() } } // Check outbound threshold and queue IO flush if needed. // This is specifically looking at situations where we are getting behind and may want // to intervene before this producer goes back to top of readloop. We are in the producer's // readloop go routine at this point. // FIXME(dlc) - We may call this alot, maybe suppress after first call? if len(client.out.nb) != 0 { client.flushSignal() } // Add the data size we are responsible for here. This will be processed when we // return to the top of the readLoop. c.addToPCD(client) if client.trace { client.traceOutOp(bytesToString(mh[:len(mh)-LEN_CR_LF]), nil) } client.mu.Unlock() return true } // Add the given sub's client to the list of clients that need flushing. // This must be invoked from `c`'s readLoop. No lock for c is required, // however, `client` lock must be held on entry. This holds true even // if `client` is same than `c`. func (c *client) addToPCD(client *client) { if _, ok := c.pcd[client]; !ok { client.out.fsp++ c.pcd[client] = needFlush } } // This will track a remote reply for an exported service that has requested // latency tracking. // Lock assumed to be held. func (c *client) trackRemoteReply(subject, reply string) { a := c.acc if a == nil { return } var lrt time.Duration var respThresh time.Duration a.mu.RLock() se := a.getServiceExport(subject) if se != nil { lrt = a.lowestServiceExportResponseTime() respThresh = se.respThresh } a.mu.RUnlock() if se == nil { return } if c.rrTracking == nil { c.rrTracking = &rrTracking{ rmap: make(map[string]*remoteLatency), ptmr: time.AfterFunc(lrt, c.pruneRemoteTracking), lrt: lrt, } } rl := remoteLatency{ Account: a.Name, ReqId: reply, respThresh: respThresh, } rl.M2.RequestStart = time.Now().UTC() c.rrTracking.rmap[reply] = &rl } // pruneRemoteTracking will prune any remote tracking objects // that are too old. These are orphaned when a service is not // sending reponses etc. // Lock should be held upon entry. func (c *client) pruneRemoteTracking() { c.mu.Lock() if c.rrTracking == nil { c.mu.Unlock() return } now := time.Now() for subject, rl := range c.rrTracking.rmap { if now.After(rl.M2.RequestStart.Add(rl.respThresh)) { delete(c.rrTracking.rmap, subject) } } if len(c.rrTracking.rmap) > 0 { t := c.rrTracking.ptmr t.Stop() t.Reset(c.rrTracking.lrt) } else { c.rrTracking.ptmr.Stop() c.rrTracking = nil } c.mu.Unlock() } // pruneReplyPerms will remove any stale or expired entries // in our reply cache. We make sure to not check too often. func (c *client) pruneReplyPerms() { // Make sure we do not check too often. if c.perms.resp == nil { return } mm := c.perms.resp.MaxMsgs ttl := c.perms.resp.Expires now := time.Now() for k, resp := range c.replies { if mm > 0 && resp.n >= mm { delete(c.replies, k) } else if ttl > 0 && now.Sub(resp.t) > ttl { delete(c.replies, k) } } c.repliesSincePrune = 0 c.lastReplyPrune = now } // pruneDenyCache will prune the deny cache via randomly // deleting items. Doing so pruneSize items at a time. // Lock must be held for this one since it is shared under // deliverMsg. func (c *client) pruneDenyCache() { r := 0 for subject := range c.mperms.dcache { delete(c.mperms.dcache, subject) if r++; r > pruneSize { break } } } // prunePubPermsCache will prune the cache via randomly // deleting items. Doing so pruneSize items at a time. func (c *client) prunePubPermsCache() { // There is a case where we can invoke this from multiple go routines, // (in deliverMsg() if sub.client is a LEAF), so we make sure to prune // from only one go routine at a time. if !atomic.CompareAndSwapInt32(&c.perms.prun, 0, 1) { return } const maxPruneAtOnce = 1000 r := 0 c.perms.pcache.Range(func(k, _ any) bool { c.perms.pcache.Delete(k) if r++; (r > pruneSize && atomic.LoadInt32(&c.perms.pcsz) < int32(maxPermCacheSize)) || (r > maxPruneAtOnce) { return false } return true }) atomic.AddInt32(&c.perms.pcsz, -int32(r)) atomic.StoreInt32(&c.perms.prun, 0) } // pubAllowed checks on publish permissioning. // Lock should not be held. func (c *client) pubAllowed(subject string) bool { return c.pubAllowedFullCheck(subject, true, false) } // pubAllowedFullCheck checks on all publish permissioning depending // on the flag for dynamic reply permissions. func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bool { if c.perms == nil || (c.perms.pub.allow == nil && c.perms.pub.deny == nil) { return true } // Check if published subject is allowed if we have permissions in place. v, ok := c.perms.pcache.Load(subject) if ok { return v.(bool) } allowed := true // Cache miss, check allow then deny as needed. if c.perms.pub.allow != nil { np, _ := c.perms.pub.allow.NumInterest(subject) allowed = np != 0 } // If we have a deny list and are currently allowed, check that as well. if allowed && c.perms.pub.deny != nil { np, _ := c.perms.pub.deny.NumInterest(subject) allowed = np == 0 } // If we are tracking reply subjects // dynamically, check to see if we are allowed here but avoid pcache. // We need to acquire the lock though. if !allowed && fullCheck && c.perms.resp != nil { if !hasLock { c.mu.Lock() } if resp := c.replies[subject]; resp != nil { resp.n++ // Check if we have sent too many responses. if c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs { delete(c.replies, subject) } else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires { delete(c.replies, subject) } else { allowed = true } } if !hasLock { c.mu.Unlock() } } else { // Update our cache here. c.perms.pcache.Store(subject, allowed) if n := atomic.AddInt32(&c.perms.pcsz, 1); n > maxPermCacheSize { c.prunePubPermsCache() } } return allowed } // Test whether a reply subject is a service import reply. func isServiceReply(reply []byte) bool { // This function is inlined and checking this way is actually faster // than byte-by-byte comparison. return len(reply) > 3 && bytesToString(reply[:4]) == replyPrefix } // Test whether a reply subject is a service import or a gateway routed reply. func isReservedReply(reply []byte) bool { if isServiceReply(reply) { return true } rLen := len(reply) // Faster to check with string([:]) than byte-by-byte if rLen > jsAckPreLen && bytesToString(reply[:jsAckPreLen]) == jsAckPre { return true } else if rLen > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix { return true } return false } // This will decide to call the client code or router code. func (c *client) processInboundMsg(msg []byte) { switch c.kind { case CLIENT: c.processInboundClientMsg(msg) case ROUTER: c.processInboundRoutedMsg(msg) case GATEWAY: c.processInboundGatewayMsg(msg) case LEAF: c.processInboundLeafMsg(msg) } } // selectMappedSubject will choose the mapped subject based on the client's inbound subject. func (c *client) selectMappedSubject() bool { nsubj, changed := c.acc.selectMappedSubject(bytesToString(c.pa.subject)) if changed { c.pa.mapped = c.pa.subject c.pa.subject = []byte(nsubj) } return changed } // processInboundClientMsg is called to process an inbound msg from a client. // Return if the message was delivered, and if the message was not delivered // due to a permission issue. func (c *client) processInboundClientMsg(msg []byte) (bool, bool) { // Update statistics // The msg includes the CR_LF, so pull back out for accounting. c.in.msgs++ c.in.bytes += int32(len(msg) - LEN_CR_LF) // Check that client (could be here with SYSTEM) is not publishing on reserved "$GNR" prefix. if c.kind == CLIENT && hasGWRoutedReplyPrefix(c.pa.subject) { c.pubPermissionViolation(c.pa.subject) return false, true } // Mostly under testing scenarios. c.mu.Lock() if c.srv == nil || c.acc == nil { c.mu.Unlock() return false, false } acc := c.acc genidAddr := &acc.sl.genid // Check pub permissions if c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) && !c.pubAllowedFullCheck(string(c.pa.subject), true, true) { c.mu.Unlock() c.pubPermissionViolation(c.pa.subject) return false, true } c.mu.Unlock() // Now check for reserved replies. These are used for service imports. if c.kind == CLIENT && len(c.pa.reply) > 0 && isReservedReply(c.pa.reply) { c.replySubjectViolation(c.pa.reply) return false, true } if c.opts.Verbose { c.sendOK() } // If MQTT client, check for retain flag now that we have passed permissions check if c.isMqtt() { c.mqttHandlePubRetain() } // Doing this inline as opposed to create a function (which otherwise has a measured // performance impact reported in our bench) var isGWRouted bool if c.kind != CLIENT { if atomic.LoadInt32(&acc.gwReplyMapping.check) > 0 { acc.mu.RLock() c.pa.subject, isGWRouted = acc.gwReplyMapping.get(c.pa.subject) acc.mu.RUnlock() } } else if atomic.LoadInt32(&c.gwReplyMapping.check) > 0 { c.mu.Lock() c.pa.subject, isGWRouted = c.gwReplyMapping.get(c.pa.subject) c.mu.Unlock() } // If we have an exported service and we are doing remote tracking, check this subject // to see if we need to report the latency. if c.rrTracking != nil { c.mu.Lock() rl := c.rrTracking.rmap[string(c.pa.subject)] if rl != nil { delete(c.rrTracking.rmap, bytesToString(c.pa.subject)) } c.mu.Unlock() if rl != nil { sl := &rl.M2 // Fill this in and send it off to the other side. sl.Status = 200 sl.Responder = c.getClientInfo(true) sl.ServiceLatency = time.Since(sl.RequestStart) - sl.Responder.RTT sl.TotalLatency = sl.ServiceLatency + sl.Responder.RTT sanitizeLatencyMetric(sl) lsub := remoteLatencySubjectForResponse(c.pa.subject) c.srv.sendInternalAccountMsg(nil, lsub, rl) // Send to SYS account } } // If the subject was converted to the gateway routed subject, then handle it now // and be done with the rest of this function. if isGWRouted { c.handleGWReplyMap(msg) return true, false } // Match the subscriptions. We will use our own L1 map if // it's still valid, avoiding contention on the shared sublist. var r *SublistResult var ok bool genid := atomic.LoadUint64(genidAddr) if genid == c.in.genid && c.in.results != nil { r, ok = c.in.results[string(c.pa.subject)] } else { // Reset our L1 completely. c.in.results = make(map[string]*SublistResult) c.in.genid = genid } // Go back to the sublist data structure. if !ok { // Match may use the subject here to populate a cache, so can not use bytesToString here. r = acc.sl.Match(string(c.pa.subject)) if len(r.psubs)+len(r.qsubs) > 0 { // Prune the results cache. Keeps us from unbounded growth. Random delete. if len(c.in.results) >= maxResultCacheSize { n := 0 for subject := range c.in.results { delete(c.in.results, subject) if n++; n > pruneSize { break } } } // Then add the new cache entry. c.in.results[string(c.pa.subject)] = r } } // Indication if we attempted to deliver the message to anyone. var didDeliver bool var qnames [][]byte // Check for no interest, short circuit if so. // This is the fanout scale. if len(r.psubs)+len(r.qsubs) > 0 { flag := pmrNoFlag // If there are matching queue subs and we are in gateway mode, // we need to keep track of the queue names the messages are // delivered to. When sending to the GWs, the RMSG will include // those names so that the remote clusters do not deliver messages // to their queue subs of the same names. if len(r.qsubs) > 0 && c.srv.gateway.enabled && atomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 { flag |= pmrCollectQueueNames } didDeliver, qnames = c.processMsgResults(acc, r, msg, c.pa.deliver, c.pa.subject, c.pa.reply, flag) } // Now deal with gateways if c.srv.gateway.enabled { reply := c.pa.reply if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 { reply = append(reply, '@') reply = append(reply, c.pa.deliver...) } didDeliver = c.sendMsgToGateways(acc, msg, c.pa.subject, reply, qnames, false) || didDeliver } // Check to see if we did not deliver to anyone and the client has a reply subject set // and wants notification of no_responders. if !didDeliver && len(c.pa.reply) > 0 { c.mu.Lock() if c.opts.NoResponders { if sub := c.subForReply(c.pa.reply); sub != nil { proto := fmt.Sprintf("HMSG %s %s 16 16\r\nNATS/1.0 503\r\n\r\n\r\n", c.pa.reply, sub.sid) c.queueOutbound([]byte(proto)) c.addToPCD(c) } } c.mu.Unlock() } return didDeliver, false } // Return the subscription for this reply subject. Only look at normal subs for this client. func (c *client) subForReply(reply []byte) *subscription { r := c.acc.sl.Match(string(reply)) for _, sub := range r.psubs { if sub.client == c { return sub } } return nil } // This is invoked knowing that c.pa.subject has been set to the gateway routed subject. // This function will send the message to possibly LEAFs and directly back to the origin // gateway. func (c *client) handleGWReplyMap(msg []byte) bool { // Check for leaf nodes if c.srv.gwLeafSubs.Count() > 0 { if r := c.srv.gwLeafSubs.MatchBytes(c.pa.subject); len(r.psubs) > 0 { c.processMsgResults(c.acc, r, msg, c.pa.deliver, c.pa.subject, c.pa.reply, pmrNoFlag) } } if c.srv.gateway.enabled { reply := c.pa.reply if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 { reply = append(reply, '@') reply = append(reply, c.pa.deliver...) } c.sendMsgToGateways(c.acc, msg, c.pa.subject, reply, nil, false) } return true } // Used to setup the response map for a service import request that has a reply subject. func (c *client) setupResponseServiceImport(acc *Account, si *serviceImport, tracking bool, header http.Header) *serviceImport { rsi := si.acc.addRespServiceImport(acc, string(c.pa.reply), si, tracking, header) if si.latency != nil { if c.rtt == 0 { // We have a service import that we are tracking but have not established RTT. c.sendRTTPing() } si.acc.mu.Lock() rsi.rc = c si.acc.mu.Unlock() } return rsi } // Will remove a header if present. func removeHeaderIfPresent(hdr []byte, key string) []byte { start := bytes.Index(hdr, []byte(key)) // key can't be first and we want to check that it is preceded by a '\n' if start < 1 || hdr[start-1] != '\n' { return hdr } index := start + len(key) if index >= len(hdr) || hdr[index] != ':' { return hdr } end := bytes.Index(hdr[start:], []byte(_CRLF_)) if end < 0 { return hdr } hdr = append(hdr[:start], hdr[start+end+len(_CRLF_):]...) if len(hdr) <= len(emptyHdrLine) { return nil } return hdr } func removeHeaderIfPrefixPresent(hdr []byte, prefix string) []byte { var index int for { if index >= len(hdr) { return hdr } start := bytes.Index(hdr[index:], []byte(prefix)) if start < 0 { return hdr } index += start if index < 1 || hdr[index-1] != '\n' { return hdr } end := bytes.Index(hdr[index+len(prefix):], []byte(_CRLF_)) if end < 0 { return hdr } hdr = append(hdr[:index], hdr[index+end+len(prefix)+len(_CRLF_):]...) if len(hdr) <= len(emptyHdrLine) { return nil } } } // Generate a new header based on optional original header and key value. // More used in JetStream layers. func genHeader(hdr []byte, key, value string) []byte { var bb bytes.Buffer if len(hdr) > LEN_CR_LF { bb.Write(hdr[:len(hdr)-LEN_CR_LF]) } else { bb.WriteString(hdrLine) } http.Header{key: []string{value}}.Write(&bb) bb.WriteString(CR_LF) return bb.Bytes() } // This will set a header for the message. // Lock does not need to be held but this should only be called // from the inbound go routine. We will update the pubArgs. // This will replace any previously set header and not add to it per normal spec. func (c *client) setHeader(key, value string, msg []byte) []byte { var bb bytes.Buffer var omi int // Write original header if present. if c.pa.hdr > LEN_CR_LF { omi = c.pa.hdr hdr := removeHeaderIfPresent(msg[:c.pa.hdr-LEN_CR_LF], key) if len(hdr) == 0 { bb.WriteString(hdrLine) } else { bb.Write(hdr) } } else { bb.WriteString(hdrLine) } http.Header{key: []string{value}}.Write(&bb) bb.WriteString(CR_LF) nhdr := bb.Len() // Put the original message back. // FIXME(dlc) - This is inefficient. bb.Write(msg[omi:]) nsize := bb.Len() - LEN_CR_LF // MQTT producers don't have CRLF, so add it back. if c.isMqtt() { nsize += LEN_CR_LF } // Update pubArgs // If others will use this later we need to save and restore original. c.pa.hdr = nhdr c.pa.size = nsize c.pa.hdb = []byte(strconv.Itoa(nhdr)) c.pa.szb = []byte(strconv.Itoa(nsize)) return bb.Bytes() } // Will return a copy of the value for the header denoted by key or nil if it does not exist. // If you know that it is safe to refer to the underlying hdr slice for the period that the // return value is used, then sliceHeader() will be faster. func getHeader(key string, hdr []byte) []byte { v := sliceHeader(key, hdr) if v == nil { return nil } return append(make([]byte, 0, len(v)), v...) } // Will return the sliced value for the header denoted by key or nil if it does not exists. // This function ignores errors and tries to achieve speed and no additional allocations. func sliceHeader(key string, hdr []byte) []byte { if len(hdr) == 0 { return nil } index := bytes.Index(hdr, []byte(key)) hdrLen := len(hdr) // Check that we have enough characters, this will handle the -1 case of the key not // being found and will also handle not having enough characters for trailing CRLF. if index < 2 { return nil } // There should be a terminating CRLF. if index >= hdrLen-1 || hdr[index-1] != '\n' || hdr[index-2] != '\r' { return nil } // The key should be immediately followed by a : separator. index += len(key) + 1 if index >= hdrLen || hdr[index-1] != ':' { return nil } // Skip over whitespace before the value. for index < hdrLen && hdr[index] == ' ' { index++ } // Collect together the rest of the value until we hit a CRLF. start := index for index < hdrLen { if hdr[index] == '\r' && index < hdrLen-1 && hdr[index+1] == '\n' { break } index++ } return hdr[start:index:index] } // For bytes.HasPrefix below. var ( jsRequestNextPreB = []byte(jsRequestNextPre) jsDirectGetPreB = []byte(jsDirectGetPre) ) // processServiceImport is an internal callback when a subscription matches an imported service // from another account. This includes response mappings as well. func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byte) bool { // If we are a GW and this is not a direct serviceImport ignore. isResponse := si.isRespServiceImport() if (c.kind == GATEWAY || c.kind == ROUTER) && !isResponse { return false } // Detect cycles and ignore (return) when we detect one. if len(c.pa.psi) > 0 { for i := len(c.pa.psi) - 1; i >= 0; i-- { if psi := c.pa.psi[i]; psi.se == si.se { return false } } } acc.mu.RLock() var checkJS bool shouldReturn := si.invalid || acc.sl == nil if !shouldReturn && !isResponse && si.to == jsAllAPI { if bytes.HasPrefix(c.pa.subject, jsDirectGetPreB) || bytes.HasPrefix(c.pa.subject, jsRequestNextPreB) { checkJS = true } } siAcc := si.acc acc.mu.RUnlock() // We have a special case where JetStream pulls in all service imports through one export. // However the GetNext for consumers and DirectGet for streams are a no-op and causes buildups of service imports, // response service imports and rrMap entries which all will need to simply expire. // TODO(dlc) - Come up with something better. if shouldReturn || (checkJS && si.se != nil && si.se.acc == c.srv.SystemAccount()) { return false } var nrr []byte var rsi *serviceImport // Check if there is a reply present and set up a response. tracking, headers := shouldSample(si.latency, c) if len(c.pa.reply) > 0 { // Special case for now, need to formalize. // TODO(dlc) - Formalize as a service import option for reply rewrite. // For now we can't do $JS.ACK since that breaks pull consumers across accounts. if !bytes.HasPrefix(c.pa.reply, []byte(jsAckPre)) { if rsi = c.setupResponseServiceImport(acc, si, tracking, headers); rsi != nil { nrr = []byte(rsi.from) } } else { // This only happens when we do a pull subscriber that trampolines through another account. // Normally this code is not called. nrr = c.pa.reply } } else if !isResponse && si.latency != nil && tracking { // Check to see if this was a bad request with no reply and we were supposed to be tracking. siAcc.sendBadRequestTrackingLatency(si, c, headers) } // Send tracking info here if we are tracking this response. // This is always a response. var didSendTL bool if si.tracking && !si.didDeliver { // Stamp that we attempted delivery. si.didDeliver = true didSendTL = acc.sendTrackingLatency(si, c) } // Pick correct "to" subject. If we matched on a wildcard use the literal publish subject. to, subject := si.to, string(c.pa.subject) if si.tr != nil { // FIXME(dlc) - This could be slow, may want to look at adding cache to bare transforms? to = si.tr.TransformSubject(subject) } else if si.usePub { to = subject } // Copy our pubArg since this gets modified as we process the service import itself. pacopy := c.pa // Now check to see if this account has mappings that could affect the service import. // Can't use non-locked trick like in processInboundClientMsg, so just call into selectMappedSubject // so we only lock once. nsubj, changed := siAcc.selectMappedSubject(to) if changed { c.pa.mapped = []byte(to) to = nsubj } // Set previous service import to detect chaining. lpsi := len(c.pa.psi) hadPrevSi, share := lpsi > 0, si.share if hadPrevSi { share = c.pa.psi[lpsi-1].share } c.pa.psi = append(c.pa.psi, si) // Place our client info for the request in the original message. // This will survive going across routes, etc. if !isResponse { isSysImport := siAcc == c.srv.SystemAccount() var ci *ClientInfo if hadPrevSi && c.pa.hdr >= 0 { var cis ClientInfo if err := json.Unmarshal(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr]), &cis); err == nil { ci = &cis ci.Service = acc.Name // Check if we are moving into a share details account from a non-shared // and add in server and cluster details. if !share && (si.share || isSysImport) { c.addServerAndClusterInfo(ci) } } } else if c.kind != LEAF || c.pa.hdr < 0 || len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) == 0 { ci = c.getClientInfo(share) // If we did not share but the imports destination is the system account add in the server and cluster info. if !share && isSysImport { c.addServerAndClusterInfo(ci) } } else if c.kind == LEAF && (si.share || isSysImport) { // We have a leaf header here for ci, augment as above. ci = c.getClientInfo(si.share) if !si.share && isSysImport { c.addServerAndClusterInfo(ci) } } // Set clientInfo if present. if ci != nil { if b, _ := json.Marshal(ci); b != nil { msg = c.setHeader(ClientInfoHdr, bytesToString(b), msg) } } } // Set our optional subject(to) and reply. if !isResponse && to != subject { c.pa.subject = []byte(to) } c.pa.reply = nrr if changed && c.isMqtt() && c.pa.hdr > 0 { c.srv.mqttStoreQoSMsgForAccountOnNewSubject(c.pa.hdr, msg, siAcc.GetName(), to) } // FIXME(dlc) - Do L1 cache trick like normal client? rr := siAcc.sl.Match(to) // If we are a route or gateway or leafnode and this message is flipped to a queue subscriber we // need to handle that since the processMsgResults will want a queue filter. flags := pmrMsgImportedFromService if c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF { flags |= pmrIgnoreEmptyQueueFilter } // We will be calling back into processMsgResults since we are now being called as a normal sub. // We need to take care of the c.in.rts, so save off what is there and use a local version. We // will put back what was there after. orts := c.in.rts var lrts [routeTargetInit]routeTarget c.in.rts = lrts[:0] var didDeliver bool // If this is not a gateway connection but gateway is enabled, // try to send this converted message to all gateways. if c.srv.gateway.enabled { flags |= pmrCollectQueueNames var queues [][]byte didDeliver, queues = c.processMsgResults(siAcc, rr, msg, c.pa.deliver, []byte(to), nrr, flags) didDeliver = c.sendMsgToGateways(siAcc, msg, []byte(to), nrr, queues, false) || didDeliver } else { didDeliver, _ = c.processMsgResults(siAcc, rr, msg, c.pa.deliver, []byte(to), nrr, flags) } // Restore to original values. c.in.rts = orts c.pa = pacopy // Before we undo didDeliver based on tracing and last mile, mark in the c.pa which informs us of no responders status. // If we override due to tracing and traceOnly we do not want to send back a no responders. c.pa.delivered = didDeliver // Determine if we should remove this service import. This is for response service imports. // We will remove if we did not deliver, or if we are a response service import and we are // a singleton, or we have an EOF message. shouldRemove := !didDeliver || (isResponse && (si.rt == Singleton || len(msg) == LEN_CR_LF)) // If we are tracking and we did not actually send the latency info we need to suppress the removal. if si.tracking && !didSendTL { shouldRemove = false } // If we are streamed or chunked we need to update our timestamp to avoid cleanup. if si.rt != Singleton && didDeliver { acc.mu.Lock() si.ts = time.Now().UnixNano() acc.mu.Unlock() } // Cleanup of a response service import if shouldRemove { reason := rsiOk if !didDeliver { reason = rsiNoDelivery } if isResponse { acc.removeRespServiceImport(si, reason) } else { // This is a main import and since we could not even deliver to the exporting account // go ahead and remove the respServiceImport we created above. siAcc.removeRespServiceImport(rsi, reason) } } return didDeliver } func (c *client) addSubToRouteTargets(sub *subscription) { if c.in.rts == nil { c.in.rts = make([]routeTarget, 0, routeTargetInit) } for i := range c.in.rts { rt := &c.in.rts[i] if rt.sub.client == sub.client { if sub.queue != nil { rt.qs = append(rt.qs, sub.queue...) rt.qs = append(rt.qs, ' ') } return } } var rt *routeTarget lrts := len(c.in.rts) // If we are here we do not have the sub yet in our list // If we have to grow do so here. if lrts == cap(c.in.rts) { c.in.rts = append(c.in.rts, routeTarget{}) } c.in.rts = c.in.rts[:lrts+1] rt = &c.in.rts[lrts] rt.sub = sub rt.qs = rt._qs[:0] if sub.queue != nil { rt.qs = append(rt.qs, sub.queue...) rt.qs = append(rt.qs, ' ') } } // This processes the sublist results for a given message. // Returns if the message was delivered to at least target and queue filters. func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, deliver, subject, reply []byte, flags int) (bool, [][]byte) { // For sending messages across routes and leafnodes. // Reset if we have one since we reuse this data structure. if c.in.rts != nil { c.in.rts = c.in.rts[:0] } var rplyHasGWPrefix bool var creply = reply // If the reply subject is a GW routed reply, we will perform some // tracking in deliverMsg(). We also want to send to the user the // reply without the prefix. `creply` will be set to that and be // used to create the message header for client connections. if rplyHasGWPrefix = isGWRoutedReply(reply); rplyHasGWPrefix { creply = reply[gwSubjectOffset:] } // With JetStream we now have times where we want to match a subscription // on one subject, but deliver it with another. e.g. JetStream deliverables. // This only works for last mile, meaning to a client. For other types we need // to use the original subject. subj := subject if len(deliver) > 0 { subj = deliver } // Check for JetStream encoded reply subjects. // For now these will only be on $JS.ACK prefixed reply subjects. var remapped bool if len(creply) > 0 && c.kind != CLIENT && c.kind != SYSTEM && c.kind != JETSTREAM && c.kind != ACCOUNT && bytes.HasPrefix(creply, []byte(jsAckPre)) { // We need to rewrite the subject and the reply. if li := bytes.LastIndex(creply, []byte("@")); li != -1 && li < len(creply)-1 { remapped = true subj, creply = creply[li+1:], creply[:li] } } var didDeliver bool // delivery subject for clients var dsubj []byte // Used as scratch if mapping var _dsubj [128]byte // For stats, we will keep track of the number of messages that have been // delivered and then multiply by the size of that message and update // server and account stats in a "single" operation (instead of per-sub). // However, we account for situations where the message is possibly changed // by having an extra size var dlvMsgs int64 var dlvExtraSize int64 // We need to know if this is a MQTT producer because they send messages // without CR_LF (we otherwise remove the size of CR_LF from message size). prodIsMQTT := c.isMqtt() updateStats := func() { if dlvMsgs == 0 { return } totalBytes := dlvMsgs*int64(len(msg)) + dlvExtraSize // For non MQTT producers, remove the CR_LF * number of messages if !prodIsMQTT { totalBytes -= dlvMsgs * int64(LEN_CR_LF) } if acc != nil { atomic.AddInt64(&acc.outMsgs, dlvMsgs) atomic.AddInt64(&acc.outBytes, totalBytes) } if srv := c.srv; srv != nil { atomic.AddInt64(&srv.outMsgs, dlvMsgs) atomic.AddInt64(&srv.outBytes, totalBytes) } } // Loop over all normal subscriptions that match. for _, sub := range r.psubs { // Check if this is a send to a ROUTER. We now process // these after everything else. switch sub.client.kind { case ROUTER: if (c.kind != ROUTER && !c.isSpokeLeafNode()) || (flags&pmrAllowSendFromRouteToRoute != 0) { c.addSubToRouteTargets(sub) } continue case GATEWAY: // Never send to gateway from here. continue case LEAF: // We handle similarly to routes and use the same data structures. // Leaf node delivery audience is different however. // Also leaf nodes are always no echo, so we make sure we are not // going to send back to ourselves here. For messages from routes we want // to suppress in general unless we know from the hub or its a service reply. if c != sub.client && (c.kind != ROUTER || sub.client.isHubLeafNode() || isServiceReply(c.pa.subject)) { c.addSubToRouteTargets(sub) } continue } // Assume delivery subject is the normal subject to this point. dsubj = subj // Check for stream import mapped subs (shadow subs). These apply to local subs only. if sub.im != nil { // If this message was a service import do not re-export to an exported stream. if flags&pmrMsgImportedFromService != 0 { continue } if sub.im.tr != nil { to := sub.im.tr.TransformSubject(bytesToString(subject)) dsubj = append(_dsubj[:0], to...) } else if sub.im.usePub { dsubj = append(_dsubj[:0], subj...) } else { dsubj = append(_dsubj[:0], sub.im.to...) } // Make sure deliver is set if inbound from a route. if remapped && (c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF) { deliver = subj } // If we are mapping for a deliver subject we will reverse roles. // The original subj we set from above is correct for the msg header, // but we need to transform the deliver subject to properly route. if len(deliver) > 0 { dsubj, subj = subj, dsubj } } // Remap to the original subject if internal. if sub.icb != nil && sub.rsi { dsubj = subject } // Normal delivery mh := c.msgHeader(dsubj, creply, sub) if c.deliverMsg(prodIsMQTT, sub, acc, dsubj, creply, mh, msg, rplyHasGWPrefix) { // We don't count internal deliveries, so do only when sub.icb is nil. if sub.icb == nil { dlvMsgs++ } didDeliver = true } } // Set these up to optionally filter based on the queue lists. // This is for messages received from routes which will have directed // guidance on which queue groups we should deliver to. qf := c.pa.queues // Declared here because of goto. var queues [][]byte var leafOrigin string switch c.kind { case ROUTER: if len(c.pa.origin) > 0 { // Picture a message sent from a leafnode to a server that then routes // this message: CluserA -leaf-> HUB1 -route-> HUB2 // Here we are in HUB2, so c.kind is a ROUTER, but the message will // contain a c.pa.origin set to "ClusterA" to indicate that this message // originated from that leafnode cluster. leafOrigin = bytesToString(c.pa.origin) } case LEAF: leafOrigin = c.remoteCluster() } // For all routes/leaf/gateway connections, we may still want to send messages to // leaf nodes or routes even if there are no queue filters since we collect // them above and do not process inline like normal clients. // However, do select queue subs if asked to ignore empty queue filter. if (c.kind == LEAF || c.kind == ROUTER || c.kind == GATEWAY) && len(qf) == 0 && flags&pmrIgnoreEmptyQueueFilter == 0 { goto sendToRoutesOrLeafs } // Process queue subs for i := 0; i < len(r.qsubs); i++ { qsubs := r.qsubs[i] // If we have a filter check that here. We could make this a map or someting more // complex but linear search since we expect queues to be small. Should be faster // and more cache friendly. if qf != nil && len(qsubs) > 0 { tqn := qsubs[0].queue for _, qn := range qf { if bytes.Equal(qn, tqn) { goto selectQSub } } continue } selectQSub: // We will hold onto remote or lead qsubs when we are coming from // a route or a leaf node just in case we can no longer do local delivery. var rsub, sub *subscription var _ql [32]*subscription src := c.kind // If we just came from a route we want to prefer local subs. // So only select from local subs but remember the first rsub // in case all else fails. if src == ROUTER { ql := _ql[:0] for i := 0; i < len(qsubs); i++ { sub = qsubs[i] if dst := sub.client.kind; dst == LEAF || dst == ROUTER { // If the destination is a LEAF, we first need to make sure // that we would not pick one that was the origin of this // message. if dst == LEAF && leafOrigin != _EMPTY_ && leafOrigin == sub.client.remoteCluster() { continue } // If we have assigned a ROUTER rsub already, replace if // the destination is a LEAF since we want to favor that. if rsub == nil || (rsub.client.kind == ROUTER && dst == LEAF) { rsub = sub } else if dst == LEAF { // We already have a LEAF and this is another one. // Flip a coin to see if we swap it or not. // See https://github.com/nats-io/nats-server/issues/6040 if fastrand.Uint32()%2 == 1 { rsub = sub } } } else { ql = append(ql, sub) } } qsubs = ql } sindex := 0 lqs := len(qsubs) if lqs > 1 { sindex = int(fastrand.Uint32() % uint32(lqs)) } // Find a subscription that is able to deliver this message starting at a random index. // Note that if the message came from a ROUTER, we will only have CLIENT or LEAF // queue subs here, otherwise we can have all types. for i := 0; i < lqs; i++ { if sindex+i < lqs { sub = qsubs[sindex+i] } else { sub = qsubs[(sindex+i)%lqs] } if sub == nil { continue } // If we are a spoke leaf node make sure to not forward across routes. // This mimics same behavior for normal subs above. if c.kind == LEAF && c.isSpokeLeafNode() && sub.client.kind == ROUTER { continue } // We have taken care of preferring local subs for a message from a route above. // Here we just care about a client or leaf and skipping a leaf and preferring locals. if dst := sub.client.kind; dst == ROUTER || dst == LEAF { if (src == LEAF || src == CLIENT) && dst == LEAF { // If we come from a LEAF and are about to pick a LEAF connection, // make sure this is not the same leaf cluster. if src == LEAF && leafOrigin != _EMPTY_ && leafOrigin == sub.client.remoteCluster() { continue } // Remember that leaf in case we don't find any other candidate. // We already start randomly in lqs slice, so we don't need // to do a random swap if we already have an rsub like we do // when src == ROUTER above. if rsub == nil { rsub = sub } continue } else { // We want to favor qsubs in our own cluster. If the routed // qsub has an origin, it means that is on behalf of a leaf. // We need to treat it differently. if len(sub.origin) > 0 { // If we already have an rsub, nothing to do. Also, do // not pick a routed qsub for a LEAF origin cluster // that is the same than where the message comes from. if rsub == nil && (leafOrigin == _EMPTY_ || leafOrigin != bytesToString(sub.origin)) { rsub = sub } continue } // This is a qsub that is local on the remote server (or // we are connected to an older server and we don't know). // Pick this one and be done. rsub = sub break } } // Assume delivery subject is normal subject to this point. dsubj = subj // Check for stream import mapped subs. These apply to local subs only. if sub.im != nil { // If this message was a service import do not re-export to an exported stream. if flags&pmrMsgImportedFromService != 0 { continue } if sub.im.tr != nil { to := sub.im.tr.TransformSubject(bytesToString(subject)) dsubj = append(_dsubj[:0], to...) } else if sub.im.usePub { dsubj = append(_dsubj[:0], subj...) } else { dsubj = append(_dsubj[:0], sub.im.to...) } // Make sure deliver is set if inbound from a route. if remapped && (c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF) { deliver = subj } // If we are mapping for a deliver subject we will reverse roles. // The original subj we set from above is correct for the msg header, // but we need to transform the deliver subject to properly route. if len(deliver) > 0 { dsubj, subj = subj, dsubj } } mh := c.msgHeader(dsubj, creply, sub) if c.deliverMsg(prodIsMQTT, sub, acc, subject, creply, mh, msg, rplyHasGWPrefix) { if sub.icb == nil { dlvMsgs++ } didDeliver = true // Clear rsub rsub = nil if flags&pmrCollectQueueNames != 0 { queues = append(queues, sub.queue) } break } } if rsub != nil { // We are here if we have selected a leaf or route as the destination, // or if we tried to deliver to a local qsub but failed. c.addSubToRouteTargets(rsub) if flags&pmrCollectQueueNames != 0 { queues = append(queues, rsub.queue) } } } sendToRoutesOrLeafs: // If no messages for routes or leafnodes return here. if len(c.in.rts) == 0 { updateStats() return didDeliver, queues } // If we do have a deliver subject we need to do something with it. // Again this is when JetStream (but possibly others) wants the system // to rewrite the delivered subject. The way we will do that is place it // at the end of the reply subject if it exists. if len(deliver) > 0 && len(reply) > 0 { reply = append(reply, '@') reply = append(reply, deliver...) } // Copy off original pa in case it changes. pa := c.pa // We address by index to avoid struct copy. // We have inline structs for memory layout and cache coherency. for i := range c.in.rts { rt := &c.in.rts[i] dc := rt.sub.client dmsg, hset := msg, false // Check if we have an origin cluster set from a leafnode message. // If so make sure we do not send it back to the same cluster for a different // leafnode. Cluster wide no echo. if dc.kind == LEAF { // Check two scenarios. One is inbound from a route (c.pa.origin), // and the other is leaf to leaf. In both case, leafOrigin is the one // to use for the comparison. if leafOrigin != _EMPTY_ && leafOrigin == dc.remoteCluster() { continue } // We need to check if this is a request that has a stamped client information header. // This will contain an account but will represent the account from the leafnode. If // they are not named the same this would cause an account lookup failure trying to // process the request for something like JetStream or other system services that rely // on the client info header. We can just check for reply and the presence of a header // to avoid slow downs for all traffic. if len(c.pa.reply) > 0 && c.pa.hdr >= 0 { dmsg, hset = c.checkLeafClientInfoHeader(msg) } } mh := c.msgHeaderForRouteOrLeaf(subject, reply, rt, acc) if c.deliverMsg(prodIsMQTT, rt.sub, acc, subject, reply, mh, dmsg, false) { if rt.sub.icb == nil { dlvMsgs++ dlvExtraSize += int64(len(dmsg) - len(msg)) } didDeliver = true } // If we set the header reset the origin pub args. if hset { c.pa = pa } } updateStats() return didDeliver, queues } // Check and swap accounts on a client info header destined across a leafnode. func (c *client) checkLeafClientInfoHeader(msg []byte) (dmsg []byte, setHdr bool) { if c.pa.hdr < 0 || len(msg) < c.pa.hdr { return msg, false } cir := sliceHeader(ClientInfoHdr, msg[:c.pa.hdr]) if len(cir) == 0 { return msg, false } dmsg = msg var ci ClientInfo if err := json.Unmarshal(cir, &ci); err == nil { if v, _ := c.srv.leafRemoteAccounts.Load(ci.Account); v != nil { remoteAcc := v.(string) if ci.Account != remoteAcc { ci.Account = remoteAcc if b, _ := json.Marshal(ci); b != nil { dmsg, setHdr = c.setHeader(ClientInfoHdr, bytesToString(b), msg), true } } } } return dmsg, setHdr } func (c *client) pubPermissionViolation(subject []byte) { c.sendErr(fmt.Sprintf("Permissions Violation for Publish to %q", subject)) c.Errorf("Publish Violation - %s, Subject %q", c.getAuthUser(), subject) } func (c *client) subPermissionViolation(sub *subscription) { errTxt := fmt.Sprintf("Permissions Violation for Subscription to %q", sub.subject) logTxt := fmt.Sprintf("Subscription Violation - %s, Subject %q, SID %s", c.getAuthUser(), sub.subject, sub.sid) if sub.queue != nil { errTxt = fmt.Sprintf("Permissions Violation for Subscription to %q using queue %q", sub.subject, sub.queue) logTxt = fmt.Sprintf("Subscription Violation - %s, Subject %q, Queue: %q, SID %s", c.getAuthUser(), sub.subject, sub.queue, sub.sid) } c.sendErr(errTxt) c.Errorf(logTxt) } func (c *client) replySubjectViolation(reply []byte) { c.sendErr(fmt.Sprintf("Permissions Violation for Publish with Reply of %q", reply)) c.Errorf("Publish Violation - %s, Reply %q", c.getAuthUser(), reply) } func (c *client) maxTokensViolation(sub *subscription) { errTxt := fmt.Sprintf("Permissions Violation for Subscription to %q, too many tokens", sub.subject) logTxt := fmt.Sprintf("Subscription Violation Too Many Tokens - %s, Subject %q, SID %s", c.getAuthUser(), sub.subject, sub.sid) c.sendErr(errTxt) c.Errorf(logTxt) } func (c *client) processPingTimer() { c.mu.Lock() c.ping.tmr = nil // Check if connection is still opened if c.isClosed() { c.mu.Unlock() return } c.Debugf("%s Ping Timer", c.kindString()) var sendPing bool opts := c.srv.getOpts() pingInterval := opts.PingInterval if c.kind == ROUTER && opts.Cluster.PingInterval > 0 { pingInterval = opts.Cluster.PingInterval } pingInterval = adjustPingInterval(c.kind, pingInterval) now := time.Now() needRTT := c.rtt == 0 || now.Sub(c.rttStart) > DEFAULT_RTT_MEASUREMENT_INTERVAL // Do not delay PINGs for ROUTER, GATEWAY or spoke LEAF connections. if c.kind == ROUTER || c.kind == GATEWAY || c.isSpokeLeafNode() { sendPing = true } else { // If we received client data or a ping from the other side within the PingInterval, // then there is no need to send a ping. if delta := now.Sub(c.lastIn); delta < pingInterval && !needRTT { c.Debugf("Delaying PING due to remote client data or ping %v ago", delta.Round(time.Second)) } else { sendPing = true } } if sendPing { // Check for violation maxPingsOut := opts.MaxPingsOut if c.kind == ROUTER && opts.Cluster.MaxPingsOut > 0 { maxPingsOut = opts.Cluster.MaxPingsOut } if c.ping.out+1 > maxPingsOut { c.Debugf("Stale Client Connection - Closing") c.enqueueProto([]byte(fmt.Sprintf(errProto, "Stale Connection"))) c.mu.Unlock() c.closeConnection(StaleConnection) return } // Send PING c.sendPing() } // Reset to fire again. c.setPingTimer() c.mu.Unlock() } // Returns the smallest value between the given `d` and some max value // based on the connection kind. func adjustPingInterval(kind int, d time.Duration) time.Duration { switch kind { case ROUTER: if d > routeMaxPingInterval { return routeMaxPingInterval } case GATEWAY: if d > gatewayMaxPingInterval { return gatewayMaxPingInterval } } return d } // This is used when a connection cannot yet start to send PINGs because // the remote would not be able to handle them (case of compression, // or outbound gateway, etc...), but we still want to close the connection // if the timer has not been reset by the time we reach the time equivalent // to have sent the max number of pings. // // Lock should be held func (c *client) watchForStaleConnection(pingInterval time.Duration, pingMax int) { c.ping.tmr = time.AfterFunc(pingInterval*time.Duration(pingMax+1), func() { c.mu.Lock() c.Debugf("Stale Client Connection - Closing") c.enqueueProto([]byte(fmt.Sprintf(errProto, "Stale Connection"))) c.mu.Unlock() c.closeConnection(StaleConnection) }) } // Lock should be held func (c *client) setPingTimer() { if c.srv == nil { return } opts := c.srv.getOpts() d := opts.PingInterval if c.kind == ROUTER && opts.Cluster.PingInterval > 0 { d = opts.Cluster.PingInterval } d = adjustPingInterval(c.kind, d) c.ping.tmr = time.AfterFunc(d, c.processPingTimer) } // Lock should be held func (c *client) clearPingTimer() { if c.ping.tmr == nil { return } c.ping.tmr.Stop() c.ping.tmr = nil } func (c *client) clearTlsToTimer() { if c.tlsTo == nil { return } c.tlsTo.Stop() c.tlsTo = nil } // Lock should be held func (c *client) setAuthTimer(d time.Duration) { c.atmr = time.AfterFunc(d, c.authTimeout) } // Lock should be held func (c *client) clearAuthTimer() bool { if c.atmr == nil { return true } stopped := c.atmr.Stop() c.atmr = nil return stopped } // We may reuse atmr for expiring user jwts, // so check connectReceived. // Lock assume held on entry. func (c *client) awaitingAuth() bool { return !c.flags.isSet(connectReceived) && c.atmr != nil } // This will set the atmr for the JWT expiration time. // We will lock on entry. func (c *client) setExpirationTimer(d time.Duration) { c.mu.Lock() c.setExpirationTimerUnlocked(d) c.mu.Unlock() } // This will set the atmr for the JWT expiration time. client lock should be held before call func (c *client) setExpirationTimerUnlocked(d time.Duration) { c.atmr = time.AfterFunc(d, c.authExpired) // This is an JWT expiration. if c.flags.isSet(connectReceived) { c.expires = time.Now().Add(d).Truncate(time.Second) } } // Return when this client expires via a claim, or 0 if not set. func (c *client) claimExpiration() time.Duration { c.mu.Lock() defer c.mu.Unlock() if c.expires.IsZero() { return 0 } return time.Until(c.expires).Truncate(time.Second) } // Possibly flush the connection and then close the low level connection. // The boolean `minimalFlush` indicates if the flush operation should have a // minimal write deadline. // Lock is held on entry. func (c *client) flushAndClose(minimalFlush bool) { if !c.flags.isSet(skipFlushOnClose) && c.out.pb > 0 { if minimalFlush { const lowWriteDeadline = 100 * time.Millisecond // Reduce the write deadline if needed. if c.out.wdl > lowWriteDeadline { c.out.wdl = lowWriteDeadline } } c.flushOutbound() } for i := range c.out.nb { nbPoolPut(c.out.nb[i]) } c.out.nb = nil // We can't touch c.out.wnb when a flushOutbound is in progress since it // is accessed outside the lock there. If in progress, the cleanup will be // done in flushOutbound when detecting that connection is closed. if !c.flags.isSet(flushOutbound) { for i := range c.out.wnb { nbPoolPut(c.out.wnb[i]) } c.out.wnb = nil } // This seem to be important (from experimentation) for the GC to release // the connection. c.out.sg = nil // Close the low level connection. if c.nc != nil { // Starting with Go 1.16, the low level close will set its own deadline // of 5 seconds, so setting our own deadline does not work. Instead, // we will close the TLS connection in separate go routine. nc := c.nc c.nc = nil if _, ok := nc.(*tls.Conn); ok { go func() { nc.Close() }() } else { nc.Close() } } } var kindStringMap = map[int]string{ CLIENT: "Client", ROUTER: "Router", GATEWAY: "Gateway", LEAF: "Leafnode", JETSTREAM: "JetStream", ACCOUNT: "Account", SYSTEM: "System", } func (c *client) kindString() string { if kindStringVal, ok := kindStringMap[c.kind]; ok { return kindStringVal } return "Unknown Type" } // swapAccountAfterReload will check to make sure the bound account for this client // is current. Under certain circumstances after a reload we could be pointing to // an older one. func (c *client) swapAccountAfterReload() { c.mu.Lock() srv := c.srv an := c.acc.GetName() c.mu.Unlock() if srv == nil { return } if acc, _ := srv.LookupAccount(an); acc != nil { c.mu.Lock() if c.acc != acc { c.acc = acc } c.mu.Unlock() } } // processSubsOnConfigReload removes any subscriptions the client has that are no // longer authorized, and checks for imports (accounts) due to a config reload. func (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) { c.mu.Lock() var ( checkPerms = c.perms != nil checkAcc = c.acc != nil acc = c.acc ) if !checkPerms && !checkAcc { c.mu.Unlock() return } var ( _subs [32]*subscription subs = _subs[:0] _removed [32]*subscription removed = _removed[:0] srv = c.srv ) if checkAcc { // We actually only want to check if stream imports have changed. if _, ok := awcsti[acc.Name]; !ok { checkAcc = false } } // We will clear any mperms we have here. It will rebuild on the fly with canSubscribe, // so we do that here as we collect them. We will check result down below. c.mperms = nil // Collect client's subs under the lock for _, sub := range c.subs { // Just checking to rebuild mperms under the lock, will collect removed though here. // Only collect under subs array of canSubscribe and checkAcc true. canSub := c.canSubscribe(string(sub.subject)) canQSub := sub.queue != nil && c.canSubscribe(string(sub.subject), string(sub.queue)) if !canSub && !canQSub { removed = append(removed, sub) } else if checkAcc { subs = append(subs, sub) } } c.mu.Unlock() // This list is all subs who are allowed and we need to check accounts. for _, sub := range subs { c.mu.Lock() oldShadows := sub.shadow sub.shadow = nil c.mu.Unlock() c.addShadowSubscriptions(acc, sub, true) for _, nsub := range oldShadows { nsub.im.acc.sl.Remove(nsub) } } // Unsubscribe all that need to be removed and report back to client and logs. for _, sub := range removed { c.unsubscribe(acc, sub, true, true) c.sendErr(fmt.Sprintf("Permissions Violation for Subscription to %q (sid %q)", sub.subject, sub.sid)) srv.Noticef("Removed sub %q (sid %q) for %s - not authorized", sub.subject, sub.sid, c.getAuthUser()) } } // Allows us to count up all the queue subscribers during close. type qsub struct { sub *subscription n int32 } func (c *client) closeConnection(reason ClosedState) { c.mu.Lock() if c.flags.isSet(closeConnection) { c.mu.Unlock() return } // Note that we may have markConnAsClosed() invoked before closeConnection(), // so don't set this to 1, instead bump the count. c.rref++ c.flags.set(closeConnection) c.clearAuthTimer() c.clearPingTimer() c.clearTlsToTimer() c.markConnAsClosed(reason) // Unblock anyone who is potentially stalled waiting on us. if c.out.stc != nil { close(c.out.stc) c.out.stc = nil } // If we have remote latency tracking running shut that down. if c.rrTracking != nil { c.rrTracking.ptmr.Stop() c.rrTracking = nil } // If we are shutting down, no need to do all the accounting on subs, etc. if reason == ServerShutdown { s := c.srv c.mu.Unlock() if s != nil { // Unregister s.removeClient(c) } return } var ( kind = c.kind srv = c.srv noReconnect = c.flags.isSet(noReconnect) acc = c.acc spoke bool ) // Snapshot for use if we are a client connection. // FIXME(dlc) - we can just stub in a new one for client // and reference existing one. var subs []*subscription if kind == CLIENT || kind == LEAF || kind == JETSTREAM { var _subs [32]*subscription subs = _subs[:0] // Do not set c.subs to nil or delete the sub from c.subs here because // it will be needed in saveClosedClient (which has been started as a // go routine in markConnAsClosed). Cleanup will be done there. for _, sub := range c.subs { // Auto-unsubscribe subscriptions must be unsubscribed forcibly. sub.max = 0 sub.close() subs = append(subs, sub) } spoke = c.isSpokeLeafNode() } c.mu.Unlock() // Remove client's or leaf node or jetstream subscriptions. if acc != nil && (kind == CLIENT || kind == LEAF || kind == JETSTREAM) { acc.sl.RemoveBatch(subs) } else if kind == ROUTER { c.removeRemoteSubs() } if srv != nil { // Unregister srv.removeClient(c) if acc != nil { // Update remote subscriptions. if kind == CLIENT || kind == LEAF || kind == JETSTREAM { qsubs := map[string]*qsub{} for _, sub := range subs { // Call unsubscribe here to cleanup shadow subscriptions and such. c.unsubscribe(acc, sub, true, false) // Update route as normal for a normal subscriber. if sub.queue == nil { if !spoke { srv.updateRouteSubscriptionMap(acc, sub, -1) if srv.gateway.enabled { srv.gatewayUpdateSubInterest(acc.Name, sub, -1) } } acc.updateLeafNodes(sub, -1) } else { // We handle queue subscribers special in case we // have a bunch we can just send one update to the // connected routes. num := int32(1) if kind == LEAF { num = sub.qw } key := keyFromSub(sub) if esub, ok := qsubs[key]; ok { esub.n += num } else { qsubs[key] = &qsub{sub, num} } } } // Process any qsubs here. for _, esub := range qsubs { if !spoke { srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n)) if srv.gateway.enabled { srv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n)) } } acc.updateLeafNodes(esub.sub, -(esub.n)) } } // Always remove from the account, otherwise we can leak clients. // Note that SYSTEM and ACCOUNT types from above cleanup their own subs. if prev := acc.removeClient(c); prev == 1 { srv.decActiveAccounts() } } } // Now that we are done with subscriptions, clear the field so that the // connection can be released and gc'ed. if kind == CLIENT || kind == LEAF { c.mu.Lock() c.subs = nil c.mu.Unlock() } // Don't reconnect connections that have been marked with // the no reconnect flag. if noReconnect { return } c.reconnect() } // Depending on the kind of connections, this may attempt to recreate a connection. // The actual reconnect attempt will be started in a go routine. func (c *client) reconnect() { var ( retryImplicit bool gwName string gwIsOutbound bool gwCfg *gatewayCfg leafCfg *leafNodeCfg ) c.mu.Lock() // Decrease the ref count and perform the reconnect only if == 0. c.rref-- if c.flags.isSet(noReconnect) || c.rref > 0 { c.mu.Unlock() return } if c.route != nil { // A route is marked as solicited if it was given an URL to connect to, // which would be the case even with implicit (due to gossip), so mark this // as a retry for a route that is solicited and not explicit. retryImplicit = c.route.retry || (c.route.didSolicit && c.route.routeType == Implicit) } kind := c.kind switch kind { case GATEWAY: gwName = c.gw.name gwIsOutbound = c.gw.outbound gwCfg = c.gw.cfg case LEAF: if c.isSolicitedLeafNode() { leafCfg = c.leaf.remote } } srv := c.srv c.mu.Unlock() // Check for a solicited route. If it was, start up a reconnect unless // we are already connected to the other end. if didSolicit := c.isSolicitedRoute(); didSolicit || retryImplicit { srv.mu.Lock() defer srv.mu.Unlock() // Capture these under lock c.mu.Lock() rid := c.route.remoteID rtype := c.route.routeType rurl := c.route.url accName := string(c.route.accName) checkRID := accName == _EMPTY_ && srv.getOpts().Cluster.PoolSize < 1 && rid != _EMPTY_ c.mu.Unlock() // It is possible that the server is being shutdown. // If so, don't try to reconnect if !srv.isRunning() { return } if checkRID && srv.routes[rid] != nil { // This is the case of "no pool". Make sure that the registered one // is upgraded to solicited if the connection trying to reconnect // was a solicited one. if didSolicit { if remote := srv.routes[rid][0]; remote != nil { upgradeRouteToSolicited(remote, rurl, rtype) } } srv.Debugf("Not attempting reconnect for solicited route, already connected to %q", rid) return } else if rid == srv.info.ID { srv.Debugf("Detected route to self, ignoring %q", rurl.Redacted()) return } else if rtype != Implicit || retryImplicit { srv.Debugf("Attempting reconnect for solicited route %q", rurl.Redacted()) // Keep track of this go-routine so we can wait for it on // server shutdown. srv.startGoRoutine(func() { srv.reConnectToRoute(rurl, rtype, accName) }) } } else if srv != nil && kind == GATEWAY && gwIsOutbound { if gwCfg != nil { srv.Debugf("Attempting reconnect for gateway %q", gwName) // Run this as a go routine since we may be called within // the solicitGateway itself if there was an error during // the creation of the gateway connection. srv.startGoRoutine(func() { srv.reconnectGateway(gwCfg) }) } else { srv.Debugf("Gateway %q not in configuration, not attempting reconnect", gwName) } } else if leafCfg != nil { // Check if this is a solicited leaf node. Start up a reconnect. srv.startGoRoutine(func() { srv.reConnectToRemoteLeafNode(leafCfg) }) } } // Set the noReconnect flag. This is used before a call to closeConnection() // to prevent the connection to reconnect (routes, gateways). func (c *client) setNoReconnect() { c.mu.Lock() c.flags.set(noReconnect) c.mu.Unlock() } // Returns the client's RTT value with the protection of the client's lock. func (c *client) getRTTValue() time.Duration { c.mu.Lock() rtt := c.rtt c.mu.Unlock() return rtt } // This function is used by ROUTER and GATEWAY connections to // look for a subject on a given account (since these type of // connections are not bound to a specific account). // If the c.pa.subject is found in the cache, the cached result // is returned, otherwse, we match the account's sublist and update // the cache. The cache is pruned if reaching a certain size. func (c *client) getAccAndResultFromCache() (*Account, *SublistResult) { var ( acc *Account pac *perAccountCache r *SublistResult ok bool ) // Check our cache. if pac, ok = c.in.pacache[string(c.pa.pacache)]; ok { // Check the genid to see if it's still valid. // Since v2.10.0, the config reload of accounts has been fixed // and an account's sublist pointer should not change, so no need to // lock to access it. sl := pac.acc.sl if genid := atomic.LoadUint64(&sl.genid); genid != pac.genid { ok = false c.in.pacache = make(map[string]*perAccountCache) } else { acc = pac.acc r = pac.results } } if !ok { if c.kind == ROUTER && len(c.route.accName) > 0 { if acc = c.acc; acc == nil { return nil, nil } } else { // Match correct account and sublist. if acc, _ = c.srv.LookupAccount(bytesToString(c.pa.account)); acc == nil { return nil, nil } } sl := acc.sl // Match against the account sublist. r = sl.MatchBytes(c.pa.subject) // Check if we need to prune. if len(c.in.pacache) >= maxPerAccountCacheSize { c.prunePerAccountCache() } // Store in our cache,make sure to do so after we prune. c.in.pacache[string(c.pa.pacache)] = &perAccountCache{acc, r, atomic.LoadUint64(&sl.genid)} } return acc, r } // Account will return the associated account for this client. func (c *client) Account() *Account { if c == nil { return nil } c.mu.Lock() acc := c.acc c.mu.Unlock() return acc } // prunePerAccountCache will prune off a random number of cache entries. func (c *client) prunePerAccountCache() { n := 0 for cacheKey := range c.in.pacache { delete(c.in.pacache, cacheKey) if n++; n > prunePerAccountCacheSize { break } } } // pruneClosedSubFromPerAccountCache remove entries that contain subscriptions // that have been closed. func (c *client) pruneClosedSubFromPerAccountCache() { for cacheKey, pac := range c.in.pacache { for _, sub := range pac.results.psubs { if sub.isClosed() { goto REMOVE } } for _, qsub := range pac.results.qsubs { for _, sub := range qsub { if sub.isClosed() { goto REMOVE } } } continue REMOVE: delete(c.in.pacache, cacheKey) } } // Returns our service account for this request. func (ci *ClientInfo) serviceAccount() string { if ci == nil { return _EMPTY_ } if ci.Service != _EMPTY_ { return ci.Service } return ci.Account } // Add in our server and cluster information to this client info. func (c *client) addServerAndClusterInfo(ci *ClientInfo) { if ci == nil { return } // Server if c.kind != LEAF { ci.Server = c.srv.Name() } else if c.kind == LEAF { ci.Server = c.leaf.remoteServer } // Cluster ci.Cluster = c.srv.cachedClusterName() // If we have gateways fill in cluster alternates. // These will be in RTT asc order. if c.srv.gateway.enabled { var gws []*client c.srv.getOutboundGatewayConnections(&gws) for _, c := range gws { c.mu.Lock() cn := c.gw.name c.mu.Unlock() ci.Alternates = append(ci.Alternates, cn) } } } // Grabs the information for this client. func (c *client) getClientInfo(detailed bool) *ClientInfo { if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) { return nil } // Result var ci ClientInfo if detailed { c.addServerAndClusterInfo(&ci) } c.mu.Lock() // RTT and Account are always added. ci.Account = accForClient(c) ci.RTT = c.rtt // Detailed signals additional opt in. if detailed { ci.Start = &c.start ci.Host = c.host ci.ID = c.cid ci.Name = c.opts.Name ci.User = c.getRawAuthUser() ci.Lang = c.opts.Lang ci.Version = c.opts.Version ci.Jwt = c.opts.JWT ci.IssuerKey = issuerForClient(c) ci.NameTag = c.nameTag ci.Tags = c.tags ci.Kind = c.kindString() ci.ClientType = c.clientTypeString() } c.mu.Unlock() return &ci } func (c *client) doTLSServerHandshake(typ string, tlsConfig *tls.Config, timeout float64, pCerts PinnedCertSet) error { _, err := c.doTLSHandshake(typ, false, nil, tlsConfig, _EMPTY_, timeout, pCerts) return err } func (c *client) doTLSClientHandshake(typ string, url *url.URL, tlsConfig *tls.Config, tlsName string, timeout float64, pCerts PinnedCertSet) (bool, error) { return c.doTLSHandshake(typ, true, url, tlsConfig, tlsName, timeout, pCerts) } // Performs either server or client side (if solicit is true) TLS Handshake. // On error, the TLS handshake error has been logged and the connection // has been closed. // // Lock is held on entry. func (c *client) doTLSHandshake(typ string, solicit bool, url *url.URL, tlsConfig *tls.Config, tlsName string, timeout float64, pCerts PinnedCertSet) (bool, error) { var host string var resetTLSName bool var err error // Capture kind for some debug/error statements. kind := c.kind // If we solicited, we will act like the client, otherwise the server. if solicit { c.Debugf("Starting TLS %s client handshake", typ) if tlsConfig.ServerName == _EMPTY_ { // If the given url is a hostname, use this hostname for the // ServerName. If it is an IP, use the cfg's tlsName. If none // is available, resort to current IP. host = url.Hostname() if tlsName != _EMPTY_ && net.ParseIP(host) != nil { host = tlsName } tlsConfig.ServerName = host } c.nc = tls.Client(c.nc, tlsConfig) } else { if kind == CLIENT { c.Debugf("Starting TLS client connection handshake") } else { c.Debugf("Starting TLS %s server handshake", typ) } c.nc = tls.Server(c.nc, tlsConfig) } conn := c.nc.(*tls.Conn) // Setup the timeout ttl := secondsToDuration(timeout) c.tlsTo = time.AfterFunc(ttl, func() { tlsTimeout(c, conn) }) conn.SetReadDeadline(time.Now().Add(ttl)) c.mu.Unlock() if err = conn.Handshake(); err != nil { if solicit { // Based on type of error, possibly clear the saved tlsName // See: https://github.com/nats-io/nats-server/issues/1256 // NOTE: As of Go 1.20, the HostnameError is wrapped so cannot // type assert to check directly. var hostnameErr x509.HostnameError if errors.As(err, &hostnameErr) { if host == tlsName { resetTLSName = true } } } } else if !c.matchesPinnedCert(pCerts) { err = ErrCertNotPinned } if err != nil { if kind == CLIENT { c.Errorf("TLS handshake error: %v", err) } else { c.Errorf("TLS %s handshake error: %v", typ, err) } c.closeConnection(TLSHandshakeError) // Grab the lock before returning since the caller was holding the lock on entry c.mu.Lock() // Returning any error is fine. Since the connection is closed ErrConnectionClosed // is appropriate. return resetTLSName, ErrConnectionClosed } // Reset the read deadline conn.SetReadDeadline(time.Time{}) // Re-Grab lock c.mu.Lock() // To be consistent with client, set this flag to indicate that handshake is done c.flags.set(handshakeComplete) // The connection still may have been closed on success handshake due // to a race with tls timeout. If that the case, return error indicating // that the connection is closed. if c.isClosed() { err = ErrConnectionClosed } return false, err } // getRawAuthUserLock returns the raw auth user for the client. // Will acquire the client lock. func (c *client) getRawAuthUserLock() string { c.mu.Lock() defer c.mu.Unlock() return c.getRawAuthUser() } // getRawAuthUser returns the raw auth user for the client. // Lock should be held. func (c *client) getRawAuthUser() string { switch { case c.opts.Nkey != _EMPTY_: return c.opts.Nkey case c.opts.Username != _EMPTY_: return c.opts.Username case c.opts.JWT != _EMPTY_: return c.pubKey case c.opts.Token != _EMPTY_: return c.opts.Token default: return _EMPTY_ } } // getAuthUser returns the auth user for the client. // Lock should be held. func (c *client) getAuthUser() string { switch { case c.opts.Nkey != _EMPTY_: return fmt.Sprintf("Nkey %q", c.opts.Nkey) case c.opts.Username != _EMPTY_: return fmt.Sprintf("User %q", c.opts.Username) case c.opts.JWT != _EMPTY_: return fmt.Sprintf("JWT User %q", c.pubKey) case c.opts.Token != _EMPTY_: return fmt.Sprintf("Token %q", c.opts.Token) default: return `User "N/A"` } } // Given an array of strings, this function converts it to a map as long // as all the content (converted to upper-case) matches some constants. // Converts the given array of strings to a map of string. // The strings are converted to upper-case and added to the map only // if the server recognize them as valid connection types. // If there are unknown connection types, the map of valid ones is returned // along with an error that contains the name of the unknown. func convertAllowedConnectionTypes(cts []string) (map[string]struct{}, error) { var unknown []string m := make(map[string]struct{}, len(cts)) for _, i := range cts { i = strings.ToUpper(i) switch i { case jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket, jwt.ConnectionTypeLeafnode, jwt.ConnectionTypeLeafnodeWS, jwt.ConnectionTypeMqtt, jwt.ConnectionTypeMqttWS: m[i] = struct{}{} default: unknown = append(unknown, i) } } var err error // We will still return the map of valid ones. if len(unknown) != 0 { err = fmt.Errorf("invalid connection types %q", unknown) } return m, err } // This will return true if the connection is of a type present in the given `acts` map. // Note that so far this is used only for CLIENT or LEAF connections. // But a CLIENT can be standard or websocket (and other types in the future). func (c *client) connectionTypeAllowed(acts map[string]struct{}) bool { // Empty means all type of clients are allowed if len(acts) == 0 { return true } var want string switch c.kind { case CLIENT: switch c.clientType() { case NATS: want = jwt.ConnectionTypeStandard case WS: want = jwt.ConnectionTypeWebsocket case MQTT: if c.isWebsocket() { want = jwt.ConnectionTypeMqttWS } else { want = jwt.ConnectionTypeMqtt } } case LEAF: if c.isWebsocket() { want = jwt.ConnectionTypeLeafnodeWS } else { want = jwt.ConnectionTypeLeafnode } } _, ok := acts[want] return ok } // isClosed returns true if either closeConnection or connMarkedClosed // flag have been set, or if `nc` is nil, which may happen in tests. func (c *client) isClosed() bool { return c.flags.isSet(closeConnection) || c.flags.isSet(connMarkedClosed) || c.nc == nil } // Logging functionality scoped to a client or route. func (c *client) Error(err error) { c.srv.Errors(c, err) } func (c *client) Errorf(format string, v ...any) { format = fmt.Sprintf("%s - %s", c, format) c.srv.Errorf(format, v...) } func (c *client) Debugf(format string, v ...any) { format = fmt.Sprintf("%s - %s", c, format) c.srv.Debugf(format, v...) } func (c *client) Noticef(format string, v ...any) { format = fmt.Sprintf("%s - %s", c, format) c.srv.Noticef(format, v...) } func (c *client) Tracef(format string, v ...any) { format = fmt.Sprintf("%s - %s", c, format) c.srv.Tracef(format, v...) } func (c *client) Warnf(format string, v ...any) { format = fmt.Sprintf("%s - %s", c, format) c.srv.Warnf(format, v...) } func (c *client) rateLimitFormatWarnf(format string, v ...any) { if _, loaded := c.srv.rateLimitLogging.LoadOrStore(format, time.Now()); loaded { return } statement := fmt.Sprintf(format, v...) c.Warnf("%s", statement) } func (c *client) RateLimitWarnf(format string, v ...any) { // Do the check before adding the client info to the format... statement := fmt.Sprintf(format, v...) if _, loaded := c.srv.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { return } c.Warnf("%s", statement) } // Set the very first PING to a lower interval to capture the initial RTT. // After that the PING interval will be set to the user defined value. // Client lock should be held. func (c *client) setFirstPingTimer() { s := c.srv if s == nil { return } opts := s.getOpts() d := opts.PingInterval if c.kind == ROUTER && opts.Cluster.PingInterval > 0 { d = opts.Cluster.PingInterval } if !opts.DisableShortFirstPing { if c.kind != CLIENT { if d > firstPingInterval { d = firstPingInterval } d = adjustPingInterval(c.kind, d) } else if d > firstClientPingInterval { d = firstClientPingInterval } } // We randomize the first one by an offset up to 20%, e.g. 2m ~= max 24s. addDelay := rand.Int63n(int64(d / 5)) d += time.Duration(addDelay) // In the case of ROUTER/LEAF and when compression is configured, it is possible // that this timer was already set, but just to detect a stale connection // since we have to delay the first PING after compression negotiation // occurred. if c.ping.tmr != nil { c.ping.tmr.Stop() } c.ping.tmr = time.AfterFunc(d, c.processPingTimer) } nats-server-2.10.27/server/client_test.go000066400000000000000000002347141477524627100203400ustar00rootroot00000000000000// Copyright 2012-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "encoding/json" "fmt" "io" "math" "net" "net/url" "reflect" "regexp" "runtime" "strings" "sync" "sync/atomic" "testing" "time" "crypto/tls" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) type serverInfo struct { ID string `json:"server_id"` Host string `json:"host"` Port uint `json:"port"` Version string `json:"version"` AuthRequired bool `json:"auth_required"` TLSRequired bool `json:"tls_required"` MaxPayload int64 `json:"max_payload"` Headers bool `json:"headers"` ConnectURLs []string `json:"connect_urls,omitempty"` LameDuckMode bool `json:"ldm,omitempty"` CID uint64 `json:"client_id,omitempty"` } type testAsyncClient struct { *client parseAsync func(string) quitCh chan bool } func (c *testAsyncClient) close() { c.client.closeConnection(ClientClosed) c.quitCh <- true } func (c *testAsyncClient) parse(proto []byte) error { err := c.client.parse(proto) c.client.flushClients(0) return err } func (c *testAsyncClient) parseAndClose(proto []byte) { c.client.parse(proto) c.client.flushClients(0) c.closeConnection(ClientClosed) } func createClientAsync(ch chan *client, s *Server, cli net.Conn) { // Normally, those type of clients are used against non running servers. // However, some don't, which would then cause the writeLoop to be // started twice for the same client (since createClient() start both // read and write loop if it is detected as running). startWriteLoop := !s.isRunning() if startWriteLoop { s.grWG.Add(1) } go func() { c := s.createClient(cli) // Must be here to suppress +OK c.opts.Verbose = false if startWriteLoop { go c.writeLoop() } ch <- c }() } func newClientForServer(s *Server) (*testAsyncClient, *bufio.Reader, string) { cli, srv := net.Pipe() cr := bufio.NewReaderSize(cli, maxBufSize) ch := make(chan *client) createClientAsync(ch, s, srv) // So failing tests don't just hang. cli.SetReadDeadline(time.Now().Add(10 * time.Second)) l, _ := cr.ReadString('\n') // Grab client c := <-ch parse, quitCh := genAsyncParser(c) asyncClient := &testAsyncClient{ client: c, parseAsync: parse, quitCh: quitCh, } return asyncClient, cr, l } func genAsyncParser(c *client) (func(string), chan bool) { pab := make(chan []byte, 16) pas := func(cs string) { pab <- []byte(cs) } quit := make(chan bool) go func() { for { select { case cs := <-pab: c.parse(cs) c.flushClients(0) case <-quit: return } } }() return pas, quit } var defaultServerOptions = Options{ Host: "127.0.0.1", Port: -1, Trace: true, Debug: true, DisableShortFirstPing: true, NoLog: true, NoSigs: true, } func rawSetup(serverOptions Options) (*Server, *testAsyncClient, *bufio.Reader, string) { s := New(&serverOptions) c, cr, l := newClientForServer(s) return s, c, cr, l } func setUpClientWithResponse() (*testAsyncClient, string) { _, c, _, l := rawSetup(defaultServerOptions) return c, l } func setupClient() (*Server, *testAsyncClient, *bufio.Reader) { s, c, cr, _ := rawSetup(defaultServerOptions) return s, c, cr } func checkClientsCount(t *testing.T, s *Server, expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if nc := s.NumClients(); nc != expected { return fmt.Errorf("The number of expected connections was %v, got %v", expected, nc) } return nil }) } func checkAccClientsCount(t *testing.T, acc *Account, expected int) { t.Helper() checkFor(t, 4*time.Second, 10*time.Millisecond, func() error { if nc := acc.NumConnections(); nc != expected { return fmt.Errorf("Expected account %q to have %v clients, got %v", acc.Name, expected, nc) } return nil }) } func TestAsyncClientWithRunningServer(t *testing.T) { o := DefaultOptions() s := RunServer(o) defer s.Shutdown() c, _, _ := newClientForServer(s) defer c.close() buf := make([]byte, 1000000) writeLoopTxt := fmt.Sprintf("writeLoop(%p)", c.client) checkFor(t, time.Second, 15*time.Millisecond, func() error { n := runtime.Stack(buf, true) if count := strings.Count(string(buf[:n]), writeLoopTxt); count != 1 { return fmt.Errorf("writeLoop for client should have been started only once: %v", count) } return nil }) } func TestClientCreateAndInfo(t *testing.T) { s, c, _, l := rawSetup(defaultServerOptions) defer c.close() if c.cid != 1 { t.Fatalf("Expected cid of 1 vs %d\n", c.cid) } if c.state != OP_START { t.Fatal("Expected state to be OP_START") } if !strings.HasPrefix(l, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", l) } // Make sure payload is proper json var info serverInfo err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } // Sanity checks if info.MaxPayload != MAX_PAYLOAD_SIZE || info.AuthRequired || info.TLSRequired || int(info.Port) != s.opts.Port { t.Fatalf("INFO inconsistent: %+v\n", info) } } func TestClientNoResponderSupport(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, _, _ := newClientForServer(s) defer c.close() // Force header support if you want to do no_responders. Make sure headers are set. if err := c.parse([]byte("CONNECT {\"no_responders\":true}\r\n")); err == nil { t.Fatalf("Expected error") } c, cr, _ := newClientForServer(s) defer c.close() c.parseAsync("CONNECT {\"headers\":true, \"no_responders\":true}\r\nSUB reply 1\r\nPUB foo reply 2\r\nok\r\n") l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := hmsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a match for %q", l) } checkPayload(cr, []byte("NATS/1.0 503\r\n\r\n"), t) } func TestServerHeaderSupport(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, _, l := newClientForServer(s) defer c.close() if !strings.HasPrefix(l, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", l) } var info serverInfo if err := json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if !info.Headers { t.Fatalf("Expected by default for header support to be enabled") } opts.NoHeaderSupport = true opts.Port = -1 s = New(&opts) c, _, l = newClientForServer(s) defer c.close() if err := json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Headers { t.Fatalf("Expected header support to be disabled") } } // This test specifically is not testing how headers are encoded in a raw msg. // It wants to make sure the serve and clients agreement on when to use headers // is bi-directional and functions properly. func TestClientHeaderSupport(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, _, _ := newClientForServer(s) defer c.close() // Even though the server supports headers we need to explicitly say we do in the // CONNECT. If we do not we should get an error. if err := c.parse([]byte("CONNECT {}\r\nHPUB foo 0 2\r\nok\r\n")); err != ErrMsgHeadersNotSupported { t.Fatalf("Expected to receive an error, got %v", err) } // This should succeed. c, _, _ = newClientForServer(s) defer c.close() if err := c.parse([]byte("CONNECT {\"headers\":true}\r\nHPUB foo 0 2\r\nok\r\n")); err != nil { t.Fatalf("Unexpected error %v", err) } // Now start a server without support. opts.NoHeaderSupport = true opts.Port = -1 s = New(&opts) c, _, _ = newClientForServer(s) defer c.close() if err := c.parse([]byte("CONNECT {\"headers\":true}\r\nHPUB foo 0 2\r\nok\r\n")); err != ErrMsgHeadersNotSupported { t.Fatalf("Expected to receive an error, got %v", err) } } var hmsgPat = regexp.MustCompile(`HMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)[^\S\r\n]+(\d+)\r\n`) func TestClientHeaderDeliverMsg(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, cr, _ := newClientForServer(s) defer c.close() connect := "CONNECT {\"headers\":true}" subOp := "SUB foo 1" pubOp := "HPUB foo 12 14\r\nName:Derek\r\nOK\r\n" cmd := strings.Join([]string{connect, subOp, pubOp}, "\r\n") c.parseAsync(cmd) l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := hmsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a match for %q", l) } matches := am[0] if len(matches) != 7 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 7) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[HDR_INDEX] != "12" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[HDR_INDEX]) } if matches[TLEN_INDEX] != "14" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[TLEN_INDEX]) } checkPayload(cr, []byte("Name:Derek\r\nOK\r\n"), t) } var smsgPat = regexp.MustCompile(`^MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\r\n`) func TestClientHeaderDeliverStrippedMsg(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, _, _ := newClientForServer(s) defer c.close() b, br, _ := newClientForServer(s) defer b.close() // Does not support headers b.parseAsync("SUB foo 1\r\nPING\r\n") if _, err := br.ReadString('\n'); err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } connect := "CONNECT {\"headers\":true}" pubOp := "HPUB foo 12 14\r\nName:Derek\r\nOK\r\n" cmd := strings.Join([]string{connect, pubOp}, "\r\n") c.parseAsync(cmd) // Read from 'b' client. l, err := br.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := smsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a correct match for %q", l) } matches := am[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[LEN_INDEX] != "2" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } checkPayload(br, []byte("OK\r\n"), t) if br.Buffered() != 0 { t.Fatalf("Expected no extra bytes to be buffered, got %d", br.Buffered()) } } func TestClientHeaderDeliverQueueSubStrippedMsg(t *testing.T) { opts := defaultServerOptions s := New(&opts) c, _, _ := newClientForServer(s) defer c.close() b, br, _ := newClientForServer(s) defer b.close() // Does not support headers b.parseAsync("SUB foo bar 1\r\nPING\r\n") if _, err := br.ReadString('\n'); err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } connect := "CONNECT {\"headers\":true}" pubOp := "HPUB foo 12 14\r\nName:Derek\r\nOK\r\n" cmd := strings.Join([]string{connect, pubOp}, "\r\n") c.parseAsync(cmd) // Read from 'b' client. l, err := br.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := smsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a correct match for %q", l) } matches := am[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[LEN_INDEX] != "2" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } checkPayload(br, []byte("OK\r\n"), t) } func TestNonTLSConnectionState(t *testing.T) { _, c, _ := setupClient() defer c.close() state := c.GetTLSConnectionState() if state != nil { t.Error("GetTLSConnectionState() returned non-nil") } } func TestClientConnect(t *testing.T) { _, c, _ := setupClient() defer c.close() // Basic Connect setting flags connectOp := []byte("CONNECT {\"verbose\":true,\"pedantic\":true,\"tls_required\":false,\"echo\":false}\r\n") err := c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Verbose: true, Pedantic: true, Echo: false}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } // Test that we can capture user/pass connectOp = []byte("CONNECT {\"user\":\"derek\",\"pass\":\"foo\"}\r\n") c.opts = defaultOpts err = c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Username: "derek", Password: "foo"}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } // Test that we can capture client name connectOp = []byte("CONNECT {\"user\":\"derek\",\"pass\":\"foo\",\"name\":\"router\"}\r\n") c.opts = defaultOpts err = c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Username: "derek", Password: "foo", Name: "router"}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } // Test that we correctly capture auth tokens connectOp = []byte("CONNECT {\"auth_token\":\"YZZ222\",\"name\":\"router\"}\r\n") c.opts = defaultOpts err = c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Token: "YZZ222", Name: "router"}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } } func TestClientConnectProto(t *testing.T) { _, c, r := setupClient() defer c.close() // Basic Connect setting flags, proto should be zero (original proto) connectOp := []byte("CONNECT {\"verbose\":true,\"pedantic\":true,\"tls_required\":false}\r\n") err := c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Protocol: ClientProtoZero}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } // ProtoInfo connectOp = []byte(fmt.Sprintf("CONNECT {\"verbose\":true,\"pedantic\":true,\"tls_required\":false,\"protocol\":%d}\r\n", ClientProtoInfo)) err = c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected state of OP_START vs %d\n", c.state) } if !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Protocol: ClientProtoInfo}) { t.Fatalf("Did not parse connect options correctly: %+v\n", c.opts) } if c.opts.Protocol != ClientProtoInfo { t.Fatalf("Protocol should have been set to %v, but is set to %v", ClientProtoInfo, c.opts.Protocol) } // Illegal Option connectOp = []byte("CONNECT {\"protocol\":22}\r\n") wg := sync.WaitGroup{} wg.Add(1) // The client here is using a pipe, we need to be dequeuing // data otherwise the server would be blocked trying to send // the error back to it. go func() { defer wg.Done() for { if _, _, err := r.ReadLine(); err != nil { return } } }() err = c.parse(connectOp) if err == nil { t.Fatalf("Expected to receive an error\n") } if err != ErrBadClientProtocol { t.Fatalf("Expected err of %q, got %q\n", ErrBadClientProtocol, err) } wg.Wait() } func TestRemoteAddress(t *testing.T) { rc := &client{} // though in reality this will panic if it does not, adding coverage anyway if rc.RemoteAddress() != nil { t.Errorf("RemoteAddress() did not handle nil connection correctly") } _, c, _ := setupClient() defer c.close() addr := c.RemoteAddress() if addr.Network() != "pipe" { t.Errorf("RemoteAddress() returned invalid network: %s", addr.Network()) } if addr.String() != "pipe" { t.Errorf("RemoteAddress() returned invalid string: %s", addr.String()) } } func TestClientPing(t *testing.T) { _, c, cr := setupClient() defer c.close() // PING pingOp := "PING\r\n" c.parseAsync(pingOp) l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response incorrect: %s\n", l) } } var msgPat = regexp.MustCompile(`MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\r\n`) const ( SUB_INDEX = 1 SID_INDEX = 2 REPLY_INDEX = 4 LEN_INDEX = 5 HDR_INDEX = 5 TLEN_INDEX = 6 ) func grabPayload(cr *bufio.Reader, expected int) []byte { d := make([]byte, expected) n, _ := cr.Read(d) cr.ReadString('\n') return d[:n] } func checkPayload(cr *bufio.Reader, expected []byte, t *testing.T) { t.Helper() // Read in payload d := make([]byte, len(expected)) n, err := cr.Read(d) if err != nil { t.Fatalf("Error receiving msg payload from server: %v\n", err) } if n != len(expected) { t.Fatalf("Did not read correct amount of bytes: %d vs %d\n", n, len(expected)) } if !bytes.Equal(d, expected) { t.Fatalf("Did not read correct payload:: <%s>\n", d) } } func TestClientSimplePubSub(t *testing.T) { _, c, cr := setupClient() defer c.close() // SUB/PUB c.parseAsync("SUB foo 1\r\nPUB foo 5\r\nhello\r\nPING\r\n") l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[LEN_INDEX] != "5" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } checkPayload(cr, []byte("hello\r\n"), t) } func TestClientPubSubNoEcho(t *testing.T) { _, c, cr := setupClient() defer c.close() // Specify no echo connectOp := []byte("CONNECT {\"echo\":false}\r\n") err := c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } // SUB/PUB c.parseAsync("SUB foo 1\r\nPUB foo 5\r\nhello\r\nPING\r\n") l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } // We should not receive anything but a PONG since we specified no echo. if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response incorrect: %q\n", l) } } func TestClientSimplePubSubWithReply(t *testing.T) { _, c, cr := setupClient() defer c.close() // SUB/PUB c.parseAsync("SUB foo 1\r\nPUB foo bar 5\r\nhello\r\nPING\r\n") l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[REPLY_INDEX] != "bar" { t.Fatalf("Did not get correct reply subject: '%s'\n", matches[REPLY_INDEX]) } if matches[LEN_INDEX] != "5" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } } func TestClientNoBodyPubSubWithReply(t *testing.T) { _, c, cr := setupClient() defer c.close() // SUB/PUB c.parseAsync("SUB foo 1\r\nPUB foo bar 0\r\n\r\nPING\r\n") l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[REPLY_INDEX] != "bar" { t.Fatalf("Did not get correct reply subject: '%s'\n", matches[REPLY_INDEX]) } if matches[LEN_INDEX] != "0" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } } func TestClientPubWithQueueSub(t *testing.T) { _, c, cr := setupClient() defer c.close() num := 100 // Queue SUB/PUB subs := []byte("SUB foo g1 1\r\nSUB foo g1 2\r\n") pubs := []byte("PUB foo bar 5\r\nhello\r\n") op := []byte{} op = append(op, subs...) for i := 0; i < num; i++ { op = append(op, pubs...) } go c.parseAndClose(op) var n1, n2, received int for ; ; received++ { l, err := cr.ReadString('\n') if err != nil { break } matches := msgPat.FindAllStringSubmatch(l, -1)[0] // Count which sub switch matches[SID_INDEX] { case "1": n1++ case "2": n2++ } checkPayload(cr, []byte("hello\r\n"), t) } if received != num { t.Fatalf("Received wrong # of msgs: %d vs %d\n", received, num) } // Threshold for randomness for now if n1 < 20 || n2 < 20 { t.Fatalf("Received wrong # of msgs per subscriber: %d - %d\n", n1, n2) } } func TestSplitSubjectQueue(t *testing.T) { cases := []struct { name string sq string wantSubject []byte wantQueue []byte wantErr bool }{ {name: "single subject", sq: "foo", wantSubject: []byte("foo"), wantQueue: nil}, {name: "subject and queue", sq: "foo bar", wantSubject: []byte("foo"), wantQueue: []byte("bar")}, {name: "subject and queue with surrounding spaces", sq: " foo bar ", wantSubject: []byte("foo"), wantQueue: []byte("bar")}, {name: "subject and queue with extra spaces in the middle", sq: "foo bar", wantSubject: []byte("foo"), wantQueue: []byte("bar")}, {name: "subject, queue, and extra token", sq: "foo bar fizz", wantSubject: []byte(nil), wantQueue: []byte(nil), wantErr: true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { sub, que, err := splitSubjectQueue(c.sq) if err == nil && c.wantErr { t.Fatal("Expected error, but got nil") } if err != nil && !c.wantErr { t.Fatalf("Expected nil error, but got %v", err) } if !reflect.DeepEqual(sub, c.wantSubject) { t.Fatalf("Expected to get subject %#v, but instead got %#v", c.wantSubject, sub) } if !reflect.DeepEqual(que, c.wantQueue) { t.Fatalf("Expected to get queue %#v, but instead got %#v", c.wantQueue, que) } }) } } func TestTypeString(t *testing.T) { cases := []struct { intType int stringType string }{ { intType: CLIENT, stringType: "Client", }, { intType: ROUTER, stringType: "Router", }, { intType: GATEWAY, stringType: "Gateway", }, { intType: LEAF, stringType: "Leafnode", }, { intType: JETSTREAM, stringType: "JetStream", }, { intType: ACCOUNT, stringType: "Account", }, { intType: SYSTEM, stringType: "System", }, { intType: -1, stringType: "Unknown Type", }, } for _, cs := range cases { c := &client{kind: cs.intType} typeStringVal := c.kindString() if typeStringVal != cs.stringType { t.Fatalf("Expected typeString value %q, but instead received %q", cs.stringType, typeStringVal) } } } func TestQueueSubscribePermissions(t *testing.T) { cases := []struct { name string perms *SubjectPermission subject string queue string want string }{ { name: "plain subscription on foo", perms: &SubjectPermission{Allow: []string{"foo"}}, subject: "foo", want: "+OK\r\n", }, { name: "queue subscribe with allowed group", perms: &SubjectPermission{Allow: []string{"foo bar"}}, subject: "foo", queue: "bar", want: "+OK\r\n", }, { name: "queue subscribe with wildcard allowed group", perms: &SubjectPermission{Allow: []string{"foo bar.*"}}, subject: "foo", queue: "bar.fizz", want: "+OK\r\n", }, { name: "queue subscribe with full wildcard subject and subgroup", perms: &SubjectPermission{Allow: []string{"> bar.>"}}, subject: "whizz", queue: "bar.bang", want: "+OK\r\n", }, { name: "plain subscribe with full wildcard subject and subgroup", perms: &SubjectPermission{Allow: []string{"> bar.>"}}, subject: "whizz", want: "-ERR 'Permissions Violation for Subscription to \"whizz\"'\r\n", }, { name: "deny plain subscription on foo", perms: &SubjectPermission{Allow: []string{">"}, Deny: []string{"foo"}}, subject: "foo", queue: "bar", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"bar\"'\r\n", }, { name: "allow plain subscription, except foo", perms: &SubjectPermission{Allow: []string{">"}, Deny: []string{"foo"}}, subject: "bar", want: "+OK\r\n", }, { name: "deny everything", perms: &SubjectPermission{Allow: []string{">"}, Deny: []string{">"}}, subject: "foo", queue: "bar", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"bar\"'\r\n", }, { name: "can only subscribe to queues v1", perms: &SubjectPermission{Allow: []string{"> v1.>"}}, subject: "foo", queue: "v1.prod", want: "+OK\r\n", }, { name: "cannot subscribe to queues, plain subscribe ok", perms: &SubjectPermission{Allow: []string{">"}, Deny: []string{"> >"}}, subject: "foo", want: "+OK\r\n", }, { name: "cannot subscribe to queues, queue subscribe not ok", perms: &SubjectPermission{Deny: []string{"> >"}}, subject: "foo", queue: "bar", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"bar\"'\r\n", }, { name: "deny all queue subscriptions on dev or stg only", perms: &SubjectPermission{Deny: []string{"> *.dev", "> *.stg"}}, subject: "foo", queue: "bar", want: "+OK\r\n", }, { name: "allow only queue subscription on dev or stg", perms: &SubjectPermission{Allow: []string{"> *.dev", "> *.stg"}}, subject: "foo", queue: "bar", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"bar\"'\r\n", }, { name: "deny queue subscriptions with subject foo", perms: &SubjectPermission{Deny: []string{"foo >"}}, subject: "foo", queue: "bar", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"bar\"'\r\n", }, { name: "plain sub is allowed, but queue subscribe with queue not in list", perms: &SubjectPermission{Allow: []string{"foo bar"}}, subject: "foo", queue: "fizz", want: "-ERR 'Permissions Violation for Subscription to \"foo\" using queue \"fizz\"'\r\n", }, { name: "allow plain sub, but do queue subscribe", perms: &SubjectPermission{Allow: []string{"foo"}}, subject: "foo", queue: "bar", want: "+OK\r\n", }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { _, client, r := setupClient() defer client.close() client.RegisterUser(&User{ Permissions: &Permissions{Subscribe: c.perms}, }) connect := []byte("CONNECT {\"verbose\":true}\r\n") qsub := []byte(fmt.Sprintf("SUB %s %s 1\r\n", c.subject, c.queue)) go client.parseAndClose(append(connect, qsub...)) var buf bytes.Buffer if _, err := io.Copy(&buf, r); err != nil { t.Fatal(err) } // Extra OK is from the successful CONNECT. want := "+OK\r\n" + c.want if got := buf.String(); got != want { t.Fatalf("Expected to receive %q, but instead received %q", want, got) } }) } } func TestClientPubWithQueueSubNoEcho(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() nc1, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() // Grab the client from server and set no echo by hand. s.mu.Lock() lc := len(s.clients) c := s.clients[s.gcid] s.mu.Unlock() if lc != 1 { t.Fatalf("Expected only 1 client but got %d\n", lc) } if c == nil { t.Fatal("Expected to retrieve client\n") } c.mu.Lock() c.echo = false c.mu.Unlock() // Queue sub on nc1. _, err = nc1.QueueSubscribe("foo", "bar", func(*nats.Msg) {}) if err != nil { t.Fatalf("Error on subscribe: %v", err) } nc1.Flush() nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() n := int32(0) cb := func(m *nats.Msg) { atomic.AddInt32(&n, 1) } _, err = nc2.QueueSubscribe("foo", "bar", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } nc2.Flush() // Now publish 100 messages on nc1 which does not allow echo. for i := 0; i < 100; i++ { nc1.Publish("foo", []byte("Hello")) } nc1.Flush() nc2.Flush() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { num := atomic.LoadInt32(&n) if num != int32(100) { return fmt.Errorf("Expected all the msgs to be received by nc2, got %d\n", num) } return nil }) } func TestClientUnSub(t *testing.T) { _, c, cr := setupClient() defer c.close() num := 1 // SUB/PUB subs := []byte("SUB foo 1\r\nSUB foo 2\r\n") unsub := []byte("UNSUB 1\r\n") pub := []byte("PUB foo bar 5\r\nhello\r\n") op := []byte{} op = append(op, subs...) op = append(op, unsub...) op = append(op, pub...) go c.parseAndClose(op) var received int for ; ; received++ { l, err := cr.ReadString('\n') if err != nil { break } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if matches[SID_INDEX] != "2" { t.Fatalf("Received msg on unsubscribed subscription!\n") } checkPayload(cr, []byte("hello\r\n"), t) } if received != num { t.Fatalf("Received wrong # of msgs: %d vs %d\n", received, num) } } func TestClientUnSubMax(t *testing.T) { _, c, cr := setupClient() defer c.close() num := 10 exp := 5 // SUB/PUB subs := []byte("SUB foo 1\r\n") unsub := []byte("UNSUB 1 5\r\n") pub := []byte("PUB foo bar 5\r\nhello\r\n") op := []byte{} op = append(op, subs...) op = append(op, unsub...) for i := 0; i < num; i++ { op = append(op, pub...) } go c.parseAndClose(op) var received int for ; ; received++ { l, err := cr.ReadString('\n') if err != nil { break } matches := msgPat.FindAllStringSubmatch(l, -1)[0] if matches[SID_INDEX] != "1" { t.Fatalf("Received msg on unsubscribed subscription!\n") } checkPayload(cr, []byte("hello\r\n"), t) } if received != exp { t.Fatalf("Received wrong # of msgs: %d vs %d\n", received, exp) } } func TestClientAutoUnsubExactReceived(t *testing.T) { _, c, _ := setupClient() defer c.close() // SUB/PUB subs := []byte("SUB foo 1\r\n") unsub := []byte("UNSUB 1 1\r\n") pub := []byte("PUB foo bar 2\r\nok\r\n") op := []byte{} op = append(op, subs...) op = append(op, unsub...) op = append(op, pub...) c.parse(op) // We should not have any subscriptions in place here. if len(c.subs) != 0 { t.Fatalf("Wrong number of subscriptions: expected 0, got %d\n", len(c.subs)) } } func TestClientUnsubAfterAutoUnsub(t *testing.T) { _, c, _ := setupClient() defer c.close() // SUB/UNSUB/UNSUB subs := []byte("SUB foo 1\r\n") asub := []byte("UNSUB 1 1\r\n") unsub := []byte("UNSUB 1\r\n") op := []byte{} op = append(op, subs...) op = append(op, asub...) op = append(op, unsub...) c.parse(op) // We should not have any subscriptions in place here. if len(c.subs) != 0 { t.Fatalf("Wrong number of subscriptions: expected 0, got %d\n", len(c.subs)) } } func TestClientRemoveSubsOnDisconnect(t *testing.T) { s, c, _ := setupClient() defer c.close() subs := []byte("SUB foo 1\r\nSUB bar 2\r\n") c.parse(subs) if s.NumSubscriptions() != 2 { t.Fatalf("Should have 2 subscriptions, got %d\n", s.NumSubscriptions()) } c.closeConnection(ClientClosed) checkExpectedSubs(t, 0, s) } func TestClientDoesNotAddSubscriptionsWhenConnectionClosed(t *testing.T) { _, c, _ := setupClient() c.close() subs := []byte("SUB foo 1\r\nSUB bar 2\r\n") c.parse(subs) if c.acc.sl.Count() != 0 { t.Fatalf("Should have no subscriptions after close, got %d\n", c.acc.sl.Count()) } } func TestClientMapRemoval(t *testing.T) { s, c, _ := setupClient() c.close() checkClientsCount(t, s, 0) } func TestAuthorizationTimeout(t *testing.T) { serverOptions := DefaultOptions() serverOptions.Authorization = "my_token" serverOptions.AuthTimeout = 0.4 s := RunServer(serverOptions) defer s.Shutdown() conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", serverOptions.Host, serverOptions.Port)) if err != nil { t.Fatalf("Error dialing server: %v\n", err) } defer conn.Close() client := bufio.NewReaderSize(conn, maxBufSize) if _, err := client.ReadString('\n'); err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } time.Sleep(3 * secondsToDuration(serverOptions.AuthTimeout)) l, err := client.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } if !strings.Contains(l, "Authentication Timeout") { t.Fatalf("Authentication Timeout response incorrect: %q\n", l) } } // This is from bug report #18 func TestTwoTokenPubMatchSingleTokenSub(t *testing.T) { _, c, cr := setupClient() defer c.close() test := "PUB foo.bar 5\r\nhello\r\nSUB foo 1\r\nPING\r\nPUB foo.bar 5\r\nhello\r\nPING\r\n" c.parseAsync(test) l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response incorrect: %q\n", l) } // Expect just a pong, no match should exist here.. l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "PONG\r\n") { t.Fatalf("PONG response was expected, got: %q\n", l) } } func TestUnsubRace(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("nats://%s:%d", s.getOpts().Host, s.Addr().(*net.TCPAddr).Port, ) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error creating client to %s: %v\n", url, err) } defer nc.Close() ncp, err := nats.Connect(url) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer ncp.Close() sub, _ := nc.Subscribe("foo", func(m *nats.Msg) { // Just eat it.. }) nc.Flush() var wg sync.WaitGroup wg.Add(1) go func() { for i := 0; i < 10000; i++ { ncp.Publish("foo", []byte("hello")) } wg.Done() }() time.Sleep(5 * time.Millisecond) sub.Unsubscribe() wg.Wait() } func TestClientCloseTLSConnection(t *testing.T) { opts, err := ProcessConfigFile("./configs/tls.conf") if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.TLSTimeout = 100 opts.NoLog = true opts.NoSigs = true s := RunServer(opts) defer s.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) conn, err := net.DialTimeout("tcp", endpoint, 2*time.Second) if err != nil { t.Fatalf("Unexpected error on dial: %v", err) } defer conn.Close() br := bufio.NewReaderSize(conn, 100) if _, err := br.ReadString('\n'); err != nil { t.Fatalf("Unexpected error reading INFO: %v", err) } tlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true}) defer tlsConn.Close() if err := tlsConn.Handshake(); err != nil { t.Fatalf("Unexpected error during handshake: %v", err) } br = bufio.NewReaderSize(tlsConn, 100) connectOp := []byte("CONNECT {\"user\":\"derek\",\"pass\":\"foo\",\"verbose\":false,\"pedantic\":false,\"tls_required\":true}\r\n") if _, err := tlsConn.Write(connectOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } if _, err := tlsConn.Write([]byte("PING\r\n")); err != nil { t.Fatalf("Unexpected error writing PING: %v", err) } if _, err := br.ReadString('\n'); err != nil { t.Fatalf("Unexpected error reading PONG: %v", err) } // Check that client is registered. checkClientsCount(t, s, 1) var cli *client s.mu.Lock() for _, c := range s.clients { cli = c break } s.mu.Unlock() if cli == nil { t.Fatal("Did not register client on time") } // Test GetTLSConnectionState state := cli.GetTLSConnectionState() if state == nil { t.Error("GetTLSConnectionState() returned nil") } // Test RemoteAddress addr := cli.RemoteAddress() if addr == nil { t.Error("RemoteAddress() returned nil") } if addr.(*net.TCPAddr).IP.String() != "127.0.0.1" { t.Error("RemoteAddress() returned incorrect ip " + addr.String()) } // Fill the buffer. We want to timeout on write so that nc.Close() // would block due to a write that cannot complete. buf := make([]byte, 64*1024) done := false for !done { cli.nc.SetWriteDeadline(time.Now().Add(time.Second)) if _, err := cli.nc.Write(buf); err != nil { done = true } cli.nc.SetWriteDeadline(time.Time{}) } ch := make(chan bool) go func() { select { case <-ch: return case <-time.After(3 * time.Second): fmt.Println("!!!! closeConnection is blocked, test will hang !!!") return } }() // Close the client cli.closeConnection(ClientClosed) ch <- true } // This tests issue #558 func TestWildcardCharsInLiteralSubjectWorks(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() ch := make(chan bool, 1) // This subject is a literal even though it contains `*` and `>`, // they are not treated as wildcards. subj := "foo.bar,*,>,baz" cb := func(_ *nats.Msg) { ch <- true } for i := 0; i < 2; i++ { sub, err := nc.Subscribe(subj, cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := nc.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } if err := nc.LastError(); err != nil { t.Fatalf("Server reported error: %v", err) } if err := nc.Publish(subj, []byte("msg")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: case <-time.After(time.Second): t.Fatalf("Should have received the message") } if err := sub.Unsubscribe(); err != nil { t.Fatalf("Error on unsubscribe: %v", err) } } } // This test ensures that coalescing into the fixed-size output // queues works as expected. When bytes are queued up, they should // not overflow a buffer until the capacity is exceeded, at which // point a new buffer should be added. func TestClientOutboundQueueCoalesce(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() clients := s.GlobalAccount().getClients() if len(clients) != 1 { t.Fatal("Expecting a client to exist") } client := clients[0] client.mu.Lock() defer client.mu.Unlock() // First up, queue something small into the queue. client.queueOutbound([]byte{1, 2, 3, 4, 5}) if len(client.out.nb) != 1 { t.Fatal("Expecting a single queued buffer") } if l := len(client.out.nb[0]); l != 5 { t.Fatalf("Expecting only 5 bytes in the first queued buffer, found %d instead", l) } // Then queue up a few more bytes, but not enough // to overflow into the next buffer. client.queueOutbound([]byte{6, 7, 8, 9, 10}) if len(client.out.nb) != 1 { t.Fatal("Expecting a single queued buffer") } if l := len(client.out.nb[0]); l != 10 { t.Fatalf("Expecting 10 bytes in the first queued buffer, found %d instead", l) } // Finally, queue up something that is guaranteed // to overflow. b := nbPoolSmall.Get().(*[nbPoolSizeSmall]byte)[:] b = b[:cap(b)] client.queueOutbound(b) if len(client.out.nb) != 2 { t.Fatal("Expecting buffer to have overflowed") } if l := len(client.out.nb[0]); l != cap(b) { t.Fatalf("Expecting %d bytes in the first queued buffer, found %d instead", cap(b), l) } if l := len(client.out.nb[1]); l != 10 { t.Fatalf("Expecting 10 bytes in the second queued buffer, found %d instead", l) } } func TestClientTraceRace(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() // Activate trace logging s.SetLogger(&DummyLogger{}, false, true) nc1, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() total := 10000 count := 0 ch := make(chan bool, 1) if _, err := nc1.Subscribe("foo", func(_ *nats.Msg) { count++ if count == total { ch <- true } }); err != nil { t.Fatalf("Error on subscribe: %v", err) } nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for i := 0; i < total; i++ { nc1.Publish("bar", []byte("hello")) } }() for i := 0; i < total; i++ { nc2.Publish("foo", []byte("hello")) } if err := wait(ch); err != nil { t.Fatal("Did not get all our messages") } wg.Wait() } func TestClientUserInfo(t *testing.T) { pnkey := "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" c := &client{ cid: 1024, opts: ClientOpts{ Nkey: pnkey, }, } got := c.getAuthUser() expected := `Nkey "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI"` if got != expected { t.Errorf("Expected %q, got %q", expected, got) } c = &client{ cid: 1024, opts: ClientOpts{ Username: "foo", }, } got = c.getAuthUser() expected = `User "foo"` if got != expected { t.Errorf("Expected %q, got %q", expected, got) } c = &client{ cid: 1024, opts: ClientOpts{}, } got = c.getAuthUser() expected = `User "N/A"` if got != expected { t.Errorf("Expected %q, got %q", expected, got) } } type captureWarnLogger struct { DummyLogger warn chan string } func (l *captureWarnLogger) Warnf(format string, v ...any) { select { case l.warn <- fmt.Sprintf(format, v...): default: } } func TestReadloopWarning(t *testing.T) { readLoopReportThreshold = 100 * time.Millisecond defer func() { readLoopReportThreshold = readLoopReport }() opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() l := &captureWarnLogger{warn: make(chan string, 1)} s.SetLogger(l, false, false) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc := natsConnect(t, url) defer nc.Close() natsSubSync(t, nc, "foo") natsFlush(t, nc) cid, _ := nc.GetClientID() sender := natsConnect(t, url) defer sender.Close() wg := sync.WaitGroup{} wg.Add(1) c := s.getClient(cid) c.mu.Lock() go func() { defer wg.Done() time.Sleep(250 * time.Millisecond) c.mu.Unlock() }() natsPub(t, sender, "foo", make([]byte, 100)) natsFlush(t, sender) select { case warn := <-l.warn: if !strings.Contains(warn, "Readloop") { t.Fatalf("unexpected warning: %v", warn) } case <-time.After(2 * time.Second): t.Fatalf("No warning printed") } wg.Wait() } func TestTraceMsg(t *testing.T) { c := &client{} // Enable message trace c.trace = true cases := []struct { Desc string Msg []byte Wanted string MaxTracedMsgLen int }{ { Desc: "normal length", Msg: []byte(fmt.Sprintf("normal%s", CR_LF)), Wanted: " - <<- MSG_PAYLOAD: [\"normal\"]", MaxTracedMsgLen: 10, }, { Desc: "over length", Msg: []byte(fmt.Sprintf("over length%s", CR_LF)), Wanted: " - <<- MSG_PAYLOAD: [\"over lengt...\"]", MaxTracedMsgLen: 10, }, { Desc: "unlimited length", Msg: []byte(fmt.Sprintf("unlimited length%s", CR_LF)), Wanted: " - <<- MSG_PAYLOAD: [\"unlimited length\"]", MaxTracedMsgLen: 0, }, { Desc: "negative max traced msg len", Msg: []byte(fmt.Sprintf("negative max traced msg len%s", CR_LF)), Wanted: " - <<- MSG_PAYLOAD: [\"negative max traced msg len\"]", MaxTracedMsgLen: -1, }, } for _, ut := range cases { c.srv = &Server{ opts: &Options{MaxTracedMsgLen: ut.MaxTracedMsgLen}, } c.srv.SetLogger(&DummyLogger{}, true, true) c.traceMsg(ut.Msg) got := c.srv.logging.logger.(*DummyLogger).Msg if !reflect.DeepEqual(ut.Wanted, got) { t.Errorf("Desc: %s. Msg %q. Traced msg want: %s, got: %s", ut.Desc, ut.Msg, ut.Wanted, got) } } } func TestClientMaxPending(t *testing.T) { opts := DefaultOptions() opts.MaxPending = math.MaxInt32 + 1 s := RunServer(opts) defer s.Shutdown() nc := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) defer nc.Close() sub := natsSubSync(t, nc, "foo") natsPub(t, nc, "foo", []byte("msg")) natsNexMsg(t, sub, 100*time.Millisecond) } func TestResponsePermissions(t *testing.T) { for i, test := range []struct { name string perms *ResponsePermission }{ {"max_msgs", &ResponsePermission{MaxMsgs: 2, Expires: time.Hour}}, {"no_expire_limit", &ResponsePermission{MaxMsgs: 3, Expires: -1 * time.Millisecond}}, {"expire", &ResponsePermission{MaxMsgs: 1000, Expires: 100 * time.Millisecond}}, {"no_msgs_limit", &ResponsePermission{MaxMsgs: -1, Expires: 100 * time.Millisecond}}, } { t.Run(test.name, func(t *testing.T) { opts := DefaultOptions() u1 := &User{ Username: "service", Password: "pwd", Permissions: &Permissions{Response: test.perms}, } u2 := &User{Username: "ivan", Password: "pwd"} opts.Users = []*User{u1, u2} s := RunServer(opts) defer s.Shutdown() svcNC := natsConnect(t, fmt.Sprintf("nats://service:pwd@%s:%d", opts.Host, opts.Port)) defer svcNC.Close() reqSub := natsSubSync(t, svcNC, "request") natsFlush(t, svcNC) nc := natsConnect(t, fmt.Sprintf("nats://ivan:pwd@%s:%d", opts.Host, opts.Port)) defer nc.Close() replySub := natsSubSync(t, nc, "reply") natsPubReq(t, nc, "request", "reply", []byte("req1")) req1 := natsNexMsg(t, reqSub, 100*time.Millisecond) checkFailed := func(t *testing.T) { t.Helper() if reply, err := replySub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { if reply != nil { t.Fatalf("Expected to receive timeout, got reply=%q", reply.Data) } else { t.Fatalf("Unexpected error: %v", err) } } } switch i { case 0: // Should allow only 2 replies... for i := 0; i < 10; i++ { natsPub(t, svcNC, req1.Reply, []byte("reply")) } natsNexMsg(t, replySub, 100*time.Millisecond) natsNexMsg(t, replySub, 100*time.Millisecond) // The next should fail... checkFailed(t) case 1: // Expiration is set to -1ms, which should count as infinite... natsPub(t, svcNC, req1.Reply, []byte("reply")) // Sleep a bit before next send time.Sleep(50 * time.Millisecond) natsPub(t, svcNC, req1.Reply, []byte("reply")) // Make sure we receive both natsNexMsg(t, replySub, 100*time.Millisecond) natsNexMsg(t, replySub, 100*time.Millisecond) case 2: fallthrough case 3: // Expire set to 100ms so make sure we wait more between // next publish natsPub(t, svcNC, req1.Reply, []byte("reply")) time.Sleep(200 * time.Millisecond) natsPub(t, svcNC, req1.Reply, []byte("reply")) // Should receive one, and fail on the other natsNexMsg(t, replySub, 100*time.Millisecond) checkFailed(t) } // When testing expiration, sleep before sending next reply if i >= 2 { time.Sleep(400 * time.Millisecond) } }) } } func TestPingNotSentTooSoon(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() doneCh := make(chan bool, 1) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { s.Connz(nil) select { case <-doneCh: return case <-time.After(time.Millisecond): } } }() for i := 0; i < 100; i++ { nc, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Close() } close(doneCh) wg.Wait() c, br, _ := newClientForServer(s) defer c.close() connectOp := []byte("CONNECT {\"user\":\"ivan\",\"pass\":\"bar\"}\r\n") c.parse(connectOp) // Since client has not send PING, having server try to send RTT ping // to client should not do anything if c.sendRTTPing() { t.Fatalf("RTT ping should not have been sent") } // Speed up detection of time elapsed by moving the c.start to more than // 2 secs in the past. c.mu.Lock() c.start = time.Unix(0, c.start.UnixNano()-int64(maxNoRTTPingBeforeFirstPong+time.Second)) c.mu.Unlock() errCh := make(chan error, 1) go func() { l, _ := br.ReadString('\n') if l != "PING\r\n" { errCh <- fmt.Errorf("expected to get PING, got %s", l) return } errCh <- nil }() if !c.sendRTTPing() { t.Fatalf("RTT ping should have been sent") } wg.Wait() if e := <-errCh; e != nil { t.Fatal(e.Error()) } } func TestClientCheckUseOfGWReplyPrefix(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() ech := make(chan error, 1) nc, err := nats.Connect(s.ClientURL(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) { ech <- e })) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Expect to fail if publish on gateway reply prefix nc.Publish(gwReplyPrefix+"anything", []byte("should fail")) // Wait for publish violation error select { case e := <-ech: if e == nil || !strings.Contains(strings.ToLower(e.Error()), "violation for publish") { t.Fatalf("Expected violation error, got %v", e) } case <-time.After(time.Second): t.Fatalf("Did not receive permissions violation error") } // Now publish a message with a reply set to the prefix, // it should be rejected too. nc.PublishRequest("foo", gwReplyPrefix+"anything", []byte("should fail")) // Wait for publish violation error with reply select { case e := <-ech: if e == nil || !strings.Contains(strings.ToLower(e.Error()), "violation for publish with reply") { t.Fatalf("Expected violation error, got %v", e) } case <-time.After(time.Second): t.Fatalf("Did not receive permissions violation error") } } func TestNoClientLeakOnSlowConsumer(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer c.Close() cr := bufio.NewReader(c) // Wait for INFO... line, _, _ := cr.ReadLine() var info serverInfo if err = json.Unmarshal(line[5:], &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } // Send our connect if _, err := c.Write([]byte("CONNECT {\"verbose\": false}\r\nSUB foo 1\r\nPING\r\n")); err != nil { t.Fatalf("Error sending CONNECT and SUB: %v", err) } // Wait for PONG line, _, _ = cr.ReadLine() if string(line) != "PONG" { t.Fatalf("Expected 'PONG' but got %q", line) } // Get the client from server map cli := s.GetClient(info.CID) if cli == nil { t.Fatalf("No client registered") } // Change the write deadline to very low value cli.mu.Lock() cli.out.wdl = time.Nanosecond cli.mu.Unlock() nc := natsConnect(t, s.ClientURL()) defer nc.Close() // Send some messages to cause write deadline error on "cli" payload := make([]byte, 1000) for i := 0; i < 100; i++ { natsPub(t, nc, "foo", payload) } natsFlush(t, nc) nc.Close() // Now make sure that the number of clients goes to 0. checkClientsCount(t, s, 0) } func TestClientSlowConsumerWithoutConnect(t *testing.T) { opts := DefaultOptions() opts.WriteDeadline = 100 * time.Millisecond s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("127.0.0.1:%d", opts.Port) c, err := net.Dial("tcp", url) if err != nil { t.Fatalf("Error on dial: %v", err) } defer c.Close() c.Write([]byte("SUB foo 1\r\n")) payload := make([]byte, 10000) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() for i := 0; i < 10000; i++ { nc.Publish("foo", payload) } nc.Flush() checkFor(t, time.Second, 15*time.Millisecond, func() error { // Expect slow consumer.. if n := atomic.LoadInt64(&s.slowConsumers); n != 1 { return fmt.Errorf("Expected 1 slow consumer, got: %v", n) } if n := s.scStats.clients.Load(); n != 1 { return fmt.Errorf("Expected 1 slow consumer, got: %v", n) } return nil }) varz, err := s.Varz(nil) if err != nil { t.Fatal(err) } if varz.SlowConsumersStats.Clients != 1 { t.Error("Expected a slow consumer client in varz") } } func TestClientNoSlowConsumerIfConnectExpected(t *testing.T) { opts := DefaultOptions() opts.Username = "ivan" opts.Password = "pass" // Make it very slow so that the INFO sent to client fails... opts.WriteDeadline = time.Nanosecond s := RunServer(opts) defer s.Shutdown() // Expect server to close the connection, but will bump the slow // consumer count. nc, err := nats.Connect(fmt.Sprintf("nats://ivan:pass@%s:%d", opts.Host, opts.Port)) if err == nil { nc.Close() t.Fatal("Expected connect error") } if n := atomic.LoadInt64(&s.slowConsumers); n != 0 { t.Fatalf("Expected 0 slow consumer, got: %v", n) } } func TestClientIPv6Address(t *testing.T) { opts := DefaultOptions() opts.Host = "0.0.0.0" s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://[::1]:%v", opts.Port)) // Travis may not accept IPv6, in that case, skip the test. if err != nil { t.Skipf("Skipping test because could not connect: %v", err) } defer nc.Close() cid, _ := nc.GetClientID() c := s.GetClient(cid) c.mu.Lock() ncs := c.String() c.mu.Unlock() if !strings.HasPrefix(ncs, "[::1]") { t.Fatalf("Wrong string representation of an IPv6 address: %q", ncs) } } func TestPBNotIncreasedOnMaxPending(t *testing.T) { opts := DefaultOptions() opts.MaxPending = 100 s := &Server{opts: opts} c := &client{srv: s} c.initClient() c.mu.Lock() c.queueOutbound(make([]byte, 200)) pb := c.out.pb c.mu.Unlock() if pb != 0 { t.Fatalf("c.out.pb should be 0, got %v", pb) } } type testConnWritePartial struct { net.Conn partial bool buf bytes.Buffer } func (c *testConnWritePartial) Write(p []byte) (int, error) { n := len(p) if c.partial { n = n/2 + 1 } return c.buf.Write(p[:n]) } func (c *testConnWritePartial) RemoteAddr() net.Addr { return nil } func (c *testConnWritePartial) SetWriteDeadline(_ time.Time) error { return nil } func TestFlushOutboundNoSliceReuseIfPartial(t *testing.T) { opts := DefaultOptions() opts.MaxPending = 1024 s := &Server{opts: opts} fakeConn := &testConnWritePartial{partial: true} c := &client{srv: s, nc: fakeConn} c.initClient() bufs := [][]byte{ []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), []byte("------"), []byte("0123456789"), } expected := bytes.Buffer{} for _, buf := range bufs { expected.Write(buf) c.mu.Lock() c.queueOutbound(buf) c.flushOutbound() fakeConn.partial = false c.mu.Unlock() } // Ensure everything is flushed. for done := false; !done; { c.mu.Lock() if c.out.pb > 0 { c.flushOutbound() } else { done = true } c.mu.Unlock() } if !bytes.Equal(expected.Bytes(), fakeConn.buf.Bytes()) { t.Fatalf("Expected\n%q\ngot\n%q", expected.String(), fakeConn.buf.String()) } } type captureNoticeLogger struct { DummyLogger notices []string } func (l *captureNoticeLogger) Noticef(format string, v ...any) { l.Lock() l.notices = append(l.notices, fmt.Sprintf(format, v...)) l.Unlock() } func TestCloseConnectionLogsReason(t *testing.T) { o1 := DefaultOptions() s1 := RunServer(o1) defer s1.Shutdown() l := &captureNoticeLogger{} s1.SetLogger(l, true, true) o2 := DefaultOptions() o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) s2.Shutdown() checkFor(t, time.Second, 15*time.Millisecond, func() error { if s1.NumRoutes() != 0 { return fmt.Errorf("route still connected") } return nil }) // Now check that s1 has logged that the connection is closed and that the reason is included. ok := false l.Lock() for _, n := range l.notices { if strings.Contains(n, "connection closed: "+ClientClosed.String()) { ok = true break } } l.Unlock() if !ok { t.Fatal("Log does not contain closed reason") } } func TestCloseConnectionVeryEarly(t *testing.T) { for _, test := range []struct { name string useTLS bool }{ {"no_tls", false}, {"tls", true}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() if test.useTLS { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConfig, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } o.TLSConfig = tlsConfig } s := RunServer(o) defer s.Shutdown() // The issue was with a connection that would break right when // server was sending the INFO. Creating a bare TCP connection // and closing it right away won't help reproduce the problem. // So testing in 2 steps. // Get a normal TCP connection to the server. c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", o.Port)) if err != nil { t.Fatalf("Unable to create tcp connection") } // Now close it. c.Close() // Wait that num clients falls to 0. checkClientsCount(t, s, 0) // Call again with this closed connection. Alternatively, we // would have to call with a fake connection that implements // net.Conn but returns an error on Write. s.createClient(c) // This connection should not have been added to the server. checkClientsCount(t, s, 0) }) } } type connAddrString struct { net.Addr } func (a *connAddrString) String() string { return "[fe80::abc:def:ghi:123%utun0]:4222" } type connString struct { net.Conn } func (c *connString) RemoteAddr() net.Addr { return &connAddrString{} } func TestClientConnectionName(t *testing.T) { s, err := NewServer(DefaultOptions()) if err != nil { t.Fatalf("Error creating server: %v", err) } l := &DummyLogger{} s.SetLogger(l, true, true) for _, test := range []struct { name string kind int kindStr string ws bool mqtt bool }{ {"client", CLIENT, "cid:", false, false}, {"ws client", CLIENT, "wid:", true, false}, {"mqtt client", CLIENT, "mid:", false, true}, {"route", ROUTER, "rid:", false, false}, {"gateway", GATEWAY, "gid:", false, false}, {"leafnode", LEAF, "lid:", false, false}, } { t.Run(test.name, func(t *testing.T) { c := &client{srv: s, nc: &connString{}, kind: test.kind} if test.ws { c.ws = &websocket{} } if test.mqtt { c.mqtt = &mqtt{} } c.initClient() if host := "fe80::abc:def:ghi:123%utun0"; host != c.host { t.Fatalf("expected host to be %q, got %q", host, c.host) } if port := uint16(4222); port != c.port { t.Fatalf("expected port to be %v, got %v", port, c.port) } checkLog := func(suffix string) { t.Helper() l.Lock() msg := l.Msg l.Unlock() if strings.Contains(msg, "(MISSING)") { t.Fatalf("conn name was not escaped properly, got MISSING: %s", msg) } if !strings.Contains(l.Msg, test.kindStr) { t.Fatalf("expected kind to be %q, got: %s", test.kindStr, msg) } if !strings.HasSuffix(l.Msg, suffix) { t.Fatalf("expected statement to end with %q, got %s", suffix, msg) } } c.Debugf("debug: %v", 1) checkLog(" 1") c.Tracef("trace: %s", "2") checkLog(" 2") c.Warnf("warn: %s %d", "3", 4) checkLog(" 3 4") c.Errorf("error: %v %s", 5, "6") checkLog(" 5 6") }) } } func TestClientLimits(t *testing.T) { accKp, err := nkeys.CreateAccount() if err != nil { t.Fatalf("Error creating account key: %v", err) } uKp, err := nkeys.CreateUser() if err != nil { t.Fatalf("Error creating user key: %v", err) } uPub, err := uKp.PublicKey() if err != nil { t.Fatalf("Error obtaining publicKey: %v", err) } s, err := NewServer(DefaultOptions()) if err != nil { t.Fatalf("Error creating server: %v", err) } for _, test := range []struct { client int32 acc int32 srv int32 expect int32 }{ // all identical {1, 1, 1, 1}, {-1, -1, 0, -1}, // only one value unlimited {1, -1, 0, 1}, {-1, 1, 0, 1}, {-1, -1, 1, 1}, // all combinations of distinct values {1, 2, 3, 1}, {1, 3, 2, 1}, {2, 1, 3, 1}, {2, 3, 1, 1}, {3, 1, 2, 1}, {3, 2, 1, 1}, } { t.Run("", func(t *testing.T) { s.opts.MaxPayload = test.srv s.opts.MaxSubs = int(test.srv) c := &client{srv: s, acc: &Account{ limits: limits{mpay: test.acc, msubs: test.acc}, }} uc := jwt.NewUserClaims(uPub) uc.Limits.Subs = int64(test.client) uc.Limits.Payload = int64(test.client) c.opts.JWT, err = uc.Encode(accKp) if err != nil { t.Fatalf("Error encoding jwt: %v", err) } c.applyAccountLimits() if c.mpay != test.expect { t.Fatalf("payload %d not as expected %d", c.mpay, test.expect) } if c.msubs != test.expect { t.Fatalf("subscriber %d not as expected %d", c.msubs, test.expect) } }) } } func TestClientClampMaxSubsErrReport(t *testing.T) { maxSubLimitReportThreshold = int64(100 * time.Millisecond) defer func() { maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold }() o1 := DefaultOptions() o1.MaxSubs = 1 o1.LeafNode.Host = "127.0.0.1" o1.LeafNode.Port = -1 s1 := RunServer(o1) defer s1.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} s1.SetLogger(l, false, false) o2 := DefaultOptions() o2.Cluster.Name = "xyz" u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", o1.LeafNode.Port)) o2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} s2 := RunServer(o2) defer s2.Shutdown() checkLeafNodeConnected(t, s1) checkLeafNodeConnected(t, s2) nc := natsConnect(t, s2.ClientURL()) defer nc.Close() natsSubSync(t, nc, "foo") natsSubSync(t, nc, "bar") // Make sure we receive only 1 check := func() { t.Helper() for i := 0; i < 2; i++ { select { case errStr := <-l.errCh: if i > 0 { t.Fatalf("Should not have logged a second time: %s", errStr) } if !strings.Contains(errStr, "maximum subscriptions") { t.Fatalf("Unexpected error: %s", errStr) } case <-time.After(300 * time.Millisecond): if i == 0 { t.Fatal("Error should have been logged") } } } } check() // The above will have waited long enough to clear the report threshold. // So create two new subs and check again that we get only 1 report. natsSubSync(t, nc, "baz") natsSubSync(t, nc, "bat") check() } func TestClientDenySysGroupSub(t *testing.T) { s := RunServer(DefaultOptions()) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL(), nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) require_NoError(t, err) defer nc.Close() _, err = nc.QueueSubscribeSync("foo", sysGroup) require_NoError(t, err) nc.Flush() err = nc.LastError() require_Error(t, err) require_Contains(t, err.Error(), "Permissions Violation") } func TestClientAuthRequiredNoAuthUser(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ { user: user, password: pass } ] } } no_auth_user: user `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() if nc.AuthRequired() { t.Fatalf("Expected AuthRequired to be false due to 'no_auth_user'") } } func TestClientUserInfoReq(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 PERMS = { publish = { allow: "$SYS.REQ.>", deny: "$SYS.REQ.ACCOUNT.>" } subscribe = "_INBOX.>" allow_responses: true } accounts: { A: { users: [ { user: dlc, password: pass, permissions: $PERMS } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } no_auth_user: dlc `)) defer removeFile(t, conf) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() resp, err := nc.Request("$SYS.REQ.USER.INFO", nil, time.Second) require_NoError(t, err) response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) dlc := &UserInfo{ UserID: "dlc", Account: "A", Permissions: &Permissions{ Publish: &SubjectPermission{ Allow: []string{"$SYS.REQ.>"}, Deny: []string{"$SYS.REQ.ACCOUNT.>"}, }, Subscribe: &SubjectPermission{ Allow: []string{"_INBOX.>"}, }, Response: &ResponsePermission{ MaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS, Expires: DEFAULT_ALLOW_RESPONSE_EXPIRATION, }, }, } if !reflect.DeepEqual(dlc, userInfo) { t.Fatalf("User info for %q did not match", "dlc") } // Make sure system users work ok too. nc, err = nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() resp, err = nc.Request("$SYS.REQ.USER.INFO", nil, time.Second) require_NoError(t, err) response = ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo = response.Data.(*UserInfo) admin := &UserInfo{ UserID: "admin", Account: "$SYS", } if !reflect.DeepEqual(admin, userInfo) { t.Fatalf("User info for %q did not match", "admin") } } func TestTLSClientHandshakeFirst(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" timeout: 1 first: %s } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "true"))) s, o := RunServerWithConfig(conf) defer s.Shutdown() connect := func(tlsfirst, expectedOk bool) { opts := []nats.Option{nats.RootCAs("../test/configs/certs/ca.pem")} if tlsfirst { opts = append(opts, nats.TLSHandshakeFirst()) } nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", o.Port), opts...) if expectedOk { defer nc.Close() if err != nil { t.Fatalf("Unexpected error: %v", err) } if tlsfirst { cz, err := s.Connz(nil) if err != nil { t.Fatalf("Error getting connz: %v", err) } if !cz.Conns[0].TLSFirst { t.Fatal("Expected TLSFirst boolean to be set, it was not") } } } else if !expectedOk && err == nil { nc.Close() t.Fatal("Expected error, got none") } } // Server is TLS first, but client is not, so should fail. connect(false, false) // Now client is TLS first too, so should work. connect(true, true) // Config reload the server and disable tls first reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "false")) // Now if client wants TLS first, connection should fail. connect(true, false) // But if it does not, should be ok. connect(false, true) // Config reload the server again and enable tls first reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "true")) // If both client and server are TLS first, this should work. connect(true, true) } func TestTLSClientHandshakeFirstFallbackDelayConfigValues(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" timeout: 1 first: %s } ` for _, test := range []struct { name string val string first bool delay time.Duration }{ {"first as boolean true", "true", true, 0}, {"first as boolean false", "false", false, 0}, {"first as string true", "\"true\"", true, 0}, {"first as string false", "\"false\"", false, 0}, {"first as string on", "on", true, 0}, {"first as string off", "off", false, 0}, {"first as string auto", "auto", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY}, {"first as string auto_fallback", "auto_fallback", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY}, {"first as fallback duration", "300ms", true, 300 * time.Millisecond}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.val))) s, o := RunServerWithConfig(conf) defer s.Shutdown() if test.first { if !o.TLSHandshakeFirst { t.Fatal("Expected tls first to be true, was not") } if test.delay != o.TLSHandshakeFirstFallback { t.Fatalf("Expected fallback delay to be %v, got %v", test.delay, o.TLSHandshakeFirstFallback) } } else { if o.TLSHandshakeFirst { t.Fatal("Expected tls first to be false, was not") } if o.TLSHandshakeFirstFallback != 0 { t.Fatalf("Expected fallback delay to be 0, got %v", o.TLSHandshakeFirstFallback) } } }) } } type pauseAfterDial struct { delay time.Duration } func (d *pauseAfterDial) Dial(network, address string) (net.Conn, error) { c, err := net.Dial(network, address) if err != nil { return nil, err } time.Sleep(d.delay) return c, nil } func TestTLSClientHandshakeFirstFallbackDelay(t *testing.T) { // Using certificates with RSA 4K to make sure that the fallback does // not prevent a client with TLS first to successfully connect. tmpl := ` listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/tls/benchmark-server-cert-rsa-4096.pem" key_file: "./configs/certs/tls/benchmark-server-key-rsa-4096.pem" timeout: 1 first: %s } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "auto"))) s, o := RunServerWithConfig(conf) defer s.Shutdown() url := fmt.Sprintf("tls://localhost:%d", o.Port) d := &pauseAfterDial{delay: DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY + 100*time.Millisecond} // Connect a client without "TLS first" and it should be accepted. nc, err := nats.Connect(url, nats.SetCustomDialer(d), nats.Secure(&tls.Config{ ServerName: "reuben.nats.io", MinVersion: tls.VersionTLS12, }), nats.RootCAs("./configs/certs/tls/benchmark-ca-cert.pem")) require_NoError(t, err) defer nc.Close() // Check that the TLS first in monitoring is set to false cs, err := s.Connz(nil) require_NoError(t, err) if cs.Conns[0].TLSFirst { t.Fatal("Expected monitoring ConnInfo.TLSFirst to be false, it was not") } nc.Close() // Wait for the client to be removed checkClientsCount(t, s, 0) // Increase the fallback delay with config reload. reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "\"1s\"")) // This time, start the client with "TLS first". // We will also make sure that we did not wait for the fallback delay // in order to connect. start := time.Now() nc, err = nats.Connect(url, nats.SetCustomDialer(d), nats.Secure(&tls.Config{ ServerName: "reuben.nats.io", MinVersion: tls.VersionTLS12, }), nats.RootCAs("./configs/certs/tls/benchmark-ca-cert.pem"), nats.TLSHandshakeFirst()) require_NoError(t, err) require_True(t, time.Since(start) < 500*time.Millisecond) defer nc.Close() // Check that the TLS first in monitoring is set to true. cs, err = s.Connz(nil) require_NoError(t, err) if !cs.Conns[0].TLSFirst { t.Fatal("Expected monitoring ConnInfo.TLSFirst to be true, it was not") } nc.Close() } func TestTLSClientHandshakeFirstFallbackDelayAndAllowNonTLS(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" timeout: 1 first: %s } allow_non_tls: true ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "true"))) s, o := RunServerWithConfig(conf) defer s.Shutdown() // We first start with a server that has handshake first set to true // and allow_non_tls. In that case, only "TLS first" clients should be // accepted. url := fmt.Sprintf("tls://localhost:%d", o.Port) nc, err := nats.Connect(url, nats.RootCAs("../test/configs/certs/ca.pem"), nats.TLSHandshakeFirst()) require_NoError(t, err) defer nc.Close() // Check that the TLS first in monitoring is set to true cs, err := s.Connz(nil) require_NoError(t, err) if !cs.Conns[0].TLSFirst { t.Fatal("Expected monitoring ConnInfo.TLSFirst to be true, it was not") } nc.Close() // Client not using "TLS First" should fail. nc, err = nats.Connect(url, nats.RootCAs("../test/configs/certs/ca.pem")) if err == nil { nc.Close() t.Fatal("Expected connection to fail, it did not") } // And non TLS clients should also fail to connect. nc, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", o.Port)) if err == nil { nc.Close() t.Fatal("Expected connection to fail, it did not") } // Now we will replace TLS first in server with a fallback delay. reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "\"25ms\"")) // Clients with "TLS first" should still be able to connect nc, err = nats.Connect(url, nats.RootCAs("../test/configs/certs/ca.pem"), nats.TLSHandshakeFirst()) require_NoError(t, err) defer nc.Close() checkConnInfo := func(isTLS, isTLSFirst bool) { t.Helper() cs, err = s.Connz(nil) require_NoError(t, err) conn := cs.Conns[0] if !isTLS { if conn.TLSVersion != _EMPTY_ { t.Fatalf("Being a non TLS client, there should not be TLSVersion set, got %v", conn.TLSVersion) } if conn.TLSFirst { t.Fatal("Being a non TLS client, TLSFirst should not be set, but it was") } return } if isTLSFirst && !conn.TLSFirst { t.Fatal("Expected monitoring ConnInfo.TLSFirst to be true, it was not") } else if !isTLSFirst && conn.TLSFirst { t.Fatal("Expected monitoring ConnInfo.TLSFirst to be false, it was not") } nc.Close() checkClientsCount(t, s, 0) } checkConnInfo(true, true) // Clients with TLS but not "TLS first" should also be able to connect. nc, err = nats.Connect(url, nats.RootCAs("../test/configs/certs/ca.pem")) require_NoError(t, err) defer nc.Close() checkConnInfo(true, false) // And non TLS clients should also be able to connect. nc, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", o.Port)) require_NoError(t, err) defer nc.Close() checkConnInfo(false, false) } func TestTLSClientHandshakeFirstAndInProcessConnection(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" timeout: 1 first: true } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Check that we can create an in process connection that does not use TLS nc, err := nats.Connect(_EMPTY_, nats.InProcessServer(s)) require_NoError(t, err) defer nc.Close() if nc.TLSRequired() { t.Fatalf("Shouldn't have required TLS for in-process connection") } if _, err = nc.TLSConnectionState(); err == nil { t.Fatal("Should have got an error retrieving TLS connection state") } nc.Close() // If the client wants TLS, it should get a TLS connection. nc, err = nats.Connect(_EMPTY_, nats.InProcessServer(s), nats.RootCAs("../test/configs/certs/ca.pem")) require_NoError(t, err) defer nc.Close() if _, err = nc.TLSConnectionState(); err != nil { t.Fatal("Should have not got an error retrieving TLS connection state") } // However, the server would not have sent that TLS was required, // but instead it is available. if nc.TLSRequired() { t.Fatalf("Shouldn't have required TLS for in-process connection") } nc.Close() // The in-process connection with TLS and "TLS first" should also be working. nc, err = nats.Connect(_EMPTY_, nats.InProcessServer(s), nats.RootCAs("../test/configs/certs/ca.pem"), nats.TLSHandshakeFirst()) require_NoError(t, err) defer nc.Close() if !nc.TLSRequired() { t.Fatalf("The server should have sent that TLS is required") } if _, err = nc.TLSConnectionState(); err != nil { t.Fatal("Should have not got an error retrieving TLS connection state") } } func TestRemoveHeaderIfPrefixPresent(t *testing.T) { hdr := []byte("NATS/1.0\r\n\r\n") hdr = genHeader(hdr, "a", "1") hdr = genHeader(hdr, JSExpectedStream, "my-stream") hdr = genHeader(hdr, JSExpectedLastSeq, "22") hdr = genHeader(hdr, "b", "2") hdr = genHeader(hdr, JSExpectedLastSubjSeq, "24") hdr = genHeader(hdr, JSExpectedLastMsgId, "1") hdr = genHeader(hdr, "c", "3") hdr = removeHeaderIfPrefixPresent(hdr, "Nats-Expected-") if !bytes.Equal(hdr, []byte("NATS/1.0\r\na: 1\r\nb: 2\r\nc: 3\r\n\r\n")) { t.Fatalf("Expected headers to be stripped, got %q", hdr) } } func TestSliceHeader(t *testing.T) { hdr := []byte("NATS/1.0\r\n\r\n") hdr = genHeader(hdr, "a", "1") hdr = genHeader(hdr, JSExpectedStream, "my-stream") hdr = genHeader(hdr, JSExpectedLastSeq, "22") hdr = genHeader(hdr, "b", "2") hdr = genHeader(hdr, JSExpectedLastSubjSeq, "24") hdr = genHeader(hdr, JSExpectedLastMsgId, "1") hdr = genHeader(hdr, "c", "3") sliced := sliceHeader(JSExpectedLastSubjSeq, hdr) copied := getHeader(JSExpectedLastSubjSeq, hdr) require_NotNil(t, sliced) require_Equal(t, cap(sliced), 2) require_NotNil(t, copied) require_Equal(t, cap(copied), len(copied)) require_True(t, bytes.Equal(sliced, copied)) } func TestClientFlushOutboundNoSlowConsumer(t *testing.T) { opts := DefaultOptions() opts.MaxPending = 1024 * 1024 * 140 // 140MB opts.MaxPayload = 1024 * 1024 * 16 // 16MB opts.WriteDeadline = time.Second * 30 s := RunServer(opts) defer s.Shutdown() nc := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) defer nc.Close() proxy := newNetProxy(0, 1024*1024*8, 1024*1024*8, s.ClientURL()) // 8MB/s defer proxy.stop() wait := make(chan error) nca, err := nats.Connect(proxy.clientURL(), nats.NoCallbacksAfterClientClose()) require_NoError(t, err) defer nca.Close() nca.SetDisconnectErrHandler(func(c *nats.Conn, err error) { wait <- err close(wait) }) ncb, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer ncb.Close() _, err = nca.Subscribe("test", func(msg *nats.Msg) { wait <- nil }) require_NoError(t, err) // Publish 128MB of data onto the test subject. This will // mean that the outbound queue for nca has more than 64MB, // which is the max we will send into a single writev call. payload := make([]byte, 1024*1024*16) // 16MB for i := 0; i < 8; i++ { require_NoError(t, ncb.Publish("test", payload)) } // Get the client ID for nca. cid, err := nca.GetClientID() require_NoError(t, err) // Check that the client queue has more than 64MB queued // up in it. s.mu.RLock() ca := s.clients[cid] s.mu.RUnlock() ca.mu.Lock() pba := ca.out.pb ca.mu.Unlock() require_True(t, pba > 1024*1024*64) // Wait for our messages to be delivered. This will take // a few seconds as the client is limited to 8MB/s, so it // can't deliver messages to us as quickly as the other // client can publish them. var msgs int for err := range wait { require_NoError(t, err) msgs++ if msgs == 8 { break } } require_Equal(t, msgs, 8) } nats-server-2.10.27/server/closed_conns_test.go000066400000000000000000000156361477524627100215330ustar00rootroot00000000000000// Copyright 2018-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "net" "strings" "testing" "time" "github.com/nats-io/nats.go" ) func checkClosedConns(t *testing.T, s *Server, num int, wait time.Duration) { t.Helper() checkFor(t, wait, 5*time.Millisecond, func() error { if nc := s.numClosedConns(); nc != num { return fmt.Errorf("Closed conns expected to be %v, got %v", num, nc) } return nil }) } func checkTotalClosedConns(t *testing.T, s *Server, num uint64, wait time.Duration) { t.Helper() checkFor(t, wait, 5*time.Millisecond, func() error { if nc := s.totalClosedConns(); nc != num { return fmt.Errorf("Total closed conns expected to be %v, got %v", num, nc) } return nil }) } func TestClosedConnsAccounting(t *testing.T) { opts := DefaultOptions() opts.MaxClosedClients = 10 opts.NoSystemAccount = true s := RunServer(opts) defer s.Shutdown() wait := time.Second nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } id, _ := nc.GetClientID() nc.Close() checkClosedConns(t, s, 1, wait) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } if conns[0].Cid != id { t.Fatalf("Expected CID to be %d, got %d\n", id, conns[0].Cid) } // Now create 21 more for i := 0; i < 21; i++ { nc, err = nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Close() checkTotalClosedConns(t, s, uint64(i+2), wait) } checkClosedConns(t, s, opts.MaxClosedClients, wait) checkTotalClosedConns(t, s, 22, wait) conns = s.closedClients() if lc := len(conns); lc != opts.MaxClosedClients { t.Fatalf("len(conns) expected to be %d, got %d\n", opts.MaxClosedClients, lc) } // Set it to the start after overflow. cid := uint64(22 - opts.MaxClosedClients) for _, ci := range conns { cid++ if ci.Cid != cid { t.Fatalf("Expected cid of %d, got %d\n", cid, ci.Cid) } } } func TestClosedConnsSubsAccounting(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer nc.Close() // Now create some subscriptions numSubs := 10 for i := 0; i < numSubs; i++ { subj := fmt.Sprintf("foo.%d", i) nc.Subscribe(subj, func(m *nats.Msg) {}) } nc.Flush() nc.Close() checkClosedConns(t, s, 1, 20*time.Millisecond) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be 1, got %d\n", lc) } ci := conns[0] if len(ci.subs) != numSubs { t.Fatalf("Expected number of Subs to be %d, got %d\n", numSubs, len(ci.subs)) } } func checkReason(t *testing.T, reason string, expected ClosedState) { if !strings.Contains(reason, expected.String()) { t.Fatalf("Expected closed connection with `%s` state, got `%s`\n", expected, reason) } } func TestClosedAuthorizationTimeout(t *testing.T) { serverOptions := DefaultOptions() serverOptions.Authorization = "my_token" serverOptions.AuthTimeout = 0.4 s := RunServer(serverOptions) defer s.Shutdown() conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", serverOptions.Host, serverOptions.Port)) if err != nil { t.Fatalf("Error dialing server: %v\n", err) } defer conn.Close() checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, AuthenticationTimeout) } func TestClosedAuthorizationViolation(t *testing.T) { serverOptions := DefaultOptions() serverOptions.Authorization = "my_token" s := RunServer(serverOptions) defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err == nil { nc.Close() t.Fatal("Expected failure for connection") } checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, AuthenticationViolation) } func TestClosedUPAuthorizationViolation(t *testing.T) { serverOptions := DefaultOptions() serverOptions.Username = "my_user" serverOptions.Password = "my_secret" s := RunServer(serverOptions) defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err == nil { nc.Close() t.Fatal("Expected failure for connection") } url2 := fmt.Sprintf("nats://my_user:wrong_pass@%s:%d", opts.Host, opts.Port) nc, err = nats.Connect(url2) if err == nil { nc.Close() t.Fatal("Expected failure for connection") } checkClosedConns(t, s, 2, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 2 { t.Fatalf("len(conns) expected to be %d, got %d\n", 2, lc) } checkReason(t, conns[0].Reason, AuthenticationViolation) checkReason(t, conns[1].Reason, AuthenticationViolation) } func TestClosedMaxPayload(t *testing.T) { serverOptions := DefaultOptions() serverOptions.MaxPayload = 100 s := RunServer(serverOptions) defer s.Shutdown() opts := s.getOpts() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) conn, err := net.DialTimeout("tcp", endpoint, time.Second) if err != nil { t.Fatalf("Could not make a raw connection to the server: %v", err) } defer conn.Close() // This should trigger it. pub := "PUB foo.bar 1024\r\n" conn.Write([]byte(pub)) checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, MaxPayloadExceeded) } func TestClosedTLSHandshake(t *testing.T) { opts, err := ProcessConfigFile("./configs/tls.conf") if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.TLSVerify = true opts.NoLog = true opts.NoSigs = true s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port)) if err == nil { nc.Close() t.Fatal("Expected failure for connection") } checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, TLSHandshakeError) } nats-server-2.10.27/server/config_check_test.go000066400000000000000000001375471477524627100214720ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "errors" "fmt" "strings" "testing" ) func TestConfigCheck(t *testing.T) { tests := []struct { // name is the name of the test. name string // config is content of the configuration file. config string // warningErr is an error that does not prevent server from starting. warningErr error // errorLine is the location of the error. errorLine int // errorPos is the position of the error. errorPos int // warning errors also include a reason optionally. reason string // newDefaultErr is a configuration error that includes source of error. err error }{ { name: "when unknown field is used at top level", config: ` monitor = "127.0.0.1:4442" `, err: errors.New(`unknown field "monitor"`), errorLine: 2, errorPos: 17, }, { name: "when default permissions are used at top level", config: ` "default_permissions" { publish = ["_SANDBOX.>"] subscribe = ["_SANDBOX.>"] } `, err: errors.New(`unknown field "default_permissions"`), errorLine: 2, errorPos: 18, }, { name: "when authorization config is empty", config: ` authorization = { } `, err: nil, }, { name: "when authorization config has unknown fields", config: ` authorization = { foo = "bar" } `, err: errors.New(`unknown field "foo"`), errorLine: 3, errorPos: 5, }, { name: "when authorization config has unknown fields", config: ` port = 4222 authorization = { user = "hello" foo = "bar" password = "world" } `, err: errors.New(`unknown field "foo"`), errorLine: 6, errorPos: 5, }, { name: "when user authorization config has unknown fields", config: ` authorization = { users = [ { user = "foo" pass = "bar" token = "quux" } ] } `, err: errors.New(`unknown field "token"`), errorLine: 7, errorPos: 9, }, { name: "when user authorization permissions config has unknown fields", config: ` authorization { permissions { subscribe = {} inboxes = {} publish = {} } } `, err: errors.New(`Unknown field "inboxes" parsing permissions`), errorLine: 5, errorPos: 7, }, { name: "when user authorization permissions config has unknown fields within allow or deny", config: ` authorization { permissions { subscribe = { allow = ["hello", "world"] deny = ["foo", "bar"] denied = "_INBOX.>" } publish = {} } } `, err: errors.New(`Unknown field name "denied" parsing subject permissions, only 'allow' or 'deny' are permitted`), errorLine: 7, errorPos: 9, }, { name: "when user authorization permissions config has unknown fields within allow or deny", config: ` authorization { permissions { publish = { allow = ["hello", "world"] deny = ["foo", "bar"] allowed = "_INBOX.>" } subscribe = {} } } `, err: errors.New(`Unknown field name "allowed" parsing subject permissions, only 'allow' or 'deny' are permitted`), errorLine: 7, errorPos: 9, }, { name: "when user authorization permissions config has unknown fields using arrays", config: ` authorization { default_permissions { subscribe = ["a"] publish = ["b"] inboxes = ["c"] } users = [ { user = "foo" pass = "bar" } ] } `, err: errors.New(`Unknown field "inboxes" parsing permissions`), errorLine: 7, errorPos: 6, }, { name: "when user authorization permissions config has unknown fields using strings", config: ` authorization { default_permissions { subscribe = "a" requests = "b" publish = "c" } users = [ { user = "foo" pass = "bar" } ] } `, err: errors.New(`Unknown field "requests" parsing permissions`), errorLine: 6, errorPos: 6, }, { name: "when user authorization permissions config is empty", config: ` authorization = { users = [ { user = "foo", pass = "bar", permissions = { } } ] } `, err: nil, }, { name: "when unknown permissions are included in user config", config: ` authorization = { users = [ { user = "foo", pass = "bar", permissions { inboxes = true } } ] } `, err: errors.New(`Unknown field "inboxes" parsing permissions`), errorLine: 6, errorPos: 11, }, { name: "when clustering config is empty", config: ` cluster = { } `, err: nil, }, { name: "when unknown option is in clustering config", config: ` # NATS Server Configuration port = 4222 cluster = { port = 6222 foo = "bar" authorization { user = "hello" pass = "world" } } `, err: errors.New(`unknown field "foo"`), errorLine: 9, errorPos: 5, }, { name: "when unknown option is in clustering authorization config", config: ` cluster = { authorization { foo = "bar" } } `, err: errors.New(`unknown field "foo"`), errorLine: 4, errorPos: 7, }, { name: "when unknown option is in tls config", config: ` tls = { hello = "world" } `, err: errors.New(`error parsing tls config, unknown field "hello"`), errorLine: 3, errorPos: 5, }, { name: "when unknown option is in cluster tls config", config: ` cluster { tls = { foo = "bar" } } `, err: errors.New(`error parsing tls config, unknown field "foo"`), errorLine: 4, errorPos: 7, }, { name: "when using cipher suites in the TLS config", config: ` tls = { cipher_suites: [ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" ] preferences = [] } `, err: errors.New(`error parsing tls config, unknown field "preferences"`), errorLine: 7, errorPos: 7, }, { name: "when using curve preferences in the TLS config", config: ` tls = { curve_preferences: [ "CurveP256", "CurveP384", "CurveP521" ] suites = [] } `, err: errors.New(`error parsing tls config, unknown field "suites"`), errorLine: 8, errorPos: 7, }, { name: "when using curve preferences in the TLS config", config: ` tls = { curve_preferences: [ "CurveP5210000" ] } `, err: errors.New(`unrecognized curve preference CurveP5210000`), errorLine: 4, errorPos: 5, }, { name: "verify_cert_and_check_known_urls not support for clients", config: ` tls = { cert_file: "configs/certs/server.pem" key_file: "configs/certs/key.pem" verify_cert_and_check_known_urls: true } `, err: errors.New("verify_cert_and_check_known_urls not supported in this context"), errorLine: 5, errorPos: 10, }, { name: "when unknown option is in cluster config with defined routes", config: ` cluster { port = 6222 routes = [ nats://127.0.0.1:6222 ] peers = [] } `, err: errors.New(`unknown field "peers"`), errorLine: 7, errorPos: 5, }, { name: "when used as variable in authorization block it should not be considered as unknown field", config: ` # listen: 127.0.0.1:-1 listen: 127.0.0.1:4222 authorization { # Superuser can do anything. super_user = { publish = ">" subscribe = ">" } # Can do requests on foo or bar, and subscribe to anything # that is a response to an _INBOX. # # Notice that authorization filters can be singletons or arrays. req_pub_user = { publish = ["req.foo", "req.bar"] subscribe = "_INBOX.>" } # Setup a default user that can subscribe to anything, but has # no publish capabilities. default_user = { subscribe = "PUBLIC.>" } unused = "hello" # Default permissions if none presented. e.g. susan below. default_permissions: $default_user # Users listed with persmissions. users = [ {user: alice, password: foo, permissions: $super_user} {user: bob, password: bar, permissions: $req_pub_user} {user: susan, password: baz} ] } `, err: errors.New(`unknown field "unused"`), errorLine: 27, errorPos: 5, }, { name: "when used as variable in top level config it should not be considered as unknown field", config: ` monitoring_port = 8222 http_port = $monitoring_port port = 4222 `, err: nil, }, { name: "when used as variable in cluster config it should not be considered as unknown field", config: ` cluster { clustering_port = 6222 port = $clustering_port } `, err: nil, }, { name: "when setting permissions within cluster authorization block", config: ` cluster { authorization { permissions = { publish = { allow = ["foo", "bar"] } } } permissions = { publish = { deny = ["foo", "bar"] } } } `, warningErr: errors.New(`invalid use of field "authorization"`), errorLine: 3, errorPos: 5, reason: `setting "permissions" within cluster authorization block is deprecated`, }, { name: "when write deadline is used with deprecated usage", config: ` write_deadline = 100 `, warningErr: errors.New(`invalid use of field "write_deadline"`), errorLine: 2, errorPos: 17, reason: `write_deadline should be converted to a duration`, }, ///////////////////// // ACCOUNTS // ///////////////////// { name: "when accounts block is correctly configured", config: ` http_port = 8222 accounts { # # synadia > nats.io, cncf # synadia { # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" users [ { # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" } ] exports = [ { service: "synadia.requests", accounts: [nats, cncf] } ] } # # nats < synadia # nats { # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE nkey = "ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS" users [ { # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM nkey = "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" } ] imports = [ # This account has to send requests to 'nats.requests' subject { service: { account: "synadia", subject: "synadia.requests" }, to: "nats.requests" } ] } # # cncf < synadia # cncf { # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4 nkey = "AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT" users [ { # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ nkey = "UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ" } ] imports = [ # This account has to send requests to 'synadia.requests' subject { service: { account: "synadia", subject: "synadia.requests" } } ] } } `, err: nil, }, { name: "when nkey is invalid within accounts block", config: ` accounts { # # synadia > nats.io, cncf # synadia { # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" users [ { # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM nkey = "SCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" } ] exports = [ { service: "synadia.requests", accounts: [nats, cncf] } ] } # # nats < synadia # nats { # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE nkey = "ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS" users [ { # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM nkey = "UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" } ] imports = [ # This account has to send requests to 'nats.requests' subject { service: { account: "synadia", subject: "synadia.requests" }, to: "nats.requests" } ] } # # cncf < synadia # cncf { # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4 nkey = "AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT" users [ { # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ nkey = "UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ" } ] imports = [ # This account has to send requests to 'synadia.requests' subject { service: { account: "synadia", subject: "synadia.requests" } } ] } } `, err: errors.New(`Not a valid public nkey for a user`), errorLine: 14, errorPos: 11, }, { name: "when accounts block has unknown fields", config: ` http_port = 8222 accounts { foo = "bar" }`, err: errors.New(`Expected map entries for accounts`), errorLine: 5, errorPos: 19, }, { name: "when accounts has a referenced config variable within same block", config: ` accounts { PERMISSIONS = { publish = { allow = ["foo","bar"] deny = ["quux"] } } synadia { nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" users [ { nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" permissions = $PERMISSIONS } ] exports = [ { stream: "synadia.>" } ] } }`, err: nil, }, { name: "when accounts has an unreferenced config variables within same block", config: ` accounts { PERMISSIONS = { publish = { allow = ["foo","bar"] deny = ["quux"] } } synadia { nkey = "AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" users [ { nkey = "UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" } ] exports = [ { stream: "synadia.>" } ] } }`, err: errors.New(`unknown field "publish"`), errorLine: 4, errorPos: 5, }, { name: "when accounts block defines a global account", config: ` http_port = 8222 accounts { $G = { } } `, err: errors.New(`"$G" is a Reserved Account`), errorLine: 5, errorPos: 19, }, { name: "when accounts block uses an invalid public key", config: ` accounts { synadia = { nkey = "invalid" } } `, err: errors.New(`Not a valid public nkey for an account: "invalid"`), errorLine: 4, errorPos: 21, }, { name: "when accounts list includes reserved account", config: ` port = 4222 accounts = [foo, bar, "$G"] http_port = 8222 `, err: errors.New(`"$G" is a Reserved Account`), errorLine: 4, errorPos: 26, }, { name: "when accounts list includes a dupe entry", config: ` port = 4222 accounts = [foo, bar, bar] http_port = 8222 `, err: errors.New(`Duplicate Account Entry: bar`), errorLine: 4, errorPos: 25, }, { name: "when accounts block includes a dupe user", config: ` port = 4222 accounts = { nats { users = [ { user: "foo", pass: "bar" }, { user: "hello", pass: "world" }, { user: "foo", pass: "bar" } ] } } http_port = 8222 `, err: errors.New(`Duplicate user "foo" detected`), errorLine: 6, errorPos: 21, }, { name: "when accounts block imports are not a list", config: ` port = 4222 accounts = { nats { imports = true } } http_port = 8222 `, err: errors.New(`Imports should be an array, got bool`), errorLine: 6, errorPos: 21, }, { name: "when accounts block exports are not a list", config: ` port = 4222 accounts = { nats { exports = true } } http_port = 8222 `, err: errors.New(`Exports should be an array, got bool`), errorLine: 6, errorPos: 21, }, { name: "when accounts block imports items are not a map", config: ` port = 4222 accounts = { nats { imports = [ false ] } } http_port = 8222 `, err: errors.New(`Import Items should be a map with type entry, got bool`), errorLine: 7, errorPos: 23, }, { name: "when accounts block export items are not a map", config: ` port = 4222 accounts = { nats { exports = [ false ] } } http_port = 8222 `, err: errors.New(`Export Items should be a map with type entry, got bool`), errorLine: 7, errorPos: 23, }, { name: "when accounts exports has a stream name that is not a string", config: ` port = 4222 accounts = { nats { exports = [ { stream: false } ] } } http_port = 8222 `, err: errors.New(`Expected stream name to be string, got bool`), errorLine: 8, errorPos: 25, }, { name: "when accounts exports has a service name that is not a string", config: ` accounts = { nats { exports = [ { service: false } ] } } `, err: errors.New(`Expected service name to be string, got bool`), errorLine: 6, errorPos: 25, }, { name: "when accounts imports stream without name", config: ` port = 4222 accounts = { nats { imports = [ { stream: { }} ] } } http_port = 8222 `, err: errors.New(`Expect an account name and a subject`), errorLine: 7, errorPos: 25, }, { name: "when accounts imports service without name", config: ` port = 4222 accounts = { nats { imports = [ { service: { }} ] } } http_port = 8222 `, err: errors.New(`Expect an account name and a subject`), errorLine: 7, errorPos: 25, }, { name: "when user authorization config has both token and users", config: ` authorization = { token = "s3cr3t" users = [ { user = "foo" pass = "bar" } ] } `, err: errors.New(`Can not have a token and a users array`), errorLine: 2, errorPos: 3, }, { name: "when user authorization config has both token and user", config: ` authorization = { user = "foo" pass = "bar" token = "baz" } `, err: errors.New(`Cannot have a user/pass and token`), errorLine: 2, errorPos: 3, }, { name: "when user authorization config has both user and users array", config: ` authorization = { user = "user1" pass = "pwd1" users = [ { user = "user2" pass = "pwd2" } ] } `, err: errors.New(`Can not have a single user/pass and a users array`), errorLine: 2, errorPos: 3, }, { name: "when user authorization has duplicate users", config: ` authorization = { users = [ {user: "user1", pass: "pwd"} {user: "user2", pass: "pwd"} {user: "user1", pass: "pwd"} ] } `, err: fmt.Errorf(`Duplicate user %q detected`, "user1"), errorLine: 2, errorPos: 3, }, { name: "when user authorization has duplicate nkeys", config: ` authorization = { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX } {nkey: UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E } {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX } ] } `, err: fmt.Errorf(`Duplicate nkey %q detected`, "UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX"), errorLine: 2, errorPos: 3, }, { name: "when user authorization config has users not as a list", config: ` authorization = { users = false } `, err: errors.New(`Expected users field to be an array, got false`), errorLine: 3, errorPos: 5, }, { name: "when user authorization config has users not as a map", config: ` authorization = { users = [false] } `, err: errors.New(`Expected user entry to be a map/struct, got false`), errorLine: 3, errorPos: 14, }, { name: "when user authorization config has permissions not as a map", config: ` authorization = { users = [{user: hello, pass: world}] permissions = false } `, err: errors.New(`Expected permissions to be a map/struct, got false`), errorLine: 4, errorPos: 19, }, { name: "when user authorization permissions config has invalid fields within allow", config: ` authorization { permissions { publish = { allow = [false, "hello", "world"] deny = ["foo", "bar"] } subscribe = {} } } `, err: errors.New(`Subject in permissions array cannot be cast to string`), errorLine: 5, errorPos: 18, }, { name: "when user authorization permissions config has invalid fields within deny", config: ` authorization { permissions { publish = { allow = ["hello", "world"] deny = [true, "foo", "bar"] } subscribe = {} } } `, err: errors.New(`Subject in permissions array cannot be cast to string`), errorLine: 6, errorPos: 17, }, { name: "when user authorization permissions config has invalid type", config: ` authorization { permissions { publish = { allow = false } subscribe = {} } } `, err: errors.New(`Expected subject permissions to be a subject, or array of subjects, got bool`), errorLine: 5, errorPos: 9, }, { name: "when user authorization permissions subject is invalid", config: ` authorization { permissions { publish = { allow = ["foo..bar"] } subscribe = {} } } `, err: errors.New(`subject "foo..bar" is not a valid subject`), errorLine: 5, errorPos: 9, }, { name: "when cluster config listen is invalid", config: ` cluster { listen = "0.0.0.0:XXXX" } `, err: errors.New(`could not parse port "XXXX"`), errorLine: 3, errorPos: 5, }, { name: "when cluster config includes multiple users", config: ` cluster { authorization { users = [] } } `, err: errors.New(`Cluster authorization does not allow multiple users`), errorLine: 3, errorPos: 5, }, { name: "when cluster routes are invalid", config: ` cluster { routes = [ "0.0.0.0:XXXX" # "0.0.0.0:YYYY" # "0.0.0.0:ZZZZ" ] } `, err: errors.New(`error parsing route url ["0.0.0.0:XXXX"]`), errorLine: 4, errorPos: 22, }, { name: "when setting invalid TLS config within cluster block", config: ` cluster { tls { } } `, err: nil, errorLine: 0, errorPos: 0, }, { name: "invalid lame_duck_duration type", config: ` lame_duck_duration: abc `, err: errors.New(`error parsing lame_duck_duration: time: invalid duration`), errorLine: 2, errorPos: 5, }, { name: "lame_duck_duration too small", config: ` lame_duck_duration: "5s" `, err: errors.New(`invalid lame_duck_duration of 5s, minimum is 30 seconds`), errorLine: 2, errorPos: 5, }, { name: "invalid lame_duck_grace_period type", config: ` lame_duck_grace_period: abc `, err: errors.New(`error parsing lame_duck_grace_period: time: invalid duration`), errorLine: 2, errorPos: 5, }, { name: "lame_duck_grace_period should be positive", config: ` lame_duck_grace_period: "-5s" `, err: errors.New(`invalid lame_duck_grace_period, needs to be positive`), errorLine: 2, errorPos: 5, }, { name: "when only setting TLS timeout for a leafnode remote", config: ` leafnodes { remotes = [ { url: "tls://nats:7422" tls { timeout: 0.01 } } ] }`, err: nil, errorLine: 0, errorPos: 0, }, { name: "verify_cert_and_check_known_urls do not work for leaf nodes", config: ` leafnodes { remotes = [ { url: "tls://nats:7422" tls { timeout: 0.01 verify_cert_and_check_known_urls: true } } ] }`, //Unexpected error after processing config: /var/folders/9h/6g_c9l6n6bb8gp331d_9y0_w0000gn/T/057996446:8:5: err: errors.New("verify_cert_and_check_known_urls not supported in this context"), errorLine: 8, errorPos: 5, }, { name: "when leafnode remotes use wrong type", config: ` leafnodes { remotes: { url: "tls://nats:7422" } }`, err: errors.New(`Expected remotes field to be an array, got map[string]interface {}`), errorLine: 3, errorPos: 5, }, { name: "when leafnode remotes url uses wrong type", config: ` leafnodes { remotes: [ { urls: 1234 } ] }`, err: errors.New(`Expected remote leafnode url to be an array or string, got 1234`), errorLine: 4, errorPos: 18, }, { name: "when leafnode min_version is wrong type", config: ` leafnodes { port: -1 min_version = 123 }`, err: errors.New(`interface conversion: interface {} is int64, not string`), errorLine: 4, errorPos: 6, }, { name: "when leafnode min_version has parsing error", config: ` leafnodes { port: -1 min_version = bad.version }`, err: errors.New(`invalid leafnode's minimum version: invalid semver`), errorLine: 4, errorPos: 6, }, { name: "when leafnode min_version is too low", config: ` leafnodes { port: -1 min_version = 2.7.9 }`, err: errors.New(`the minimum version should be at least 2.8.0`), errorLine: 4, errorPos: 6, }, { name: "when setting latency tracking with a system account", config: ` system_account: sys accounts { sys { users = [ {user: sys, pass: "" } ] } nats.io: { users = [ { user : bar, pass: "" } ] exports = [ { service: "nats.add" response: singleton latency: { sampling: 100% subject: "latency.tracking.add" } } ] } } `, err: nil, errorLine: 0, errorPos: 0, }, { name: "when setting latency tracking with an invalid publish subject", config: ` system_account = sys accounts { sys { users = [ {user: sys, pass: "" } ] } nats.io: { users = [ { user : bar, pass: "" } ] exports = [ { service: "nats.add" response: singleton latency: "*" } ] } } `, err: errors.New(`Error adding service latency sampling for "nats.add" on subject "*": invalid publish subject`), errorLine: 3, errorPos: 17, }, { name: "when setting latency tracking on a stream", config: ` system_account = sys accounts { sys { users = [ {user: sys, pass: "" } ] } nats.io: { users = [ { user : bar, pass: "" } ] exports = [ { stream: "nats.add" latency: "foo" } ] } } `, err: errors.New(`Detected latency directive on non-service`), errorLine: 11, errorPos: 25, }, { name: "when using duplicate service import subject", config: ` accounts { A: { users = [ {user: user1, pass: ""} ] exports = [ {service: "remote1"} {service: "remote2"} ] } B: { users = [ {user: user2, pass: ""} ] imports = [ {service: {account: "A", subject: "remote1"}, to: "local"} {service: {account: "A", subject: "remote2"}, to: "local"} ] } } `, err: errors.New(`Duplicate service import subject "local", previously used in import for account "A", subject "remote1"`), errorLine: 14, errorPos: 71, }, { name: "mixing single and multi users in leafnode authorization", config: ` leafnodes { authorization { user: user1 password: pwd users = [{user: user2, password: pwd}] } } `, err: errors.New("can not have a single user/pass and a users array"), errorLine: 3, errorPos: 20, }, { name: "duplicate usernames in leafnode authorization", config: ` leafnodes { authorization { users = [ {user: user, password: pwd} {user: user, password: pwd} ] } } `, err: errors.New(`duplicate user "user" detected in leafnode authorization`), errorLine: 3, errorPos: 21, }, { name: "mqtt bad type", config: ` mqtt [ "wrong" ] `, err: errors.New(`Expected mqtt to be a map, got []interface {}`), errorLine: 2, errorPos: 17, }, { name: "mqtt bad listen", config: ` mqtt { listen: "xxxxxxxx" } `, err: errors.New(`could not parse address string "xxxxxxxx"`), errorLine: 3, errorPos: 21, }, { name: "mqtt bad host", config: ` mqtt { host: 1234 } `, err: errors.New(`interface conversion: interface {} is int64, not string`), errorLine: 3, errorPos: 21, }, { name: "mqtt bad port", config: ` mqtt { port: "abc" } `, err: errors.New(`interface conversion: interface {} is string, not int64`), errorLine: 3, errorPos: 21, }, { name: "mqtt bad TLS", config: ` mqtt { port: -1 tls { cert_file: "./configs/certs/server.pem" } } `, err: errors.New(`missing 'key_file' in TLS configuration`), errorLine: 4, errorPos: 21, }, { name: "connection types wrong type", config: ` authorization { users [ {user: a, password: pwd, allowed_connection_types: 123} ] } `, err: errors.New(`error parsing allowed connection types: unsupported type int64`), errorLine: 4, errorPos: 53, }, { name: "connection types content wrong type", config: ` authorization { users [ {user: a, password: pwd, allowed_connection_types: [ 123 WEBSOCKET ]} ] } `, err: errors.New(`error parsing allowed connection types: unsupported type in array int64`), errorLine: 5, errorPos: 32, }, { name: "connection types type unknown", config: ` authorization { users [ {user: a, password: pwd, allowed_connection_types: [ "UNKNOWN" ]} ] } `, err: fmt.Errorf("invalid connection types [%q]", "UNKNOWN"), errorLine: 4, errorPos: 53, }, { name: "websocket auth unknown var", config: ` websocket { authorization { unknown: "field" } } `, err: fmt.Errorf("unknown field %q", "unknown"), errorLine: 4, errorPos: 25, }, { name: "websocket bad tls", config: ` websocket { tls { cert_file: "configs/certs/server.pem" } } `, err: fmt.Errorf("missing 'key_file' in TLS configuration"), errorLine: 3, errorPos: 21, }, { name: "verify_cert_and_check_known_urls not support for websockets", config: ` websocket { tls { cert_file: "configs/certs/server.pem" key_file: "configs/certs/key.pem" verify_cert_and_check_known_urls: true } } `, err: fmt.Errorf("verify_cert_and_check_known_urls not supported in this context"), errorLine: 6, errorPos: 10, }, { name: "ambiguous store dir", config: ` store_dir: "foo" jetstream { store_dir: "bar" } `, err: fmt.Errorf(`Duplicate 'store_dir' configuration`), }, { name: "token not supported in cluster", config: ` cluster { port: -1 authorization { token: "my_token" } } `, err: fmt.Errorf("Cluster authorization does not support tokens"), errorLine: 4, errorPos: 6, }, { name: "token not supported in gateway", config: ` gateway { port: -1 name: "A" authorization { token: "my_token" } } `, err: fmt.Errorf("Gateway authorization does not support tokens"), errorLine: 5, errorPos: 6, }, { name: "wrong type for cluster pool size", config: ` cluster { port: -1 pool_size: "abc" } `, err: fmt.Errorf("interface conversion: interface {} is string, not int64"), errorLine: 4, errorPos: 6, }, { name: "wrong type for cluster accounts", config: ` cluster { port: -1 accounts: 123 } `, err: fmt.Errorf("error parsing accounts: unsupported type int64"), errorLine: 4, errorPos: 6, }, { name: "wrong type for cluster compression", config: ` cluster { port: -1 compression: 123 } `, err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), errorLine: 4, errorPos: 6, }, { name: "wrong type for cluster compression mode", config: ` cluster { port: -1 compression: { mode: 123 } } `, err: fmt.Errorf("interface conversion: interface {} is int64, not string"), errorLine: 5, errorPos: 7, }, { name: "wrong type for cluster compression rtt thresholds", config: ` cluster { port: -1 compression: { mode: "s2_auto" rtt_thresholds: 123 } } `, err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), errorLine: 6, errorPos: 7, }, { name: "invalid durations for cluster compression rtt thresholds", config: ` cluster { port: -1 compression: { mode: "s2_auto" rtt_thresholds: [abc] } } `, err: fmt.Errorf("time: invalid duration %q", "abc"), errorLine: 6, errorPos: 7, }, { name: "invalid durations for cluster ping interval", config: ` cluster { port: -1 ping_interval: -1 ping_max: 6 } `, err: fmt.Errorf(`invalid use of field "ping_interval": ping_interval should be converted to a duration`), errorLine: 4, errorPos: 6, }, { name: "invalid durations for cluster ping interval", config: ` cluster { port: -1 ping_interval: '2m' ping_max: 6 } `, warningErr: fmt.Errorf(`Cluster 'ping_interval' will reset to 30s which is the max for routes`), errorLine: 4, errorPos: 6, }, { name: "wrong type for leafnodes compression", config: ` leafnodes { port: -1 compression: 123 } `, err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), errorLine: 4, errorPos: 6, }, { name: "wrong type for leafnodes compression mode", config: ` leafnodes { port: -1 compression: { mode: 123 } } `, err: fmt.Errorf("interface conversion: interface {} is int64, not string"), errorLine: 5, errorPos: 7, }, { name: "wrong type for leafnodes compression rtt thresholds", config: ` leafnodes { port: -1 compression: { mode: "s2_auto" rtt_thresholds: 123 } } `, err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), errorLine: 6, errorPos: 7, }, { name: "invalid durations for leafnodes compression rtt thresholds", config: ` leafnodes { port: -1 compression: { mode: "s2_auto" rtt_thresholds: [abc] } } `, err: fmt.Errorf("time: invalid duration %q", "abc"), errorLine: 6, errorPos: 7, }, { name: "wrong type for remote leafnodes compression", config: ` leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:123" compression: 123 } ] } `, err: fmt.Errorf("field %q should be a boolean or a structure, got int64", "compression"), errorLine: 7, errorPos: 8, }, { name: "wrong type for remote leafnodes compression mode", config: ` leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:123" compression: { mode: 123 } } ] } `, err: fmt.Errorf("interface conversion: interface {} is int64, not string"), errorLine: 8, errorPos: 9, }, { name: "wrong type for remote leafnodes compression rtt thresholds", config: ` leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:123" compression: { mode: "s2_auto" rtt_thresholds: 123 } } ] } `, err: fmt.Errorf("interface conversion: interface {} is int64, not []interface {}"), errorLine: 9, errorPos: 9, }, { name: "invalid durations for remote leafnodes compression rtt thresholds", config: ` leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:123" compression: { mode: "s2_auto" rtt_thresholds: [abc] } } ] } `, err: fmt.Errorf("time: invalid duration %q", "abc"), errorLine: 9, errorPos: 9, }, { name: "invalid duration for remote leafnode first info timeout", config: ` leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:123" first_info_timeout: abc } ] } `, err: fmt.Errorf("error parsing first_info_timeout: time: invalid duration %q", "abc"), errorLine: 7, errorPos: 8, }, { name: "show warnings on empty configs without values", config: ``, warningErr: errors.New(`config has no values or is empty`), errorLine: 0, errorPos: 0, reason: "", }, { name: "show warnings on empty configs without values and only comments", config: `# Valid file but has no usable values. `, warningErr: errors.New(`config has no values or is empty`), errorLine: 0, errorPos: 0, reason: "", }, { name: "TLS handshake first, wrong type", config: ` port: -1 tls { first: 123 } `, err: fmt.Errorf("field %q should be a boolean or a string, got int64", "first"), errorLine: 4, errorPos: 6, }, { name: "TLS handshake first, wrong value", config: ` port: -1 tls { first: "123" } `, err: fmt.Errorf("field %q's value %q is invalid", "first", "123"), errorLine: 4, errorPos: 6, }, { name: "TLS multiple certs", config: ` port: -1 tls { certs: [ { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"}, { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, ] } `, err: nil, }, { name: "TLS multiple certs, bad type", config: ` port: -1 tls { certs: [ { cert_file: "configs/certs/server.pem", key_file: 123 }, { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, ] } `, err: fmt.Errorf("error parsing certificates config: unsupported type int64"), errorLine: 5, errorPos: 49, }, { name: "TLS multiple certs, missing key_file", config: ` port: -1 tls { certs: [ { cert_file: "configs/certs/server.pem" } { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"} ] } `, err: fmt.Errorf("error parsing certificates config: both 'cert_file' and 'cert_key' options are required"), errorLine: 5, errorPos: 10, }, { name: "TLS multiple certs and single cert options at the same time", config: ` port: -1 tls { cert_file: "configs/certs/server.pem" key_file: "configs/certs/key.pem" certs: [ { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.pem"}, { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"}, ] } `, err: fmt.Errorf("error parsing tls config, cannot combine 'cert_file' option with 'certs' option"), errorLine: 3, errorPos: 5, }, { name: "TLS multiple certs used but not configured, but cert_file configured", config: ` port: -1 tls { cert_file: "configs/certs/server.pem" key_file: "configs/certs/key.pem" certs: [] } `, err: nil, }, { name: "TLS multiple certs, missing bad path", config: ` port: -1 tls { certs: [ { cert_file: "configs/certs/cert.new.pem", key_file: "configs/certs/key.new.pem"} { cert_file: "configs/certs/server.pem", key_file: "configs/certs/key.new.pom" } ] } `, err: fmt.Errorf("error parsing X509 certificate/key pair 2/2: open configs/certs/key.new.pom: no such file or directory"), errorLine: 3, errorPos: 5, }, } checkConfig := func(config string) error { opts := &Options{ CheckConfig: true, } return opts.ProcessConfigFile(config) } checkErr := func(t *testing.T, err, expectedErr error) { t.Helper() switch { case err == nil && expectedErr == nil: // OK case err != nil && expectedErr == nil: t.Errorf("Unexpected error after processing config: %s", err) case err == nil && expectedErr != nil: t.Errorf("Expected %q error after processing invalid config but got nothing", expectedErr) } } for _, test := range tests { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(test.config)) err := checkConfig(conf) var expectedErr error // Check for either warnings or errors. if test.err != nil { expectedErr = test.err } else if test.warningErr != nil { expectedErr = test.warningErr } if err != nil && expectedErr != nil { var msg string if test.errorPos > 0 { msg = fmt.Sprintf("%s:%d:%d: %s", conf, test.errorLine, test.errorPos, expectedErr.Error()) if test.reason != "" { msg += ": " + test.reason } } else if test.warningErr != nil { msg = expectedErr.Error() } else { msg = test.reason } if !strings.Contains(err.Error(), msg) { t.Errorf("Expected:\n%q\ngot:\n%q", msg, err.Error()) } } checkErr(t, err, expectedErr) }) } } func TestConfigCheckIncludes(t *testing.T) { // Check happy path first. opts := &Options{ CheckConfig: true, } err := opts.ProcessConfigFile("./configs/include_conf_check_a.conf") if err != nil { t.Errorf("Unexpected error processing include files with configuration check enabled: %v", err) } opts = &Options{ CheckConfig: true, } err = opts.ProcessConfigFile("./configs/include_bad_conf_check_a.conf") if err == nil { t.Errorf("Expected error processing include files with configuration check enabled: %v", err) } expectedErr := `include_bad_conf_check_b.conf:10:19: unknown field "monitoring_port"` + "\n" if err != nil && !strings.HasSuffix(err.Error(), expectedErr) { t.Errorf("Expected: \n%q, got\n: %q", expectedErr, err.Error()) } } func TestConfigCheckMultipleErrors(t *testing.T) { opts := &Options{ CheckConfig: true, } err := opts.ProcessConfigFile("./configs/multiple_errors.conf") if err == nil { t.Errorf("Expected error processing config files with multiple errors check enabled: %v", err) } cerr, ok := err.(*processConfigErr) if !ok { t.Fatalf("Expected a configuration process error") } got := len(cerr.Warnings()) expected := 1 if got != expected { t.Errorf("Expected a %d warning, got: %d", expected, got) } got = len(cerr.Errors()) // Could be 7 or 8 errors depending on internal ordering of the parsing. if got != 7 && got != 8 { t.Errorf("Expected 7 or 8 errors, got: %d", got) } errMsg := err.Error() errs := []string{ `./configs/multiple_errors.conf:12:1: invalid use of field "write_deadline": write_deadline should be converted to a duration`, `./configs/multiple_errors.conf:2:1: Cannot have a user/pass and token`, `./configs/multiple_errors.conf:10:1: unknown field "monitoring"`, `./configs/multiple_errors.conf:67:3: Cluster authorization does not allow multiple users`, `./configs/multiple_errors.conf:21:5: Not a valid public nkey for an account: "OC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL"`, `./configs/multiple_errors.conf:26:9: Not a valid public nkey for a user`, `./configs/multiple_errors.conf:36:5: Not a valid public nkey for an account: "ODRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS"`, `./configs/multiple_errors.conf:41:9: Not a valid public nkey for a user`, } for _, msg := range errs { found := strings.Contains(errMsg, msg) if !found { t.Fatalf("Expected to find error %q", msg) } } if got == 8 { extra := "./configs/multiple_errors.conf:54:5: Can not have a single user/pass and accounts" if !strings.Contains(errMsg, extra) { t.Fatalf("Expected to find error %q (%s)", extra, errMsg) } } } nats-server-2.10.27/server/configs/000077500000000000000000000000001477524627100171115ustar00rootroot00000000000000nats-server-2.10.27/server/configs/accounts.conf000066400000000000000000000026771477524627100216130ustar00rootroot00000000000000 accounts: { synadia: { nkey: ADMHMDX2LEUJRZQHGVSVRWZEJ2CPNHYO6TB4ZCZ37LXAX5SYNEW252GF users = [ # Bob {nkey : UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} # Alice {nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E} ] exports = [ {stream: "public.>"} # No accounts means public. {stream: "synadia.private.>", accounts: [cncf, nats.io]} {service: "pub.request"} # No accounts means public. {service: "pub.special.request", accounts: [nats.io]} ] imports = [ {service: {account: "nats.io", subject: "nats.time"}} ] } nats.io: { nkey: AB5UKNPVHDWBP5WODG742274I3OGY5FM3CBIFCYI4OFEH7Y23GNZPXFE users = [ # Ivan {nkey : UBRYMDSRTC6AVJL6USKKS3FIOE466GMEU67PZDGOWYSYHWA7GSKO42VW} # Derek {nkey : UDEREK22W43P2NFQCSKGM6BWD23OVWEDR7JE7LSNCD232MZIC4X2MEKZ} ] imports = [ {stream: {account: "synadia", subject:"public.synadia"}, prefix: "imports.synadia"} {stream: {account: "synadia", subject:"synadia.private.*"}} {service: {account: "synadia", subject: "pub.special.request"}, to: "synadia.request"} ] exports = [ {service: "nats.time", response: stream} {service: "nats.photo", response: chunked} {service: "nats.add", response: singleton, accounts: [cncf]} {service: "nats.sub"} ] } cncf: { nkey: ABDAYEV6KZVLW3GSJ3V7IWC542676TFYILXF2C7Z56LCPSMVHJE5BVYO} } nats-server-2.10.27/server/configs/authorization.conf000066400000000000000000000025201477524627100226570ustar00rootroot00000000000000listen: 127.0.0.1:4222 authorization { # Our role based permissions. # Superuser can do anything. super_user = { publish = "*" subscribe = ">" } # Can do requests on foo or bar, and subscribe to anything # that is a response to an _INBOX. # # Notice that authorization filters can be singletons or arrays. req_pub_user = { publish = ["req.foo", "req.bar"] subscribe = "_INBOX.>" } # Setup a default user that can subscribe to anything, but has # no publish capabilities. default_user = { subscribe = "PUBLIC.>" } # Service can listen on the request subject and respond to any # received reply subject. my_service = { subscribe = "my.service.req" publish_allow_responses: true } # Can support a map with max messages and expiration of the permission. my_stream_service = { subscribe = "my.service.req" allow_responses: {max: 10, expires: "1m"} } # Default permissions if none presented. e.g. susan below. default_permissions: $default_user # Users listed with persmissions. users = [ {user: alice, password: foo, permissions: $super_user} {user: bob, password: bar, permissions: $req_pub_user} {user: susan, password: baz} {user: svca, password: pc, permissions: $my_service} {user: svcb, password: sam, permissions: $my_stream_service} ] } nats-server-2.10.27/server/configs/certs/000077500000000000000000000000001477524627100202315ustar00rootroot00000000000000nats-server-2.10.27/server/configs/certs/cert.new.pem000066400000000000000000000024421477524627100224630ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDnjCCAoagAwIBAgIJAM/HacKKaH7zMA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD VQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD VQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu aW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIyODIxWhcN MjkxMDE0MTIyODIxWjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD VQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u cyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz LmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBIAVUr4gCEBRVbA HSweCLExOrMZmII4AdvgIqT+svjBkJd+vdkbd5b/SC1HQx1E14kiRJ/JrZIZoMqi +7pK3kFM63Fkhkg8rWOxn0tQznSymKTpha5NdDWxnB0dXlXFCQG1e/cuDalR7UhF LPHiuK42gAvhivBcymDPV0hTYt4rHb71SQ1DwfCYzcLkDvDFA/W7kronaEhRyWn6 uvZvHkdvScoubdzoW/kNBH4JYZw5svLzGz3z20rUGeLttF4ge5SCAz9unZk96HUO EFmUDvFmxdnTXrYINjraNvgu7fPFmVzupWrWPA/7U+cxOvm3qBMdqxFxNr7bq+Md UPKi3QIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB AQCCgFKP/bMi5cvasJOzXKLpwOneW+oItL2t/5NxPumIBMDo5NnShzsFLfGmujYw fjcMharynkkbz/oeDAm8h1mySAJrnqtabiWgaW8zbNJfJOrAq4Jvs9COMAzjJclL +h9GWtvVylDnsNtd18n1gA5OYv0A6YjuSrWINL8Sp5QvTF/5tT8jFrDOIjZl7m50 lX4R70N9GLt2jIlKro+qdsi6qUZccuJmoQxUpG1iQcRNFHtWfDPr5KFEXaO6IoYt D1kYWmo/A3WQm7nbXeZw/zaSGSS0t6/hKZwm+gPCL6TEdDrjhjxpCZQTwaMd6jj2 bvT2OA0ZpzUWyxnaX6u+cDfM -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/key.new.pem000066400000000000000000000032501477524627100223140ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8EgBVSviAIQFF VsAdLB4IsTE6sxmYgjgB2+AipP6y+MGQl3692Rt3lv9ILUdDHUTXiSJEn8mtkhmg yqL7ukreQUzrcWSGSDytY7GfS1DOdLKYpOmFrk10NbGcHR1eVcUJAbV79y4NqVHt SEUs8eK4rjaAC+GK8FzKYM9XSFNi3isdvvVJDUPB8JjNwuQO8MUD9buSuidoSFHJ afq69m8eR29Jyi5t3Ohb+Q0EfglhnDmy8vMbPfPbStQZ4u20XiB7lIIDP26dmT3o dQ4QWZQO8WbF2dNetgg2Oto2+C7t88WZXO6latY8D/tT5zE6+beoEx2rEXE2vtur 4x1Q8qLdAgMBAAECggEBAI0zJrTi2Le6D+4zN4Gvgc0c280qcmkiO1KJ9jmMjv1n 7kvUsf9vZUPgRkG2XO5ypyD7gJLtNMnwCvXBraQ5NcSwWkPampKG4ad8Vfs23LBk xUH9bqZDOzuopHSFF2ugEZK1icBM0HLJUQ1JWUZpRMNLaPex8+AQnloDXSg20QRE Z53CDYZVEVaLAYIkfgZ6QgBSbJ0ob/a+ZJkChWs3f6cpNHige4tR2RQ0aJFwvlwu eC8G3RDNzUMhPf4TSf/XUlHXngyHKa7MS3feEtuKIQ4gb8hjESDcZznJUGG9qyMT 7445mapc60t8z46o9iVeVJQmQxmjFUDkLcZ9ckcWfzECgYEA4OiolNUiTnXbvP0H 3rxAnhx9GlyHiRaIZvgsAHH/6aMbJIXnH89cAR38PL79OaQpWZqK8Z1vrCHC8q7s Wb1RehgnwPxtTl4RTZ0wj3Ri5BIq32nzgla0w/5RAmAw9HrYXjp4TR4Quc+bHLcr ZM2gA8nZ4PG73rYvd/Mu3lCQiS8CgYEA1hGoaduGG7eHTMCct/kCupFfYXgEuUDh V/nO1SYJbIkMuSONNaQy1oUc5o0BB/JTcslZWqtuGa9snKPVd3/rJz75YGt1QR6u MukCYjbWGAWXfDh8G8aGSr3k1k3JmKNE7fI7L5DuvKCGoGjzzC+kclyoij8jwXk0 zghaYPvM+bMCgYEA3/fwgR3p5vZJF19mqfEP7CP0lP7V3bd5qAi1UNA1h4VsrydF LRFCzr38hMWwx+jpYJicitU78s9AIon9RbRY4dwSIoV9mE/mrUK+q+y72eEZnpgU 7ZPIuXCVXWdK+PsoYlWZnTo2b8ME9UiWxvBZy8wD05UGgFcu2CVsY+kYtfMCgYAs Vn/xXPyL5Rlq9kH/gN3l0pJU18zyqdOCq0UBtN0i08gE2K44vAejkvKHdhEOmkxa bAXL19H4E/OFBhICrEYCXPK928PvdvFRrh1GRmFVnGLh4bki752FAYvSL05gBQET 36YOlhA8lWsM8m/8jKmc3kAyUh2PxxD+05AUolK0LQKBgHqPgMj5rfiR7MoVIaXr K8h7g+HejppmCxzENc98hvIQIO2aWeC4Hi/EMe+XYSN/Gl6d5+aw80JYRgIv6dFV hDEEt8lEI4sFhKD0oG1dQo+BdDNXOLbpHq+RQRPHa8J/pmSjI5Joqb2mCaAcm/KJ nYE20ox06YMSb+ZrjP2p4qw3 -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/key.pem000066400000000000000000000032501477524627100215240ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDc8YaxliKNsa8M idwQsDXS0MI4z+ZP1UMW9U/Vt2qTienYs3L6KqezRfwEH2AoE7XyvQdzePV3aN1g A0L3mAKiS2FN3DhxR6Y8D/JNb5YA6Pl0flaMKnWktma/SMOf7o7eS7Zt6ZmIvtBg p1ZtRSZSH64KSqC467eGByHnXm25Z7Uf/2rO8K1jsmRwgDEJoJL5Z4va4pypX5Kq 7OxWk8Za50Kuk6DtdRfIcnUjoDadsYzGHzF7kp1/31elQ2aUPp9e6X3aWu++gJv5 brBMsWeQ+ETmjCMPqxoXDM2AJqwP73irELU7POUR/q3gooqy16BvTp7TaLvAtKdP SK7kl1QVAgMBAAECggEAKVT1gZs4c5Fd0daoWGbeasWqz730s9KNbmoNuUlKorcW C/fl9m6sWJkrAApujpDBZNt/3VYvdVskrqVtjaveMkYvucUMugRRUMMa7OmHAjyT DfvkbOw0fc+xgO83yV9zUpqPUhh1JGOtz6b+QArDlHFguYQkdPODrYsACKxdkmU/ A+ZJkC2Sdn0L7ARmROxvoelxV3HNbC+p8lqrIVarpni8oeDF2JxQfyJETl3Df099 /KA/IQvXytEQBJpguh0j7sK81cI40TKO3kNWsCe46S2nkCuEeEums7BqAspmdNMS gBW0rpFA9Ep5AyT4+A2XhWFQAwf/PI26JUVDhbClAQKBgQD/XlONHf5tNiOEtW88 Whd9ha7oLgoBhr4gFOcPM4MUStRDftRXY1ty0IGseV0Rj1xQRUp31Pl6Kep/udy1 r/6ML/IQNaht1VJOf2+2v7oZFoLRGLvR3qSRFNVtfLWp1Dg5OPgyi00OpFnxz/6o iuWlh1EWwSm9vPl91HcWPPcwBQKBgQDdfWfDfy8q9jExj8tVNh5++haeemprkZxk QuduIkahlbQ3V+SLaSWQrwzcB4PWL+45fnIWwK3Vmm2POBv+OWfKGjJniSd4Zrrf H+lSg4+hpivvoaYf1TsCTHy/af/5pYIkEUmbPM0EqwubyoW6oMAgGIAMyH6XBLaY a4vWZo6g0QKBgFXDXhgUrLAM8JzPOk5wi1cSoI1FeQLON+gaXQdT63/TKbqJS9MV gU7sC8Da+ZC+LuiefMYF9ss6bJD84Mz8EGcQayFag/hvHjdSwTgE6AEo+EI1Jk7z kR1Qe+VLbs9cgI1nPqPq+LQkKjj1+7aq/zk6WtdjhBs+7iN+SYhkbTfpAoGAeCbv gz4beFLVnO1EgJU7Nea1HoOJ95CmJj2lDjnJ1x4/BUbI1FfV6QcNEs+A1VBrCwVt HqqnopiDlo35oY/CngBYF5JvtwEDnsbQ69IyuJ5Md1JZrCsgN78GbVAzbFo8nxRB udTh4wZm5byXjwZlMwQXctfQ1FvaMhUlZsl71gECgYA82osmLHxZZOiQrYJ7z302 CmPohwcTtUASqeA0kQOX/OdFJ1IxWIFpzq2ffMipLTfzk7OeFS2hqvNo/NE8NHlR /79IycYwlO99T3e2yQhYKW5ZMT3Q8YtyLLPtBzPPtO/Tc8yKhjX8wc9w9sxq9kqg i5Z+6xfO+l/NBT/JHF4yEw== -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/server.pem000066400000000000000000000024421477524627100222440ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDnjCCAoagAwIBAgIJAOXpOWk41Pd7MA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD VQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD VQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu aW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIzMjQ5WhcN MjkxMDE0MTIzMjQ5WjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD VQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u cyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz LmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3PGGsZYijbGvDInc ELA10tDCOM/mT9VDFvVP1bdqk4np2LNy+iqns0X8BB9gKBO18r0Hc3j1d2jdYANC 95gCokthTdw4cUemPA/yTW+WAOj5dH5WjCp1pLZmv0jDn+6O3ku2bemZiL7QYKdW bUUmUh+uCkqguOu3hgch515tuWe1H/9qzvCtY7JkcIAxCaCS+WeL2uKcqV+Squzs VpPGWudCrpOg7XUXyHJ1I6A2nbGMxh8xe5Kdf99XpUNmlD6fXul92lrvvoCb+W6w TLFnkPhE5owjD6saFwzNgCasD+94qxC1OzzlEf6t4KKKstegb06e02i7wLSnT0iu 5JdUFQIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB AQDIMNgMqgGdXwZTIDxz4iq76iHdSHxZK4YwJr/4xRSq1uEddxmUfAQzo+gBboA5 c0XxYxc0xViuueThnqdLhHmEyCs8uGFRLaQAI5Bq1PdMQP12m5fCWRAyKrCxdtli zm8ByDq7mWpkfMTd/rJGR4wCR9qI9Y5Bp6p4FBKZ3pzEanFXMV9IHhkm1BGh9tbe l6GQyBptEpfTiRwNCC/ympeiL3G8hfDCPkcLed5sQ+OhPe5iWMVPncZh/qehnUJK B5CIXcagcROFutsDYPCurKcfQOsfqulu0q95h7FQUOsIeU7jIlcLxIii19qjlTZh sFjsul5G7qqMEgIUsx3U985v -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/000077500000000000000000000000001477524627100210335ustar00rootroot00000000000000nats-server-2.10.27/server/configs/certs/tls/benchmark-ca-cert.pem000066400000000000000000000025671477524627100250160ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIID2zCCAsOgAwIBAgIUZj0PngA93uUSShcRndTQju/J88YwDQYJKoZIhvcNAQEL BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 WhgPMjEyMzA3MjExNTI3NTdaMHwxEzARBgNVBAMMCm5hdHMuaW8uQ0ExIjAgBgNV BAsMGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWEx EDAOBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNB MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2cCyJL+DExUyZto2eFLm MBRSkQLxM9pOWB9O8TecHlPcc/SPGq/x9lpguJ/IiaUj+VffVWy236KW2JL5Xj83 PZwhXi1yZzxlIBsKAgAUeNfWuTAc0K0Qm9pR5Wjv5eNcT0mw6JX0SPgUQAl9BSwU WvtMOTxOt0hBjHmZaEamp7nLmwogpvgPsrubD6U4O/vUQm3JTsbp2rFQxXPpkG19 69PGsT37r0/w9Zv0xNAcB/zCWdNBXCTA2ACV2IpJedWm8Jrjcn3Kp4Fv3TKTsCZl eWtfxCdljndk88+NFK7cEw7b9Bs5R5Zhu20C+Ne8vmMWhYbVBFYws5/jGzPBkVTD 7wIDAQABo1MwUTAdBgNVHQ4EFgQUEqfeAemfeIp4MM4C7H1bJS+mra4wHwYDVR0j BBgwFoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDwYDVR0TAQH/BAUwAwEB/zANBgkq hkiG9w0BAQsFAAOCAQEAiamiPxOlZ0pwOJvv0ylDreHVk2kast67YlhAOcZoSMvi e2jbKL98U3+ZznGj21AKqEaOkO7UmKoJ/3QlrjgElXzcMUrUrJ1WNowlbXPlAhyL KhNthLKUr72Tv6wv5GZdAR6DaAwq3iYTbpnLq4oCnFHiXgDWgWyJDLsTGulWve/K GGM2JMcnacNgNC18uki440Wcfp0vGj9HhO6I/u63oGewZnIK87GQMQCt3JLFyiUc hrn9nWoixFWcJfCjBcMlwZXMIAlDdelU1/hWtSknKCs57GvZuACcicAYiYIkWCkd p1pF4G0Ic6irAnLTqhdGwL4+5pjNd1Ih0Gezn9hJLg== -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/benchmark-ca-key.pem000066400000000000000000000032501477524627100246370ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZwLIkv4MTFTJm 2jZ4UuYwFFKRAvEz2k5YH07xN5weU9xz9I8ar/H2WmC4n8iJpSP5V99VbLbfopbY kvlePzc9nCFeLXJnPGUgGwoCABR419a5MBzQrRCb2lHlaO/l41xPSbDolfRI+BRA CX0FLBRa+0w5PE63SEGMeZloRqanucubCiCm+A+yu5sPpTg7+9RCbclOxunasVDF c+mQbX3r08axPfuvT/D1m/TE0BwH/MJZ00FcJMDYAJXYikl51abwmuNyfcqngW/d MpOwJmV5a1/EJ2WOd2Tzz40UrtwTDtv0GzlHlmG7bQL417y+YxaFhtUEVjCzn+Mb M8GRVMPvAgMBAAECggEASvFWfm5JMtqjP6HPiGbjoV2FMzJjiFiUiSCxXzSn7wom v+PGEsXGTWa6jiAz+SeUc38KNtDVOa+wIfani4fPP82J8GtMyfoPxdZ4gcq8QQDr /k1wRWOi6Tjg4cdVdXXkMcencs0VR73V3lpFpG+Qy+VcTQCUCOF96dZ59VkHh4a4 CHX6PegOWwr0TSaCUacwhua+rPmCar/btAYv7Wp7+c+Zf7Rn2WYTV7ol4sYXR4ZQ Sy/ROijeFTMkYNpaW/KOO2/pn3OJA8ycYH8UWZpsenPfIajC0Eka7BoEQpw6M8HR wRWrKwssBEs0psFiq9s8J+6resPgXfU/9pf+mTkTqQKBgQD8kktUqN+vYc6t22DE tSkg8y8OsGh9VTYfp7x5hu9qEC9t4mAKjqA/rRLiTXze/wInntreRTjjMb/NlqMy PvI0Z+dM1UuqcF1axgKrIYsgnLJWuunOhaj5K3LhiNcHznlCtN9601dbccwLlQhL 5jdjnOuJ0i+Nh9v5oiu37SfldwKBgQDctWdbF4hJrBPnS6CcojuQj6ha10wYYe46 ZVcxKe5hFBs1q975YCHnEntyCDvXGfOTRgbKeZbwNhMvAc7Pp6eGMR/9SpiRwTt4 567hUz56WXVmp4gSvxoNuYRlWiMI8rZkyKJ8KFipvHgRa8nuamh0QBB4ShEJiVk8 fhaUiZeTSQKBgF0hAD/OKPR9Jv06J9tARVMN+Cr9ZvnXwqY3biqNU5gTMbndv7YE 0xfHlG/3THTZKI09aMyOT6SOQn/m7HPpe9tQ+Jt/BnBpEDMZUgCR1MAIp0WNlAp/ hEej+q8oiskpG9M56DFc3hgsxKT8pdt+nqvPP5ZI9xnDn5vTbTVbb9uPAoGAFRvU csXhZwpqLOjyx4hMohrbQzTsNjjHjBY9LJqSDf7aS1vQy5ECLRN7cwCOmJgGz8MW yy6t3POPCiPmH74tK4xvPs5voSEWCw49j5dillkP/W1wejqEx2NC4l6okyaDg0gd IjrJoBJCeYgRnBfZPaUS7i5HSt40BrEYf8RZFuECgYAjSnvY8nYFRudsylYOu3TL AcGbAdpDfL2H4G9z7qEQC6t1LqqGNcfMan/qS4J/n5GCOedVWHcfCraROOMil52v 3ZDXyyjGEO08XgKnoa2ZL/z2a6s077+hAAnbcywyi2Qz8Yfi6GnwYfU1u8SN0APd T71HPNsWkU4zkxmP4S0Olg== -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-cert-ed25519.pem000066400000000000000000000021631477524627100267250ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDHTCCAgWgAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsYwDQYJKoZIhvcNAQEL BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMCow BQYDK2VwAyEAyyc9y9iZgWWSsPRahbeGxF6XN3VOFPZBvD/HQps6jr6jgZEwgY4w CwYDVR0PBAQDAgQwMBMGA1UdJQQMMAoGCCsGAQUFBwMBMCoGA1UdEQQjMCGCDnJl dWJlbi5uYXRzLmlvgg9yZXViZW4ubmF0cy5jb20wHQYDVR0OBBYEFBwkwMU8xuQO FN1Ck5o2qQ4Dz87ZMB8GA1UdIwQYMBaAFBKn3gHpn3iKeDDOAux9WyUvpq2uMA0G CSqGSIb3DQEBCwUAA4IBAQALjCynuxEobk1MYQAFhkrfAD29H6yRpOcKigHCZjTJ Dnpupip1xvaFPPvhi4nxtuWcXgKpWEfd1jOPaiNV6lrefahitZpzcflD7wNOxqvx Hau2U3lFnjnGaC0ppp66x26cQznp6YcTdxrJ1QF4vkOejxqNvaTzmiwzSPIIYm7+ iKVWT+Z86WKof3vAdsX/f148YH1YSPk0ykiBzlbLScbyWebbaydrAIpU01IkSvMo qDYu+Fba0tpONLe1BUklc608riwQjw9HiJJ2zJIAOBAUev5+48RP91/K111Ix1bl fGPT8/1TJbyGG2jeJwyLoSIu72aDnnIBfqGkVunRTmeg -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-cert-rsa-1024.pem000066400000000000000000000024261477524627100271020ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDkzCCAnugAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsMwDQYJKoZIhvcNAQEL BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIGf MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyHHaVHinB3jBsicR4hp7uopz0u3+O kUicIUSQXDcWiPzdvE+7YZ/s4+Ud4aw4g9q0wHzkZSaMg8nil4tCKmTrUKolVTVj CCCBmtqq3LwzNLapyoDJRyXsWqHt5TWYSxaf/UQT6sWOgqHOLrbd4J8F0sjxEniB GDHR1ZXpJCBaIQIDAQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggr BgEFBQcDATAqBgNVHREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMu Y29tMB0GA1UdDgQWBBQk5kWOUcUNn7FppddLANe3droUlzAfBgNVHSMEGDAWgBQS p94B6Z94ingwzgLsfVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEA2Njy2f1PUZRf G1/oZ0El7J8L6Ql1HmEC7tOTzbORg7U9uMHKqIFL/IXXAdAlE/EjFEA2riPO8cu/ bvL2A4CapYzt2kDD9PPYfVtniRr7mv0EVntPwEvfiySMAEeZuW/M2liPfgPpQkhL fzwPeCOfqM8AjpyDab8NEGX5Bbf421oQorlENpm4PKQCXoUN5cWpBwuwWxj7yndj 256MevLDKKe/ALSLQEo/2Jgpnmp7Qol0GtomCzsLgZ+ASuVtCsGTFmaRrsqVPspJ oOl6qby5gYwN9TR8zfRYL1m1sbYROz+5+ofEoiTnaOoOSjiBIoYoMeSC/jvJQTPT VdD8QeQ6Og== -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-cert-rsa-2048.pem000066400000000000000000000027101477524627100271050ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEFzCCAv+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsQwDQYJKoZIhvcNAQEL BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIB IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyx3O+Z6u8Y1SiuHu3szWbLvL WrZpSpEiZkll+wk5205S1FRcQLccfr4ubdtjOBdi+RzILCtkflUI01Dbqu6cV7/2 yfLthxBeNDiXMhjyOFkYLwwE4w7CdTwWWsmW31oUH1rYXIDPoeb7WPF7w3NwaUJu ZXnqM98LRgWDTmh+nsqDDW/bz1fYIdxcO9az6iBOnJ2AGWI2ur5GzWc4+gNMOZiZ Xj657g0MbyVM4Gzyc4Au22hShZ/YorLP8NAiwNJamlrCFzlnZN/ePjuQPcI6glnb oO9IAGfPdAOJptfayuPAZgUngzewB38yY0Q/rKG1GJKSkQ8X6/lXiWaRPZJjYwID AQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggrBgEFBQcDATAqBgNV HREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMuY29tMB0GA1UdDgQW BBRtanJZScdSlsPsi58lBcpdj+bV/zAfBgNVHSMEGDAWgBQSp94B6Z94ingwzgLs fVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEAV4TZ3b8cYO7ZeRyoCQtCBAab9gNe kbQpWqICvkVQOk5Anq3opwAWk2FuIRs5KoT7ssckHpXwTwWLs+KuIVo+Fet19IH6 BQfck1jwhzM04MA6zLO/F2j548XlrJy3IzViPM/VxwMMTt5YSoogrz/3TzzJPIe0 eQomf5HbpVgrf08pMVkdaI7PCd7N/CxeWiD5zEWqBu9FqofO188Kb/umx0VwgBju dX46MKO5TyUc91UrG3M35/r4Z7fd52SWWWFQiI7UBOl2L27samjHlJsKjyFoBF3Z alvnoUVzo7zwAYmhEdPYDNVceF4KtAFpGipoQPRMg83G87LgYBA4Sa6uKw== -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-cert-rsa-4096.pem000066400000000000000000000034421477524627100271150ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIFFzCCA/+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsUwDQYJKoZIhvcNAQEL BQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB bmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250 bzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3 WhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM GVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO BgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIC IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAj/rw0WpCnizUSs06NxSsjuYb 6sWtYQjLd6O6SCoQwrPSY2Zv0u28RywBJkSiIbv12kWxrhqv+kDqhmt7tfwEdHAE gOl6E2P/CFDnyAJ4it5qgFRHcEItGp1Ap4ZrZ3OTl41DDe28giflHb7+VAzxUV6r zkHmrqkaJd41YrJoKyZ3u+/FHs6CudO+jGXC2ubH1i71ARXHn6tkmOTwFf1N49wB Hr144xBmPyvH/A/elgXBjR28/7x64MO/qBZsr/lbTaS4YwN1+rod/HRc2GIiJxpS tB1bh/dXZCfa0QyhTzCNLG/j7IzetDrPBGzpw2WjufhkSuaxoMDWlDkkfqmQxtHJ 5L+PqIiPT69tzEbfuS9Ogz7DW10CcpSdXW13sWCdSNCGEvPqLFka36q9B82V0GHz tmx8VqdfWSu4qMyVmTsxxzLTTxpQU4X1Q2RnT/1igbsOM620LuvrvnXlh5Rdg7tp T++QJ/b4xCCg62tv4VwORe27xHYXMeDn4aoRdyoI45/+ZK6yqwNAOrKKpse/M8uz mJK2i8pfEFmitIKoNYn3MR2dFrCqifZkFf9rX9A/1Ym+WKAPLmWdGp13fvTdzxQG Y44f9tBL2RWsoGX++01XEwIiWz7kqObC0L8fz3EdIPaULX7MZiQrxzzhRCcJhyn/ aOrJfLYj0GAmIaHElHkCAwEAAaOBkTCBjjALBgNVHQ8EBAMCBDAwEwYDVR0lBAww CgYIKwYBBQUHAwEwKgYDVR0RBCMwIYIOcmV1YmVuLm5hdHMuaW+CD3JldWJlbi5u YXRzLmNvbTAdBgNVHQ4EFgQUbJSM8LNWmc9IgLe0X53yE3c3h1cwHwYDVR0jBBgw FoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDQYJKoZIhvcNAQELBQADggEBAEGmLvEE +MTE1bHMbl/5QG+/xusmervIuxkfAfId0H+8TWB75y+yhVZpEdM7knfl+lexmtGQ Gr4HNGTZhAZ3NYFaBw7nfeqO48He7gHUKfJA/zv7FREF3Fy+Qe/hydDJQzBzZfaU 64XqhY6jOurpZhTAoOXfjpYzZaLi7+rdpTAbfxHCCAC8SxZD3++Q97ZeoT6en47O 8SQQ7FIzWxs15k88oYalw51vZujxX7dz4l+LxsLXtlYW7ZM1163cgU7lF/jQqDcN z8X8jk1AjQY7AuFPuOzQ1hLXcZySm8rUG5pPgHrZ1QKmkFFWRaeCiO2hU794wI5C vIGR8lIhkNwEJnQ= -----END CERTIFICATE----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-key-ed25519.pem000066400000000000000000000001671477524627100265620ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MC4CAQAwBQYDK2VwBCIEIJRCtUNxUuutNs9j8OtcwFw1xkbs+zxjHhpAqVuqDNo5 -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-key-rsa-1024.pem000066400000000000000000000016241477524627100267340ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBALIcdpUeKcHeMGyJ xHiGnu6inPS7f46RSJwhRJBcNxaI/N28T7thn+zj5R3hrDiD2rTAfORlJoyDyeKX i0IqZOtQqiVVNWMIIIGa2qrcvDM0tqnKgMlHJexaoe3lNZhLFp/9RBPqxY6Coc4u tt3gnwXSyPESeIEYMdHVlekkIFohAgMBAAECgYAwt8RfyV5WnvXT2mMZLIlwcJ5J +rdLQcYAnsDoU7DlwxaXeBi/AlcCLtvOrpmy464A3t3KgzhmGu4vwo/ey0XK+nTQ tzORP/PXTaVC8DzJ8PnJmUaB7l+H7a88OSPLbjgnbpw4SyvDpKUHiiw0EDYC7L6Y 1vvCOlnprptXbE5eeQJBAOpjwdBVWkVtmStjsxbxZsUTI7XKxS2VZinRLH0l5/hI hIHRxwy9oRbeNrf5815lGolTUD0mq+N0dJRlMop1yKsCQQDCiGDkH/pQqhB8ibmD 0XNw0EzxJmPFACO/x49VCfCPE5p1FQhpyIl6JkyAFNN7Xs4HX8jMHTuvNgJVti61 O0BjAkEAj0wr2vXDubyWrztF61nszcG0zFjKkeLL0fcLLvv0xQt4z3F0MyrgCH4U kAflLSm8voZMAQbagbXZ7DuuWY5G/wJAWyKnOdidXZL+3ElthwrmKVD86vEQRqe1 F9C3HqDkeTM25mkvItfXSEmPB2Y6WY7luOCv4qhDYOdNmrgaE7+pfwJAcbV5ZVJW OZvH1ofsJVvUA8J58tzv1+KPb96pI3YRAu8xbMC0mzezPsYjg2wjaRgJ2C+7On27 BaArNo75B20AkA== -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-key-rsa-2048.pem000066400000000000000000000032501477524627100267400ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLHc75nq7xjVKK 4e7ezNZsu8tatmlKkSJmSWX7CTnbTlLUVFxAtxx+vi5t22M4F2L5HMgsK2R+VQjT UNuq7pxXv/bJ8u2HEF40OJcyGPI4WRgvDATjDsJ1PBZayZbfWhQfWthcgM+h5vtY 8XvDc3BpQm5leeoz3wtGBYNOaH6eyoMNb9vPV9gh3Fw71rPqIE6cnYAZYja6vkbN Zzj6A0w5mJlePrnuDQxvJUzgbPJzgC7baFKFn9iiss/w0CLA0lqaWsIXOWdk394+ O5A9wjqCWdug70gAZ890A4mm19rK48BmBSeDN7AHfzJjRD+sobUYkpKRDxfr+VeJ ZpE9kmNjAgMBAAECggEADicdEWKfsQAWZMv6X3bpZ/kr4b29F2+GdJcfrn7Fk8Tg 25+nL/EyYJhy1r/HKZTjlhUN05oQbgcRztue+smWhjy/fvHY4CThU4Uv79MyKX/3 wet1+bZBEXcm3ZuXUifOKCMl2Ug2b4MPN3LYG1XTWtpAo/x7N7MOb4oZzKBWVk0J GEfxvJVZyVe3libIN2WWZFQJ1620AxaWP2jZ83PhRR/TJIBiq+pI/lBIyt+nu8Z3 3vzB020R2uvENOhnaDjzNVfnJxSvlAAQkN47zo2Tf4UnB0792Y7DEEjFKX4XCFVd zxmjmw7VcRcnyruDhCoRC6mNraaAHuMqPwuBoC6GtQKBgQD/7CV8XKzcPPfFJY9e aHPzgXJwK+5u3jq7tNYUksVfv0s2lLQnRdbqAhHzxNYLQjdVd7J6t55h2z8scYaP oB7TTwszhKZS2sQ/lcpfOFoFN6KjN0iOnXVFucGoQ36gexNqPw894NFKWX/RHrnZ UfL/OnOUPpra3w+WMxjUYQ5ivQKBgQDLLZDIpM8fSRKeqZzjv/eoKnoDhqmzPipj bvNXAkIr/nWfUHL9YRnpxX7PW8DqFWIYoM8b0uOKO4vLJnQUMyc2jPq1rPXEBrjk w+xKWCipKdPrqwttiLKme+ZArRT5CJ9qcqxSX0yNp69xwXtyHe+zY18F/a4+70wT 5wJwYmhQnwKBgHpICTk8xtOMxg6K/c/sNMr65QU32Htc789Ufp3h6zDupC92KgZB 1oiFaLKDMIq8ntfVk5ATQDgdnDfOHq9toIzyzbVWAmrAYNjI56NLt6eah7lY5vBN yAUC1sdhSJXBeOthKhU04IuX6/yto7t07piJA0SoDTHbNwVbcNe5cDg5AoGBALjR jxVlDd+4mc5oHYXy1rZLRUg10+JvlyFyCLrKHCVmx9oO1Tr1fBvxggPfw+FraBtd FGiL8l2JAwXdydOiIHZ30Ys3dSxGrSOzsRqDjSEsIlEK+088/L2CkRWeHCjYliK/ g08+zyVANtC0nrVU0/mLWCHb/AfVp4+nIMnYSmmjAoGAEMyBqq2AyUmx1xAmhw36 LqgKy+vgHEAFRFPD8IttHFLOlUdXlvxoDq4xW2a7bJsJrs9ZrluRFKVh7QnSckmP Jt/Plg+XYB3B2exD5Xyh9xNYNVW/Aqvg+NuiWeCGK/o7mUfGWd9qWrD2aw51m+X3 Svtkgck1kulqPoUFG1b3R4k= -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/certs/tls/benchmark-server-key-rsa-4096.pem000066400000000000000000000063101477524627100267450ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCP+vDRakKeLNRK zTo3FKyO5hvqxa1hCMt3o7pIKhDCs9JjZm/S7bxHLAEmRKIhu/XaRbGuGq/6QOqG a3u1/AR0cASA6XoTY/8IUOfIAniK3mqAVEdwQi0anUCnhmtnc5OXjUMN7byCJ+Ud vv5UDPFRXqvOQeauqRol3jVismgrJne778UezoK5076MZcLa5sfWLvUBFcefq2SY 5PAV/U3j3AEevXjjEGY/K8f8D96WBcGNHbz/vHrgw7+oFmyv+VtNpLhjA3X6uh38 dFzYYiInGlK0HVuH91dkJ9rRDKFPMI0sb+PsjN60Os8EbOnDZaO5+GRK5rGgwNaU OSR+qZDG0cnkv4+oiI9Pr23MRt+5L06DPsNbXQJylJ1dbXexYJ1I0IYS8+osWRrf qr0HzZXQYfO2bHxWp19ZK7iozJWZOzHHMtNPGlBThfVDZGdP/WKBuw4zrbQu6+u+ deWHlF2Du2lP75An9vjEIKDra2/hXA5F7bvEdhcx4OfhqhF3Kgjjn/5krrKrA0A6 soqmx78zy7OYkraLyl8QWaK0gqg1ifcxHZ0WsKqJ9mQV/2tf0D/Vib5YoA8uZZ0a nXd+9N3PFAZjjh/20EvZFaygZf77TVcTAiJbPuSo5sLQvx/PcR0g9pQtfsxmJCvH POFEJwmHKf9o6sl8tiPQYCYhocSUeQIDAQABAoICAAEYUQ4KqeyJiJN0DvIIdc0q v1dwVEKlaOS6nvSkYJcWe9kL2E8JRfy1lIxTCneethPdoqhhZWljZn/fX+QRAVir BGxq5NCHxWgDCNble+oJydNlhgXldEcxnyJXBvM/trA49Q5X+pkeeWMvRsBiuSVx BqCi3CtZDQzw18TtzessF6OF2IuE2bYqP9anVyLH4rOvN5JKn9zH69PaLorq76rL YMp4JEFNKIs+R5RUFjwxBC6Q+r+9fM1qTQdtJOZMC293pfusylAoll315ZczXIah xYi/ThOmjtMrwWyEan1PcCIVzMI0X7o3q0eMQNU0ApmBY9k0+sYEnsJ8Um8l1icb WE/KBDzQk2mSoIzEdo99gYiuLAhi5WTEcQzeovZKHKkhRxLn3GUK/ZznpTuI0Voh 1Qbfw05wf62yh8/nc747n5ULPgf8/nUH5G8jUeSis9gMGtFloqIxUEBphzIHwj7W FUIKoKpGkHL4Jkg9wXIe7lg5Qp+cdCrkWKKvQc57EbWxsM1Pjktw6X5ctNQclWG0 WQJUw+BJhq6InKAV4aDW4/QlCOjvGtDDkYvwghE+ZgIzm9BARM4+DbeoPTmAkwwz 6NTM8DYWvdcYVpgchEwphS6OFkOD1Dc6Wg2OUUtLjvl4NVf/NDYN8wIWNwmMm4aj +Is12NwoUqZ44E4pSfohAoIBAQDLloatDuObuLzInBqER2tByPNnDV9AVTR2rm6w TgBmslFHBCf/uSzq0ivh1oZmFfDhcqdrtheTL0BWXbFxR/GelDm9N4d/LqOGRyWV Qmy5dGp2wE6gsZGZuvsbVst6eFO+zgDLvDMLrVqP+577g/QbrSR8gPz39WCgLpvf RcNpM2rHeKkm09IdUe0G+YEnW3fD7rxtMS+j8J57yuuxLO+SemGJuJQqGHYOukxF KMZxX2wMs1YMNwKEXY0/9M+LAtW9J3azkRp5xd+E2kzXGoWcl2nS/3+J6nh1+HWq SZE7SDWSimzut/aPTOM7YxLVqtANWHzVBBNpTbOMSrSZTZQJAoIBAQC1C/afOROb GQMb1LOt95AxvGyYo62mOkcPxhxQGojVIi+Yz8Q1ZWTuT5qyyWSofR3bE9mQwQGW KQ/znJKiwA48BS0osnmQi3Bi6eVD73NfwxMkL6zrGFxf336DNQi9mbidJvOaHNsT wXGPqRQXDcLa3s3WqerDzKJ2gxG22rtkoC0uCw+J9NlkJTTI9bJ9rjDZIh6mE/M0 3ye19IgkBdbMv7FSjGVpovdqWZ2HELDYAXJftzuMPUO1GNYbyMHAHnWH0a1TQRM8 ELHzmPRaFBeegmpBVaenlxKoi72iwqrVFrEr5FLAKxLq/9RwizJ8FzySIpH/5+hU Ky4mG93lmHjxAoIBAQDLOXs+jTpPW923M3yUxuYeSQYPvJ10jplMT1tWysZDvYS8 qz1yW9qmnR4I1ihbB1Po+JZ/QsnNtsE2dViHiBV9AuGQLDopjtjVVXgCwsfdaIRN /jF+30JEfw3igIWlvy95rBHHThp2cZmRWM+eql2msvNVBT2AF4VY4K3f9rfV7+mL LLtNcuyvL/S3naB7NSccgte89/hiYfMSB8G2nvCW+2saGxJr4vcWRImWD9nnmiU1 mF8w2ki88NXrHel/Dlll9FrdbN9M52T0LSW/I050vgB5C2q4tUGCIX7zeXRsBOzV VzDeKu0Ipuu9gGxwtY3xhH839FWcLGAqjvgwf+xhAoIBAQCuvWs9ZoNr0QpVFEiQ Aj9kIa7W7DOwGtN3gAjXr6Sdwa8a2H1R5Bk0ghSXtxW2IXxtdI0qz35OhjdlM5u8 BY43k+9wNkJqporEjWfA2B4NMWUKKhHFnu+ZgUbEMK3NAc9TrsKz3mH8gVqwA8rm LVwCj8UwCTQT4zBzHjI8wITZrFeu9vH6fx5LMDXwOGQcNcHj8LCQLvUv9KqJTgkQ a6pUWDg3qlY/TRFrzi7iq9NjyJGxnFKXGpJ8+gm9K1kFquBZRKD7l/WOpbZ7nQdK 4dWiIdGYWanFcWSK1MUlkKn9nTdHW8oau/g4ZM+QCGmjp3HIwiEUU6rDgiG6mm7j KPShAoIBAG1sFn8X841038Z/sp4JzYypQjj7g9UOTkghHcJHIZzRXLiTqW3lKxbt Gz93DjWRxXD404hem8dOUf22VgfjB4z1mSrV5SWtLjLf5wD8gl0eUNKP2lZFLggz O6nHCLLzlKs2RH5pDo3c8qyjLRIfCXy0YGnr+9RErVJG+TVB/MOSgHagmkdVOYCH phw4EiwJ+rPFy/xm5D6+BuOyt7hw7boQsw3EHpZTyQHWObcIggjlopTX2VSG8Dx+ /iQRTuVRVyNAhYwCuNtSh27zawWr+A40acFJsJpvkFkbZBEH1IqoCteUaiEVU1qm 51lKgt3ZVAXuecJ1U/0u4HtC0QdEBGE= -----END PRIVATE KEY----- nats-server-2.10.27/server/configs/cluster.conf000066400000000000000000000012051477524627100214370ustar00rootroot00000000000000# Cluster config file port: 4242 net: 127.0.0.1 authorization { user: derek password: porkchop timeout: 1 } pid_file: '/tmp/nats-server/nats_cluster_test.pid' cluster { host: 127.0.0.1 port: 4244 name: "abc" authorization { user: route_user password: top_secret timeout: 1 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://foo:bar@127.0.0.1:4245 nats-route://foo:bar@127.0.0.1:4246 ] no_advertise: true connect_retries: 2 } nats-server-2.10.27/server/configs/gwa.conf000066400000000000000000000001451477524627100205360ustar00rootroot00000000000000listen: "127.0.0.1:-1" gateway { name: "A" listen: "127.0.0.1:5227" include 'gws.conf' } nats-server-2.10.27/server/configs/gwb.conf000066400000000000000000000001451477524627100205370ustar00rootroot00000000000000listen: "127.0.0.1:-1" gateway { name: "B" listen: "127.0.0.1:5228" include 'gws.conf' } nats-server-2.10.27/server/configs/gws.conf000066400000000000000000000002231477524627100205550ustar00rootroot00000000000000gateways [ { name: "A" url: "nats://127.0.0.1:5227" } { name: "B" url: "nats://127.0.0.1:5228" } ] nats-server-2.10.27/server/configs/include_bad_conf_check_a.conf000066400000000000000000000001261477524627100246520ustar00rootroot00000000000000 port = 4222 include "include_bad_conf_check_b.conf" # http_port = $monitoring_port nats-server-2.10.27/server/configs/include_bad_conf_check_b.conf000066400000000000000000000001271477524627100246540ustar00rootroot00000000000000 monitoring_port = 8222 include "include_conf_check_c.conf" nats-server-2.10.27/server/configs/include_conf_check_a.conf000066400000000000000000000001201477524627100240360ustar00rootroot00000000000000 port = 4222 include "include_conf_check_b.conf" http_port = $monitoring_port nats-server-2.10.27/server/configs/include_conf_check_b.conf000066400000000000000000000000751477524627100240500ustar00rootroot00000000000000 monitoring_port = 8222 include "include_conf_check_c.conf" nats-server-2.10.27/server/configs/include_conf_check_c.conf000066400000000000000000000000611477524627100240440ustar00rootroot00000000000000 authorization { user = "foo" pass = "bar" } nats-server-2.10.27/server/configs/js-op.conf000066400000000000000000000052661477524627100210210ustar00rootroot00000000000000# Server that loads an operator JWT port: -1 server_name: "S1" # The test will cleanup this directory so if you change it search for the test. jetstream { store_dir: "/tmp/nats-server/js-op-test" max_memory_store: 1073741824 max_file_store: 10737418240 } operator = "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIyUkc0WjJKVzYzSUtBS1czSFg0SkpMQVhRN1ZSM0NKVlRQU1FHUVRCU0ZFQjVKTkNISUpRIiwiaWF0IjoxNjE2MTg2MjAxLCJpc3MiOiJPRDRNNklBQUtRU000RFFTTFdaSVZCQUNZVFlTMlM2WFFTN1U2N1hYWVhKNDRJTE5FMzVZUEdKSyIsIm5hbWUiOiJqcy10ZXN0Iiwic3ViIjoiT0Q0TTZJQUFLUVNNNERRU0xXWklWQkFDWVRZUzJTNlhRUzdVNjdYWFlYSjQ0SUxORTM1WVBHSksiLCJuYXRzIjp7InR5cGUiOiJvcGVyYXRvciIsInZlcnNpb24iOjJ9fQ.VkjSK2BlMmtYfVJSkC9aZEFvjg4BXzbd0oXkQa3Rlkhh8EuSRU7-Tp1zUm1SveBb6dZXsE51vhIQFQY66fO0Bw" system_account = "AD4M34OPRUPWDYTIQLWKMEZZM5MK7PQQDSKNELCZDRZZEVFWOWPLFTUE" # configuration of the nats based resolver resolver { type: full # Directory in which the account jwt will be stored dir: "/tmp/nats-server/js-op-test/jwts" allow_delete: true interval: "10s" } # This is a map that can preload keys:jwts into a memory resolver. resolver_preload = { # system AD4M34OPRUPWDYTIQLWKMEZZM5MK7PQQDSKNELCZDRZZEVFWOWPLFTUE : "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIyQ1dSRk80MlZVMk1TVlI0QkhIUFBEVjdJNERaTFZHWURZUlNXQ0pIVUM0Q1dRVkhXWjNBIiwiaWF0IjoxNjE2MTg2MzYxLCJpc3MiOiJPRDRNNklBQUtRU000RFFTTFdaSVZCQUNZVFlTMlM2WFFTN1U2N1hYWVhKNDRJTE5FMzVZUEdKSyIsIm5hbWUiOiJzeXMiLCJzdWIiOiJBRDRNMzRPUFJVUFdEWVRJUUxXS01FWlpNNU1LN1BRUURTS05FTENaRFJaWkVWRldPV1BMRlRVRSIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJ0eXBlIjoiYWNjb3VudCIsInZlcnNpb24iOjJ9fQ.gExWTSaWWSYh9N_KsrOGPAJk0KJUn1VjVo00yfE6NoMXrmjkATDMoDLpKC19mrBNaMa6RZjhXyTdzJulxzxGBw" # one ABVRZKJ6Z7NIMPIYJRCHEYFRUO7ENN6NWOKQDFLFDFVPSM36UPX2UCPP : "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJRSVFDRFpRRkRIUU9WQlc3Q0tDNFQzVzJJV0dZV1ZISjZTVTRSTlYzUFo3NUhTUFVQS1ZBIiwiaWF0IjoxNjE2MTg3MjUwLCJpc3MiOiJPRDRNNklBQUtRU000RFFTTFdaSVZCQUNZVFlTMlM2WFFTN1U2N1hYWVhKNDRJTE5FMzVZUEdKSyIsIm5hbWUiOiJvbmUiLCJzdWIiOiJBQlZSWktKNlo3TklNUElZSlJDSEVZRlJVTzdFTk42TldPS1FERkxGREZWUFNNMzZVUFgyVUNQUCIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTEsIm1lbV9zdG9yYWdlIjoxMDczNzQxODI0LCJkaXNrX3N0b3JhZ2UiOjEwNzM3NDE4MjQwLCJzdHJlYW1zIjoxMDB9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJ0eXBlIjoiYWNjb3VudCIsInZlcnNpb24iOjJ9fQ.yYdE3AjpWMBt_rZVLSNmuxR28FP5-6fO_x6FMQl7xe7hmeEc--bbCBcw4ujEjtVkE62I21asKtu06vmW1fEsAg" } nats-server-2.10.27/server/configs/listen-1.conf000066400000000000000000000001311477524627100214070ustar00rootroot00000000000000# Make sure -1 works in listen directives too. listen: 10.0.1.22:-1 http: -1 https: :-1 nats-server-2.10.27/server/configs/listen.conf000066400000000000000000000003051477524627100212540ustar00rootroot00000000000000# Test all permutations of listen address parsing, client, cluster and http. listen: 10.0.1.22:4422 http: 127.0.0.1:8422 https: 127.0.0.1:9443 cluster { listen: 127.0.0.1:4244 name: "abc" } nats-server-2.10.27/server/configs/listen_port.conf000066400000000000000000000000151477524627100223160ustar00rootroot00000000000000listen: 8922 nats-server-2.10.27/server/configs/listen_port_with_colon.conf000066400000000000000000000000161477524627100245440ustar00rootroot00000000000000listen: :8922 nats-server-2.10.27/server/configs/malformed_cluster_address.conf000066400000000000000000000001141477524627100251700ustar00rootroot00000000000000# Test malformed cluster listen address cluster { listen: 266.0.0.1:foo } nats-server-2.10.27/server/configs/malformed_listen_address.conf000066400000000000000000000000771477524627100250150ustar00rootroot00000000000000# test garbage listen address for failure listen: 10.0.1.22:foonats-server-2.10.27/server/configs/multiple_errors.conf000066400000000000000000000022371477524627100232130ustar00rootroot00000000000000 authorization { user = foo pass = bar token = quux } http_port = 8222 monitoring = 8222 write_deadline = 5 accounts { synadia { exports = [ { stream: "synadia.>" } ] # Malformed nkeys nkey = "OC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL" users [ { # Malformed nkeys nkey = "OCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3" } ] } # # + nats < synadia # nats { # Malformed nkeys nkey = "ODRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS" users [ { # Malformed nkeys nkey = "OD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI" } ] imports = [ { stream: { account: "synadia", subject: "synadia.>" }, prefix: "imports.nats" } ] } # + cncf < synadia cncf { nkey = "AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT" users [ { nkey = "UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ" } ] imports = [ { stream: { account: "synadia", subject: "synadia.>" }, prefix: "imports.cncf" } ] } } cluster { authorization { users = [] } } nats-server-2.10.27/server/configs/multiple_users.conf000066400000000000000000000002251477524627100230330ustar00rootroot00000000000000listen: 127.0.0.1:4443 authorization { users = [ {user: alice, password: foo} {user: bob, password: bar} ] timeout: 0.5 } nats-server-2.10.27/server/configs/new_style_authorization.conf000066400000000000000000000013161477524627100247520ustar00rootroot00000000000000listen: 127.0.0.1:4222 authorization { # Our new style role based permissions. # These support both allow and deny. # If allow is empty it means all or ">" # If deny is empty it means none, or empty list. normal_user = { # Can send to foo, bar or baz only. publish = { allow = ["foo", "bar", "baz"] } # Can subscribe to everything but $SYS prefixed subjects. subscribe = { deny = "$SYS.>" } } admin_user = { publish = "$SYS.>" subscribe = { deny = ["foo", "bar", "baz"] } } # Users listed with persmissions. users = [ {user: alice, password: foo, permissions: $normal_user} {user: bob, password: special, permissions: $admin_user} ] } nats-server-2.10.27/server/configs/one.creds000066400000000000000000000017141477524627100207170ustar00rootroot00000000000000-----BEGIN NATS USER JWT----- eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJHRUNQVEpISE1TM01DTUtMVFBHWUdBTzQ1R1E2TjZRUFlXUTRHUExBRUIzM1ZDUkpOUlZRIiwiaWF0IjoxNjE2MjQ3MjMyLCJpc3MiOiJBQlZSWktKNlo3TklNUElZSlJDSEVZRlJVTzdFTk42TldPS1FERkxGREZWUFNNMzZVUFgyVUNQUCIsIm5hbWUiOiJvbmUiLCJzdWIiOiJVRENJQkdHR0hDSkJRUE9PNFNDSkpCSFpEUjM3TlFHR0NJV01ORzJEREFLSjZXTUtCTUFLWElNTyIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.VLhSDtGZEF_jdvgmhgkdISXAt5wFMMZxxwm5w8UrsnlM1hkUvtxBlTe4IP0xJIf4xf8JOR2Bmf73xUGJKUZECQ ------END NATS USER JWT------ ************************* IMPORTANT ************************* NKEY Seed printed below can be used to sign and prove identity. NKEYs are sensitive and should be treated as secrets. -----BEGIN USER NKEY SEED----- SUAPCDMU5TSHHLWUUZSOUABJXP2GXRCZVEOVWPSVM5XRSXYGQMRRFDYNMY ------END USER NKEY SEED------ ************************************************************* nats-server-2.10.27/server/configs/reload/000077500000000000000000000000001477524627100203575ustar00rootroot00000000000000nats-server-2.10.27/server/configs/reload/authorization_1.conf000066400000000000000000000016351477524627100243530ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { # Our role based permissions. # Superuser can do anything. super_user = { publish = ">" subscribe = ">" } # Can do requests on foo or bar, and subscribe to anything # that is a response to an _INBOX. # # Notice that authorization filters can be singletons or arrays. req_pub_user = { publish = ["req.foo", "req.bar"] subscribe = "_INBOX.>" } # Setup a default user that can subscribe to anything, but has # no publish capabilities. default_user = { subscribe = { allow: ["PUBLIC.>", "foo.*"] deny: "foo.bar" } } # Default permissions if none presented. e.g. susan below. default_permissions: $default_user # Users listed with persmissions. users = [ {user: alice, password: foo, permissions: $super_user} {user: bob, password: bar, permissions: $req_pub_user} {user: susan, password: baz} ] } nats-server-2.10.27/server/configs/reload/authorization_2.conf000066400000000000000000000016521477524627100243530ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { # Our role based permissions. # Superuser can do anything. super_user = { publish = ">" subscribe = ">" } # Can do requests on _INBOX.foo.bar, and subscribe to anything # that is a response to an _INBOX.foo. # # Notice that authorization filters can be singletons or arrays. req_pub_user = { publish = ["_INBOX.foo.bar"] subscribe = "_INBOX.foo.>" } # Setup a default user that can subscribe to anything, but has # no publish capabilities. default_user = { subscribe = { allow: ["PUBLIC.>", "foo.*"] deny: ["PUBLIC.foo"] } } # Default permissions if none presented. e.g. susan below. default_permissions: $default_user # Users listed with persmissions. users = [ {user: alice, password: foo, permissions: $super_user} {user: bob, password: bar, permissions: $req_pub_user} {user: susan, password: baz} ] } nats-server-2.10.27/server/configs/reload/basic.conf000066400000000000000000000000271477524627100223060ustar00rootroot00000000000000listen: 127.0.0.1:-1 nats-server-2.10.27/server/configs/reload/file_rotate.conf000066400000000000000000000001061477524627100235200ustar00rootroot00000000000000listen: 127.0.0.1:-1 logfile: "log.txt" pid_file: "nats-server.pid" nats-server-2.10.27/server/configs/reload/file_rotate1.conf000066400000000000000000000001101477524627100235740ustar00rootroot00000000000000listen: 127.0.0.1:-1 logfile: "log1.txt" pid_file: "nats-server1.pid" nats-server-2.10.27/server/configs/reload/invalid.conf000066400000000000000000000000351477524627100226520ustar00rootroot00000000000000# Invalid config file trace: nats-server-2.10.27/server/configs/reload/max_connections.conf000066400000000000000000000000531477524627100244130ustar00rootroot00000000000000listen: 127.0.0.1:-1 max_connections: 1 nats-server-2.10.27/server/configs/reload/max_payload.conf000066400000000000000000000000471477524627100235250ustar00rootroot00000000000000listen: 127.0.0.1:-1 max_payload: 1 nats-server-2.10.27/server/configs/reload/multiple_users_1.conf000066400000000000000000000002251477524627100245210ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { users = [ {user: alice, password: foo} {user: bob, password: bar} ] timeout: 0.5 } nats-server-2.10.27/server/configs/reload/multiple_users_2.conf000066400000000000000000000002251477524627100245220ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { users = [ {user: alice, password: baz} {user: bob, password: bar} ] timeout: 0.5 } nats-server-2.10.27/server/configs/reload/reload.conf000066400000000000000000000017251477524627100225010ustar00rootroot00000000000000include 'platform.conf' port: 2233 # logging options debug: true # enable on reload trace: true # enable on reload logtime: true # enable on reload logtime_utc: true # enable on reload log_file: "nats-server.log" # change on reload pid_file: "nats-server.pid" # change on reload max_control_line: 512 # change on reload ping_interval: 5 # change on reload ping_max: 1 # change on reload write_deadline: "3s" # change on reload max_payload: 1024 # change on reload # Enable TLS on reload tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" verify: true } # Enable authorization on reload authorization { user: tyler password: T0pS3cr3t timeout: 2 } cluster { listen: 127.0.0.1:-1 name: "abc" no_advertise: true # enable on reload ping_interval: '20s' ping_max: 8 } nats-server-2.10.27/server/configs/reload/reload_unsupported.conf000066400000000000000000000001541477524627100251440ustar00rootroot00000000000000# logging options debug: false trace: true logtime: true # Removes cluster host, which is unsupported. nats-server-2.10.27/server/configs/reload/single_user_authentication_1.conf000066400000000000000000000001261477524627100270630ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { user: tyler password: T0pS3cr3t } nats-server-2.10.27/server/configs/reload/single_user_authentication_2.conf000066400000000000000000000001251477524627100270630ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { user: derek password: passw0rd } nats-server-2.10.27/server/configs/reload/srv_a_1.conf000066400000000000000000000002411477524627100225550ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7244 name: "abc" routes = [ nats-route://127.0.0.1:7246 ] } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_a_2.conf000066400000000000000000000003031477524627100225550ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7244 name: "abc" routes = [ nats-route://tyler:foo@127.0.0.1:7246 # Use route credentials ] } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_a_3.conf000066400000000000000000000003041477524627100225570ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7244 name: "abc" routes = [ nats-route://127.0.0.1:7247 # Remove srv b route and add srv c ] } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_a_4.conf000066400000000000000000000001571477524627100225660ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7244 name: "abc" } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_b_1.conf000066400000000000000000000001571477524627100225640ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7246 name: "abc" } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_b_2.conf000066400000000000000000000003141477524627100225600ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7246 name: "abc" # Enable route authorization. authorization { user: tyler password: foo } } no_sys_acc: true nats-server-2.10.27/server/configs/reload/srv_c_1.conf000066400000000000000000000001571477524627100225650ustar00rootroot00000000000000# Cluster Server C listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:7247 name: "abc" } no_sys_acc: true nats-server-2.10.27/server/configs/reload/test.conf000066400000000000000000000002471477524627100222100ustar00rootroot00000000000000port: 2233 # logging options debug: false trace: false logtime: false cluster { listen: 127.0.0.1:-1 name: "abc" no_advertise: false } nats-server-2.10.27/server/configs/reload/tls_multi_cert_1.conf000066400000000000000000000003231477524627100244750ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:-1 tls { certs = [ { cert_file: "../test/configs/certs/srva-cert.pem" key_file: "../test/configs/certs/srva-key.pem" } ] timeout: 2 } nats-server-2.10.27/server/configs/reload/tls_multi_cert_2.conf000066400000000000000000000006041477524627100245000ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:-1 tls { certs = [ { cert_file: "../test/configs/certs/srva-cert.pem" key_file: "../test/configs/certs/srva-key.pem" }, { cert_file: "../test/configs/certs/srvb-cert.pem" key_file: "../test/configs/certs/srvb-key.pem" } ] ca_file: "../test/configs/certs/ca.pem" verify: true timeout: 2 } nats-server-2.10.27/server/configs/reload/tls_multi_cert_3.conf000066400000000000000000000003201477524627100244740ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:-1 tls { certs = [ { cert_file: "../test/configs/certs/srvb-cert.pem" key_file: "../test/configs/certs/srvb-key.pem" } ] timeout: 2 } nats-server-2.10.27/server/configs/reload/tls_test.conf000066400000000000000000000002351477524627100230670ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 } nats-server-2.10.27/server/configs/reload/tls_verify_test.conf000066400000000000000000000003431477524627100244530ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/cert.new.pem" key_file: "./configs/certs/key.new.pem" ca_file: "./configs/certs/cert.new.pem" verify: true timeout: 2 } nats-server-2.10.27/server/configs/reload/token_authentication_1.conf000066400000000000000000000000771477524627100256710ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { token: T0pS3cr3t } nats-server-2.10.27/server/configs/reload/token_authentication_2.conf000066400000000000000000000000761477524627100256710ustar00rootroot00000000000000listen: 127.0.0.1:-1 authorization { token: passw0rd } nats-server-2.10.27/server/configs/seed.conf000066400000000000000000000001661477524627100207030ustar00rootroot00000000000000# Cluster Seed Node listen: 127.0.0.1:7222 http: 127.0.0.1:9222 cluster { listen: 127.0.0.1:7248 name: "abc" } nats-server-2.10.27/server/configs/seed_tls.conf000066400000000000000000000007611477524627100215660ustar00rootroot00000000000000# Cluster Seed Node listen: 127.0.0.1:7222 http: 127.0.0.1:9222 cluster { listen: 127.0.0.1:7248 name: "abc" tls { # Route cert cert_file: "../test/configs/certs/server-cert.pem" # Private key key_file: "../test/configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 # Optional certificate authority verifying connected routes # Required when we have self-signed CA, etc. ca_file: "../test/configs/certs/ca.pem" } } nats-server-2.10.27/server/configs/srv_a.conf000066400000000000000000000006601477524627100210740ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:7222 cluster { listen: 127.0.0.1:7244 name: "abc" authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:7246 ] } nats-server-2.10.27/server/configs/srv_a_bcrypt.conf000066400000000000000000000004731477524627100224610ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:7222 authorization { user: user password: foo timeout: 2 } cluster { listen: 127.0.0.1:7244 name: "abc" authorization { user: ruser # bcrypt version of 'bar' password: $2a$10$LoRPzN3GtF2pNX5QgCBBHeUr6/zVN./RVGOu5U8SpHyg2sfzvfXji timeout: 5 } } nats-server-2.10.27/server/configs/srv_b.conf000066400000000000000000000006601477524627100210750ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:7224 cluster { listen: 127.0.0.1:7246 name: "abc" authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:7244 ] } nats-server-2.10.27/server/configs/srv_b_bcrypt.conf000066400000000000000000000010641477524627100224570ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:7224 authorization { user: user password: foo timeout: 2 } cluster { listen: 127.0.0.1:7246 name: "abc" authorization { user: ruser # bcrypt version of 'bar' password: $2a$10$LoRPzN3GtF2pNX5QgCBBHeUr6/zVN./RVGOu5U8SpHyg2sfzvfXji timeout: 5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:bar@127.0.0.1:7244 ] } nats-server-2.10.27/server/configs/test.conf000066400000000000000000000017141477524627100207420ustar00rootroot00000000000000# Simple config file server_name: testing_server listen: 127.0.0.1:4242 http: 8222 http_base_path: /nats authorization { user: derek password: porkchop timeout: 1 } # logging options debug: false trace: true logtime: false syslog: true remote_syslog: "udp://foo.com:33" # pid file pid_file: "/tmp/nats-server/nats-server.pid" # prof_port prof_port: 6543 # max_connections max_connections: 100 # max_subscriptions (per connection) max_subscriptions: 1000 # max_pending max_pending: 10000000 # maximum control line max_control_line: 2048 # maximum payload max_payload: 65536 # ping interval and no pong threshold ping_interval: "60s" ping_max: 3 # how long server can block on a socket write to a client write_deadline: "3s" lame_duck_duration: "4m" # report repeated failed route/gateway/leafNode connection # every 24hour (24*60*60) connect_error_reports: 86400 # report failed reconnect events every 5 attempts reconnect_error_reports: 5 nats-server-2.10.27/server/configs/tls.conf000066400000000000000000000003431477524627100205620ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: "2s" } authorization { user: derek password: foo timeout: 1 } nats-server-2.10.27/server/configs/tls/000077500000000000000000000000001477524627100177135ustar00rootroot00000000000000nats-server-2.10.27/server/configs/tls/tls-ed25519.conf000066400000000000000000000004321477524627100223570ustar00rootroot00000000000000# Simple TLS (ed25519) config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/tls/benchmark-server-cert-ed25519.pem" key_file: "./configs/certs/tls/benchmark-server-key-ed25519.pem" ca_file: "./configs/certs/tls/benchmark-ca-cert.pem" timeout: "5s" } nats-server-2.10.27/server/configs/tls/tls-none.conf000066400000000000000000000000541477524627100223200ustar00rootroot00000000000000# Simple config file listen: 127.0.0.1:-1 nats-server-2.10.27/server/configs/tls/tls-rsa-1024.conf000066400000000000000000000004351477524627100225350ustar00rootroot00000000000000# Simple TLS (rsa-1024) config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/tls/benchmark-server-cert-rsa-1024.pem" key_file: "./configs/certs/tls/benchmark-server-key-rsa-1024.pem" ca_file: "./configs/certs/tls/benchmark-ca-cert.pem" timeout: "5s" } nats-server-2.10.27/server/configs/tls/tls-rsa-2048.conf000066400000000000000000000004351477524627100225440ustar00rootroot00000000000000# Simple TLS (rsa-2048) config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/tls/benchmark-server-cert-rsa-2048.pem" key_file: "./configs/certs/tls/benchmark-server-key-rsa-2048.pem" ca_file: "./configs/certs/tls/benchmark-ca-cert.pem" timeout: "5s" } nats-server-2.10.27/server/configs/tls/tls-rsa-4096.conf000066400000000000000000000004351477524627100225510ustar00rootroot00000000000000# Simple TLS (rsa-4096) config file listen: 127.0.0.1:-1 tls { cert_file: "./configs/certs/tls/benchmark-server-cert-rsa-4096.pem" key_file: "./configs/certs/tls/benchmark-server-key-rsa-4096.pem" ca_file: "./configs/certs/tls/benchmark-ca-cert.pem" timeout: "5s" } nats-server-2.10.27/server/configs/tls_bad_cipher.conf000066400000000000000000000005421477524627100227230ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 # this should generate an error cipher_suites: [ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", "BAD_CIPHER_SPEC", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", ] } nats-server-2.10.27/server/configs/tls_bad_curve_prefs.conf000066400000000000000000000003011477524627100237650ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 curve_preferences: [ "GARBAGE" ] } nats-server-2.10.27/server/configs/tls_ciphers.conf000066400000000000000000000013351477524627100223010ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 cipher_suites: [ "TLS_RSA_WITH_RC4_128_SHA", "TLS_RSA_WITH_3DES_EDE_CBC_SHA", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_RC4_128_SHA", "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" ] } authorization { user: derek password: monkey timeout: 1 } nats-server-2.10.27/server/configs/tls_curve_prefs.conf000066400000000000000000000003031477524627100231610ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 curve_preferences: [ "CurveP256" ] } nats-server-2.10.27/server/configs/tls_empty_cipher.conf000066400000000000000000000003241477524627100233310ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 # this should generate an error cipher_suites: [ ] } nats-server-2.10.27/server/configs/tls_empty_curve_prefs.conf000066400000000000000000000002651477524627100244060ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:4443 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 2 curve_preferences: [ ] } nats-server-2.10.27/server/const.go000066400000000000000000000201331477524627100171350ustar00rootroot00000000000000// Copyright 2012-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "runtime/debug" "time" ) // Command is a signal used to control a running nats-server process. type Command string // Valid Command values. const ( CommandStop = Command("stop") CommandQuit = Command("quit") CommandReopen = Command("reopen") CommandReload = Command("reload") // private for now commandLDMode = Command("ldm") commandTerm = Command("term") ) var ( // gitCommit and serverVersion injected at build. gitCommit, serverVersion string // trustedKeys is a whitespace separated array of trusted operator's public nkeys. trustedKeys string ) func init() { // Use build info if present, it would be if building using 'go build .' // or when using a release. if info, ok := debug.ReadBuildInfo(); ok { for _, setting := range info.Settings { switch setting.Key { case "vcs.revision": gitCommit = setting.Value[:7] } } } } const ( // VERSION is the current version for the server. VERSION = "2.10.27" // PROTO is the currently supported protocol. // 0 was the original // 1 maintains proto 0, adds echo abilities for CONNECT from the client. Clients // should not send echo unless proto in INFO is >= 1. PROTO = 1 // DEFAULT_PORT is the default port for client connections. DEFAULT_PORT = 4222 // RANDOM_PORT is the value for port that, when supplied, will cause the // server to listen on a randomly-chosen available port. The resolved port // is available via the Addr() method. RANDOM_PORT = -1 // DEFAULT_HOST defaults to all interfaces. DEFAULT_HOST = "0.0.0.0" // MAX_CONTROL_LINE_SIZE is the maximum allowed protocol control line size. // 4k should be plenty since payloads sans connect/info string are separate. MAX_CONTROL_LINE_SIZE = 4096 // MAX_PAYLOAD_SIZE is the maximum allowed payload size. Should be using // something different if > 1MB payloads are needed. MAX_PAYLOAD_SIZE = (1024 * 1024) // MAX_PAYLOAD_MAX_SIZE is the size at which the server will warn about // max_payload being too high. In the future, the server may enforce/reject // max_payload above this value. MAX_PAYLOAD_MAX_SIZE = (8 * 1024 * 1024) // MAX_PENDING_SIZE is the maximum outbound pending bytes per client. MAX_PENDING_SIZE = (64 * 1024 * 1024) // DEFAULT_MAX_CONNECTIONS is the default maximum connections allowed. DEFAULT_MAX_CONNECTIONS = (64 * 1024) // TLS_TIMEOUT is the TLS wait time. TLS_TIMEOUT = 2 * time.Second // DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY is the default amount of // time for the server to wait for the TLS handshake with a client to // be initiated before falling back to sending the INFO protocol first. // See TLSHandshakeFirst and TLSHandshakeFirstFallback options. DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY = 50 * time.Millisecond // AUTH_TIMEOUT is the authorization wait time. AUTH_TIMEOUT = 2 * time.Second // DEFAULT_PING_INTERVAL is how often pings are sent to clients, etc... DEFAULT_PING_INTERVAL = 2 * time.Minute // DEFAULT_PING_MAX_OUT is maximum allowed pings outstanding before disconnect. DEFAULT_PING_MAX_OUT = 2 // CR_LF string CR_LF = "\r\n" // LEN_CR_LF hold onto the computed size. LEN_CR_LF = len(CR_LF) // DEFAULT_FLUSH_DEADLINE is the write/flush deadlines. DEFAULT_FLUSH_DEADLINE = 10 * time.Second // DEFAULT_HTTP_PORT is the default monitoring port. DEFAULT_HTTP_PORT = 8222 // DEFAULT_HTTP_BASE_PATH is the default base path for monitoring. DEFAULT_HTTP_BASE_PATH = "/" // ACCEPT_MIN_SLEEP is the minimum acceptable sleep times on temporary errors. ACCEPT_MIN_SLEEP = 10 * time.Millisecond // ACCEPT_MAX_SLEEP is the maximum acceptable sleep times on temporary errors ACCEPT_MAX_SLEEP = 1 * time.Second // DEFAULT_ROUTE_CONNECT Route solicitation intervals. DEFAULT_ROUTE_CONNECT = 1 * time.Second // DEFAULT_ROUTE_RECONNECT Route reconnect intervals. DEFAULT_ROUTE_RECONNECT = 1 * time.Second // DEFAULT_ROUTE_DIAL Route dial timeout. DEFAULT_ROUTE_DIAL = 1 * time.Second // DEFAULT_ROUTE_POOL_SIZE Route default pool size DEFAULT_ROUTE_POOL_SIZE = 3 // DEFAULT_LEAF_NODE_RECONNECT LeafNode reconnect interval. DEFAULT_LEAF_NODE_RECONNECT = time.Second // DEFAULT_LEAF_TLS_TIMEOUT TLS timeout for LeafNodes DEFAULT_LEAF_TLS_TIMEOUT = 2 * time.Second // PROTO_SNIPPET_SIZE is the default size of proto to print on parse errors. PROTO_SNIPPET_SIZE = 32 // MAX_CONTROL_LINE_SNIPPET_SIZE is the default size of proto to print on max control line errors. MAX_CONTROL_LINE_SNIPPET_SIZE = 128 // MAX_MSG_ARGS Maximum possible number of arguments from MSG proto. MAX_MSG_ARGS = 4 // MAX_RMSG_ARGS Maximum possible number of arguments from RMSG proto. MAX_RMSG_ARGS = 6 // MAX_HMSG_ARGS Maximum possible number of arguments from HMSG proto. MAX_HMSG_ARGS = 7 // MAX_PUB_ARGS Maximum possible number of arguments from PUB proto. MAX_PUB_ARGS = 3 // MAX_HPUB_ARGS Maximum possible number of arguments from HPUB proto. MAX_HPUB_ARGS = 4 // MAX_RSUB_ARGS Maximum possible number of arguments from a RS+/LS+ proto. MAX_RSUB_ARGS = 6 // DEFAULT_MAX_CLOSED_CLIENTS is the maximum number of closed connections we hold onto. DEFAULT_MAX_CLOSED_CLIENTS = 10000 // DEFAULT_LAME_DUCK_DURATION is the time in which the server spreads // the closing of clients when signaled to go in lame duck mode. DEFAULT_LAME_DUCK_DURATION = 2 * time.Minute // DEFAULT_LAME_DUCK_GRACE_PERIOD is the duration the server waits, after entering // lame duck mode, before starting closing client connections. DEFAULT_LAME_DUCK_GRACE_PERIOD = 10 * time.Second // DEFAULT_LEAFNODE_INFO_WAIT Route dial timeout. DEFAULT_LEAFNODE_INFO_WAIT = 1 * time.Second // DEFAULT_LEAFNODE_PORT is the default port for remote leafnode connections. DEFAULT_LEAFNODE_PORT = 7422 // DEFAULT_CONNECT_ERROR_REPORTS is the number of attempts at which a // repeated failed route, gateway or leaf node connection is reported. // This is used for initial connection, that is, when the server has // never had a connection to the given endpoint. Once connected, and // if a disconnect occurs, DEFAULT_RECONNECT_ERROR_REPORTS is used // instead. // The default is to report every 3600 attempts (roughly every hour). DEFAULT_CONNECT_ERROR_REPORTS = 3600 // DEFAULT_RECONNECT_ERROR_REPORTS is the default number of failed // attempt to reconnect a route, gateway or leaf node connection. // The default is to report every attempt. DEFAULT_RECONNECT_ERROR_REPORTS = 1 // DEFAULT_RTT_MEASUREMENT_INTERVAL is how often we want to measure RTT from // this server to clients, routes, gateways or leafnode connections. DEFAULT_RTT_MEASUREMENT_INTERVAL = time.Hour // DEFAULT_ALLOW_RESPONSE_MAX_MSGS is the default number of responses allowed // for a reply subject. DEFAULT_ALLOW_RESPONSE_MAX_MSGS = 1 // DEFAULT_ALLOW_RESPONSE_EXPIRATION is the default time allowed for a given // dynamic response permission. DEFAULT_ALLOW_RESPONSE_EXPIRATION = 2 * time.Minute // DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD is the default time that the system will // expect a service export response to be delivered. This is used in corner cases for // time based cleanup of reverse mapping structures. DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD = 2 * time.Minute // DEFAULT_SERVICE_LATENCY_SAMPLING is the default sampling rate for service // latency metrics DEFAULT_SERVICE_LATENCY_SAMPLING = 100 // DEFAULT_SYSTEM_ACCOUNT DEFAULT_SYSTEM_ACCOUNT = "$SYS" // DEFAULT GLOBAL_ACCOUNT DEFAULT_GLOBAL_ACCOUNT = "$G" // DEFAULT_FETCH_TIMEOUT is the default time that the system will wait for an account fetch to return. DEFAULT_ACCOUNT_FETCH_TIMEOUT = 1900 * time.Millisecond ) nats-server-2.10.27/server/consumer.go000066400000000000000000004632101477524627100176510ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "encoding/binary" "encoding/json" "errors" "fmt" "math/rand" "reflect" "slices" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/nats-io/nats-server/v2/server/avl" "github.com/nats-io/nuid" "golang.org/x/time/rate" ) // Headers sent with Request Timeout const ( JSPullRequestPendingMsgs = "Nats-Pending-Messages" JSPullRequestPendingBytes = "Nats-Pending-Bytes" ) // Headers sent when batch size was completed, but there were remaining bytes. const JsPullRequestRemainingBytesT = "NATS/1.0 409 Batch Completed\r\n%s: %d\r\n%s: %d\r\n\r\n" type ConsumerInfo struct { Stream string `json:"stream_name"` Name string `json:"name"` Created time.Time `json:"created"` Config *ConsumerConfig `json:"config,omitempty"` Delivered SequenceInfo `json:"delivered"` AckFloor SequenceInfo `json:"ack_floor"` NumAckPending int `json:"num_ack_pending"` NumRedelivered int `json:"num_redelivered"` NumWaiting int `json:"num_waiting"` NumPending uint64 `json:"num_pending"` Cluster *ClusterInfo `json:"cluster,omitempty"` PushBound bool `json:"push_bound,omitempty"` // TimeStamp indicates when the info was gathered TimeStamp time.Time `json:"ts"` } type ConsumerConfig struct { Durable string `json:"durable_name,omitempty"` Name string `json:"name,omitempty"` Description string `json:"description,omitempty"` DeliverPolicy DeliverPolicy `json:"deliver_policy"` OptStartSeq uint64 `json:"opt_start_seq,omitempty"` OptStartTime *time.Time `json:"opt_start_time,omitempty"` AckPolicy AckPolicy `json:"ack_policy"` AckWait time.Duration `json:"ack_wait,omitempty"` MaxDeliver int `json:"max_deliver,omitempty"` BackOff []time.Duration `json:"backoff,omitempty"` FilterSubject string `json:"filter_subject,omitempty"` FilterSubjects []string `json:"filter_subjects,omitempty"` ReplayPolicy ReplayPolicy `json:"replay_policy"` RateLimit uint64 `json:"rate_limit_bps,omitempty"` // Bits per sec SampleFrequency string `json:"sample_freq,omitempty"` MaxWaiting int `json:"max_waiting,omitempty"` MaxAckPending int `json:"max_ack_pending,omitempty"` Heartbeat time.Duration `json:"idle_heartbeat,omitempty"` FlowControl bool `json:"flow_control,omitempty"` HeadersOnly bool `json:"headers_only,omitempty"` // Pull based options. MaxRequestBatch int `json:"max_batch,omitempty"` MaxRequestExpires time.Duration `json:"max_expires,omitempty"` MaxRequestMaxBytes int `json:"max_bytes,omitempty"` // Push based consumers. DeliverSubject string `json:"deliver_subject,omitempty"` DeliverGroup string `json:"deliver_group,omitempty"` // Ephemeral inactivity threshold. InactiveThreshold time.Duration `json:"inactive_threshold,omitempty"` // Generally inherited by parent stream and other markers, now can be configured directly. Replicas int `json:"num_replicas"` // Force memory storage. MemoryStorage bool `json:"mem_storage,omitempty"` // Don't add to general clients. Direct bool `json:"direct,omitempty"` // Metadata is additional metadata for the Consumer. Metadata map[string]string `json:"metadata,omitempty"` } // SequenceInfo has both the consumer and the stream sequence and last activity. type SequenceInfo struct { Consumer uint64 `json:"consumer_seq"` Stream uint64 `json:"stream_seq"` Last *time.Time `json:"last_active,omitempty"` } type CreateConsumerRequest struct { Stream string `json:"stream_name"` Config ConsumerConfig `json:"config"` Action ConsumerAction `json:"action"` } type ConsumerAction int const ( ActionCreateOrUpdate ConsumerAction = iota ActionUpdate ActionCreate ) const ( actionUpdateJSONString = `"update"` actionCreateJSONString = `"create"` actionCreateOrUpdateJSONString = `""` ) var ( actionUpdateJSONBytes = []byte(actionUpdateJSONString) actionCreateJSONBytes = []byte(actionCreateJSONString) actionCreateOrUpdateJSONBytes = []byte(actionCreateOrUpdateJSONString) ) func (a ConsumerAction) String() string { switch a { case ActionCreateOrUpdate: return actionCreateOrUpdateJSONString case ActionCreate: return actionCreateJSONString case ActionUpdate: return actionUpdateJSONString } return actionCreateOrUpdateJSONString } func (a ConsumerAction) MarshalJSON() ([]byte, error) { switch a { case ActionCreate: return actionCreateJSONBytes, nil case ActionUpdate: return actionUpdateJSONBytes, nil case ActionCreateOrUpdate: return actionCreateOrUpdateJSONBytes, nil default: return nil, fmt.Errorf("can not marshal %v", a) } } func (a *ConsumerAction) UnmarshalJSON(data []byte) error { switch string(data) { case actionCreateJSONString: *a = ActionCreate case actionUpdateJSONString: *a = ActionUpdate case actionCreateOrUpdateJSONString: *a = ActionCreateOrUpdate default: return fmt.Errorf("unknown consumer action: %v", string(data)) } return nil } // ConsumerNakOptions is for optional NAK values, e.g. delay. type ConsumerNakOptions struct { Delay time.Duration `json:"delay"` } // DeliverPolicy determines how the consumer should select the first message to deliver. type DeliverPolicy int const ( // DeliverAll will be the default so can be omitted from the request. DeliverAll DeliverPolicy = iota // DeliverLast will start the consumer with the last sequence received. DeliverLast // DeliverNew will only deliver new messages that are sent after the consumer is created. DeliverNew // DeliverByStartSequence will look for a defined starting sequence to start. DeliverByStartSequence // DeliverByStartTime will select the first messsage with a timestamp >= to StartTime. DeliverByStartTime // DeliverLastPerSubject will start the consumer with the last message for all subjects received. DeliverLastPerSubject ) func (dp DeliverPolicy) String() string { switch dp { case DeliverAll: return "all" case DeliverLast: return "last" case DeliverNew: return "new" case DeliverByStartSequence: return "by_start_sequence" case DeliverByStartTime: return "by_start_time" case DeliverLastPerSubject: return "last_per_subject" default: return "undefined" } } // AckPolicy determines how the consumer should acknowledge delivered messages. type AckPolicy int const ( // AckNone requires no acks for delivered messages. AckNone AckPolicy = iota // AckAll when acking a sequence number, this implicitly acks all sequences below this one as well. AckAll // AckExplicit requires ack or nack for all messages. AckExplicit ) func (a AckPolicy) String() string { switch a { case AckNone: return "none" case AckAll: return "all" default: return "explicit" } } // ReplayPolicy determines how the consumer should replay messages it already has queued in the stream. type ReplayPolicy int const ( // ReplayInstant will replay messages as fast as possible. ReplayInstant ReplayPolicy = iota // ReplayOriginal will maintain the same timing as the messages were received. ReplayOriginal ) func (r ReplayPolicy) String() string { switch r { case ReplayInstant: return replayInstantPolicyJSONString default: return replayOriginalPolicyJSONString } } // OK const OK = "+OK" // Ack responses. Note that a nil or no payload is same as AckAck var ( // Ack AckAck = []byte("+ACK") // nil or no payload to ack subject also means ACK AckOK = []byte(OK) // deprecated but +OK meant ack as well. // Nack AckNak = []byte("-NAK") // Progress indicator AckProgress = []byte("+WPI") // Ack + Deliver the next message(s). AckNext = []byte("+NXT") // Terminate delivery of the message. AckTerm = []byte("+TERM") ) const ( // reasons to supply when terminating messages using limits ackTermLimitsReason = "Message deleted by stream limits" ackTermUnackedLimitsReason = "Unacknowledged message was deleted" ) // Calculate accurate replicas for the consumer config with the parent stream config. func (consCfg ConsumerConfig) replicas(strCfg *StreamConfig) int { if consCfg.Replicas == 0 || consCfg.Replicas > strCfg.Replicas { if !isDurableConsumer(&consCfg) && strCfg.Retention == LimitsPolicy && consCfg.Replicas == 0 { // Matches old-school ephemerals only, where the replica count is 0. return 1 } return strCfg.Replicas } return consCfg.Replicas } // Consumer is a jetstream consumer. type consumer struct { // Atomic used to notify that we want to process an ack. // This will be checked in checkPending to abort processing // and let ack be processed in priority. awl int64 leader atomic.Bool mu sync.RWMutex js *jetStream mset *stream acc *Account srv *Server client *client sysc *client sid int name string stream string sseq uint64 // next stream sequence subjf subjectFilters // subject filters and their sequences filters *Sublist // When we have multiple filters we will use LoadNextMsgMulti and pass this in. dseq uint64 // delivered consumer sequence adflr uint64 // ack delivery floor asflr uint64 // ack store floor chkflr uint64 // our check floor, interest streams only. npc int64 // Num Pending Count npf uint64 // Num Pending Floor Sequence dsubj string qgroup string lss *lastSeqSkipList rlimit *rate.Limiter reqSub *subscription ackSub *subscription ackReplyT string ackSubj string nextMsgSubj string nextMsgReqs *ipQueue[*nextMsgReq] maxp int pblimit int maxpb int pbytes int fcsz int fcid string fcSub *subscription outq *jsOutQ pending map[uint64]*Pending ptmr *time.Timer ptmrEnd time.Time rdq []uint64 rdqi avl.SequenceSet rdc map[uint64]uint64 replies map[uint64]string maxdc uint64 waiting *waitQueue cfg ConsumerConfig ici *ConsumerInfo store ConsumerStore active bool replay bool dtmr *time.Timer gwdtmr *time.Timer dthresh time.Duration mch chan struct{} qch chan struct{} inch chan bool sfreq int32 ackEventT string nakEventT string deliveryExcEventT string created time.Time ldt time.Time lat time.Time lwqic time.Time closed bool // Clustered. ca *consumerAssignment node RaftNode infoSub *subscription lqsent time.Time prm map[string]struct{} prOk bool uch chan struct{} retention RetentionPolicy monitorWg sync.WaitGroup inMonitor bool // R>1 proposals pch chan struct{} phead *proposal ptail *proposal // Ack queue ackMsgs *ipQueue[*jsAckMsg] // for stream signaling when multiple filters are set. sigSubs []string } // A single subject filter. type subjectFilter struct { subject string tokenizedSubject []string hasWildcard bool } type subjectFilters []*subjectFilter // subjects is a helper function used for updating consumers. // It is not used and should not be used in hotpath. func (s subjectFilters) subjects() []string { subjects := make([]string, 0, len(s)) for _, filter := range s { subjects = append(subjects, filter.subject) } return subjects } type proposal struct { data []byte next *proposal } const ( // JsAckWaitDefault is the default AckWait, only applicable on explicit ack policy consumers. JsAckWaitDefault = 30 * time.Second // JsDeleteWaitTimeDefault is the default amount of time we will wait for non-durable // consumers to be in an inactive state before deleting them. JsDeleteWaitTimeDefault = 5 * time.Second // JsFlowControlMaxPending specifies default pending bytes during flow control that can be outstanding. JsFlowControlMaxPending = 32 * 1024 * 1024 // JsDefaultMaxAckPending is set for consumers with explicit ack that do not set the max ack pending. JsDefaultMaxAckPending = 1000 ) // Helper function to set consumer config defaults from above. func setConsumerConfigDefaults(config *ConsumerConfig, streamCfg *StreamConfig, lim *JSLimitOpts, accLim *JetStreamAccountLimits) { // Set to default if not specified. if config.DeliverSubject == _EMPTY_ && config.MaxWaiting == 0 { config.MaxWaiting = JSWaitQueueDefaultMax } // Setup proper default for ack wait if we are in explicit ack mode. if config.AckWait == 0 && (config.AckPolicy == AckExplicit || config.AckPolicy == AckAll) { config.AckWait = JsAckWaitDefault } // Setup default of -1, meaning no limit for MaxDeliver. if config.MaxDeliver == 0 { config.MaxDeliver = -1 } // If BackOff was specified that will override the AckWait and the MaxDeliver. if len(config.BackOff) > 0 { config.AckWait = config.BackOff[0] } if config.MaxAckPending == 0 { config.MaxAckPending = streamCfg.ConsumerLimits.MaxAckPending } if config.InactiveThreshold == 0 { config.InactiveThreshold = streamCfg.ConsumerLimits.InactiveThreshold } // Set proper default for max ack pending if we are ack explicit and none has been set. if (config.AckPolicy == AckExplicit || config.AckPolicy == AckAll) && config.MaxAckPending == 0 { accPending := JsDefaultMaxAckPending if lim.MaxAckPending > 0 && lim.MaxAckPending < accPending { accPending = lim.MaxAckPending } if accLim.MaxAckPending > 0 && accLim.MaxAckPending < accPending { accPending = accLim.MaxAckPending } config.MaxAckPending = accPending } // if applicable set max request batch size if config.DeliverSubject == _EMPTY_ && config.MaxRequestBatch == 0 && lim.MaxRequestBatch > 0 { config.MaxRequestBatch = lim.MaxRequestBatch } } // Check the consumer config. If we are recovering don't check filter subjects. func checkConsumerCfg( config *ConsumerConfig, srvLim *JSLimitOpts, cfg *StreamConfig, _ *Account, accLim *JetStreamAccountLimits, isRecovering bool, ) *ApiError { // Check if replicas is defined but exceeds parent stream. if config.Replicas > 0 && config.Replicas > cfg.Replicas { return NewJSConsumerReplicasExceedsStreamError() } // Check that it is not negative if config.Replicas < 0 { return NewJSReplicasCountCannotBeNegativeError() } // If the stream is interest or workqueue retention make sure the replicas // match that of the stream. This is REQUIRED for now. if cfg.Retention == InterestPolicy || cfg.Retention == WorkQueuePolicy { // Only error here if not recovering. // We handle recovering in a different spot to allow consumer to come up // if previous version allowed it to be created. We do not want it to not come up. if !isRecovering && config.Replicas != 0 && config.Replicas != cfg.Replicas { return NewJSConsumerReplicasShouldMatchStreamError() } } // Check if we have a BackOff defined that MaxDeliver is within range etc. if lbo := len(config.BackOff); lbo > 0 && config.MaxDeliver != -1 && lbo > config.MaxDeliver { return NewJSConsumerMaxDeliverBackoffError() } if len(config.Description) > JSMaxDescriptionLen { return NewJSConsumerDescriptionTooLongError(JSMaxDescriptionLen) } // For now expect a literal subject if its not empty. Empty means work queue mode (pull mode). if config.DeliverSubject != _EMPTY_ { if !subjectIsLiteral(config.DeliverSubject) { return NewJSConsumerDeliverToWildcardsError() } if !IsValidSubject(config.DeliverSubject) { return NewJSConsumerInvalidDeliverSubjectError() } if deliveryFormsCycle(cfg, config.DeliverSubject) { return NewJSConsumerDeliverCycleError() } if config.MaxWaiting != 0 { return NewJSConsumerPushMaxWaitingError() } if config.MaxAckPending > 0 && config.AckPolicy == AckNone { return NewJSConsumerMaxPendingAckPolicyRequiredError() } if config.Heartbeat > 0 && config.Heartbeat < 100*time.Millisecond { return NewJSConsumerSmallHeartbeatError() } } else { // Pull mode with work queue retention from the stream requires an explicit ack. if config.AckPolicy == AckNone && cfg.Retention == WorkQueuePolicy { return NewJSConsumerPullRequiresAckError() } if config.RateLimit > 0 { return NewJSConsumerPullWithRateLimitError() } if config.MaxWaiting < 0 { return NewJSConsumerMaxWaitingNegativeError() } if config.Heartbeat > 0 { return NewJSConsumerHBRequiresPushError() } if config.FlowControl { return NewJSConsumerFCRequiresPushError() } if config.MaxRequestBatch < 0 { return NewJSConsumerMaxRequestBatchNegativeError() } if config.MaxRequestExpires != 0 && config.MaxRequestExpires < time.Millisecond { return NewJSConsumerMaxRequestExpiresToSmallError() } if srvLim.MaxRequestBatch > 0 && config.MaxRequestBatch > srvLim.MaxRequestBatch { return NewJSConsumerMaxRequestBatchExceededError(srvLim.MaxRequestBatch) } } if srvLim.MaxAckPending > 0 && config.MaxAckPending > srvLim.MaxAckPending { return NewJSConsumerMaxPendingAckExcessError(srvLim.MaxAckPending) } if accLim.MaxAckPending > 0 && config.MaxAckPending > accLim.MaxAckPending { return NewJSConsumerMaxPendingAckExcessError(accLim.MaxAckPending) } if cfg.ConsumerLimits.MaxAckPending > 0 && config.MaxAckPending > cfg.ConsumerLimits.MaxAckPending { return NewJSConsumerMaxPendingAckExcessError(cfg.ConsumerLimits.MaxAckPending) } if cfg.ConsumerLimits.InactiveThreshold > 0 && config.InactiveThreshold > cfg.ConsumerLimits.InactiveThreshold { return NewJSConsumerInactiveThresholdExcessError(cfg.ConsumerLimits.InactiveThreshold) } // Direct need to be non-mapped ephemerals. if config.Direct { if config.DeliverSubject == _EMPTY_ { return NewJSConsumerDirectRequiresPushError() } if isDurableConsumer(config) { return NewJSConsumerDirectRequiresEphemeralError() } } // Do not allow specifying both FilterSubject and FilterSubjects, // as that's probably unintentional without any difference from passing // all filters in FilterSubjects. if config.FilterSubject != _EMPTY_ && len(config.FilterSubjects) > 0 { return NewJSConsumerDuplicateFilterSubjectsError() } if config.FilterSubject != _EMPTY_ && !IsValidSubject(config.FilterSubject) { return NewJSStreamInvalidConfigError(ErrBadSubject) } // We treat FilterSubjects: []string{""} as a misconfig, so we validate against it. for _, filter := range config.FilterSubjects { if filter == _EMPTY_ { return NewJSConsumerEmptyFilterError() } } subjectFilters := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects) // Check subject filters do not overlap. for outer, subject := range subjectFilters { if !IsValidSubject(subject) { return NewJSStreamInvalidConfigError(ErrBadSubject) } for inner, ssubject := range subjectFilters { if inner != outer && SubjectsCollide(subject, ssubject) { return NewJSConsumerOverlappingSubjectFiltersError() } } } // Helper function to formulate similar errors. badStart := func(dp, start string) error { return fmt.Errorf("consumer delivery policy is deliver %s, but optional start %s is also set", dp, start) } notSet := func(dp, notSet string) error { return fmt.Errorf("consumer delivery policy is deliver %s, but optional %s is not set", dp, notSet) } // Check on start position conflicts. switch config.DeliverPolicy { case DeliverAll: if config.OptStartSeq > 0 { return NewJSConsumerInvalidPolicyError(badStart("all", "sequence")) } if config.OptStartTime != nil { return NewJSConsumerInvalidPolicyError(badStart("all", "time")) } case DeliverLast: if config.OptStartSeq > 0 { return NewJSConsumerInvalidPolicyError(badStart("last", "sequence")) } if config.OptStartTime != nil { return NewJSConsumerInvalidPolicyError(badStart("last", "time")) } case DeliverLastPerSubject: if config.OptStartSeq > 0 { return NewJSConsumerInvalidPolicyError(badStart("last per subject", "sequence")) } if config.OptStartTime != nil { return NewJSConsumerInvalidPolicyError(badStart("last per subject", "time")) } if config.FilterSubject == _EMPTY_ && len(config.FilterSubjects) == 0 { return NewJSConsumerInvalidPolicyError(notSet("last per subject", "filter subject")) } case DeliverNew: if config.OptStartSeq > 0 { return NewJSConsumerInvalidPolicyError(badStart("new", "sequence")) } if config.OptStartTime != nil { return NewJSConsumerInvalidPolicyError(badStart("new", "time")) } case DeliverByStartSequence: if config.OptStartSeq == 0 { return NewJSConsumerInvalidPolicyError(notSet("by start sequence", "start sequence")) } if config.OptStartTime != nil { return NewJSConsumerInvalidPolicyError(badStart("by start sequence", "time")) } case DeliverByStartTime: if config.OptStartTime == nil { return NewJSConsumerInvalidPolicyError(notSet("by start time", "start time")) } if config.OptStartSeq != 0 { return NewJSConsumerInvalidPolicyError(badStart("by start time", "start sequence")) } } if config.SampleFrequency != _EMPTY_ { s := strings.TrimSuffix(config.SampleFrequency, "%") if sampleFreq, err := strconv.Atoi(s); err != nil || sampleFreq < 0 { return NewJSConsumerInvalidSamplingError(err) } } // We reject if flow control is set without heartbeats. if config.FlowControl && config.Heartbeat == 0 { return NewJSConsumerWithFlowControlNeedsHeartbeatsError() } if config.Durable != _EMPTY_ && config.Name != _EMPTY_ { if config.Name != config.Durable { return NewJSConsumerCreateDurableAndNameMismatchError() } } var metadataLen int for k, v := range config.Metadata { metadataLen += len(k) + len(v) } if metadataLen > JSMaxMetadataLen { return NewJSConsumerMetadataLengthError(fmt.Sprintf("%dKB", JSMaxMetadataLen/1024)) } return nil } func (mset *stream) addConsumerWithAction(config *ConsumerConfig, action ConsumerAction) (*consumer, error) { return mset.addConsumerWithAssignment(config, _EMPTY_, nil, false, action) } func (mset *stream) addConsumer(config *ConsumerConfig) (*consumer, error) { return mset.addConsumerWithAction(config, ActionCreateOrUpdate) } func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname string, ca *consumerAssignment, isRecovering bool, action ConsumerAction) (*consumer, error) { // Check if this stream has closed. if mset.closed.Load() { return nil, NewJSStreamInvalidError() } mset.mu.RLock() s, jsa, cfg, acc := mset.srv, mset.jsa, mset.cfg, mset.acc mset.mu.RUnlock() // If we do not have the consumer currently assigned to us in cluster mode we will proceed but warn. // This can happen on startup with restored state where on meta replay we still do not have // the assignment. Running in single server mode this always returns true. if oname != _EMPTY_ && !jsa.consumerAssigned(mset.name(), oname) { s.Debugf("Consumer %q > %q does not seem to be assigned to this server", mset.name(), oname) } if config == nil { return nil, NewJSConsumerConfigRequiredError() } selectedLimits, _, _, _ := acc.selectLimits(config.replicas(&cfg)) if selectedLimits == nil { return nil, NewJSNoLimitsError() } srvLim := &s.getOpts().JetStreamLimits // Make sure we have sane defaults. Do so with the JS lock, otherwise a // badly timed meta snapshot can result in a race condition. mset.js.mu.Lock() setConsumerConfigDefaults(config, &cfg, srvLim, selectedLimits) mset.js.mu.Unlock() if err := checkConsumerCfg(config, srvLim, &cfg, acc, selectedLimits, isRecovering); err != nil { return nil, err } sampleFreq := 0 if config.SampleFrequency != _EMPTY_ { // Can't fail as checkConsumerCfg checks correct format sampleFreq, _ = strconv.Atoi(strings.TrimSuffix(config.SampleFrequency, "%")) } // Grab the client, account and server reference. c := mset.client if c == nil { return nil, NewJSStreamInvalidError() } var accName string c.mu.Lock() s, a := c.srv, c.acc if a != nil { accName = a.Name } c.mu.Unlock() // Hold mset lock here. mset.mu.Lock() if mset.client == nil || mset.store == nil || mset.consumers == nil { mset.mu.Unlock() return nil, NewJSStreamInvalidError() } // If this one is durable and already exists, we let that be ok as long as only updating what should be allowed. var cName string if isDurableConsumer(config) { cName = config.Durable } else if config.Name != _EMPTY_ { cName = config.Name } if cName != _EMPTY_ { if eo, ok := mset.consumers[cName]; ok { mset.mu.Unlock() if action == ActionCreate && !reflect.DeepEqual(*config, eo.config()) { return nil, NewJSConsumerAlreadyExistsError() } // Check for overlapping subjects if we are a workqueue if cfg.Retention == WorkQueuePolicy { subjects := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects) if !mset.partitionUnique(cName, subjects) { return nil, NewJSConsumerWQConsumerNotUniqueError() } } err := eo.updateConfig(config) if err == nil { return eo, nil } return nil, NewJSConsumerCreateError(err, Unless(err)) } } if action == ActionUpdate { mset.mu.Unlock() return nil, NewJSConsumerDoesNotExistError() } // Check for any limits, if the config for the consumer sets a limit we check against that // but if not we use the value from account limits, if account limits is more restrictive // than stream config we prefer the account limits to handle cases where account limits are // updated during the lifecycle of the stream maxc := cfg.MaxConsumers if maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) { maxc = selectedLimits.MaxConsumers } if maxc > 0 && mset.numPublicConsumers() >= maxc { mset.mu.Unlock() return nil, NewJSMaximumConsumersLimitError() } // Check on stream type conflicts with WorkQueues. if cfg.Retention == WorkQueuePolicy && !config.Direct { // Force explicit acks here. if config.AckPolicy != AckExplicit { mset.mu.Unlock() return nil, NewJSConsumerWQRequiresExplicitAckError() } if len(mset.consumers) > 0 { subjects := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects) if len(subjects) == 0 { mset.mu.Unlock() return nil, NewJSConsumerWQMultipleUnfilteredError() } else if !mset.partitionUnique(cName, subjects) { // Prior to v2.9.7, on a stream with WorkQueue policy, the servers // were not catching the error of having multiple consumers with // overlapping filter subjects depending on the scope, for instance // creating "foo.*.bar" and then "foo.>" was not detected, while // "foo.>" and then "foo.*.bar" would have been. Failing here // in recovery mode would leave the rejected consumer in a bad state, // so we will simply warn here, asking the user to remove this // consumer administratively. Otherwise, if this is the creation // of a new consumer, we will return the error. if isRecovering { s.Warnf("Consumer %q > %q has a filter subject that overlaps "+ "with other consumers, which is not allowed for a stream "+ "with WorkQueue policy, it should be administratively deleted", cfg.Name, cName) } else { // We have a partition but it is not unique amongst the others. mset.mu.Unlock() return nil, NewJSConsumerWQConsumerNotUniqueError() } } } if config.DeliverPolicy != DeliverAll { mset.mu.Unlock() return nil, NewJSConsumerWQConsumerNotDeliverAllError() } } // Set name, which will be durable name if set, otherwise we create one at random. o := &consumer{ mset: mset, js: s.getJetStream(), acc: a, srv: s, client: s.createInternalJetStreamClient(), sysc: s.createInternalJetStreamClient(), cfg: *config, dsubj: config.DeliverSubject, outq: mset.outq, active: true, qch: make(chan struct{}), uch: make(chan struct{}, 1), mch: make(chan struct{}, 1), sfreq: int32(sampleFreq), maxdc: uint64(config.MaxDeliver), maxp: config.MaxAckPending, retention: cfg.Retention, created: time.Now().UTC(), } // Bind internal client to the user account. o.client.registerWithAccount(a) // Bind to the system account. o.sysc.registerWithAccount(s.SystemAccount()) if isDurableConsumer(config) { if len(config.Durable) > JSMaxNameLen { mset.mu.Unlock() o.deleteWithoutAdvisory() return nil, NewJSConsumerNameTooLongError(JSMaxNameLen) } o.name = config.Durable } else if oname != _EMPTY_ { o.name = oname } else { if config.Name != _EMPTY_ { o.name = config.Name } else { // Legacy ephemeral auto-generated. for { o.name = createConsumerName() if _, ok := mset.consumers[o.name]; !ok { break } } config.Name = o.name } } // Create ackMsgs queue now that we have a consumer name o.ackMsgs = newIPQueue[*jsAckMsg](s, fmt.Sprintf("[ACC:%s] consumer '%s' on stream '%s' ackMsgs", accName, o.name, cfg.Name)) // Create our request waiting queue. if o.isPullMode() { o.waiting = newWaitQueue(config.MaxWaiting) // Create our internal queue for next msg requests. o.nextMsgReqs = newIPQueue[*nextMsgReq](s, fmt.Sprintf("[ACC:%s] consumer '%s' on stream '%s' pull requests", accName, o.name, cfg.Name)) } // already under lock, mset.Name() would deadlock o.stream = cfg.Name o.ackEventT = JSMetricConsumerAckPre + "." + o.stream + "." + o.name o.nakEventT = JSAdvisoryConsumerMsgNakPre + "." + o.stream + "." + o.name o.deliveryExcEventT = JSAdvisoryConsumerMaxDeliveryExceedPre + "." + o.stream + "." + o.name if !isValidName(o.name) { mset.mu.Unlock() o.deleteWithoutAdvisory() return nil, NewJSConsumerBadDurableNameError() } // Setup our storage if not a direct consumer. if !config.Direct { store, err := mset.store.ConsumerStore(o.name, config) if err != nil { mset.mu.Unlock() o.deleteWithoutAdvisory() return nil, NewJSConsumerStoreFailedError(err) } o.store = store } for _, filter := range gatherSubjectFilters(o.cfg.FilterSubject, o.cfg.FilterSubjects) { sub := &subjectFilter{ subject: filter, hasWildcard: subjectHasWildcard(filter), tokenizedSubject: tokenizeSubjectIntoSlice(nil, filter), } o.subjf = append(o.subjf, sub) } // If we have multiple filter subjects, create a sublist which we will use // in calling store.LoadNextMsgMulti. if len(o.cfg.FilterSubjects) > 0 { o.filters = NewSublistNoCache() for _, filter := range o.cfg.FilterSubjects { o.filters.Insert(&subscription{subject: []byte(filter)}) } } else { // Make sure this is nil otherwise. o.filters = nil } if o.store != nil && o.store.HasState() { // Restore our saved state. o.mu.Lock() o.readStoredState(0) o.mu.Unlock() } else { // Select starting sequence number o.selectStartingSeqNo() } // Now register with mset and create the ack subscription. // Check if we already have this one registered. if eo, ok := mset.consumers[o.name]; ok { mset.mu.Unlock() if !o.isDurable() || !o.isPushMode() { o.name = _EMPTY_ // Prevent removal since same name. o.deleteWithoutAdvisory() return nil, NewJSConsumerNameExistError() } // If we are here we have already registered this durable. If it is still active that is an error. if eo.isActive() { o.name = _EMPTY_ // Prevent removal since same name. o.deleteWithoutAdvisory() return nil, NewJSConsumerExistingActiveError() } // Since we are here this means we have a potentially new durable so we should update here. // Check that configs are the same. if !configsEqualSansDelivery(o.cfg, eo.cfg) { o.name = _EMPTY_ // Prevent removal since same name. o.deleteWithoutAdvisory() return nil, NewJSConsumerReplacementWithDifferentNameError() } // Once we are here we have a replacement push-based durable. eo.updateDeliverSubject(o.cfg.DeliverSubject) return eo, nil } // Set up the ack subscription for this consumer. Will use wildcard for all acks. // We will remember the template to generate replies with sequence numbers and use // that to scanf them back in. // Escape '%' in consumer and stream names, as `pre` is used as a template later // in consumer.ackReply(), resulting in erroneous formatting of the ack subject. mn := strings.ReplaceAll(cfg.Name, "%", "%%") pre := fmt.Sprintf(jsAckT, mn, strings.ReplaceAll(o.name, "%", "%%")) o.ackReplyT = fmt.Sprintf("%s.%%d.%%d.%%d.%%d.%%d", pre) o.ackSubj = fmt.Sprintf("%s.*.*.*.*.*", pre) o.nextMsgSubj = fmt.Sprintf(JSApiRequestNextT, mn, o.name) // Check/update the inactive threshold o.updateInactiveThreshold(&o.cfg) if o.isPushMode() { // Check if we are running only 1 replica and that the delivery subject has interest. // Check in place here for interest. Will setup properly in setLeader. if config.replicas(&cfg) == 1 { interest := o.acc.sl.HasInterest(o.cfg.DeliverSubject) if !o.hasDeliveryInterest(interest) { // Let the interest come to us eventually, but setup delete timer. o.updateDeliveryInterest(false) } } } // Set our ca. if ca != nil { o.setConsumerAssignment(ca) } // Check if we have a rate limit set. if config.RateLimit != 0 { o.setRateLimit(config.RateLimit) } mset.setConsumer(o) mset.mu.Unlock() if config.Direct || (!s.JetStreamIsClustered() && s.standAloneMode()) { o.setLeader(true) } // This is always true in single server mode. if o.IsLeader() { // Send advisory. var suppress bool if !s.standAloneMode() && ca == nil { suppress = true } else if ca != nil { suppress = ca.responded } if !suppress { o.sendCreateAdvisory() } } return o, nil } // Updates the consumer `dthresh` delete timer duration and set // cfg.InactiveThreshold to JsDeleteWaitTimeDefault for ephemerals // if not explicitly already specified by the user. // Lock should be held. func (o *consumer) updateInactiveThreshold(cfg *ConsumerConfig) { // Ephemerals will always have inactive thresholds. if !o.isDurable() && cfg.InactiveThreshold <= 0 { // Add in 1 sec of jitter above and beyond the default of 5s. o.dthresh = JsDeleteWaitTimeDefault + 100*time.Millisecond + time.Duration(rand.Int63n(900))*time.Millisecond // Only stamp config with default sans jitter. cfg.InactiveThreshold = JsDeleteWaitTimeDefault } else if cfg.InactiveThreshold > 0 { // Add in up to 1 sec of jitter if pull mode. if o.isPullMode() { o.dthresh = cfg.InactiveThreshold + 100*time.Millisecond + time.Duration(rand.Int63n(900))*time.Millisecond } else { o.dthresh = cfg.InactiveThreshold } } else if cfg.InactiveThreshold <= 0 { // We accept InactiveThreshold be set to 0 (for durables) o.dthresh = 0 } } func (o *consumer) consumerAssignment() *consumerAssignment { o.mu.RLock() defer o.mu.RUnlock() return o.ca } func (o *consumer) setConsumerAssignment(ca *consumerAssignment) { o.mu.Lock() defer o.mu.Unlock() o.ca = ca if ca == nil { return } // Set our node. o.node = ca.Group.node // Trigger update chan. select { case o.uch <- struct{}{}: default: } } func (o *consumer) updateC() <-chan struct{} { o.mu.RLock() defer o.mu.RUnlock() return o.uch } // checkQueueInterest will check on our interest's queue group status. // Lock should be held. func (o *consumer) checkQueueInterest() { if !o.active || o.cfg.DeliverSubject == _EMPTY_ { return } subj := o.dsubj if subj == _EMPTY_ { subj = o.cfg.DeliverSubject } if rr := o.acc.sl.Match(subj); len(rr.qsubs) > 0 { // Just grab first if qsubs := rr.qsubs[0]; len(qsubs) > 0 { if sub := rr.qsubs[0][0]; len(sub.queue) > 0 { o.qgroup = string(sub.queue) } } } } // clears our node if we have one. When we scale down to 1. func (o *consumer) clearNode() { o.mu.Lock() defer o.mu.Unlock() if o.node != nil { o.node.Delete() o.node = nil } } // IsLeader will return if we are the current leader. func (o *consumer) IsLeader() bool { return o.isLeader() } // Lock should be held. func (o *consumer) isLeader() bool { return o.leader.Load() } func (o *consumer) setLeader(isLeader bool) { o.mu.RLock() mset, closed := o.mset, o.closed movingToClustered := o.node != nil && o.pch == nil movingToNonClustered := o.node == nil && o.pch != nil wasLeader := o.leader.Swap(isLeader) o.mu.RUnlock() // If we are here we have a change in leader status. if isLeader { if closed || mset == nil { return } if wasLeader { // If we detect we are scaling up, make sure to create clustered routines and channels. if movingToClustered { o.mu.Lock() // We are moving from R1 to clustered. o.pch = make(chan struct{}, 1) go o.loopAndForwardProposals(o.qch) if o.phead != nil { select { case o.pch <- struct{}{}: default: } } o.mu.Unlock() } else if movingToNonClustered { // We are moving from clustered to non-clustered now. // Set pch to nil so if we scale back up we will recreate the loopAndForward from above. o.mu.Lock() pch := o.pch o.pch = nil select { case pch <- struct{}{}: default: } o.mu.Unlock() } return } mset.mu.RLock() s, jsa, stream, lseq := mset.srv, mset.jsa, mset.getCfgName(), mset.lseq mset.mu.RUnlock() o.mu.Lock() o.rdq = nil o.rdqi.Empty() // Restore our saved state. During non-leader status we just update our underlying store. o.readStoredState(lseq) // Setup initial num pending. o.streamNumPending() // Cleanup lss when we take over in clustered mode. if o.hasSkipListPending() && o.sseq >= o.lss.resume { o.lss = nil } // Update the group on the our starting sequence if we are starting but we skipped some in the stream. if o.dseq == 1 && o.sseq > 1 { o.updateSkipped(o.sseq) } // Do info sub. if o.infoSub == nil && jsa != nil { isubj := fmt.Sprintf(clusterConsumerInfoT, jsa.acc(), stream, o.name) // Note below the way we subscribe here is so that we can send requests to ourselves. o.infoSub, _ = s.systemSubscribe(isubj, _EMPTY_, false, o.sysc, o.handleClusterConsumerInfoRequest) } var err error if o.cfg.AckPolicy != AckNone { if o.ackSub, err = o.subscribeInternal(o.ackSubj, o.pushAck); err != nil { o.mu.Unlock() o.deleteWithoutAdvisory() return } } // Setup the internal sub for next message requests regardless. // Will error if wrong mode to provide feedback to users. if o.reqSub, err = o.subscribeInternal(o.nextMsgSubj, o.processNextMsgReq); err != nil { o.mu.Unlock() o.deleteWithoutAdvisory() return } // Check on flow control settings. if o.cfg.FlowControl { o.setMaxPendingBytes(JsFlowControlMaxPending) fcsubj := fmt.Sprintf(jsFlowControl, stream, o.name) if o.fcSub, err = o.subscribeInternal(fcsubj, o.processFlowControl); err != nil { o.mu.Unlock() o.deleteWithoutAdvisory() return } } // If push mode, register for notifications on interest. if o.isPushMode() { o.inch = make(chan bool, 8) o.acc.sl.registerNotification(o.cfg.DeliverSubject, o.cfg.DeliverGroup, o.inch) if o.active = <-o.inch; o.active { o.checkQueueInterest() } // Check gateways in case they are enabled. if s.gateway.enabled { if !o.active { o.active = s.hasGatewayInterest(o.acc.Name, o.cfg.DeliverSubject) } stopAndClearTimer(&o.gwdtmr) o.gwdtmr = time.AfterFunc(time.Second, func() { o.watchGWinterest() }) } } if o.dthresh > 0 && (o.isPullMode() || !o.active) { // Pull consumer. We run the dtmr all the time for this one. stopAndClearTimer(&o.dtmr) o.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive) } // If we are not in ReplayInstant mode mark us as in replay state until resolved. if o.cfg.ReplayPolicy != ReplayInstant { o.replay = true } // Recreate quit channel. o.qch = make(chan struct{}) qch := o.qch node := o.node if node != nil && o.pch == nil { o.pch = make(chan struct{}, 1) } pullMode := o.isPullMode() o.mu.Unlock() // Check if there are any pending we might need to clean up etc. o.checkPending() // Snapshot initial info. o.infoWithSnap(true) // These are the labels we will use to annotate our goroutines. labels := pprofLabels{ "type": "consumer", "account": mset.accName(), "stream": mset.name(), "consumer": o.name, } // Now start up Go routine to deliver msgs. go func() { setGoRoutineLabels(labels) o.loopAndGatherMsgs(qch) }() // Now start up Go routine to process acks. go func() { setGoRoutineLabels(labels) o.processInboundAcks(qch) }() if pullMode { // Now start up Go routine to process inbound next message requests. go func() { setGoRoutineLabels(labels) o.processInboundNextMsgReqs(qch) }() } // If we are R>1 spin up our proposal loop. if node != nil { // Determine if we can send pending requests info to the group. // They must be on server versions >= 2.7.1 o.checkAndSetPendingRequestsOk() o.checkPendingRequests() go func() { setGoRoutineLabels(labels) o.loopAndForwardProposals(qch) }() } } else { // Shutdown the go routines and the subscriptions. o.mu.Lock() if o.qch != nil { close(o.qch) o.qch = nil } // Stop any inactivity timers. Should only be running on leaders. stopAndClearTimer(&o.dtmr) // Make sure to clear out any re-deliver queues o.stopAndClearPtmr() o.rdq = nil o.rdqi.Empty() o.pending = nil // ok if they are nil, we protect inside unsubscribe() o.unsubscribe(o.ackSub) o.unsubscribe(o.reqSub) o.unsubscribe(o.fcSub) o.ackSub, o.reqSub, o.fcSub = nil, nil, nil if o.infoSub != nil { o.srv.sysUnsubscribe(o.infoSub) o.infoSub = nil } // Reset waiting if we are in pull mode. if o.isPullMode() { o.waiting = newWaitQueue(o.cfg.MaxWaiting) o.nextMsgReqs.drain() } else if o.srv.gateway.enabled { stopAndClearTimer(&o.gwdtmr) } // If we were the leader make sure to drain queued up acks. if wasLeader { o.ackMsgs.drain() // Reset amount of acks that need to be processed. atomic.StoreInt64(&o.awl, 0) // Also remove any pending replies since we should not be the one to respond at this point. o.replies = nil } o.mu.Unlock() } } // This is coming on the wire so do not block here. func (o *consumer) handleClusterConsumerInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { go o.infoWithSnapAndReply(false, reply) } // Lock should be held. func (o *consumer) subscribeInternal(subject string, cb msgHandler) (*subscription, error) { c := o.client if c == nil { return nil, fmt.Errorf("invalid consumer") } if !c.srv.EventsEnabled() { return nil, ErrNoSysAccount } if cb == nil { return nil, fmt.Errorf("undefined message handler") } o.sid++ // Now create the subscription return c.processSub([]byte(subject), nil, []byte(strconv.Itoa(o.sid)), cb, false) } // Unsubscribe from our subscription. // Lock should be held. func (o *consumer) unsubscribe(sub *subscription) { if sub == nil || o.client == nil { return } o.client.processUnsub(sub.sid) } // We need to make sure we protect access to the outq. // Do all advisory sends here. func (o *consumer) sendAdvisory(subject string, e any) { if o.acc == nil { return } // If there is no one listening for this advisory then save ourselves the effort // and don't bother encoding the JSON or sending it. if sl := o.acc.sl; (sl != nil && !sl.HasInterest(subject)) && !o.srv.hasGatewayInterest(o.acc.Name, subject) { return } j, err := json.Marshal(e) if err != nil { return } o.outq.sendMsg(subject, j) } func (o *consumer) sendDeleteAdvisoryLocked() { e := JSConsumerActionAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerActionAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: o.stream, Consumer: o.name, Action: DeleteEvent, Domain: o.srv.getOpts().JetStreamDomain, } subj := JSAdvisoryConsumerDeletedPre + "." + o.stream + "." + o.name o.sendAdvisory(subj, e) } func (o *consumer) sendCreateAdvisory() { o.mu.Lock() defer o.mu.Unlock() e := JSConsumerActionAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerActionAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: o.stream, Consumer: o.name, Action: CreateEvent, Domain: o.srv.getOpts().JetStreamDomain, } subj := JSAdvisoryConsumerCreatedPre + "." + o.stream + "." + o.name o.sendAdvisory(subj, e) } // Created returns created time. func (o *consumer) createdTime() time.Time { o.mu.Lock() created := o.created o.mu.Unlock() return created } // Internal to allow creation time to be restored. func (o *consumer) setCreatedTime(created time.Time) { o.mu.Lock() o.created = created o.mu.Unlock() } // This will check for extended interest in a subject. If we have local interest we just return // that, but in the absence of local interest and presence of gateways or service imports we need // to check those as well. func (o *consumer) hasDeliveryInterest(localInterest bool) bool { o.mu.RLock() mset := o.mset if mset == nil { o.mu.RUnlock() return false } acc := o.acc deliver := o.cfg.DeliverSubject o.mu.RUnlock() if localInterest { return true } // If we are here check gateways. if s := acc.srv; s != nil && s.hasGatewayInterest(acc.Name, deliver) { return true } return false } func (s *Server) hasGatewayInterest(account, subject string) bool { gw := s.gateway if !gw.enabled { return false } gw.RLock() defer gw.RUnlock() for _, gwc := range gw.outo { psi, qr := gwc.gatewayInterest(account, stringToBytes(subject)) if psi || qr != nil { return true } } return false } // This processes an update to the local interest for a deliver subject. func (o *consumer) updateDeliveryInterest(localInterest bool) bool { interest := o.hasDeliveryInterest(localInterest) o.mu.Lock() defer o.mu.Unlock() mset := o.mset if mset == nil || o.isPullMode() { return false } if interest && !o.active { o.signalNewMessages() } // Update active status, if not active clear any queue group we captured. if o.active = interest; !o.active { o.qgroup = _EMPTY_ } else { o.checkQueueInterest() } // If the delete timer has already been set do not clear here and return. // Note that durable can now have an inactive threshold, so don't check // for durable status, instead check for dthresh > 0. if o.dtmr != nil && o.dthresh > 0 && !interest { return true } // Stop and clear the delete timer always. stopAndClearTimer(&o.dtmr) // If we do not have interest anymore and have a delete threshold set, then set // a timer to delete us. We wait for a bit in case of server reconnect. if !interest && o.dthresh > 0 { o.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive) return true } return false } const ( defaultConsumerNotActiveStartInterval = 30 * time.Second defaultConsumerNotActiveMaxInterval = 5 * time.Minute ) var ( consumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval consumerNotActiveMaxInterval = defaultConsumerNotActiveMaxInterval ) // deleteNotActive must only be called from time.AfterFunc or in its own // goroutine, as it can block on clean-up. func (o *consumer) deleteNotActive() { // Take a copy of these when the goroutine starts, mostly it avoids a // race condition with tests that modify these consts, such as // TestJetStreamClusterGhostEphemeralsAfterRestart. cnaMax := consumerNotActiveMaxInterval cnaStart := consumerNotActiveStartInterval o.mu.Lock() if o.mset == nil { o.mu.Unlock() return } // Push mode just look at active. if o.isPushMode() { // If we are active simply return. if o.active { o.mu.Unlock() return } } else { // Pull mode. elapsed := time.Since(o.waiting.last) if elapsed <= o.cfg.InactiveThreshold { // These need to keep firing so reset but use delta. if o.dtmr != nil { o.dtmr.Reset(o.dthresh - elapsed) } else { o.dtmr = time.AfterFunc(o.dthresh-elapsed, o.deleteNotActive) } o.mu.Unlock() return } // Check if we still have valid requests waiting. if o.checkWaitingForInterest() { if o.dtmr != nil { o.dtmr.Reset(o.dthresh) } else { o.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive) } o.mu.Unlock() return } } s, js := o.mset.srv, o.srv.js.Load() acc, stream, name, isDirect := o.acc.Name, o.stream, o.name, o.cfg.Direct var qch, cqch chan struct{} if o.srv != nil { qch = o.srv.quitCh } o.mu.Unlock() if js != nil { cqch = js.clusterQuitC() } // Useful for pprof. setGoRoutineLabels(pprofLabels{ "account": acc, "stream": stream, "consumer": name, }) // We will delete locally regardless. defer o.delete() // If we are clustered, check if we still have this consumer assigned. // If we do forward a proposal to delete ourselves to the metacontroller leader. if !isDirect && s.JetStreamIsClustered() { js.mu.RLock() var ( cca consumerAssignment meta RaftNode removeEntry []byte ) ca, cc := js.consumerAssignment(acc, stream, name), js.cluster if ca != nil && cc != nil { meta = cc.meta cca = *ca cca.Reply = _EMPTY_ removeEntry = encodeDeleteConsumerAssignment(&cca) meta.ForwardProposal(removeEntry) } js.mu.RUnlock() if ca != nil && cc != nil { // Check to make sure we went away. // Don't think this needs to be a monitored go routine. jitter := time.Duration(rand.Int63n(int64(cnaStart))) interval := cnaStart + jitter ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ticker.C: case <-qch: return case <-cqch: return } js.mu.RLock() if js.shuttingDown { js.mu.RUnlock() return } nca := js.consumerAssignment(acc, stream, name) js.mu.RUnlock() // Make sure this is not a new consumer with the same name. if nca != nil && nca == ca { s.Warnf("Consumer assignment for '%s > %s > %s' not cleaned up, retrying", acc, stream, name) meta.ForwardProposal(removeEntry) if interval < cnaMax { interval *= 2 ticker.Reset(interval) } continue } // We saw that consumer has been removed, all done. return } } } } func (o *consumer) watchGWinterest() { pa := o.isActive() // If there is no local interest... if o.hasNoLocalInterest() { o.updateDeliveryInterest(false) if !pa && o.isActive() { o.signalNewMessages() } } // We want this to always be running so we can also pick up on interest returning. o.mu.Lock() if o.gwdtmr != nil { o.gwdtmr.Reset(time.Second) } else { stopAndClearTimer(&o.gwdtmr) o.gwdtmr = time.AfterFunc(time.Second, func() { o.watchGWinterest() }) } o.mu.Unlock() } // Config returns the consumer's configuration. func (o *consumer) config() ConsumerConfig { o.mu.Lock() defer o.mu.Unlock() return o.cfg } // Check if we have hit max deliveries. If so do notification and cleanup. // Return whether or not the max was hit. // Lock should be held. func (o *consumer) hasMaxDeliveries(seq uint64) bool { if o.maxdc == 0 { return false } if dc := o.deliveryCount(seq); dc >= o.maxdc { // We have hit our max deliveries for this sequence. // Only send the advisory once. if dc == o.maxdc { o.notifyDeliveryExceeded(seq, dc) } // Determine if we signal to start flow of messages again. if o.maxp > 0 && len(o.pending) >= o.maxp { o.signalNewMessages() } // Make sure to remove from pending. if p, ok := o.pending[seq]; ok && p != nil { delete(o.pending, seq) o.updateDelivered(p.Sequence, seq, dc, p.Timestamp) } // Ensure redelivered state is set, if not already. if o.rdc == nil { o.rdc = make(map[uint64]uint64) } o.rdc[seq] = dc return true } return false } // Force expiration of all pending. // Lock should be held. func (o *consumer) forceExpirePending() { var expired []uint64 for seq := range o.pending { if !o.onRedeliverQueue(seq) && !o.hasMaxDeliveries(seq) { expired = append(expired, seq) } } if len(expired) > 0 { slices.Sort(expired) o.addToRedeliverQueue(expired...) // Now we should update the timestamp here since we are redelivering. // We will use an incrementing time to preserve order for any other redelivery. off := time.Now().UnixNano() - o.pending[expired[0]].Timestamp for _, seq := range expired { if p, ok := o.pending[seq]; ok && p != nil { p.Timestamp += off } } o.resetPtmr(o.ackWait(0)) } o.signalNewMessages() } // Acquire proper locks and update rate limit. // Will use what is in config. func (o *consumer) setRateLimitNeedsLocks() { o.mu.RLock() mset := o.mset o.mu.RUnlock() if mset == nil { return } mset.mu.RLock() o.mu.Lock() o.setRateLimit(o.cfg.RateLimit) o.mu.Unlock() mset.mu.RUnlock() } // Set the rate limiter // Both mset and consumer lock should be held. func (o *consumer) setRateLimit(bps uint64) { if bps == 0 { o.rlimit = nil return } // TODO(dlc) - Make sane values or error if not sane? // We are configured in bits per sec so adjust to bytes. rl := rate.Limit(bps / 8) mset := o.mset // Burst should be set to maximum msg size for this account, etc. var burst int // We don't need to get cfgMu's rlock here since this function // is already invoked under mset.mu.RLock(), which superseeds cfgMu. if mset.cfg.MaxMsgSize > 0 { burst = int(mset.cfg.MaxMsgSize) } else if mset.jsa.account.limits.mpay > 0 { burst = int(mset.jsa.account.limits.mpay) } else { s := mset.jsa.account.srv burst = int(s.getOpts().MaxPayload) } o.rlimit = rate.NewLimiter(rl, burst) } // Check if new consumer config allowed vs old. func (acc *Account) checkNewConsumerConfig(cfg, ncfg *ConsumerConfig) error { if reflect.DeepEqual(cfg, ncfg) { return nil } // Something different, so check since we only allow certain things to be updated. if cfg.DeliverPolicy != ncfg.DeliverPolicy { return errors.New("deliver policy can not be updated") } if cfg.OptStartSeq != ncfg.OptStartSeq { return errors.New("start sequence can not be updated") } if cfg.OptStartTime != nil && ncfg.OptStartTime != nil { // Both have start times set, compare them directly: if !cfg.OptStartTime.Equal(*ncfg.OptStartTime) { return errors.New("start time can not be updated") } } else if cfg.OptStartTime != nil || ncfg.OptStartTime != nil { // At least one start time is set and the other is not return errors.New("start time can not be updated") } if cfg.AckPolicy != ncfg.AckPolicy { return errors.New("ack policy can not be updated") } if cfg.ReplayPolicy != ncfg.ReplayPolicy { return errors.New("replay policy can not be updated") } if cfg.Heartbeat != ncfg.Heartbeat { return errors.New("heart beats can not be updated") } if cfg.FlowControl != ncfg.FlowControl { return errors.New("flow control can not be updated") } // Deliver Subject is conditional on if its bound. if cfg.DeliverSubject != ncfg.DeliverSubject { if cfg.DeliverSubject == _EMPTY_ { return errors.New("can not update pull consumer to push based") } if ncfg.DeliverSubject == _EMPTY_ { return errors.New("can not update push consumer to pull based") } if acc.sl.HasInterest(cfg.DeliverSubject) { return NewJSConsumerNameExistError() } } if cfg.MaxWaiting != ncfg.MaxWaiting { return errors.New("max waiting can not be updated") } // Check if BackOff is defined, MaxDeliver is within range. if lbo := len(ncfg.BackOff); lbo > 0 && ncfg.MaxDeliver != -1 && lbo > ncfg.MaxDeliver { return NewJSConsumerMaxDeliverBackoffError() } return nil } // Update the config based on the new config, or error if update not allowed. func (o *consumer) updateConfig(cfg *ConsumerConfig) error { o.mu.Lock() defer o.mu.Unlock() if o.closed || o.mset == nil { return NewJSConsumerDoesNotExistError() } if err := o.acc.checkNewConsumerConfig(&o.cfg, cfg); err != nil { return err } if o.store != nil { // Update local state always. if err := o.store.UpdateConfig(cfg); err != nil { return err } } // DeliverSubject if cfg.DeliverSubject != o.cfg.DeliverSubject { o.updateDeliverSubjectLocked(cfg.DeliverSubject) } // MaxAckPending if cfg.MaxAckPending != o.cfg.MaxAckPending { o.maxp = cfg.MaxAckPending o.signalNewMessages() } // AckWait if cfg.AckWait != o.cfg.AckWait { if o.ptmr != nil { o.resetPtmr(100 * time.Millisecond) } } // Rate Limit if cfg.RateLimit != o.cfg.RateLimit { // We need both locks here so do in Go routine. go o.setRateLimitNeedsLocks() } if cfg.SampleFrequency != o.cfg.SampleFrequency { s := strings.TrimSuffix(cfg.SampleFrequency, "%") // String has been already verified for validity up in the stack, so no // need to check for error here. sampleFreq, _ := strconv.Atoi(s) o.sfreq = int32(sampleFreq) } // Set MaxDeliver if changed if cfg.MaxDeliver != o.cfg.MaxDeliver { o.maxdc = uint64(cfg.MaxDeliver) } // Set InactiveThreshold if changed. if val := cfg.InactiveThreshold; val != o.cfg.InactiveThreshold { o.updateInactiveThreshold(cfg) stopAndClearTimer(&o.dtmr) // Restart timer only if we are the leader. if o.isLeader() && o.dthresh > 0 { o.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive) } } // Check for Subject Filters update. newSubjects := gatherSubjectFilters(cfg.FilterSubject, cfg.FilterSubjects) if !subjectSliceEqual(newSubjects, o.subjf.subjects()) { newSubjf := make(subjectFilters, 0, len(newSubjects)) for _, newFilter := range newSubjects { fs := &subjectFilter{ subject: newFilter, hasWildcard: subjectHasWildcard(newFilter), tokenizedSubject: tokenizeSubjectIntoSlice(nil, newFilter), } newSubjf = append(newSubjf, fs) } // Make sure we have correct signaling setup. // Consumer lock can not be held. mset := o.mset o.mu.Unlock() mset.swapSigSubs(o, newSubjf.subjects()) o.mu.Lock() // When we're done with signaling, we can replace the subjects. // If filters were removed, set `o.subjf` to nil. if len(newSubjf) == 0 { o.subjf = nil o.filters = nil } else { o.subjf = newSubjf if len(o.subjf) == 1 { o.filters = nil } else { o.filters = NewSublistNoCache() for _, filter := range o.subjf { o.filters.Insert(&subscription{subject: []byte(filter.subject)}) } } } } // Record new config for others that do not need special handling. // Allowed but considered no-op, [Description, SampleFrequency, MaxWaiting, HeadersOnly] o.cfg = *cfg // Cleanup messages that lost interest. if o.retention == InterestPolicy { o.mu.Unlock() o.cleanupNoInterestMessages(o.mset, false) o.mu.Lock() } // Re-calculate num pending on update. o.streamNumPending() return nil } // This is a config change for the delivery subject for a // push based consumer. func (o *consumer) updateDeliverSubject(newDeliver string) { // Update the config and the dsubj o.mu.Lock() defer o.mu.Unlock() o.updateDeliverSubjectLocked(newDeliver) } // This is a config change for the delivery subject for a // push based consumer. func (o *consumer) updateDeliverSubjectLocked(newDeliver string) { if o.closed || o.isPullMode() || o.cfg.DeliverSubject == newDeliver { return } // Force redeliver of all pending on change of delivery subject. if len(o.pending) > 0 { o.forceExpirePending() } o.acc.sl.clearNotification(o.dsubj, o.cfg.DeliverGroup, o.inch) o.dsubj, o.cfg.DeliverSubject = newDeliver, newDeliver // When we register new one it will deliver to update state loop. o.acc.sl.registerNotification(newDeliver, o.cfg.DeliverGroup, o.inch) } // Check that configs are equal but allow delivery subjects to be different. func configsEqualSansDelivery(a, b ConsumerConfig) bool { // These were copied in so can set Delivery here. a.DeliverSubject, b.DeliverSubject = _EMPTY_, _EMPTY_ return reflect.DeepEqual(a, b) } // Helper to send a reply to an ack. func (o *consumer) sendAckReply(subj string) { o.mu.RLock() defer o.mu.RUnlock() o.outq.sendMsg(subj, nil) } type jsAckMsg struct { subject string reply string hdr int msg []byte } var jsAckMsgPool sync.Pool func newJSAckMsg(subj, reply string, hdr int, msg []byte) *jsAckMsg { var m *jsAckMsg am := jsAckMsgPool.Get() if am != nil { m = am.(*jsAckMsg) } else { m = &jsAckMsg{} } // When getting something from a pool it is critical that all fields are // initialized. Doing this way guarantees that if someone adds a field to // the structure, the compiler will fail the build if this line is not updated. (*m) = jsAckMsg{subj, reply, hdr, msg} return m } func (am *jsAckMsg) returnToPool() { if am == nil { return } am.subject, am.reply, am.hdr, am.msg = _EMPTY_, _EMPTY_, -1, nil jsAckMsgPool.Put(am) } // Push the ack message to the consumer's ackMsgs queue func (o *consumer) pushAck(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { atomic.AddInt64(&o.awl, 1) o.ackMsgs.push(newJSAckMsg(subject, reply, c.pa.hdr, copyBytes(rmsg))) } // Processes a message for the ack reply subject delivered with a message. func (o *consumer) processAck(subject, reply string, hdr int, rmsg []byte) { defer atomic.AddInt64(&o.awl, -1) var msg []byte if hdr > 0 { msg = rmsg[hdr:] } else { msg = rmsg } sseq, dseq, dc := ackReplyInfo(subject) skipAckReply := sseq == 0 switch { case len(msg) == 0, bytes.Equal(msg, AckAck), bytes.Equal(msg, AckOK): if !o.processAckMsg(sseq, dseq, dc, reply, true) { // We handle replies for acks in updateAcks skipAckReply = true } case bytes.HasPrefix(msg, AckNext): o.processAckMsg(sseq, dseq, dc, _EMPTY_, true) o.processNextMsgRequest(reply, msg[len(AckNext):]) skipAckReply = true case bytes.HasPrefix(msg, AckNak): o.processNak(sseq, dseq, dc, msg) case bytes.Equal(msg, AckProgress): o.progressUpdate(sseq) case bytes.HasPrefix(msg, AckTerm): var reason string if buf := msg[len(AckTerm):]; len(buf) > 0 { reason = string(bytes.TrimSpace(buf)) } if !o.processTerm(sseq, dseq, dc, reason, reply) { // We handle replies for acks in updateAcks skipAckReply = true } } // Ack the ack if requested. if len(reply) > 0 && !skipAckReply { o.sendAckReply(reply) } } // Used to process a working update to delay redelivery. func (o *consumer) progressUpdate(seq uint64) { o.mu.Lock() defer o.mu.Unlock() if p, ok := o.pending[seq]; ok { p.Timestamp = time.Now().UnixNano() // Update store system. o.updateDelivered(p.Sequence, seq, 1, p.Timestamp) } } // Lock should be held. func (o *consumer) updateSkipped(seq uint64) { // Clustered mode and R>1 only. if o.node == nil || !o.isLeader() { return } var b [1 + 8]byte b[0] = byte(updateSkipOp) var le = binary.LittleEndian le.PutUint64(b[1:], seq) o.propose(b[:]) } func (o *consumer) loopAndForwardProposals(qch chan struct{}) { // On exit make sure we nil out pch. defer func() { o.mu.Lock() o.pch = nil o.mu.Unlock() }() o.mu.RLock() node, pch := o.node, o.pch o.mu.RUnlock() if node == nil || pch == nil { return } forwardProposals := func() error { o.mu.Lock() if o.node == nil || o.node.State() != Leader { o.mu.Unlock() return errors.New("no longer leader") } proposal := o.phead o.phead, o.ptail = nil, nil o.mu.Unlock() // 256k max for now per batch. const maxBatch = 256 * 1024 var entries []*Entry for sz := 0; proposal != nil; proposal = proposal.next { entries = append(entries, newEntry(EntryNormal, proposal.data)) sz += len(proposal.data) if sz > maxBatch { node.ProposeMulti(entries) // We need to re-create `entries` because there is a reference // to it in the node's pae map. sz, entries = 0, nil } } if len(entries) > 0 { node.ProposeMulti(entries) } return nil } // In case we have anything pending on entry. forwardProposals() for { select { case <-qch: forwardProposals() return case <-pch: if err := forwardProposals(); err != nil { return } } } } // Lock should be held. func (o *consumer) propose(entry []byte) { p := &proposal{data: entry} if o.phead == nil { o.phead = p } else { o.ptail.next = p } o.ptail = p // Kick our looper routine. select { case o.pch <- struct{}{}: default: } } // Lock should be held. func (o *consumer) updateDelivered(dseq, sseq, dc uint64, ts int64) { // Clustered mode and R>1. if o.node != nil { // Inline for now, use variable compression. var b [4*binary.MaxVarintLen64 + 1]byte b[0] = byte(updateDeliveredOp) n := 1 n += binary.PutUvarint(b[n:], dseq) n += binary.PutUvarint(b[n:], sseq) n += binary.PutUvarint(b[n:], dc) n += binary.PutVarint(b[n:], ts) o.propose(b[:n]) } else if o.store != nil { o.store.UpdateDelivered(dseq, sseq, dc, ts) } // Update activity. o.ldt = time.Now() } // Used to remember a pending ack reply in a replicated consumer. // Lock should be held. func (o *consumer) addAckReply(sseq uint64, reply string) { if o.replies == nil { o.replies = make(map[uint64]string) } o.replies[sseq] = reply } // Lock should be held. func (o *consumer) updateAcks(dseq, sseq uint64, reply string) { if o.node != nil { // Inline for now, use variable compression. var b [2*binary.MaxVarintLen64 + 1]byte b[0] = byte(updateAcksOp) n := 1 n += binary.PutUvarint(b[n:], dseq) n += binary.PutUvarint(b[n:], sseq) o.propose(b[:n]) if reply != _EMPTY_ { o.addAckReply(sseq, reply) } } else if o.store != nil { o.store.UpdateAcks(dseq, sseq) if reply != _EMPTY_ { // Already locked so send direct. o.outq.sendMsg(reply, nil) } } // Update activity. o.lat = time.Now() } // Communicate to the cluster an addition of a pending request. // Lock should be held. func (o *consumer) addClusterPendingRequest(reply string) { if o.node == nil || !o.pendingRequestsOk() { return } b := make([]byte, len(reply)+1) b[0] = byte(addPendingRequest) copy(b[1:], reply) o.propose(b) } // Communicate to the cluster a removal of a pending request. // Lock should be held. func (o *consumer) removeClusterPendingRequest(reply string) { if o.node == nil || !o.pendingRequestsOk() { return } b := make([]byte, len(reply)+1) b[0] = byte(removePendingRequest) copy(b[1:], reply) o.propose(b) } // Set whether or not we can send pending requests to followers. func (o *consumer) setPendingRequestsOk(ok bool) { o.mu.Lock() o.prOk = ok o.mu.Unlock() } // Lock should be held. func (o *consumer) pendingRequestsOk() bool { return o.prOk } // Set whether or not we can send info about pending pull requests to our group. // Will require all peers have a minimum version. func (o *consumer) checkAndSetPendingRequestsOk() { o.mu.RLock() s, isValid := o.srv, o.mset != nil o.mu.RUnlock() if !isValid { return } if ca := o.consumerAssignment(); ca != nil && len(ca.Group.Peers) > 1 { for _, pn := range ca.Group.Peers { if si, ok := s.nodeToInfo.Load(pn); ok { if !versionAtLeast(si.(nodeInfo).version, 2, 7, 1) { // We expect all of our peers to eventually be up to date. // So check again in awhile. time.AfterFunc(eventsHBInterval, func() { o.checkAndSetPendingRequestsOk() }) o.setPendingRequestsOk(false) return } } } } o.setPendingRequestsOk(true) } // On leadership change make sure we alert the pending requests that they are no longer valid. func (o *consumer) checkPendingRequests() { o.mu.Lock() defer o.mu.Unlock() if o.mset == nil || o.outq == nil { return } hdr := []byte("NATS/1.0 409 Leadership Change\r\n\r\n") for reply := range o.prm { o.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } o.prm = nil } // This will release any pending pull requests if applicable. // Should be called only by the leader being deleted or stopped. // Lock should be held. func (o *consumer) releaseAnyPendingRequests(isAssigned bool) { if o.mset == nil || o.outq == nil || o.waiting.len() == 0 { return } var hdr []byte if !isAssigned { hdr = []byte("NATS/1.0 409 Consumer Deleted\r\n\r\n") } wq := o.waiting for wr := wq.head; wr != nil; { if hdr != nil { o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } next := wr.next wr.recycle() wr = next } // Nil out old queue. o.waiting = nil } // Process a NAK. func (o *consumer) processNak(sseq, dseq, dc uint64, nak []byte) { o.mu.Lock() defer o.mu.Unlock() // Check for out of range. if dseq <= o.adflr || dseq > o.dseq { return } // If we are explicit ack make sure this is still on our pending list. if _, ok := o.pending[sseq]; !ok { return } // Deliver an advisory e := JSConsumerDeliveryNakAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerDeliveryNakAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: o.stream, Consumer: o.name, ConsumerSeq: dseq, StreamSeq: sseq, Deliveries: dc, Domain: o.srv.getOpts().JetStreamDomain, } o.sendAdvisory(o.nakEventT, e) // Check to see if we have delays attached. if len(nak) > len(AckNak) { arg := bytes.TrimSpace(nak[len(AckNak):]) if len(arg) > 0 { var d time.Duration var err error if arg[0] == '{' { var nd ConsumerNakOptions if err = json.Unmarshal(arg, &nd); err == nil { d = nd.Delay } } else { d, err = time.ParseDuration(string(arg)) } if err != nil { // Treat this as normal NAK. o.srv.Warnf("JetStream consumer '%s > %s > %s' bad NAK delay value: %q", o.acc.Name, o.stream, o.name, arg) } else { // We have a parsed duration that the user wants us to wait before retrying. // Make sure we are not on the rdq. o.removeFromRedeliverQueue(sseq) if p, ok := o.pending[sseq]; ok { // now - ackWait is expired now, so offset from there. p.Timestamp = time.Now().Add(-o.cfg.AckWait).Add(d).UnixNano() // Update store system which will update followers as well. o.updateDelivered(p.Sequence, sseq, dc, p.Timestamp) if o.ptmr != nil { // Want checkPending to run and figure out the next timer ttl. // TODO(dlc) - We could optimize this maybe a bit more and track when we expect the timer to fire. o.resetPtmr(10 * time.Millisecond) } } // Nothing else for use to do now so return. return } } } // If already queued up also ignore. if !o.onRedeliverQueue(sseq) { o.addToRedeliverQueue(sseq) } o.signalNewMessages() } // Process a TERM // Returns `true` if the ack was processed in place and the sender can now respond // to the client, or `false` if there was an error or the ack is replicated (in which // case the reply will be sent later). func (o *consumer) processTerm(sseq, dseq, dc uint64, reason, reply string) bool { // Treat like an ack to suppress redelivery. ackedInPlace := o.processAckMsg(sseq, dseq, dc, reply, false) o.mu.Lock() defer o.mu.Unlock() // Deliver an advisory e := JSConsumerDeliveryTerminatedAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerDeliveryTerminatedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: o.stream, Consumer: o.name, ConsumerSeq: dseq, StreamSeq: sseq, Deliveries: dc, Reason: reason, Domain: o.srv.getOpts().JetStreamDomain, } subj := JSAdvisoryConsumerMsgTerminatedPre + "." + o.stream + "." + o.name o.sendAdvisory(subj, e) return ackedInPlace } // Introduce a small delay in when timer fires to check pending. // Allows bursts to be treated in same time frame. const ackWaitDelay = time.Millisecond // ackWait returns how long to wait to fire the pending timer. func (o *consumer) ackWait(next time.Duration) time.Duration { if next > 0 { return next + ackWaitDelay } return o.cfg.AckWait + ackWaitDelay } // Due to bug in calculation of sequences on restoring redelivered let's do quick sanity check. // Lock should be held. func (o *consumer) checkRedelivered(slseq uint64) { var lseq uint64 if mset := o.mset; mset != nil { lseq = slseq } var shouldUpdateState bool for sseq := range o.rdc { if sseq <= o.asflr || (lseq > 0 && sseq > lseq) { delete(o.rdc, sseq) o.removeFromRedeliverQueue(sseq) shouldUpdateState = true } } if shouldUpdateState { if err := o.writeStoreStateUnlocked(); err != nil && o.srv != nil && o.mset != nil && !o.closed { s, acc, mset, name := o.srv, o.acc, o.mset, o.name s.Warnf("Consumer '%s > %s > %s' error on write store state from check redelivered: %v", acc, mset.getCfgName(), name, err) } } } // This will restore the state from disk. // Lock should be held. func (o *consumer) readStoredState(slseq uint64) error { if o.store == nil { return nil } state, err := o.store.State() if err == nil { o.applyState(state) if len(o.rdc) > 0 { o.checkRedelivered(slseq) } } return err } // Apply the consumer stored state. // Lock should be held. func (o *consumer) applyState(state *ConsumerState) { if state == nil { return } // If o.sseq is greater don't update. Don't go backwards on o.sseq if leader. if !o.isLeader() || o.sseq <= state.Delivered.Stream { o.sseq = state.Delivered.Stream + 1 } o.dseq = state.Delivered.Consumer + 1 o.adflr = state.AckFloor.Consumer o.asflr = state.AckFloor.Stream o.pending = state.Pending o.rdc = state.Redelivered // Setup tracking timer if we have restored pending. if o.isLeader() && len(o.pending) > 0 { // This is on startup or leader change. We want to check pending // sooner in case there are inconsistencies etc. Pick between 500ms - 1.5s delay := 500*time.Millisecond + time.Duration(rand.Int63n(1000))*time.Millisecond // If normal is lower than this just use that. if o.cfg.AckWait < delay { delay = o.ackWait(0) } o.resetPtmr(delay) } } // Sets our store state from another source. Used in clustered mode on snapshot restore. // Lock should be held. func (o *consumer) setStoreState(state *ConsumerState) error { if state == nil || o.store == nil { return nil } err := o.store.Update(state) if err == nil { o.applyState(state) } return err } // Update our state to the store. func (o *consumer) writeStoreState() error { o.mu.Lock() defer o.mu.Unlock() return o.writeStoreStateUnlocked() } // Update our state to the store. // Lock should be held. func (o *consumer) writeStoreStateUnlocked() error { if o.store == nil { return nil } state := ConsumerState{ Delivered: SequencePair{ Consumer: o.dseq - 1, Stream: o.sseq - 1, }, AckFloor: SequencePair{ Consumer: o.adflr, Stream: o.asflr, }, Pending: o.pending, Redelivered: o.rdc, } return o.store.Update(&state) } // Returns an initial info. Only applicable for non-clustered consumers. // We will clear after we return it, so one shot. func (o *consumer) initialInfo() *ConsumerInfo { o.mu.Lock() ici := o.ici o.ici = nil // gc friendly o.mu.Unlock() if ici == nil { ici = o.info() } return ici } // Clears our initial info. // Used when we have a leader change in cluster mode but do not send a response. func (o *consumer) clearInitialInfo() { o.mu.Lock() o.ici = nil // gc friendly o.mu.Unlock() } // Info returns our current consumer state. func (o *consumer) info() *ConsumerInfo { return o.infoWithSnap(false) } func (o *consumer) infoWithSnap(snap bool) *ConsumerInfo { return o.infoWithSnapAndReply(snap, _EMPTY_) } func (o *consumer) infoWithSnapAndReply(snap bool, reply string) *ConsumerInfo { o.mu.Lock() mset := o.mset if o.closed || mset == nil || mset.srv == nil { o.mu.Unlock() return nil } js := o.js if js == nil { o.mu.Unlock() return nil } // Capture raftGroup. var rg *raftGroup if o.ca != nil { rg = o.ca.Group } cfg := o.cfg info := &ConsumerInfo{ Stream: o.stream, Name: o.name, Created: o.created, Config: &cfg, Delivered: SequenceInfo{ Consumer: o.dseq - 1, Stream: o.sseq - 1, }, AckFloor: SequenceInfo{ Consumer: o.adflr, Stream: o.asflr, }, NumAckPending: len(o.pending), NumRedelivered: len(o.rdc), NumPending: o.checkNumPending(), PushBound: o.isPushMode() && o.active, TimeStamp: time.Now().UTC(), } // If we are replicated, we need to pull certain data from our store. if rg != nil && rg.node != nil && o.store != nil { state, err := o.store.BorrowState() if err != nil { o.mu.Unlock() return nil } // If we are the leader we could have o.sseq that is skipped ahead. // To maintain consistency in reporting (e.g. jsz) we always take the state for our delivered/ackfloor stream sequence. info.Delivered.Consumer, info.Delivered.Stream = state.Delivered.Consumer, state.Delivered.Stream info.AckFloor.Consumer, info.AckFloor.Stream = state.AckFloor.Consumer, state.AckFloor.Stream if !o.isLeader() { info.NumAckPending = len(state.Pending) info.NumRedelivered = len(state.Redelivered) } } // Adjust active based on non-zero etc. Also make UTC here. if !o.ldt.IsZero() { ldt := o.ldt.UTC() // This copies as well. info.Delivered.Last = &ldt } if !o.lat.IsZero() { lat := o.lat.UTC() // This copies as well. info.AckFloor.Last = &lat } // If we are a pull mode consumer, report on number of waiting requests. if o.isPullMode() { o.processWaiting(false) info.NumWaiting = o.waiting.len() } // If we were asked to snapshot do so here. if snap { o.ici = info } sysc := o.sysc o.mu.Unlock() // Do cluster. if rg != nil { info.Cluster = js.clusterInfo(rg) } // If we have a reply subject send the response here. if reply != _EMPTY_ && sysc != nil { sysc.sendInternalMsg(reply, _EMPTY_, nil, info) } return info } // Will signal us that new messages are available. Will break out of waiting. func (o *consumer) signalNewMessages() { // Kick our new message channel select { case o.mch <- struct{}{}: default: } } // shouldSample lets us know if we are sampling metrics on acks. func (o *consumer) shouldSample() bool { switch { case o.sfreq <= 0: return false case o.sfreq >= 100: return true } // TODO(ripienaar) this is a tad slow so we need to rethink here, however this will only // hit for those with sampling enabled and its not the default return rand.Int31n(100) <= o.sfreq } func (o *consumer) sampleAck(sseq, dseq, dc uint64) { if !o.shouldSample() { return } now := time.Now().UTC() unow := now.UnixNano() e := JSConsumerAckMetric{ TypedEvent: TypedEvent{ Type: JSConsumerAckMetricType, ID: nuid.Next(), Time: now, }, Stream: o.stream, Consumer: o.name, ConsumerSeq: dseq, StreamSeq: sseq, Delay: unow - o.pending[sseq].Timestamp, Deliveries: dc, Domain: o.srv.getOpts().JetStreamDomain, } o.sendAdvisory(o.ackEventT, e) } // Process an ACK. // Returns `true` if the ack was processed in place and the sender can now respond // to the client, or `false` if there was an error or the ack is replicated (in which // case the reply will be sent later). func (o *consumer) processAckMsg(sseq, dseq, dc uint64, reply string, doSample bool) bool { o.mu.Lock() if o.closed { o.mu.Unlock() return false } mset := o.mset if mset == nil || mset.closed.Load() { o.mu.Unlock() return false } // Check if this ack is above the current pointer to our next to deliver. // This could happen on a cooperative takeover with high speed deliveries. if sseq >= o.sseq { // Let's make sure this is valid. // This is only received on the consumer leader, so should never be higher // than the last stream sequence. But could happen if we've just become // consumer leader, and we are not up-to-date on the stream yet. var ss StreamState mset.store.FastState(&ss) if sseq > ss.LastSeq { o.srv.Warnf("JetStream consumer '%s > %s > %s' ACK sequence %d past last stream sequence of %d", o.acc.Name, o.stream, o.name, sseq, ss.LastSeq) // FIXME(dlc) - For 2.11 onwards should we return an error here to the caller? } // Even though another leader must have delivered a message with this sequence, we must not adjust // the current pointer. This could otherwise result in a stuck consumer, where messages below this // sequence can't be redelivered, and we'll have incorrect pending state and ack floors. o.mu.Unlock() return false } // Let the owning stream know if we are interest or workqueue retention based. // If this consumer is clustered (o.node != nil) this will be handled by // processReplicatedAck after the ack has propagated. ackInPlace := o.node == nil && o.retention != LimitsPolicy var sgap, floor uint64 var needSignal bool switch o.cfg.AckPolicy { case AckExplicit: if p, ok := o.pending[sseq]; ok { if doSample { o.sampleAck(sseq, dseq, dc) } if o.maxp > 0 && len(o.pending) >= o.maxp { needSignal = true } delete(o.pending, sseq) // Use the original deliver sequence from our pending record. dseq = p.Sequence // Only move floors if we matched an existing pending. if len(o.pending) == 0 { o.adflr = o.dseq - 1 o.asflr = o.sseq - 1 } else if dseq == o.adflr+1 { o.adflr, o.asflr = dseq, sseq for ss := sseq + 1; ss < o.sseq; ss++ { if p, ok := o.pending[ss]; ok { if p.Sequence > 0 { o.adflr, o.asflr = p.Sequence-1, ss-1 } break } } } } delete(o.rdc, sseq) o.removeFromRedeliverQueue(sseq) case AckAll: // no-op if dseq <= o.adflr || sseq <= o.asflr { o.mu.Unlock() // Return true to let caller respond back to the client. return true } if o.maxp > 0 && len(o.pending) >= o.maxp { needSignal = true } sgap = sseq - o.asflr floor = sgap // start at same and set lower as we go. o.adflr, o.asflr = dseq, sseq remove := func(seq uint64) { delete(o.pending, seq) delete(o.rdc, seq) o.removeFromRedeliverQueue(seq) if seq < floor { floor = seq } } // Determine if smarter to walk all of pending vs the sequence range. if sgap > uint64(len(o.pending)) { for seq := range o.pending { if seq <= sseq { remove(seq) } } } else { for seq := sseq; seq > sseq-sgap && len(o.pending) > 0; seq-- { remove(seq) } } case AckNone: // FIXME(dlc) - This is error but do we care? o.mu.Unlock() return ackInPlace } // No ack replication, so we set reply to "" so that updateAcks does not // send the reply. The caller will. if ackInPlace { reply = _EMPTY_ } // Update underlying store. o.updateAcks(dseq, sseq, reply) o.mu.Unlock() if ackInPlace { if sgap > 1 { // FIXME(dlc) - This can very inefficient, will need to fix. for seq := sseq; seq >= floor; seq-- { mset.ackMsg(o, seq) } } else { mset.ackMsg(o, sseq) } } // If we had max ack pending set and were at limit we need to unblock ourselves. if needSignal { o.signalNewMessages() } return ackInPlace } // Determine if this is a truly filtered consumer. Modern clients will place filtered subjects // even if the stream only has a single non-wildcard subject designation. // Read lock should be held. func (o *consumer) isFiltered() bool { if o.subjf == nil { return false } // If we are here we want to check if the filtered subject is // a direct match for our only listed subject. mset := o.mset if mset == nil { return true } // Protect access to mset.cfg with the cfgMu mutex. mset.cfgMu.RLock() msetSubjects := mset.cfg.Subjects mset.cfgMu.RUnlock() // `isFiltered` need to be performant, so we do // as any checks as possible to avoid unnecessary work. // Here we avoid iteration over slices if there is only one subject in stream // and one filter for the consumer. if len(msetSubjects) == 1 && len(o.subjf) == 1 { return msetSubjects[0] != o.subjf[0].subject } // if the list is not equal length, we can return early, as this is filtered. if len(msetSubjects) != len(o.subjf) { return true } // if in rare case scenario that user passed all stream subjects as consumer filters, // we need to do a more expensive operation. // reflect.DeepEqual would return false if the filters are the same, but in different order // so it can't be used here. cfilters := make(map[string]struct{}, len(o.subjf)) for _, val := range o.subjf { cfilters[val.subject] = struct{}{} } for _, val := range msetSubjects { if _, ok := cfilters[val]; !ok { return true } } return false } // Check if we need an ack for this store seq. // This is called for interest based retention streams to remove messages. func (o *consumer) needAck(sseq uint64, subj string) bool { var needAck bool var asflr, osseq uint64 var pending map[uint64]*Pending var rdc map[uint64]uint64 o.mu.RLock() defer o.mu.RUnlock() isFiltered := o.isFiltered() if isFiltered && o.mset == nil { return false } // Check if we are filtered, and if so check if this is even applicable to us. if isFiltered { if subj == _EMPTY_ { var svp StoreMsg if _, err := o.mset.store.LoadMsg(sseq, &svp); err != nil { return false } subj = svp.subj } if !o.isFilteredMatch(subj) { return false } } if o.isLeader() { asflr, osseq = o.asflr, o.sseq pending, rdc = o.pending, o.rdc } else { if o.store == nil { return false } state, err := o.store.BorrowState() if err != nil || state == nil { // Fall back to what we track internally for now. return sseq > o.asflr && !o.isFiltered() } // If loading state as here, the osseq is +1. asflr, osseq, pending, rdc = state.AckFloor.Stream, state.Delivered.Stream+1, state.Pending, state.Redelivered } switch o.cfg.AckPolicy { case AckNone, AckAll: needAck = sseq > asflr case AckExplicit: if sseq > asflr { if sseq >= osseq { needAck = true } else { _, needAck = pending[sseq] } } } // Finally check if redelivery of this message is tracked. // If the message is not pending, it should be preserved if it reached max delivery. if !needAck { _, needAck = rdc[sseq] } return needAck } // Used in nextReqFromMsg, since the json.Unmarshal causes the request // struct to escape to the heap always. This should reduce GC pressure. var jsGetNextPool = sync.Pool{ New: func() any { return &JSApiConsumerGetNextRequest{} }, } // Helper for the next message requests. func nextReqFromMsg(msg []byte) (time.Time, int, int, bool, time.Duration, time.Time, error) { req := bytes.TrimSpace(msg) switch { case len(req) == 0: return time.Time{}, 1, 0, false, 0, time.Time{}, nil case req[0] == '{': cr := jsGetNextPool.Get().(*JSApiConsumerGetNextRequest) defer func() { *cr = JSApiConsumerGetNextRequest{} jsGetNextPool.Put(cr) }() if err := json.Unmarshal(req, &cr); err != nil { return time.Time{}, -1, 0, false, 0, time.Time{}, err } var hbt time.Time if cr.Heartbeat > 0 { if cr.Heartbeat*2 > cr.Expires { return time.Time{}, 1, 0, false, 0, time.Time{}, errors.New("heartbeat value too large") } hbt = time.Now().Add(cr.Heartbeat) } if cr.Expires == time.Duration(0) { return time.Time{}, cr.Batch, cr.MaxBytes, cr.NoWait, cr.Heartbeat, hbt, nil } return time.Now().Add(cr.Expires), cr.Batch, cr.MaxBytes, cr.NoWait, cr.Heartbeat, hbt, nil default: if n, err := strconv.Atoi(string(req)); err == nil { return time.Time{}, n, 0, false, 0, time.Time{}, nil } } return time.Time{}, 1, 0, false, 0, time.Time{}, nil } // Represents a request that is on the internal waiting queue type waitingRequest struct { next *waitingRequest acc *Account interest string reply string n int // For batching d int // num delivered b int // For max bytes tracking expires time.Time received time.Time hb time.Duration hbt time.Time noWait bool } // sync.Pool for waiting requests. var wrPool = sync.Pool{ New: func() any { return new(waitingRequest) }, } // Recycle this request. This request can not be accessed after this call. func (wr *waitingRequest) recycleIfDone() bool { if wr != nil && wr.n <= 0 { wr.recycle() return true } return false } // Force a recycle. func (wr *waitingRequest) recycle() { if wr != nil { wr.next, wr.acc, wr.interest, wr.reply = nil, nil, _EMPTY_, _EMPTY_ wrPool.Put(wr) } } // waiting queue for requests that are waiting for new messages to arrive. type waitQueue struct { n, max int last time.Time head *waitingRequest tail *waitingRequest } // Create a new ring buffer with at most max items. func newWaitQueue(max int) *waitQueue { return &waitQueue{max: max} } var ( errWaitQueueFull = errors.New("wait queue is full") errWaitQueueNil = errors.New("wait queue is nil") ) // Adds in a new request. func (wq *waitQueue) add(wr *waitingRequest) error { if wq == nil { return errWaitQueueNil } if wq.isFull() { return errWaitQueueFull } if wq.head == nil { wq.head = wr } else { wq.tail.next = wr } // Always set tail. wq.tail = wr // Make sure nil wr.next = nil // Track last active via when we receive a request. wq.last = wr.received wq.n++ return nil } func (wq *waitQueue) isFull() bool { if wq == nil { return false } return wq.n == wq.max } func (wq *waitQueue) isEmpty() bool { if wq == nil { return true } return wq.n == 0 } func (wq *waitQueue) len() int { if wq == nil { return 0 } return wq.n } // Peek will return the next request waiting or nil if empty. func (wq *waitQueue) peek() *waitingRequest { if wq == nil { return nil } return wq.head } // pop will return the next request and move the read cursor. // This will now place a request that still has pending items at the ends of the list. func (wq *waitQueue) pop() *waitingRequest { wr := wq.peek() if wr != nil { wr.d++ wr.n-- // Always remove current now on a pop, and move to end if still valid. // If we were the only one don't need to remove since this can be a no-op. if wr.n > 0 && wq.n > 1 { wq.removeCurrent() wq.add(wr) } else if wr.n <= 0 { wq.removeCurrent() } } return wr } // Removes the current read pointer (head FIFO) entry. func (wq *waitQueue) removeCurrent() { wq.remove(nil, wq.head) } // Remove the wr element from the wait queue. func (wq *waitQueue) remove(pre, wr *waitingRequest) { if wr == nil { return } if pre != nil { pre.next = wr.next } else if wr == wq.head { // We are removing head here. wq.head = wr.next } // Check if wr was our tail. if wr == wq.tail { // Check if we need to assign to pre. if wr.next == nil { wq.tail = pre } else { wq.tail = wr.next } } wq.n-- } // Return the map of pending requests keyed by the reply subject. // No-op if push consumer or invalid etc. func (o *consumer) pendingRequests() map[string]*waitingRequest { if o.waiting == nil { return nil } wq, m := o.waiting, make(map[string]*waitingRequest) for wr := wq.head; wr != nil; wr = wr.next { m[wr.reply] = wr } return m } // Return next waiting request. This will check for expirations but not noWait or interest. // That will be handled by processWaiting. // Lock should be held. func (o *consumer) nextWaiting(sz int) *waitingRequest { if o.waiting == nil || o.waiting.isEmpty() { return nil } for wr := o.waiting.peek(); !o.waiting.isEmpty(); wr = o.waiting.peek() { if wr == nil { break } // Check if we have max bytes set. if wr.b > 0 { if sz <= wr.b { wr.b -= sz // If we are right now at zero, set batch to 1 to deliver this one but stop after. if wr.b == 0 { wr.n = 1 } } else { // Since we can't send that message to the requestor, we need to // notify that we are closing the request. const maxBytesT = "NATS/1.0 409 Message Size Exceeds MaxBytes\r\n%s: %d\r\n%s: %d\r\n\r\n" hdr := fmt.Appendf(nil, maxBytesT, JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b) o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) // Remove the current one, no longer valid due to max bytes limit. o.waiting.removeCurrent() if o.node != nil { o.removeClusterPendingRequest(wr.reply) } wr.recycle() continue } } if wr.expires.IsZero() || time.Now().Before(wr.expires) { if wr.acc.sl.HasInterest(wr.interest) { return o.waiting.pop() } else if time.Since(wr.received) < defaultGatewayRecentSubExpiration && (o.srv.leafNodeEnabled || o.srv.gateway.enabled) { return o.waiting.pop() } else if o.srv.gateway.enabled && o.srv.hasGatewayInterest(wr.acc.Name, wr.interest) { return o.waiting.pop() } } else { // We do check for expiration in `processWaiting`, but it is possible to hit the expiry here, and not there. hdr := fmt.Appendf(nil, "NATS/1.0 408 Request Timeout\r\n%s: %d\r\n%s: %d\r\n\r\n", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b) o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) o.waiting.removeCurrent() if o.node != nil { o.removeClusterPendingRequest(wr.reply) } wr.recycle() continue } if wr.interest != wr.reply { const intExpT = "NATS/1.0 408 Interest Expired\r\n%s: %d\r\n%s: %d\r\n\r\n" hdr := fmt.Appendf(nil, intExpT, JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b) o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } // Remove the current one, no longer valid. o.waiting.removeCurrent() if o.node != nil { o.removeClusterPendingRequest(wr.reply) } wr.recycle() } return nil } // Next message request. type nextMsgReq struct { reply string msg []byte } var nextMsgReqPool sync.Pool func newNextMsgReq(reply string, msg []byte) *nextMsgReq { var nmr *nextMsgReq m := nextMsgReqPool.Get() if m != nil { nmr = m.(*nextMsgReq) } else { nmr = &nextMsgReq{} } // When getting something from a pool it is critical that all fields are // initialized. Doing this way guarantees that if someone adds a field to // the structure, the compiler will fail the build if this line is not updated. (*nmr) = nextMsgReq{reply, msg} return nmr } func (nmr *nextMsgReq) returnToPool() { if nmr == nil { return } nmr.reply, nmr.msg = _EMPTY_, nil nextMsgReqPool.Put(nmr) } // processNextMsgReq will process a request for the next message available. A nil message payload means deliver // a single message. If the payload is a formal request or a number parseable with Atoi(), then we will send a // batch of messages without requiring another request to this endpoint, or an ACK. func (o *consumer) processNextMsgReq(_ *subscription, c *client, _ *Account, _, reply string, msg []byte) { if reply == _EMPTY_ { return } // Short circuit error here. if o.nextMsgReqs == nil { hdr := []byte("NATS/1.0 409 Consumer is push based\r\n\r\n") o.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } _, msg = c.msgParts(msg) o.nextMsgReqs.push(newNextMsgReq(reply, copyBytes(msg))) } func (o *consumer) processNextMsgRequest(reply string, msg []byte) { o.mu.Lock() defer o.mu.Unlock() mset := o.mset if mset == nil { return } sendErr := func(status int, description string) { hdr := fmt.Appendf(nil, "NATS/1.0 %d %s\r\n\r\n", status, description) o.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } if o.isPushMode() || o.waiting == nil { sendErr(409, "Consumer is push based") return } // Check payload here to see if they sent in batch size or a formal request. expires, batchSize, maxBytes, noWait, hb, hbt, err := nextReqFromMsg(msg) if err != nil { sendErr(400, fmt.Sprintf("Bad Request - %v", err)) return } // Check for request limits if o.cfg.MaxRequestBatch > 0 && batchSize > o.cfg.MaxRequestBatch { sendErr(409, fmt.Sprintf("Exceeded MaxRequestBatch of %d", o.cfg.MaxRequestBatch)) return } if !expires.IsZero() && o.cfg.MaxRequestExpires > 0 && expires.After(time.Now().Add(o.cfg.MaxRequestExpires)) { sendErr(409, fmt.Sprintf("Exceeded MaxRequestExpires of %v", o.cfg.MaxRequestExpires)) return } if maxBytes > 0 && o.cfg.MaxRequestMaxBytes > 0 && maxBytes > o.cfg.MaxRequestMaxBytes { sendErr(409, fmt.Sprintf("Exceeded MaxRequestMaxBytes of %v", o.cfg.MaxRequestMaxBytes)) return } // If we have the max number of requests already pending try to expire. if o.waiting.isFull() { // Try to expire some of the requests. // We do not want to push too hard here so at maximum process once per sec. if time.Since(o.lwqic) > time.Second { o.processWaiting(false) } } // If the request is for noWait and we have pending requests already, check if we have room. if noWait { msgsPending := o.numPending() + uint64(len(o.rdq)) // If no pending at all, decide what to do with request. // If no expires was set then fail. if msgsPending == 0 && expires.IsZero() { o.waiting.last = time.Now() sendErr(404, "No Messages") return } if msgsPending > 0 { _, _, batchPending, _ := o.processWaiting(false) if msgsPending < uint64(batchPending) { o.waiting.last = time.Now() sendErr(408, "Requests Pending") return } } // If we are here this should be considered a one-shot situation. // We will wait for expires but will return as soon as we have any messages. } // If we receive this request though an account export, we need to track that interest subject and account. acc, interest := trackDownAccountAndInterest(o.acc, reply) // Create a waiting request. wr := wrPool.Get().(*waitingRequest) wr.acc, wr.interest, wr.reply, wr.n, wr.d, wr.noWait, wr.expires, wr.hb, wr.hbt = acc, interest, reply, batchSize, 0, noWait, expires, hb, hbt wr.b = maxBytes wr.received = time.Now() if err := o.waiting.add(wr); err != nil { sendErr(409, "Exceeded MaxWaiting") wr.recycle() return } o.signalNewMessages() // If we are clustered update our followers about this request. if o.node != nil { o.addClusterPendingRequest(wr.reply) } } func trackDownAccountAndInterest(acc *Account, interest string) (*Account, string) { for strings.HasPrefix(interest, replyPrefix) { oa := acc oa.mu.RLock() if oa.exports.responses == nil { oa.mu.RUnlock() break } si := oa.exports.responses[interest] if si == nil { oa.mu.RUnlock() break } acc, interest = si.acc, si.to oa.mu.RUnlock() } return acc, interest } // Return current delivery count for a given sequence. func (o *consumer) deliveryCount(seq uint64) uint64 { if o.rdc == nil { return 1 } if dc := o.rdc[seq]; dc >= 1 { return dc } return 1 } // Increase the delivery count for this message. // ONLY used on redelivery semantics. // Lock should be held. func (o *consumer) incDeliveryCount(sseq uint64) uint64 { if o.rdc == nil { o.rdc = make(map[uint64]uint64) } o.rdc[sseq] += 1 return o.rdc[sseq] + 1 } // Used if we have to adjust on failed delivery or bad lookups. // Those failed attempts should not increase deliver count. // Lock should be held. func (o *consumer) decDeliveryCount(sseq uint64) { if o.rdc == nil { o.rdc = make(map[uint64]uint64) } o.rdc[sseq] -= 1 } // send a delivery exceeded advisory. func (o *consumer) notifyDeliveryExceeded(sseq, dc uint64) { e := JSConsumerDeliveryExceededAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerDeliveryExceededAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: o.stream, Consumer: o.name, StreamSeq: sseq, Deliveries: dc, Domain: o.srv.getOpts().JetStreamDomain, } o.sendAdvisory(o.deliveryExcEventT, e) } // Check if the candidate subject matches a filter if its present. // Lock should be held. func (o *consumer) isFilteredMatch(subj string) bool { // No filter is automatic match. if o.subjf == nil { return true } for _, filter := range o.subjf { if !filter.hasWildcard && subj == filter.subject { return true } } // It's quicker to first check for non-wildcard filters, then // iterate again to check for subset match. tsa := [32]string{} tts := tokenizeSubjectIntoSlice(tsa[:0], subj) for _, filter := range o.subjf { if isSubsetMatchTokenized(tts, filter.tokenizedSubject) { return true } } return false } // Check if the candidate filter subject is equal to or a subset match // of one of the filter subjects. // Lock should be held. func (o *consumer) isEqualOrSubsetMatch(subj string) bool { for _, filter := range o.subjf { if !filter.hasWildcard && subj == filter.subject { return true } } tsa := [32]string{} tts := tokenizeSubjectIntoSlice(tsa[:0], subj) for _, filter := range o.subjf { if isSubsetMatchTokenized(filter.tokenizedSubject, tts) { return true } } return false } var ( errMaxAckPending = errors.New("max ack pending reached") errBadConsumer = errors.New("consumer not valid") errNoInterest = errors.New("consumer requires interest for delivery subject when ephemeral") ) // Get next available message from underlying store. // Is partition aware and redeliver aware. // Lock should be held. func (o *consumer) getNextMsg() (*jsPubMsg, uint64, error) { if o.mset == nil || o.mset.store == nil { return nil, 0, errBadConsumer } // Process redelivered messages before looking at possibly "skip list" (deliver last per subject) if o.hasRedeliveries() { var seq, dc uint64 for seq = o.getNextToRedeliver(); seq > 0; seq = o.getNextToRedeliver() { dc = o.incDeliveryCount(seq) if o.maxdc > 0 && dc > o.maxdc { // Only send once if dc == o.maxdc+1 { o.notifyDeliveryExceeded(seq, dc-1) } // Make sure to remove from pending. if p, ok := o.pending[seq]; ok && p != nil { delete(o.pending, seq) o.updateDelivered(p.Sequence, seq, dc, p.Timestamp) } continue } pmsg := getJSPubMsgFromPool() sm, err := o.mset.store.LoadMsg(seq, &pmsg.StoreMsg) if sm == nil || err != nil { pmsg.returnToPool() pmsg, dc = nil, 0 // Adjust back deliver count. o.decDeliveryCount(seq) } // Message was scheduled for redelivery but was removed in the meantime. if err == ErrStoreMsgNotFound || err == errDeletedMsg { // This is a race condition where the message is still in o.pending and // scheduled for redelivery, but it has been removed from the stream. // o.processTerm is called in a goroutine so could run after we get here. // That will correct the pending state and delivery/ack floors, so just skip here. continue } return pmsg, dc, err } } // Check if we have max pending. if o.maxp > 0 && len(o.pending) >= o.maxp { // maxp only set when ack policy != AckNone and user set MaxAckPending // Stall if we have hit max pending. return nil, 0, errMaxAckPending } if o.hasSkipListPending() { seq := o.lss.seqs[0] if len(o.lss.seqs) == 1 { o.sseq = o.lss.resume o.lss = nil o.updateSkipped(o.sseq) } else { o.lss.seqs = o.lss.seqs[1:] } pmsg := getJSPubMsgFromPool() sm, err := o.mset.store.LoadMsg(seq, &pmsg.StoreMsg) if sm == nil || err != nil { pmsg.returnToPool() } o.sseq++ return pmsg, 1, err } // Hold onto this since we release the lock. store := o.mset.store var sseq uint64 var err error var sm *StoreMsg var pmsg = getJSPubMsgFromPool() // Grab next message applicable to us. filters, subjf, fseq := o.filters, o.subjf, o.sseq // Check if we are multi-filtered or not. if filters != nil { sm, sseq, err = store.LoadNextMsgMulti(filters, fseq, &pmsg.StoreMsg) } else if len(subjf) > 0 { // Means single filtered subject since o.filters means > 1. filter, wc := subjf[0].subject, subjf[0].hasWildcard sm, sseq, err = store.LoadNextMsg(filter, wc, fseq, &pmsg.StoreMsg) } else { // No filter here. sm, sseq, err = store.LoadNextMsg(_EMPTY_, false, fseq, &pmsg.StoreMsg) } if sm == nil { pmsg.returnToPool() pmsg = nil } // Check if we should move our o.sseq. if sseq >= o.sseq { // If we are moving step by step then sseq == o.sseq. // If we have jumped we should update skipped for other replicas. if sseq != o.sseq && err == ErrStoreEOF { o.updateSkipped(sseq + 1) } o.sseq = sseq + 1 } return pmsg, 1, err } // Will check for expiration and lack of interest on waiting requests. // Will also do any heartbeats and return the next expiration or HB interval. func (o *consumer) processWaiting(eos bool) (int, int, int, time.Time) { var fexp time.Time if o.srv == nil || o.waiting.isEmpty() { return 0, 0, 0, fexp } // Mark our last check time. o.lwqic = time.Now() var expired, brp int s, now := o.srv, time.Now() wq := o.waiting remove := func(pre, wr *waitingRequest) *waitingRequest { expired++ if o.node != nil { o.removeClusterPendingRequest(wr.reply) } next := wr.next wq.remove(pre, wr) wr.recycle() return next } var pre *waitingRequest for wr := wq.head; wr != nil; { // Check expiration. if (eos && wr.noWait && wr.d > 0) || (!wr.expires.IsZero() && now.After(wr.expires)) { hdr := fmt.Appendf(nil, "NATS/1.0 408 Request Timeout\r\n%s: %d\r\n%s: %d\r\n\r\n", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b) o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) wr = remove(pre, wr) continue } // Now check interest. interest := wr.acc.sl.HasInterest(wr.interest) if !interest && (s.leafNodeEnabled || s.gateway.enabled) { // If we are here check on gateways and leaf nodes (as they can mask gateways on the other end). // If we have interest or the request is too young break and do not expire. if time.Since(wr.received) < defaultGatewayRecentSubExpiration { interest = true } else if s.gateway.enabled && s.hasGatewayInterest(wr.acc.Name, wr.interest) { interest = true } } // Check if we have interest. if !interest { // No more interest here so go ahead and remove this one from our list. wr = remove(pre, wr) continue } // If interest, update batch pending requests counter and update fexp timer. brp += wr.n if !wr.hbt.IsZero() { if now.After(wr.hbt) { // Fire off a heartbeat here. o.sendIdleHeartbeat(wr.reply) // Update next HB. wr.hbt = now.Add(wr.hb) } if fexp.IsZero() || wr.hbt.Before(fexp) { fexp = wr.hbt } } if !wr.expires.IsZero() && (fexp.IsZero() || wr.expires.Before(fexp)) { fexp = wr.expires } // Update pre and wr here. pre = wr wr = wr.next } return expired, wq.len(), brp, fexp } // Will check to make sure those waiting still have registered interest. func (o *consumer) checkWaitingForInterest() bool { o.processWaiting(true) return o.waiting.len() > 0 } // Lock should be held. func (o *consumer) hbTimer() (time.Duration, *time.Timer) { if o.cfg.Heartbeat == 0 { return 0, nil } return o.cfg.Heartbeat, time.NewTimer(o.cfg.Heartbeat) } // Check here for conditions when our ack floor may have drifted below the streams first sequence. // In general this is accounted for in normal operations, but if the consumer misses the signal from // the stream it will not clear the message and move the ack state. // Should only be called from consumer leader. func (o *consumer) checkAckFloor() { o.mu.RLock() mset, closed, asflr, numPending := o.mset, o.closed, o.asflr, len(o.pending) o.mu.RUnlock() if asflr == 0 || closed || mset == nil { return } var ss StreamState mset.store.FastState(&ss) // If our floor is equal or greater that is normal and nothing for us to do. if ss.FirstSeq == 0 || asflr >= ss.FirstSeq-1 { return } // Check which linear space is less to walk. if ss.FirstSeq-asflr-1 < uint64(numPending) { // Process all messages that no longer exist. for seq := asflr + 1; seq < ss.FirstSeq; seq++ { // Check if this message was pending. o.mu.RLock() p, isPending := o.pending[seq] rdc := o.deliveryCount(seq) o.mu.RUnlock() // If it was pending for us, get rid of it. if isPending { o.processTerm(seq, p.Sequence, rdc, ackTermLimitsReason, _EMPTY_) } } } else if numPending > 0 { // here it is shorter to walk pending. // toTerm is seq, dseq, rcd for each entry. toTerm := make([]uint64, 0, numPending*3) o.mu.RLock() for seq, p := range o.pending { if seq < ss.FirstSeq { var dseq uint64 = 1 if p != nil { dseq = p.Sequence } rdc := o.deliveryCount(seq) toTerm = append(toTerm, seq, dseq, rdc) } } o.mu.RUnlock() for i := 0; i < len(toTerm); i += 3 { seq, dseq, rdc := toTerm[i], toTerm[i+1], toTerm[i+2] o.processTerm(seq, dseq, rdc, ackTermLimitsReason, _EMPTY_) } } // Do one final check here. o.mu.Lock() defer o.mu.Unlock() // If we are closed do not change anything and simply return. if o.closed { return } // If we are here, and this should be rare, we still are off with our ack floor. // We will make sure we are not doing un-necessary work here if only off by a bit // since this could be normal for a high activity wq or stream. // We will set it explicitly to 1 behind our current lowest in pending, or if // pending is empty, to our current delivered -1. const minOffThreshold = 50 if ss.FirstSeq >= minOffThreshold && o.asflr < ss.FirstSeq-minOffThreshold { var psseq, pdseq uint64 for seq, p := range o.pending { if psseq == 0 || seq < psseq { psseq, pdseq = seq, p.Sequence } } // If we still have none, set to current delivered -1. if psseq == 0 { psseq, pdseq = o.sseq-1, o.dseq-1 // If still not adjusted. if psseq < ss.FirstSeq-1 { psseq = ss.FirstSeq - 1 } } else { // Since this was set via the pending, we should not include // it directly but set floors to -1. psseq, pdseq = psseq-1, pdseq-1 } o.asflr, o.adflr = psseq, pdseq } } func (o *consumer) processInboundAcks(qch chan struct{}) { // Grab the server lock to watch for server quit. o.mu.RLock() s, mset := o.srv, o.mset hasInactiveThresh := o.cfg.InactiveThreshold > 0 o.mu.RUnlock() if s == nil || mset == nil { return } // We will check this on entry and periodically. o.checkAckFloor() // How often we will check for ack floor drift. // Spread these out for large numbers on a server restart. delta := time.Duration(rand.Int63n(int64(time.Minute))) ticker := time.NewTicker(time.Minute + delta) defer ticker.Stop() for { select { case <-o.ackMsgs.ch: acks := o.ackMsgs.pop() for _, ack := range acks { o.processAck(ack.subject, ack.reply, ack.hdr, ack.msg) ack.returnToPool() } o.ackMsgs.recycle(&acks) // If we have an inactiveThreshold set, mark our activity. if hasInactiveThresh { o.suppressDeletion() } case <-ticker.C: o.checkAckFloor() case <-qch: return case <-s.quitCh: return } } } // Process inbound next message requests. func (o *consumer) processInboundNextMsgReqs(qch chan struct{}) { // Grab the server lock to watch for server quit. o.mu.RLock() s := o.srv o.mu.RUnlock() for { select { case <-o.nextMsgReqs.ch: reqs := o.nextMsgReqs.pop() for _, req := range reqs { o.processNextMsgRequest(req.reply, req.msg) req.returnToPool() } o.nextMsgReqs.recycle(&reqs) case <-qch: return case <-s.quitCh: return } } } // Suppress auto cleanup on ack activity of any kind. func (o *consumer) suppressDeletion() { o.mu.Lock() defer o.mu.Unlock() if o.closed { return } if o.isPushMode() && o.dtmr != nil { // if dtmr is not nil we have started the countdown, simply reset to threshold. o.dtmr.Reset(o.dthresh) } else if o.isPullMode() && o.waiting != nil { // Pull mode always has timer running, just update last on waiting queue. o.waiting.last = time.Now() } } func (o *consumer) loopAndGatherMsgs(qch chan struct{}) { // On startup check to see if we are in a reply situation where replay policy is not instant. var ( lts int64 // last time stamp seen, used for replay. lseq uint64 ) o.mu.RLock() mset := o.mset getLSeq := o.replay o.mu.RUnlock() // consumer is closed when mset is set to nil. if mset == nil { return } if getLSeq { lseq = mset.state().LastSeq } o.mu.Lock() s := o.srv // need to check again if consumer is closed if o.mset == nil { o.mu.Unlock() return } // For idle heartbeat support. var hbc <-chan time.Time hbd, hb := o.hbTimer() if hb != nil { hbc = hb.C } // Interest changes. inch := o.inch o.mu.Unlock() // Grab the stream's retention policy and name mset.cfgMu.RLock() stream, rp := mset.cfg.Name, mset.cfg.Retention mset.cfgMu.RUnlock() var err error // Deliver all the msgs we have now, once done or on a condition, we wait for new ones. for { var ( pmsg *jsPubMsg dc uint64 dsubj string ackReply string delay time.Duration sz int wrn, wrb int ) o.mu.Lock() // consumer is closed when mset is set to nil. if o.closed || o.mset == nil { o.mu.Unlock() return } // Clear last error. err = nil // If we are in push mode and not active or under flowcontrol let's stop sending. if o.isPushMode() { if !o.active || (o.maxpb > 0 && o.pbytes > o.maxpb) { goto waitForMsgs } } else if o.waiting.isEmpty() { // If we are in pull mode and no one is waiting already break and wait. goto waitForMsgs } // Grab our next msg. pmsg, dc, err = o.getNextMsg() // We can release the lock now under getNextMsg so need to check this condition again here. if o.closed || o.mset == nil { o.mu.Unlock() return } // On error either wait or return. if err != nil || pmsg == nil { // On EOF we can optionally fast sync num pending state. if err == ErrStoreEOF { o.checkNumPendingOnEOF() } if err == ErrStoreMsgNotFound || err == errDeletedMsg || err == ErrStoreEOF || err == errMaxAckPending { goto waitForMsgs } else if err == errPartialCache { s.Warnf("Unexpected partial cache error looking up message for consumer '%s > %s > %s'", o.mset.acc, stream, o.cfg.Name) goto waitForMsgs } else { s.Errorf("Received an error looking up message for consumer '%s > %s > %s': %v", o.mset.acc, stream, o.cfg.Name, err) goto waitForMsgs } } // Update our cached num pending here first. if dc == 1 { o.npc-- } // Pre-calculate ackReply ackReply = o.ackReply(pmsg.seq, o.dseq, dc, pmsg.ts, o.numPending()) // If headers only do not send msg payload. // Add in msg size itself as header. if o.cfg.HeadersOnly { convertToHeadersOnly(pmsg) } // Calculate payload size. This can be calculated on client side. // We do not include transport subject here since not generally known on client. sz = len(pmsg.subj) + len(ackReply) + len(pmsg.hdr) + len(pmsg.msg) if o.isPushMode() { dsubj = o.dsubj } else if wr := o.nextWaiting(sz); wr != nil { wrn, wrb = wr.n, wr.b dsubj = wr.reply if done := wr.recycleIfDone(); done && o.node != nil { o.removeClusterPendingRequest(dsubj) } else if !done && wr.hb > 0 { wr.hbt = time.Now().Add(wr.hb) } } else { // We will redo this one as long as this is not a redelivery. // Need to also test that this is not going backwards since if // we fail to deliver we can end up here from rdq but we do not // want to decrement o.sseq if that is the case. if dc == 1 && pmsg.seq == o.sseq-1 { o.sseq-- o.npc++ } else if !o.onRedeliverQueue(pmsg.seq) { // We are not on the rdq so decrement the delivery count // and add it back. o.decDeliveryCount(pmsg.seq) o.addToRedeliverQueue(pmsg.seq) } pmsg.returnToPool() goto waitForMsgs } // If we are in a replay scenario and have not caught up check if we need to delay here. if o.replay && lts > 0 { if delay = time.Duration(pmsg.ts - lts); delay > time.Millisecond { o.mu.Unlock() select { case <-qch: pmsg.returnToPool() return case <-time.After(delay): } o.mu.Lock() } } // Track this regardless. lts = pmsg.ts // If we have a rate limit set make sure we check that here. if o.rlimit != nil { now := time.Now() r := o.rlimit.ReserveN(now, sz) delay := r.DelayFrom(now) if delay > 0 { o.mu.Unlock() select { case <-qch: pmsg.returnToPool() return case <-time.After(delay): } o.mu.Lock() } } // Do actual delivery. o.deliverMsg(dsubj, ackReply, pmsg, dc, rp) // If given request fulfilled batch size, but there are still pending bytes, send information about it. if wrn <= 0 && wrb > 0 { msg := fmt.Appendf(nil, JsPullRequestRemainingBytesT, JSPullRequestPendingMsgs, wrn, JSPullRequestPendingBytes, wrb) o.outq.send(newJSPubMsg(dsubj, _EMPTY_, _EMPTY_, msg, nil, nil, 0)) } // Reset our idle heartbeat timer if set. if hb != nil { hb.Reset(hbd) } o.mu.Unlock() continue waitForMsgs: // If we were in a replay state check to see if we are caught up. If so clear. if o.replay && o.sseq > lseq { o.replay = false } // Make sure to process any expired requests that are pending. var wrExp <-chan time.Time if o.isPullMode() { // Dont expire oneshots if we are here because of max ack pending limit. _, _, _, fexp := o.processWaiting(err != errMaxAckPending) if !fexp.IsZero() { expires := time.Until(fexp) if expires <= 0 { expires = time.Millisecond } wrExp = time.NewTimer(expires).C } } // We will wait here for new messages to arrive. mch, odsubj := o.mch, o.cfg.DeliverSubject o.mu.Unlock() select { case <-mch: // Messages are waiting. case interest := <-inch: // inch can be nil on pull-based, but then this will // just block and not fire. o.updateDeliveryInterest(interest) case <-qch: return case <-wrExp: o.mu.Lock() o.processWaiting(true) o.mu.Unlock() case <-hbc: if o.isActive() { o.mu.RLock() o.sendIdleHeartbeat(odsubj) o.mu.RUnlock() } // Reset our idle heartbeat timer. hb.Reset(hbd) } } } // Lock should be held. func (o *consumer) sendIdleHeartbeat(subj string) { const t = "NATS/1.0 100 Idle Heartbeat\r\n%s: %d\r\n%s: %d\r\n\r\n" sseq, dseq := o.sseq-1, o.dseq-1 hdr := fmt.Appendf(nil, t, JSLastConsumerSeq, dseq, JSLastStreamSeq, sseq) if fcp := o.fcid; fcp != _EMPTY_ { // Add in that we are stalled on flow control here. addOn := fmt.Appendf(nil, "%s: %s\r\n\r\n", JSConsumerStalled, fcp) hdr = append(hdr[:len(hdr)-LEN_CR_LF], []byte(addOn)...) } o.outq.send(newJSPubMsg(subj, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } func (o *consumer) ackReply(sseq, dseq, dc uint64, ts int64, pending uint64) string { return fmt.Sprintf(o.ackReplyT, dc, sseq, dseq, ts, pending) } // Used mostly for testing. Sets max pending bytes for flow control setups. func (o *consumer) setMaxPendingBytes(limit int) { o.pblimit = limit o.maxpb = limit / 16 if o.maxpb == 0 { o.maxpb = 1 } } // Does some sanity checks to see if we should re-calculate. // Since there is a race when decrementing when there is contention at the beginning of the stream. // The race is a getNextMsg skips a deleted msg, and then the decStreamPending call fires. // This does some quick sanity checks to see if we should re-calculate num pending. // Lock should be held. func (o *consumer) checkNumPending() uint64 { if o.mset != nil { var state StreamState o.mset.store.FastState(&state) npc := o.numPending() if o.sseq > state.LastSeq && npc > 0 || npc > state.Msgs { // Re-calculate. o.streamNumPending() } } return o.numPending() } // Lock should be held. func (o *consumer) numPending() uint64 { if o.npc < 0 { return 0 } return uint64(o.npc) } // This will do a quick sanity check on num pending when we encounter // and EOF in the loop and gather. // Lock should be held. func (o *consumer) checkNumPendingOnEOF() { if o.mset == nil { return } var state StreamState o.mset.store.FastState(&state) if o.sseq > state.LastSeq && o.npc != 0 { // We know here we can reset our running state for num pending. o.npc, o.npf = 0, state.LastSeq } } // Call into streamNumPending after acquiring the consumer lock. func (o *consumer) streamNumPendingLocked() uint64 { o.mu.Lock() defer o.mu.Unlock() return o.streamNumPending() } // Will force a set from the stream store of num pending. // Depends on delivery policy, for last per subject we calculate differently. // Lock should be held. func (o *consumer) streamNumPending() uint64 { if o.mset == nil || o.mset.store == nil { o.npc, o.npf = 0, 0 return 0 } npc, npf := o.calculateNumPending() o.npc, o.npf = int64(npc), npf return o.numPending() } // Will calculate num pending but only requires a read lock. // Depends on delivery policy, for last per subject we calculate differently. // At least RLock should be held. func (o *consumer) calculateNumPending() (npc, npf uint64) { if o.mset == nil || o.mset.store == nil { return 0, 0 } isLastPerSubject := o.cfg.DeliverPolicy == DeliverLastPerSubject filters, subjf := o.filters, o.subjf if filters != nil { return o.mset.store.NumPendingMulti(o.sseq, filters, isLastPerSubject) } else if len(subjf) > 0 { filter := subjf[0].subject return o.mset.store.NumPending(o.sseq, filter, isLastPerSubject) } return o.mset.store.NumPending(o.sseq, _EMPTY_, isLastPerSubject) } func convertToHeadersOnly(pmsg *jsPubMsg) { // If headers only do not send msg payload. // Add in msg size itself as header. hdr, msg := pmsg.hdr, pmsg.msg var bb bytes.Buffer if len(hdr) == 0 { bb.WriteString(hdrLine) } else { bb.Write(hdr) bb.Truncate(len(hdr) - LEN_CR_LF) } bb.WriteString(JSMsgSize) bb.WriteString(": ") bb.WriteString(strconv.FormatInt(int64(len(msg)), 10)) bb.WriteString(CR_LF) bb.WriteString(CR_LF) // Replace underlying buf which we can use directly when we send. // TODO(dlc) - Probably just use directly when forming bytes.Buffer? pmsg.buf = pmsg.buf[:0] pmsg.buf = append(pmsg.buf, bb.Bytes()...) // Replace with new header. pmsg.hdr = pmsg.buf // Cancel msg payload pmsg.msg = nil } // Deliver a msg to the consumer. // Lock should be held and o.mset validated to be non-nil. func (o *consumer) deliverMsg(dsubj, ackReply string, pmsg *jsPubMsg, dc uint64, rp RetentionPolicy) { if o.mset == nil { pmsg.returnToPool() return } dseq := o.dseq o.dseq++ pmsg.dsubj, pmsg.reply, pmsg.o = dsubj, ackReply, o psz := pmsg.size() if o.maxpb > 0 { o.pbytes += psz } mset := o.mset ap := o.cfg.AckPolicy // Cant touch pmsg after this sending so capture what we need. seq, ts := pmsg.seq, pmsg.ts // Update delivered first. o.updateDelivered(dseq, seq, dc, ts) // Send message. o.outq.send(pmsg) if ap == AckExplicit || ap == AckAll { o.trackPending(seq, dseq) } else if ap == AckNone { o.adflr = dseq o.asflr = seq } // Flow control. if o.maxpb > 0 && o.needFlowControl(psz) { o.sendFlowControl() } // If pull mode and we have inactivity threshold, signaled by dthresh, update last activity. if o.isPullMode() && o.dthresh > 0 { o.waiting.last = time.Now() } // If we are ack none and mset is interest only we should make sure stream removes interest. if ap == AckNone && rp != LimitsPolicy { if mset != nil && mset.ackq != nil && (o.node == nil || o.cfg.Direct) { mset.ackq.push(seq) } else { o.updateAcks(dseq, seq, _EMPTY_) } } } func (o *consumer) needFlowControl(sz int) bool { if o.maxpb == 0 { return false } // Decide whether to send a flow control message which we will need the user to respond. // We send when we are over 50% of our current window limit. if o.fcid == _EMPTY_ && o.pbytes > o.maxpb/2 { return true } // If we have an existing outstanding FC, check to see if we need to expand the o.fcsz if o.fcid != _EMPTY_ && (o.pbytes-o.fcsz) >= o.maxpb { o.fcsz += sz } return false } func (o *consumer) processFlowControl(_ *subscription, c *client, _ *Account, subj, _ string, _ []byte) { o.mu.Lock() defer o.mu.Unlock() // Ignore if not the latest we have sent out. if subj != o.fcid { return } // For slow starts and ramping up. if o.maxpb < o.pblimit { o.maxpb *= 2 if o.maxpb > o.pblimit { o.maxpb = o.pblimit } } // Update accounting. o.pbytes -= o.fcsz if o.pbytes < 0 { o.pbytes = 0 } o.fcid, o.fcsz = _EMPTY_, 0 o.signalNewMessages() } // Lock should be held. func (o *consumer) fcReply() string { var sb strings.Builder sb.WriteString(jsFlowControlPre) sb.WriteString(o.stream) sb.WriteByte(btsep) sb.WriteString(o.name) sb.WriteByte(btsep) var b [4]byte rn := rand.Int63() for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } sb.Write(b[:]) return sb.String() } // sendFlowControl will send a flow control packet to the consumer. // Lock should be held. func (o *consumer) sendFlowControl() { if !o.isPushMode() { return } subj, rply := o.cfg.DeliverSubject, o.fcReply() o.fcsz, o.fcid = o.pbytes, rply hdr := []byte("NATS/1.0 100 FlowControl Request\r\n\r\n") o.outq.send(newJSPubMsg(subj, _EMPTY_, rply, hdr, nil, nil, 0)) } // Tracks our outstanding pending acks. Only applicable to AckExplicit mode. // Lock should be held. func (o *consumer) trackPending(sseq, dseq uint64) { if o.pending == nil { o.pending = make(map[uint64]*Pending) } // We could have a backoff that set a timer higher than what we need for this message. // In that case, reset to lowest backoff required for a message redelivery. minDelay := o.ackWait(0) if l := len(o.cfg.BackOff); l > 0 { bi := int(o.rdc[sseq]) if bi < 0 { bi = 0 } else if bi >= l { bi = l - 1 } minDelay = o.ackWait(o.cfg.BackOff[bi]) } minDeadline := time.Now().Add(minDelay) if o.ptmr == nil || o.ptmrEnd.After(minDeadline) { o.resetPtmr(minDelay) } if p, ok := o.pending[sseq]; ok { // Update timestamp but keep original consumer delivery sequence. // So do not update p.Sequence. p.Timestamp = time.Now().UnixNano() } else { o.pending[sseq] = &Pending{dseq, time.Now().UnixNano()} } } // Credit back a failed delivery. // lock should be held. func (o *consumer) creditWaitingRequest(reply string) { wq := o.waiting for wr := wq.head; wr != nil; wr = wr.next { if wr.reply == reply { wr.n++ wr.d-- return } } } // didNotDeliver is called when a delivery for a consumer message failed. // Depending on our state, we will process the failure. func (o *consumer) didNotDeliver(seq uint64, subj string) { o.mu.Lock() mset := o.mset if mset == nil { o.mu.Unlock() return } // Adjust back deliver count. o.decDeliveryCount(seq) var checkDeliveryInterest bool if o.isPushMode() { o.active = false checkDeliveryInterest = true } else if o.pending != nil { // Good chance we did not deliver because no interest so force a check. o.processWaiting(false) // If it is still there credit it back. o.creditWaitingRequest(subj) // pull mode and we have pending. if _, ok := o.pending[seq]; ok { // We found this messsage on pending, we need // to queue it up for immediate redelivery since // we know it was not delivered if !o.onRedeliverQueue(seq) { o.addToRedeliverQueue(seq) if !o.waiting.isEmpty() { o.signalNewMessages() } } } } o.mu.Unlock() // If we do not have interest update that here. if checkDeliveryInterest && o.hasNoLocalInterest() { o.updateDeliveryInterest(false) } } // Lock should be held. func (o *consumer) addToRedeliverQueue(seqs ...uint64) { o.rdq = append(o.rdq, seqs...) for _, seq := range seqs { o.rdqi.Insert(seq) } } // Lock should be held. func (o *consumer) hasRedeliveries() bool { return len(o.rdq) > 0 } func (o *consumer) getNextToRedeliver() uint64 { if len(o.rdq) == 0 { return 0 } seq := o.rdq[0] if len(o.rdq) == 1 { o.rdq = nil o.rdqi.Empty() } else { o.rdq = append(o.rdq[:0], o.rdq[1:]...) o.rdqi.Delete(seq) } return seq } // This checks if we already have this sequence queued for redelivery. // FIXME(dlc) - This is O(n) but should be fast with small redeliver size. // Lock should be held. func (o *consumer) onRedeliverQueue(seq uint64) bool { return o.rdqi.Exists(seq) } // Remove a sequence from the redelivery queue. // Lock should be held. func (o *consumer) removeFromRedeliverQueue(seq uint64) bool { if !o.onRedeliverQueue(seq) { return false } for i, rseq := range o.rdq { if rseq == seq { if len(o.rdq) == 1 { o.rdq = nil o.rdqi.Empty() } else { o.rdq = append(o.rdq[:i], o.rdq[i+1:]...) o.rdqi.Delete(seq) } return true } } return false } // Checks the pending messages. func (o *consumer) checkPending() { o.mu.Lock() defer o.mu.Unlock() mset := o.mset // On stop, mset and timer will be nil. if o.closed || mset == nil || o.ptmr == nil { o.stopAndClearPtmr() return } var shouldUpdateState bool var state StreamState mset.store.FastState(&state) fseq := state.FirstSeq now := time.Now().UnixNano() ttl := int64(o.cfg.AckWait) next := int64(o.ackWait(0)) // However, if there is backoff, initializes with the largest backoff. // It will be adjusted as needed. if l := len(o.cfg.BackOff); l > 0 { next = int64(o.cfg.BackOff[l-1]) } // Since we can update timestamps, we have to review all pending. // We will now bail if we see an ack pending inbound to us via o.awl. var expired []uint64 check := len(o.pending) > 1024 for seq, p := range o.pending { if check && atomic.LoadInt64(&o.awl) > 0 { o.resetPtmr(100 * time.Millisecond) return } // Check if these are no longer valid. if seq < fseq || seq <= o.asflr { delete(o.pending, seq) delete(o.rdc, seq) o.removeFromRedeliverQueue(seq) shouldUpdateState = true // Check if we need to move ack floors. if seq > o.asflr { o.asflr = seq } if p.Sequence > o.adflr { o.adflr = p.Sequence } continue } elapsed, deadline := now-p.Timestamp, ttl if len(o.cfg.BackOff) > 0 { // This is ok even if o.rdc is nil, we would get dc == 0, which is what we want. dc := int(o.rdc[seq]) if dc < 0 { // Prevent consumer backoff from going backwards. dc = 0 } // This will be the index for the next backoff, will set to last element if needed. nbi := dc + 1 if dc+1 >= len(o.cfg.BackOff) { dc = len(o.cfg.BackOff) - 1 nbi = dc } deadline = int64(o.cfg.BackOff[dc]) // Set `next` to the next backoff (if smaller than current `next` value). if nextBackoff := int64(o.cfg.BackOff[nbi]); nextBackoff < next { next = nextBackoff } } if elapsed >= deadline { // We will check if we have hit our max deliveries. Previously we would do this on getNextMsg() which // worked well for push consumers, but with pull based consumers would require a new pull request to be // present to process and redelivered could be reported incorrectly. if !o.onRedeliverQueue(seq) && !o.hasMaxDeliveries(seq) { expired = append(expired, seq) } } else if deadline-elapsed < next { // Update when we should fire next. next = deadline - elapsed } } if len(expired) > 0 { // We need to sort. slices.Sort(expired) o.addToRedeliverQueue(expired...) // Now we should update the timestamp here since we are redelivering. // We will use an incrementing time to preserve order for any other redelivery. off := now - o.pending[expired[0]].Timestamp for _, seq := range expired { if p, ok := o.pending[seq]; ok { p.Timestamp += off } } o.signalNewMessages() } if len(o.pending) > 0 { o.resetPtmr(time.Duration(next)) } else { // Make sure to stop timer and clear out any re delivery queues o.stopAndClearPtmr() o.rdq = nil o.rdqi.Empty() o.pending = nil // Mimic behavior in processAckMsg when pending is empty. o.adflr, o.asflr = o.dseq-1, o.sseq-1 } // Update our state if needed. if shouldUpdateState { if err := o.writeStoreStateUnlocked(); err != nil && o.srv != nil && o.mset != nil && !o.closed { s, acc, mset, name := o.srv, o.acc, o.mset, o.name s.Warnf("Consumer '%s > %s > %s' error on write store state from check pending: %v", acc, mset.getCfgName(), name, err) } } } // SeqFromReply will extract a sequence number from a reply subject. func (o *consumer) seqFromReply(reply string) uint64 { _, dseq, _ := ackReplyInfo(reply) return dseq } // StreamSeqFromReply will extract the stream sequence from the reply subject. func (o *consumer) streamSeqFromReply(reply string) uint64 { sseq, _, _ := ackReplyInfo(reply) return sseq } // Quick parser for positive numbers in ack reply encoding. func parseAckReplyNum(d string) (n int64) { if len(d) == 0 { return -1 } for _, dec := range d { if dec < asciiZero || dec > asciiNine { return -1 } n = n*10 + (int64(dec) - asciiZero) } return n } const expectedNumReplyTokens = 9 // Grab encoded information in the reply subject for a delivered message. func replyInfo(subject string) (sseq, dseq, dc uint64, ts int64, pending uint64) { tsa := [expectedNumReplyTokens]string{} start, tokens := 0, tsa[:0] for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) if len(tokens) != expectedNumReplyTokens || tokens[0] != "$JS" || tokens[1] != "ACK" { return 0, 0, 0, 0, 0 } // TODO(dlc) - Should we error if we do not match consumer name? // stream is tokens[2], consumer is 3. dc = uint64(parseAckReplyNum(tokens[4])) sseq, dseq = uint64(parseAckReplyNum(tokens[5])), uint64(parseAckReplyNum(tokens[6])) ts = parseAckReplyNum(tokens[7]) pending = uint64(parseAckReplyNum(tokens[8])) return sseq, dseq, dc, ts, pending } func ackReplyInfo(subject string) (sseq, dseq, dc uint64) { tsa := [expectedNumReplyTokens]string{} start, tokens := 0, tsa[:0] for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) if len(tokens) != expectedNumReplyTokens || tokens[0] != "$JS" || tokens[1] != "ACK" { return 0, 0, 0 } dc = uint64(parseAckReplyNum(tokens[4])) sseq, dseq = uint64(parseAckReplyNum(tokens[5])), uint64(parseAckReplyNum(tokens[6])) return sseq, dseq, dc } // NextSeq returns the next delivered sequence number for this consumer. func (o *consumer) nextSeq() uint64 { o.mu.RLock() dseq := o.dseq o.mu.RUnlock() return dseq } // Used to hold skip list when deliver policy is last per subject. type lastSeqSkipList struct { resume uint64 seqs []uint64 } // Let's us know we have a skip list, which is for deliver last per subject and we are just starting. // Lock should be held. func (o *consumer) hasSkipListPending() bool { return o.lss != nil && len(o.lss.seqs) > 0 } // Will select the starting sequence. func (o *consumer) selectStartingSeqNo() { if o.mset == nil || o.mset.store == nil { o.sseq = 1 } else { var state StreamState o.mset.store.FastState(&state) if o.cfg.OptStartSeq == 0 { if o.cfg.DeliverPolicy == DeliverAll { o.sseq = state.FirstSeq } else if o.cfg.DeliverPolicy == DeliverLast { if o.subjf == nil { o.sseq = state.LastSeq return } // If we are partitioned here this will be properly set when we become leader. for _, filter := range o.subjf { ss := o.mset.store.FilteredState(1, filter.subject) if ss.Last > o.sseq { o.sseq = ss.Last } } } else if o.cfg.DeliverPolicy == DeliverLastPerSubject { // If our parent stream is set to max msgs per subject of 1 this is just // a normal consumer at this point. We can avoid any heavy lifting. o.mset.cfgMu.RLock() mmp := o.mset.cfg.MaxMsgsPer o.mset.cfgMu.RUnlock() if mmp == 1 { o.sseq = state.FirstSeq } else { // A threshold for when we switch from get last msg to subjects state. const numSubjectsThresh = 256 lss := &lastSeqSkipList{resume: state.LastSeq} var filters []string if o.subjf == nil { filters = append(filters, o.cfg.FilterSubject) } else { for _, filter := range o.subjf { filters = append(filters, filter.subject) } } for _, filter := range filters { if st := o.mset.store.SubjectsTotals(filter); len(st) < numSubjectsThresh { var smv StoreMsg for subj := range st { if sm, err := o.mset.store.LoadLastMsg(subj, &smv); err == nil { lss.seqs = append(lss.seqs, sm.seq) } } } else if mss := o.mset.store.SubjectsState(filter); len(mss) > 0 { for _, ss := range mss { lss.seqs = append(lss.seqs, ss.Last) } } } // Sort the skip list if needed. if len(lss.seqs) > 1 { slices.Sort(lss.seqs) } if len(lss.seqs) == 0 { o.sseq = state.LastSeq } else { o.sseq = lss.seqs[0] } // Assign skip list. o.lss = lss } } else if o.cfg.OptStartTime != nil { // If we are here we are time based. // TODO(dlc) - Once clustered can't rely on this. o.sseq = o.mset.store.GetSeqFromTime(*o.cfg.OptStartTime) // Here we want to see if we are filtered, and if so possibly close the gap // to the nearest first given our starting sequence from time. This is so we do // not force the system to do a linear walk between o.sseq and the real first. if len(o.subjf) > 0 { nseq := state.LastSeq for _, filter := range o.subjf { // Use first sequence since this is more optimized atm. ss := o.mset.store.FilteredState(state.FirstSeq, filter.subject) if ss.First >= o.sseq && ss.First < nseq { nseq = ss.First } } // Skip ahead if possible. if nseq > o.sseq && nseq < state.LastSeq { o.sseq = nseq } } } else { // DeliverNew o.sseq = state.LastSeq + 1 } } else { o.sseq = o.cfg.OptStartSeq } if state.FirstSeq == 0 { o.sseq = 1 } else if o.sseq < state.FirstSeq { o.sseq = state.FirstSeq } else if o.sseq > state.LastSeq { o.sseq = state.LastSeq + 1 } } // Always set delivery sequence to 1. o.dseq = 1 // Set ack delivery floor to delivery-1 o.adflr = o.dseq - 1 // Set ack store floor to store-1 o.asflr = o.sseq - 1 // Set our starting sequence state. if o.store != nil && o.sseq > 0 { o.store.SetStarting(o.sseq - 1) } } // Test whether a config represents a durable subscriber. func isDurableConsumer(config *ConsumerConfig) bool { return config != nil && config.Durable != _EMPTY_ } func (o *consumer) isDurable() bool { return o.cfg.Durable != _EMPTY_ } // Are we in push mode, delivery subject, etc. func (o *consumer) isPushMode() bool { return o.cfg.DeliverSubject != _EMPTY_ } func (o *consumer) isPullMode() bool { return o.cfg.DeliverSubject == _EMPTY_ } // Name returns the name of this consumer. func (o *consumer) String() string { o.mu.RLock() n := o.name o.mu.RUnlock() return n } func createConsumerName() string { return getHash(nuid.Next()) } // deleteConsumer will delete the consumer from this stream. func (mset *stream) deleteConsumer(o *consumer) error { return o.delete() } func (o *consumer) getStream() *stream { o.mu.RLock() mset := o.mset o.mu.RUnlock() return mset } func (o *consumer) streamName() string { o.mu.RLock() mset := o.mset o.mu.RUnlock() if mset != nil { return mset.name() } return _EMPTY_ } // Active indicates if this consumer is still active. func (o *consumer) isActive() bool { o.mu.RLock() active := o.active && o.mset != nil o.mu.RUnlock() return active } // hasNoLocalInterest return true if we have no local interest. func (o *consumer) hasNoLocalInterest() bool { o.mu.RLock() interest := o.acc.sl.HasInterest(o.cfg.DeliverSubject) o.mu.RUnlock() return !interest } // This is when the underlying stream has been purged. // sseq is the new first seq for the stream after purge. // Lock should NOT be held. func (o *consumer) purge(sseq uint64, slseq uint64, isWider bool) { // Do not update our state unless we know we are the leader. if !o.isLeader() { return } // Signals all have been purged for this consumer. if sseq == 0 && !isWider { sseq = slseq + 1 } var store StreamStore if isWider { o.mu.RLock() if o.mset != nil { store = o.mset.store } o.mu.RUnlock() } o.mu.Lock() // Do not go backwards if o.sseq < sseq { o.sseq = sseq } if o.asflr < sseq { o.asflr = sseq - 1 // We need to remove those no longer relevant from pending. for seq, p := range o.pending { if seq <= o.asflr { if p.Sequence > o.adflr { o.adflr = p.Sequence if o.adflr > o.dseq { o.dseq = o.adflr } } delete(o.pending, seq) delete(o.rdc, seq) // rdq handled below. } if isWider && store != nil { // Our filtered subject, which could be all, is wider than the underlying purge. // We need to check if the pending items left are still valid. var smv StoreMsg if _, err := store.LoadMsg(seq, &smv); err == errDeletedMsg || err == ErrStoreMsgNotFound { if p.Sequence > o.adflr { o.adflr = p.Sequence if o.adflr > o.dseq { o.dseq = o.adflr } } delete(o.pending, seq) delete(o.rdc, seq) } } } } // This means we can reset everything at this point. if len(o.pending) == 0 { o.pending, o.rdc = nil, nil o.adflr, o.asflr = o.dseq-1, o.sseq-1 } // We need to remove all those being queued for redelivery under o.rdq if len(o.rdq) > 0 { rdq := o.rdq o.rdq = nil o.rdqi.Empty() for _, sseq := range rdq { if sseq >= o.sseq { o.addToRedeliverQueue(sseq) } } } // Grab some info in case of error below. s, acc, mset, name := o.srv, o.acc, o.mset, o.name o.mu.Unlock() if err := o.writeStoreState(); err != nil && s != nil && mset != nil { s.Warnf("Consumer '%s > %s > %s' error on write store state from purge: %v", acc, mset.name(), name, err) } } func stopAndClearTimer(tp **time.Timer) { if *tp == nil { return } // Will get drained in normal course, do not try to // drain here. (*tp).Stop() *tp = nil } // Stop will shutdown the consumer for the associated stream. func (o *consumer) stop() error { return o.stopWithFlags(false, false, true, false) } func (o *consumer) deleteWithoutAdvisory() error { return o.stopWithFlags(true, false, true, false) } // Delete will delete the consumer for the associated stream and send advisories. func (o *consumer) delete() error { return o.stopWithFlags(true, false, true, true) } // To test for closed state. func (o *consumer) isClosed() bool { o.mu.RLock() defer o.mu.RUnlock() return o.closed } func (o *consumer) stopWithFlags(dflag, sdflag, doSignal, advisory bool) error { // If dflag is true determine if we are still assigned. var isAssigned bool if dflag { o.mu.RLock() acc, stream, consumer := o.acc, o.stream, o.name isClustered := o.js != nil && o.js.isClustered() o.mu.RUnlock() if isClustered { // Grab jsa to check assignment. var jsa *jsAccount if acc != nil { // Need lock here to avoid data race. acc.mu.RLock() jsa = acc.js acc.mu.RUnlock() } if jsa != nil { isAssigned = jsa.consumerAssigned(stream, consumer) } } } o.mu.Lock() if o.closed { o.mu.Unlock() return nil } o.closed = true // Check if we are the leader and are being deleted (as a node). if dflag && o.isLeader() { // If we are clustered and node leader (probable from above), stepdown. if node := o.node; node != nil && node.Leader() { node.StepDown() } // dflag does not necessarily mean that the consumer is being deleted, // just that the consumer node is being removed from this peer, so we // send delete advisories only if we are no longer assigned at the meta layer, // or we are not clustered. if !isAssigned && advisory { o.sendDeleteAdvisoryLocked() } if o.isPullMode() { // Release any pending. o.releaseAnyPendingRequests(isAssigned) } } if o.qch != nil { close(o.qch) o.qch = nil } a := o.acc store := o.store mset := o.mset o.mset = nil o.active = false o.unsubscribe(o.ackSub) o.unsubscribe(o.reqSub) o.unsubscribe(o.fcSub) o.ackSub = nil o.reqSub = nil o.fcSub = nil if o.infoSub != nil { o.srv.sysUnsubscribe(o.infoSub) o.infoSub = nil } c := o.client o.client = nil sysc := o.sysc o.sysc = nil o.stopAndClearPtmr() stopAndClearTimer(&o.dtmr) stopAndClearTimer(&o.gwdtmr) delivery := o.cfg.DeliverSubject o.waiting = nil // Break us out of the readLoop. if doSignal { o.signalNewMessages() } n := o.node qgroup := o.cfg.DeliverGroup o.ackMsgs.unregister() if o.nextMsgReqs != nil { o.nextMsgReqs.unregister() } // For cleaning up the node assignment. var ca *consumerAssignment if dflag { ca = o.ca } js := o.js o.mu.Unlock() if c != nil { c.closeConnection(ClientClosed) } if sysc != nil { sysc.closeConnection(ClientClosed) } if delivery != _EMPTY_ { a.sl.clearNotification(delivery, qgroup, o.inch) } var rp RetentionPolicy if mset != nil { mset.mu.Lock() mset.removeConsumer(o) // No need for cfgMu's lock since mset.mu.Lock superseeds it. rp = mset.cfg.Retention mset.mu.Unlock() } // Cleanup messages that lost interest. if dflag && rp == InterestPolicy { o.cleanupNoInterestMessages(mset, true) } // Cluster cleanup. if n != nil { if dflag { n.Delete() } else { n.Stop() } } if ca != nil { js.mu.Lock() if ca.Group != nil { ca.Group.node = nil } js.mu.Unlock() } // Clean up our store. var err error if store != nil { if dflag { if sdflag { err = store.StreamDelete() } else { err = store.Delete() } } else { err = store.Stop() } } return err } // We need to optionally remove all messages since we are interest based retention. // We will do this consistently on all replicas. Note that if in clustered mode the non-leader // consumers will need to restore state first. // ignoreInterest marks whether the consumer should be ignored when determining interest. // No lock held on entry. func (o *consumer) cleanupNoInterestMessages(mset *stream, ignoreInterest bool) { o.mu.Lock() if !o.isLeader() { o.readStoredState(0) } start := o.asflr o.mu.Unlock() // Make sure we start at worst with first sequence in the stream. state := mset.state() if start < state.FirstSeq { start = state.FirstSeq } stop := state.LastSeq // Consumer's interests are ignored by default. If we should not ignore interest, unset. co := o if !ignoreInterest { co = nil } var rmseqs []uint64 mset.mu.RLock() // If over this amount of messages to check, defer to checkInterestState() which // will do the right thing since we are now removed. // TODO(dlc) - Better way? const bailThresh = 100_000 // Check if we would be spending too much time here and defer to separate go routine. if len(mset.consumers) == 0 { mset.mu.RUnlock() mset.mu.Lock() defer mset.mu.Unlock() mset.store.Purge() var state StreamState mset.store.FastState(&state) mset.lseq = state.LastSeq // Also make sure we clear any pending acks. mset.clearAllPreAcksBelowFloor(state.FirstSeq) return } else if stop-start > bailThresh { mset.mu.RUnlock() go mset.checkInterestState() return } mset.mu.RUnlock() mset.mu.Lock() for seq := start; seq <= stop; seq++ { if mset.noInterest(seq, co) { rmseqs = append(rmseqs, seq) } } mset.mu.Unlock() // These can be removed. for _, seq := range rmseqs { mset.store.RemoveMsg(seq) } } // Check that we do not form a cycle by delivering to a delivery subject // that is part of the interest group. func deliveryFormsCycle(cfg *StreamConfig, deliverySubject string) bool { for _, subject := range cfg.Subjects { if subjectIsSubsetMatch(deliverySubject, subject) { return true } } return false } // switchToEphemeral is called on startup when recovering ephemerals. func (o *consumer) switchToEphemeral() { o.mu.Lock() o.cfg.Durable = _EMPTY_ store, ok := o.store.(*consumerFileStore) interest := o.acc.sl.HasInterest(o.cfg.DeliverSubject) // Setup dthresh. o.updateInactiveThreshold(&o.cfg) o.mu.Unlock() // Update interest o.updateDeliveryInterest(interest) // Write out new config if ok { store.updateConfig(o.cfg) } } // RequestNextMsgSubject returns the subject to request the next message when in pull or worker mode. // Returns empty otherwise. func (o *consumer) requestNextMsgSubject() string { return o.nextMsgSubj } func (o *consumer) decStreamPending(sseq uint64, subj string) { o.mu.Lock() // Update our cached num pending only if we think deliverMsg has not done so. if sseq >= o.sseq && o.isFilteredMatch(subj) { o.npc-- } // Check if this message was pending. p, wasPending := o.pending[sseq] rdc := o.deliveryCount(sseq) o.mu.Unlock() // If it was pending process it like an ack. if wasPending { // We could have the lock for the stream so do this in a go routine. // TODO(dlc) - We should do this with ipq vs naked go routines. go o.processTerm(sseq, p.Sequence, rdc, ackTermUnackedLimitsReason, _EMPTY_) } } func (o *consumer) account() *Account { o.mu.RLock() a := o.acc o.mu.RUnlock() return a } // Creates a sublist for consumer. // All subjects share the same callback. func (o *consumer) signalSubs() []string { o.mu.Lock() defer o.mu.Unlock() if o.sigSubs != nil { return o.sigSubs } if len(o.subjf) == 0 { subs := []string{fwcs} o.sigSubs = subs return subs } subs := make([]string, 0, len(o.subjf)) for _, filter := range o.subjf { subs = append(subs, filter.subject) } o.sigSubs = subs return subs } // This is what will be called when our parent stream wants to kick us regarding a new message. // We know that this subject matches us by how the parent handles registering us with the signaling sublist, // but we must check if we are leader. // We do need the sequence of the message however and we use the msg as the encoded seq. func (o *consumer) processStreamSignal(seq uint64) { // We can get called here now when not leader, so bail fast // and without acquiring any locks. if !o.leader.Load() { return } o.mu.Lock() defer o.mu.Unlock() if o.mset == nil { return } if seq > o.npf { o.npc++ } if seq < o.sseq { return } if o.isPushMode() && o.active || o.isPullMode() && !o.waiting.isEmpty() { o.signalNewMessages() } } // Used to compare if two multiple filtered subject lists are equal. func subjectSliceEqual(slice1 []string, slice2 []string) bool { if len(slice1) != len(slice2) { return false } set2 := make(map[string]struct{}, len(slice2)) for _, val := range slice2 { set2[val] = struct{}{} } for _, val := range slice1 { if _, ok := set2[val]; !ok { return false } } return true } // Utility for simpler if conditions in Consumer config checks. // In future iteration, we can immediately create `o.subjf` and // use it to validate things. func gatherSubjectFilters(filter string, filters []string) []string { if filter != _EMPTY_ { filters = append(filters, filter) } // list of filters should never contain non-empty filter. return filters } // shouldStartMonitor will return true if we should start a monitor // goroutine or will return false if one is already running. func (o *consumer) shouldStartMonitor() bool { o.mu.Lock() defer o.mu.Unlock() if o.inMonitor { return false } o.monitorWg.Add(1) o.inMonitor = true return true } // Clear the monitor running state. The monitor goroutine should // call this in a defer to clean up on exit. func (o *consumer) clearMonitorRunning() { o.mu.Lock() defer o.mu.Unlock() if o.inMonitor { o.monitorWg.Done() o.inMonitor = false } } // Test whether we are in the monitor routine. func (o *consumer) isMonitorRunning() bool { o.mu.RLock() defer o.mu.RUnlock() return o.inMonitor } // If we detect that our ackfloor is higher than the stream's last sequence, return this error. var errAckFloorHigherThanLastSeq = errors.New("consumer ack floor is higher than streams last sequence") var errAckFloorInvalid = errors.New("consumer ack floor is invalid") // If we are a consumer of an interest or workqueue policy stream, process that state and make sure consistent. func (o *consumer) checkStateForInterestStream(ss *StreamState) error { o.mu.RLock() // See if we need to process this update if our parent stream is not a limits policy stream. mset := o.mset shouldProcessState := mset != nil && o.retention != LimitsPolicy if o.closed || !shouldProcessState || o.store == nil || ss == nil { o.mu.RUnlock() return nil } store := mset.store state, err := o.store.State() filters, subjf, filter := o.filters, o.subjf, _EMPTY_ var wc bool if filters == nil && subjf != nil { filter, wc = subjf[0].subject, subjf[0].hasWildcard } chkfloor := o.chkflr o.mu.RUnlock() if err != nil { return err } asflr := state.AckFloor.Stream // Protect ourselves against rolling backwards. if asflr&(1<<63) != 0 { return errAckFloorInvalid } // Check if the underlying stream's last sequence is less than our floor. // This can happen if the stream has been reset and has not caught up yet. if asflr > ss.LastSeq { return errAckFloorHigherThanLastSeq } var smv StoreMsg var seq, nseq uint64 // Start at first stream seq or a previous check floor, whichever is higher. // Note this will really help for interest retention, with WQ the loadNextMsg // gets us a long way already since it will skip deleted msgs not for our filter. fseq := ss.FirstSeq if chkfloor > fseq { fseq = chkfloor } var retryAsflr uint64 for seq = fseq; asflr > 0 && seq <= asflr; seq++ { if filters != nil { _, nseq, err = store.LoadNextMsgMulti(filters, seq, &smv) } else { _, nseq, err = store.LoadNextMsg(filter, wc, seq, &smv) } // if we advanced sequence update our seq. This can be on no error and EOF. if nseq > seq { seq = nseq } // Only ack though if no error and seq <= ack floor. if err == nil && seq <= asflr { didRemove := mset.ackMsg(o, seq) // Removing the message could fail. For example if we're behind on stream applies. // Overwrite retry floor (only the first time) to allow us to check next time if the removal was successful. if didRemove && retryAsflr == 0 { retryAsflr = seq } } } // If retry floor was not overwritten, set to ack floor+1, we don't need to account for any retries below it. if retryAsflr == 0 { retryAsflr = asflr + 1 } o.mu.Lock() // Update our check floor. // Check floor must never be greater than ack floor+1, otherwise subsequent calls to this function would skip work. if retryAsflr > o.chkflr { o.chkflr = retryAsflr } // See if we need to process this update if our parent stream is not a limits policy stream. state, _ = o.store.State() o.mu.Unlock() // If we have pending, we will need to walk through to delivered in case we missed any of those acks as well. if state != nil && len(state.Pending) > 0 && state.AckFloor.Stream > 0 { for seq := state.AckFloor.Stream + 1; seq <= state.Delivered.Stream; seq++ { if _, ok := state.Pending[seq]; !ok { // Want to call needAck since it is filter aware. if o.needAck(seq, _EMPTY_) { mset.ackMsg(o, seq) } } } } return nil } func (o *consumer) resetPtmr(delay time.Duration) { if o.ptmr == nil { o.ptmr = time.AfterFunc(delay, o.checkPending) } else { o.ptmr.Reset(delay) } o.ptmrEnd = time.Now().Add(delay) } func (o *consumer) stopAndClearPtmr() { stopAndClearTimer(&o.ptmr) o.ptmrEnd = time.Time{} } nats-server-2.10.27/server/core_benchmarks_test.go000066400000000000000000000367771477524627100222200ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "crypto/tls" "errors" "fmt" "math" "math/rand" "os" "strconv" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/internal/fastrand" "github.com/nats-io/nats.go" ) func BenchmarkCoreRequestReply(b *testing.B) { const ( subject = "test-subject" ) messageSizes := []int64{ 1024, // 1kb 4096, // 4kb 40960, // 40kb 409600, // 400kb } for _, messageSize := range messageSizes { b.Run(fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { // Start server serverOpts := DefaultOptions() server := RunServer(serverOpts) defer server.Shutdown() clientUrl := server.ClientURL() // Create "echo" subscriber ncSub, err := nats.Connect(clientUrl) if err != nil { b.Fatal(err) } defer ncSub.Close() sub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) { // Responder echoes the request payload as-is msg.Respond(msg.Data) }) defer sub.Unsubscribe() if err != nil { b.Fatal(err) } // Create publisher ncPub, err := nats.Connect(clientUrl) if err != nil { b.Fatal(err) } defer ncPub.Close() var errors = 0 // Create message messageData := make([]byte, messageSize) rand.New(rand.NewSource(12345)).Read(messageData) b.SetBytes(messageSize) // Benchmark b.ResetTimer() for i := 0; i < b.N; i++ { fastRandomMutation(messageData, 10) _, err := ncPub.Request(subject, messageData, time.Second) if err != nil { errors++ } } b.StopTimer() b.ReportMetric(float64(errors), "errors") }) } } func BenchmarkCoreTLSFanOut(b *testing.B) { const ( subject = "test-subject" configsBasePath = "./configs/tls" maxPendingMessages = 25 maxPendingBytes = 15 * 1024 * 1024 // 15MiB ) keyTypeCases := []string{ "none", "ed25519", "rsa-1024", "rsa-2048", "rsa-4096", } messageSizeCases := []int64{ 512 * 1024, // 512Kib } numSubsCases := []int{ 5, } // Custom error handler that ignores ErrSlowConsumer. // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher // than what the server can relay to subscribers. ignoreSlowConsumerErrorHandler := func(conn *nats.Conn, s *nats.Subscription, err error) { if errors.Is(err, nats.ErrSlowConsumer) { // Swallow this error } else { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) } } for _, keyType := range keyTypeCases { b.Run( fmt.Sprintf("keyType=%s", keyType), func(b *testing.B) { for _, messageSize := range messageSizeCases { b.Run( fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { for _, numSubs := range numSubsCases { b.Run( fmt.Sprintf("subs=%d", numSubs), func(b *testing.B) { // Start server configPath := fmt.Sprintf("%s/tls-%s.conf", configsBasePath, keyType) server, _ := RunServerWithConfig(configPath) defer server.Shutdown() opts := []nats.Option{ nats.MaxReconnects(-1), nats.ReconnectWait(0), nats.ErrorHandler(ignoreSlowConsumerErrorHandler), } if keyType != "none" { opts = append(opts, nats.Secure(&tls.Config{ InsecureSkipVerify: true, })) } clientUrl := server.ClientURL() // Count of messages received for by each subscriber counters := make([]int, numSubs) // Wait group for subscribers to signal they received b.N messages var wg sync.WaitGroup wg.Add(numSubs) // Create subscribers for i := 0; i < numSubs; i++ { subIndex := i ncSub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncSub.Close() sub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) { counters[subIndex] += 1 if counters[subIndex] == b.N { wg.Done() } }) if err != nil { b.Fatalf("failed to subscribe: %s", err) } err = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes) if err != nil { b.Fatalf("failed to set pending limits: %s", err) } defer sub.Unsubscribe() if err != nil { b.Fatal(err) } } // publisher ncPub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncPub.Close() var errorCount = 0 // random bytes as payload messageData := make([]byte, messageSize) rand.New(rand.NewSource(12345)).Read(messageData) quitCh := make(chan bool, 1) publish := func() { for { select { case <-quitCh: return default: // continue publishing } fastRandomMutation(messageData, 10) err := ncPub.Publish(subject, messageData) if err != nil { errorCount += 1 } } } // Set bytes per operation b.SetBytes(messageSize) // Start the clock b.ResetTimer() // Start publishing as fast as the server allows go publish() // Wait for all subscribers to have delivered b.N messages wg.Wait() // Stop the clock b.StopTimer() // Stop publisher quitCh <- true b.ReportMetric(float64(errorCount), "errors") }, ) } }, ) } }, ) } } func BenchmarkCoreFanOut(b *testing.B) { const ( subject = "test-subject" maxPendingMessages = 25 maxPendingBytes = 15 * 1024 * 1024 // 15MiB ) messageSizeCases := []int64{ 100, // 100B 1024, // 1KiB 10240, // 10KiB 512 * 1024, // 512KiB } numSubsCases := []int{ 3, 5, 10, } // Custom error handler that ignores ErrSlowConsumer. // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher // than what the server can relay to subscribers. ignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) { if errors.Is(err, nats.ErrSlowConsumer) { // Swallow this error } else { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) } } for _, messageSize := range messageSizeCases { b.Run( fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { for _, numSubs := range numSubsCases { b.Run( fmt.Sprintf("subs=%d", numSubs), func(b *testing.B) { // Start server defaultOpts := DefaultOptions() server := RunServer(defaultOpts) defer server.Shutdown() opts := []nats.Option{ nats.MaxReconnects(-1), nats.ReconnectWait(0), nats.ErrorHandler(ignoreSlowConsumerErrorHandler), } clientUrl := server.ClientURL() // Count of messages received for by each subscriber counters := make([]int, numSubs) // Wait group for subscribers to signal they received b.N messages var wg sync.WaitGroup wg.Add(numSubs) // Create subscribers for i := 0; i < numSubs; i++ { subIndex := i ncSub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncSub.Close() sub, err := ncSub.Subscribe(subject, func(_ *nats.Msg) { counters[subIndex] += 1 if counters[subIndex] == b.N { wg.Done() } }) if err != nil { b.Fatalf("failed to subscribe: %s", err) } err = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes) if err != nil { b.Fatalf("failed to set pending limits: %s", err) } defer sub.Unsubscribe() } // publisher ncPub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncPub.Close() var errorCount = 0 // random bytes as payload messageData := make([]byte, messageSize) rand.New(rand.NewSource(123456)).Read(messageData) quitCh := make(chan bool, 1) publish := func() { for { select { case <-quitCh: return default: // continue publishing } fastRandomMutation(messageData, 10) err := ncPub.Publish(subject, messageData) if err != nil { errorCount += 1 } } } // Set bytes per operation b.SetBytes(messageSize) // Start the clock b.ResetTimer() // Start publishing as fast as the server allows go publish() // Wait for all subscribers to have delivered b.N messages wg.Wait() // Stop the clock b.StopTimer() // Stop publisher quitCh <- true b.ReportMetric(100*float64(errorCount)/float64(b.N), "%error") }, ) } }, ) } } func BenchmarkCoreFanIn(b *testing.B) { type BenchPublisher struct { // nats connection for this publisher conn *nats.Conn // number of publishing errors encountered publishErrors int // number of messages published publishCounter int // quit channel which will terminate publishing quitCh chan bool // message data buffer messageData []byte } const subjectBaseName = "test-subject" messageSizeCases := []int64{ 100, // 100B 1024, // 1KiB 10240, // 10KiB 512 * 1024, // 512KiB } numPubsCases := []int{ 3, 5, 10, } // Custom error handler that ignores ErrSlowConsumer. // Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher // than what the server can relay to subscribers. ignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) { if errors.Is(err, nats.ErrSlowConsumer) { // Swallow this error } else { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", err) } } workload := func(b *testing.B, clientUrl string, numPubs int, messageSize int64) { // connection options opts := []nats.Option{ nats.MaxReconnects(-1), nats.ReconnectWait(0), nats.ErrorHandler(ignoreSlowConsumerErrorHandler), } // waits for all publishers sub-routines and for main thread to be ready var publishersReadyWg sync.WaitGroup publishersReadyWg.Add(numPubs + 1) // wait group to ensure all publishers have been torn down var finishedPublishersWg sync.WaitGroup finishedPublishersWg.Add(numPubs) publishers := make([]BenchPublisher, numPubs) // create N publishers for i := range publishers { // create publisher connection ncPub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncPub.Close() // create bench publisher object publisher := BenchPublisher{ conn: ncPub, publishErrors: 0, publishCounter: 0, quitCh: make(chan bool, 1), messageData: make([]byte, messageSize), } rand.New(rand.NewSource(int64(i))).Read(publisher.messageData) publishers[i] = publisher } // total number of publishers that have published b.N to the subscriber successfully completedPublishersCount := 0 // wait group blocks main thread until publish workload is completed, it is decremented after subscriber receives b.N messages from all publishers var benchCompleteWg sync.WaitGroup benchCompleteWg.Add(1) // start subscriber ncSub, err := nats.Connect(clientUrl, opts...) if err != nil { b.Fatal(err) } defer ncSub.Close() // subscriber ncSub.Subscribe(fmt.Sprintf("%s.*", subjectBaseName), func(msg *nats.Msg) { // get the publisher id from subject pubIdx, err := strconv.Atoi(msg.Subject[len(subjectBaseName)+1:]) if err != nil { b.Fatal(err) } // message successfully received from publisher publishers[pubIdx].publishCounter += 1 // subscriber has received a total of b.N messages from this publisher if publishers[pubIdx].publishCounter == b.N { completedPublishersCount++ // every publisher has successfully sent b.N messages to subscriber if completedPublishersCount == numPubs { benchCompleteWg.Done() } } }) // start publisher sub-routines for i := range publishers { go func(pubId int) { // publisher sub-routine initialized publishersReadyWg.Done() publisher := publishers[pubId] subject := fmt.Sprintf("%s.%d", subjectBaseName, pubId) // signal that this publisher has been torn down defer finishedPublishersWg.Done() // wait till all other publishers are ready to start workload publishersReadyWg.Wait() // publish until quitCh is closed for { select { case <-publisher.quitCh: return default: // continue publishing } fastRandomMutation(publisher.messageData, 10) err := publisher.conn.Publish(subject, publisher.messageData) if err != nil { publisher.publishErrors += 1 } } }(i) } // set bytes per operation b.SetBytes(messageSize) // main thread is ready publishersReadyWg.Done() // wait till publishers are ready publishersReadyWg.Wait() // start the clock b.ResetTimer() // wait till termination cond reached benchCompleteWg.Wait() // stop the clock b.StopTimer() // send quit signal to all publishers for i := range publishers { publishers[i].quitCh <- true } // wait for all publishers to shutdown finishedPublishersWg.Wait() // sum errors from all publishers totalErrors := 0 for _, publisher := range publishers { totalErrors += publisher.publishErrors } // sum total messages sent from all publishers totalMessages := 0 for _, publisher := range publishers { totalMessages += publisher.publishCounter } errorRate := 100 * float64(totalErrors) / float64(totalMessages) // report error rate b.ReportMetric(errorRate, "%error") } // benchmark case matrix for _, messageSize := range messageSizeCases { b.Run( fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { for _, numPubs := range numPubsCases { b.Run( fmt.Sprintf("pubs=%d", numPubs), func(b *testing.B) { // start server defaultOpts := DefaultOptions() server := RunServer(defaultOpts) defer server.Shutdown() // get connection string clientUrl := server.ClientURL() // run fan-in workload workload(b, clientUrl, numPubs, messageSize) }) } }) } } // fastRandomMutation performs a minor in-place mutation to the given buffer. // This is useful in benchmark to avoid sending the same payload every time (which could result in some optimizations // we do not want to measure), while not slowing down the benchmark with a full payload generated for each operation. func fastRandomMutation(data []byte, mutations int) { for i := 0; i < mutations; i++ { data[fastrand.Uint32n(uint32(len(data)))] = byte(fastrand.Uint32() % math.MaxUint8) } } nats-server-2.10.27/server/dirstore.go000066400000000000000000000461601477524627100176520ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "container/heap" "container/list" "crypto/sha256" "errors" "fmt" "math" "os" "path/filepath" "strings" "sync" "time" "github.com/nats-io/nkeys" "github.com/nats-io/jwt/v2" // only used to decode, not for storage ) const ( fileExtension = ".jwt" ) // validatePathExists checks that the provided path exists and is a dir if requested func validatePathExists(path string, dir bool) (string, error) { if path == _EMPTY_ { return _EMPTY_, errors.New("path is not specified") } abs, err := filepath.Abs(path) if err != nil { return _EMPTY_, fmt.Errorf("error parsing path [%s]: %v", abs, err) } var finfo os.FileInfo if finfo, err = os.Stat(abs); os.IsNotExist(err) { return _EMPTY_, fmt.Errorf("the path [%s] doesn't exist", abs) } mode := finfo.Mode() if dir && mode.IsRegular() { return _EMPTY_, fmt.Errorf("the path [%s] is not a directory", abs) } if !dir && mode.IsDir() { return _EMPTY_, fmt.Errorf("the path [%s] is not a file", abs) } return abs, nil } // ValidateDirPath checks that the provided path exists and is a dir func validateDirPath(path string) (string, error) { return validatePathExists(path, true) } // JWTChanged functions are called when the store file watcher notices a JWT changed type JWTChanged func(publicKey string) // DirJWTStore implements the JWT Store interface, keeping JWTs in an optionally sharded // directory structure type DirJWTStore struct { sync.Mutex directory string shard bool readonly bool deleteType deleteType operator map[string]struct{} expiration *expirationTracker changed JWTChanged deleted JWTChanged } func newDir(dirPath string, create bool) (string, error) { fullPath, err := validateDirPath(dirPath) if err != nil { if !create { return _EMPTY_, err } if err = os.MkdirAll(dirPath, defaultDirPerms); err != nil { return _EMPTY_, err } if fullPath, err = validateDirPath(dirPath); err != nil { return _EMPTY_, err } } return fullPath, nil } // future proofing in case new options will be added type dirJWTStoreOption any // Creates a directory based jwt store. // Reads files only, does NOT watch directories and files. func NewImmutableDirJWTStore(dirPath string, shard bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) { theStore, err := NewDirJWTStore(dirPath, shard, false, nil) if err != nil { return nil, err } theStore.readonly = true return theStore, nil } // Creates a directory based jwt store. // Operates on files only, does NOT watch directories and files. func NewDirJWTStore(dirPath string, shard bool, create bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) { fullPath, err := newDir(dirPath, create) if err != nil { return nil, err } theStore := &DirJWTStore{ directory: fullPath, shard: shard, } return theStore, nil } type deleteType int const ( NoDelete deleteType = iota RenameDeleted HardDelete ) // Creates a directory based jwt store. // // When ttl is set deletion of file is based on it and not on the jwt expiration // To completely disable expiration (including expiration in jwt) set ttl to max duration time.Duration(math.MaxInt64) // // limit defines how many files are allowed at any given time. Set to math.MaxInt64 to disable. // evictOnLimit determines the behavior once limit is reached. // * true - Evict based on lru strategy // * false - return an error func NewExpiringDirJWTStore(dirPath string, shard bool, create bool, delete deleteType, expireCheck time.Duration, limit int64, evictOnLimit bool, ttl time.Duration, changeNotification JWTChanged, _ ...dirJWTStoreOption) (*DirJWTStore, error) { fullPath, err := newDir(dirPath, create) if err != nil { return nil, err } theStore := &DirJWTStore{ directory: fullPath, shard: shard, deleteType: delete, changed: changeNotification, } if expireCheck <= 0 { if ttl != 0 { expireCheck = ttl / 2 } if expireCheck == 0 || expireCheck > time.Minute { expireCheck = time.Minute } } if limit <= 0 { limit = math.MaxInt64 } theStore.startExpiring(expireCheck, limit, evictOnLimit, ttl) theStore.Lock() err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { if strings.HasSuffix(path, fileExtension) { if theJwt, err := os.ReadFile(path); err == nil { hash := sha256.Sum256(theJwt) _, file := filepath.Split(path) theStore.expiration.track(strings.TrimSuffix(file, fileExtension), &hash, string(theJwt)) } } return nil }) theStore.Unlock() if err != nil { theStore.Close() return nil, err } return theStore, err } func (store *DirJWTStore) IsReadOnly() bool { return store.readonly } func (store *DirJWTStore) LoadAcc(publicKey string) (string, error) { return store.load(publicKey) } func (store *DirJWTStore) SaveAcc(publicKey string, theJWT string) error { return store.save(publicKey, theJWT) } func (store *DirJWTStore) LoadAct(hash string) (string, error) { return store.load(hash) } func (store *DirJWTStore) SaveAct(hash string, theJWT string) error { return store.save(hash, theJWT) } func (store *DirJWTStore) Close() { store.Lock() defer store.Unlock() if store.expiration != nil { store.expiration.close() store.expiration = nil } } // Pack up to maxJWTs into a package func (store *DirJWTStore) Pack(maxJWTs int) (string, error) { count := 0 var pack []string if maxJWTs > 0 { pack = make([]string, 0, maxJWTs) } else { pack = []string{} } store.Lock() err := filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error { if !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT if count == maxJWTs { // won't match negative return nil } pubKey := strings.TrimSuffix(filepath.Base(path), fileExtension) if store.expiration != nil { if _, ok := store.expiration.idx[pubKey]; !ok { return nil // only include indexed files } } jwtBytes, err := os.ReadFile(path) if err != nil { return err } if store.expiration != nil { claim, err := jwt.DecodeGeneric(string(jwtBytes)) if err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() { return nil } } pack = append(pack, fmt.Sprintf("%s|%s", pubKey, string(jwtBytes))) count++ } return nil }) store.Unlock() if err != nil { return _EMPTY_, err } else { return strings.Join(pack, "\n"), nil } } // Pack up to maxJWTs into a message and invoke callback with it func (store *DirJWTStore) PackWalk(maxJWTs int, cb func(partialPackMsg string)) error { if maxJWTs <= 0 || cb == nil { return errors.New("bad arguments to PackWalk") } var packMsg []string store.Lock() dir := store.directory exp := store.expiration store.Unlock() err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if info != nil && !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT pubKey := strings.TrimSuffix(filepath.Base(path), fileExtension) store.Lock() if exp != nil { if _, ok := exp.idx[pubKey]; !ok { store.Unlock() return nil // only include indexed files } } store.Unlock() jwtBytes, err := os.ReadFile(path) if err != nil { return err } if len(jwtBytes) == 0 { // Skip if no contents in the JWT. return nil } if exp != nil { claim, err := jwt.DecodeGeneric(string(jwtBytes)) if err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() { return nil } } packMsg = append(packMsg, fmt.Sprintf("%s|%s", pubKey, string(jwtBytes))) if len(packMsg) == maxJWTs { // won't match negative cb(strings.Join(packMsg, "\n")) packMsg = nil } } return nil }) if packMsg != nil { cb(strings.Join(packMsg, "\n")) } return err } // Merge takes the JWTs from package and adds them to the store // Merge is destructive in the sense that it doesn't check if the JWT // is newer or anything like that. func (store *DirJWTStore) Merge(pack string) error { newJWTs := strings.Split(pack, "\n") for _, line := range newJWTs { if line == _EMPTY_ { // ignore blank lines continue } split := strings.Split(line, "|") if len(split) != 2 { return fmt.Errorf("line in package didn't contain 2 entries: %q", line) } pubKey := split[0] if !nkeys.IsValidPublicAccountKey(pubKey) { return fmt.Errorf("key to merge is not a valid public account key") } if err := store.saveIfNewer(pubKey, split[1]); err != nil { return err } } return nil } func (store *DirJWTStore) Reload() error { store.Lock() exp := store.expiration if exp == nil || store.readonly { store.Unlock() return nil } idx := exp.idx changed := store.changed isCache := store.expiration.evictOnLimit // clear out indexing data structures exp.heap = make([]*jwtItem, 0, len(exp.heap)) exp.idx = make(map[string]*list.Element) exp.lru = list.New() exp.hash = [sha256.Size]byte{} store.Unlock() return filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error { if strings.HasSuffix(path, fileExtension) { if theJwt, err := os.ReadFile(path); err == nil { hash := sha256.Sum256(theJwt) _, file := filepath.Split(path) pkey := strings.TrimSuffix(file, fileExtension) notify := isCache // for cache, issue cb even when file not present (may have been evicted) if i, ok := idx[pkey]; ok { notify = !bytes.Equal(i.Value.(*jwtItem).hash[:], hash[:]) } store.Lock() exp.track(pkey, &hash, string(theJwt)) store.Unlock() if notify && changed != nil { changed(pkey) } } } return nil }) } func (store *DirJWTStore) pathForKey(publicKey string) string { if len(publicKey) < 2 { return _EMPTY_ } if !nkeys.IsValidPublicKey(publicKey) { return _EMPTY_ } fileName := fmt.Sprintf("%s%s", publicKey, fileExtension) if store.shard { last := publicKey[len(publicKey)-2:] return filepath.Join(store.directory, last, fileName) } else { return filepath.Join(store.directory, fileName) } } // Load checks the memory store and returns the matching JWT or an error // Assumes lock is NOT held func (store *DirJWTStore) load(publicKey string) (string, error) { store.Lock() defer store.Unlock() if path := store.pathForKey(publicKey); path == _EMPTY_ { return _EMPTY_, fmt.Errorf("invalid public key") } else if data, err := os.ReadFile(path); err != nil { return _EMPTY_, err } else { if store.expiration != nil { store.expiration.updateTrack(publicKey) } return string(data), nil } } // write that keeps hash of all jwt in sync // Assumes the lock is held. Does return true or an error never both. func (store *DirJWTStore) write(path string, publicKey string, theJWT string) (bool, error) { if len(theJWT) == 0 { return false, fmt.Errorf("invalid JWT") } var newHash *[sha256.Size]byte if store.expiration != nil { h := sha256.Sum256([]byte(theJWT)) newHash = &h if v, ok := store.expiration.idx[publicKey]; ok { store.expiration.updateTrack(publicKey) // this write is an update, move to back it := v.Value.(*jwtItem) oldHash := it.hash[:] if bytes.Equal(oldHash, newHash[:]) { return false, nil } } else if int64(store.expiration.Len()) >= store.expiration.limit { if !store.expiration.evictOnLimit { return false, errors.New("jwt store is full") } // this write is an add, pick the least recently used value for removal i := store.expiration.lru.Front().Value.(*jwtItem) if err := os.Remove(store.pathForKey(i.publicKey)); err != nil { return false, err } else { store.expiration.unTrack(i.publicKey) } } } if err := os.WriteFile(path, []byte(theJWT), defaultFilePerms); err != nil { return false, err } else if store.expiration != nil { store.expiration.track(publicKey, newHash, theJWT) } return true, nil } func (store *DirJWTStore) delete(publicKey string) error { if store.readonly { return fmt.Errorf("store is read-only") } else if store.deleteType == NoDelete { return fmt.Errorf("store is not set up to for delete") } store.Lock() defer store.Unlock() name := store.pathForKey(publicKey) if store.deleteType == RenameDeleted { if err := os.Rename(name, name+".deleted"); err != nil { if os.IsNotExist(err) { return nil } return err } } else if err := os.Remove(name); err != nil { if os.IsNotExist(err) { return nil } return err } store.expiration.unTrack(publicKey) store.deleted(publicKey) return nil } // Save puts the JWT in a map by public key and performs update callbacks // Assumes lock is NOT held func (store *DirJWTStore) save(publicKey string, theJWT string) error { if store.readonly { return fmt.Errorf("store is read-only") } store.Lock() path := store.pathForKey(publicKey) if path == _EMPTY_ { store.Unlock() return fmt.Errorf("invalid public key") } dirPath := filepath.Dir(path) if _, err := validateDirPath(dirPath); err != nil { if err := os.MkdirAll(dirPath, defaultDirPerms); err != nil { store.Unlock() return err } } changed, err := store.write(path, publicKey, theJWT) cb := store.changed store.Unlock() if changed && cb != nil { cb(publicKey) } return err } // Assumes the lock is NOT held, and only updates if the jwt is new, or the one on disk is older // When changed, invokes jwt changed callback func (store *DirJWTStore) saveIfNewer(publicKey string, theJWT string) error { if store.readonly { return fmt.Errorf("store is read-only") } path := store.pathForKey(publicKey) if path == _EMPTY_ { return fmt.Errorf("invalid public key") } dirPath := filepath.Dir(path) if _, err := validateDirPath(dirPath); err != nil { if err := os.MkdirAll(dirPath, defaultDirPerms); err != nil { return err } } if _, err := os.Stat(path); err == nil { if newJWT, err := jwt.DecodeGeneric(theJWT); err != nil { return err } else if existing, err := os.ReadFile(path); err != nil { return err } else if existingJWT, err := jwt.DecodeGeneric(string(existing)); err != nil { // skip if it can't be decoded } else if existingJWT.ID == newJWT.ID { return nil } else if existingJWT.IssuedAt > newJWT.IssuedAt { return nil } else if newJWT.Subject != publicKey { return fmt.Errorf("jwt subject nkey and provided nkey do not match") } else if existingJWT.Subject != newJWT.Subject { return fmt.Errorf("subject of existing and new jwt do not match") } } store.Lock() cb := store.changed changed, err := store.write(path, publicKey, theJWT) store.Unlock() if err != nil { return err } else if changed && cb != nil { cb(publicKey) } return nil } func xorAssign(lVal *[sha256.Size]byte, rVal [sha256.Size]byte) { for i := range rVal { (*lVal)[i] ^= rVal[i] } } // returns a hash representing all indexed jwt func (store *DirJWTStore) Hash() [sha256.Size]byte { store.Lock() defer store.Unlock() if store.expiration == nil { return [sha256.Size]byte{} } else { return store.expiration.hash } } // An jwtItem is something managed by the priority queue type jwtItem struct { index int publicKey string expiration int64 // consists of unix time of expiration (ttl when set or jwt expiration) in seconds hash [sha256.Size]byte } // A expirationTracker implements heap.Interface and holds Items. type expirationTracker struct { heap []*jwtItem // sorted by jwtItem.expiration idx map[string]*list.Element lru *list.List // keep which jwt are least used limit int64 // limit how many jwt are being tracked evictOnLimit bool // when limit is hit, error or evict using lru ttl time.Duration hash [sha256.Size]byte // xor of all jwtItem.hash in idx quit chan struct{} wg sync.WaitGroup } func (q *expirationTracker) Len() int { return len(q.heap) } func (q *expirationTracker) Less(i, j int) bool { pq := q.heap return pq[i].expiration < pq[j].expiration } func (q *expirationTracker) Swap(i, j int) { pq := q.heap pq[i], pq[j] = pq[j], pq[i] pq[i].index = i pq[j].index = j } func (q *expirationTracker) Push(x any) { n := len(q.heap) item := x.(*jwtItem) item.index = n q.heap = append(q.heap, item) q.idx[item.publicKey] = q.lru.PushBack(item) } func (q *expirationTracker) Pop() any { old := q.heap n := len(old) item := old[n-1] old[n-1] = nil // avoid memory leak item.index = -1 q.heap = old[0 : n-1] q.lru.Remove(q.idx[item.publicKey]) delete(q.idx, item.publicKey) return item } func (pq *expirationTracker) updateTrack(publicKey string) { if e, ok := pq.idx[publicKey]; ok { i := e.Value.(*jwtItem) if pq.ttl != 0 { // only update expiration when set i.expiration = time.Now().Add(pq.ttl).UnixNano() heap.Fix(pq, i.index) } if pq.evictOnLimit { pq.lru.MoveToBack(e) } } } func (pq *expirationTracker) unTrack(publicKey string) { if it, ok := pq.idx[publicKey]; ok { xorAssign(&pq.hash, it.Value.(*jwtItem).hash) heap.Remove(pq, it.Value.(*jwtItem).index) delete(pq.idx, publicKey) } } func (pq *expirationTracker) track(publicKey string, hash *[sha256.Size]byte, theJWT string) { var exp int64 // prioritize ttl over expiration if pq.ttl != 0 { if pq.ttl == time.Duration(math.MaxInt64) { exp = math.MaxInt64 } else { exp = time.Now().Add(pq.ttl).UnixNano() } } else { if g, err := jwt.DecodeGeneric(theJWT); err == nil { exp = time.Unix(g.Expires, 0).UnixNano() } if exp == 0 { exp = math.MaxInt64 // default to indefinite } } if e, ok := pq.idx[publicKey]; ok { i := e.Value.(*jwtItem) xorAssign(&pq.hash, i.hash) // remove old hash i.expiration = exp i.hash = *hash heap.Fix(pq, i.index) } else { heap.Push(pq, &jwtItem{-1, publicKey, exp, *hash}) } xorAssign(&pq.hash, *hash) // add in new hash } func (pq *expirationTracker) close() { if pq == nil || pq.quit == nil { return } close(pq.quit) pq.quit = nil } func (store *DirJWTStore) startExpiring(reCheck time.Duration, limit int64, evictOnLimit bool, ttl time.Duration) { store.Lock() defer store.Unlock() quit := make(chan struct{}) pq := &expirationTracker{ make([]*jwtItem, 0, 10), make(map[string]*list.Element), list.New(), limit, evictOnLimit, ttl, [sha256.Size]byte{}, quit, sync.WaitGroup{}, } store.expiration = pq pq.wg.Add(1) go func() { t := time.NewTicker(reCheck) defer t.Stop() defer pq.wg.Done() for { now := time.Now().UnixNano() store.Lock() if pq.Len() > 0 { if it := pq.heap[0]; it.expiration <= now { path := store.pathForKey(it.publicKey) if err := os.Remove(path); err == nil { heap.Pop(pq) pq.unTrack(it.publicKey) xorAssign(&pq.hash, it.hash) store.Unlock() continue // we removed an entry, check next one right away } } } store.Unlock() select { case <-t.C: case <-quit: return } } }() } nats-server-2.10.27/server/dirstore_test.go000066400000000000000000000617021477524627100207100ustar00rootroot00000000000000// Copyright 2012-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/sha256" "fmt" "math" "math/rand" "os" "path/filepath" "strings" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" ) var ( one, two, three, four = "", "", "", "" jwt1, jwt2, jwt3, jwt4 = "", "", "", "" op nkeys.KeyPair ) func init() { op, _ = nkeys.CreateOperator() nkone, _ := nkeys.CreateAccount() pub, _ := nkone.PublicKey() one = pub ac := jwt.NewAccountClaims(pub) jwt1, _ = ac.Encode(op) nktwo, _ := nkeys.CreateAccount() pub, _ = nktwo.PublicKey() two = pub ac = jwt.NewAccountClaims(pub) jwt2, _ = ac.Encode(op) nkthree, _ := nkeys.CreateAccount() pub, _ = nkthree.PublicKey() three = pub ac = jwt.NewAccountClaims(pub) jwt3, _ = ac.Encode(op) nkfour, _ := nkeys.CreateAccount() pub, _ = nkfour.PublicKey() four = pub ac = jwt.NewAccountClaims(pub) jwt4, _ = ac.Encode(op) } func TestShardedDirStoreWriteAndReadonly(t *testing.T) { t.Parallel() dir := t.TempDir() store, err := NewDirJWTStore(dir, true, false) require_NoError(t, err) expected := map[string]string{ one: "alpha", two: "beta", three: "gamma", four: "delta", } for k, v := range expected { store.SaveAcc(k, v) } for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err := store.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) got, err = store.LoadAcc("") require_Error(t, err) require_Equal(t, "", got) err = store.SaveAcc("", "onetwothree") require_Error(t, err) store.Close() // re-use the folder for readonly mode store, err = NewImmutableDirJWTStore(dir, true) require_NoError(t, err) require_True(t, store.IsReadOnly()) err = store.SaveAcc("five", "omega") require_Error(t, err) for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } store.Close() } func TestUnshardedDirStoreWriteAndReadonly(t *testing.T) { t.Parallel() dir := t.TempDir() store, err := NewDirJWTStore(dir, false, false) require_NoError(t, err) expected := map[string]string{ one: "alpha", two: "beta", three: "gamma", four: "delta", } require_False(t, store.IsReadOnly()) for k, v := range expected { store.SaveAcc(k, v) } for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err := store.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) got, err = store.LoadAcc("") require_Error(t, err) require_Equal(t, "", got) err = store.SaveAcc("", "onetwothree") require_Error(t, err) store.Close() // re-use the folder for readonly mode store, err = NewImmutableDirJWTStore(dir, false) require_NoError(t, err) require_True(t, store.IsReadOnly()) err = store.SaveAcc("five", "omega") require_Error(t, err) for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } store.Close() } func TestNoCreateRequiresDir(t *testing.T) { t.Parallel() _, err := NewDirJWTStore("/a/b/c", true, false) require_Error(t, err) } func TestCreateMakesDir(t *testing.T) { t.Parallel() dir := t.TempDir() fullPath := filepath.Join(dir, "a/b") _, err := os.Stat(fullPath) require_Error(t, err) require_True(t, os.IsNotExist(err)) s, err := NewDirJWTStore(fullPath, false, true) require_NoError(t, err) s.Close() _, err = os.Stat(fullPath) require_NoError(t, err) } func TestShardedDirStorePackMerge(t *testing.T) { t.Parallel() dir := t.TempDir() dir2 := t.TempDir() dir3 := t.TempDir() store, err := NewDirJWTStore(dir, true, false) require_NoError(t, err) expected := map[string]string{ one: "alpha", two: "beta", three: "gamma", four: "delta", } require_False(t, store.IsReadOnly()) for k, v := range expected { store.SaveAcc(k, v) } for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err := store.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) pack, err := store.Pack(-1) require_NoError(t, err) inc, err := NewDirJWTStore(dir2, true, false) require_NoError(t, err) inc.Merge(pack) for k, v := range expected { got, err := inc.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err = inc.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) limitedPack, err := inc.Pack(1) require_NoError(t, err) limited, err := NewDirJWTStore(dir3, true, false) require_NoError(t, err) limited.Merge(limitedPack) count := 0 for k, v := range expected { got, err := limited.LoadAcc(k) if err == nil { count++ require_Equal(t, v, got) } } require_Len(t, 1, count) got, err = inc.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) } func TestShardedToUnsharedDirStorePackMerge(t *testing.T) { t.Parallel() dir := t.TempDir() dir2 := t.TempDir() store, err := NewDirJWTStore(dir, true, false) require_NoError(t, err) expected := map[string]string{ one: "alpha", two: "beta", three: "gamma", four: "delta", } require_False(t, store.IsReadOnly()) for k, v := range expected { store.SaveAcc(k, v) } for k, v := range expected { got, err := store.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err := store.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) pack, err := store.Pack(-1) require_NoError(t, err) inc, err := NewDirJWTStore(dir2, false, false) require_NoError(t, err) inc.Merge(pack) for k, v := range expected { got, err := inc.LoadAcc(k) require_NoError(t, err) require_Equal(t, v, got) } got, err = inc.LoadAcc("random") require_Error(t, err) require_Equal(t, "", got) err = store.Merge("foo") require_Error(t, err) err = store.Merge("") // will skip it require_NoError(t, err) err = store.Merge("a|something") // should fail on a for sharding require_Error(t, err) } func TestMergeOnlyOnNewer(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewDirJWTStore(dir, true, false) require_NoError(t, err) accountKey, err := nkeys.CreateAccount() require_NoError(t, err) pubKey, err := accountKey.PublicKey() require_NoError(t, err) account := jwt.NewAccountClaims(pubKey) account.Name = "old" olderJWT, err := account.Encode(accountKey) require_NoError(t, err) time.Sleep(2 * time.Second) account.Name = "new" newerJWT, err := account.Encode(accountKey) require_NoError(t, err) // Should work err = dirStore.SaveAcc(pubKey, olderJWT) require_NoError(t, err) fromStore, err := dirStore.LoadAcc(pubKey) require_NoError(t, err) require_Equal(t, olderJWT, fromStore) // should replace err = dirStore.saveIfNewer(pubKey, newerJWT) require_NoError(t, err) fromStore, err = dirStore.LoadAcc(pubKey) require_NoError(t, err) require_Equal(t, newerJWT, fromStore) // should fail err = dirStore.saveIfNewer(pubKey, olderJWT) require_NoError(t, err) fromStore, err = dirStore.LoadAcc(pubKey) require_NoError(t, err) require_Equal(t, newerJWT, fromStore) } func createTestAccount(t *testing.T, dirStore *DirJWTStore, expSec int, accKey nkeys.KeyPair) string { t.Helper() pubKey, err := accKey.PublicKey() require_NoError(t, err) account := jwt.NewAccountClaims(pubKey) if expSec > 0 { account.Expires = time.Now().Round(time.Second).Add(time.Second * time.Duration(expSec)).Unix() } jwt, err := account.Encode(accKey) require_NoError(t, err) err = dirStore.SaveAcc(pubKey, jwt) require_NoError(t, err) return jwt } func assertStoreSize(t *testing.T, dirStore *DirJWTStore, length int) { t.Helper() f, err := os.ReadDir(dirStore.directory) require_NoError(t, err) require_Len(t, len(f), length) dirStore.Lock() require_Len(t, len(dirStore.expiration.idx), length) require_Len(t, dirStore.expiration.lru.Len(), length) require_Len(t, len(dirStore.expiration.heap), length) dirStore.Unlock() } func TestExpiration(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil) require_NoError(t, err) defer dirStore.Close() account := func(expSec int) { accountKey, err := nkeys.CreateAccount() require_NoError(t, err) createTestAccount(t, dirStore, expSec, accountKey) } hBegin := dirStore.Hash() account(100) hNoExp := dirStore.Hash() require_NotEqual(t, hBegin, hNoExp) account(1) nh2 := dirStore.Hash() require_NotEqual(t, hNoExp, nh2) assertStoreSize(t, dirStore, 2) failAt := time.Now().Add(4 * time.Second) for time.Now().Before(failAt) { time.Sleep(100 * time.Millisecond) f, err := os.ReadDir(dir) require_NoError(t, err) if len(f) == 1 { lh := dirStore.Hash() require_Equal(t, string(hNoExp[:]), string(lh[:])) return } } t.Fatalf("Waited more than 4 seconds for the file with expiration 1 second to expire") } func TestLimit(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 5, true, 0, nil) require_NoError(t, err) defer dirStore.Close() account := func(expSec int) { accountKey, err := nkeys.CreateAccount() require_NoError(t, err) createTestAccount(t, dirStore, expSec, accountKey) } h := dirStore.Hash() accountKey, err := nkeys.CreateAccount() require_NoError(t, err) // update first account for i := 0; i < 10; i++ { createTestAccount(t, dirStore, 50, accountKey) assertStoreSize(t, dirStore, 1) } // new accounts for i := 0; i < 10; i++ { account(i) nh := dirStore.Hash() require_NotEqual(t, h, nh) h = nh } // first account should be gone now accountKey.PublicKey() key, _ := accountKey.PublicKey() _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, key)) require_True(t, os.IsNotExist(err)) // update first account for i := 0; i < 10; i++ { createTestAccount(t, dirStore, 50, accountKey) assertStoreSize(t, dirStore, 5) } } func TestLimitNoEvict(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, false, 0, nil) require_NoError(t, err) defer dirStore.Close() accountKey1, err := nkeys.CreateAccount() require_NoError(t, err) pKey1, err := accountKey1.PublicKey() require_NoError(t, err) accountKey2, err := nkeys.CreateAccount() require_NoError(t, err) accountKey3, err := nkeys.CreateAccount() require_NoError(t, err) pKey3, err := accountKey3.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 100, accountKey1) assertStoreSize(t, dirStore, 1) createTestAccount(t, dirStore, 1, accountKey2) assertStoreSize(t, dirStore, 2) hBefore := dirStore.Hash() // 2 jwt are already stored. third must result in an error pubKey, err := accountKey3.PublicKey() require_NoError(t, err) account := jwt.NewAccountClaims(pubKey) jwt, err := account.Encode(accountKey3) require_NoError(t, err) err = dirStore.SaveAcc(pubKey, jwt) require_Error(t, err) assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) require_NoError(t, err) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) require_True(t, os.IsNotExist(err)) // check that the hash did not change hAfter := dirStore.Hash() require_True(t, bytes.Equal(hBefore[:], hAfter[:])) // wait for expiration of account2 time.Sleep(2200 * time.Millisecond) err = dirStore.SaveAcc(pubKey, jwt) require_NoError(t, err) assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) require_NoError(t, err) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) require_NoError(t, err) } func TestLruLoad(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, nil) require_NoError(t, err) defer dirStore.Close() accountKey1, err := nkeys.CreateAccount() require_NoError(t, err) pKey1, err := accountKey1.PublicKey() require_NoError(t, err) accountKey2, err := nkeys.CreateAccount() require_NoError(t, err) accountKey3, err := nkeys.CreateAccount() require_NoError(t, err) pKey3, err := accountKey3.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 10, accountKey1) assertStoreSize(t, dirStore, 1) createTestAccount(t, dirStore, 10, accountKey2) assertStoreSize(t, dirStore, 2) dirStore.LoadAcc(pKey1) // will reorder 1/2 createTestAccount(t, dirStore, 10, accountKey3) assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) require_NoError(t, err) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) require_NoError(t, err) } func TestLruVolume(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil) require_NoError(t, err) defer dirStore.Close() replaceCnt := 500 // needs to be bigger than 2 due to loop unrolling keys := make([]string, replaceCnt) key, err := nkeys.CreateAccount() require_NoError(t, err) keys[0], err = key.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 10000, key) // not intended to expire assertStoreSize(t, dirStore, 1) key, err = nkeys.CreateAccount() require_NoError(t, err) keys[1], err = key.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 10000, key) assertStoreSize(t, dirStore, 2) for i := 2; i < replaceCnt; i++ { k, err := nkeys.CreateAccount() require_NoError(t, err) keys[i], err = k.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 10000+rand.Intn(10000), k) // not intended to expire assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i-2])) require_Error(t, err) require_True(t, os.IsNotExist(err)) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i-1])) require_NoError(t, err) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, keys[i])) require_NoError(t, err) } } func TestLru(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil) require_NoError(t, err) defer dirStore.Close() accountKey1, err := nkeys.CreateAccount() require_NoError(t, err) pKey1, err := accountKey1.PublicKey() require_NoError(t, err) accountKey2, err := nkeys.CreateAccount() require_NoError(t, err) accountKey3, err := nkeys.CreateAccount() require_NoError(t, err) pKey3, err := accountKey3.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 1000, accountKey1) assertStoreSize(t, dirStore, 1) createTestAccount(t, dirStore, 1000, accountKey2) assertStoreSize(t, dirStore, 2) createTestAccount(t, dirStore, 1000, accountKey3) assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) require_Error(t, err) require_True(t, os.IsNotExist(err)) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) require_NoError(t, err) // update -> will change this keys position for eviction createTestAccount(t, dirStore, 1000, accountKey2) assertStoreSize(t, dirStore, 2) // recreate -> will evict 3 createTestAccount(t, dirStore, 1, accountKey1) assertStoreSize(t, dirStore, 2) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey3)) require_True(t, os.IsNotExist(err)) // let key1 expire. sleep expSec=1 + 1 for rounding time.Sleep(2200 * time.Millisecond) assertStoreSize(t, dirStore, 1) _, err = os.Stat(fmt.Sprintf("%s/%s.jwt", dir, pKey1)) require_True(t, os.IsNotExist(err)) // recreate key3 - no eviction createTestAccount(t, dirStore, 1000, accountKey3) assertStoreSize(t, dirStore, 2) } func TestReload(t *testing.T) { t.Parallel() dir := t.TempDir() notificationChan := make(chan struct{}, 5) dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, func(publicKey string) { notificationChan <- struct{}{} }) require_NoError(t, err) defer dirStore.Close() newAccount := func() string { t.Helper() accKey, err := nkeys.CreateAccount() require_NoError(t, err) pKey, err := accKey.PublicKey() require_NoError(t, err) pubKey, err := accKey.PublicKey() require_NoError(t, err) account := jwt.NewAccountClaims(pubKey) jwt, err := account.Encode(accKey) require_NoError(t, err) file := fmt.Sprintf("%s/%s.jwt", dir, pKey) err = os.WriteFile(file, []byte(jwt), 0644) require_NoError(t, err) return file } files := make(map[string]struct{}) assertStoreSize(t, dirStore, 0) hash := dirStore.Hash() emptyHash := [sha256.Size]byte{} require_True(t, bytes.Equal(hash[:], emptyHash[:])) for i := 0; i < 5; i++ { files[newAccount()] = struct{}{} err = dirStore.Reload() require_NoError(t, err) <-notificationChan assertStoreSize(t, dirStore, i+1) hash = dirStore.Hash() require_False(t, bytes.Equal(hash[:], emptyHash[:])) msg, err := dirStore.Pack(-1) require_NoError(t, err) require_Len(t, len(strings.Split(msg, "\n")), len(files)) } for k := range files { hash = dirStore.Hash() require_False(t, bytes.Equal(hash[:], emptyHash[:])) removeFile(t, k) err = dirStore.Reload() require_NoError(t, err) assertStoreSize(t, dirStore, len(files)-1) delete(files, k) msg, err := dirStore.Pack(-1) require_NoError(t, err) if len(files) != 0 { // when len is 0, we have an empty line require_Len(t, len(strings.Split(msg, "\n")), len(files)) } } require_True(t, len(notificationChan) == 0) hash = dirStore.Hash() require_True(t, bytes.Equal(hash[:], emptyHash[:])) } func TestExpirationUpdate(t *testing.T) { t.Parallel() dir := t.TempDir() dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil) require_NoError(t, err) defer dirStore.Close() accountKey, err := nkeys.CreateAccount() require_NoError(t, err) h := dirStore.Hash() createTestAccount(t, dirStore, 0, accountKey) nh := dirStore.Hash() require_NotEqual(t, h, nh) h = nh time.Sleep(1500 * time.Millisecond) f, err := os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 1) createTestAccount(t, dirStore, 2, accountKey) nh = dirStore.Hash() require_NotEqual(t, h, nh) h = nh time.Sleep(1500 * time.Millisecond) f, err = os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 1) createTestAccount(t, dirStore, 0, accountKey) nh = dirStore.Hash() require_NotEqual(t, h, nh) h = nh time.Sleep(1500 * time.Millisecond) f, err = os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 1) createTestAccount(t, dirStore, 1, accountKey) nh = dirStore.Hash() require_NotEqual(t, h, nh) time.Sleep(1500 * time.Millisecond) f, err = os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 0) empty := [32]byte{} h = dirStore.Hash() require_Equal(t, string(h[:]), string(empty[:])) } func TestTTL(t *testing.T) { t.Parallel() dir := t.TempDir() require_OneJWT := func() { t.Helper() f, err := os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 1) } test := func(op func(store *DirJWTStore, accountKey nkeys.KeyPair, accountPubKey string, jwt string)) { dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, 50*time.Millisecond, 10, true, 200*time.Millisecond, nil) require_NoError(t, err) defer dirStore.Close() accountKey, err := nkeys.CreateAccount() require_NoError(t, err) pubKey, err := accountKey.PublicKey() require_NoError(t, err) jwt := createTestAccount(t, dirStore, 0, accountKey) require_OneJWT() // observe non expiration due to activity for i := 0; i < 4; i++ { time.Sleep(110 * time.Millisecond) op(dirStore, accountKey, pubKey, jwt) require_OneJWT() } // observe expiration for i := 0; i < 40; i++ { time.Sleep(50 * time.Millisecond) f, err := os.ReadDir(dir) require_NoError(t, err) if len(f) == 0 { return } } t.Fatalf("jwt should have expired by now") } t.Run("no expiration due to load", func(t *testing.T) { test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { store.LoadAcc(pubKey) }) }) t.Run("no expiration due to store", func(t *testing.T) { test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { store.SaveAcc(pubKey, jwt) }) }) t.Run("no expiration due to overwrite", func(t *testing.T) { test(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) { createTestAccount(t, store, 0, accountKey) }) }) } func TestRemove(t *testing.T) { for deleteType, test := range map[deleteType]struct { expected int moved int }{ HardDelete: {0, 0}, RenameDeleted: {0, 1}, NoDelete: {1, 0}, } { deleteType, test := deleteType, test // fixes govet capturing loop variables t.Run("", func(t *testing.T) { t.Parallel() dir := t.TempDir() require_OneJWT := func() { t.Helper() f, err := os.ReadDir(dir) require_NoError(t, err) require_Len(t, len(f), 1) } dirStore, err := NewExpiringDirJWTStore(dir, false, false, deleteType, 0, 10, true, 0, nil) delPubKey := "" dirStore.deleted = func(publicKey string) { delPubKey = publicKey } require_NoError(t, err) defer dirStore.Close() accountKey, err := nkeys.CreateAccount() require_NoError(t, err) pubKey, err := accountKey.PublicKey() require_NoError(t, err) createTestAccount(t, dirStore, 0, accountKey) require_OneJWT() dirStore.delete(pubKey) if deleteType == NoDelete { require_True(t, delPubKey == "") } else { require_True(t, delPubKey == pubKey) } f, err := filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt") require_NoError(t, err) require_Len(t, len(f), test.expected) f, err = filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt.deleted") require_NoError(t, err) require_Len(t, len(f), test.moved) }) } } const infDur = time.Duration(math.MaxInt64) func TestNotificationOnPack(t *testing.T) { t.Parallel() jwts := map[string]string{ one: jwt1, two: jwt2, three: jwt3, four: jwt4, } notificationChan := make(chan struct{}, len(jwts)) // set to same len so all extra will block notification := func(pubKey string) { if _, ok := jwts[pubKey]; !ok { t.Fatalf("Key not found: %s", pubKey) } notificationChan <- struct{}{} } dirPack := t.TempDir() packStore, err := NewExpiringDirJWTStore(dirPack, false, false, NoDelete, infDur, 0, true, 0, notification) require_NoError(t, err) // prefill the store with data for k, v := range jwts { require_NoError(t, packStore.SaveAcc(k, v)) } for i := 0; i < len(jwts); i++ { <-notificationChan } msg, err := packStore.Pack(-1) require_NoError(t, err) packStore.Close() hash := packStore.Hash() for _, shard := range []bool{true, false, true, false} { dirMerge := t.TempDir() mergeStore, err := NewExpiringDirJWTStore(dirMerge, shard, false, NoDelete, infDur, 0, true, 0, notification) require_NoError(t, err) // set err = mergeStore.Merge(msg) require_NoError(t, err) assertStoreSize(t, mergeStore, len(jwts)) hash1 := packStore.Hash() require_True(t, bytes.Equal(hash[:], hash1[:])) for i := 0; i < len(jwts); i++ { <-notificationChan } // overwrite - assure err = mergeStore.Merge(msg) require_NoError(t, err) assertStoreSize(t, mergeStore, len(jwts)) hash2 := packStore.Hash() require_True(t, bytes.Equal(hash1[:], hash2[:])) hash = hash1 msg, err = mergeStore.Pack(-1) require_NoError(t, err) mergeStore.Close() require_True(t, len(notificationChan) == 0) for k, v := range jwts { j, err := packStore.LoadAcc(k) require_NoError(t, err) require_Equal(t, j, v) } } } func TestNotificationOnPackWalk(t *testing.T) { t.Parallel() const storeCnt = 5 const keyCnt = 50 const iterCnt = 8 store := [storeCnt]*DirJWTStore{} for i := 0; i < storeCnt; i++ { dirMerge := t.TempDir() mergeStore, err := NewExpiringDirJWTStore(dirMerge, true, false, NoDelete, infDur, 0, true, 0, nil) require_NoError(t, err) store[i] = mergeStore } for i := 0; i < iterCnt; i++ { //iterations jwts := make(map[string]string) for j := 0; j < keyCnt; j++ { kp, _ := nkeys.CreateAccount() key, _ := kp.PublicKey() ac := jwt.NewAccountClaims(key) jwts[key], _ = ac.Encode(op) require_NoError(t, store[0].SaveAcc(key, jwts[key])) } for j := 0; j < storeCnt-1; j++ { // stores err := store[j].PackWalk(3, func(partialPackMsg string) { err := store[j+1].Merge(partialPackMsg) require_NoError(t, err) }) require_NoError(t, err) } for i := 0; i < storeCnt-1; i++ { h1 := store[i].Hash() h2 := store[i+1].Hash() require_True(t, bytes.Equal(h1[:], h2[:])) } } for i := 0; i < storeCnt; i++ { store[i].Close() } } nats-server-2.10.27/server/disk_avail.go000066400000000000000000000022311477524627100201140ustar00rootroot00000000000000// Copyright 2020-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !openbsd && !netbsd && !wasm // +build !windows,!openbsd,!netbsd,!wasm package server import ( "os" "syscall" ) func diskAvailable(storeDir string) int64 { var ba int64 if _, err := os.Stat(storeDir); os.IsNotExist(err) { os.MkdirAll(storeDir, defaultDirPerms) } var fs syscall.Statfs_t if err := syscall.Statfs(storeDir, &fs); err == nil { // Estimate 75% of available storage. ba = int64(uint64(fs.Bavail) * uint64(fs.Bsize) / 4 * 3) } else { // Used 1TB default as a guess if all else fails. ba = JetStreamMaxStoreDefault } return ba } nats-server-2.10.27/server/disk_avail_netbsd.go000066400000000000000000000014061477524627100214560ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build netbsd // +build netbsd package server // TODO - See if there is a version of this for NetBSD. func diskAvailable(storeDir string) int64 { return JetStreamMaxStoreDefault } nats-server-2.10.27/server/disk_avail_openbsd.go000066400000000000000000000021371477524627100216330ustar00rootroot00000000000000// Copyright 2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build openbsd // +build openbsd package server import ( "os" "syscall" ) func diskAvailable(storeDir string) int64 { var ba int64 if _, err := os.Stat(storeDir); os.IsNotExist(err) { os.MkdirAll(storeDir, defaultDirPerms) } var fs syscall.Statfs_t if err := syscall.Statfs(storeDir, &fs); err == nil { // Estimate 75% of available storage. ba = int64(uint64(fs.F_bavail) * uint64(fs.F_bsize) / 4 * 3) } else { // Used 1TB default as a guess if all else fails. ba = JetStreamMaxStoreDefault } return ba } nats-server-2.10.27/server/disk_avail_wasm.go000066400000000000000000000013171477524627100211470ustar00rootroot00000000000000// Copyright 2022-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build wasm // +build wasm package server func diskAvailable(storeDir string) int64 { return JetStreamMaxStoreDefault } nats-server-2.10.27/server/disk_avail_windows.go000066400000000000000000000014231477524627100216700ustar00rootroot00000000000000// Copyright 2020-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows // +build windows package server // TODO(dlc) - See if there is a version of this for windows. func diskAvailable(storeDir string) int64 { return JetStreamMaxStoreDefault } nats-server-2.10.27/server/errors.go000066400000000000000000000374711477524627100173400ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "errors" "fmt" ) var ( // ErrConnectionClosed represents an error condition on a closed connection. ErrConnectionClosed = errors.New("connection closed") // ErrAuthentication represents an error condition on failed authentication. ErrAuthentication = errors.New("authentication error") // ErrAuthTimeout represents an error condition on failed authorization due to timeout. ErrAuthTimeout = errors.New("authentication timeout") // ErrAuthExpired represents an expired authorization due to timeout. ErrAuthExpired = errors.New("authentication expired") // ErrMaxPayload represents an error condition when the payload is too big. ErrMaxPayload = errors.New("maximum payload exceeded") // ErrMaxControlLine represents an error condition when the control line is too big. ErrMaxControlLine = errors.New("maximum control line exceeded") // ErrReservedPublishSubject represents an error condition when sending to a reserved subject, e.g. _SYS.> ErrReservedPublishSubject = errors.New("reserved internal subject") // ErrBadPublishSubject represents an error condition for an invalid publish subject. ErrBadPublishSubject = errors.New("invalid publish subject") // ErrBadSubject represents an error condition for an invalid subject. ErrBadSubject = errors.New("invalid subject") // ErrBadQualifier is used to error on a bad qualifier for a transform. ErrBadQualifier = errors.New("bad qualifier") // ErrBadClientProtocol signals a client requested an invalid client protocol. ErrBadClientProtocol = errors.New("invalid client protocol") // ErrTooManyConnections signals a client that the maximum number of connections supported by the // server has been reached. ErrTooManyConnections = errors.New("maximum connections exceeded") // ErrTooManyAccountConnections signals that an account has reached its maximum number of active // connections. ErrTooManyAccountConnections = errors.New("maximum account active connections exceeded") // ErrLeafNodeLoop signals a leafnode is trying to register for a cluster we already have registered. ErrLeafNodeLoop = errors.New("leafnode loop detected") // ErrTooManySubs signals a client that the maximum number of subscriptions per connection // has been reached. ErrTooManySubs = errors.New("maximum subscriptions exceeded") // ErrTooManySubTokens signals a client that the subject has too many tokens. ErrTooManySubTokens = errors.New("subject has exceeded number of tokens limit") // ErrClientConnectedToRoutePort represents an error condition when a client // attempted to connect to the route listen port. ErrClientConnectedToRoutePort = errors.New("attempted to connect to route port") // ErrClientConnectedToLeafNodePort represents an error condition when a client // attempted to connect to the leaf node listen port. ErrClientConnectedToLeafNodePort = errors.New("attempted to connect to leaf node port") // ErrLeafNodeHasSameClusterName represents an error condition when a leafnode is a cluster // and it has the same cluster name as the hub cluster. ErrLeafNodeHasSameClusterName = errors.New("remote leafnode has same cluster name") // ErrLeafNodeDisabled is when we disable leafnodes. ErrLeafNodeDisabled = errors.New("leafnodes disabled") // ErrConnectedToWrongPort represents an error condition when a connection is attempted // to the wrong listen port (for instance a LeafNode to a client port, etc...) ErrConnectedToWrongPort = errors.New("attempted to connect to wrong port") // ErrAccountExists is returned when an account is attempted to be registered // but already exists. ErrAccountExists = errors.New("account exists") // ErrBadAccount represents a malformed or incorrect account. ErrBadAccount = errors.New("bad account") // ErrReservedAccount represents a reserved account that can not be created. ErrReservedAccount = errors.New("reserved account") // ErrMissingAccount is returned when an account does not exist. ErrMissingAccount = errors.New("account missing") // ErrMissingService is returned when an account does not have an exported service. ErrMissingService = errors.New("service missing") // ErrBadServiceType is returned when latency tracking is being applied to non-singleton response types. ErrBadServiceType = errors.New("bad service response type") // ErrBadSampling is returned when the sampling for latency tracking is not 1 >= sample <= 100. ErrBadSampling = errors.New("bad sampling percentage, should be 1-100") // ErrAccountValidation is returned when an account has failed validation. ErrAccountValidation = errors.New("account validation failed") // ErrAccountExpired is returned when an account has expired. ErrAccountExpired = errors.New("account expired") // ErrNoAccountResolver is returned when we attempt an update but do not have an account resolver. ErrNoAccountResolver = errors.New("account resolver missing") // ErrAccountResolverUpdateTooSoon is returned when we attempt an update too soon to last request. ErrAccountResolverUpdateTooSoon = errors.New("account resolver update too soon") // ErrAccountResolverSameClaims is returned when same claims have been fetched. ErrAccountResolverSameClaims = errors.New("account resolver no new claims") // ErrStreamImportAuthorization is returned when a stream import is not authorized. ErrStreamImportAuthorization = errors.New("stream import not authorized") // ErrStreamImportBadPrefix is returned when a stream import prefix contains wildcards. ErrStreamImportBadPrefix = errors.New("stream import prefix can not contain wildcard tokens") // ErrStreamImportDuplicate is returned when a stream import is a duplicate of one that already exists. ErrStreamImportDuplicate = errors.New("stream import already exists") // ErrServiceImportAuthorization is returned when a service import is not authorized. ErrServiceImportAuthorization = errors.New("service import not authorized") // ErrImportFormsCycle is returned when an import would form a cycle. ErrImportFormsCycle = errors.New("import forms a cycle") // ErrCycleSearchDepth is returned when we have exceeded our maximum search depth.. ErrCycleSearchDepth = errors.New("search cycle depth exhausted") // ErrClientOrRouteConnectedToGatewayPort represents an error condition when // a client or route attempted to connect to the Gateway port. ErrClientOrRouteConnectedToGatewayPort = errors.New("attempted to connect to gateway port") // ErrWrongGateway represents an error condition when a server receives a connect // request from a remote Gateway with a destination name that does not match the server's // Gateway's name. ErrWrongGateway = errors.New("wrong gateway") // ErrNoSysAccount is returned when an attempt to publish or subscribe is made // when there is no internal system account defined. ErrNoSysAccount = errors.New("system account not setup") // ErrRevocation is returned when a credential has been revoked. ErrRevocation = errors.New("credentials have been revoked") // ErrServerNotRunning is used to signal an error that a server is not running. ErrServerNotRunning = errors.New("server is not running") // ErrBadMsgHeader signals the parser detected a bad message header ErrBadMsgHeader = errors.New("bad message header detected") // ErrMsgHeadersNotSupported signals the parser detected a message header // but they are not supported on this server. ErrMsgHeadersNotSupported = errors.New("message headers not supported") // ErrNoRespondersRequiresHeaders signals that a client needs to have headers // on if they want no responders behavior. ErrNoRespondersRequiresHeaders = errors.New("no responders requires headers support") // ErrClusterNameConfigConflict signals that the options for cluster name in cluster and gateway are in conflict. ErrClusterNameConfigConflict = errors.New("cluster name conflicts between cluster and gateway definitions") // ErrClusterNameRemoteConflict signals that a remote server has a different cluster name. ErrClusterNameRemoteConflict = errors.New("cluster name from remote server conflicts") // ErrClusterNameHasSpaces signals that the cluster name contains spaces, which is not allowed. ErrClusterNameHasSpaces = errors.New("cluster name cannot contain spaces or new lines") // ErrMalformedSubject is returned when a subscription is made with a subject that does not conform to subject rules. ErrMalformedSubject = errors.New("malformed subject") // ErrSubscribePermissionViolation is returned when processing of a subscription fails due to permissions. ErrSubscribePermissionViolation = errors.New("subscribe permission violation") // ErrNoTransforms signals no subject transforms are available to map this subject. ErrNoTransforms = errors.New("no matching transforms available") // ErrCertNotPinned is returned when pinned certs are set and the certificate is not in it ErrCertNotPinned = errors.New("certificate not pinned") // ErrDuplicateServerName is returned when processing a server remote connection and // the server reports that this server name is already used in the cluster. ErrDuplicateServerName = errors.New("duplicate server name") // ErrMinimumVersionRequired is returned when a connection is not at the minimum version required. ErrMinimumVersionRequired = errors.New("minimum version required") // ErrInvalidMappingDestination is used for all subject mapping destination errors ErrInvalidMappingDestination = errors.New("invalid mapping destination") // ErrInvalidMappingDestinationSubject is used to error on a bad transform destination mapping ErrInvalidMappingDestinationSubject = fmt.Errorf("%w: invalid subject", ErrInvalidMappingDestination) // ErrMappingDestinationNotUsingAllWildcards is used to error on a transform destination not using all of the token wildcards ErrMappingDestinationNotUsingAllWildcards = fmt.Errorf("%w: not using all of the token wildcard(s)", ErrInvalidMappingDestination) // ErrUnknownMappingDestinationFunction is returned when a subject mapping destination contains an unknown mustache-escaped mapping function. ErrUnknownMappingDestinationFunction = fmt.Errorf("%w: unknown function", ErrInvalidMappingDestination) // ErrMappingDestinationIndexOutOfRange is returned when the mapping destination function is passed an out of range wildcard index value for one of it's arguments ErrMappingDestinationIndexOutOfRange = fmt.Errorf("%w: wildcard index out of range", ErrInvalidMappingDestination) // ErrMappingDestinationNotEnoughArgs is returned when the mapping destination function is not passed enough arguments ErrMappingDestinationNotEnoughArgs = fmt.Errorf("%w: not enough arguments passed to the function", ErrInvalidMappingDestination) // ErrMappingDestinationInvalidArg is returned when the mapping destination function is passed and invalid argument ErrMappingDestinationInvalidArg = fmt.Errorf("%w: function argument is invalid or in the wrong format", ErrInvalidMappingDestination) // ErrMappingDestinationTooManyArgs is returned when the mapping destination function is passed too many arguments ErrMappingDestinationTooManyArgs = fmt.Errorf("%w: too many arguments passed to the function", ErrInvalidMappingDestination) // ErrMappingDestinationNotSupportedForImport is returned when you try to use a mapping function other than wildcard in a transform that needs to be reversible (i.e. an import) ErrMappingDestinationNotSupportedForImport = fmt.Errorf("%w: the only mapping function allowed for import transforms is {{Wildcard()}}", ErrInvalidMappingDestination) ) // mappingDestinationErr is a type of subject mapping destination error type mappingDestinationErr struct { token string err error } func (e *mappingDestinationErr) Error() string { return fmt.Sprintf("%s in %s", e.err, e.token) } func (e *mappingDestinationErr) Is(target error) bool { return target == ErrInvalidMappingDestination } // configErr is a configuration error. type configErr struct { token token reason string } // Source reports the location of a configuration error. func (e *configErr) Source() string { return fmt.Sprintf("%s:%d:%d", e.token.SourceFile(), e.token.Line(), e.token.Position()) } // Error reports the location and reason from a configuration error. func (e *configErr) Error() string { if e.token != nil { return fmt.Sprintf("%s: %s", e.Source(), e.reason) } return e.reason } // unknownConfigFieldErr is an error reported in pedantic mode. type unknownConfigFieldErr struct { configErr field string } // Error reports that an unknown field was in the configuration. func (e *unknownConfigFieldErr) Error() string { return fmt.Sprintf("%s: unknown field %q", e.Source(), e.field) } // configWarningErr is an error reported in pedantic mode. type configWarningErr struct { configErr field string } // Error reports a configuration warning. func (e *configWarningErr) Error() string { return fmt.Sprintf("%s: invalid use of field %q: %s", e.Source(), e.field, e.reason) } // processConfigErr is the result of processing the configuration from the server. type processConfigErr struct { errors []error warnings []error } // Error returns the collection of errors separated by new lines, // warnings appear first then hard errors. func (e *processConfigErr) Error() string { var msg string for _, err := range e.Warnings() { msg += err.Error() + "\n" } for _, err := range e.Errors() { msg += err.Error() + "\n" } return msg } // Warnings returns the list of warnings. func (e *processConfigErr) Warnings() []error { return e.warnings } // Errors returns the list of errors. func (e *processConfigErr) Errors() []error { return e.errors } // errCtx wraps an error and stores additional ctx information for tracing. // Does not print or return it unless explicitly requested. type errCtx struct { error ctx string } func NewErrorCtx(err error, format string, args ...any) error { return &errCtx{err, fmt.Sprintf(format, args...)} } // Unwrap implement to work with errors.Is and errors.As func (e *errCtx) Unwrap() error { if e == nil { return nil } return e.error } // Context for error func (e *errCtx) Context() string { if e == nil { return "" } return e.ctx } // UnpackIfErrorCtx return Error or, if type is right error and context func UnpackIfErrorCtx(err error) string { if e, ok := err.(*errCtx); ok { if _, ok := e.error.(*errCtx); ok { return fmt.Sprint(UnpackIfErrorCtx(e.error), ": ", e.Context()) } return fmt.Sprint(e.Error(), ": ", e.Context()) } return err.Error() } // implements: go 1.13 errors.Unwrap(err error) error // TODO replace with native code once we no longer support go1.12 func errorsUnwrap(err error) error { u, ok := err.(interface { Unwrap() error }) if !ok { return nil } return u.Unwrap() } // ErrorIs implements: go 1.13 errors.Is(err, target error) bool // TODO replace with native code once we no longer support go1.12 func ErrorIs(err, target error) bool { // this is an outright copy of go 1.13 errors.Is(err, target error) bool // removed isComparable if err == nil || target == nil { return err == target } for { if err == target { return true } if x, ok := err.(interface{ Is(error) bool }); ok && x.Is(target) { return true } // TODO: consider supporing target.Is(err). This would allow // user-definable predicates, but also may allow for coping with sloppy // APIs, thereby making it easier to get away with them. if err = errorsUnwrap(err); err == nil { return false } } } nats-server-2.10.27/server/errors.json000066400000000000000000001072501477524627100176750ustar00rootroot00000000000000[ { "constant": "JSClusterPeerNotMemberErr", "code": 400, "error_code": 10040, "description": "peer not a member", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerEphemeralWithDurableInSubjectErr", "code": 400, "error_code": 10019, "description": "consumer expected to be ephemeral but detected a durable name set in subject", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamExternalDelPrefixOverlapsErrF", "code": 400, "error_code": 10022, "description": "stream external delivery prefix {prefix} overlaps with stream subject {subject}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSAccountResourcesExceededErr", "code": 400, "error_code": 10002, "description": "resource limits exceeded for account", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterNotAvailErr", "code": 503, "error_code": 10008, "description": "JetStream system temporarily unavailable", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamSubjectOverlapErr", "code": 400, "error_code": 10065, "description": "subjects overlap with an existing stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamWrongLastSequenceErrF", "code": 400, "error_code": 10071, "description": "wrong last sequence: {seq}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSTemplateNameNotMatchSubjectErr", "code": 400, "error_code": 10073, "description": "template name in subject does not match request", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterNoPeersErrF", "code": 400, "error_code": 10005, "description": "{err}", "comment": "Error causing no peers to be available", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerEphemeralWithDurableNameErr", "code": 400, "error_code": 10020, "description": "consumer expected to be ephemeral but a durable name was set in request", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSInsufficientResourcesErr", "code": 503, "error_code": 10023, "description": "insufficient resources", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamResourcesExceeded" }, { "constant": "JSMirrorMaxMessageSizeTooBigErr", "code": 400, "error_code": 10030, "description": "stream mirror must have max message size \u003e= source", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamTemplateDeleteErrF", "code": 500, "error_code": 10067, "description": "{err}", "comment": "Generic stream template deletion failed error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSBadRequestErr", "code": 400, "error_code": 10003, "description": "bad request", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterUnSupportFeatureErr", "code": 503, "error_code": 10036, "description": "not currently supported in clustered mode", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerNotFoundErr", "code": 404, "error_code": 10014, "description": "consumer not found", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceMaxMessageSizeTooBigErr", "code": 400, "error_code": 10046, "description": "stream source must have max message size \u003e= target", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamAssignmentErrF", "code": 500, "error_code": 10048, "description": "{err}", "comment": "Generic stream assignment error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMessageExceedsMaximumErr", "code": 400, "error_code": 10054, "description": "message size exceeds maximum allowed", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamTemplateCreateErrF", "code": 500, "error_code": 10066, "description": "{err}", "comment": "Generic template creation failed string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSInvalidJSONErr", "code": 400, "error_code": 10025, "description": "invalid JSON", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamInvalidExternalDeliverySubjErrF", "code": 400, "error_code": 10024, "description": "stream external delivery prefix {prefix} must not contain wildcards", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamRestoreErrF", "code": 500, "error_code": 10062, "description": "restore failed: {err}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterIncompleteErr", "code": 503, "error_code": 10004, "description": "incomplete results", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSNoAccountErr", "code": 503, "error_code": 10035, "description": "account not found", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSRaftGeneralErrF", "code": 500, "error_code": 10041, "description": "{err}", "comment": "General RAFT error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSRestoreSubscribeFailedErrF", "code": 500, "error_code": 10042, "description": "JetStream unable to subscribe to restore snapshot {subject}: {err}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamDeleteErrF", "code": 500, "error_code": 10050, "description": "{err}", "comment": "General stream deletion error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamExternalApiOverlapErrF", "code": 400, "error_code": 10021, "description": "stream external api prefix {prefix} must not overlap with {subject}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorWithSubjectsErr", "code": 400, "error_code": 10034, "description": "stream mirrors can not contain subjects", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorWithFirstSeqErr", "code": 400, "error_code": 10143, "description": "stream mirrors can not have first sequence configured", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSNotEnabledErr", "code": 503, "error_code": 10076, "description": "JetStream not enabled", "comment": "", "help": "This error indicates that JetStream is not enabled at a global level", "url": "", "deprecates": "ErrJetStreamNotEnabled" }, { "constant": "JSNotEnabledForAccountErr", "code": 503, "error_code": 10039, "description": "JetStream not enabled for account", "comment": "", "help": "This error indicates that JetStream is not enabled for an account account level", "url": "", "deprecates": "" }, { "constant": "JSSequenceNotFoundErrF", "code": 400, "error_code": 10043, "description": "sequence {seq} not found", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMirrorNotUpdatableErr", "code": 400, "error_code": 10055, "description": "stream mirror configuration can not be updated", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamSequenceNotMatchErr", "code": 503, "error_code": 10063, "description": "expected stream sequence does not match", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamWrongLastMsgIDErrF", "code": 400, "error_code": 10070, "description": "wrong last msg ID: {id}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSTempStorageFailedErr", "code": 500, "error_code": 10072, "description": "JetStream unable to open temp storage for restore", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStorageResourcesExceededErr", "code": 500, "error_code": 10047, "description": "insufficient storage resources available", "comment": "", "help": "", "url": "", "deprecates": "ErrStorageResourcesExceeded" }, { "constant": "JSStreamMismatchErr", "code": 400, "error_code": 10056, "description": "stream name in subject does not match request", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamNotMatchErr", "code": 400, "error_code": 10060, "description": "expected stream does not match", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorConsumerSetupFailedErrF", "code": 500, "error_code": 10029, "description": "{err}", "comment": "generic mirror consumer setup failure string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSNotEmptyRequestErr", "code": 400, "error_code": 10038, "description": "expected an empty request payload", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamNameExistErr", "code": 400, "error_code": 10058, "description": "stream name already in use with a different configuration", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamStreamAlreadyUsed" }, { "constant": "JSClusterTagsErr", "code": 400, "error_code": 10011, "description": "tags placement not supported for operation", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMaximumConsumersLimitErr", "code": 400, "error_code": 10026, "description": "maximum consumers limit reached", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceConsumerSetupFailedErrF", "code": 500, "error_code": 10045, "description": "{err}", "comment": "General source consumer setup failure string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerCreateErrF", "code": 500, "error_code": 10012, "description": "{err}", "comment": "General consumer creation failure string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDurableNameNotInSubjectErr", "code": 400, "error_code": 10016, "description": "consumer expected to be durable but no durable name set in subject", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamLimitsErrF", "code": 500, "error_code": 10053, "description": "{err}", "comment": "General stream limits exceeded error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamReplicasNotUpdatableErr", "code": 400, "error_code": 10061, "description": "Replicas configuration can not be updated", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamTemplateNotFoundErr", "code": 404, "error_code": 10068, "description": "template not found", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterNotAssignedErr", "code": 500, "error_code": 10007, "description": "JetStream cluster not assigned to this server", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamNotAssigned" }, { "constant": "JSClusterNotLeaderErr", "code": 500, "error_code": 10009, "description": "JetStream cluster can not handle request", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamNotLeader" }, { "constant": "JSConsumerNameExistErr", "code": 400, "error_code": 10013, "description": "consumer name already in use", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamConsumerAlreadyUsed" }, { "constant": "JSMirrorWithSourcesErr", "code": 400, "error_code": 10031, "description": "stream mirrors can not also contain other sources", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamNotFoundErr", "code": 404, "error_code": 10059, "description": "stream not found", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamStreamNotFound" }, { "constant": "JSClusterRequiredErr", "code": 503, "error_code": 10010, "description": "JetStream clustering support required", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDurableNameNotSetErr", "code": 400, "error_code": 10018, "description": "consumer expected to be durable but a durable name was not set", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMaximumStreamsLimitErr", "code": 400, "error_code": 10027, "description": "maximum number of streams reached", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorWithStartSeqAndTimeErr", "code": 400, "error_code": 10032, "description": "stream mirrors can not have both start seq and start time configured", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamSnapshotErrF", "code": 500, "error_code": 10064, "description": "snapshot failed: {err}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamUpdateErrF", "code": 500, "error_code": 10069, "description": "{err}", "comment": "Generic stream update error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterNotActiveErr", "code": 500, "error_code": 10006, "description": "JetStream not in clustered mode", "comment": "", "help": "", "url": "", "deprecates": "ErrJetStreamNotClustered" }, { "constant": "JSConsumerDurableNameNotMatchSubjectErr", "code": 400, "error_code": 10017, "description": "consumer name in subject does not match durable name in request", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMemoryResourcesExceededErr", "code": 500, "error_code": 10028, "description": "insufficient memory resources available", "comment": "", "help": "", "url": "", "deprecates": "ErrMemoryResourcesExceeded" }, { "constant": "JSMirrorWithSubjectFiltersErr", "code": 400, "error_code": 10033, "description": "stream mirrors can not contain filtered subjects", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamCreateErrF", "code": 500, "error_code": 10049, "description": "{err}", "comment": "Generic stream creation error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSClusterServerNotMemberErr", "code": 400, "error_code": 10044, "description": "server is not a member of the cluster", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSNoMessageFoundErr", "code": 404, "error_code": 10037, "description": "no message found", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSnapshotDeliverSubjectInvalidErr", "code": 400, "error_code": 10015, "description": "deliver subject not valid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamGeneralErrorF", "code": 500, "error_code": 10051, "description": "{err}", "comment": "General stream failure string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamInvalidConfigF", "code": 500, "error_code": 10052, "description": "{err}", "comment": "Stream configuration validation error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamReplicasNotSupportedErr", "code": 500, "error_code": 10074, "description": "replicas \u003e 1 not supported in non-clustered mode", "comment": "", "help": "", "url": "", "deprecates": "ErrReplicasNotSupported" }, { "constant": "JSStreamMsgDeleteFailedF", "code": 500, "error_code": 10057, "description": "{err}", "comment": "Generic message deletion failure error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSPeerRemapErr", "code": 503, "error_code": 10075, "description": "peer remap failed", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamStoreFailedF", "code": 503, "error_code": 10077, "description": "{err}", "comment": "Generic error when storing a message failed", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerConfigRequiredErr", "code": 400, "error_code": 10078, "description": "consumer config required", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDeliverToWildcardsErr", "code": 400, "error_code": 10079, "description": "consumer deliver subject has wildcards", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerPushMaxWaitingErr", "code": 400, "error_code": 10080, "description": "consumer in push mode can not set max waiting", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDeliverCycleErr", "code": 400, "error_code": 10081, "description": "consumer deliver subject forms a cycle", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxPendingAckPolicyRequiredErr", "code": 400, "error_code": 10082, "description": "consumer requires ack policy for max ack pending", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerSmallHeartbeatErr", "code": 400, "error_code": 10083, "description": "consumer idle heartbeat needs to be \u003e= 100ms", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerPullRequiresAckErr", "code": 400, "error_code": 10084, "description": "consumer in pull mode requires ack policy", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerPullNotDurableErr", "code": 400, "error_code": 10085, "description": "consumer in pull mode requires a durable name", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerPullWithRateLimitErr", "code": 400, "error_code": 10086, "description": "consumer in pull mode can not have rate limit set", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxWaitingNegativeErr", "code": 400, "error_code": 10087, "description": "consumer max waiting needs to be positive", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerHBRequiresPushErr", "code": 400, "error_code": 10088, "description": "consumer idle heartbeat requires a push based consumer", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerFCRequiresPushErr", "code": 400, "error_code": 10089, "description": "consumer flow control requires a push based consumer", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDirectRequiresPushErr", "code": 400, "error_code": 10090, "description": "consumer direct requires a push based consumer", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDirectRequiresEphemeralErr", "code": 400, "error_code": 10091, "description": "consumer direct requires an ephemeral consumer", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerOnMappedErr", "code": 400, "error_code": 10092, "description": "consumer direct on a mapped consumer", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerFilterNotSubsetErr", "code": 400, "error_code": 10093, "description": "consumer filter subject is not a valid subset of the interest subjects", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerInvalidPolicyErrF", "code": 400, "error_code": 10094, "description": "{err}", "comment": "Generic delivery policy error", "help": "Error returned for impossible deliver policies when combined with start sequences etc", "url": "", "deprecates": "" }, { "constant": "JSConsumerInvalidSamplingErrF", "code": 400, "error_code": 10095, "description": "failed to parse consumer sampling configuration: {err}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamInvalidErr", "code": 500, "error_code": 10096, "description": "stream not valid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerWQRequiresExplicitAckErr", "code": 400, "error_code": 10098, "description": "workqueue stream requires explicit ack", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerWQMultipleUnfilteredErr", "code": 400, "error_code": 10099, "description": "multiple non-filtered consumers not allowed on workqueue stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerWQConsumerNotUniqueErr", "code": 400, "error_code": 10100, "description": "filtered consumer not unique on workqueue stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerWQConsumerNotDeliverAllErr", "code": 400, "error_code": 10101, "description": "consumer must be deliver all on workqueue stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerNameTooLongErrF", "code": 400, "error_code": 10102, "description": "consumer name is too long, maximum allowed is {max}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerBadDurableNameErr", "code": 400, "error_code": 10103, "description": "durable name can not contain '.', '*', '\u003e'", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerStoreFailedErrF", "code": 500, "error_code": 10104, "description": "error creating store for consumer: {err}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerExistingActiveErr", "code": 400, "error_code": 10105, "description": "consumer already exists and is still active", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerReplacementWithDifferentNameErr", "code": 400, "error_code": 10106, "description": "consumer replacement durable config not the same", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDescriptionTooLongErrF", "code": 400, "error_code": 10107, "description": "consumer description is too long, maximum allowed is {max}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamHeaderExceedsMaximumErr", "code": 400, "error_code": 10097, "description": "header size exceeds maximum allowed of 64k", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerWithFlowControlNeedsHeartbeats", "code": 400, "error_code": 10108, "description": "consumer with flow control also needs heartbeats", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamSealedErr", "code": 400, "error_code": 10109, "description": "invalid operation on sealed stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamPurgeFailedF", "code": 500, "error_code": 10110, "description": "{err}", "comment": "Generic stream purge failure error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamRollupFailedF", "code": 500, "error_code": 10111, "description": "{err}", "comment": "Generic stream rollup failure error string", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerInvalidDeliverSubject", "code": 400, "error_code": 10112, "description": "invalid push consumer deliver subject", "comment": "", "help": "Returned when the delivery subject on a Push Consumer is not a valid NATS Subject", "url": "", "deprecates": "" }, { "constant": "JSStreamMaxBytesRequired", "code": 400, "error_code": 10113, "description": "account requires a stream config to have max bytes set", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxRequestBatchNegativeErr", "code": 400, "error_code": 10114, "description": "consumer max request batch needs to be \u003e 0", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxRequestExpiresToSmall", "code": 400, "error_code": 10115, "description": "consumer max request expires needs to be \u003e= 1ms", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxDeliverBackoffErr", "code": 400, "error_code": 10116, "description": "max deliver is required to be \u003e length of backoff values", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamInfoMaxSubjectsErr", "code": 500, "error_code": 10117, "description": "subject details would exceed maximum allowed", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamOfflineErr", "code": 500, "error_code": 10118, "description": "stream is offline", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerOfflineErr", "code": 500, "error_code": 10119, "description": "consumer is offline", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSNoLimitsErr", "code": 400, "error_code": 10120, "description": "no JetStream default or applicable tiered limit present", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxPendingAckExcessErrF", "code": 400, "error_code": 10121, "description": "consumer max ack pending exceeds system limit of {limit}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMaxStreamBytesExceeded", "code": 400, "error_code": 10122, "description": "stream max bytes exceeds account limit max stream bytes", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMoveAndScaleErr", "code": 400, "error_code": 10123, "description": "can not move and scale a stream in a single update", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMoveInProgressF", "code": 400, "error_code": 10124, "description": "stream move already in progress: {msg}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMaxRequestBatchExceededF", "code": 400, "error_code": 10125, "description": "consumer max request batch exceeds server limit of {limit}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerReplicasExceedsStream", "code": 400, "error_code": 10126, "description": "consumer config replica count exceeds parent stream", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerNameContainsPathSeparatorsErr", "code": 400, "error_code": 10127, "description": "Consumer name can not contain path separators", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamNameContainsPathSeparatorsErr", "code": 400, "error_code": 10128, "description": "Stream name can not contain path separators", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamMoveNotInProgress", "code": 400, "error_code": 10129, "description": "stream move not in progress", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSStreamNameExistRestoreFailedErr", "code": 400, "error_code": 10130, "description": "stream name already in use, cannot restore", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerCreateFilterSubjectMismatchErr", "code": 400, "error_code": 10131, "description": "Consumer create request did not match filtered subject from create subject", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerCreateDurableAndNameMismatch", "code": 400, "error_code": 10132, "description": "Consumer Durable and Name have to be equal if both are provided", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSReplicasCountCannotBeNegative", "code": 400, "error_code": 10133, "description": "replicas count cannot be negative", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerReplicasShouldMatchStream", "code": 400, "error_code": 10134, "description": "consumer config replicas must match interest retention stream's replicas", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMetadataLengthErrF", "code": 400, "error_code": 10135, "description": "consumer metadata exceeds maximum size of {limit}", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDuplicateFilterSubjects", "code": 400, "error_code": 10136, "description": "consumer cannot have both FilterSubject and FilterSubjects specified", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerMultipleFiltersNotAllowed", "code": 400, "error_code": 10137, "description": "consumer with multiple subject filters cannot use subject based API", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerOverlappingSubjectFilters", "code": 400, "error_code": 10138, "description": "consumer subject filters cannot overlap", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerEmptyFilter", "code": 400, "error_code": 10139, "description": "consumer filter in FilterSubjects cannot be empty", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceDuplicateDetected", "code": 400, "error_code": 10140, "description": "duplicate source configuration detected", "comment": "source stream, filter and transform (plus external if present) must form a unique combination", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceInvalidStreamName", "code": 400, "error_code": 10141, "description": "sourced stream name is invalid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorInvalidStreamName", "code": 400, "error_code": 10142, "description": "mirrored stream name is invalid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceMultipleFiltersNotAllowed", "code": 400, "error_code": 10144, "description": "source with multiple subject transforms cannot also have a single subject filter", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceInvalidSubjectFilter", "code": 400, "error_code": 10145, "description": "source subject filter is invalid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceInvalidTransformDestination", "code": 400, "error_code": 10146, "description": "source transform destination is invalid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSSourceOverlappingSubjectFilters", "code": 400, "error_code": 10147, "description": "source filters can not overlap", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerAlreadyExists", "code": 400, "error_code": 10148, "description": "consumer already exists", "comment": "action CREATE is used for a existing consumer with a different config", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerDoesNotExist", "code": 400, "error_code": 10149, "description": "consumer does not exist", "comment": "action UPDATE is used for a nonexisting consumer", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorMultipleFiltersNotAllowed", "code": 400, "error_code": 10150, "description": "mirror with multiple subject transforms cannot also have a single subject filter", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorInvalidSubjectFilter", "code": 400, "error_code": 10151, "description": "mirror subject filter is invalid", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSMirrorOverlappingSubjectFilters", "code": 400, "error_code": 10152, "description": "mirror subject filters can not overlap", "comment": "", "help": "", "url": "", "deprecates": "" }, { "constant": "JSConsumerInactiveThresholdExcess", "code": 400, "error_code": 10153, "description": "consumer inactive threshold exceeds system limit of {limit}", "comment": "", "help": "", "url": "", "deprecates": "" } ] nats-server-2.10.27/server/errors_gen.go000066400000000000000000000123721477524627100201620ustar00rootroot00000000000000//go:build ignore // +build ignore package main import ( "encoding/json" "fmt" "log" "os" "os/exec" "regexp" "sort" "strings" "text/template" "github.com/nats-io/nats-server/v2/server" ) var tagRe = regexp.MustCompile("\\{(.+?)}") var templ = ` // Generated code, do not edit. See errors.json and run go generate to update package server import "strings" const ( {{- range $i, $error := . }} {{- if .Comment }} // {{ .Constant }} {{ .Comment }} ({{ .Description | print }}) {{- else }} // {{ .Constant }} {{ .Description | print }} {{- end }} {{ .Constant }} ErrorIdentifier = {{ .ErrCode }} {{ end }} ) var ( ApiErrors = map[ErrorIdentifier]*ApiError{ {{- range $i, $error := . }} {{ .Constant }}: {Code: {{ .Code }},ErrCode: {{ .ErrCode }},Description: {{ .Description | printf "%q" }}},{{- end }} } {{- range $i, $error := . }} {{- if .Deprecates }} // {{ .Deprecates }} Deprecated by {{ .Constant }} ApiError, use IsNatsError() for comparisons {{ .Deprecates }} = ApiErrors[{{ .Constant }}] {{- end }} {{- end }} ) {{- range $i, $error := . }} // {{ .Constant | funcNameForConstant }} creates a new {{ .Constant }} error: {{ .Description | printf "%q" }} func {{ .Constant | funcNameForConstant }}({{ .Description | funcArgsForTags }}) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } {{ if .Description | hasTags }} e:=ApiErrors[{{.Constant}}] args:=e.toReplacerArgs([]interface{}{ {{.Description | replacerArgsForTags }} }) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } {{- else }} return ApiErrors[{{.Constant}}] {{- end }} } {{- end }} ` func panicIfErr(err error) { if err == nil { return } panic(err) } func goFmt(file string) error { c := exec.Command("go", "fmt", file) out, err := c.CombinedOutput() if err != nil { log.Printf("go fmt failed: %s", string(out)) } return err } func checkIncrements(errs []server.ErrorsData) error { sort.Slice(errs, func(i, j int) bool { return errs[i].ErrCode < errs[j].ErrCode }) last := errs[0].ErrCode gaps := []uint16{} for i := 1; i < len(errs); i++ { if errs[i].ErrCode != last+1 { gaps = append(gaps, last) } last = errs[i].ErrCode } if len(gaps) > 0 { return fmt.Errorf("gaps found in sequences: %v", gaps) } return nil } func checkDupes(errs []server.ErrorsData) error { codes := []uint16{} highest := uint16(0) for _, err := range errs { codes = append(codes, err.ErrCode) if highest < err.ErrCode { highest = err.ErrCode } } codeKeys := make(map[uint16]bool) constKeys := make(map[string]bool) for _, entry := range errs { if _, found := codeKeys[entry.ErrCode]; found { return fmt.Errorf("duplicate error code %+v, highest code is %d", entry, highest) } if _, found := constKeys[entry.Constant]; found { return fmt.Errorf("duplicate error constant %+v", entry) } codeKeys[entry.ErrCode] = true constKeys[entry.Constant] = true } return nil } func findTags(d string) []string { tags := []string{} for _, tag := range tagRe.FindAllStringSubmatch(d, -1) { if len(tag) != 2 { continue } tags = append(tags, tag[1]) } sort.Strings(tags) return tags } func main() { ej, err := os.ReadFile("server/errors.json") panicIfErr(err) errs := []server.ErrorsData{} panicIfErr(json.Unmarshal(ej, &errs)) panicIfErr(checkDupes(errs)) panicIfErr(checkIncrements(errs)) sort.Slice(errs, func(i, j int) bool { return errs[i].Constant < errs[j].Constant }) t := template.New("errors").Funcs( template.FuncMap{ "inc": func(i int) int { return i + 1 }, "hasTags": func(d string) bool { return strings.Contains(d, "{") && strings.Contains(d, "}") }, "replacerArgsForTags": func(d string) string { res := []string{} for _, tag := range findTags(d) { res = append(res, `"{`+tag+`}"`) res = append(res, tag) } return strings.Join(res, ", ") }, "funcArgsForTags": func(d string) string { res := []string{} for _, tag := range findTags(d) { if tag == "err" { res = append(res, "err error") } else if tag == "seq" { res = append(res, "seq uint64") } else { res = append(res, fmt.Sprintf("%s interface{}", tag)) } } res = append(res, "opts ...ErrorOption") return strings.Join(res, ", ") }, "funcNameForConstant": func(c string) string { res := "" switch { case strings.HasSuffix(c, "ErrF"): res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "ErrF")) case strings.HasSuffix(c, "Err"): res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "Err")) case strings.HasSuffix(c, "ErrorF"): res = fmt.Sprintf("New%s", strings.TrimSuffix(c, "F")) case strings.HasSuffix(c, "F"): res = fmt.Sprintf("New%sError", strings.TrimSuffix(c, "F")) default: res = fmt.Sprintf("New%s", c) } if !strings.HasSuffix(res, "Error") { res = fmt.Sprintf("%sError", res) } return res }, }) p, err := t.Parse(templ) panicIfErr(err) tf, err := os.CreateTemp("", "") panicIfErr(err) defer tf.Close() panicIfErr(p.Execute(tf, errs)) panicIfErr(os.Rename(tf.Name(), "server/jetstream_errors_generated.go")) panicIfErr(goFmt("server/jetstream_errors_generated.go")) } nats-server-2.10.27/server/errors_test.go000066400000000000000000000044341477524627100203700ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "strings" "testing" ) func TestErrCtx(t *testing.T) { ctx := "Extra context information" e := NewErrorCtx(ErrWrongGateway, "%s", ctx) if e.Error() != ErrWrongGateway.Error() { t.Fatalf("%v and %v are supposed to be identical", e, ErrWrongGateway) } if e == ErrWrongGateway { t.Fatalf("%v and %v can't be compared this way", e, ErrWrongGateway) } if !ErrorIs(e, ErrWrongGateway) { t.Fatalf("%s and %s ", e, ErrWrongGateway) } if UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() { t.Fatalf("Error of different type should be processed unchanged") } trace := UnpackIfErrorCtx(e) if !strings.HasPrefix(trace, ErrWrongGateway.Error()) { t.Fatalf("original error needs to remain") } if !strings.HasSuffix(trace, ctx) { t.Fatalf("ctx needs to be added") } } func TestErrCtxWrapped(t *testing.T) { ctxO := "Original Ctx" eO := NewErrorCtx(ErrWrongGateway, "%s", ctxO) ctx := "Extra context information" e := NewErrorCtx(eO, "%s", ctx) if e.Error() != ErrWrongGateway.Error() { t.Fatalf("%v and %v are supposed to be identical", e, ErrWrongGateway) } if e == ErrWrongGateway { t.Fatalf("%v and %v can't be compared this way", e, ErrWrongGateway) } if !ErrorIs(e, ErrWrongGateway) { t.Fatalf("%s and %s ", e, ErrWrongGateway) } if UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() { t.Fatalf("Error of different type should be processed unchanged") } trace := UnpackIfErrorCtx(e) if !strings.HasPrefix(trace, ErrWrongGateway.Error()) { t.Fatalf("original error needs to remain") } if !strings.HasSuffix(trace, ctx) { t.Fatalf("ctx needs to be added") } if !strings.Contains(trace, ctxO) { t.Fatalf("Needs to contain every context") } } nats-server-2.10.27/server/events.go000066400000000000000000002727341477524627100173330ustar00rootroot00000000000000// Copyright 2018-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "compress/gzip" "crypto/sha256" "crypto/x509" "encoding/json" "errors" "fmt" "math/rand" "net/http" "runtime" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/certidp" "github.com/nats-io/nats-server/v2/server/pse" ) const ( accLookupReqTokens = 6 accLookupReqSubj = "$SYS.REQ.ACCOUNT.%s.CLAIMS.LOOKUP" accPackReqSubj = "$SYS.REQ.CLAIMS.PACK" accListReqSubj = "$SYS.REQ.CLAIMS.LIST" accClaimsReqSubj = "$SYS.REQ.CLAIMS.UPDATE" accDeleteReqSubj = "$SYS.REQ.CLAIMS.DELETE" connectEventSubj = "$SYS.ACCOUNT.%s.CONNECT" disconnectEventSubj = "$SYS.ACCOUNT.%s.DISCONNECT" accDirectReqSubj = "$SYS.REQ.ACCOUNT.%s.%s" accPingReqSubj = "$SYS.REQ.ACCOUNT.PING.%s" // atm. only used for STATZ and CONNZ import from system account // kept for backward compatibility when using http resolver // this overlaps with the names for events but you'd have to have the operator private key in order to succeed. accUpdateEventSubjOld = "$SYS.ACCOUNT.%s.CLAIMS.UPDATE" accUpdateEventSubjNew = "$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE" connsRespSubj = "$SYS._INBOX_.%s" accConnsEventSubjNew = "$SYS.ACCOUNT.%s.SERVER.CONNS" accConnsEventSubjOld = "$SYS.SERVER.ACCOUNT.%s.CONNS" // kept for backward compatibility lameDuckEventSubj = "$SYS.SERVER.%s.LAMEDUCK" shutdownEventSubj = "$SYS.SERVER.%s.SHUTDOWN" clientKickReqSubj = "$SYS.REQ.SERVER.%s.KICK" clientLDMReqSubj = "$SYS.REQ.SERVER.%s.LDM" authErrorEventSubj = "$SYS.SERVER.%s.CLIENT.AUTH.ERR" authErrorAccountEventSubj = "$SYS.ACCOUNT.CLIENT.AUTH.ERR" serverStatsSubj = "$SYS.SERVER.%s.STATSZ" serverDirectReqSubj = "$SYS.REQ.SERVER.%s.%s" serverPingReqSubj = "$SYS.REQ.SERVER.PING.%s" serverStatsPingReqSubj = "$SYS.REQ.SERVER.PING" // use $SYS.REQ.SERVER.PING.STATSZ instead serverReloadReqSubj = "$SYS.REQ.SERVER.%s.RELOAD" // with server ID leafNodeConnectEventSubj = "$SYS.ACCOUNT.%s.LEAFNODE.CONNECT" // for internal use only remoteLatencyEventSubj = "$SYS.LATENCY.M2.%s" inboxRespSubj = "$SYS._INBOX.%s.%s" // Used to return information to a user on bound account and user permissions. userDirectInfoSubj = "$SYS.REQ.USER.INFO" userDirectReqSubj = "$SYS.REQ.USER.%s.INFO" // FIXME(dlc) - Should account scope, even with wc for now, but later on // we can then shard as needed. accNumSubsReqSubj = "$SYS.REQ.ACCOUNT.NSUBS" // These are for exported debug services. These are local to this server only. accSubsSubj = "$SYS.DEBUG.SUBSCRIBERS" shutdownEventTokens = 4 serverSubjectIndex = 2 accUpdateTokensNew = 6 accUpdateTokensOld = 5 accUpdateAccIdxOld = 2 accReqTokens = 5 accReqAccIndex = 3 ocspPeerRejectEventSubj = "$SYS.SERVER.%s.OCSP.PEER.CONN.REJECT" ocspPeerChainlinkInvalidEventSubj = "$SYS.SERVER.%s.OCSP.PEER.LINK.INVALID" ) // FIXME(dlc) - make configurable. var eventsHBInterval = 30 * time.Second var statsHBInterval = 10 * time.Second // Default minimum wait time for sending statsz const defaultStatszRateLimit = 1 * time.Second // Variable version so we can set in tests. var statszRateLimit = defaultStatszRateLimit type sysMsgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, hdr, msg []byte) // Used if we have to queue things internally to avoid the route/gw path. type inSysMsg struct { sub *subscription c *client acc *Account subj string rply string hdr []byte msg []byte cb sysMsgHandler } // Used to send and receive messages from inside the server. type internal struct { account *Account client *client seq uint64 sid int servers map[string]*serverUpdate sweeper *time.Timer stmr *time.Timer replies map[string]msgHandler sendq *ipQueue[*pubMsg] recvq *ipQueue[*inSysMsg] recvqp *ipQueue[*inSysMsg] // For STATSZ/Pings resetCh chan struct{} wg sync.WaitGroup sq *sendq orphMax time.Duration chkOrph time.Duration statsz time.Duration cstatsz time.Duration shash string inboxPre string remoteStatsSub *subscription lastStatsz time.Time } // ServerStatsMsg is sent periodically with stats updates. type ServerStatsMsg struct { Server ServerInfo `json:"server"` Stats ServerStats `json:"statsz"` } // ConnectEventMsg is sent when a new connection is made that is part of an account. type ConnectEventMsg struct { TypedEvent Server ServerInfo `json:"server"` Client ClientInfo `json:"client"` } // ConnectEventMsgType is the schema type for ConnectEventMsg const ConnectEventMsgType = "io.nats.server.advisory.v1.client_connect" // DisconnectEventMsg is sent when a new connection previously defined from a // ConnectEventMsg is closed. type DisconnectEventMsg struct { TypedEvent Server ServerInfo `json:"server"` Client ClientInfo `json:"client"` Sent DataStats `json:"sent"` Received DataStats `json:"received"` Reason string `json:"reason"` } // DisconnectEventMsgType is the schema type for DisconnectEventMsg const DisconnectEventMsgType = "io.nats.server.advisory.v1.client_disconnect" // OCSPPeerRejectEventMsg is sent when a peer TLS handshake is ultimately rejected due to OCSP invalidation. // A "peer" can be an inbound client connection or a leaf connection to a remote server. Peer in event payload // is always the peer's (TLS) leaf cert, which may or may be the invalid cert (See also OCSPPeerChainlinkInvalidEventMsg) type OCSPPeerRejectEventMsg struct { TypedEvent Kind string `json:"kind"` Peer certidp.CertInfo `json:"peer"` Server ServerInfo `json:"server"` Reason string `json:"reason"` } // OCSPPeerRejectEventMsgType is the schema type for OCSPPeerRejectEventMsg const OCSPPeerRejectEventMsgType = "io.nats.server.advisory.v1.ocsp_peer_reject" // OCSPPeerChainlinkInvalidEventMsg is sent when a certificate (link) in a valid TLS chain is found to be OCSP invalid // during a peer TLS handshake. A "peer" can be an inbound client connection or a leaf connection to a remote server. // Peer and Link may be the same if the invalid cert was the peer's leaf cert type OCSPPeerChainlinkInvalidEventMsg struct { TypedEvent Link certidp.CertInfo `json:"link"` Peer certidp.CertInfo `json:"peer"` Server ServerInfo `json:"server"` Reason string `json:"reason"` } // OCSPPeerChainlinkInvalidEventMsgType is the schema type for OCSPPeerChainlinkInvalidEventMsg const OCSPPeerChainlinkInvalidEventMsgType = "io.nats.server.advisory.v1.ocsp_peer_link_invalid" // AccountNumConns is an event that will be sent from a server that is tracking // a given account when the number of connections changes. It will also HB // updates in the absence of any changes. type AccountNumConns struct { TypedEvent Server ServerInfo `json:"server"` AccountStat } // AccountStat contains the data common between AccountNumConns and AccountStatz type AccountStat struct { Account string `json:"acc"` Conns int `json:"conns"` LeafNodes int `json:"leafnodes"` TotalConns int `json:"total_conns"` NumSubs uint32 `json:"num_subscriptions"` Sent DataStats `json:"sent"` Received DataStats `json:"received"` SlowConsumers int64 `json:"slow_consumers"` } const AccountNumConnsMsgType = "io.nats.server.advisory.v1.account_connections" // accNumConnsReq is sent when we are starting to track an account for the first // time. We will request others send info to us about their local state. type accNumConnsReq struct { Server ServerInfo `json:"server"` Account string `json:"acc"` } // ServerID is basic static info for a server. type ServerID struct { Name string `json:"name"` Host string `json:"host"` ID string `json:"id"` } // Type for our server capabilities. type ServerCapability uint64 // ServerInfo identifies remote servers. type ServerInfo struct { Name string `json:"name"` Host string `json:"host"` ID string `json:"id"` Cluster string `json:"cluster,omitempty"` Domain string `json:"domain,omitempty"` Version string `json:"ver"` Tags []string `json:"tags,omitempty"` // Whether JetStream is enabled (deprecated in favor of the `ServerCapability`). JetStream bool `json:"jetstream"` // Generic capability flags Flags ServerCapability `json:"flags"` // Sequence and Time from the remote server for this message. Seq uint64 `json:"seq"` Time time.Time `json:"time"` } const ( JetStreamEnabled ServerCapability = 1 << iota // Server had JetStream enabled. BinaryStreamSnapshot // New stream snapshot capability. ) // Set JetStream capability. func (si *ServerInfo) SetJetStreamEnabled() { si.Flags |= JetStreamEnabled // Still set old version. si.JetStream = true } // JetStreamEnabled indicates whether or not we have JetStream enabled. func (si *ServerInfo) JetStreamEnabled() bool { // Take into account old version. return si.Flags&JetStreamEnabled != 0 || si.JetStream } // Set binary stream snapshot capability. func (si *ServerInfo) SetBinaryStreamSnapshot() { si.Flags |= BinaryStreamSnapshot } // JetStreamEnabled indicates whether or not we have binary stream snapshot capbilities. func (si *ServerInfo) BinaryStreamSnapshot() bool { return si.Flags&BinaryStreamSnapshot != 0 } // ClientInfo is detailed information about the client forming a connection. type ClientInfo struct { Start *time.Time `json:"start,omitempty"` Host string `json:"host,omitempty"` ID uint64 `json:"id,omitempty"` Account string `json:"acc,omitempty"` Service string `json:"svc,omitempty"` User string `json:"user,omitempty"` Name string `json:"name,omitempty"` Lang string `json:"lang,omitempty"` Version string `json:"ver,omitempty"` RTT time.Duration `json:"rtt,omitempty"` Server string `json:"server,omitempty"` Cluster string `json:"cluster,omitempty"` Alternates []string `json:"alts,omitempty"` Stop *time.Time `json:"stop,omitempty"` Jwt string `json:"jwt,omitempty"` IssuerKey string `json:"issuer_key,omitempty"` NameTag string `json:"name_tag,omitempty"` Tags jwt.TagList `json:"tags,omitempty"` Kind string `json:"kind,omitempty"` ClientType string `json:"client_type,omitempty"` MQTTClient string `json:"client_id,omitempty"` // This is the MQTT client ID Nonce string `json:"nonce,omitempty"` } // forAssignmentSnap returns the minimum amount of ClientInfo we need for assignment snapshots. func (ci *ClientInfo) forAssignmentSnap() *ClientInfo { return &ClientInfo{ Account: ci.Account, Service: ci.Service, Cluster: ci.Cluster, } } // forProposal returns the minimum amount of ClientInfo we need for assignment proposals. func (ci *ClientInfo) forProposal() *ClientInfo { if ci == nil { return nil } cci := *ci cci.Jwt = _EMPTY_ cci.IssuerKey = _EMPTY_ return &cci } // forAdvisory returns the minimum amount of ClientInfo we need for JS advisory events. func (ci *ClientInfo) forAdvisory() *ClientInfo { if ci == nil { return nil } cci := *ci cci.Jwt = _EMPTY_ cci.Alternates = nil return &cci } // ServerStats hold various statistics that we will periodically send out. type ServerStats struct { Start time.Time `json:"start"` Mem int64 `json:"mem"` Cores int `json:"cores"` CPU float64 `json:"cpu"` Connections int `json:"connections"` TotalConnections uint64 `json:"total_connections"` ActiveAccounts int `json:"active_accounts"` NumSubs uint32 `json:"subscriptions"` Sent DataStats `json:"sent"` Received DataStats `json:"received"` SlowConsumers int64 `json:"slow_consumers"` Routes []*RouteStat `json:"routes,omitempty"` Gateways []*GatewayStat `json:"gateways,omitempty"` ActiveServers int `json:"active_servers,omitempty"` JetStream *JetStreamVarz `json:"jetstream,omitempty"` } // RouteStat holds route statistics. type RouteStat struct { ID uint64 `json:"rid"` Name string `json:"name,omitempty"` Sent DataStats `json:"sent"` Received DataStats `json:"received"` Pending int `json:"pending"` } // GatewayStat holds gateway statistics. type GatewayStat struct { ID uint64 `json:"gwid"` Name string `json:"name"` Sent DataStats `json:"sent"` Received DataStats `json:"received"` NumInbound int `json:"inbound_connections"` } // DataStats reports how may msg and bytes. Applicable for both sent and received. type DataStats struct { Msgs int64 `json:"msgs"` Bytes int64 `json:"bytes"` } // Used for internally queueing up messages that the server wants to send. type pubMsg struct { c *client sub string rply string si *ServerInfo hdr map[string]string msg any oct compressionType echo bool last bool } var pubMsgPool sync.Pool func newPubMsg(c *client, sub, rply string, si *ServerInfo, hdr map[string]string, msg any, oct compressionType, echo, last bool) *pubMsg { var m *pubMsg pm := pubMsgPool.Get() if pm != nil { m = pm.(*pubMsg) } else { m = &pubMsg{} } // When getting something from a pool it is critical that all fields are // initialized. Doing this way guarantees that if someone adds a field to // the structure, the compiler will fail the build if this line is not updated. (*m) = pubMsg{c, sub, rply, si, hdr, msg, oct, echo, last} return m } func (pm *pubMsg) returnToPool() { if pm == nil { return } pm.c, pm.sub, pm.rply, pm.si, pm.hdr, pm.msg = nil, _EMPTY_, _EMPTY_, nil, nil, nil pubMsgPool.Put(pm) } // Used to track server updates. type serverUpdate struct { seq uint64 ltime time.Time } // TypedEvent is a event or advisory sent by the server that has nats type hints // typically used for events that might be consumed by 3rd party event systems type TypedEvent struct { Type string `json:"type"` ID string `json:"id"` Time time.Time `json:"timestamp"` } // internalReceiveLoop will be responsible for dispatching all messages that // a server receives and needs to internally process, e.g. internal subs. func (s *Server) internalReceiveLoop(recvq *ipQueue[*inSysMsg]) { for s.eventsRunning() { select { case <-recvq.ch: msgs := recvq.pop() for _, m := range msgs { if m.cb != nil { m.cb(m.sub, m.c, m.acc, m.subj, m.rply, m.hdr, m.msg) } } recvq.recycle(&msgs) case <-s.quitCh: return } } } // internalSendLoop will be responsible for serializing all messages that // a server wants to send. func (s *Server) internalSendLoop(wg *sync.WaitGroup) { defer wg.Done() RESET: s.mu.RLock() if s.sys == nil || s.sys.sendq == nil { s.mu.RUnlock() return } sysc := s.sys.client resetCh := s.sys.resetCh sendq := s.sys.sendq id := s.info.ID host := s.info.Host servername := s.info.Name domain := s.info.Domain seqp := &s.sys.seq js := s.info.JetStream cluster := s.info.Cluster if s.gateway.enabled { cluster = s.getGatewayName() } s.mu.RUnlock() // Grab tags. tags := s.getOpts().Tags for s.eventsRunning() { select { case <-sendq.ch: msgs := sendq.pop() for _, pm := range msgs { if si := pm.si; si != nil { si.Name = servername si.Domain = domain si.Host = host si.Cluster = cluster si.ID = id si.Seq = atomic.AddUint64(seqp, 1) si.Version = VERSION si.Time = time.Now().UTC() si.Tags = tags if js { // New capability based flags. si.SetJetStreamEnabled() si.SetBinaryStreamSnapshot() } } var b []byte if pm.msg != nil { switch v := pm.msg.(type) { case string: b = []byte(v) case []byte: b = v default: b, _ = json.Marshal(pm.msg) } } // Setup our client. If the user wants to use a non-system account use our internal // account scoped here so that we are not changing out accounts for the system client. var c *client if pm.c != nil { c = pm.c } else { c = sysc } // Grab client lock. c.mu.Lock() // Prep internal structures needed to send message. c.pa.subject, c.pa.reply = []byte(pm.sub), []byte(pm.rply) c.pa.size, c.pa.szb = len(b), []byte(strconv.FormatInt(int64(len(b)), 10)) c.pa.hdr, c.pa.hdb = -1, nil trace := c.trace // Now check for optional compression. var contentHeader string var bb bytes.Buffer if len(b) > 0 { switch pm.oct { case gzipCompression: zw := gzip.NewWriter(&bb) zw.Write(b) zw.Close() b = bb.Bytes() contentHeader = "gzip" case snappyCompression: sw := s2.NewWriter(&bb, s2.WriterSnappyCompat()) sw.Write(b) sw.Close() b = bb.Bytes() contentHeader = "snappy" case unsupportedCompression: contentHeader = "identity" } } // Optional Echo replaceEcho := c.echo != pm.echo if replaceEcho { c.echo = !c.echo } c.mu.Unlock() // Add in NL b = append(b, _CRLF_...) // Check if we should set content-encoding if contentHeader != _EMPTY_ { b = c.setHeader(contentEncodingHeader, contentHeader, b) } // Optional header processing. if pm.hdr != nil { for k, v := range pm.hdr { b = c.setHeader(k, v, b) } } // Tracing if trace { c.traceInOp(fmt.Sprintf("PUB %s %s %d", c.pa.subject, c.pa.reply, c.pa.size), nil) c.traceMsg(b) } // Process like a normal inbound msg. c.processInboundClientMsg(b) // Put echo back if needed. if replaceEcho { c.mu.Lock() c.echo = !c.echo c.mu.Unlock() } // See if we are doing graceful shutdown. if !pm.last { c.flushClients(0) // Never spend time in place. } else { // For the Shutdown event, we need to send in place otherwise // there is a chance that the process will exit before the // writeLoop has a chance to send it. c.flushClients(time.Second) sendq.recycle(&msgs) return } pm.returnToPool() } sendq.recycle(&msgs) case <-resetCh: goto RESET case <-s.quitCh: return } } } // Will send a shutdown message for lame-duck. Unlike sendShutdownEvent, this will // not close off the send queue or reply handler, as we may still have a workload // that needs migrating off. // Lock should be held. func (s *Server) sendLDMShutdownEventLocked() { if s.sys == nil || s.sys.sendq == nil { return } subj := fmt.Sprintf(lameDuckEventSubj, s.info.ID) si := &ServerInfo{} s.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true)) } // Will send a shutdown message. func (s *Server) sendShutdownEvent() { s.mu.Lock() if s.sys == nil || s.sys.sendq == nil { s.mu.Unlock() return } subj := fmt.Sprintf(shutdownEventSubj, s.info.ID) sendq := s.sys.sendq // Stop any more messages from queueing up. s.sys.sendq = nil // Unhook all msgHandlers. Normal client cleanup will deal with subs, etc. s.sys.replies = nil // Send to the internal queue and mark as last. si := &ServerInfo{} sendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true)) s.mu.Unlock() } // Used to send an internal message to an arbitrary account. func (s *Server) sendInternalAccountMsg(a *Account, subject string, msg any) error { return s.sendInternalAccountMsgWithReply(a, subject, _EMPTY_, nil, msg, false) } // Used to send an internal message with an optional reply to an arbitrary account. func (s *Server) sendInternalAccountMsgWithReply(a *Account, subject, reply string, hdr map[string]string, msg any, echo bool) error { s.mu.RLock() if s.sys == nil || s.sys.sendq == nil { s.mu.RUnlock() if s.isShuttingDown() { // Skip in case this was called at the end phase during shut down // to avoid too many entries in the logs. return nil } return ErrNoSysAccount } c := s.sys.client // Replace our client with the account's internal client. if a != nil { a.mu.Lock() c = a.internalClient() a.mu.Unlock() } s.sys.sendq.push(newPubMsg(c, subject, reply, nil, hdr, msg, noCompression, echo, false)) s.mu.RUnlock() return nil } // Send system style message to an account scope. func (s *Server) sendInternalAccountSysMsg(a *Account, subj string, si *ServerInfo, msg interface{}) { s.mu.RLock() if s.sys == nil || s.sys.sendq == nil || a == nil { s.mu.RUnlock() return } sendq := s.sys.sendq s.mu.RUnlock() a.mu.Lock() c := a.internalClient() a.mu.Unlock() sendq.push(newPubMsg(c, subj, _EMPTY_, si, nil, msg, noCompression, false, false)) } // This will queue up a message to be sent. // Lock should not be held. func (s *Server) sendInternalMsgLocked(subj, rply string, si *ServerInfo, msg any) { s.mu.RLock() s.sendInternalMsg(subj, rply, si, msg) s.mu.RUnlock() } // This will queue up a message to be sent. // Assumes lock is held on entry. func (s *Server) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) { if s.sys == nil || s.sys.sendq == nil { return } s.sys.sendq.push(newPubMsg(nil, subj, rply, si, nil, msg, noCompression, false, false)) } // Will send an api response. func (s *Server) sendInternalResponse(subj string, response *ServerAPIResponse) { s.mu.RLock() if s.sys == nil || s.sys.sendq == nil { s.mu.RUnlock() return } s.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, response.Server, nil, response, response.compress, false, false)) s.mu.RUnlock() } // Used to send internal messages from other system clients to avoid no echo issues. func (c *client) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) { if c == nil { return } s := c.srv if s == nil { return } s.mu.RLock() if s.sys == nil || s.sys.sendq == nil { s.mu.RUnlock() return } s.sys.sendq.push(newPubMsg(c, subj, rply, si, nil, msg, noCompression, false, false)) s.mu.RUnlock() } // Locked version of checking if events system running. Also checks server. func (s *Server) eventsRunning() bool { if s == nil { return false } s.mu.RLock() er := s.isRunning() && s.eventsEnabled() s.mu.RUnlock() return er } // EventsEnabled will report if the server has internal events enabled via // a defined system account. func (s *Server) EventsEnabled() bool { s.mu.RLock() defer s.mu.RUnlock() return s.eventsEnabled() } // eventsEnabled will report if events are enabled. // Lock should be held. func (s *Server) eventsEnabled() bool { return s.sys != nil && s.sys.client != nil && s.sys.account != nil } // TrackedRemoteServers returns how many remote servers we are tracking // from a system events perspective. func (s *Server) TrackedRemoteServers() int { s.mu.RLock() defer s.mu.RUnlock() if !s.isRunning() || !s.eventsEnabled() { return -1 } return len(s.sys.servers) } // Check for orphan servers who may have gone away without notification. // This should be wrapChk() to setup common locking. func (s *Server) checkRemoteServers() { now := time.Now() for sid, su := range s.sys.servers { if now.Sub(su.ltime) > s.sys.orphMax { s.Debugf("Detected orphan remote server: %q", sid) // Simulate it going away. s.processRemoteServerShutdown(sid) } } if s.sys.sweeper != nil { s.sys.sweeper.Reset(s.sys.chkOrph) } } // Grab RSS and PCPU // Server lock will be held but released. func (s *Server) updateServerUsage(v *ServerStats) { var vss int64 pse.ProcUsage(&v.CPU, &v.Mem, &vss) v.Cores = runtime.NumCPU() } // Generate a route stat for our statz update. func routeStat(r *client) *RouteStat { if r == nil { return nil } r.mu.Lock() // Note: *client.out[Msgs|Bytes] are not set using atomics, // unlike in[Msgs|Bytes]. rs := &RouteStat{ ID: r.cid, Sent: DataStats{ Msgs: r.outMsgs, Bytes: r.outBytes, }, Received: DataStats{ Msgs: atomic.LoadInt64(&r.inMsgs), Bytes: atomic.LoadInt64(&r.inBytes), }, Pending: int(r.out.pb), } if r.route != nil { rs.Name = r.route.remoteName } r.mu.Unlock() return rs } // Actual send method for statz updates. // Lock should be held. func (s *Server) sendStatsz(subj string) { var m ServerStatsMsg s.updateServerUsage(&m.Stats) if s.limitStatsz(subj) { return } s.mu.RLock() defer s.mu.RUnlock() // Check that we have a system account, etc. if s.sys == nil || s.sys.account == nil { return } shouldCheckInterest := func() bool { opts := s.getOpts() if opts.Cluster.Port != 0 || opts.Gateway.Port != 0 || opts.LeafNode.Port != 0 { return false } // If we are here we have no clustering or gateways and are not a leafnode hub. // Check for leafnode remotes that connect the system account. if len(opts.LeafNode.Remotes) > 0 { sysAcc := s.sys.account.GetName() for _, r := range opts.LeafNode.Remotes { if r.LocalAccount == sysAcc { return false } } } return true } // if we are running standalone, check for interest. if shouldCheckInterest() { // Check if we even have interest in this subject. sacc := s.sys.account rr := sacc.sl.Match(subj) totalSubs := len(rr.psubs) + len(rr.qsubs) if totalSubs == 0 { return } else if totalSubs == 1 && len(rr.psubs) == 1 { // For the broadcast subject we listen to that ourselves with no echo for remote updates. // If we are the only ones listening do not send either. if rr.psubs[0] == s.sys.remoteStatsSub { return } } } m.Stats.Start = s.start m.Stats.Connections = len(s.clients) m.Stats.TotalConnections = s.totalClients m.Stats.ActiveAccounts = int(atomic.LoadInt32(&s.activeAccounts)) m.Stats.Received.Msgs = atomic.LoadInt64(&s.inMsgs) m.Stats.Received.Bytes = atomic.LoadInt64(&s.inBytes) m.Stats.Sent.Msgs = atomic.LoadInt64(&s.outMsgs) m.Stats.Sent.Bytes = atomic.LoadInt64(&s.outBytes) m.Stats.SlowConsumers = atomic.LoadInt64(&s.slowConsumers) m.Stats.NumSubs = s.numSubscriptions() // Routes s.forEachRoute(func(r *client) { m.Stats.Routes = append(m.Stats.Routes, routeStat(r)) }) // Gateways if s.gateway.enabled { gw := s.gateway gw.RLock() for name, c := range gw.out { gs := &GatewayStat{Name: name} c.mu.Lock() gs.ID = c.cid // Note that *client.out[Msgs|Bytes] are not set using atomic, // unlike the in[Msgs|bytes]. gs.Sent = DataStats{ Msgs: c.outMsgs, Bytes: c.outBytes, } c.mu.Unlock() // Gather matching inbound connections gs.Received = DataStats{} for _, c := range gw.in { c.mu.Lock() if c.gw.name == name { gs.Received.Msgs += atomic.LoadInt64(&c.inMsgs) gs.Received.Bytes += atomic.LoadInt64(&c.inBytes) gs.NumInbound++ } c.mu.Unlock() } m.Stats.Gateways = append(m.Stats.Gateways, gs) } gw.RUnlock() } // Active Servers m.Stats.ActiveServers = len(s.sys.servers) + 1 // JetStream if js := s.js.Load(); js != nil { jStat := &JetStreamVarz{} s.mu.RUnlock() js.mu.RLock() c := js.config c.StoreDir = _EMPTY_ jStat.Config = &c js.mu.RUnlock() jStat.Stats = js.usageStats() // Update our own usage since we do not echo so we will not hear ourselves. ourNode := getHash(s.serverName()) if v, ok := s.nodeToInfo.Load(ourNode); ok && v != nil { ni := v.(nodeInfo) ni.stats = jStat.Stats ni.cfg = jStat.Config s.optsMu.RLock() ni.tags = copyStrings(s.opts.Tags) s.optsMu.RUnlock() s.nodeToInfo.Store(ourNode, ni) } // Metagroup info. if mg := js.getMetaGroup(); mg != nil { if mg.Leader() { if ci := s.raftNodeToClusterInfo(mg); ci != nil { jStat.Meta = &MetaClusterInfo{ Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Replicas: ci.Replicas, Size: mg.ClusterSize(), } } } else { // non leader only include a shortened version without peers leader := s.serverNameForNode(mg.GroupLeader()) jStat.Meta = &MetaClusterInfo{ Name: mg.Group(), Leader: leader, Peer: getHash(leader), Size: mg.ClusterSize(), } } if ipq := s.jsAPIRoutedReqs; ipq != nil && jStat.Meta != nil { jStat.Meta.Pending = ipq.len() } } m.Stats.JetStream = jStat s.mu.RLock() } // Send message. s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) } // Limit updates to the heartbeat interval, max one second by default. func (s *Server) limitStatsz(subj string) bool { s.mu.Lock() defer s.mu.Unlock() if s.sys == nil { return true } // Only limit the normal broadcast subject. if subj != fmt.Sprintf(serverStatsSubj, s.ID()) { return false } interval := statszRateLimit if s.sys.cstatsz < interval { interval = s.sys.cstatsz } if time.Since(s.sys.lastStatsz) < interval { // Reschedule heartbeat for the next interval. if s.sys.stmr != nil { s.sys.stmr.Reset(time.Until(s.sys.lastStatsz.Add(interval))) } return true } s.sys.lastStatsz = time.Now() return false } // Send out our statz update. // This should be wrapChk() to setup common locking. func (s *Server) heartbeatStatsz() { if s.sys.stmr != nil { // Increase after startup to our max. if s.sys.cstatsz < s.sys.statsz { s.sys.cstatsz *= 2 if s.sys.cstatsz > s.sys.statsz { s.sys.cstatsz = s.sys.statsz } } s.sys.stmr.Reset(s.sys.cstatsz) } // Do in separate Go routine. go s.sendStatszUpdate() } // Reset statsz rate limit for the next broadcast. // This should be wrapChk() to setup common locking. func (s *Server) resetLastStatsz() { s.sys.lastStatsz = time.Time{} } func (s *Server) sendStatszUpdate() { s.sendStatsz(fmt.Sprintf(serverStatsSubj, s.ID())) } // This should be wrapChk() to setup common locking. func (s *Server) startStatszTimer() { // We will start by sending out more of these and trail off to the statsz being the max. s.sys.cstatsz = 250 * time.Millisecond // Send out the first one quickly, we will slowly back off. s.sys.stmr = time.AfterFunc(s.sys.cstatsz, s.wrapChk(s.heartbeatStatsz)) } // Start a ticker that will fire periodically and check for orphaned servers. // This should be wrapChk() to setup common locking. func (s *Server) startRemoteServerSweepTimer() { s.sys.sweeper = time.AfterFunc(s.sys.chkOrph, s.wrapChk(s.checkRemoteServers)) } // Length of our system hash used for server targeted messages. const sysHashLen = 8 // Computes a hash of 8 characters for the name. func getHash(name string) string { return getHashSize(name, sysHashLen) } // Computes a hash for the given `name`. The result will be `size` characters long. func getHashSize(name string, size int) string { sha := sha256.New() sha.Write([]byte(name)) b := sha.Sum(nil) for i := 0; i < size; i++ { b[i] = digits[int(b[i]%base)] } return string(b[:size]) } // Returns the node name for this server which is a hash of the server name. func (s *Server) Node() string { s.mu.RLock() defer s.mu.RUnlock() if s.sys != nil { return s.sys.shash } return _EMPTY_ } // This will setup our system wide tracking subs. // For now we will setup one wildcard subscription to // monitor all accounts for changes in number of connections. // We can make this on a per account tracking basis if needed. // Tradeoff is subscription and interest graph events vs connect and // disconnect events, etc. func (s *Server) initEventTracking() { // Capture sys in case we are shutdown while setting up. s.mu.RLock() sys := s.sys s.mu.RUnlock() if sys == nil || sys.client == nil || sys.account == nil { return } // Create a system hash which we use for other servers to target us specifically. sys.shash = getHash(s.info.Name) // This will be for all inbox responses. subject := fmt.Sprintf(inboxRespSubj, sys.shash, "*") if _, err := s.sysSubscribe(subject, s.inboxReply); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } sys.inboxPre = subject // This is for remote updates for connection accounting. subject = fmt.Sprintf(accConnsEventSubjOld, "*") if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil { s.Errorf("Error setting up internal tracking for %s: %v", subject, err) return } // This will be for responses for account info that we send out. subject = fmt.Sprintf(connsRespSubj, s.info.ID) if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // Listen for broad requests to respond with number of subscriptions for a given subject. if _, err := s.sysSubscribe(accNumSubsReqSubj, s.noInlineCallback(s.nsubsRequest)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // Listen for statsz from others. subject = fmt.Sprintf(serverStatsSubj, "*") if sub, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerUpdate)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } else { // Keep track of this one. sys.remoteStatsSub = sub } // Listen for all server shutdowns. subject = fmt.Sprintf(shutdownEventSubj, "*") if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // Listen for servers entering lame-duck mode. // NOTE: This currently is handled in the same way as a server shutdown, but has // a different subject in case we need to handle differently in future. subject = fmt.Sprintf(lameDuckEventSubj, "*") if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // Listen for account claims updates. subscribeToUpdate := true if s.accResolver != nil { subscribeToUpdate = !s.accResolver.IsTrackingUpdate() } if subscribeToUpdate { for _, sub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} { if _, err := s.sysSubscribe(fmt.Sprintf(sub, "*"), s.noInlineCallback(s.accountClaimUpdate)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } } } // Listen for ping messages that will be sent to all servers for statsz. // This subscription is kept for backwards compatibility. Got replaced by ...PING.STATZ from below if _, err := s.sysSubscribe(serverStatsPingReqSubj, s.noInlineCallbackStatsz(s.statszReq)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } monSrvc := map[string]sysMsgHandler{ "IDZ": s.idzReq, "STATSZ": s.statszReq, "VARZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &VarzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Varz(&optz.VarzOptions) }) }, "SUBSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &SubszEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Subsz(&optz.SubszOptions) }) }, "CONNZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &ConnzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Connz(&optz.ConnzOptions) }) }, "ROUTEZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &RoutezEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Routez(&optz.RoutezOptions) }) }, "GATEWAYZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &GatewayzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Gatewayz(&optz.GatewayzOptions) }) }, "LEAFZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &LeafzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Leafz(&optz.LeafzOptions) }) }, "ACCOUNTZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &AccountzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Accountz(&optz.AccountzOptions) }) }, "JSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &JszEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Jsz(&optz.JSzOptions) }) }, "HEALTHZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &HealthzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.healthz(&optz.HealthzOptions), nil }) }, "PROFILEZ": nil, // Special case, see below "EXPVARZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &ExpvarzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.expvarz(optz), nil }) }, "IPQUEUESZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &IpqueueszEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Ipqueuesz(&optz.IpqueueszOptions), nil }) }, "RAFTZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &RaftzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Raftz(&optz.RaftzOptions), nil }) }, } profilez := func(_ *subscription, c *client, _ *Account, _, rply string, rmsg []byte) { hdr, msg := c.msgParts(rmsg) // Need to copy since we are passing those to the go routine below. hdr, msg = copyBytes(hdr), copyBytes(msg) // Execute in its own go routine because CPU profiling, for instance, // could take several seconds to complete. go func() { optz := &ProfilezEventOptions{} s.zReq(c, rply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.profilez(&optz.ProfilezOptions), nil }) }() } for name, req := range monSrvc { var h msgHandler switch name { case "PROFILEZ": h = profilez case "STATSZ": h = s.noInlineCallbackStatsz(req) default: h = s.noInlineCallback(req) } subject = fmt.Sprintf(serverDirectReqSubj, s.info.ID, name) if _, err := s.sysSubscribe(subject, h); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } subject = fmt.Sprintf(serverPingReqSubj, name) if _, err := s.sysSubscribe(subject, h); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } } extractAccount := func(subject string) (string, error) { if tk := strings.Split(subject, tsep); len(tk) != accReqTokens { return _EMPTY_, fmt.Errorf("subject %q is malformed", subject) } else { return tk[accReqAccIndex], nil } } monAccSrvc := map[string]sysMsgHandler{ "SUBSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &SubszEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else { optz.SubszOptions.Subscriptions = true optz.SubszOptions.Account = acc return s.Subsz(&optz.SubszOptions) } }) }, "CONNZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &ConnzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else { optz.ConnzOptions.Account = acc return s.Connz(&optz.ConnzOptions) } }) }, "LEAFZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &LeafzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else { optz.LeafzOptions.Account = acc return s.Leafz(&optz.LeafzOptions) } }) }, "JSZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &JszEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else { optz.Account = acc return s.JszAccount(&optz.JSzOptions) } }) }, "INFO": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &AccInfoEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else { return s.accountInfo(acc) } }) }, // STATZ is essentially a duplicate of CONNS with an envelope identical to the others. // For historical reasons CONNS is the odd one out. // STATZ is also less heavy weight than INFO "STATZ": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &AccountStatzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if acc, err := extractAccount(subject); err != nil { return nil, err } else if acc == "PING" { // Filter PING subject. Happens for server as well. But wildcards are not used return nil, errSkipZreq } else { optz.Accounts = []string{acc} if stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil { return nil, err } else if len(stz.Accounts) == 0 && !optz.IncludeUnused { return nil, errSkipZreq } else { return stz, nil } } }) }, "CONNS": s.connsRequest, } for name, req := range monAccSrvc { if _, err := s.sysSubscribe(fmt.Sprintf(accDirectReqSubj, "*", name), s.noInlineCallback(req)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } } // User info. // TODO(dlc) - Can be internal and not forwarded since bound server for the client connection // is only one that will answer. This breaks tests since we still forward on remote server connect. if _, err := s.sysSubscribe(fmt.Sprintf(userDirectReqSubj, "*"), s.userInfoReq); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // For now only the STATZ subject has an account specific ping equivalent. if _, err := s.sysSubscribe(fmt.Sprintf(accPingReqSubj, "STATZ"), s.noInlineCallback(func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { optz := &AccountStatzEventOptions{} s.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { if stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil { return nil, err } else if len(stz.Accounts) == 0 && !optz.IncludeUnused { return nil, errSkipZreq } else { return stz, nil } }) })); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // Listen for updates when leaf nodes connect for a given account. This will // force any gateway connections to move to `modeInterestOnly` subject = fmt.Sprintf(leafNodeConnectEventSubj, "*") if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.leafNodeConnected)); err != nil { s.Errorf("Error setting up internal tracking: %v", err) return } // For tracking remote latency measurements. subject = fmt.Sprintf(remoteLatencyEventSubj, sys.shash) if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteLatencyUpdate)); err != nil { s.Errorf("Error setting up internal latency tracking: %v", err) return } // This is for simple debugging of number of subscribers that exist in the system. if _, err := s.sysSubscribeInternal(accSubsSubj, s.noInlineCallback(s.debugSubscribers)); err != nil { s.Errorf("Error setting up internal debug service for subscribers: %v", err) return } // Listen for requests to reload the server configuration. subject = fmt.Sprintf(serverReloadReqSubj, s.info.ID) if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.reloadConfig)); err != nil { s.Errorf("Error setting up server reload handler: %v", err) return } // Client connection kick subject = fmt.Sprintf(clientKickReqSubj, s.info.ID) if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.kickClient)); err != nil { s.Errorf("Error setting up client kick service: %v", err) return } // Client connection LDM subject = fmt.Sprintf(clientLDMReqSubj, s.info.ID) if _, err := s.sysSubscribe(subject, s.noInlineCallback(s.ldmClient)); err != nil { s.Errorf("Error setting up client LDM service: %v", err) return } } // UserInfo returns basic information to a user about bound account and user permissions. // For account information they will need to ping that separately, and this allows security // controls on each subsystem if desired, e.g. account info, jetstream account info, etc. type UserInfo struct { UserID string `json:"user"` Account string `json:"account"` Permissions *Permissions `json:"permissions,omitempty"` Expires time.Duration `json:"expires,omitempty"` } // Process a user info request. func (s *Server) userInfoReq(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { if !s.EventsEnabled() || reply == _EMPTY_ { return } response := &ServerAPIResponse{Server: &ServerInfo{}} ci, _, _, _, err := s.getRequestInfo(c, msg) if err != nil { response.Error = &ApiError{Code: http.StatusBadRequest} s.sendInternalResponse(reply, response) return } response.Data = &UserInfo{ UserID: ci.User, Account: ci.Account, Permissions: c.publicPermissions(), Expires: c.claimExpiration(), } s.sendInternalResponse(reply, response) } // register existing accounts with any system exports. func (s *Server) registerSystemImportsForExisting() { var accounts []*Account s.mu.RLock() if s.sys == nil { s.mu.RUnlock() return } sacc := s.sys.account s.accounts.Range(func(k, v any) bool { a := v.(*Account) if a != sacc { accounts = append(accounts, a) } return true }) s.mu.RUnlock() for _, a := range accounts { s.registerSystemImports(a) } } // add all exports a system account will need func (s *Server) addSystemAccountExports(sacc *Account) { if !s.EventsEnabled() { return } accConnzSubj := fmt.Sprintf(accDirectReqSubj, "*", "CONNZ") // prioritize not automatically added exports if !sacc.hasServiceExportMatching(accConnzSubj) { // pick export type that clamps importing account id into subject if err := sacc.addServiceExportWithResponseAndAccountPos(accConnzSubj, Streamed, nil, 4); err != nil { //if err := sacc.AddServiceExportWithResponse(accConnzSubj, Streamed, nil); err != nil { s.Errorf("Error adding system service export for %q: %v", accConnzSubj, err) } } // prioritize not automatically added exports accStatzSubj := fmt.Sprintf(accDirectReqSubj, "*", "STATZ") if !sacc.hasServiceExportMatching(accStatzSubj) { // pick export type that clamps importing account id into subject if err := sacc.addServiceExportWithResponseAndAccountPos(accStatzSubj, Streamed, nil, 4); err != nil { s.Errorf("Error adding system service export for %q: %v", accStatzSubj, err) } } // FIXME(dlc) - Old experiment, Remove? if !sacc.hasServiceExportMatching(accSubsSubj) { if err := sacc.AddServiceExport(accSubsSubj, nil); err != nil { s.Errorf("Error adding system service export for %q: %v", accSubsSubj, err) } } // User info export. userInfoSubj := fmt.Sprintf(userDirectReqSubj, "*") if !sacc.hasServiceExportMatching(userInfoSubj) { if err := sacc.AddServiceExport(userInfoSubj, nil); err != nil { s.Errorf("Error adding system service export for %q: %v", userInfoSubj, err) } mappedSubj := fmt.Sprintf(userDirectReqSubj, sacc.GetName()) if err := sacc.AddServiceImport(sacc, userDirectInfoSubj, mappedSubj); err != nil { s.Errorf("Error setting up system service import %s: %v", mappedSubj, err) } // Make sure to share details. sacc.setServiceImportSharing(sacc, mappedSubj, false, true) } // Register any accounts that existed prior. s.registerSystemImportsForExisting() // in case of a mixed mode setup, enable js exports anyway if s.JetStreamEnabled() || !s.standAloneMode() { s.checkJetStreamExports() } } // accountClaimUpdate will receive claim updates for accounts. func (s *Server) accountClaimUpdate(sub *subscription, c *client, _ *Account, subject, resp string, hdr, msg []byte) { if !s.EventsEnabled() { return } var pubKey string toks := strings.Split(subject, tsep) if len(toks) == accUpdateTokensNew { pubKey = toks[accReqAccIndex] } else if len(toks) == accUpdateTokensOld { pubKey = toks[accUpdateAccIdxOld] } else { s.Debugf("Received account claims update on bad subject %q", subject) return } if len(msg) == 0 { err := errors.New("request body is empty") respondToUpdate(s, resp, pubKey, "jwt update error", err) } else if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil { respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else if claim.Subject != pubKey { err := errors.New("subject does not match jwt content") respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else if v, ok := s.accounts.Load(pubKey); !ok { respondToUpdate(s, resp, pubKey, "jwt update skipped", nil) } else if err := s.updateAccountWithClaimJWT(v.(*Account), string(msg)); err != nil { respondToUpdate(s, resp, pubKey, "jwt update resulted in error", err) } else { respondToUpdate(s, resp, pubKey, "jwt updated", nil) } } // processRemoteServerShutdown will update any affected accounts. // Will update the remote count for clients. // Lock assume held. func (s *Server) processRemoteServerShutdown(sid string) { s.accounts.Range(func(k, v any) bool { v.(*Account).removeRemoteServer(sid) return true }) // Update any state in nodeInfo. s.nodeToInfo.Range(func(k, v any) bool { ni := v.(nodeInfo) if ni.id == sid { ni.offline = true s.nodeToInfo.Store(k, ni) return false } return true }) delete(s.sys.servers, sid) } func (s *Server) sameDomain(domain string) bool { return domain == _EMPTY_ || s.info.Domain == _EMPTY_ || domain == s.info.Domain } // remoteServerShutdown is called when we get an event from another server shutting down. func (s *Server) remoteServerShutdown(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { s.mu.Lock() defer s.mu.Unlock() if !s.eventsEnabled() { return } toks := strings.Split(subject, tsep) if len(toks) < shutdownEventTokens { s.Debugf("Received remote server shutdown on bad subject %q", subject) return } if len(msg) == 0 { s.Errorf("Remote server sent invalid (empty) shutdown message to %q", subject) return } // We have an optional serverInfo here, remove from nodeToX lookups. var si ServerInfo if err := json.Unmarshal(msg, &si); err != nil { s.Debugf("Received bad server info for remote server shutdown") return } // JetStream node updates if applicable. node := getHash(si.Name) if v, ok := s.nodeToInfo.Load(node); ok && v != nil { ni := v.(nodeInfo) ni.offline = true s.nodeToInfo.Store(node, ni) } sid := toks[serverSubjectIndex] if su := s.sys.servers[sid]; su != nil { s.processRemoteServerShutdown(sid) } } // remoteServerUpdate listens for statsz updates from other servers. func (s *Server) remoteServerUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { var ssm ServerStatsMsg if len(msg) == 0 { s.Debugf("Received empty server info for remote server update") return } else if err := json.Unmarshal(msg, &ssm); err != nil { s.Debugf("Received bad server info for remote server update") return } si := ssm.Server // Should do normal updates before bailing if wrong domain. s.mu.Lock() if s.isRunning() && s.eventsEnabled() && ssm.Server.ID != s.info.ID { s.updateRemoteServer(&si) } s.mu.Unlock() // JetStream node updates. if !s.sameDomain(si.Domain) { return } var cfg *JetStreamConfig var stats *JetStreamStats if ssm.Stats.JetStream != nil { cfg = ssm.Stats.JetStream.Config stats = ssm.Stats.JetStream.Stats } node := getHash(si.Name) s.nodeToInfo.Store(node, nodeInfo{ si.Name, si.Version, si.Cluster, si.Domain, si.ID, si.Tags, cfg, stats, false, si.JetStreamEnabled(), si.BinaryStreamSnapshot(), }) } // updateRemoteServer is called when we have an update from a remote server. // This allows us to track remote servers, respond to shutdown messages properly, // make sure that messages are ordered, and allow us to prune dead servers. // Lock should be held upon entry. func (s *Server) updateRemoteServer(si *ServerInfo) { su := s.sys.servers[si.ID] if su == nil { s.sys.servers[si.ID] = &serverUpdate{si.Seq, time.Now()} s.processNewServer(si) } else { // Should always be going up. if si.Seq <= su.seq { s.Errorf("Received out of order remote server update from: %q", si.ID) return } su.seq = si.Seq su.ltime = time.Now() } } // processNewServer will hold any logic we want to use when we discover a new server. // Lock should be held upon entry. func (s *Server) processNewServer(si *ServerInfo) { // Right now we only check if we have leafnode servers and if so send another // connect update to make sure they switch this account to interest only mode. s.ensureGWsInterestOnlyForLeafNodes() // Add to our nodeToName if s.sameDomain(si.Domain) { node := getHash(si.Name) // Only update if non-existent if _, ok := s.nodeToInfo.Load(node); !ok { s.nodeToInfo.Store(node, nodeInfo{ si.Name, si.Version, si.Cluster, si.Domain, si.ID, si.Tags, nil, nil, false, si.JetStreamEnabled(), si.BinaryStreamSnapshot(), }) } } // Announce ourselves.. // Do this in a separate Go routine. go s.sendStatszUpdate() } // If GW is enabled on this server and there are any leaf node connections, // this function will send a LeafNode connect system event to the super cluster // to ensure that the GWs are in interest-only mode for this account. // Lock should be held upon entry. // TODO(dlc) - this will cause this account to be loaded on all servers. Need a better // way with GW2. func (s *Server) ensureGWsInterestOnlyForLeafNodes() { if !s.gateway.enabled || len(s.leafs) == 0 { return } sent := make(map[*Account]bool, len(s.leafs)) for _, c := range s.leafs { if !sent[c.acc] { s.sendLeafNodeConnectMsg(c.acc.Name) sent[c.acc] = true } } } // shutdownEventing will clean up all eventing state. func (s *Server) shutdownEventing() { if !s.eventsRunning() { return } s.mu.Lock() clearTimer(&s.sys.sweeper) clearTimer(&s.sys.stmr) rc := s.sys.resetCh s.sys.resetCh = nil wg := &s.sys.wg s.mu.Unlock() // We will queue up a shutdown event and wait for the // internal send loop to exit. s.sendShutdownEvent() wg.Wait() s.mu.Lock() defer s.mu.Unlock() // Whip through all accounts. s.accounts.Range(func(k, v any) bool { v.(*Account).clearEventing() return true }) // Turn everything off here. s.sys = nil // Make sure this is done after s.sys = nil, so that we don't // get sends to closed channels on badly-timed config reloads. close(rc) } // Request for our local connection count. func (s *Server) connsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } tk := strings.Split(subject, tsep) if len(tk) != accReqTokens { s.sys.client.Errorf("Bad subject account connections request message") return } a := tk[accReqAccIndex] m := accNumConnsReq{Account: a} if len(msg) > 0 { if err := json.Unmarshal(msg, &m); err != nil { s.sys.client.Errorf("Error unmarshalling account connections request message: %v", err) return } } if m.Account != a { s.sys.client.Errorf("Error unmarshalled account does not match subject") return } // Here we really only want to lookup the account if its local. We do not want to fetch this // account if we have no interest in it. var acc *Account if v, ok := s.accounts.Load(m.Account); ok { acc = v.(*Account) } if acc == nil { return } // We know this is a local connection. if nlc := acc.NumLocalConnections(); nlc > 0 { s.mu.Lock() s.sendAccConnsUpdate(acc, reply) s.mu.Unlock() } } // leafNodeConnected is an event we will receive when a leaf node for a given account connects. func (s *Server) leafNodeConnected(sub *subscription, _ *client, _ *Account, subject, reply string, hdr, msg []byte) { m := accNumConnsReq{} if err := json.Unmarshal(msg, &m); err != nil { s.sys.client.Errorf("Error unmarshalling account connections request message: %v", err) return } s.mu.RLock() na := m.Account == _EMPTY_ || !s.eventsEnabled() || !s.gateway.enabled s.mu.RUnlock() if na { return } if acc, _ := s.lookupAccount(m.Account); acc != nil { s.switchAccountToInterestMode(acc.Name) } } // Common filter options for system requests STATSZ VARZ SUBSZ CONNZ ROUTEZ GATEWAYZ LEAFZ type EventFilterOptions struct { Name string `json:"server_name,omitempty"` // filter by server name Cluster string `json:"cluster,omitempty"` // filter by cluster name Host string `json:"host,omitempty"` // filter by host name Tags []string `json:"tags,omitempty"` // filter by tags (must match all tags) Domain string `json:"domain,omitempty"` // filter by JS domain } // StatszEventOptions are options passed to Statsz type StatszEventOptions struct { // No actual options yet EventFilterOptions } // Options for account Info type AccInfoEventOptions struct { // No actual options yet EventFilterOptions } // In the context of system events, ConnzEventOptions are options passed to Connz type ConnzEventOptions struct { ConnzOptions EventFilterOptions } // In the context of system events, RoutezEventOptions are options passed to Routez type RoutezEventOptions struct { RoutezOptions EventFilterOptions } // In the context of system events, SubzEventOptions are options passed to Subz type SubszEventOptions struct { SubszOptions EventFilterOptions } // In the context of system events, VarzEventOptions are options passed to Varz type VarzEventOptions struct { VarzOptions EventFilterOptions } // In the context of system events, GatewayzEventOptions are options passed to Gatewayz type GatewayzEventOptions struct { GatewayzOptions EventFilterOptions } // In the context of system events, LeafzEventOptions are options passed to Leafz type LeafzEventOptions struct { LeafzOptions EventFilterOptions } // In the context of system events, AccountzEventOptions are options passed to Accountz type AccountzEventOptions struct { AccountzOptions EventFilterOptions } // In the context of system events, AccountzEventOptions are options passed to Accountz type AccountStatzEventOptions struct { AccountStatzOptions EventFilterOptions } // In the context of system events, JszEventOptions are options passed to Jsz type JszEventOptions struct { JSzOptions EventFilterOptions } // In the context of system events, HealthzEventOptions are options passed to Healthz type HealthzEventOptions struct { HealthzOptions EventFilterOptions } // In the context of system events, ProfilezEventOptions are options passed to Profilez type ProfilezEventOptions struct { ProfilezOptions EventFilterOptions } // In the context of system events, ExpvarzEventOptions are options passed to Expvarz type ExpvarzEventOptions struct { EventFilterOptions } // In the context of system events, IpqueueszEventOptions are options passed to Ipqueuesz type IpqueueszEventOptions struct { EventFilterOptions IpqueueszOptions } // In the context of system events, RaftzEventOptions are options passed to Raftz type RaftzEventOptions struct { EventFilterOptions RaftzOptions } // returns true if the request does NOT apply to this server and can be ignored. // DO NOT hold the server lock when func (s *Server) filterRequest(fOpts *EventFilterOptions) bool { if fOpts.Name != _EMPTY_ && !strings.Contains(s.info.Name, fOpts.Name) { return true } if fOpts.Host != _EMPTY_ && !strings.Contains(s.info.Host, fOpts.Host) { return true } if fOpts.Cluster != _EMPTY_ { if !strings.Contains(s.ClusterName(), fOpts.Cluster) { return true } } if len(fOpts.Tags) > 0 { opts := s.getOpts() for _, t := range fOpts.Tags { if !opts.Tags.Contains(t) { return true } } } if fOpts.Domain != _EMPTY_ && s.getOpts().JetStreamDomain != fOpts.Domain { return true } return false } // Encoding support (compression) type compressionType int8 const ( noCompression = compressionType(iota) gzipCompression snappyCompression unsupportedCompression ) // ServerAPIResponse is the response type for the server API like varz, connz etc. type ServerAPIResponse struct { Server *ServerInfo `json:"server"` Data any `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` // Private to indicate compression if any. compress compressionType } // Specialized response types for unmarshalling. These structures are not // used in the server code and only there for users of the Z endpoints to // unmarshal the data without having to create these structs in their code // ServerAPIConnzResponse is the response type connz type ServerAPIConnzResponse struct { Server *ServerInfo `json:"server"` Data *Connz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIRoutezResponse is the response type for routez type ServerAPIRoutezResponse struct { Server *ServerInfo `json:"server"` Data *Routez `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIGatewayzResponse is the response type for gatewayz type ServerAPIGatewayzResponse struct { Server *ServerInfo `json:"server"` Data *Gatewayz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIJszResponse is the response type for jsz type ServerAPIJszResponse struct { Server *ServerInfo `json:"server"` Data *JSInfo `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIHealthzResponse is the response type for healthz type ServerAPIHealthzResponse struct { Server *ServerInfo `json:"server"` Data *HealthStatus `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIVarzResponse is the response type for varz type ServerAPIVarzResponse struct { Server *ServerInfo `json:"server"` Data *Varz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPISubszResponse is the response type for subsz type ServerAPISubszResponse struct { Server *ServerInfo `json:"server"` Data *Subsz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPILeafzResponse is the response type for leafz type ServerAPILeafzResponse struct { Server *ServerInfo `json:"server"` Data *Leafz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIAccountzResponse is the response type for accountz type ServerAPIAccountzResponse struct { Server *ServerInfo `json:"server"` Data *Accountz `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIExpvarzResponse is the response type for expvarz type ServerAPIExpvarzResponse struct { Server *ServerInfo `json:"server"` Data *ExpvarzStatus `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIpqueueszResponse is the response type for ipqueuesz type ServerAPIpqueueszResponse struct { Server *ServerInfo `json:"server"` Data *IpqueueszStatus `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // ServerAPIRaftzResponse is the response type for raftz type ServerAPIRaftzResponse struct { Server *ServerInfo `json:"server"` Data *RaftzStatus `json:"data,omitempty"` Error *ApiError `json:"error,omitempty"` } // statszReq is a request for us to respond with current statsz. func (s *Server) statszReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.EventsEnabled() { return } // No reply is a signal that we should use our normal broadcast subject. if reply == _EMPTY_ { reply = fmt.Sprintf(serverStatsSubj, s.info.ID) s.wrapChk(s.resetLastStatsz) } opts := StatszEventOptions{} if len(msg) != 0 { if err := json.Unmarshal(msg, &opts); err != nil { response := &ServerAPIResponse{ Server: &ServerInfo{}, Error: &ApiError{Code: http.StatusBadRequest, Description: err.Error()}, } s.sendInternalMsgLocked(reply, _EMPTY_, response.Server, response) return } else if ignore := s.filterRequest(&opts.EventFilterOptions); ignore { return } } s.sendStatsz(reply) } // idzReq is for a request for basic static server info. // Try to not hold the write lock or dynamically create data. func (s *Server) idzReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { s.mu.RLock() defer s.mu.RUnlock() id := &ServerID{ Name: s.info.Name, Host: s.info.Host, ID: s.info.ID, } s.sendInternalMsg(reply, _EMPTY_, nil, &id) } var errSkipZreq = errors.New("filtered response") const ( acceptEncodingHeader = "Accept-Encoding" contentEncodingHeader = "Content-Encoding" ) // This is not as formal as it could be. We see if anything has s2 or snappy first, then gzip. func getAcceptEncoding(hdr []byte) compressionType { ae := strings.ToLower(string(getHeader(acceptEncodingHeader, hdr))) if ae == _EMPTY_ { return noCompression } if strings.Contains(ae, "snappy") || strings.Contains(ae, "s2") { return snappyCompression } if strings.Contains(ae, "gzip") { return gzipCompression } return unsupportedCompression } func (s *Server) zReq(_ *client, reply string, hdr, msg []byte, fOpts *EventFilterOptions, optz any, respf func() (any, error)) { if !s.EventsEnabled() || reply == _EMPTY_ { return } response := &ServerAPIResponse{Server: &ServerInfo{}} var err error status := 0 if len(msg) != 0 { if err = json.Unmarshal(msg, optz); err != nil { status = http.StatusBadRequest // status is only included on error, so record how far execution got } else if s.filterRequest(fOpts) { return } } if err == nil { response.Data, err = respf() if errors.Is(err, errSkipZreq) { return } else if err != nil { status = http.StatusInternalServerError } } if err != nil { response.Error = &ApiError{Code: status, Description: err.Error()} } else if len(hdr) > 0 { response.compress = getAcceptEncoding(hdr) } s.sendInternalResponse(reply, response) } // remoteConnsUpdate gets called when we receive a remote update from another server. func (s *Server) remoteConnsUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } var m AccountNumConns if len(msg) == 0 { s.sys.client.Errorf("No message body provided") return } else if err := json.Unmarshal(msg, &m); err != nil { s.sys.client.Errorf("Error unmarshalling account connection event message: %v", err) return } // See if we have the account registered, if not drop it. // Make sure this does not force us to load this account here. var acc *Account if v, ok := s.accounts.Load(m.Account); ok { acc = v.(*Account) } // Silently ignore these if we do not have local interest in the account. if acc == nil { return } s.mu.Lock() // check again here if we have been shutdown. if !s.isRunning() || !s.eventsEnabled() { s.mu.Unlock() return } // Double check that this is not us, should never happen, so error if it does. if m.Server.ID == s.info.ID { s.sys.client.Errorf("Processing our own account connection event message: ignored") s.mu.Unlock() return } // If we are here we have interest in tracking this account. Update our accounting. clients := acc.updateRemoteServer(&m) s.updateRemoteServer(&m.Server) s.mu.Unlock() // Need to close clients outside of server lock for _, c := range clients { c.maxAccountConnExceeded() } } // This will import any system level exports. func (s *Server) registerSystemImports(a *Account) { if a == nil || !s.EventsEnabled() { return } sacc := s.SystemAccount() if sacc == nil || sacc == a { return } // FIXME(dlc) - make a shared list between sys exports etc. importSrvc := func(subj, mappedSubj string) { if !a.serviceImportExists(subj) { if err := a.addServiceImportWithClaim(sacc, subj, mappedSubj, nil, true); err != nil { s.Errorf("Error setting up system service import %s -> %s for account: %v", subj, mappedSubj, err) } } } // Add in this to the account in 2 places. // "$SYS.REQ.SERVER.PING.CONNZ" and "$SYS.REQ.ACCOUNT.PING.CONNZ" mappedConnzSubj := fmt.Sprintf(accDirectReqSubj, a.Name, "CONNZ") importSrvc(fmt.Sprintf(accPingReqSubj, "CONNZ"), mappedConnzSubj) importSrvc(fmt.Sprintf(serverPingReqSubj, "CONNZ"), mappedConnzSubj) importSrvc(fmt.Sprintf(accPingReqSubj, "STATZ"), fmt.Sprintf(accDirectReqSubj, a.Name, "STATZ")) // This is for user's looking up their own info. mappedSubject := fmt.Sprintf(userDirectReqSubj, a.Name) importSrvc(userDirectInfoSubj, mappedSubject) // Make sure to share details. a.setServiceImportSharing(sacc, mappedSubject, false, true) } // Setup tracking for this account. This allows us to track global account activity. // Lock should be held on entry. func (s *Server) enableAccountTracking(a *Account) { if a == nil || !s.eventsEnabled() { return } // TODO(ik): Generate payload although message may not be sent. // May need to ensure we do so only if there is a known interest. // This can get complicated with gateways. subj := fmt.Sprintf(accDirectReqSubj, a.Name, "CONNS") reply := fmt.Sprintf(connsRespSubj, s.info.ID) m := accNumConnsReq{Account: a.Name} s.sendInternalMsg(subj, reply, &m.Server, &m) } // Event on leaf node connect. // Lock should NOT be held on entry. func (s *Server) sendLeafNodeConnect(a *Account) { s.mu.Lock() // If we are not in operator mode, or do not have any gateways defined, this should also be a no-op. if a == nil || !s.eventsEnabled() || !s.gateway.enabled { s.mu.Unlock() return } s.sendLeafNodeConnectMsg(a.Name) s.mu.Unlock() s.switchAccountToInterestMode(a.Name) } // Send the leafnode connect message. // Lock should be held. func (s *Server) sendLeafNodeConnectMsg(accName string) { subj := fmt.Sprintf(leafNodeConnectEventSubj, accName) m := accNumConnsReq{Account: accName} s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) } // sendAccConnsUpdate is called to send out our information on the // account's local connections. // Lock should be held on entry. func (s *Server) sendAccConnsUpdate(a *Account, subj ...string) { if !s.eventsEnabled() || a == nil { return } sendQ := s.sys.sendq if sendQ == nil { return } // Build event with account name and number of local clients and leafnodes. eid := s.nextEventID() a.mu.Lock() stat := a.statz() m := AccountNumConns{ TypedEvent: TypedEvent{ Type: AccountNumConnsMsgType, ID: eid, Time: time.Now().UTC(), }, AccountStat: *stat, } // Set timer to fire again unless we are at zero. if m.TotalConns == 0 { clearTimer(&a.ctmr) } else { // Check to see if we have an HB running and update. if a.ctmr == nil { a.ctmr = time.AfterFunc(eventsHBInterval, func() { s.accConnsUpdate(a) }) } else { a.ctmr.Reset(eventsHBInterval) } } for _, sub := range subj { msg := newPubMsg(nil, sub, _EMPTY_, &m.Server, nil, &m, noCompression, false, false) sendQ.push(msg) } a.mu.Unlock() } // Lock should be held on entry. func (a *Account) statz() *AccountStat { localConns := a.numLocalConnections() leafConns := a.numLocalLeafNodes() return &AccountStat{ Account: a.Name, Conns: localConns, LeafNodes: leafConns, TotalConns: localConns + leafConns, NumSubs: a.sl.Count(), Received: DataStats{ Msgs: atomic.LoadInt64(&a.inMsgs), Bytes: atomic.LoadInt64(&a.inBytes), }, Sent: DataStats{ Msgs: atomic.LoadInt64(&a.outMsgs), Bytes: atomic.LoadInt64(&a.outBytes), }, SlowConsumers: atomic.LoadInt64(&a.slowConsumers), } } // accConnsUpdate is called whenever there is a change to the account's // number of active connections, or during a heartbeat. // We will not send for $G. func (s *Server) accConnsUpdate(a *Account) { s.mu.Lock() defer s.mu.Unlock() if !s.eventsEnabled() || a == nil || a == s.gacc { return } s.sendAccConnsUpdate(a, fmt.Sprintf(accConnsEventSubjOld, a.Name), fmt.Sprintf(accConnsEventSubjNew, a.Name)) } // server lock should be held func (s *Server) nextEventID() string { return s.eventIds.Next() } // accountConnectEvent will send an account client connect event if there is interest. // This is a billing event. func (s *Server) accountConnectEvent(c *client) { s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return } gacc := s.gacc eid := s.nextEventID() s.mu.Unlock() c.mu.Lock() // Ignore global account activity if c.acc == nil || c.acc == gacc { c.mu.Unlock() return } m := ConnectEventMsg{ TypedEvent: TypedEvent{ Type: ConnectEventMsgType, ID: eid, Time: time.Now().UTC(), }, Client: ClientInfo{ Start: &c.start, Host: c.host, ID: c.cid, Account: accForClient(c), User: c.getRawAuthUser(), Name: c.opts.Name, Lang: c.opts.Lang, Version: c.opts.Version, Jwt: c.opts.JWT, IssuerKey: issuerForClient(c), Tags: c.tags, NameTag: c.nameTag, Kind: c.kindString(), ClientType: c.clientTypeString(), MQTTClient: c.getMQTTClientID(), }, } subj := fmt.Sprintf(connectEventSubj, c.acc.Name) c.mu.Unlock() s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) } // accountDisconnectEvent will send an account client disconnect event if there is interest. // This is a billing event. func (s *Server) accountDisconnectEvent(c *client, now time.Time, reason string) { s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return } gacc := s.gacc eid := s.nextEventID() s.mu.Unlock() c.mu.Lock() // Ignore global account activity if c.acc == nil || c.acc == gacc { c.mu.Unlock() return } m := DisconnectEventMsg{ TypedEvent: TypedEvent{ Type: DisconnectEventMsgType, ID: eid, Time: now, }, Client: ClientInfo{ Start: &c.start, Stop: &now, Host: c.host, ID: c.cid, Account: accForClient(c), User: c.getRawAuthUser(), Name: c.opts.Name, Lang: c.opts.Lang, Version: c.opts.Version, RTT: c.getRTT(), Jwt: c.opts.JWT, IssuerKey: issuerForClient(c), Tags: c.tags, NameTag: c.nameTag, Kind: c.kindString(), ClientType: c.clientTypeString(), MQTTClient: c.getMQTTClientID(), }, Sent: DataStats{ Msgs: atomic.LoadInt64(&c.inMsgs), Bytes: atomic.LoadInt64(&c.inBytes), }, Received: DataStats{ Msgs: c.outMsgs, Bytes: c.outBytes, }, Reason: reason, } accName := c.acc.Name c.mu.Unlock() subj := fmt.Sprintf(disconnectEventSubj, accName) s.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m) } // This is the system level event sent to the system account for operators. func (s *Server) sendAuthErrorEvent(c *client) { s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return } eid := s.nextEventID() s.mu.Unlock() now := time.Now().UTC() c.mu.Lock() m := DisconnectEventMsg{ TypedEvent: TypedEvent{ Type: DisconnectEventMsgType, ID: eid, Time: now, }, Client: ClientInfo{ Start: &c.start, Stop: &now, Host: c.host, ID: c.cid, Account: accForClient(c), User: c.getRawAuthUser(), Name: c.opts.Name, Lang: c.opts.Lang, Version: c.opts.Version, RTT: c.getRTT(), Jwt: c.opts.JWT, IssuerKey: issuerForClient(c), Tags: c.tags, NameTag: c.nameTag, Kind: c.kindString(), ClientType: c.clientTypeString(), MQTTClient: c.getMQTTClientID(), }, Sent: DataStats{ Msgs: c.inMsgs, Bytes: c.inBytes, }, Received: DataStats{ Msgs: c.outMsgs, Bytes: c.outBytes, }, Reason: AuthenticationViolation.String(), } c.mu.Unlock() s.mu.Lock() subj := fmt.Sprintf(authErrorEventSubj, s.info.ID) s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) s.mu.Unlock() } // This is the account level event sent to the origin account for account owners. func (s *Server) sendAccountAuthErrorEvent(c *client, acc *Account, reason string) { if acc == nil { return } s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return } eid := s.nextEventID() s.mu.Unlock() now := time.Now().UTC() c.mu.Lock() m := DisconnectEventMsg{ TypedEvent: TypedEvent{ Type: DisconnectEventMsgType, ID: eid, Time: now, }, Client: ClientInfo{ Start: &c.start, Stop: &now, Host: c.host, ID: c.cid, Account: acc.Name, User: c.getRawAuthUser(), Name: c.opts.Name, Lang: c.opts.Lang, Version: c.opts.Version, RTT: c.getRTT(), Jwt: c.opts.JWT, IssuerKey: issuerForClient(c), Tags: c.tags, NameTag: c.nameTag, Kind: c.kindString(), ClientType: c.clientTypeString(), MQTTClient: c.getMQTTClientID(), }, Sent: DataStats{ Msgs: c.inMsgs, Bytes: c.inBytes, }, Received: DataStats{ Msgs: c.outMsgs, Bytes: c.outBytes, }, Reason: reason, } c.mu.Unlock() s.sendInternalAccountSysMsg(acc, authErrorAccountEventSubj, &m.Server, &m) } // Internal message callback. // If the msg is needed past the callback it is required to be copied. // rmsg contains header and the message. use client.msgParts(rmsg) to split them apart type msgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, rmsg []byte) const ( recvQMuxed = 1 recvQStatsz = 2 ) // Create a wrapped callback handler for the subscription that will move it to an // internal recvQ for processing not inline with routes etc. func (s *Server) noInlineCallback(cb sysMsgHandler) msgHandler { return s.noInlineCallbackRecvQSelect(cb, recvQMuxed) } // Create a wrapped callback handler for the subscription that will move it to an // internal recvQ for Statsz/Pings for processing not inline with routes etc. func (s *Server) noInlineCallbackStatsz(cb sysMsgHandler) msgHandler { return s.noInlineCallbackRecvQSelect(cb, recvQStatsz) } // Create a wrapped callback handler for the subscription that will move it to an // internal IPQueue for processing not inline with routes etc. func (s *Server) noInlineCallbackRecvQSelect(cb sysMsgHandler, recvQSelect int) msgHandler { s.mu.RLock() if !s.eventsEnabled() { s.mu.RUnlock() return nil } // Capture here for direct reference to avoid any unnecessary blocking inline with routes, gateways etc. var recvq *ipQueue[*inSysMsg] switch recvQSelect { case recvQStatsz: recvq = s.sys.recvqp default: recvq = s.sys.recvq } s.mu.RUnlock() return func(sub *subscription, c *client, acc *Account, subj, rply string, rmsg []byte) { // Need to copy and split here. hdr, msg := c.msgParts(rmsg) recvq.push(&inSysMsg{sub, c, acc, subj, rply, copyBytes(hdr), copyBytes(msg), cb}) } } // Create an internal subscription. sysSubscribeQ for queue groups. func (s *Server) sysSubscribe(subject string, cb msgHandler) (*subscription, error) { return s.systemSubscribe(subject, _EMPTY_, false, nil, cb) } // Create an internal subscription with queue func (s *Server) sysSubscribeQ(subject, queue string, cb msgHandler) (*subscription, error) { return s.systemSubscribe(subject, queue, false, nil, cb) } // Create an internal subscription but do not forward interest. func (s *Server) sysSubscribeInternal(subject string, cb msgHandler) (*subscription, error) { return s.systemSubscribe(subject, _EMPTY_, true, nil, cb) } func (s *Server) systemSubscribe(subject, queue string, internalOnly bool, c *client, cb msgHandler) (*subscription, error) { s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return nil, ErrNoSysAccount } if cb == nil { s.mu.Unlock() return nil, fmt.Errorf("undefined message handler") } if c == nil { c = s.sys.client } trace := c.trace s.sys.sid++ sid := strconv.Itoa(s.sys.sid) s.mu.Unlock() // Now create the subscription if trace { c.traceInOp("SUB", []byte(subject+" "+queue+" "+sid)) } var q []byte if queue != _EMPTY_ { q = []byte(queue) } // Now create the subscription return c.processSub([]byte(subject), q, []byte(sid), cb, internalOnly) } func (s *Server) sysUnsubscribe(sub *subscription) { if sub == nil { return } s.mu.RLock() if !s.eventsEnabled() { s.mu.RUnlock() return } c := sub.client s.mu.RUnlock() if c != nil { c.processUnsub(sub.sid) } } // This will generate the tracking subject for remote latency from the response subject. func remoteLatencySubjectForResponse(subject []byte) string { if !isTrackedReply(subject) { return "" } toks := bytes.Split(subject, []byte(tsep)) // FIXME(dlc) - Sprintf may become a performance concern at some point. return fmt.Sprintf(remoteLatencyEventSubj, toks[len(toks)-2]) } // remoteLatencyUpdate is used to track remote latency measurements for tracking on exported services. func (s *Server) remoteLatencyUpdate(sub *subscription, _ *client, _ *Account, subject, _ string, hdr, msg []byte) { if !s.eventsRunning() { return } var rl remoteLatency if err := json.Unmarshal(msg, &rl); err != nil { s.Errorf("Error unmarshalling remote latency measurement: %v", err) return } // Now we need to look up the responseServiceImport associated with this measurement. acc, err := s.LookupAccount(rl.Account) if err != nil { s.Warnf("Could not lookup account %q for latency measurement", rl.Account) return } // Now get the request id / reply. We need to see if we have a GW prefix and if so strip that off. reply := rl.ReqId if gwPrefix, old := isGWRoutedSubjectAndIsOldPrefix([]byte(reply)); gwPrefix { reply = string(getSubjectFromGWRoutedReply([]byte(reply), old)) } acc.mu.RLock() si := acc.exports.responses[reply] if si == nil { acc.mu.RUnlock() return } lsub := si.latency.subject acc.mu.RUnlock() si.acc.mu.Lock() m1 := si.m1 m2 := rl.M2 // So we have not processed the response tracking measurement yet. if m1 == nil { // Store our value there for them to pick up. si.m1 = &m2 } si.acc.mu.Unlock() if m1 == nil { return } // Calculate the correct latencies given M1 and M2. m1.merge(&m2) // Clear the requesting client since we send the result here. acc.mu.Lock() si.rc = nil acc.mu.Unlock() // Make sure we remove the entry here. acc.removeServiceImport(si.from) // Send the metrics s.sendInternalAccountMsg(acc, lsub, m1) } // This is used for all inbox replies so that we do not send supercluster wide interest // updates for every request. Same trick used in modern NATS clients. func (s *Server) inboxReply(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) { s.mu.RLock() if !s.eventsEnabled() || s.sys.replies == nil { s.mu.RUnlock() return } cb, ok := s.sys.replies[subject] s.mu.RUnlock() if ok && cb != nil { cb(sub, c, acc, subject, reply, msg) } } // Copied from go client. // We could use serviceReply here instead to save some code. // I prefer these semantics for the moment, when tracing you know what this is. const ( InboxPrefix = "$SYS._INBOX." inboxPrefixLen = len(InboxPrefix) respInboxPrefixLen = inboxPrefixLen + sysHashLen + 1 replySuffixLen = 8 // Gives us 62^8 ) // Creates an internal inbox used for replies that will be processed by the global wc handler. func (s *Server) newRespInbox() string { var b [respInboxPrefixLen + replySuffixLen]byte pres := b[:respInboxPrefixLen] copy(pres, s.sys.inboxPre) rn := rand.Int63() for i, l := respInboxPrefixLen, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } return string(b[:]) } // accNumSubsReq is sent when we need to gather remote info on subs. type accNumSubsReq struct { Account string `json:"acc"` Subject string `json:"subject"` Queue []byte `json:"queue,omitempty"` } // helper function to total information from results to count subs. func totalSubs(rr *SublistResult, qg []byte) (nsubs int32) { if rr == nil { return } checkSub := func(sub *subscription) { // TODO(dlc) - This could be smarter. if qg != nil && !bytes.Equal(qg, sub.queue) { return } if sub.client.kind == CLIENT || sub.client.isHubLeafNode() { nsubs++ } } if qg == nil { for _, sub := range rr.psubs { checkSub(sub) } } for _, qsub := range rr.qsubs { for _, sub := range qsub { checkSub(sub) } } return } // Allows users of large systems to debug active subscribers for a given subject. // Payload should be the subject of interest. func (s *Server) debugSubscribers(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { // Even though this is an internal only subscription, meaning interest was not forwarded, we could // get one here from a GW in optimistic mode. Ignore for now. // FIXME(dlc) - Should we send no interest here back to the GW? if c.kind != CLIENT { return } var ci ClientInfo if len(hdr) > 0 { if err := json.Unmarshal(getHeader(ClientInfoHdr, hdr), &ci); err != nil { return } } var acc *Account if ci.Service != _EMPTY_ { acc, _ = s.LookupAccount(ci.Service) } else if ci.Account != _EMPTY_ { acc, _ = s.LookupAccount(ci.Account) } else { // Direct $SYS access. acc = c.acc if acc == nil { acc = s.SystemAccount() } } if acc == nil { return } // We could have a single subject or we could have a subject and a wildcard separated by whitespace. args := strings.Split(strings.TrimSpace(string(msg)), " ") if len(args) == 0 { s.sendInternalAccountMsg(acc, reply, 0) return } tsubj := args[0] var qgroup []byte if len(args) > 1 { qgroup = []byte(args[1]) } var nsubs int32 if subjectIsLiteral(tsubj) { // We will look up subscribers locally first then determine if we need to solicit other servers. rr := acc.sl.Match(tsubj) nsubs = totalSubs(rr, qgroup) } else { // We have a wildcard, so this is a bit slower path. var _subs [32]*subscription subs := _subs[:0] acc.sl.All(&subs) for _, sub := range subs { if subjectIsSubsetMatch(string(sub.subject), tsubj) { if qgroup != nil && !bytes.Equal(qgroup, sub.queue) { continue } if sub.client.kind == CLIENT || sub.client.isHubLeafNode() { nsubs++ } } } } // We should have an idea of how many responses to expect from remote servers. var expected = acc.expectedRemoteResponses() // If we are only local, go ahead and return. if expected == 0 { s.sendInternalAccountMsg(nil, reply, nsubs) return } // We need to solicit from others. // To track status. responses := int32(0) done := make(chan (bool)) s.mu.Lock() // Create direct reply inbox that we multiplex under the WC replies. replySubj := s.newRespInbox() // Store our handler. s.sys.replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { if n, err := strconv.Atoi(string(msg)); err == nil { atomic.AddInt32(&nsubs, int32(n)) } if atomic.AddInt32(&responses, 1) >= expected { select { case done <- true: default: } } } // Send the request to the other servers. request := &accNumSubsReq{ Account: acc.Name, Subject: tsubj, Queue: qgroup, } s.sendInternalMsg(accNumSubsReqSubj, replySubj, nil, request) s.mu.Unlock() // FIXME(dlc) - We should rate limit here instead of blind Go routine. go func() { select { case <-done: case <-time.After(500 * time.Millisecond): } // Cleanup the WC entry. var sendResponse bool s.mu.Lock() if s.sys != nil && s.sys.replies != nil { delete(s.sys.replies, replySubj) sendResponse = true } s.mu.Unlock() if sendResponse { // Send the response. s.sendInternalAccountMsg(nil, reply, atomic.LoadInt32(&nsubs)) } }() } // Request for our local subscription count. This will come from a remote origin server // that received the initial request. func (s *Server) nsubsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } m := accNumSubsReq{} if len(msg) == 0 { s.sys.client.Errorf("request requires a body") return } else if err := json.Unmarshal(msg, &m); err != nil { s.sys.client.Errorf("Error unmarshalling account nsubs request message: %v", err) return } // Grab account. acc, _ := s.lookupAccount(m.Account) if acc == nil || acc.numLocalAndLeafConnections() == 0 { return } // We will look up subscribers locally first then determine if we need to solicit other servers. var nsubs int32 if subjectIsLiteral(m.Subject) { rr := acc.sl.Match(m.Subject) nsubs = totalSubs(rr, m.Queue) } else { // We have a wildcard, so this is a bit slower path. var _subs [32]*subscription subs := _subs[:0] acc.sl.All(&subs) for _, sub := range subs { if (sub.client.kind == CLIENT || sub.client.isHubLeafNode()) && subjectIsSubsetMatch(string(sub.subject), m.Subject) { if m.Queue != nil && !bytes.Equal(m.Queue, sub.queue) { continue } nsubs++ } } } s.sendInternalMsgLocked(reply, _EMPTY_, nil, nsubs) } func (s *Server) reloadConfig(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } optz := &EventFilterOptions{} s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { // Reload the server config, as requested. return nil, s.Reload() }) } type KickClientReq struct { CID uint64 `json:"cid"` } type LDMClientReq struct { CID uint64 `json:"cid"` } func (s *Server) kickClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } var req KickClientReq if err := json.Unmarshal(msg, &req); err != nil { s.sys.client.Errorf("Error unmarshalling kick client request: %v", err) return } optz := &EventFilterOptions{} s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { return nil, s.DisconnectClientByID(req.CID) }) } func (s *Server) ldmClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) { if !s.eventsRunning() { return } var req LDMClientReq if err := json.Unmarshal(msg, &req); err != nil { s.sys.client.Errorf("Error unmarshalling kick client request: %v", err) return } optz := &EventFilterOptions{} s.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) { return nil, s.LDMClientByID(req.CID) }) } // Helper to grab account name for a client. func accForClient(c *client) string { if c.acc != nil { return c.acc.Name } return "N/A" } // Helper to grab issuer for a client. func issuerForClient(c *client) (issuerKey string) { if c == nil || c.user == nil { return } issuerKey = c.user.SigningKey if issuerKey == _EMPTY_ && c.user.Account != nil { issuerKey = c.user.Account.Name } return } // Helper to clear timers. func clearTimer(tp **time.Timer) { if t := *tp; t != nil { t.Stop() *tp = nil } } // Helper function to wrap functions with common test // to lock server and return if events not enabled. func (s *Server) wrapChk(f func()) func() { return func() { s.mu.Lock() if !s.eventsEnabled() { s.mu.Unlock() return } f() s.mu.Unlock() } } // sendOCSPPeerRejectEvent sends a system level event to system account when a peer connection is // rejected due to OCSP invalid status of its trust chain(s). func (s *Server) sendOCSPPeerRejectEvent(kind string, peer *x509.Certificate, reason string) { s.mu.Lock() defer s.mu.Unlock() if !s.eventsEnabled() { return } if peer == nil { s.Errorf(certidp.ErrPeerEmptyNoEvent) return } eid := s.nextEventID() now := time.Now().UTC() m := OCSPPeerRejectEventMsg{ TypedEvent: TypedEvent{ Type: OCSPPeerRejectEventMsgType, ID: eid, Time: now, }, Kind: kind, Peer: certidp.CertInfo{ Subject: certidp.GetSubjectDNForm(peer), Issuer: certidp.GetIssuerDNForm(peer), Fingerprint: certidp.GenerateFingerprint(peer), Raw: peer.Raw, }, Reason: reason, } subj := fmt.Sprintf(ocspPeerRejectEventSubj, s.info.ID) s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) } // sendOCSPPeerChainlinkInvalidEvent sends a system level event to system account when a link in a peer's trust chain // is OCSP invalid. func (s *Server) sendOCSPPeerChainlinkInvalidEvent(peer *x509.Certificate, link *x509.Certificate, reason string) { s.mu.Lock() defer s.mu.Unlock() if !s.eventsEnabled() { return } if peer == nil || link == nil { s.Errorf(certidp.ErrPeerEmptyNoEvent) return } eid := s.nextEventID() now := time.Now().UTC() m := OCSPPeerChainlinkInvalidEventMsg{ TypedEvent: TypedEvent{ Type: OCSPPeerChainlinkInvalidEventMsgType, ID: eid, Time: now, }, Link: certidp.CertInfo{ Subject: certidp.GetSubjectDNForm(link), Issuer: certidp.GetIssuerDNForm(link), Fingerprint: certidp.GenerateFingerprint(link), Raw: link.Raw, }, Peer: certidp.CertInfo{ Subject: certidp.GetSubjectDNForm(peer), Issuer: certidp.GetIssuerDNForm(peer), Fingerprint: certidp.GenerateFingerprint(peer), Raw: peer.Raw, }, Reason: reason, } subj := fmt.Sprintf(ocspPeerChainlinkInvalidEventSubj, s.info.ID) s.sendInternalMsg(subj, _EMPTY_, &m.Server, &m) } nats-server-2.10.27/server/events_test.go000066400000000000000000003166421477524627100203670ustar00rootroot00000000000000// Copyright 2018-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/sha256" "encoding/json" "errors" "fmt" "math/rand" "net/http" "net/http/httptest" "os" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) func createAccount(s *Server) (*Account, nkeys.KeyPair) { okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) jwt, _ := nac.Encode(okp) addAccountToMemResolver(s, pub, jwt) acc, err := s.LookupAccount(pub) if err != nil { panic(err) } return acc, akp } func createUserCredsEx(t *testing.T, nuc *jwt.UserClaims, akp nkeys.KeyPair) nats.Option { t.Helper() kp, _ := nkeys.CreateUser() nuc.Subject, _ = kp.PublicKey() ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } userCB := func() (string, error) { return ujwt, nil } sigCB := func(nonce []byte) ([]byte, error) { sig, _ := kp.Sign(nonce) return sig, nil } return nats.UserJWT(userCB, sigCB) } func createUserCreds(t *testing.T, _ *Server, akp nkeys.KeyPair) nats.Option { return createUserCredsEx(t, jwt.NewUserClaims("test"), akp) } func runTrustedServer(t *testing.T) (*Server, *Options) { t.Helper() opts := DefaultOptions() kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() opts.TrustedKeys = []string{pub} opts.AccountResolver = &MemAccResolver{} s := RunServer(opts) return s, opts } func runTrustedCluster(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) { t.Helper() kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() mr := &MemAccResolver{} // Now create a system account. // NOTE: This can NOT be shared directly between servers. // Set via server options. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) jwt, _ := nac.Encode(okp) mr.Store(apub, jwt) optsA := DefaultOptions() optsA.Cluster.Name = "TEST CLUSTER 22" optsA.Cluster.Host = "127.0.0.1" optsA.TrustedKeys = []string{pub} optsA.AccountResolver = mr optsA.SystemAccount = apub optsA.ServerName = "A_SRV" // Add in dummy gateway optsA.Gateway.Name = "TEST CLUSTER 22" optsA.Gateway.Host = "127.0.0.1" optsA.Gateway.Port = -1 optsA.gatewaysSolicitDelay = 30 * time.Second sa := RunServer(optsA) optsB := nextServerOpts(optsA) optsB.ServerName = "B_SRV" optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) sb := RunServer(optsB) checkClusterFormed(t, sa, sb) return sa, optsA, sb, optsB, akp } func runTrustedGateways(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) { t.Helper() kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() mr := &MemAccResolver{} // Now create a system account. // NOTE: This can NOT be shared directly between servers. // Set via server options. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) jwt, _ := nac.Encode(okp) mr.Store(apub, jwt) optsA := testDefaultOptionsForGateway("A") optsA.Cluster.Name = "A" optsA.Cluster.Host = "127.0.0.1" optsA.TrustedKeys = []string{pub} optsA.AccountResolver = mr optsA.SystemAccount = apub sa := RunServer(optsA) optsB := testGatewayOptionsFromToWithServers(t, "B", "A", sa) optsB.Cluster.Name = "B" optsB.TrustedKeys = []string{pub} optsB.AccountResolver = mr optsB.SystemAccount = apub sb := RunServer(optsB) waitForInboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) return sa, optsA, sb, optsB, akp } func TestSystemAccount(t *testing.T) { s, _ := runTrustedServer(t) defer s.Shutdown() acc, _ := createAccount(s) s.setSystemAccount(acc) s.mu.Lock() defer s.mu.Unlock() if s.sys == nil || s.sys.account == nil { t.Fatalf("Expected sys.account to be non-nil") } if s.sys.client == nil { t.Fatalf("Expected sys.client to be non-nil") } s.sys.client.mu.Lock() defer s.sys.client.mu.Unlock() if s.sys.client.echo { t.Fatalf("Internal clients should always have echo false") } } func TestSystemAccountNewConnection(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() // We may not be able to hear ourselves (if the event is processed // before we create the sub), so we need to create a second client to // trigger the connect/disconnect events. acc2, akp2 := createAccount(s) // Be explicit to only receive the event for acc2. sub, _ := ncs.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.>", acc2.Name)) defer sub.Unsubscribe() ncs.Flush() nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } connsMsg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { msg, connsMsg = connsMsg, msg } if !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) } conns := AccountNumConns{} if err := json.Unmarshal(connsMsg.Data, &conns); err != nil { t.Fatalf("Error unmarshalling conns event message: %v", err) } else if conns.Account != acc2.Name { t.Fatalf("Wrong account in conns message: %v", conns) } else if conns.Conns != 1 || conns.TotalConns != 1 || conns.LeafNodes != 0 { t.Fatalf("Wrong counts in conns message: %v", conns) } if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", acc2.Name)) { t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) } tokens := strings.Split(msg.Subject, ".") if len(tokens) < 4 { t.Fatalf("Expected 4 tokens, got %d", len(tokens)) } account := tokens[2] if account != acc2.Name { t.Fatalf("Expected %q for account, got %q", acc2.Name, account) } cem := ConnectEventMsg{} if err := json.Unmarshal(msg.Data, &cem); err != nil { t.Fatalf("Error unmarshalling connect event message: %v", err) } if cem.Type != ConnectEventMsgType { t.Fatalf("Incorrect schema in connect event: %s", cem.Type) } if cem.Time.IsZero() { t.Fatalf("Event time is not set") } if len(cem.ID) != 22 { t.Fatalf("Event ID is incorrectly set to len %d", len(cem.ID)) } if cem.Server.ID != s.ID() { t.Fatalf("Expected server to be %q, got %q", s.ID(), cem.Server.ID) } if cem.Server.Seq == 0 { t.Fatalf("Expected sequence to be non-zero") } if cem.Client.Name != "TEST EVENTS" { t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", cem.Client.Name) } if cem.Client.Lang != "go" { t.Fatalf("Expected client lang to be \"go\", got %q", cem.Client.Lang) } // Now close the other client. Should fire a disconnect event. // First send and receive some messages. sub2, _ := nc.SubscribeSync("foo") defer sub2.Unsubscribe() sub3, _ := nc.SubscribeSync("*") defer sub3.Unsubscribe() for i := 0; i < 10; i++ { nc.Publish("foo", []byte("HELLO WORLD")) } nc.Flush() nc.Close() msg, err = sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } connsMsg, err = sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { msg, connsMsg = connsMsg, msg } if !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", acc2.Name)) { t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..CONNECT", msg.Subject) } else if !strings.Contains(string(connsMsg.Data), `"total_conns":0`) { t.Fatalf("Expected event to reflect created connection, got: %s", string(connsMsg.Data)) } conns = AccountNumConns{} if err := json.Unmarshal(connsMsg.Data, &conns); err != nil { t.Fatalf("Error unmarshalling conns event message: %v", err) } else if conns.Account != acc2.Name { t.Fatalf("Wrong account in conns message: %v", conns) } else if conns.Conns != 0 || conns.TotalConns != 0 || conns.LeafNodes != 0 { t.Fatalf("Wrong counts in conns message: %v", conns) } if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", acc2.Name)) { t.Fatalf("Expected subject to start with %q, got %q", "$SYS.ACCOUNT..DISCONNECT", msg.Subject) } tokens = strings.Split(msg.Subject, ".") if len(tokens) < 4 { t.Fatalf("Expected 4 tokens, got %d", len(tokens)) } account = tokens[2] if account != acc2.Name { t.Fatalf("Expected %q for account, got %q", acc2.Name, account) } dem := DisconnectEventMsg{} if err := json.Unmarshal(msg.Data, &dem); err != nil { t.Fatalf("Error unmarshalling disconnect event message: %v", err) } if dem.Type != DisconnectEventMsgType { t.Fatalf("Incorrect schema in connect event: %s", cem.Type) } if dem.Time.IsZero() { t.Fatalf("Event time is not set") } if len(dem.ID) != 22 { t.Fatalf("Event ID is incorrectly set to len %d", len(cem.ID)) } if dem.Server.ID != s.ID() { t.Fatalf("Expected server to be %q, got %q", s.ID(), dem.Server.ID) } if dem.Server.Seq == 0 { t.Fatalf("Expected sequence to be non-zero") } if dem.Server.Seq <= cem.Server.Seq { t.Fatalf("Expected sequence to be increasing") } if cem.Client.Name != "TEST EVENTS" { t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", dem.Client.Name) } if dem.Client.Lang != "go" { t.Fatalf("Expected client lang to be \"go\", got %q", dem.Client.Lang) } if dem.Sent.Msgs != 10 { t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs) } if dem.Sent.Bytes != 110 { t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes) } if dem.Received.Msgs != 20 { t.Fatalf("Expected 20 msgs received, got %d", dem.Sent.Msgs) } if dem.Received.Bytes != 220 { t.Fatalf("Expected 220 bytes sent, got %d", dem.Sent.Bytes) } } func runTrustedLeafServer(t *testing.T) (*Server, *Options) { t.Helper() opts := DefaultOptions() kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() opts.TrustedKeys = []string{pub} opts.AccountResolver = &MemAccResolver{} opts.LeafNode.Port = -1 s := RunServer(opts) return s, opts } func genCredsFile(t *testing.T, jwt string, seed []byte) string { creds := ` -----BEGIN NATS USER JWT----- %s ------END NATS USER JWT------ ************************* IMPORTANT ************************* NKEY Seed printed below can be used to sign and prove identity. NKEYs are sensitive and should be treated as secrets. -----BEGIN USER NKEY SEED----- %s ------END USER NKEY SEED------ ************************************************************* ` return createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), "\t\t", "", -1))) } func runSolicitWithCredentials(t *testing.T, opts *Options, creds string) (*Server, *Options, string) { content := ` port: -1 leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:%d credentials: '%s' } ] } ` config := fmt.Sprintf(content, opts.LeafNode.Port, creds) conf := createConfFile(t, []byte(config)) s, opts := RunServerWithConfig(conf) return s, opts, conf } // Helper function to check that a leaf node has connected to our server. func checkLeafNodeConnected(t testing.TB, s *Server) { t.Helper() checkLeafNodeConnectedCount(t, s, 1) } // Helper function to check that a leaf node has connected to n server. func checkLeafNodeConnectedCount(t testing.TB, s *Server, lnCons int) { t.Helper() checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { if nln := s.NumLeafNodes(); nln != lnCons { return fmt.Errorf("Expected %d connected leafnode(s) for server %v, got %d", lnCons, s, nln) } return nil }) } func TestSystemAccountingWithLeafNodes(t *testing.T) { s, opts := runTrustedLeafServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() acc2, akp2 := createAccount(s) // Be explicit to only receive the event for acc2 account. sub, _ := ncs.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", acc2.Name)) defer sub.Unsubscribe() ncs.Flush() kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp2) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) // Create a server that solicits a leafnode connection. sl, slopts, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s) // Compute the expected number of subs on "sl" based on number // of existing subs before creating the sub on "s". expected := int(sl.NumSubscriptions() + 1) nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST LEAFNODE EVENTS")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() fooSub := natsSubSync(t, nc, "foo") natsFlush(t, nc) checkExpectedSubs(t, expected, sl) surl := fmt.Sprintf("nats://%s:%d", slopts.Host, slopts.Port) nc2, err := nats.Connect(surl, nats.Name("TEST LEAFNODE EVENTS")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Compute the expected number of subs on "s" based on number // of existing subs before creating the sub on "sl". expected = int(s.NumSubscriptions() + 1) m := []byte("HELLO WORLD") // Now generate some traffic starSub := natsSubSync(t, nc2, "*") for i := 0; i < 10; i++ { nc2.Publish("foo", m) nc2.Publish("bar", m) } natsFlush(t, nc2) checkExpectedSubs(t, expected, s) // Now send some from the cluster side too. for i := 0; i < 10; i++ { nc.Publish("foo", m) nc.Publish("bar", m) } nc.Flush() // Make sure all messages are received for i := 0; i < 20; i++ { if _, err := fooSub.NextMsg(time.Second); err != nil { t.Fatalf("Did not get message: %v", err) } } for i := 0; i < 40; i++ { if _, err := starSub.NextMsg(time.Second); err != nil { t.Fatalf("Did not get message: %v", err) } } // Now shutdown the leafnode server since this is where the event tracking should // happen. Right now we do not track local clients to the leafnode server that // solicited to the cluster, but we should track usage once the leafnode connection stops. sl.Shutdown() // Make sure we get disconnect event and that tracking is correct. msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } dem := DisconnectEventMsg{} if err := json.Unmarshal(msg.Data, &dem); err != nil { t.Fatalf("Error unmarshalling disconnect event message: %v", err) } if dem.Sent.Msgs != 10 { t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs) } if dem.Sent.Bytes != 110 { t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes) } if dem.Received.Msgs != 20 { t.Fatalf("Expected 20 msgs received, got %d", dem.Received.Msgs) } if dem.Received.Bytes != 220 { t.Fatalf("Expected 220 bytes sent, got %d", dem.Received.Bytes) } } func TestSystemAccountDisconnectBadLogin(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() // We should never hear $G account events for bad logins. sub, _ := ncs.SubscribeSync("$SYS.ACCOUNT.$G.*") defer sub.Unsubscribe() // Listen for auth error events though. asub, _ := ncs.SubscribeSync("$SYS.SERVER.*.CLIENT.AUTH.ERR") defer asub.Unsubscribe() ncs.Flush() nats.Connect(url, nats.Name("TEST BAD LOGIN")) // Should not hear these. if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Received a disconnect message from bad login, expected none") } m, err := asub.NextMsg(100 * time.Millisecond) if err != nil { t.Fatalf("Should have heard an auth error event") } dem := DisconnectEventMsg{} if err := json.Unmarshal(m.Data, &dem); err != nil { t.Fatalf("Error unmarshalling disconnect event message: %v", err) } if dem.Reason != "Authentication Failure" { t.Fatalf("Expected auth error, got %q", dem.Reason) } } func TestSysSubscribeRace(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() done := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { nc.Publish("foo", []byte("hello")) select { case <-done: return default: } } }() time.Sleep(10 * time.Millisecond) received := make(chan struct{}) // Create message callback handler. cb := func(sub *subscription, producer *client, _ *Account, subject, reply string, msg []byte) { select { case received <- struct{}{}: default: } } // Now create an internal subscription sub, err := s.sysSubscribe("foo", cb) if sub == nil || err != nil { t.Fatalf("Expected to subscribe, got %v", err) } select { case <-received: close(done) case <-time.After(time.Second): t.Fatalf("Did not receive the message") } wg.Wait() } func TestSystemAccountInternalSubscriptions(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sub, err := s.sysSubscribe("foo", nil) if sub != nil || err != ErrNoSysAccount { t.Fatalf("Expected to get proper error, got %v", err) } acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() sub, err = s.sysSubscribe("foo", nil) if sub != nil || err == nil { t.Fatalf("Expected to get error for no handler, got %v", err) } received := make(chan *nats.Msg) // Create message callback handler. cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { copy := append([]byte(nil), msg...) received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy} } // Now create an internal subscription sub, err = s.sysSubscribe("foo", cb) if sub == nil || err != nil { t.Fatalf("Expected to subscribe, got %v", err) } // Now send out a message from our normal client. nc.Publish("foo", []byte("HELLO WORLD")) var msg *nats.Msg select { case msg = <-received: if msg.Subject != "foo" { t.Fatalf("Expected \"foo\" as subject, got %q", msg.Subject) } if msg.Reply != "" { t.Fatalf("Expected no reply, got %q", msg.Reply) } if !bytes.Equal(msg.Data, []byte("HELLO WORLD")) { t.Fatalf("Got the wrong msg payload: %q", msg.Data) } break case <-time.After(time.Second): t.Fatalf("Did not receive the message") } s.sysUnsubscribe(sub) // Now send out a message from our normal client. // We should not see this one. nc.Publish("foo", []byte("You There?")) select { case <-received: t.Fatalf("Received a message when we should not have") case <-time.After(100 * time.Millisecond): break } // Now make sure we do not hear ourselves. We optimize this for internally // generated messages. s.mu.Lock() s.sendInternalMsg("foo", "", nil, msg.Data) s.mu.Unlock() select { case <-received: t.Fatalf("Received a message when we should not have") case <-time.After(100 * time.Millisecond): break } } func TestSystemAccountConnectionUpdatesStopAfterNoLocal(t *testing.T) { sa, _, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // Normal Account okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 4 // Limit to 4 connections. jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) // Listen for updates to the new account connection activity. received := make(chan *nats.Msg, 10) cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { copy := append([]byte(nil), msg...) received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy} } subj := fmt.Sprintf(accConnsEventSubjNew, pub) sub, err := sa.sysSubscribe(subj, cb) if sub == nil || err != nil { t.Fatalf("Expected to subscribe, got %v", err) } defer sa.sysUnsubscribe(sub) // Create a few users on the new account. clients := []*nats.Conn{} url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) for i := 0; i < 4; i++ { nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() clients = append(clients, nc) } // Wait for all 4 notifications. checkFor(t, time.Second, 50*time.Millisecond, func() error { if len(received) == 4 { return nil } return fmt.Errorf("Not enough messages, %d vs 4", len(received)) }) // Now lookup the account doing the events on sb. acc, _ := sb.LookupAccount(pub) // Make sure we have the timer running. acc.mu.RLock() ctmr := acc.ctmr acc.mu.RUnlock() if ctmr == nil { t.Fatalf("Expected event timer for acc conns to be running") } // Now close all of the connections. for _, nc := range clients { nc.Close() } // Wait for the 4 new notifications, 8 total (4 for connect, 4 for disconnect) checkFor(t, time.Second, 50*time.Millisecond, func() error { if len(received) == 8 { return nil } return fmt.Errorf("Not enough messages, %d vs 4", len(received)) }) // Drain the messages. for i := 0; i < 7; i++ { <-received } // Check last one. msg := <-received m := AccountNumConns{} if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatalf("Error unmarshalling account connections request message: %v", err) } if m.Conns != 0 { t.Fatalf("Expected Conns to be 0, got %d", m.Conns) } // Should not receive any more messages.. select { case <-received: t.Fatalf("Did not expect a message here") case <-time.After(50 * time.Millisecond): break } // Make sure we have the timer is NOT running. acc.mu.RLock() ctmr = acc.ctmr acc.mu.RUnlock() if ctmr != nil { t.Fatalf("Expected event timer for acc conns to NOT be running after reaching zero local clients") } } func TestSystemAccountConnectionLimits(t *testing.T) { sa, optsA, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // We want to test that we are limited to a certain number of active connections // across multiple servers. // Let's create a user account. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 4 // Limit to 4 connections. jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) // Create a user on each server. Break on first failure. for { nca1, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) if err != nil { break } defer nca1.Close() ncb1, err := nats.Connect(urlB, createUserCreds(t, sb, akp)) if err != nil { break } defer ncb1.Close() } checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { total := sa.NumClients() + sb.NumClients() if total > int(nac.Limits.Conn) { return fmt.Errorf("Expected only %d connections, was allowed to connect %d", nac.Limits.Conn, total) } return nil }) } func TestBadAccountUpdate(t *testing.T) { sa, _ := runTrustedServer(t) defer sa.Shutdown() akp1, _ := nkeys.CreateAccount() pub, _ := akp1.PublicKey() nac := jwt.NewAccountClaims(pub) ajwt1, err := nac.Encode(oKp) require_NoError(t, err) addAccountToMemResolver(sa, pub, ajwt1) akp2, _ := nkeys.CreateAccount() pub2, _ := akp2.PublicKey() nac.Subject = pub2 // maliciously use a different subject but pretend to remain pub ajwt2, err := nac.Encode(oKp) require_NoError(t, err) acc, err := sa.fetchAccount(pub) require_NoError(t, err) if err := sa.updateAccountWithClaimJWT(acc, ajwt2); err != ErrAccountValidation { t.Fatalf("expected %v but got %v", ErrAccountValidation, err) } } // Make sure connection limits apply to the system account itself. func TestSystemAccountSystemConnectionLimitsHonored(t *testing.T) { sa, optsA, sb, optsB, sakp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() okp, _ := nkeys.FromSeed(oSeed) // Update system account to have 10 connections pub, _ := sakp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 10 ajwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, ajwt) addAccountToMemResolver(sb, pub, ajwt) // Update the accounts on each server with new claims to force update. sysAccA := sa.SystemAccount() sa.updateAccountWithClaimJWT(sysAccA, ajwt) sysAccB := sb.SystemAccount() sb.updateAccountWithClaimJWT(sysAccB, ajwt) // Check system here first, with no external it should be zero. sacc := sa.SystemAccount() if nlc := sacc.NumLocalConnections(); nlc != 0 { t.Fatalf("Expected no local connections, got %d", nlc) } urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) // Create a user on each server. Break on first failure. tc := 0 for { nca1, err := nats.Connect(urlA, createUserCreds(t, sa, sakp)) if err != nil { break } defer nca1.Close() tc++ ncb1, err := nats.Connect(urlB, createUserCreds(t, sb, sakp)) if err != nil { break } defer ncb1.Close() tc++ // The account's connection count is exchanged between servers // so that the local count on each server reflects the total count. // Pause a bit to give a chance to each server to process the update. time.Sleep(15 * time.Millisecond) } if tc != 10 { t.Fatalf("Expected to get 10 external connections, got %d", tc) } checkFor(t, 1*time.Second, 50*time.Millisecond, func() error { total := sa.NumClients() + sb.NumClients() if total > int(nac.Limits.Conn) { return fmt.Errorf("Expected only %d connections, was allowed to connect %d", nac.Limits.Conn, total) } return nil }) } // Test that the remote accounting works when a server is started some time later. func TestSystemAccountConnectionLimitsServersStaggered(t *testing.T) { sa, optsA, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() sb.Shutdown() // Let's create a user account. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 4 // Limit to 4 connections. jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) // Create max connections on sa. for i := 0; i < int(nac.Limits.Conn); i++ { nc, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Unexpected error on #%d try: %v", i+1, err) } defer nc.Close() } // Restart server B. optsB.AccountResolver = sa.AccountResolver() optsB.SystemAccount = sa.SystemAccount().Name sb = RunServer(optsB) defer sb.Shutdown() checkClusterFormed(t, sa, sb) // Trigger a load of the user account on the new server // NOTE: If we do not load the user, the user can be the first // to request this account, hence the connection will succeed. checkFor(t, time.Second, 15*time.Millisecond, func() error { if acc, err := sb.LookupAccount(pub); acc == nil || err != nil { return fmt.Errorf("LookupAccount did not return account or failed, err=%v", err) } return nil }) // Expect this to fail. urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) if _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil { t.Fatalf("Expected connection to fail due to max limit") } } // Test that the remote accounting works when a server is shutdown. func TestSystemAccountConnectionLimitsServerShutdownGraceful(t *testing.T) { sa, optsA, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // Let's create a user account. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 10 // Limit to 10 connections. jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) addAccountToMemResolver(sb, pub, jwt) urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) for i := 0; i < 5; i++ { nc, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc.Close() nc, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc.Close() } // We are at capacity so both of these should fail. if _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil { t.Fatalf("Expected connection to fail due to max limit") } if _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil { t.Fatalf("Expected connection to fail due to max limit") } // Now shutdown Server B. sb.Shutdown() // Now we should be able to create more on A now. for i := 0; i < 5; i++ { nc, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Expected to connect on %d, got %v", i, err) } defer nc.Close() } } // Test that the remote accounting works when a server goes away. func TestSystemAccountConnectionLimitsServerShutdownForced(t *testing.T) { sa, optsA, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() // Let's create a user account. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 20 // Limit to 20 connections. jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) addAccountToMemResolver(sb, pub, jwt) urlA := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) for i := 0; i < 10; i++ { c, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer c.Close() c, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer c.Close() } // Now shutdown Server B. Do so such that no communications go out. sb.mu.Lock() sb.sys = nil sb.mu.Unlock() sb.Shutdown() if _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil { t.Fatalf("Expected connection to fail due to max limit") } // Let's speed up the checking process. sa.mu.Lock() sa.sys.chkOrph = 10 * time.Millisecond sa.sys.orphMax = 30 * time.Millisecond sa.sys.sweeper.Reset(sa.sys.chkOrph) sa.mu.Unlock() // We should eventually be able to connect. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { if c, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err != nil { return err } else { c.Close() } return nil }) } func TestSystemAccountFromConfig(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) opub, _ := kp.PublicKey() akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(kp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(ajwt)) })) defer ts.Close() confTemplate := ` listen: -1 trusted: %s system_account: %s resolver: URL("%s/jwt/v1/accounts/") ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, opub, apub, ts.URL))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() if acc := s.SystemAccount(); acc == nil || acc.Name != apub { t.Fatalf("System Account not properly set") } } func TestAccountClaimsUpdates(t *testing.T) { test := func(subj string) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, sakp := createAccount(s) s.setSystemAccount(sacc) // Let's create a normal account with limits we can update. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 4 ajwt, _ := nac.Encode(okp) addAccountToMemResolver(s, pub, ajwt) acc, _ := s.LookupAccount(pub) if acc.MaxActiveConnections() != 4 { t.Fatalf("Expected to see a limit of 4 connections") } // Simulate a systems publisher so we can do an account claims update. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, sakp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Update the account nac = jwt.NewAccountClaims(pub) nac.Limits.Conn = 8 issAt := time.Now().Add(-30 * time.Second).Unix() nac.IssuedAt = issAt expires := time.Now().Add(2 * time.Second).Unix() nac.Expires = expires ajwt, _ = nac.Encode(okp) // Publish to the system update subject. claimUpdateSubj := fmt.Sprintf(subj, pub) nc.Publish(claimUpdateSubj, []byte(ajwt)) nc.Flush() time.Sleep(200 * time.Millisecond) acc, _ = s.LookupAccount(pub) if acc.MaxActiveConnections() != 8 { t.Fatalf("Account was not updated") } } t.Run("new", func(t *testing.T) { test(accUpdateEventSubjNew) }) t.Run("old", func(t *testing.T) { test(accUpdateEventSubjOld) }) } func TestAccountReqMonitoring(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, sakp := createAccount(s) s.setSystemAccount(sacc) s.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()}) unusedAcc, _ := createAccount(s) acc, akp := createAccount(s) acc.EnableJetStream(nil) subsz := fmt.Sprintf(accDirectReqSubj, acc.Name, "SUBSZ") connz := fmt.Sprintf(accDirectReqSubj, acc.Name, "CONNZ") jsz := fmt.Sprintf(accDirectReqSubj, acc.Name, "JSZ") pStatz := fmt.Sprintf(accPingReqSubj, "STATZ") statz := func(name string) string { return fmt.Sprintf(accDirectReqSubj, name, "STATZ") } // Create system account connection to query url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncSys, err := nats.Connect(url, createUserCreds(t, s, sakp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncSys.Close() // Create a connection that we can query nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // query SUBSZ for account resp, err := ncSys.Request(subsz, nil, time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), `"num_subscriptions":5,`) // create a subscription sub, err := nc.Subscribe("foo", func(msg *nats.Msg) {}) require_NoError(t, err) defer sub.Unsubscribe() require_NoError(t, nc.Flush()) // query SUBSZ for account resp, err = ncSys.Request(subsz, nil, time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), `"num_subscriptions":6,`, `"subject":"foo"`) // query connections for account resp, err = ncSys.Request(connz, nil, time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), `"num_connections":1,`, `"total":1,`) // query connections for js account resp, err = ncSys.Request(jsz, nil, time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), `"memory":0,`, `"storage":0,`) // query statz/conns for account resp, err = ncSys.Request(statz(acc.Name), nil, time.Second) require_NoError(t, err) respContentAcc := []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"sent":{"msgs":0,"bytes":0}`, `"received":{"msgs":0,"bytes":0}`, `"num_subscriptions":`, fmt.Sprintf(`"acc":"%s"`, acc.Name)} require_Contains(t, string(resp.Data), respContentAcc...) rIb := ncSys.NewRespInbox() rSub, err := ncSys.SubscribeSync(rIb) require_NoError(t, err) require_NoError(t, ncSys.PublishRequest(pStatz, rIb, nil)) minRespContentForBothAcc := []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"acc":"`, `"num_subscriptions":`} resp, err = rSub.NextMsg(time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), minRespContentForBothAcc...) // expect one entry per account require_Contains(t, string(resp.Data), fmt.Sprintf(`"acc":"%s"`, acc.Name), fmt.Sprintf(`"acc":"%s"`, sacc.Name)) // Test ping with filter by account name require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, sacc.Name)))) m, err := rSub.NextMsg(time.Second) require_NoError(t, err) require_Contains(t, string(m.Data), minRespContentForBothAcc...) require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, acc.Name)))) m, err = rSub.NextMsg(time.Second) require_NoError(t, err) require_Contains(t, string(m.Data), respContentAcc...) // Test include unused for statz and ping of statz unusedContent := []string{`"conns":0,`, `"total_conns":0`, `"slow_consumers":0`, fmt.Sprintf(`"acc":"%s"`, unusedAcc.Name)} resp, err = ncSys.Request(statz(unusedAcc.Name), []byte(fmt.Sprintf(`{"accounts":["%s"], "include_unused":true}`, unusedAcc.Name)), time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), unusedContent...) require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"], "include_unused":true}`, unusedAcc.Name)))) resp, err = rSub.NextMsg(time.Second) require_NoError(t, err) require_Contains(t, string(resp.Data), unusedContent...) require_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{"accounts":["%s"]}`, unusedAcc.Name)))) _, err = rSub.NextMsg(200 * time.Millisecond) require_Error(t, err) // Test ping from within account, send extra message to check counters. require_NoError(t, nc.Publish("foo", nil)) ib := nc.NewRespInbox() rSub, err = nc.SubscribeSync(ib) require_NoError(t, err) require_NoError(t, nc.PublishRequest(pStatz, ib, nil)) require_NoError(t, nc.Flush()) resp, err = rSub.NextMsg(time.Second) require_NoError(t, err) // Since we now have processed our own message, sent msgs will be at least 1. payload := string(resp.Data) respContentAcc = []string{`"conns":1,`, `"total_conns":1`, `"slow_consumers":0`, `"sent":{"msgs":1,"bytes":0}`, fmt.Sprintf(`"acc":"%s"`, acc.Name)} require_Contains(t, payload, respContentAcc...) // Depending on timing, statz message could be accounted too. receivedOK := strings.Contains(payload, `"received":{"msgs":1,"bytes":0}`) || strings.Contains(payload, `"received":{"msgs":2,"bytes":0}`) require_True(t, receivedOK) _, err = rSub.NextMsg(200 * time.Millisecond) require_Error(t, err) } func TestAccountReqInfo(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, sakp := createAccount(s) s.setSystemAccount(sacc) // Let's create an account with service export. akp, _ := nkeys.CreateAccount() pub1, _ := akp.PublicKey() nac1 := jwt.NewAccountClaims(pub1) nac1.Exports.Add(&jwt.Export{Subject: "req.*", Type: jwt.Service}) ajwt1, _ := nac1.Encode(oKp) addAccountToMemResolver(s, pub1, ajwt1) s.LookupAccount(pub1) info1 := fmt.Sprintf(accDirectReqSubj, pub1, "INFO") // Now add an account with service imports. akp2, _ := nkeys.CreateAccount() pub2, _ := akp2.PublicKey() nac2 := jwt.NewAccountClaims(pub2) nac2.Imports.Add(&jwt.Import{Account: pub1, Subject: "req.1", Type: jwt.Service}) ajwt2, _ := nac2.Encode(oKp) addAccountToMemResolver(s, pub2, ajwt2) s.LookupAccount(pub2) info2 := fmt.Sprintf(accDirectReqSubj, pub2, "INFO") // Create system account connection to query url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncSys, err := nats.Connect(url, createUserCreds(t, s, sakp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncSys.Close() checkCommon := func(info *AccountInfo, srv *ServerInfo, pub, jwt string) { if info.Complete != true { t.Fatalf("Unexpected value: %v", info.Complete) } else if info.Expired != false { t.Fatalf("Unexpected value: %v", info.Expired) } else if info.JetStream != false { t.Fatalf("Unexpected value: %v", info.JetStream) } else if info.ClientCnt != 0 { t.Fatalf("Unexpected value: %v", info.ClientCnt) } else if info.AccountName != pub { t.Fatalf("Unexpected value: %v", info.AccountName) } else if info.LeafCnt != 0 { t.Fatalf("Unexpected value: %v", info.LeafCnt) } else if info.Jwt != jwt { t.Fatalf("Unexpected value: %v", info.Jwt) } else if srv.Cluster != "abc" { t.Fatalf("Unexpected value: %v", srv.Cluster) } else if srv.Name != s.Name() { t.Fatalf("Unexpected value: %v", srv.Name) } else if srv.Host != opts.Host { t.Fatalf("Unexpected value: %v", srv.Host) } else if srv.Seq < 1 { t.Fatalf("Unexpected value: %v", srv.Seq) } } info := AccountInfo{} srv := ServerInfo{} msg := struct { Data *AccountInfo `json:"data"` Srv *ServerInfo `json:"server"` }{ &info, &srv, } if resp, err := ncSys.Request(info1, nil, time.Second); err != nil { t.Fatalf("Error on request: %v", err) } else if err := json.Unmarshal(resp.Data, &msg); err != nil { t.Fatalf("Unmarshalling failed: %v", err) } else if len(info.Exports) != 1 { t.Fatalf("Unexpected value: %v", info.Exports) } else if len(info.Imports) != 4 { t.Fatalf("Unexpected value: %+v", info.Imports) } else if info.Exports[0].Subject != "req.*" { t.Fatalf("Unexpected value: %v", info.Exports) } else if info.Exports[0].Type != jwt.Service { t.Fatalf("Unexpected value: %v", info.Exports) } else if info.Exports[0].ResponseType != jwt.ResponseTypeSingleton { t.Fatalf("Unexpected value: %v", info.Exports) } else if info.SubCnt != 4 { t.Fatalf("Unexpected value: %v", info.SubCnt) } else { checkCommon(&info, &srv, pub1, ajwt1) } info = AccountInfo{} srv = ServerInfo{} if resp, err := ncSys.Request(info2, nil, time.Second); err != nil { t.Fatalf("Error on request: %v", err) } else if err := json.Unmarshal(resp.Data, &msg); err != nil { t.Fatalf("Unmarshalling failed: %v", err) } else if len(info.Exports) != 0 { t.Fatalf("Unexpected value: %v", info.Exports) } else if len(info.Imports) != 5 { t.Fatalf("Unexpected value: %+v", info.Imports) } // Here we need to find our import var si *ExtImport for _, im := range info.Imports { if im.Subject == "req.1" { si = &im break } } if si == nil { t.Fatalf("Could not find our import") } if si.Type != jwt.Service { t.Fatalf("Unexpected value: %+v", si) } else if si.Account != pub1 { t.Fatalf("Unexpected value: %+v", si) } else if info.SubCnt != 5 { t.Fatalf("Unexpected value: %+v", si) } else { checkCommon(&info, &srv, pub2, ajwt2) } } func TestAccountClaimsUpdatesWithServiceImports(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, sakp := createAccount(s) s.setSystemAccount(sacc) okp, _ := nkeys.FromSeed(oSeed) // Let's create an account with service export. akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Exports.Add(&jwt.Export{Subject: "req.*", Type: jwt.Service}) ajwt, _ := nac.Encode(okp) addAccountToMemResolver(s, pub, ajwt) s.LookupAccount(pub) // Now add an account with multiple service imports. akp2, _ := nkeys.CreateAccount() pub2, _ := akp2.PublicKey() nac2 := jwt.NewAccountClaims(pub2) nac2.Imports.Add(&jwt.Import{Account: pub, Subject: "req.1", Type: jwt.Service}) ajwt2, _ := nac2.Encode(okp) addAccountToMemResolver(s, pub2, ajwt2) s.LookupAccount(pub2) startSubs := s.NumSubscriptions() // Simulate a systems publisher so we can do an account claims update. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, sakp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Update the account several times for i := 1; i <= 10; i++ { nac2 = jwt.NewAccountClaims(pub2) nac2.Limits.Conn = int64(i) nac2.Imports.Add(&jwt.Import{Account: pub, Subject: "req.1", Type: jwt.Service}) ajwt2, _ = nac2.Encode(okp) // Publish to the system update subject. claimUpdateSubj := fmt.Sprintf(accUpdateEventSubjNew, pub2) nc.Publish(claimUpdateSubj, []byte(ajwt2)) } nc.Flush() time.Sleep(50 * time.Millisecond) if startSubs < s.NumSubscriptions() { t.Fatalf("Subscriptions leaked: %d vs %d", startSubs, s.NumSubscriptions()) } } func TestAccountConnsLimitExceededAfterUpdate(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, _ := createAccount(s) s.setSystemAccount(sacc) // Let's create a normal account with limits we can update. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 10 ajwt, _ := nac.Encode(okp) addAccountToMemResolver(s, pub, ajwt) acc, _ := s.LookupAccount(pub) // Now create the max connections. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) for { nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { break } defer nc.Close() } // We should have max here. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { if total := s.NumClients(); total != acc.MaxActiveConnections() { return fmt.Errorf("Expected %d connections, got %d", acc.MaxActiveConnections(), total) } return nil }) // Now change limits to make current connections over the limit. nac = jwt.NewAccountClaims(pub) nac.Limits.Conn = 2 ajwt, _ = nac.Encode(okp) s.updateAccountWithClaimJWT(acc, ajwt) if acc.MaxActiveConnections() != 2 { t.Fatalf("Expected max connections to be set to 2, got %d", acc.MaxActiveConnections()) } // We should have closed the excess connections. checkClientsCount(t, s, acc.MaxActiveConnections()) } func TestAccountConnsLimitExceededAfterUpdateDisconnectNewOnly(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() sacc, _ := createAccount(s) s.setSystemAccount(sacc) // Let's create a normal account with limits we can update. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 10 ajwt, _ := nac.Encode(okp) addAccountToMemResolver(s, pub, ajwt) acc, _ := s.LookupAccount(pub) // Now create the max connections. // We create half then we will wait and then create the rest. // Will test that we disconnect the newest ones. newConns := make([]*nats.Conn, 0, 5) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) for i := 0; i < 5; i++ { nc, err := nats.Connect(url, nats.NoReconnect(), createUserCreds(t, s, akp)) require_NoError(t, err) defer nc.Close() } time.Sleep(500 * time.Millisecond) for i := 0; i < 5; i++ { nc, err := nats.Connect(url, nats.NoReconnect(), createUserCreds(t, s, akp)) require_NoError(t, err) defer nc.Close() newConns = append(newConns, nc) } // We should have max here. checkClientsCount(t, s, acc.MaxActiveConnections()) // Now change limits to make current connections over the limit. nac = jwt.NewAccountClaims(pub) nac.Limits.Conn = 5 ajwt, _ = nac.Encode(okp) s.updateAccountWithClaimJWT(acc, ajwt) if acc.MaxActiveConnections() != 5 { t.Fatalf("Expected max connections to be set to 2, got %d", acc.MaxActiveConnections()) } // We should have closed the excess connections. checkClientsCount(t, s, acc.MaxActiveConnections()) // Now make sure that only the new ones were closed. var closed int for _, nc := range newConns { if !nc.IsClosed() { closed++ } } if closed != 5 { t.Fatalf("Expected all new clients to be closed, only got %d of 5", closed) } } func TestSystemAccountWithBadRemoteLatencyUpdate(t *testing.T) { s, _ := runTrustedServer(t) defer s.Shutdown() acc, _ := createAccount(s) s.setSystemAccount(acc) rl := remoteLatency{ Account: "NONSENSE", ReqId: "_INBOX.22", } b, _ := json.Marshal(&rl) s.remoteLatencyUpdate(nil, nil, nil, "foo", _EMPTY_, nil, b) } func TestSystemAccountWithGateways(t *testing.T) { sa, oa, sb, ob, akp := runTrustedGateways(t) defer sa.Shutdown() defer sb.Shutdown() // Create a client on A that will subscribe on $SYS.ACCOUNT.> urla := fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port) nca := natsConnect(t, urla, createUserCreds(t, sa, akp), nats.Name("SYS")) defer nca.Close() nca.Flush() sub, _ := nca.SubscribeSync("$SYS.ACCOUNT.>") defer sub.Unsubscribe() nca.Flush() // If this tests fails with wrong number after 10 seconds we may have // added a new initial subscription for the eventing system. checkExpectedSubs(t, 62, sa) // Create a client on B and see if we receive the event urlb := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) ncb := natsConnect(t, urlb, createUserCreds(t, sb, akp), nats.Name("TEST EVENTS")) defer ncb.Close() // space for .CONNECT and .CONNS from SYS and $G as well as one extra message msgs := [4]*nats.Msg{} var err error msgs[0], err = sub.NextMsg(time.Second) require_NoError(t, err) msgs[1], err = sub.NextMsg(time.Second) require_NoError(t, err) // TODO: There is a race currently that can cause the server to process the // system event *after* the subscription on "A" has been registered, and so // the "nca" client would receive its own CONNECT message. msgs[2], _ = sub.NextMsg(250 * time.Millisecond) findMsgs := func(sub string) []*nats.Msg { rMsgs := []*nats.Msg{} for _, m := range msgs { if m == nil { continue } if m.Subject == sub { rMsgs = append(rMsgs, m) } } return rMsgs } msg := findMsgs(fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", sa.SystemAccount().Name)) var bMsg *nats.Msg if len(msg) < 1 { t.Fatal("Expected at least one message") } bMsg = msg[len(msg)-1] require_Contains(t, string(bMsg.Data), sb.ID()) require_Contains(t, string(bMsg.Data), `"cluster":"B"`) require_Contains(t, string(bMsg.Data), `"name":"TEST EVENTS"`) connsMsgA := findMsgs(fmt.Sprintf("$SYS.ACCOUNT.%s.SERVER.CONNS", sa.SystemAccount().Name)) if len(connsMsgA) != 1 { t.Fatal("Expected a message") } } func TestSystemAccountNoAuthUser(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { $SYS { users [{user: "admin", password: "pwd"}] } } `)) defer os.Remove(conf) s, o := RunServerWithConfig(conf) defer s.Shutdown() for _, test := range []struct { name string usrInfo string ok bool account string }{ {"valid user/pwd", "admin:pwd@", true, "$SYS"}, {"invalid pwd", "admin:wrong@", false, _EMPTY_}, {"some token", "sometoken@", false, _EMPTY_}, {"user used without pwd", "admin@", false, _EMPTY_}, // will be treated as a token {"user with empty password", "admin:@", false, _EMPTY_}, {"no user means global account", _EMPTY_, true, globalAccountName}, } { t.Run(test.name, func(t *testing.T) { url := fmt.Sprintf("nats://%s127.0.0.1:%d", test.usrInfo, o.Port) nc, err := nats.Connect(url) if err != nil { if test.ok { t.Fatalf("Unexpected error: %v", err) } return } else if !test.ok { nc.Close() t.Fatalf("Should have failed, did not") } var accName string s.mu.Lock() for _, c := range s.clients { c.mu.Lock() if c.acc != nil { accName = c.acc.Name } c.mu.Unlock() break } s.mu.Unlock() nc.Close() checkClientsCount(t, s, 0) if accName != test.account { t.Fatalf("The account should have been %q, got %q", test.account, accName) } }) } } func TestServerAccountConns(t *testing.T) { // speed up hb orgHBInterval := eventsHBInterval eventsHBInterval = time.Millisecond * 100 defer func() { eventsHBInterval = orgHBInterval }() conf := createConfFile(t, []byte(` host: 127.0.0.1 port: -1 system_account: SYS accounts: { SYS: {users: [{user: s, password: s}]} ACC: {users: [{user: a, password: a}]} }`)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL(), nats.UserInfo("a", "a")) defer nc.Close() subOut, err := nc.SubscribeSync("foo") require_NoError(t, err) hw := "HELLO WORLD" nc.Publish("foo", []byte(hw)) nc.Publish("bar", []byte(hw)) // will only count towards received nc.Flush() m, err := subOut.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, string(m.Data), hw) ncs := natsConnect(t, s.ClientURL(), nats.UserInfo("s", "s")) defer ncs.Close() subs, err := ncs.SubscribeSync("$SYS.ACCOUNT.ACC.SERVER.CONNS") require_NoError(t, err) m, err = subs.NextMsg(time.Second) require_NoError(t, err) accConns := &AccountNumConns{} err = json.Unmarshal(m.Data, accConns) require_NoError(t, err) require_True(t, accConns.Received.Msgs == 2) require_True(t, accConns.Received.Bytes == 2*int64(len(hw))) require_True(t, accConns.Sent.Msgs == 1) require_True(t, accConns.Sent.Bytes == int64(len(hw))) } func TestServerEventsStatsZ(t *testing.T) { serverStatsReqSubj := "$SYS.REQ.SERVER.%s.STATSZ" preStart := time.Now().UTC() // Add little bit of delay to make sure that time check // between pre-start and actual start does not fail. time.Sleep(5 * time.Millisecond) sa, optsA, sb, _, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // Same between actual start and post start. time.Sleep(5 * time.Millisecond) postStart := time.Now().UTC() url := fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port) ncs, err := nats.Connect(url, createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() subj := fmt.Sprintf(serverStatsSubj, sa.ID()) sub, _ := ncs.SubscribeSync(subj) defer sub.Unsubscribe() ncs.Publish("foo", []byte("HELLO WORLD")) ncs.Flush() // Let's speed up the checking process. sa.mu.Lock() sa.sys.statsz = 10 * time.Millisecond sa.sys.stmr.Reset(sa.sys.statsz) sa.mu.Unlock() _, err = sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } // Get it the second time so we can check some stats msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } m := ServerStatsMsg{} if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if m.Server.ID != sa.ID() { t.Fatalf("Did not match IDs") } if m.Server.Cluster != "TEST CLUSTER 22" { t.Fatalf("Did not match cluster name") } if m.Server.Version != VERSION { t.Fatalf("Did not match server version") } if !m.Stats.Start.After(preStart) && m.Stats.Start.Before(postStart) { t.Fatalf("Got a wrong start time for the server %v", m.Stats.Start) } if m.Stats.Connections != 1 { t.Fatalf("Did not match connections of 1, got %d", m.Stats.Connections) } if m.Stats.ActiveAccounts != 1 { t.Fatalf("Did not match active accounts of 1, got %d", m.Stats.ActiveAccounts) } if m.Stats.Sent.Msgs < 1 { t.Fatalf("Did not match sent msgs of >=1, got %d", m.Stats.Sent.Msgs) } if m.Stats.Received.Msgs < 1 { t.Fatalf("Did not match received msgs of >=1, got %d", m.Stats.Received.Msgs) } // Default pool size + 1 for system account expectedRoutes := DEFAULT_ROUTE_POOL_SIZE + 1 if lr := len(m.Stats.Routes); lr != expectedRoutes { t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) } // Now let's prompt this server to send us the statsz subj = fmt.Sprintf(serverStatsReqSubj, sa.ID()) msg, err = ncs.Request(subj, nil, time.Second) if err != nil { t.Fatalf("Error trying to request statsz: %v", err) } m2 := ServerStatsMsg{} if err := json.Unmarshal(msg.Data, &m2); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if m2.Server.ID != sa.ID() { t.Fatalf("Did not match IDs") } if m2.Stats.Connections != 1 { t.Fatalf("Did not match connections of 1, got %d", m2.Stats.Connections) } if m2.Stats.ActiveAccounts != 1 { t.Fatalf("Did not match active accounts of 1, got %d", m2.Stats.ActiveAccounts) } if m2.Stats.Sent.Msgs < 3 { t.Fatalf("Did not match sent msgs of >= 3, got %d", m2.Stats.Sent.Msgs) } if m2.Stats.Received.Msgs < 1 { t.Fatalf("Did not match received msgs of >= 1, got %d", m2.Stats.Received.Msgs) } if lr := len(m2.Stats.Routes); lr != expectedRoutes { t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) } msg, err = ncs.Request(subj, nil, time.Second) if err != nil { t.Fatalf("Error trying to request statsz: %v", err) } m3 := ServerStatsMsg{} if err := json.Unmarshal(msg.Data, &m3); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if m3.Server.ID != sa.ID() { t.Fatalf("Did not match IDs") } if m3.Stats.Connections != 1 { t.Fatalf("Did not match connections of 1, got %d", m3.Stats.Connections) } if m3.Stats.ActiveAccounts != 1 { t.Fatalf("Did not match active accounts of 1, got %d", m3.Stats.ActiveAccounts) } if m3.Stats.Sent.Msgs < 4 { t.Fatalf("Did not match sent msgs of >= 4, got %d", m3.Stats.Sent.Msgs) } if m3.Stats.Received.Msgs < 2 { t.Fatalf("Did not match received msgs of >= 2, got %d", m3.Stats.Received.Msgs) } if lr := len(m3.Stats.Routes); lr != expectedRoutes { t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) } for _, sr := range m3.Stats.Routes { if sr.Name != "B_SRV" { t.Fatalf("Expected server A's route to B to have Name set to %q, got %q", "B", sr.Name) } } // Now query B and check that route's name is "A" subj = fmt.Sprintf(serverStatsReqSubj, sb.ID()) ncs.SubscribeSync(subj) msg, err = ncs.Request(subj, nil, time.Second) if err != nil { t.Fatalf("Error trying to request statsz: %v", err) } m = ServerStatsMsg{} if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if lr := len(m.Stats.Routes); lr != expectedRoutes { t.Fatalf("Expected %d routes, but got %d", expectedRoutes, lr) } for _, sr := range m.Stats.Routes { if sr.Name != "A_SRV" { t.Fatalf("Expected server B's route to A to have Name set to %q, got %q", "A_SRV", sr.Name) } } } func TestServerEventsHealthZSingleServer(t *testing.T) { type healthzResp struct { Healthz HealthStatus `json:"data"` Server ServerInfo `json:"server"` } cfg := fmt.Sprintf(`listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} no_auth_user: one accounts { ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }`, t.TempDir()) serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" s, _ := RunServerWithConfig(createConfFile(t, []byte(cfg))) defer s.Shutdown() ncs, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Error connecting to cluster: %v", err) } defer ncs.Close() ncAcc, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncAcc.Close() js, err := ncAcc.JetStream() if err != nil { t.Fatalf("Error creating JetStream context: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "test", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } _, err = js.AddConsumer("test", &nats.ConsumerConfig{ Name: "cons", }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } subj := fmt.Sprintf(serverHealthzReqSubj, s.ID()) tests := []struct { name string req *HealthzEventOptions expected HealthStatus }{ { name: "no parameters", expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with js enabled only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ JSEnabledOnly: true, }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with server only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ JSServerOnly: true, }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name and stream", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name, stream and consumer", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "cons", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with stream only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Stream: "test", }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Error: `"account" must not be empty when checking stream health`, }, }, { name: "with stream only, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Details: true, Stream: "test", }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Errors: []HealthzError{ { Type: HealthzErrorBadRequest, Error: `"account" must not be empty when checking stream health`, }, }, }, }, { name: "with account and consumer", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Consumer: "cons", }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Error: `"stream" must not be empty when checking consumer health`, }, }, { name: "with account and consumer, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Consumer: "cons", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Errors: []HealthzError{ { Type: HealthzErrorBadRequest, Error: `"stream" must not be empty when checking consumer health`, }, }, }, }, { name: "account not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream account "abc" not found`, }, }, { name: "account not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorAccount, Account: "abc", Error: `JetStream account "abc" not found`, }, }, }, }, { name: "stream not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream stream "abc" not found on account "ONE"`, }, }, { name: "stream not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorStream, Account: "ONE", Stream: "abc", Error: `JetStream stream "abc" not found on account "ONE"`, }, }, }, }, { name: "consumer not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, }, }, { name: "consumer not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorConsumer, Account: "ONE", Stream: "test", Consumer: "abc", Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var body []byte var err error if test.req != nil { body, err = json.Marshal(test.req) if err != nil { t.Fatalf("Error marshaling request body: %v", err) } } msg, err := ncs.Request(subj, body, 1*time.Second) if err != nil { t.Fatalf("Error trying to request healthz: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if !reflect.DeepEqual(health.Healthz, test.expected) { t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) } }) } } func TestServerEventsHealthZClustered(t *testing.T) { type healthzResp struct { Healthz HealthStatus `json:"data"` Server ServerInfo `json:"server"` } serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "JSC", 3) defer c.shutdown() ncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Error connecting to cluster: %v", err) } defer ncs.Close() ncAcc, err := nats.Connect(c.randomServer().ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncAcc.Close() js, err := ncAcc.JetStream() if err != nil { t.Fatalf("Error creating JetStream context: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "test", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } _, err = js.AddConsumer("test", &nats.ConsumerConfig{ Name: "cons", Replicas: 3, }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } subj := fmt.Sprintf(serverHealthzReqSubj, c.servers[0].ID()) pingSubj := fmt.Sprintf(serverHealthzReqSubj, "PING") tests := []struct { name string req *HealthzEventOptions expected HealthStatus expectedError string }{ { name: "no parameters", expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with js enabled only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ JSEnabledOnly: true, }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with server only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ JSServerOnly: true, }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name and stream", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with account name, stream and consumer", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "cons", }, }, expected: HealthStatus{Status: "ok", StatusCode: 200}, }, { name: "with stream only", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Stream: "test", }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Error: `"account" must not be empty when checking stream health`, }, expectedError: "Bad request:", }, { name: "with stream only, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Details: true, Stream: "test", }, }, expected: HealthStatus{ Status: "error", StatusCode: 400, Errors: []HealthzError{ { Type: HealthzErrorBadRequest, Error: `"account" must not be empty when checking stream health`, }, }, }, }, { name: "account not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream account "abc" not found`, }, expectedError: `account "abc" not found`, }, { name: "account not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorAccount, Account: "abc", Error: `JetStream account "abc" not found`, }, }, }, }, { name: "stream not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream stream "abc" not found on account "ONE"`, }, expectedError: `stream "abc" not found`, }, { name: "stream not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorStream, Account: "ONE", Stream: "abc", Error: `JetStream stream "abc" not found on account "ONE"`, }, }, }, }, { name: "consumer not found", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "abc", }, }, expected: HealthStatus{ Status: "unavailable", StatusCode: 404, Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, }, expectedError: `consumer "abc" not found for stream "test"`, }, { name: "consumer not found, detailed", req: &HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", Consumer: "abc", Details: true, }, }, expected: HealthStatus{ Status: "error", StatusCode: 404, Errors: []HealthzError{ { Type: HealthzErrorConsumer, Account: "ONE", Stream: "test", Consumer: "abc", Error: `JetStream consumer "abc" not found for stream "test" on account "ONE"`, }, }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var body []byte var err error if test.req != nil { body, err = json.Marshal(test.req) if err != nil { t.Fatalf("Error marshaling request body: %v", err) } } msg, err := ncs.Request(subj, body, 1*time.Second) if err != nil { t.Fatalf("Error trying to request healthz: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if !reflect.DeepEqual(health.Healthz, test.expected) { t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) } reply := ncs.NewRespInbox() sub, err := ncs.SubscribeSync(reply) if err != nil { t.Fatalf("Error creating subscription: %v", err) } defer sub.Unsubscribe() // now PING all servers if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { t.Fatalf("Publish error: %v", err) } for i := 0; i < 3; i++ { msg, err := sub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error fetching healthz PING response: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if !reflect.DeepEqual(health.Healthz, test.expected) { t.Errorf("Invalid healthz status; want: %+v; got: %+v", test.expected, health.Healthz) } } if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { t.Fatalf("Expected timeout error; got: %v", err) } }) } } func TestServerEventsHealthZClustered_NoReplicas(t *testing.T) { type healthzResp struct { Healthz HealthStatus `json:"data"` Server ServerInfo `json:"server"` } serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "JSC", 3) defer c.shutdown() ncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Error connecting to cluster: %v", err) } defer ncs.Close() ncAcc, err := nats.Connect(c.randomServer().ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncAcc.Close() js, err := ncAcc.JetStream() if err != nil { t.Fatalf("Error creating JetStream context: %v", err) } pingSubj := fmt.Sprintf(serverHealthzReqSubj, "PING") t.Run("non-replicated stream", func(t *testing.T) { _, err = js.AddStream(&nats.StreamConfig{ Name: "test", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } _, err = js.AddConsumer("test", &nats.ConsumerConfig{ Name: "cons", }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } body, err := json.Marshal(HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test", }, }) if err != nil { t.Fatalf("Error marshaling request body: %v", err) } reply := ncs.NewRespInbox() sub, err := ncs.SubscribeSync(reply) if err != nil { t.Fatalf("Error creating subscription: %v", err) } defer sub.Unsubscribe() // now PING all servers if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { t.Fatalf("Publish error: %v", err) } var healthy int for i := 0; i < 3; i++ { msg, err := sub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error fetching healthz PING response: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if health.Healthz.Status == "ok" { healthy++ continue } if !strings.Contains(health.Healthz.Error, `stream "test" not found`) { t.Errorf("Expected error to contain: %q, got: %s", `stream "test" not found`, health.Healthz.Error) } } if healthy != 1 { t.Fatalf("Expected 1 healthy server; got: %d", healthy) } if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { t.Fatalf("Expected timeout error; got: %v", err) } }) t.Run("non-replicated consumer", func(t *testing.T) { _, err = js.AddStream(&nats.StreamConfig{ Name: "test-repl", Subjects: []string{"bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } _, err = js.AddConsumer("test-repl", &nats.ConsumerConfig{ Name: "cons-single", }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } body, err := json.Marshal(HealthzEventOptions{ HealthzOptions: HealthzOptions{ Account: "ONE", Stream: "test-repl", Consumer: "cons-single", }, }) if err != nil { t.Fatalf("Error marshaling request body: %v", err) } reply := ncs.NewRespInbox() sub, err := ncs.SubscribeSync(reply) if err != nil { t.Fatalf("Error creating subscription: %v", err) } defer sub.Unsubscribe() // now PING all servers if err := ncs.PublishRequest(pingSubj, reply, body); err != nil { t.Fatalf("Publish error: %v", err) } var healthy int for i := 0; i < 3; i++ { msg, err := sub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error fetching healthz PING response: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if health.Healthz.Status == "ok" { healthy++ continue } if !strings.Contains(health.Healthz.Error, `consumer "cons-single" not found`) { t.Errorf("Expected error to contain: %q, got: %s", `consumer "cons-single" not found`, health.Healthz.Error) } } if healthy != 1 { t.Fatalf("Expected 1 healthy server; got: %d", healthy) } if _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) { t.Fatalf("Expected timeout error; got: %v", err) } }) } func TestServerEventsHealthZJetStreamNotEnabled(t *testing.T) { type healthzResp struct { Healthz HealthStatus `json:"data"` Server ServerInfo `json:"server"` } cfg := `listen: 127.0.0.1:-1 accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` serverHealthzReqSubj := "$SYS.REQ.SERVER.%s.HEALTHZ" s, _ := RunServerWithConfig(createConfFile(t, []byte(cfg))) defer s.Shutdown() ncs, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Error connecting to cluster: %v", err) } defer ncs.Close() subj := fmt.Sprintf(serverHealthzReqSubj, s.ID()) msg, err := ncs.Request(subj, nil, 1*time.Second) if err != nil { t.Fatalf("Error trying to request healthz: %v", err) } var health healthzResp if err := json.Unmarshal(msg.Data, &health); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } if health.Healthz.Status != "ok" { t.Errorf("Invalid healthz status; want: %q; got: %q", "ok", health.Healthz.Status) } if health.Healthz.Error != "" { t.Errorf("HealthZ error: %s", health.Healthz.Error) } } func TestServerEventsPingStatsZ(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() test := func(req []byte) { reply := nc.NewRespInbox() sub, _ := nc.SubscribeSync(reply) nc.PublishRequest(serverStatsPingReqSubj, reply, req) // Make sure its a statsz m := ServerStatsMsg{} // Receive both manually. msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } msg, err = sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatalf("Error unmarshalling the statz json: %v", err) } } strRequestTbl := []string{ `{"cluster":"TEST"}`, `{"cluster":"CLUSTER"}`, `{"server_name":"SRV"}`, `{"server_name":"_"}`, fmt.Sprintf(`{"host":"%s"}`, optsB.Host), fmt.Sprintf(`{"host":"%s", "cluster":"CLUSTER", "name":"SRV"}`, optsB.Host), } for i, opt := range strRequestTbl { t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { test([]byte(opt)) }) } requestTbl := []StatszEventOptions{ {EventFilterOptions: EventFilterOptions{Cluster: "TEST"}}, {EventFilterOptions: EventFilterOptions{Cluster: "CLUSTER"}}, {EventFilterOptions: EventFilterOptions{Name: "SRV"}}, {EventFilterOptions: EventFilterOptions{Name: "_"}}, {EventFilterOptions: EventFilterOptions{Host: optsB.Host}}, {EventFilterOptions: EventFilterOptions{Host: optsB.Host, Cluster: "CLUSTER", Name: "SRV"}}, } for i, opt := range requestTbl { t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { msg, _ := json.MarshalIndent(&opt, "", " ") test(msg) }) } } func TestServerEventsPingStatsZDedicatedRecvQ(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) require_NoError(t, err) defer nc.Close() // We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS // event to be pushed in the mux'ed queue. time.Sleep(300 * time.Millisecond) testReq := func(t *testing.T, subj string, expectTwo bool) { for _, s := range []*Server{sa, sb} { s.mu.RLock() recvq := s.sys.recvq s.mu.RUnlock() recvq.Lock() defer recvq.Unlock() } reply := nc.NewRespInbox() sub := natsSubSync(t, nc, reply) nc.PublishRequest(subj, reply, nil) msg := natsNexMsg(t, sub, time.Second) if len(msg.Data) == 0 { t.Fatal("Unexpected empty response") } // Make sure its a statsz m := ServerStatsMsg{} err := json.Unmarshal(msg.Data, &m) require_NoError(t, err) require_False(t, m.Stats.Start.IsZero()) if expectTwo { msg = natsNexMsg(t, sub, time.Second) err = json.Unmarshal(msg.Data, &m) require_NoError(t, err) require_False(t, m.Stats.Start.IsZero()) } } const statsz = "STATSZ" for _, test := range []struct { name string f func() string expectTwo bool }{ {"server stats ping request subject", func() string { return serverStatsPingReqSubj }, true}, {"server ping request subject", func() string { return fmt.Sprintf(serverPingReqSubj, statsz) }, true}, {"server a direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sa.ID(), statsz) }, false}, {"server b direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), statsz) }, false}, } { t.Run(test.name, func(t *testing.T) { testReq(t, test.f(), test.expectTwo) }) } } func TestServerEventsPingStatsZFilter(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() requestTbl := []string{ `{"cluster":"DOESNOTEXIST"}`, `{"host":"DOESNOTEXIST"}`, `{"server_name":"DOESNOTEXIST"}`, } for i, msg := range requestTbl { t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { // Receive both manually. if _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout { t.Fatalf("Error, expected timeout: %v", err) } }) } requestObjTbl := []EventFilterOptions{ {Cluster: "DOESNOTEXIST"}, {Host: "DOESNOTEXIST"}, {Name: "DOESNOTEXIST"}, } for i, opt := range requestObjTbl { t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { msg, _ := json.MarshalIndent(&opt, "", " ") // Receive both manually. if _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout { t.Fatalf("Error, expected timeout: %v", err) } }) } } func TestServerEventsPingStatsZFailFilter(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Receive both manually. if msg, err := nc.Request(serverStatsPingReqSubj, []byte(`{MALFORMEDJSON`), time.Second/4); err != nil { t.Fatalf("Error: %v", err) } else { resp := make(map[string]map[string]any) if err := json.Unmarshal(msg.Data, &resp); err != nil { t.Fatalf("Error unmarshalling the response json: %v", err) } if resp["error"]["code"].(float64) != http.StatusBadRequest { t.Fatal("bad error code") } } } func TestServerEventsPingMonitorz(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() sysAcc, _ := akp.PublicKey() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.Flush() tests := []struct { endpoint string opt any resp any respField []string }{ {"VARZ", nil, &Varz{}, []string{"now", "cpu", "system_account"}}, {"SUBSZ", nil, &Subsz{}, []string{"num_subscriptions", "num_cache"}}, {"CONNZ", nil, &Connz{}, []string{"now", "connections"}}, {"ROUTEZ", nil, &Routez{}, []string{"now", "routes"}}, {"GATEWAYZ", nil, &Gatewayz{}, []string{"now", "outbound_gateways", "inbound_gateways"}}, {"LEAFZ", nil, &Leafz{}, []string{"now", "leafs"}}, {"SUBSZ", &SubszOptions{}, &Subsz{}, []string{"num_subscriptions", "num_cache"}}, {"CONNZ", &ConnzOptions{}, &Connz{}, []string{"now", "connections"}}, {"ROUTEZ", &RoutezOptions{}, &Routez{}, []string{"now", "routes"}}, {"GATEWAYZ", &GatewayzOptions{}, &Gatewayz{}, []string{"now", "outbound_gateways", "inbound_gateways"}}, {"LEAFZ", &LeafzOptions{}, &Leafz{}, []string{"now", "leafs"}}, {"ACCOUNTZ", &AccountzOptions{}, &Accountz{}, []string{"now", "accounts"}}, {"SUBSZ", &SubszOptions{Limit: 5}, &Subsz{}, []string{"num_subscriptions", "num_cache"}}, {"CONNZ", &ConnzOptions{Limit: 5}, &Connz{}, []string{"now", "connections"}}, {"ROUTEZ", &RoutezOptions{SubscriptionsDetail: true}, &Routez{}, []string{"now", "routes"}}, {"GATEWAYZ", &GatewayzOptions{Accounts: true}, &Gatewayz{}, []string{"now", "outbound_gateways", "inbound_gateways"}}, {"LEAFZ", &LeafzOptions{Subscriptions: true}, &Leafz{}, []string{"now", "leafs"}}, {"ACCOUNTZ", &AccountzOptions{Account: sysAcc}, &Accountz{}, []string{"now", "account_detail"}}, {"LEAFZ", &LeafzOptions{Account: sysAcc}, &Leafz{}, []string{"now", "leafs"}}, {"ROUTEZ", json.RawMessage(`{"cluster":""}`), &Routez{}, []string{"now", "routes"}}, {"ROUTEZ", json.RawMessage(`{"name":""}`), &Routez{}, []string{"now", "routes"}}, {"ROUTEZ", json.RawMessage(`{"cluster":"TEST CLUSTER 22"}`), &Routez{}, []string{"now", "routes"}}, {"ROUTEZ", json.RawMessage(`{"cluster":"CLUSTER"}`), &Routez{}, []string{"now", "routes"}}, {"ROUTEZ", json.RawMessage(`{"cluster":"TEST CLUSTER 22", "subscriptions":true}`), &Routez{}, []string{"now", "routes"}}, {"JSZ", nil, &JSzOptions{}, []string{"now", "disabled"}}, {"HEALTHZ", nil, &JSzOptions{}, []string{"status"}}, {"HEALTHZ", &HealthzOptions{JSEnabledOnly: true}, &JSzOptions{}, []string{"status"}}, {"HEALTHZ", &HealthzOptions{JSServerOnly: true}, &JSzOptions{}, []string{"status"}}, {"EXPVARZ", nil, &ExpvarzStatus{}, []string{"memstats", "cmdline"}}, } for i, test := range tests { t.Run(fmt.Sprintf("%s-%d", test.endpoint, i), func(t *testing.T) { var opt []byte if test.opt != nil { opt, err = json.Marshal(test.opt) if err != nil { t.Fatalf("Error marshaling opts: %v", err) } } reply := nc.NewRespInbox() replySubj, _ := nc.SubscribeSync(reply) // set a header to make sure request parsing knows to ignore them nc.PublishMsg(&nats.Msg{ Subject: fmt.Sprintf("%s.%s", serverStatsPingReqSubj, test.endpoint), Reply: reply, Header: nats.Header{"header": []string{"for header sake"}}, Data: opt, }) // Receive both manually. msg, err := replySubj.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } response1 := make(map[string]map[string]any) if err := json.Unmarshal(msg.Data, &response1); err != nil { t.Fatalf("Error unmarshalling response1 json: %v", err) } serverName := "" if response1["server"]["name"] == "A_SRV" { serverName = "B_SRV" } else if response1["server"]["name"] == "B_SRV" { serverName = "A_SRV" } else { t.Fatalf("Error finding server in %s", string(msg.Data)) } if resp, ok := response1["data"]; !ok { t.Fatalf("Error finding: %s in %s", strings.ToLower(test.endpoint), string(msg.Data)) } else { for _, respField := range test.respField { if _, ok := resp[respField]; !ok { t.Fatalf("Error finding: %s in %s", respField, resp) } } } msg, err = replySubj.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } response2 := make(map[string]map[string]any) if err := json.Unmarshal(msg.Data, &response2); err != nil { t.Fatalf("Error unmarshalling the response2 json: %v", err) } if response2["server"]["name"] != serverName { t.Fatalf("Error finding server %s in %s", serverName, string(msg.Data)) } if resp, ok := response2["data"]; !ok { t.Fatalf("Error finding: %s in %s", strings.ToLower(test.endpoint), string(msg.Data)) } else { for _, respField := range test.respField { if val, ok := resp[respField]; !ok { t.Fatalf("Error finding: %s in %s", respField, resp) } else if val == nil { t.Fatalf("Nil value found: %s in %s", respField, resp) } } } }) } } func TestGatewayNameClientInfo(t *testing.T) { sa, _, sb, _, _ := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() c, _, l := newClientForServer(sa) defer c.close() var info Info err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Cluster != "TEST CLUSTER 22" { t.Fatalf("Expected a cluster name of 'TEST CLUSTER 22', got %q", info.Cluster) } } type slowAccResolver struct { sync.Mutex AccountResolver acc string } func (sr *slowAccResolver) Fetch(name string) (string, error) { sr.Lock() delay := sr.acc == name sr.Unlock() if delay { time.Sleep(200 * time.Millisecond) } return sr.AccountResolver.Fetch(name) } func TestConnectionUpdatesTimerProperlySet(t *testing.T) { origEventsHBInterval := eventsHBInterval eventsHBInterval = 50 * time.Millisecond defer func() { eventsHBInterval = origEventsHBInterval }() sa, _, sb, optsB, _ := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // Normal Account okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) nac.Limits.Conn = 10 // set any limit... jwt, _ := nac.Encode(okp) addAccountToMemResolver(sa, pub, jwt) // Listen for HB updates... count := int32(0) cb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) { atomic.AddInt32(&count, 1) } subj := fmt.Sprintf(accConnsEventSubjNew, pub) sub, err := sa.sysSubscribe(subj, cb) if sub == nil || err != nil { t.Fatalf("Expected to subscribe, got %v", err) } defer sa.sysUnsubscribe(sub) url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc := natsConnect(t, url, createUserCreds(t, sb, akp)) defer nc.Close() time.Sleep(500 * time.Millisecond) // After waiting 500ms with HB interval of 50ms, we should get // about 10 updates, no much more if n := atomic.LoadInt32(&count); n > 15 { t.Fatalf("Expected about 10 updates, got %v", n) } // Now lookup the account doing the events on sb. acc, _ := sb.LookupAccount(pub) // Make sure we have the timer running. acc.mu.RLock() ctmr := acc.ctmr acc.mu.RUnlock() if ctmr == nil { t.Fatalf("Expected event timer for acc conns to be running") } nc.Close() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { // Make sure we have the timer is NOT running. acc.mu.RLock() ctmr = acc.ctmr acc.mu.RUnlock() if ctmr != nil { return fmt.Errorf("Expected event timer for acc conns to NOT be running after reaching zero local clients") } return nil }) } func TestServerEventsReceivedByQSubs(t *testing.T) { s, opts := runTrustedServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() // Listen for auth error events. qsub, _ := ncs.QueueSubscribeSync("$SYS.SERVER.*.CLIENT.AUTH.ERR", "queue") defer qsub.Unsubscribe() ncs.Flush() nats.Connect(url, nats.Name("TEST BAD LOGIN")) m, err := qsub.NextMsg(time.Second) if err != nil { t.Fatalf("Should have heard an auth error event") } dem := DisconnectEventMsg{} if err := json.Unmarshal(m.Data, &dem); err != nil { t.Fatalf("Error unmarshalling disconnect event message: %v", err) } if dem.Reason != "Authentication Failure" { t.Fatalf("Expected auth error, got %q", dem.Reason) } } func TestServerEventsFilteredByTag(t *testing.T) { confA := createConfFile(t, []byte(` listen: -1 server_name: srv-A server_tags: ["foo", "bar"] cluster { name: clust listen: -1 no_advertise: true } system_account: SYS accounts: { SYS: { users: [ {user: b, password: b} ] } } no_auth_user: b `)) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() confB := createConfFile(t, []byte(fmt.Sprintf(` listen: -1 server_name: srv-B server_tags: ["bar", "baz"] cluster { name: clust listen: -1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } system_account: SYS accounts: { SYS: { users: [ {user: b, password: b} ] } } no_auth_user: b `, sA.opts.Cluster.Port))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() checkClusterFormed(t, sA, sB) nc := natsConnect(t, sA.ClientURL()) defer nc.Close() ib := nats.NewInbox() req := func(tags ...string) { t.Helper() r, err := json.Marshal(VarzEventOptions{EventFilterOptions: EventFilterOptions{Tags: tags}}) require_NoError(t, err) err = nc.PublishRequest(fmt.Sprintf(serverPingReqSubj, "VARZ"), ib, r) require_NoError(t, err) } msgs := make(chan *nats.Msg, 10) defer close(msgs) _, err := nc.ChanSubscribe(ib, msgs) require_NoError(t, err) req("none") select { case <-msgs: t.Fatalf("no message expected") case <-time.After(200 * time.Millisecond): } req("foo") m := <-msgs require_Contains(t, string(m.Data), "srv-A", "foo", "bar") req("foo", "bar") m = <-msgs require_Contains(t, string(m.Data), "srv-A", "foo", "bar") req("baz") m = <-msgs require_Contains(t, string(m.Data), "srv-B", "bar", "baz") req("bar") m1 := <-msgs m2 := <-msgs require_Contains(t, string(m1.Data)+string(m2.Data), "srv-A", "srv-B", "foo", "bar", "baz") require_Len(t, len(msgs), 0) } // https://github.com/nats-io/nats-server/issues/3177 func TestServerEventsAndDQSubscribers(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, "DDQ", 3) defer c.shutdown() nc, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() sub, err := nc.QueueSubscribeSync("$SYS.ACCOUNT.*.DISCONNECT", "qq") require_NoError(t, err) nc.Flush() // Create and disconnect 10 random connections. for i := 0; i < 10; i++ { nc, err := nats.Connect(c.randomServer().ClientURL()) require_NoError(t, err) nc.Close() } checkSubsPending(t, sub, 10) } func TestServerEventsStatszSingleServer(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { $SYS { users [{user: "admin", password: "p1d"}]} } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Grab internal system client. s.mu.RLock() sysc := s.sys.client wait := s.sys.cstatsz + 25*time.Millisecond s.mu.RUnlock() // Wait for when first statsz would have gone out.. time.Sleep(wait) sysc.mu.Lock() outMsgs := sysc.stats.outMsgs sysc.mu.Unlock() require_True(t, outMsgs == 0) // Connect as a system user and make sure if there is // subscription interest that we will receive updates. nc, _ := jsClientConnect(t, s, nats.UserInfo("admin", "p1d")) defer nc.Close() sub, err := nc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) require_NoError(t, err) checkSubsPending(t, sub, 1) } func TestServerEventsReload(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts: { $SYS { users [{user: "admin", password: "p1d"}]} test { users [{user: "foo", password: "bar"}]} } ping_interval: "100ms" `)) opts := LoadConfig(conf) s := RunServer(opts) defer s.Shutdown() subject := fmt.Sprintf(serverReloadReqSubj, s.info.ID) // Connect as a test user and make sure the reload endpoint is not // accessible. ncTest, _ := jsClientConnect(t, s, nats.UserInfo("foo", "bar")) defer ncTest.Close() testReply := ncTest.NewRespInbox() sub, err := ncTest.SubscribeSync(testReply) require_NoError(t, err) err = ncTest.PublishRequest(subject, testReply, nil) require_NoError(t, err) _, err = sub.NextMsg(time.Second) require_Error(t, err) require_True(t, s.getOpts().PingInterval == 100*time.Millisecond) // Connect as a system user. nc, _ := jsClientConnect(t, s, nats.UserInfo("admin", "p1d")) defer nc.Close() // rewrite the config file with a different ping interval err = os.WriteFile(conf, []byte(` listen: "127.0.0.1:-1" accounts: { $SYS { users [{user: "admin", password: "p1d"}]} test { users [{user: "foo", password: "bar"}]} } ping_interval: "200ms" `), 0666) require_NoError(t, err) msg, err := nc.Request(subject, nil, time.Second) require_NoError(t, err) var apiResp = ServerAPIResponse{} err = json.Unmarshal(msg.Data, &apiResp) require_NoError(t, err) require_True(t, apiResp.Data == nil) require_True(t, apiResp.Error == nil) // See that the ping interval has changed. require_True(t, s.getOpts().PingInterval == 200*time.Millisecond) // rewrite the config file with a different ping interval err = os.WriteFile(conf, []byte(`garbage and nonsense`), 0666) require_NoError(t, err) // Request the server to reload and wait for the response. msg, err = nc.Request(subject, nil, time.Second) require_NoError(t, err) apiResp = ServerAPIResponse{} err = json.Unmarshal(msg.Data, &apiResp) require_NoError(t, err) require_True(t, apiResp.Data == nil) require_Error(t, apiResp.Error, fmt.Errorf("Parse error on line 1: 'Expected a top-level value to end with a new line, comment or EOF, but got 'n' instead.'")) // See that the ping interval has not changed. require_True(t, s.getOpts().PingInterval == 200*time.Millisecond) } func TestServerEventsLDMKick(t *testing.T) { ldmed := make(chan bool, 1) disconnected := make(chan bool, 1) s, opts := runTrustedServer(t) defer s.Shutdown() acc, akp := createAccount(s) s.setSystemAccount(acc) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) ncs, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncs.Close() _, akp2 := createAccount(s) nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS LDM+KICK"), nats.LameDuckModeHandler(func(_ *nats.Conn) { ldmed <- true })) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { disconnected <- true }) cid, err := nc.GetClientID() if err != nil { t.Fatalf("Error on getting the CID: %v", err) } reqldm := LDMClientReq{CID: cid} reqldmpayload, _ := json.Marshal(reqldm) reqkick := KickClientReq{CID: cid} reqkickpayload, _ := json.Marshal(reqkick) // Test for data races when getting the client by ID uc := createUserCreds(t, s, akp2) totalClients := 100 someClients := make([]*nats.Conn, 0, totalClients) for i := 0; i < totalClients; i++ { nc, err := nats.Connect(s.ClientURL(), uc) require_NoError(t, err) defer nc.Close() someClients = append(someClients, nc) } var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() for i := 0; i < totalClients; i++ { someClients[i].Close() } }() defer wg.Wait() _, err = ncs.Request(fmt.Sprintf("$SYS.REQ.SERVER.%s.LDM", s.ID()), reqldmpayload, time.Second) require_NoError(t, err) select { case <-ldmed: case <-time.After(time.Second): t.Fatalf("timeout waiting for the connection to receive the LDM signal") } _, err = ncs.Request(fmt.Sprintf("$SYS.REQ.SERVER.%s.KICK", s.ID()), reqkickpayload, time.Second) require_NoError(t, err) select { case <-disconnected: case <-time.After(time.Second): t.Fatalf("timeout waiting for the client to get disconnected") } } func Benchmark_GetHash(b *testing.B) { b.StopTimer() // Get 100 random names names := make([]string, 0, 100) for i := 0; i < 100; i++ { names = append(names, nuid.Next()) } hashes := make([]string, 0, 100) for j := 0; j < 100; j++ { sha := sha256.New() sha.Write([]byte(names[j])) b := sha.Sum(nil) for i := 0; i < 8; i++ { b[i] = digits[int(b[i]%base)] } hashes = append(hashes, string(b[:8])) } wg := sync.WaitGroup{} wg.Add(8) errCh := make(chan error, 8) b.StartTimer() for i := 0; i < 8; i++ { go func() { defer wg.Done() for i := 0; i < b.N; i++ { idx := rand.Intn(100) if h := getHash(names[idx]); h != hashes[idx] { errCh <- fmt.Errorf("Hash for name %q was %q, but should be %q", names[idx], h, hashes[idx]) return } } }() } wg.Wait() select { case err := <-errCh: b.Fatal(err.Error()) default: } } func TestClusterSetupMsgs(t *testing.T) { // Tests will set this general faster, but here we want default for production. original := statszRateLimit statszRateLimit = defaultStatszRateLimit defer func() { statszRateLimit = original }() numServers := 10 c := createClusterEx(t, false, 0, false, "cluster", numServers) defer shutdownCluster(c) var totalOut int for _, server := range c.servers { totalOut += int(atomic.LoadInt64(&server.outMsgs)) } totalExpected := numServers * numServers if totalOut >= totalExpected { t.Fatalf("Total outMsgs is %d, expected < %d\n", totalOut, totalExpected) } } func TestServerEventsProfileZNotBlockingRecvQ(t *testing.T) { sa, _, sb, optsB, akp := runTrustedCluster(t) defer sa.Shutdown() defer sb.Shutdown() // For this test, we will run a single server because the profiling // would fail for the second server since it would detect that // one profiling is already running (2 servers, but same process). sa.Shutdown() url := fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port) nc, err := nats.Connect(url, createUserCreds(t, sb, akp)) require_NoError(t, err) defer nc.Close() // We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS // event to be pushed in the mux'ed queue. time.Sleep(300 * time.Millisecond) po := ProfilezOptions{Name: "cpu", Duration: 1 * time.Second} req, err := json.Marshal(po) require_NoError(t, err) testReq := func(t *testing.T, subj string) { // Block the recvQ by locking it for the duration of this test. sb.mu.RLock() recvq := sb.sys.recvq sb.mu.RUnlock() recvq.Lock() defer recvq.Unlock() // Send the profilez request on the given subject. reply := nc.NewRespInbox() sub := natsSubSync(t, nc, reply) nc.PublishRequest(subj, reply, req) msg := natsNexMsg(t, sub, 10*time.Second) if len(msg.Data) == 0 { t.Fatal("Unexpected empty response") } // Make sure its a ServerAPIResponse resp := ServerAPIResponse{Data: &ProfilezStatus{}} err := json.Unmarshal(msg.Data, &resp) require_NoError(t, err) // Check profile status to make sure that we got something. ps := resp.Data.(*ProfilezStatus) if ps.Error != _EMPTY_ { t.Fatalf("%s", ps.Error) } require_True(t, len(ps.Profile) > 0) } const profilez = "PROFILEZ" for _, test := range []struct { name string f func() string }{ {"server profilez request subject", func() string { return fmt.Sprintf(serverPingReqSubj, profilez) }}, {"server direct request subject", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), profilez) }}, } { t.Run(test.name, func(t *testing.T) { testReq(t, test.f()) }) } } nats-server-2.10.27/server/filestore.go000066400000000000000000010146341477524627100200150ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "archive/tar" "bytes" "crypto/aes" "crypto/cipher" "crypto/rand" "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "hash" "io" "io/fs" "math" mrand "math/rand" "net" "os" "path/filepath" "runtime" "slices" "sort" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/minio/highwayhash" "github.com/nats-io/nats-server/v2/server/avl" "github.com/nats-io/nats-server/v2/server/stree" "golang.org/x/crypto/chacha20" "golang.org/x/crypto/chacha20poly1305" ) type FileStoreConfig struct { // Where the parent directory for all storage will be located. StoreDir string // BlockSize is the file block size. This also represents the maximum overhead size. BlockSize uint64 // CacheExpire is how long with no activity until we expire the cache. CacheExpire time.Duration // SubjectStateExpire is how long with no activity until we expire a msg block's subject state. SubjectStateExpire time.Duration // SyncInterval is how often we sync to disk in the background. SyncInterval time.Duration // SyncAlways is when the stream should sync all data writes. SyncAlways bool // AsyncFlush allows async flush to batch write operations. AsyncFlush bool // Cipher is the cipher to use when encrypting. Cipher StoreCipher // Compression is the algorithm to use when compressing. Compression StoreCompression // Internal reference to our server. srv *Server } // FileStreamInfo allows us to remember created time. type FileStreamInfo struct { Created time.Time StreamConfig } type StoreCipher int const ( ChaCha StoreCipher = iota AES NoCipher ) func (cipher StoreCipher) String() string { switch cipher { case ChaCha: return "ChaCha20-Poly1305" case AES: return "AES-GCM" case NoCipher: return "None" default: return "Unknown StoreCipher" } } type StoreCompression uint8 const ( NoCompression StoreCompression = iota S2Compression ) func (alg StoreCompression) String() string { switch alg { case NoCompression: return "None" case S2Compression: return "S2" default: return "Unknown StoreCompression" } } func (alg StoreCompression) MarshalJSON() ([]byte, error) { var str string switch alg { case S2Compression: str = "s2" case NoCompression: str = "none" default: return nil, fmt.Errorf("unknown compression algorithm") } return json.Marshal(str) } func (alg *StoreCompression) UnmarshalJSON(b []byte) error { var str string if err := json.Unmarshal(b, &str); err != nil { return err } switch str { case "s2": *alg = S2Compression case "none": *alg = NoCompression default: return fmt.Errorf("unknown compression algorithm") } return nil } // File ConsumerInfo is used for creating consumer stores. type FileConsumerInfo struct { Created time.Time Name string ConsumerConfig } // Default file and directory permissions. const ( defaultDirPerms = os.FileMode(0700) defaultFilePerms = os.FileMode(0600) ) type psi struct { total uint64 fblk uint32 lblk uint32 } type fileStore struct { srv *Server mu sync.RWMutex state StreamState tombs []uint64 ld *LostStreamData scb StorageUpdateHandler ageChk *time.Timer syncTmr *time.Timer cfg FileStreamInfo fcfg FileStoreConfig prf keyGen oldprf keyGen aek cipher.AEAD lmb *msgBlock blks []*msgBlock bim map[uint32]*msgBlock psim *stree.SubjectTree[psi] tsl int adml int hh hash.Hash64 qch chan struct{} fsld chan struct{} cmu sync.RWMutex cfs []ConsumerStore sips int dirty int closing bool closed bool fip bool receivedAny bool firstMoved bool } // Represents a message store block and its data. type msgBlock struct { // Here for 32bit systems and atomic. first msgId last msgId mu sync.RWMutex fs *fileStore aek cipher.AEAD bek cipher.Stream seed []byte nonce []byte mfn string mfd *os.File cmp StoreCompression // Effective compression at the time of loading the block liwsz int64 index uint32 bytes uint64 // User visible bytes count. rbytes uint64 // Total bytes (raw) including deleted. Used for rolling to new blk. msgs uint64 // User visible message count. fss *stree.SubjectTree[SimpleState] kfn string lwts int64 llts int64 lrts int64 lsts int64 llseq uint64 hh hash.Hash64 cache *cache cloads uint64 cexp time.Duration fexp time.Duration ctmr *time.Timer werr error dmap avl.SequenceSet fch chan struct{} qch chan struct{} lchk [8]byte loading bool flusher bool noTrack bool needSync bool syncAlways bool noCompact bool closed bool // Used to mock write failures. mockWriteErr bool } // Write through caching layer that is also used on loading messages. type cache struct { buf []byte off int wp int idx []uint32 lrl uint32 fseq uint64 nra bool } type msgId struct { seq uint64 ts int64 } const ( // Magic is used to identify the file store files. magic = uint8(22) // Version version = uint8(1) // New IndexInfo Version newVersion = uint8(2) // hdrLen hdrLen = 2 // This is where we keep the streams. streamsDir = "streams" // This is where we keep the message store blocks. msgDir = "msgs" // This is where we temporarily move the messages dir. purgeDir = "__msgs__" // used to scan blk file names. blkScan = "%d.blk" // used for compacted blocks that are staged. newScan = "%d.new" // used to scan index file names. indexScan = "%d.idx" // used to store our block encryption key. keyScan = "%d.key" // to look for orphans keyScanAll = "*.key" // This is where we keep state on consumers. consumerDir = "obs" // Index file for a consumer. consumerState = "o.dat" // The suffix that will be given to a new temporary block during compression. compressTmpSuffix = ".tmp" // This is where we keep state on templates. tmplsDir = "templates" // Maximum size of a write buffer we may consider for re-use. maxBufReuse = 2 * 1024 * 1024 // default cache buffer expiration defaultCacheBufferExpiration = 10 * time.Second // default sync interval defaultSyncInterval = 2 * time.Minute // default idle timeout to close FDs. closeFDsIdle = 30 * time.Second // default expiration time for mb.fss when idle. defaultFssExpiration = 2 * time.Minute // coalesceMinimum coalesceMinimum = 16 * 1024 // maxFlushWait is maximum we will wait to gather messages to flush. maxFlushWait = 8 * time.Millisecond // Metafiles for streams and consumers. JetStreamMetaFile = "meta.inf" JetStreamMetaFileSum = "meta.sum" JetStreamMetaFileKey = "meta.key" // This is the full snapshotted state for the stream. streamStreamStateFile = "index.db" // AEK key sizes minMetaKeySize = 64 minBlkKeySize = 64 // Default stream block size. defaultLargeBlockSize = 8 * 1024 * 1024 // 8MB // Default for workqueue or interest based. defaultMediumBlockSize = 4 * 1024 * 1024 // 4MB // For smaller reuse buffers. Usually being generated during contention on the lead write buffer. // E.g. mirrors/sources etc. defaultSmallBlockSize = 1 * 1024 * 1024 // 1MB // Maximum size for the encrypted head block. maximumEncryptedBlockSize = 2 * 1024 * 1024 // 2MB // Default for KV based defaultKVBlockSize = defaultMediumBlockSize // max block size for now. maxBlockSize = defaultLargeBlockSize // Compact minimum threshold. compactMinimum = 2 * 1024 * 1024 // 2MB // FileStoreMinBlkSize is minimum size we will do for a blk size. FileStoreMinBlkSize = 32 * 1000 // 32kib // FileStoreMaxBlkSize is maximum size we will do for a blk size. FileStoreMaxBlkSize = maxBlockSize // Check for bad record length value due to corrupt data. rlBadThresh = 32 * 1024 * 1024 // Checksum size for hash for msg records. recordHashSize = 8 ) func newFileStore(fcfg FileStoreConfig, cfg StreamConfig) (*fileStore, error) { return newFileStoreWithCreated(fcfg, cfg, time.Now().UTC(), nil, nil) } func newFileStoreWithCreated(fcfg FileStoreConfig, cfg StreamConfig, created time.Time, prf, oldprf keyGen) (*fileStore, error) { if cfg.Name == _EMPTY_ { return nil, fmt.Errorf("name required") } if cfg.Storage != FileStorage { return nil, fmt.Errorf("fileStore requires file storage type in config") } // Default values. if fcfg.BlockSize == 0 { fcfg.BlockSize = dynBlkSize(cfg.Retention, cfg.MaxBytes, prf != nil) } if fcfg.BlockSize > maxBlockSize { return nil, fmt.Errorf("filestore max block size is %s", friendlyBytes(maxBlockSize)) } if fcfg.CacheExpire == 0 { fcfg.CacheExpire = defaultCacheBufferExpiration } if fcfg.SubjectStateExpire == 0 { fcfg.SubjectStateExpire = defaultFssExpiration } if fcfg.SyncInterval == 0 { fcfg.SyncInterval = defaultSyncInterval } // Check the directory if stat, err := os.Stat(fcfg.StoreDir); os.IsNotExist(err) { if err := os.MkdirAll(fcfg.StoreDir, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create storage directory - %v", err) } } else if stat == nil || !stat.IsDir() { return nil, fmt.Errorf("storage directory is not a directory") } tmpfile, err := os.CreateTemp(fcfg.StoreDir, "_test_") if err != nil { return nil, fmt.Errorf("storage directory is not writable") } tmpfile.Close() <-dios os.Remove(tmpfile.Name()) dios <- struct{}{} fs := &fileStore{ fcfg: fcfg, psim: stree.NewSubjectTree[psi](), bim: make(map[uint32]*msgBlock), cfg: FileStreamInfo{Created: created, StreamConfig: cfg}, prf: prf, oldprf: oldprf, qch: make(chan struct{}), fsld: make(chan struct{}), srv: fcfg.srv, } // Set flush in place to AsyncFlush which by default is false. fs.fip = !fcfg.AsyncFlush // Check if this is a new setup. mdir := filepath.Join(fcfg.StoreDir, msgDir) odir := filepath.Join(fcfg.StoreDir, consumerDir) if err := os.MkdirAll(mdir, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create message storage directory - %v", err) } if err := os.MkdirAll(odir, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create consumer storage directory - %v", err) } // Create highway hash for message blocks. Use sha256 of directory as key. key := sha256.Sum256([]byte(cfg.Name)) fs.hh, err = highwayhash.New64(key[:]) if err != nil { return nil, fmt.Errorf("could not create hash: %v", err) } keyFile := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey) // Make sure we do not have an encrypted store underneath of us but no main key. if fs.prf == nil { if _, err := os.Stat(keyFile); err == nil { return nil, errNoMainKey } } // Attempt to recover our state. err = fs.recoverFullState() if err != nil { if !os.IsNotExist(err) { fs.warn("Recovering stream state from index errored: %v", err) } // Hold onto state prior := fs.state // Reset anything that could have been set from above. fs.state = StreamState{} fs.psim, fs.tsl = fs.psim.Empty(), 0 fs.bim = make(map[uint32]*msgBlock) fs.blks = nil fs.tombs = nil // Recover our message state the old way if err := fs.recoverMsgs(); err != nil { return nil, err } // Check if our prior state remembers a last sequence past where we can see. if fs.ld != nil && prior.LastSeq > fs.state.LastSeq { fs.state.LastSeq, fs.state.LastTime = prior.LastSeq, prior.LastTime if _, err := fs.newMsgBlockForWrite(); err == nil { if err = fs.writeTombstone(prior.LastSeq, prior.LastTime.UnixNano()); err != nil { return nil, err } } else { return nil, err } } // Since we recovered here, make sure to kick ourselves to write out our stream state. fs.dirty++ } // Also make sure we get rid of old idx and fss files on return. // Do this in separate go routine vs inline and at end of processing. defer func() { go fs.cleanupOldMeta() }() // Lock while we do enforcements and removals. fs.mu.Lock() // Check if we have any left over tombstones to process. if len(fs.tombs) > 0 { for _, seq := range fs.tombs { fs.removeMsg(seq, false, true, false) fs.removeFromLostData(seq) } // Not needed after this phase. fs.tombs = nil } // Limits checks and enforcement. fs.enforceMsgLimit() fs.enforceBytesLimit() // Do age checks too, make sure to call in place. if fs.cfg.MaxAge != 0 { err := fs.expireMsgsOnRecover() if isPermissionError(err) { return nil, err } fs.startAgeChk() } // If we have max msgs per subject make sure the is also enforced. if fs.cfg.MaxMsgsPer > 0 { fs.enforceMsgPerSubjectLimit(false) } // Grab first sequence for check below while we have lock. firstSeq := fs.state.FirstSeq fs.mu.Unlock() // If the stream has an initial sequence number then make sure we // have purged up until that point. We will do this only if the // recovered first sequence number is before our configured first // sequence. Need to do this locked as by now the age check timer // has started. if cfg.FirstSeq > 0 && firstSeq <= cfg.FirstSeq { if _, err := fs.purge(cfg.FirstSeq); err != nil { return nil, err } } // Write our meta data if it does not exist or is zero'd out. meta := filepath.Join(fcfg.StoreDir, JetStreamMetaFile) fi, err := os.Stat(meta) if err != nil && os.IsNotExist(err) || fi != nil && fi.Size() == 0 { if err := fs.writeStreamMeta(); err != nil { return nil, err } } // If we expect to be encrypted check that what we are restoring is not plaintext. // This can happen on snapshot restores or conversions. if fs.prf != nil { if _, err := os.Stat(keyFile); err != nil && os.IsNotExist(err) { if err := fs.writeStreamMeta(); err != nil { return nil, err } } } // Setup our sync timer. fs.setSyncTimer() // Spin up the go routine that will write out our full state stream index. go fs.flushStreamStateLoop(fs.qch, fs.fsld) return fs, nil } // Lock all existing message blocks. // Lock held on entry. func (fs *fileStore) lockAllMsgBlocks() { for _, mb := range fs.blks { mb.mu.Lock() } } // Unlock all existing message blocks. // Lock held on entry. func (fs *fileStore) unlockAllMsgBlocks() { for _, mb := range fs.blks { mb.mu.Unlock() } } func (fs *fileStore) UpdateConfig(cfg *StreamConfig) error { start := time.Now() defer func() { if took := time.Since(start); took > time.Minute { fs.warn("UpdateConfig took %v", took.Round(time.Millisecond)) } }() if fs.isClosed() { return ErrStoreClosed } if cfg.Name == _EMPTY_ { return fmt.Errorf("name required") } if cfg.Storage != FileStorage { return fmt.Errorf("fileStore requires file storage type in config") } if cfg.MaxMsgsPer < -1 { cfg.MaxMsgsPer = -1 } fs.mu.Lock() new_cfg := FileStreamInfo{Created: fs.cfg.Created, StreamConfig: *cfg} old_cfg := fs.cfg // The reference story has changed here, so this full msg block lock // may not be needed. fs.lockAllMsgBlocks() fs.cfg = new_cfg fs.unlockAllMsgBlocks() if err := fs.writeStreamMeta(); err != nil { fs.lockAllMsgBlocks() fs.cfg = old_cfg fs.unlockAllMsgBlocks() fs.mu.Unlock() return err } // Limits checks and enforcement. fs.enforceMsgLimit() fs.enforceBytesLimit() // Do age timers. if fs.ageChk == nil && fs.cfg.MaxAge != 0 { fs.startAgeChk() } if fs.ageChk != nil && fs.cfg.MaxAge == 0 { fs.ageChk.Stop() fs.ageChk = nil } if fs.cfg.MaxMsgsPer > 0 && (old_cfg.MaxMsgsPer == 0 || fs.cfg.MaxMsgsPer < old_cfg.MaxMsgsPer) { fs.enforceMsgPerSubjectLimit(true) } fs.mu.Unlock() if cfg.MaxAge != 0 { fs.expireMsgs() } return nil } func dynBlkSize(retention RetentionPolicy, maxBytes int64, encrypted bool) uint64 { if maxBytes > 0 { blkSize := (maxBytes / 4) + 1 // (25% overhead) // Round up to nearest 100 if m := blkSize % 100; m != 0 { blkSize += 100 - m } if blkSize <= FileStoreMinBlkSize { blkSize = FileStoreMinBlkSize } else if blkSize >= FileStoreMaxBlkSize { blkSize = FileStoreMaxBlkSize } else { blkSize = defaultMediumBlockSize } if encrypted && blkSize > maximumEncryptedBlockSize { // Notes on this below. blkSize = maximumEncryptedBlockSize } return uint64(blkSize) } switch { case encrypted: // In the case of encrypted stores, large blocks can result in worsened perf // since many writes on disk involve re-encrypting the entire block. For now, // we will enforce a cap on the block size when encryption is enabled to avoid // this. return maximumEncryptedBlockSize case retention == LimitsPolicy: // TODO(dlc) - Make the blocksize relative to this if set. return defaultLargeBlockSize default: // TODO(dlc) - Make the blocksize relative to this if set. return defaultMediumBlockSize } } func genEncryptionKey(sc StoreCipher, seed []byte) (ek cipher.AEAD, err error) { if sc == ChaCha { ek, err = chacha20poly1305.NewX(seed) } else if sc == AES { block, e := aes.NewCipher(seed) if e != nil { return nil, e } ek, err = cipher.NewGCMWithNonceSize(block, block.BlockSize()) } else { err = errUnknownCipher } return ek, err } // Generate an asset encryption key from the context and server PRF. func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek cipher.Stream, seed, encrypted []byte, err error) { if fs.prf == nil { return nil, nil, nil, nil, errNoEncryption } // Generate key encryption key. rb, err := fs.prf([]byte(context)) if err != nil { return nil, nil, nil, nil, err } sc := fs.fcfg.Cipher kek, err := genEncryptionKey(sc, rb) if err != nil { return nil, nil, nil, nil, err } // Generate random asset encryption key seed. const seedSize = 32 seed = make([]byte, seedSize) if n, err := rand.Read(seed); err != nil { return nil, nil, nil, nil, err } else if n != seedSize { return nil, nil, nil, nil, fmt.Errorf("not enough seed bytes read (%d != %d", n, seedSize) } aek, err = genEncryptionKey(sc, seed) if err != nil { return nil, nil, nil, nil, err } // Generate our nonce. Use same buffer to hold encrypted seed. nonce := make([]byte, kek.NonceSize(), kek.NonceSize()+len(seed)+kek.Overhead()) if n, err := rand.Read(nonce); err != nil { return nil, nil, nil, nil, err } else if n != len(nonce) { return nil, nil, nil, nil, fmt.Errorf("not enough nonce bytes read (%d != %d)", n, len(nonce)) } bek, err = genBlockEncryptionKey(sc, seed[:], nonce) if err != nil { return nil, nil, nil, nil, err } return aek, bek, seed, kek.Seal(nonce, nonce, seed, nil), nil } // Will generate the block encryption key. func genBlockEncryptionKey(sc StoreCipher, seed, nonce []byte) (cipher.Stream, error) { if sc == ChaCha { return chacha20.NewUnauthenticatedCipher(seed, nonce) } else if sc == AES { block, err := aes.NewCipher(seed) if err != nil { return nil, err } return cipher.NewCTR(block, nonce), nil } return nil, errUnknownCipher } // Lock should be held. func (fs *fileStore) recoverAEK() error { if fs.prf != nil && fs.aek == nil { ekey, err := os.ReadFile(filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey)) if err != nil { return err } rb, err := fs.prf([]byte(fs.cfg.Name)) if err != nil { return err } kek, err := genEncryptionKey(fs.fcfg.Cipher, rb) if err != nil { return err } ns := kek.NonceSize() seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil) if err != nil { return err } aek, err := genEncryptionKey(fs.fcfg.Cipher, seed) if err != nil { return err } fs.aek = aek } return nil } // Lock should be held. func (fs *fileStore) setupAEK() error { if fs.prf != nil && fs.aek == nil { key, _, _, encrypted, err := fs.genEncryptionKeys(fs.cfg.Name) if err != nil { return err } keyFile := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey) if _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) { return err } err = fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms) if err != nil { return err } // Set our aek. fs.aek = key } return nil } // Write out meta and the checksum. // Lock should be held. func (fs *fileStore) writeStreamMeta() error { if err := fs.setupAEK(); err != nil { return err } meta := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFile) if _, err := os.Stat(meta); err != nil && !os.IsNotExist(err) { return err } b, err := json.Marshal(fs.cfg) if err != nil { return err } // Encrypt if needed. if fs.aek != nil { nonce := make([]byte, fs.aek.NonceSize(), fs.aek.NonceSize()+len(b)+fs.aek.Overhead()) if n, err := rand.Read(nonce); err != nil { return err } else if n != len(nonce) { return fmt.Errorf("not enough nonce bytes read (%d != %d)", n, len(nonce)) } b = fs.aek.Seal(nonce, nonce, b, nil) } err = fs.writeFileWithOptionalSync(meta, b, defaultFilePerms) if err != nil { return err } fs.hh.Reset() fs.hh.Write(b) checksum := hex.EncodeToString(fs.hh.Sum(nil)) sum := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileSum) err = fs.writeFileWithOptionalSync(sum, []byte(checksum), defaultFilePerms) if err != nil { return err } return nil } // Pools to recycle the blocks to help with memory pressure. var blkPoolBig sync.Pool // 16MB var blkPoolMedium sync.Pool // 8MB var blkPoolSmall sync.Pool // 2MB // Get a new msg block based on sz estimate. func getMsgBlockBuf(sz int) (buf []byte) { var pb any if sz <= defaultSmallBlockSize { pb = blkPoolSmall.Get() } else if sz <= defaultMediumBlockSize { pb = blkPoolMedium.Get() } else { pb = blkPoolBig.Get() } if pb != nil { buf = *(pb.(*[]byte)) } else { // Here we need to make a new blk. // If small leave as is.. if sz > defaultSmallBlockSize && sz <= defaultMediumBlockSize { sz = defaultMediumBlockSize } else if sz > defaultMediumBlockSize { sz = defaultLargeBlockSize } buf = make([]byte, sz) } return buf[:0] } // Recycle the msg block. func recycleMsgBlockBuf(buf []byte) { if buf == nil || cap(buf) < defaultSmallBlockSize { return } // Make sure to reset before placing back into pool. buf = buf[:0] // We need to make sure the load code gets a block that can fit the maximum for a size block. // E.g. 8, 16 etc. otherwise we thrash and actually make things worse by pulling it out, and putting // it right back in and making a new []byte. // From above we know its already >= defaultSmallBlockSize if sz := cap(buf); sz < defaultMediumBlockSize { blkPoolSmall.Put(&buf) } else if sz < defaultLargeBlockSize { blkPoolMedium.Put(&buf) } else { blkPoolBig.Put(&buf) } } const ( msgHdrSize = 22 checksumSize = 8 emptyRecordLen = msgHdrSize + checksumSize ) // Lock should be held. func (fs *fileStore) noTrackSubjects() bool { return !(fs.psim.Size() > 0 || len(fs.cfg.Subjects) > 0 || fs.cfg.Mirror != nil || len(fs.cfg.Sources) > 0) } // Will init the basics for a message block. func (fs *fileStore) initMsgBlock(index uint32) *msgBlock { mb := &msgBlock{ fs: fs, index: index, cexp: fs.fcfg.CacheExpire, fexp: fs.fcfg.SubjectStateExpire, noTrack: fs.noTrackSubjects(), syncAlways: fs.fcfg.SyncAlways, } mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) mb.mfn = filepath.Join(mdir, fmt.Sprintf(blkScan, index)) if mb.hh == nil { key := sha256.Sum256(fs.hashKeyForBlock(index)) mb.hh, _ = highwayhash.New64(key[:]) } return mb } // Lock for fs should be held. func (fs *fileStore) loadEncryptionForMsgBlock(mb *msgBlock) error { if fs.prf == nil { return nil } var createdKeys bool mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) ekey, err := os.ReadFile(filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index))) if err != nil { // We do not seem to have keys even though we should. Could be a plaintext conversion. // Create the keys and we will double check below. if err := fs.genEncryptionKeysForBlock(mb); err != nil { return err } createdKeys = true } else { if len(ekey) < minBlkKeySize { return errBadKeySize } // Recover key encryption key. rb, err := fs.prf([]byte(fmt.Sprintf("%s:%d", fs.cfg.Name, mb.index))) if err != nil { return err } sc := fs.fcfg.Cipher kek, err := genEncryptionKey(sc, rb) if err != nil { return err } ns := kek.NonceSize() seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil) if err != nil { // We may be here on a cipher conversion, so attempt to convert. if err = mb.convertCipher(); err != nil { return err } } else { mb.seed, mb.nonce = seed, ekey[:ns] } mb.aek, err = genEncryptionKey(sc, mb.seed) if err != nil { return err } if mb.bek, err = genBlockEncryptionKey(sc, mb.seed, mb.nonce); err != nil { return err } } // If we created keys here, let's check the data and if it is plaintext convert here. if createdKeys { if err := mb.convertToEncrypted(); err != nil { return err } } return nil } // Load a last checksum if needed from the block file. // Lock should be held. func (mb *msgBlock) ensureLastChecksumLoaded() { var empty [8]byte if mb.lchk != empty { return } copy(mb.lchk[0:], mb.lastChecksum()) } // Lock held on entry func (fs *fileStore) recoverMsgBlock(index uint32) (*msgBlock, error) { mb := fs.initMsgBlock(index) // Open up the message file, but we will try to recover from the index file. // We will check that the last checksums match. file, err := mb.openBlock() if err != nil { return nil, err } defer file.Close() if fi, err := file.Stat(); fi != nil { mb.rbytes = uint64(fi.Size()) } else { return nil, err } // Make sure encryption loaded if needed. fs.loadEncryptionForMsgBlock(mb) // Grab last checksum from main block file. var lchk [8]byte if mb.rbytes >= checksumSize { if mb.bek != nil { if buf, _ := mb.loadBlock(nil); len(buf) >= checksumSize { mb.bek.XORKeyStream(buf, buf) copy(lchk[0:], buf[len(buf)-checksumSize:]) } } else { file.ReadAt(lchk[:], int64(mb.rbytes)-checksumSize) } } file.Close() // Read our index file. Use this as source of truth if possible. // This not applicable in >= 2.10 servers. Here for upgrade paths from < 2.10. if err := mb.readIndexInfo(); err == nil { // Quick sanity check here. // Note this only checks that the message blk file is not newer then this file, or is empty and we expect empty. if (mb.rbytes == 0 && mb.msgs == 0) || bytes.Equal(lchk[:], mb.lchk[:]) { if mb.msgs > 0 && !mb.noTrack && fs.psim != nil { fs.populateGlobalPerSubjectInfo(mb) // Try to dump any state we needed on recovery. mb.tryForceExpireCacheLocked() } fs.addMsgBlock(mb) return mb, nil } } // If we get data loss rebuilding the message block state record that with the fs itself. ld, tombs, _ := mb.rebuildState() if ld != nil { fs.addLostData(ld) } // Collect all tombstones. if len(tombs) > 0 { fs.tombs = append(fs.tombs, tombs...) } if mb.msgs > 0 && !mb.noTrack && fs.psim != nil { fs.populateGlobalPerSubjectInfo(mb) // Try to dump any state we needed on recovery. mb.tryForceExpireCacheLocked() } mb.closeFDs() fs.addMsgBlock(mb) return mb, nil } func (fs *fileStore) lostData() *LostStreamData { fs.mu.RLock() defer fs.mu.RUnlock() if fs.ld == nil { return nil } nld := *fs.ld return &nld } // Lock should be held. func (fs *fileStore) addLostData(ld *LostStreamData) { if ld == nil { return } if fs.ld != nil { var added bool for _, seq := range ld.Msgs { if _, found := fs.ld.exists(seq); !found { fs.ld.Msgs = append(fs.ld.Msgs, seq) added = true } } if added { msgs := fs.ld.Msgs slices.Sort(msgs) fs.ld.Bytes += ld.Bytes } } else { fs.ld = ld } } // Helper to see if we already have this sequence reported in our lost data. func (ld *LostStreamData) exists(seq uint64) (int, bool) { i := slices.IndexFunc(ld.Msgs, func(i uint64) bool { return i == seq }) return i, i > -1 } func (fs *fileStore) removeFromLostData(seq uint64) { if fs.ld == nil { return } if i, found := fs.ld.exists(seq); found { fs.ld.Msgs = append(fs.ld.Msgs[:i], fs.ld.Msgs[i+1:]...) if len(fs.ld.Msgs) == 0 { fs.ld = nil } } } func (fs *fileStore) rebuildState(ld *LostStreamData) { fs.mu.Lock() defer fs.mu.Unlock() fs.rebuildStateLocked(ld) } // Lock should be held. func (fs *fileStore) rebuildStateLocked(ld *LostStreamData) { fs.addLostData(ld) fs.state.Msgs, fs.state.Bytes = 0, 0 fs.state.FirstSeq, fs.state.LastSeq = 0, 0 for _, mb := range fs.blks { mb.mu.RLock() fs.state.Msgs += mb.msgs fs.state.Bytes += mb.bytes fseq := atomic.LoadUint64(&mb.first.seq) if fs.state.FirstSeq == 0 || fseq < fs.state.FirstSeq { fs.state.FirstSeq = fseq fs.state.FirstTime = time.Unix(0, mb.first.ts).UTC() } fs.state.LastSeq = atomic.LoadUint64(&mb.last.seq) fs.state.LastTime = time.Unix(0, mb.last.ts).UTC() mb.mu.RUnlock() } } // Attempt to convert the cipher used for this message block. func (mb *msgBlock) convertCipher() error { fs := mb.fs sc := fs.fcfg.Cipher var osc StoreCipher switch sc { case ChaCha: osc = AES case AES: osc = ChaCha } mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) ekey, err := os.ReadFile(filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index))) if err != nil { return err } if len(ekey) < minBlkKeySize { return errBadKeySize } type prfWithCipher struct { keyGen StoreCipher } var prfs []prfWithCipher if fs.prf != nil { prfs = append(prfs, prfWithCipher{fs.prf, sc}) prfs = append(prfs, prfWithCipher{fs.prf, osc}) } if fs.oldprf != nil { prfs = append(prfs, prfWithCipher{fs.oldprf, sc}) prfs = append(prfs, prfWithCipher{fs.oldprf, osc}) } for _, prf := range prfs { // Recover key encryption key. rb, err := prf.keyGen([]byte(fmt.Sprintf("%s:%d", fs.cfg.Name, mb.index))) if err != nil { continue } kek, err := genEncryptionKey(prf.StoreCipher, rb) if err != nil { continue } ns := kek.NonceSize() seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil) if err != nil { continue } nonce := ekey[:ns] bek, err := genBlockEncryptionKey(prf.StoreCipher, seed, nonce) if err != nil { return err } buf, _ := mb.loadBlock(nil) bek.XORKeyStream(buf, buf) // Make sure we can parse with old cipher and key file. if err = mb.indexCacheBuf(buf); err != nil { return err } // Reset the cache since we just read everything in. mb.cache = nil // Generate new keys. If we error for some reason then we will put // the old keyfile back. if err := fs.genEncryptionKeysForBlock(mb); err != nil { keyFile := filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index)) fs.writeFileWithOptionalSync(keyFile, ekey, defaultFilePerms) return err } mb.bek.XORKeyStream(buf, buf) <-dios err = os.WriteFile(mb.mfn, buf, defaultFilePerms) dios <- struct{}{} if err != nil { return err } return nil } return fmt.Errorf("unable to recover keys") } // Convert a plaintext block to encrypted. func (mb *msgBlock) convertToEncrypted() error { if mb.bek == nil { return nil } buf, err := mb.loadBlock(nil) if err != nil { return err } if err := mb.indexCacheBuf(buf); err != nil { // This likely indicates this was already encrypted or corrupt. mb.cache = nil return err } // Undo cache from above for later. mb.cache = nil mb.bek.XORKeyStream(buf, buf) <-dios err = os.WriteFile(mb.mfn, buf, defaultFilePerms) dios <- struct{}{} if err != nil { return err } return nil } // Return the mb's index. func (mb *msgBlock) getIndex() uint32 { mb.mu.RLock() defer mb.mu.RUnlock() return mb.index } // Rebuild the state of the blk based on what we have on disk in the N.blk file. // We will return any lost data, and we will return any delete tombstones we encountered. func (mb *msgBlock) rebuildState() (*LostStreamData, []uint64, error) { mb.mu.Lock() defer mb.mu.Unlock() return mb.rebuildStateLocked() } // Rebuild the state of the blk based on what we have on disk in the N.blk file. // Lock should be held. func (mb *msgBlock) rebuildStateLocked() (*LostStreamData, []uint64, error) { startLastSeq := atomic.LoadUint64(&mb.last.seq) // Remove the .fss file and clear any cache we have set. mb.clearCacheAndOffset() buf, err := mb.loadBlock(nil) defer recycleMsgBlockBuf(buf) if err != nil || len(buf) == 0 { var ld *LostStreamData // No data to rebuild from here. if mb.msgs > 0 { // We need to declare lost data here. ld = &LostStreamData{Msgs: make([]uint64, 0, mb.msgs), Bytes: mb.bytes} firstSeq, lastSeq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) for seq := firstSeq; seq <= lastSeq; seq++ { if !mb.dmap.Exists(seq) { ld.Msgs = append(ld.Msgs, seq) } } // Clear invalid state. We will let this blk be added in here. mb.msgs, mb.bytes, mb.rbytes, mb.fss = 0, 0, 0, nil mb.dmap.Empty() atomic.StoreUint64(&mb.first.seq, atomic.LoadUint64(&mb.last.seq)+1) } return ld, nil, err } // Clear state we need to rebuild. mb.msgs, mb.bytes, mb.rbytes, mb.fss = 0, 0, 0, nil atomic.StoreUint64(&mb.last.seq, 0) mb.last.ts = 0 firstNeedsSet := true // Check if we need to decrypt. if mb.bek != nil && len(buf) > 0 { // Recreate to reset counter. mb.bek, err = genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return nil, nil, err } mb.bek.XORKeyStream(buf, buf) } // Check for compression. if buf, err = mb.decompressIfNeeded(buf); err != nil { return nil, nil, err } mb.rbytes = uint64(len(buf)) addToDmap := func(seq uint64) { if seq == 0 { return } mb.dmap.Insert(seq) } var le = binary.LittleEndian truncate := func(index uint32) { var fd *os.File if mb.mfd != nil { fd = mb.mfd } else { <-dios fd, err = os.OpenFile(mb.mfn, os.O_RDWR, defaultFilePerms) dios <- struct{}{} if err == nil { defer fd.Close() } } if fd == nil { return } if err := fd.Truncate(int64(index)); err == nil { // Update our checksum. if index >= 8 { var lchk [8]byte fd.ReadAt(lchk[:], int64(index-8)) copy(mb.lchk[0:], lchk[:]) } fd.Sync() } } gatherLost := func(lb uint32) *LostStreamData { var ld LostStreamData for seq := atomic.LoadUint64(&mb.last.seq) + 1; seq <= startLastSeq; seq++ { ld.Msgs = append(ld.Msgs, seq) } ld.Bytes = uint64(lb) return &ld } // For tombstones that we find and collect. var ( tombstones []uint64 minTombstoneSeq uint64 minTombstoneTs int64 ) // To detect gaps from compaction. var last uint64 for index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; { if index+msgHdrSize > lbuf { truncate(index) return gatherLost(lbuf - index), tombstones, nil } hdr := buf[index : index+msgHdrSize] rl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:])) hasHeaders := rl&hbit != 0 // Clear any headers bit that could be set. rl &^= hbit dlen := int(rl) - msgHdrSize // Do some quick sanity checks here. if dlen < 0 || slen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh { truncate(index) return gatherLost(lbuf - index), tombstones, errBadMsg } // Check for checksum failures before additional processing. data := buf[index+msgHdrSize : index+rl] if hh := mb.hh; hh != nil { hh.Reset() hh.Write(hdr[4:20]) hh.Write(data[:slen]) if hasHeaders { hh.Write(data[slen+4 : dlen-recordHashSize]) } else { hh.Write(data[slen : dlen-recordHashSize]) } checksum := hh.Sum(nil) if !bytes.Equal(checksum, data[len(data)-recordHashSize:]) { truncate(index) return gatherLost(lbuf - index), tombstones, errBadMsg } copy(mb.lchk[0:], checksum) } // Grab our sequence and timestamp. seq := le.Uint64(hdr[4:]) ts := int64(le.Uint64(hdr[12:])) // Check if this is a delete tombstone. if seq&tbit != 0 { seq = seq &^ tbit // Need to process this here and make sure we have accounted for this properly. tombstones = append(tombstones, seq) if minTombstoneSeq == 0 || seq < minTombstoneSeq { minTombstoneSeq, minTombstoneTs = seq, ts } index += rl continue } fseq := atomic.LoadUint64(&mb.first.seq) // This is an old erased message, or a new one that we can track. if seq == 0 || seq&ebit != 0 || seq < fseq { seq = seq &^ ebit if seq >= fseq { atomic.StoreUint64(&mb.last.seq, seq) mb.last.ts = ts if mb.msgs == 0 { atomic.StoreUint64(&mb.first.seq, seq+1) mb.first.ts = 0 } else if seq != 0 { // Only add to dmap if past recorded first seq and non-zero. addToDmap(seq) } } index += rl continue } // This is for when we have index info that adjusts for deleted messages // at the head. So the first.seq will be already set here. If this is larger // replace what we have with this seq. if firstNeedsSet && seq >= fseq { atomic.StoreUint64(&mb.first.seq, seq) firstNeedsSet, mb.first.ts = false, ts } if !mb.dmap.Exists(seq) { mb.msgs++ mb.bytes += uint64(rl) } // Check for any gaps from compaction, meaning no ebit entry. if last > 0 && seq != last+1 { for dseq := last + 1; dseq < seq; dseq++ { addToDmap(dseq) } } // Always set last last = seq atomic.StoreUint64(&mb.last.seq, last) mb.last.ts = ts // Advance to next record. index += rl } // For empty msg blocks make sure we recover last seq correctly based off of first. // Or if we seem to have no messages but had a tombstone, which we use to remember // sequences and timestamps now, use that to properly setup the first and last. if mb.msgs == 0 { fseq := atomic.LoadUint64(&mb.first.seq) if fseq > 0 { atomic.StoreUint64(&mb.last.seq, fseq-1) } else if fseq == 0 && minTombstoneSeq > 0 { atomic.StoreUint64(&mb.first.seq, minTombstoneSeq+1) mb.first.ts = 0 if mb.last.seq == 0 { atomic.StoreUint64(&mb.last.seq, minTombstoneSeq) mb.last.ts = minTombstoneTs } } } return nil, tombstones, nil } // For doing warn logging. // Lock should be held. func (fs *fileStore) warn(format string, args ...any) { // No-op if no server configured. if fs.srv == nil { return } fs.srv.Warnf(fmt.Sprintf("Filestore [%s] %s", fs.cfg.Name, format), args...) } // For doing debug logging. // Lock should be held. func (fs *fileStore) debug(format string, args ...any) { // No-op if no server configured. if fs.srv == nil { return } fs.srv.Debugf(fmt.Sprintf("Filestore [%s] %s", fs.cfg.Name, format), args...) } // Track local state but ignore timestamps here. func updateTrackingState(state *StreamState, mb *msgBlock) { if state.FirstSeq == 0 { state.FirstSeq = mb.first.seq } else if mb.first.seq < state.FirstSeq { state.FirstSeq = mb.first.seq } if mb.last.seq > state.LastSeq { state.LastSeq = mb.last.seq } state.Msgs += mb.msgs state.Bytes += mb.bytes } // Determine if our tracking states are the same. func trackingStatesEqual(fs, mb *StreamState) bool { // When a fs is brand new the fs state will have first seq of 0, but tracking mb may have 1. // If either has a first sequence that is not 0 or 1 we will check if they are the same, otherwise skip. if (fs.FirstSeq > 1 && mb.FirstSeq > 1) || mb.FirstSeq > 1 { return fs.Msgs == mb.Msgs && fs.FirstSeq == mb.FirstSeq && fs.LastSeq == mb.LastSeq && fs.Bytes == mb.Bytes } return fs.Msgs == mb.Msgs && fs.LastSeq == mb.LastSeq && fs.Bytes == mb.Bytes } // recoverFullState will attempt to receover our last full state and re-process any state changes // that happened afterwards. func (fs *fileStore) recoverFullState() (rerr error) { fs.mu.Lock() defer fs.mu.Unlock() // Check for any left over purged messages. <-dios pdir := filepath.Join(fs.fcfg.StoreDir, purgeDir) if _, err := os.Stat(pdir); err == nil { os.RemoveAll(pdir) } // Grab our stream state file and load it in. fn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) buf, err := os.ReadFile(fn) dios <- struct{}{} if err != nil { if !os.IsNotExist(err) { fs.warn("Could not read stream state file: %v", err) } return err } const minLen = 32 if len(buf) < minLen { os.Remove(fn) fs.warn("Stream state too short (%d bytes)", len(buf)) return errCorruptState } // The highwayhash will be on the end. Check that it still matches. h := buf[len(buf)-highwayhash.Size64:] buf = buf[:len(buf)-highwayhash.Size64] fs.hh.Reset() fs.hh.Write(buf) if !bytes.Equal(h, fs.hh.Sum(nil)) { os.Remove(fn) fs.warn("Stream state checksum did not match") return errCorruptState } // Decrypt if needed. if fs.prf != nil { // We can be setup for encryption but if this is a snapshot restore we will be missing the keyfile // since snapshots strip encryption. if err := fs.recoverAEK(); err == nil { ns := fs.aek.NonceSize() buf, err = fs.aek.Open(nil, buf[:ns], buf[ns:], nil) if err != nil { fs.warn("Stream state error reading encryption key: %v", err) return err } } } if buf[0] != fullStateMagic || buf[1] != fullStateVersion { os.Remove(fn) fs.warn("Stream state magic and version mismatch") return errCorruptState } bi := hdrLen readU64 := func() uint64 { if bi < 0 { return 0 } v, n := binary.Uvarint(buf[bi:]) if n <= 0 { bi = -1 return 0 } bi += n return v } readI64 := func() int64 { if bi < 0 { return 0 } v, n := binary.Varint(buf[bi:]) if n <= 0 { bi = -1 return -1 } bi += n return v } setTime := func(t *time.Time, ts int64) { if ts == 0 { *t = time.Time{} } else { *t = time.Unix(0, ts).UTC() } } var state StreamState state.Msgs = readU64() state.Bytes = readU64() state.FirstSeq = readU64() baseTime := readI64() setTime(&state.FirstTime, baseTime) state.LastSeq = readU64() setTime(&state.LastTime, readI64()) // Check for per subject info. if numSubjects := int(readU64()); numSubjects > 0 { fs.psim, fs.tsl = fs.psim.Empty(), 0 for i := 0; i < numSubjects; i++ { if lsubj := int(readU64()); lsubj > 0 { if bi+lsubj > len(buf) { os.Remove(fn) fs.warn("Stream state bad subject len (%d)", lsubj) return errCorruptState } // If we have lots of subjects this will alloc for each one. // We could reference the underlying buffer, but we could guess wrong if // number of blocks is large and subjects is low, since we would reference buf. subj := buf[bi : bi+lsubj] // We had a bug that could cause memory corruption in the PSIM that could have gotten stored to disk. // Only would affect subjects, so do quick check. if !isValidSubject(bytesToString(subj), true) { os.Remove(fn) fs.warn("Stream state corrupt subject detected") return errCorruptState } bi += lsubj psi := psi{total: readU64(), fblk: uint32(readU64())} if psi.total > 1 { psi.lblk = uint32(readU64()) } else { psi.lblk = psi.fblk } fs.psim.Insert(subj, psi) fs.tsl += lsubj } } } // Track the state as represented by the blocks themselves. var mstate StreamState if numBlocks := readU64(); numBlocks > 0 { lastIndex := int(numBlocks - 1) fs.blks = make([]*msgBlock, 0, numBlocks) for i := 0; i < int(numBlocks); i++ { index, nbytes, fseq, fts, lseq, lts, numDeleted := uint32(readU64()), readU64(), readU64(), readI64(), readU64(), readI64(), readU64() if bi < 0 { os.Remove(fn) return errCorruptState } mb := fs.initMsgBlock(index) atomic.StoreUint64(&mb.first.seq, fseq) atomic.StoreUint64(&mb.last.seq, lseq) mb.msgs, mb.bytes = lseq-fseq+1, nbytes mb.first.ts, mb.last.ts = fts+baseTime, lts+baseTime if numDeleted > 0 { dmap, n, err := avl.Decode(buf[bi:]) if err != nil { os.Remove(fn) fs.warn("Stream state error decoding avl dmap: %v", err) return errCorruptState } mb.dmap = *dmap if mb.msgs > numDeleted { mb.msgs -= numDeleted } else { mb.msgs = 0 } bi += n } // Only add in if not empty or the lmb. if mb.msgs > 0 || i == lastIndex { fs.addMsgBlock(mb) updateTrackingState(&mstate, mb) } else { // Mark dirty to cleanup. fs.dirty++ } } } // Pull in last block index for the block that had last checksum when we wrote the full state. blkIndex := uint32(readU64()) var lchk [8]byte if bi+len(lchk) > len(buf) { bi = -1 } else { copy(lchk[0:], buf[bi:bi+len(lchk)]) } // Check if we had any errors. if bi < 0 { os.Remove(fn) fs.warn("Stream state has no checksum present") return errCorruptState } // Move into place our state, msgBlks and subject info. fs.state = state // First let's check the happy path, open the blk file that was the lmb when we created the full state. // See if we have the last block available. var matched bool mb := fs.lmb if mb == nil || mb.index != blkIndex { os.Remove(fn) fs.warn("Stream state block does not exist or index mismatch") return errCorruptState } if _, err := os.Stat(mb.mfn); err != nil && os.IsNotExist(err) { // If our saved state is past what we see on disk, fallback and rebuild. if ld, _, _ := mb.rebuildState(); ld != nil { fs.addLostData(ld) } fs.warn("Stream state detected prior state, could not locate msg block %d", blkIndex) return errPriorState } if matched = bytes.Equal(mb.lastChecksum(), lchk[:]); !matched { // Detected a stale index.db, we didn't write it upon shutdown so can't rely on it being correct. fs.warn("Stream state outdated, last block has additional entries, will rebuild") return errPriorState } // We need to see if any blocks exist after our last one even though we matched the last record exactly. mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) var dirs []os.DirEntry <-dios if f, err := os.Open(mdir); err == nil { dirs, _ = f.ReadDir(-1) f.Close() } dios <- struct{}{} var index uint32 for _, fi := range dirs { if n, err := fmt.Sscanf(fi.Name(), blkScan, &index); err == nil && n == 1 { if index > blkIndex { fs.warn("Stream state outdated, found extra blocks, will rebuild") return errPriorState } } } // We check first and last seq and number of msgs and bytes. If there is a difference, // return and error so we rebuild from the message block state on disk. if !trackingStatesEqual(&fs.state, &mstate) { os.Remove(fn) fs.warn("Stream state encountered internal inconsistency on recover") return errCorruptState } return nil } // Grabs last checksum for the named block file. // Takes into account encryption etc. func (mb *msgBlock) lastChecksum() []byte { f, err := mb.openBlock() if err != nil { return nil } defer f.Close() var lchk [8]byte if fi, _ := f.Stat(); fi != nil { mb.rbytes = uint64(fi.Size()) } if mb.rbytes < checksumSize { return lchk[:] } // Encrypted? // Check for encryption, we do not load keys on startup anymore so might need to load them here. if mb.fs != nil && mb.fs.prf != nil && (mb.aek == nil || mb.bek == nil) { if err := mb.fs.loadEncryptionForMsgBlock(mb); err != nil { return nil } } if mb.bek != nil { if buf, _ := mb.loadBlock(nil); len(buf) >= checksumSize { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return nil } mb.bek = bek mb.bek.XORKeyStream(buf, buf) copy(lchk[0:], buf[len(buf)-checksumSize:]) } } else { f.ReadAt(lchk[:], int64(mb.rbytes)-checksumSize) } return lchk[:] } // This will make sure we clean up old idx and fss files. func (fs *fileStore) cleanupOldMeta() { fs.mu.RLock() mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) fs.mu.RUnlock() <-dios f, err := os.Open(mdir) dios <- struct{}{} if err != nil { return } dirs, _ := f.ReadDir(-1) f.Close() const ( minLen = 4 idxSuffix = ".idx" fssSuffix = ".fss" ) for _, fi := range dirs { if name := fi.Name(); strings.HasSuffix(name, idxSuffix) || strings.HasSuffix(name, fssSuffix) { os.Remove(filepath.Join(mdir, name)) } } } func (fs *fileStore) recoverMsgs() error { fs.mu.Lock() defer fs.mu.Unlock() // Check for any left over purged messages. <-dios pdir := filepath.Join(fs.fcfg.StoreDir, purgeDir) if _, err := os.Stat(pdir); err == nil { os.RemoveAll(pdir) } mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) f, err := os.Open(mdir) if err != nil { dios <- struct{}{} return errNotReadable } dirs, err := f.ReadDir(-1) f.Close() dios <- struct{}{} if err != nil { return errNotReadable } indices := make(sort.IntSlice, 0, len(dirs)) var index int for _, fi := range dirs { if n, err := fmt.Sscanf(fi.Name(), blkScan, &index); err == nil && n == 1 { indices = append(indices, index) } } indices.Sort() // Recover all of the msg blocks. // We now guarantee they are coming in order. for _, index := range indices { if mb, err := fs.recoverMsgBlock(uint32(index)); err == nil && mb != nil { // This is a truncate block with possibly no index. If the OS got shutdown // out from underneath of us this is possible. if mb.first.seq == 0 { mb.dirtyCloseWithRemove(true) fs.removeMsgBlockFromList(mb) continue } if fseq := atomic.LoadUint64(&mb.first.seq); fs.state.FirstSeq == 0 || fseq < fs.state.FirstSeq { fs.state.FirstSeq = fseq if mb.first.ts == 0 { fs.state.FirstTime = time.Time{} } else { fs.state.FirstTime = time.Unix(0, mb.first.ts).UTC() } } if lseq := atomic.LoadUint64(&mb.last.seq); lseq > fs.state.LastSeq { fs.state.LastSeq = lseq if mb.last.ts == 0 { fs.state.LastTime = time.Time{} } else { fs.state.LastTime = time.Unix(0, mb.last.ts).UTC() } } fs.state.Msgs += mb.msgs fs.state.Bytes += mb.bytes } else { return err } } if len(fs.blks) > 0 { fs.lmb = fs.blks[len(fs.blks)-1] } else { _, err = fs.newMsgBlockForWrite() } // Check if we encountered any lost data. if fs.ld != nil { var emptyBlks []*msgBlock for _, mb := range fs.blks { if mb.msgs == 0 && mb.rbytes == 0 { emptyBlks = append(emptyBlks, mb) } } for _, mb := range emptyBlks { // Need the mb lock here. mb.mu.Lock() fs.removeMsgBlock(mb) mb.mu.Unlock() } } if err != nil { return err } // Check for keyfiles orphans. if kms, err := filepath.Glob(filepath.Join(mdir, keyScanAll)); err == nil && len(kms) > 0 { valid := make(map[uint32]bool) for _, mb := range fs.blks { valid[mb.index] = true } for _, fn := range kms { var index uint32 shouldRemove := true if n, err := fmt.Sscanf(filepath.Base(fn), keyScan, &index); err == nil && n == 1 && valid[index] { shouldRemove = false } if shouldRemove { os.Remove(fn) } } } return nil } // Will expire msgs that have aged out on restart. // We will treat this differently in case we have a recovery // that will expire alot of messages on startup. // Should only be called on startup. func (fs *fileStore) expireMsgsOnRecover() error { if fs.state.Msgs == 0 { return nil } var minAge = time.Now().UnixNano() - int64(fs.cfg.MaxAge) var purged, bytes uint64 var deleted int var nts int64 // If we expire all make sure to write out a tombstone. Need to be done by hand here, // usually taken care of by fs.removeMsgBlock() but we do not call that here. var last msgId deleteEmptyBlock := func(mb *msgBlock) error { // If we are the last keep state to remember first/last sequence. // Do this part by hand since not deleting one by one. if mb == fs.lmb { last.seq = atomic.LoadUint64(&mb.last.seq) last.ts = mb.last.ts } // Make sure we do subject cleanup as well. mb.ensurePerSubjectInfoLoaded() mb.fss.IterOrdered(func(bsubj []byte, ss *SimpleState) bool { subj := bytesToString(bsubj) for i := uint64(0); i < ss.Msgs; i++ { fs.removePerSubject(subj) } return true }) err := mb.dirtyCloseWithRemove(true) if isPermissionError(err) { return err } deleted++ return nil } for _, mb := range fs.blks { mb.mu.Lock() if minAge < mb.first.ts { nts = mb.first.ts mb.mu.Unlock() break } // Can we remove whole block here? if mb.last.ts <= minAge { purged += mb.msgs bytes += mb.bytes err := deleteEmptyBlock(mb) mb.mu.Unlock() if isPermissionError(err) { return err } continue } // If we are here we have to process the interior messages of this blk. // This will load fss as well. if err := mb.loadMsgsWithLock(); err != nil { mb.mu.Unlock() break } var smv StoreMsg var needNextFirst bool // Walk messages and remove if expired. fseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) for seq := fseq; seq <= lseq; seq++ { sm, err := mb.cacheLookup(seq, &smv) // Process interior deleted msgs. if err == errDeletedMsg { // Update dmap. if mb.dmap.Exists(seq) { mb.dmap.Delete(seq) } // Keep this updated just in case since we are removing dmap entries. atomic.StoreUint64(&mb.first.seq, seq) needNextFirst = true continue } // Break on other errors. if err != nil || sm == nil { atomic.StoreUint64(&mb.first.seq, seq) needNextFirst = true break } // No error and sm != nil from here onward. // Check for done. if minAge < sm.ts { atomic.StoreUint64(&mb.first.seq, sm.seq) mb.first.ts = sm.ts needNextFirst = false nts = sm.ts break } // Delete the message here. if mb.msgs > 0 { atomic.StoreUint64(&mb.first.seq, seq) needNextFirst = true sz := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg) if sz > mb.bytes { sz = mb.bytes } mb.bytes -= sz bytes += sz mb.msgs-- purged++ } // Update fss // Make sure we have fss loaded. mb.removeSeqPerSubject(sm.subj, seq) fs.removePerSubject(sm.subj) } // Make sure we have a proper next first sequence. if needNextFirst { mb.selectNextFirst() } // Check if empty after processing, could happen if tail of messages are all deleted. if mb.msgs == 0 { deleteEmptyBlock(mb) } mb.mu.Unlock() break } if nts > 0 { // Make sure to set age check based on this value. fs.resetAgeChk(nts - minAge) } if deleted > 0 { // Update block map. if fs.bim != nil { for _, mb := range fs.blks[:deleted] { delete(fs.bim, mb.index) } } // Update blks slice. fs.blks = copyMsgBlocks(fs.blks[deleted:]) if lb := len(fs.blks); lb == 0 { fs.lmb = nil } else { fs.lmb = fs.blks[lb-1] } } // Update top level accounting. if purged < fs.state.Msgs { fs.state.Msgs -= purged } else { fs.state.Msgs = 0 } if bytes < fs.state.Bytes { fs.state.Bytes -= bytes } else { fs.state.Bytes = 0 } // Make sure to we properly set the fs first sequence and timestamp. fs.selectNextFirst() // Check if we have no messages and blocks left. if fs.lmb == nil && last.seq != 0 { if lmb, _ := fs.newMsgBlockForWrite(); lmb != nil { fs.writeTombstone(last.seq, last.ts) } // Clear any global subject state. fs.psim, fs.tsl = fs.psim.Empty(), 0 } // If we purged anything, make sure we kick flush state loop. if purged > 0 { fs.dirty++ } return nil } func copyMsgBlocks(src []*msgBlock) []*msgBlock { if src == nil { return nil } dst := make([]*msgBlock, len(src)) copy(dst, src) return dst } // GetSeqFromTime looks for the first sequence number that has // the message with >= timestamp. // FIXME(dlc) - inefficient, and dumb really. Make this better. func (fs *fileStore) GetSeqFromTime(t time.Time) uint64 { fs.mu.RLock() lastSeq := fs.state.LastSeq closed := fs.closed fs.mu.RUnlock() if closed { return 0 } mb := fs.selectMsgBlockForStart(t) if mb == nil { return lastSeq + 1 } fseq := atomic.LoadUint64(&mb.first.seq) lseq := atomic.LoadUint64(&mb.last.seq) var smv StoreMsg // Linear search, hence the dumb part.. ts := t.UnixNano() for seq := fseq; seq <= lseq; seq++ { sm, _, _ := mb.fetchMsg(seq, &smv) if sm != nil && sm.ts >= ts { return sm.seq } } return 0 } // Find the first matching message against a sublist. func (mb *msgBlock) firstMatchingMulti(sl *Sublist, start uint64, sm *StoreMsg) (*StoreMsg, bool, error) { mb.mu.Lock() var didLoad bool var updateLLTS bool defer func() { if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() }() // Need messages loaded from here on out. if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return nil, false, err } didLoad = true } // Make sure to start at mb.first.seq if fseq < mb.first.seq if seq := atomic.LoadUint64(&mb.first.seq); seq > start { start = seq } lseq := atomic.LoadUint64(&mb.last.seq) if sm == nil { sm = new(StoreMsg) } // If the FSS state has fewer entries than sequences in the linear scan, // then use intersection instead as likely going to be cheaper. This will // often be the case with high numbers of deletes, as well as a smaller // number of subjects in the block. if uint64(mb.fss.Size()) < lseq-start { // If there are no subject matches then this is effectively no-op. hseq := uint64(math.MaxUint64) IntersectStree(mb.fss, sl, func(subj []byte, ss *SimpleState) { if ss.firstNeedsUpdate || ss.lastNeedsUpdate { // mb is already loaded into the cache so should be fast-ish. mb.recalculateForSubj(bytesToString(subj), ss) } first := ss.First if start > first { first = start } if first > ss.Last || first >= hseq { // The start cutoff is after the last sequence for this subject, // or we think we already know of a subject with an earlier msg // than our first seq for this subject. return } if first == ss.First { // If the start floor is below where this subject starts then we can // short-circuit, avoiding needing to scan for the next message. if fsm, err := mb.cacheLookup(ss.First, sm); err == nil { sm = fsm hseq = ss.First } return } for seq := first; seq <= ss.Last; seq++ { // Otherwise we have a start floor that intersects where this subject // has messages in the block, so we need to walk up until we find a // message matching the subject. if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). // Instead we will update it only once in a defer. updateLLTS = true continue } llseq := mb.llseq fsm, err := mb.cacheLookup(seq, sm) if err != nil { continue } updateLLTS = false // cacheLookup already updated it. if sl.HasInterest(fsm.subj) { hseq = seq sm = fsm break } // If we are here we did not match, so put the llseq back. mb.llseq = llseq } }) if hseq < uint64(math.MaxUint64) && sm != nil { return sm, didLoad, nil } } else { for seq := start; seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). // Instead we will update it only once in a defer. updateLLTS = true continue } llseq := mb.llseq fsm, err := mb.cacheLookup(seq, sm) if err != nil { continue } expireOk := seq == lseq && mb.llseq == seq updateLLTS = false // cacheLookup already updated it. if sl.HasInterest(fsm.subj) { return fsm, expireOk, nil } // If we are here we did not match, so put the llseq back. mb.llseq = llseq } } return nil, didLoad, ErrStoreMsgNotFound } // Find the first matching message. // fs lock should be held. func (mb *msgBlock) firstMatching(filter string, wc bool, start uint64, sm *StoreMsg) (*StoreMsg, bool, error) { mb.mu.Lock() var updateLLTS bool defer func() { if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() }() fseq, isAll, subs := start, filter == _EMPTY_ || filter == fwcs, []string{filter} var didLoad bool if mb.fssNotLoaded() { // Make sure we have fss loaded. mb.loadMsgsWithLock() didLoad = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() if filter == _EMPTY_ { filter = fwcs wc = true } // If we only have 1 subject currently and it matches our filter we can also set isAll. if !isAll && mb.fss.Size() == 1 { if !wc { _, isAll = mb.fss.Find(stringToBytes(filter)) } else { // Since mb.fss.Find won't work if filter is a wildcard, need to use Match instead. mb.fss.Match(stringToBytes(filter), func(subject []byte, _ *SimpleState) { isAll = true }) } } // Make sure to start at mb.first.seq if fseq < mb.first.seq if seq := atomic.LoadUint64(&mb.first.seq); seq > fseq { fseq = seq } lseq := atomic.LoadUint64(&mb.last.seq) // Optionally build the isMatch for wildcard filters. _tsa, _fsa := [32]string{}, [32]string{} tsa, fsa := _tsa[:0], _fsa[:0] var isMatch func(subj string) bool // Decide to build. if wc { fsa = tokenizeSubjectIntoSlice(fsa[:0], filter) isMatch = func(subj string) bool { tsa = tokenizeSubjectIntoSlice(tsa[:0], subj) return isSubsetMatchTokenized(tsa, fsa) } } subjs := mb.fs.cfg.Subjects // If isAll or our single filter matches the filter arg do linear scan. doLinearScan := isAll || (wc && len(subjs) == 1 && subjs[0] == filter) // If we do not think we should do a linear scan check how many fss we // would need to scan vs the full range of the linear walk. Optimize for // 25th quantile of a match in a linear walk. Filter should be a wildcard. // We should consult fss if our cache is not loaded and we only have fss loaded. if !doLinearScan && wc && mb.cacheAlreadyLoaded() { doLinearScan = mb.fss.Size()*4 > int(lseq-fseq) } if !doLinearScan { // If we have a wildcard match against all tracked subjects we know about. if wc { subs = subs[:0] mb.fss.Match(stringToBytes(filter), func(bsubj []byte, _ *SimpleState) { subs = append(subs, string(bsubj)) }) // Check if we matched anything if len(subs) == 0 { return nil, didLoad, ErrStoreMsgNotFound } } fseq = lseq + 1 for _, subj := range subs { ss, _ := mb.fss.Find(stringToBytes(subj)) if ss != nil && (ss.firstNeedsUpdate || ss.lastNeedsUpdate) { mb.recalculateForSubj(subj, ss) } if ss == nil || start > ss.Last || ss.First >= fseq { continue } if ss.First < start { fseq = start } else { fseq = ss.First } } } if fseq > lseq { return nil, didLoad, ErrStoreMsgNotFound } // If we guess to not do a linear scan, but the above resulted in alot of subs that will // need to be checked for every scanned message, revert. // TODO(dlc) - we could memoize the subs across calls. if !doLinearScan && len(subs) > int(lseq-fseq) { doLinearScan = true } // Need messages loaded from here on out. if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return nil, false, err } didLoad = true } if sm == nil { sm = new(StoreMsg) } for seq := fseq; seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). // Instead we will update it only once in a defer. updateLLTS = true continue } llseq := mb.llseq fsm, err := mb.cacheLookup(seq, sm) if err != nil { if err == errPartialCache || err == errNoCache { return nil, false, err } continue } updateLLTS = false // cacheLookup already updated it. expireOk := seq == lseq && mb.llseq == seq if isAll { return fsm, expireOk, nil } if doLinearScan { if wc && isMatch(sm.subj) { return fsm, expireOk, nil } else if !wc && fsm.subj == filter { return fsm, expireOk, nil } } else { for _, subj := range subs { if fsm.subj == subj { return fsm, expireOk, nil } } } // If we are here we did not match, so put the llseq back. mb.llseq = llseq } return nil, didLoad, ErrStoreMsgNotFound } // This will traverse a message block and generate the filtered pending. func (mb *msgBlock) filteredPending(subj string, wc bool, seq uint64) (total, first, last uint64) { mb.mu.Lock() defer mb.mu.Unlock() return mb.filteredPendingLocked(subj, wc, seq) } // This will traverse a message block and generate the filtered pending. // Lock should be held. func (mb *msgBlock) filteredPendingLocked(filter string, wc bool, sseq uint64) (total, first, last uint64) { isAll := filter == _EMPTY_ || filter == fwcs // First check if we can optimize this part. // This means we want all and the starting sequence was before this block. if isAll { if fseq := atomic.LoadUint64(&mb.first.seq); sseq <= fseq { return mb.msgs, fseq, atomic.LoadUint64(&mb.last.seq) } } if filter == _EMPTY_ { filter = fwcs wc = true } update := func(ss *SimpleState) { total += ss.Msgs if first == 0 || ss.First < first { first = ss.First } if ss.Last > last { last = ss.Last } } // Make sure we have fss loaded. mb.ensurePerSubjectInfoLoaded() _tsa, _fsa := [32]string{}, [32]string{} tsa, fsa := _tsa[:0], _fsa[:0] fsa = tokenizeSubjectIntoSlice(fsa[:0], filter) // 1. See if we match any subs from fss. // 2. If we match and the sseq is past ss.Last then we can use meta only. // 3. If we match and we need to do a partial, break and clear any totals and do a full scan like num pending. isMatch := func(subj string) bool { if !wc { return subj == filter } tsa = tokenizeSubjectIntoSlice(tsa[:0], subj) return isSubsetMatchTokenized(tsa, fsa) } var havePartial bool mb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) { if havePartial { // If we already found a partial then don't do anything else. return } if ss.firstNeedsUpdate || ss.lastNeedsUpdate { mb.recalculateForSubj(bytesToString(bsubj), ss) } if sseq <= ss.First { update(ss) } else if sseq <= ss.Last { // We matched but its a partial. havePartial = true } }) // If we did not encounter any partials we can return here. if !havePartial { return total, first, last } // If we are here we need to scan the msgs. // Clear what we had. total, first, last = 0, 0, 0 // If we load the cache for a linear scan we want to expire that cache upon exit. var shouldExpire bool if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } var smv StoreMsg for seq, lseq := sseq, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ { sm, _ := mb.cacheLookup(seq, &smv) if sm == nil { continue } if isAll || isMatch(sm.subj) { total++ if first == 0 || seq < first { first = seq } if seq > last { last = seq } } } // If we loaded this block for this operation go ahead and expire it here. if shouldExpire { mb.tryForceExpireCacheLocked() } return total, first, last } // FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence. func (fs *fileStore) FilteredState(sseq uint64, subj string) SimpleState { fs.mu.RLock() defer fs.mu.RUnlock() lseq := fs.state.LastSeq if sseq < fs.state.FirstSeq { sseq = fs.state.FirstSeq } // Returned state. var ss SimpleState // If past the end no results. if sseq > lseq { // Make sure we track sequences ss.First = fs.state.FirstSeq ss.Last = fs.state.LastSeq return ss } // If we want all msgs that match we can shortcircuit. // TODO(dlc) - This can be extended for all cases but would // need to be careful on total msgs calculations etc. if sseq == fs.state.FirstSeq { fs.numFilteredPending(subj, &ss) } else { wc := subjectHasWildcard(subj) // Tracking subject state. // TODO(dlc) - Optimize for 2.10 with avl tree and no atomics per block. for _, mb := range fs.blks { // Skip blocks that are less than our starting sequence. if sseq > atomic.LoadUint64(&mb.last.seq) { continue } t, f, l := mb.filteredPending(subj, wc, sseq) ss.Msgs += t if ss.First == 0 || (f > 0 && f < ss.First) { ss.First = f } if l > ss.Last { ss.Last = l } } } return ss } // This is used to see if we can selectively jump start blocks based on filter subject and a starting block index. // Will return -1 and ErrStoreEOF if no matches at all or no more from where we are. func (fs *fileStore) checkSkipFirstBlock(filter string, wc bool, bi int) (int, error) { // If we match everything, just move to next blk. if filter == _EMPTY_ || filter == fwcs { return bi + 1, nil } // Move through psim to gather start and stop bounds. start, stop := uint32(math.MaxUint32), uint32(0) if wc { fs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) { if psi.fblk < start { start = psi.fblk } if psi.lblk > stop { stop = psi.lblk } }) } else if psi, ok := fs.psim.Find(stringToBytes(filter)); ok { start, stop = psi.fblk, psi.lblk } // Nothing was found. if start == uint32(math.MaxUint32) { return -1, ErrStoreEOF } // Can not be nil so ok to inline dereference. mbi := fs.blks[bi].getIndex() // All matching msgs are behind us. // Less than AND equal is important because we were called because we missed searching bi. if stop <= mbi { return -1, ErrStoreEOF } // If start is > index return dereference of fs.blks index. if start > mbi { if mb := fs.bim[start]; mb != nil { ni, _ := fs.selectMsgBlockWithIndex(atomic.LoadUint64(&mb.last.seq)) return ni, nil } } // Otherwise just bump to the next one. return bi + 1, nil } // Optimized way for getting all num pending matching a filter subject. // Lock should be held. func (fs *fileStore) numFilteredPending(filter string, ss *SimpleState) { fs.numFilteredPendingWithLast(filter, true, ss) } // Optimized way for getting all num pending matching a filter subject and first sequence only. // Lock should be held. func (fs *fileStore) numFilteredPendingNoLast(filter string, ss *SimpleState) { fs.numFilteredPendingWithLast(filter, false, ss) } // Optimized way for getting all num pending matching a filter subject. // Optionally look up last sequence. Sometimes do not need last and this avoids cost. // Read lock should be held. func (fs *fileStore) numFilteredPendingWithLast(filter string, last bool, ss *SimpleState) { isAll := filter == _EMPTY_ || filter == fwcs // If isAll we do not need to do anything special to calculate the first and last and total. if isAll { ss.First = fs.state.FirstSeq ss.Last = fs.state.LastSeq ss.Msgs = fs.state.Msgs return } // Always reset. ss.First, ss.Last, ss.Msgs = 0, 0, 0 // We do need to figure out the first and last sequences. wc := subjectHasWildcard(filter) start, stop := uint32(math.MaxUint32), uint32(0) if wc { fs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) { ss.Msgs += psi.total // Keep track of start and stop indexes for this subject. if psi.fblk < start { start = psi.fblk } if psi.lblk > stop { stop = psi.lblk } }) } else if psi, ok := fs.psim.Find(stringToBytes(filter)); ok { ss.Msgs += psi.total start, stop = psi.fblk, psi.lblk } // Did not find anything. if stop == 0 { return } // Do start mb := fs.bim[start] if mb != nil { _, f, _ := mb.filteredPending(filter, wc, 0) ss.First = f } if ss.First == 0 { // This is a miss. This can happen since psi.fblk is lazy. // We will make sure to update fblk. // Hold this outside loop for psim fblk updates when done. i := start + 1 for ; i <= stop; i++ { mb := fs.bim[i] if mb == nil { continue } if _, f, _ := mb.filteredPending(filter, wc, 0); f > 0 { ss.First = f break } } // Update fblk since fblk was outdated. // We only require read lock here as that is desirable, // so we need to do this in a go routine to acquire write lock. go func() { fs.mu.Lock() defer fs.mu.Unlock() if !wc { if info, ok := fs.psim.Find(stringToBytes(filter)); ok { if i > info.fblk { info.fblk = i } } } else { fs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) { if i > psi.fblk { psi.fblk = i } }) } }() } // Now gather last sequence if asked to do so. if last { if mb = fs.bim[stop]; mb != nil { _, _, l := mb.filteredPending(filter, wc, 0) ss.Last = l } } } // SubjectsState returns a map of SimpleState for all matching subjects. func (fs *fileStore) SubjectsState(subject string) map[string]SimpleState { fs.mu.RLock() defer fs.mu.RUnlock() if fs.state.Msgs == 0 || fs.noTrackSubjects() { return nil } if subject == _EMPTY_ { subject = fwcs } start, stop := fs.blks[0], fs.lmb // We can short circuit if not a wildcard using psim for start and stop. if !subjectHasWildcard(subject) { info, ok := fs.psim.Find(stringToBytes(subject)) if !ok { return nil } if f := fs.bim[info.fblk]; f != nil { start = f } if l := fs.bim[info.lblk]; l != nil { stop = l } } // Aggregate fss. fss := make(map[string]SimpleState) var startFound bool for _, mb := range fs.blks { if !startFound { if mb != start { continue } startFound = true } mb.mu.Lock() var shouldExpire bool if mb.fssNotLoaded() { // Make sure we have fss loaded. mb.loadMsgsWithLock() shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() mb.fss.Match(stringToBytes(subject), func(bsubj []byte, ss *SimpleState) { subj := string(bsubj) if ss.firstNeedsUpdate || ss.lastNeedsUpdate { mb.recalculateForSubj(subj, ss) } oss := fss[subj] if oss.First == 0 { // New fss[subj] = *ss } else { // Merge here. oss.Last, oss.Msgs = ss.Last, oss.Msgs+ss.Msgs fss[subj] = oss } }) if shouldExpire { // Expire this cache before moving on. mb.tryForceExpireCacheLocked() } mb.mu.Unlock() if mb == stop { break } } return fss } // NumPending will return the number of pending messages matching the filter subject starting at sequence. // Optimized for stream num pending calculations for consumers. func (fs *fileStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64) { fs.mu.RLock() defer fs.mu.RUnlock() // This can always be last for these purposes. validThrough = fs.state.LastSeq if fs.state.Msgs == 0 || sseq > fs.state.LastSeq { return 0, validThrough } // If sseq is less then our first set to first. if sseq < fs.state.FirstSeq { sseq = fs.state.FirstSeq } // Track starting for both block for the sseq and staring block that matches any subject. var seqStart int // See if we need to figure out starting block per sseq. if sseq > fs.state.FirstSeq { // This should not, but can return -1, so make sure we check to avoid panic below. if seqStart, _ = fs.selectMsgBlockWithIndex(sseq); seqStart < 0 { seqStart = 0 } } isAll := filter == _EMPTY_ || filter == fwcs if isAll && filter == _EMPTY_ { filter = fwcs } wc := subjectHasWildcard(filter) // See if filter was provided but its the only subject. if !isAll && !wc && fs.psim.Size() == 1 { _, isAll = fs.psim.Find(stringToBytes(filter)) } // If we are isAll and have no deleted we can do a simpler calculation. if !lastPerSubject && isAll && (fs.state.LastSeq-fs.state.FirstSeq+1) == fs.state.Msgs { if sseq == 0 { return fs.state.Msgs, validThrough } return fs.state.LastSeq - sseq + 1, validThrough } _tsa, _fsa := [32]string{}, [32]string{} tsa, fsa := _tsa[:0], _fsa[:0] if wc { fsa = tokenizeSubjectIntoSlice(fsa[:0], filter) } isMatch := func(subj string) bool { if isAll { return true } if !wc { return subj == filter } tsa = tokenizeSubjectIntoSlice(tsa[:0], subj) return isSubsetMatchTokenized(tsa, fsa) } // Handle last by subject a bit differently. // We will scan PSIM since we accurately track the last block we have seen the subject in. This // allows us to only need to load at most one block now. // For the last block, we need to track the subjects that we know are in that block, and track seen // while in the block itself, but complexity there worth it. if lastPerSubject { // If we want all and our start sequence is equal or less than first return number of subjects. if isAll && sseq <= fs.state.FirstSeq { return uint64(fs.psim.Size()), validThrough } // If we are here we need to scan. We are going to scan the PSIM looking for lblks that are >= seqStart. // This will build up a list of all subjects from the selected block onward. lbm := make(map[string]bool) mb := fs.blks[seqStart] bi := mb.index fs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) { // If the select blk start is greater than entry's last blk skip. if bi > psi.lblk { return } total++ // We will track the subjects that are an exact match to the last block. // This is needed for last block processing. if psi.lblk == bi { lbm[string(subj)] = true } }) // Now check if we need to inspect the seqStart block. // Grab write lock in case we need to load in msgs. mb.mu.Lock() var updateLLTS bool var shouldExpire bool // We need to walk this block to correct accounting from above. if sseq > mb.first.seq { // Track the ones we add back in case more than one. seen := make(map[string]bool) // We need to discount the total by subjects seen before sseq, but also add them right back in if they are >= sseq for this blk. // This only should be subjects we know have the last blk in this block. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } var smv StoreMsg for seq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). updateLLTS = true continue } sm, _ := mb.cacheLookup(seq, &smv) if sm == nil || sm.subj == _EMPTY_ || !lbm[sm.subj] { continue } updateLLTS = false // cacheLookup already updated it. if isMatch(sm.subj) { // If less than sseq adjust off of total as long as this subject matched the last block. if seq < sseq { if !seen[sm.subj] { total-- seen[sm.subj] = true } } else if seen[sm.subj] { // This is equal or more than sseq, so add back in. total++ // Make sure to not process anymore. delete(seen, sm.subj) } } } } // If we loaded the block try to force expire. if shouldExpire { mb.tryForceExpireCacheLocked() } if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() return total, validThrough } // If we would need to scan more from the beginning, revert back to calculating directly here. // TODO(dlc) - Redo properly with sublists etc for subject-based filtering. if seqStart >= (len(fs.blks) / 2) { for i := seqStart; i < len(fs.blks); i++ { var shouldExpire bool mb := fs.blks[i] // Hold write lock in case we need to load cache. mb.mu.Lock() if isAll && sseq <= atomic.LoadUint64(&mb.first.seq) { total += mb.msgs mb.mu.Unlock() continue } // If we are here we need to at least scan the subject fss. // Make sure we have fss loaded. if mb.fssNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() var t uint64 var havePartial bool mb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) { if havePartial { // If we already found a partial then don't do anything else. return } subj := bytesToString(bsubj) if ss.firstNeedsUpdate || ss.lastNeedsUpdate { mb.recalculateForSubj(subj, ss) } if sseq <= ss.First { t += ss.Msgs } else if sseq <= ss.Last { // We matched but its a partial. havePartial = true } }) // See if we need to scan msgs here. if havePartial { // Make sure we have the cache loaded. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Clear on partial. t = 0 start := sseq if fseq := atomic.LoadUint64(&mb.first.seq); fseq > start { start = fseq } var smv StoreMsg for seq, lseq := start, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ { if sm, _ := mb.cacheLookup(seq, &smv); sm != nil && isMatch(sm.subj) { t++ } } } // If we loaded this block for this operation go ahead and expire it here. if shouldExpire { mb.tryForceExpireCacheLocked() } mb.mu.Unlock() total += t } return total, validThrough } // If we are here it's better to calculate totals from psim and adjust downward by scanning less blocks. // TODO(dlc) - Eventually when sublist uses generics, make this sublist driven instead. start := uint32(math.MaxUint32) fs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) { total += psi.total // Keep track of start index for this subject. if psi.fblk < start { start = psi.fblk } }) // See if we were asked for all, if so we are done. if sseq <= fs.state.FirstSeq { return total, validThrough } // If we are here we need to calculate partials for the first blocks. firstSubjBlk := fs.bim[start] var firstSubjBlkFound bool // Adjust in case not found. if firstSubjBlk == nil { firstSubjBlkFound = true } // Track how many we need to adjust against the total. var adjust uint64 for i := 0; i <= seqStart; i++ { mb := fs.blks[i] // We can skip blks if we know they are below the first one that has any subject matches. if !firstSubjBlkFound { if firstSubjBlkFound = (mb == firstSubjBlk); !firstSubjBlkFound { continue } } // We need to scan this block. var shouldExpire bool var updateLLTS bool mb.mu.Lock() // Check if we should include all of this block in adjusting. If so work with metadata. if sseq > atomic.LoadUint64(&mb.last.seq) { if isAll { adjust += mb.msgs } else { // We need to adjust for all matches in this block. // Make sure we have fss loaded. This loads whole block now. if mb.fssNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() mb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) { adjust += ss.Msgs }) } } else { // This is the last block. We need to scan per message here. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } var last = atomic.LoadUint64(&mb.last.seq) if sseq < last { last = sseq } // We need to walk all messages in this block var smv StoreMsg for seq := atomic.LoadUint64(&mb.first.seq); seq < last; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). updateLLTS = true continue } sm, _ := mb.cacheLookup(seq, &smv) if sm == nil || sm.subj == _EMPTY_ { continue } updateLLTS = false // cacheLookup already updated it. // Check if it matches our filter. if sm.seq < sseq && isMatch(sm.subj) { adjust++ } } } // If we loaded the block try to force expire. if shouldExpire { mb.tryForceExpireCacheLocked() } if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() } // Make final adjustment. total -= adjust return total, validThrough } // NumPending will return the number of pending messages matching any subject in the sublist starting at sequence. // Optimized for stream num pending calculations for consumers with lots of filtered subjects. // Subjects should not overlap, this property is held when doing multi-filtered consumers. func (fs *fileStore) NumPendingMulti(sseq uint64, sl *Sublist, lastPerSubject bool) (total, validThrough uint64) { fs.mu.RLock() defer fs.mu.RUnlock() // This can always be last for these purposes. validThrough = fs.state.LastSeq if fs.state.Msgs == 0 || sseq > fs.state.LastSeq { return 0, validThrough } // If sseq is less then our first set to first. if sseq < fs.state.FirstSeq { sseq = fs.state.FirstSeq } // Track starting for both block for the sseq and staring block that matches any subject. var seqStart int // See if we need to figure out starting block per sseq. if sseq > fs.state.FirstSeq { // This should not, but can return -1, so make sure we check to avoid panic below. if seqStart, _ = fs.selectMsgBlockWithIndex(sseq); seqStart < 0 { seqStart = 0 } } isAll := sl == nil // See if filter was provided but its the only subject. if !isAll && fs.psim.Size() == 1 { fs.psim.IterFast(func(subject []byte, _ *psi) bool { isAll = sl.HasInterest(bytesToString(subject)) return true }) } // If we are isAll and have no deleted we can do a simpler calculation. if !lastPerSubject && isAll && (fs.state.LastSeq-fs.state.FirstSeq+1) == fs.state.Msgs { if sseq == 0 { return fs.state.Msgs, validThrough } return fs.state.LastSeq - sseq + 1, validThrough } // Setup the isMatch function. isMatch := func(subj string) bool { if isAll { return true } return sl.HasInterest(subj) } // Handle last by subject a bit differently. // We will scan PSIM since we accurately track the last block we have seen the subject in. This // allows us to only need to load at most one block now. // For the last block, we need to track the subjects that we know are in that block, and track seen // while in the block itself, but complexity there worth it. if lastPerSubject { // If we want all and our start sequence is equal or less than first return number of subjects. if isAll && sseq <= fs.state.FirstSeq { return uint64(fs.psim.Size()), validThrough } // If we are here we need to scan. We are going to scan the PSIM looking for lblks that are >= seqStart. // This will build up a list of all subjects from the selected block onward. lbm := make(map[string]bool) mb := fs.blks[seqStart] bi := mb.index subs := make([]*subscription, 0, sl.Count()) sl.All(&subs) for _, sub := range subs { fs.psim.Match(sub.subject, func(subj []byte, psi *psi) { // If the select blk start is greater than entry's last blk skip. if bi > psi.lblk { return } total++ // We will track the subjects that are an exact match to the last block. // This is needed for last block processing. if psi.lblk == bi { lbm[string(subj)] = true } }) } // Now check if we need to inspect the seqStart block. // Grab write lock in case we need to load in msgs. mb.mu.Lock() var shouldExpire bool var updateLLTS bool // We need to walk this block to correct accounting from above. if sseq > mb.first.seq { // Track the ones we add back in case more than one. seen := make(map[string]bool) // We need to discount the total by subjects seen before sseq, but also add them right back in if they are >= sseq for this blk. // This only should be subjects we know have the last blk in this block. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } var smv StoreMsg for seq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). updateLLTS = true continue } sm, _ := mb.cacheLookup(seq, &smv) if sm == nil || sm.subj == _EMPTY_ || !lbm[sm.subj] { continue } updateLLTS = false // cacheLookup already updated it. if isMatch(sm.subj) { // If less than sseq adjust off of total as long as this subject matched the last block. if seq < sseq { if !seen[sm.subj] { total-- seen[sm.subj] = true } } else if seen[sm.subj] { // This is equal or more than sseq, so add back in. total++ // Make sure to not process anymore. delete(seen, sm.subj) } } } } // If we loaded the block try to force expire. if shouldExpire { mb.tryForceExpireCacheLocked() } if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() return total, validThrough } // If we would need to scan more from the beginning, revert back to calculating directly here. if seqStart >= (len(fs.blks) / 2) { for i := seqStart; i < len(fs.blks); i++ { var shouldExpire bool mb := fs.blks[i] // Hold write lock in case we need to load cache. mb.mu.Lock() if isAll && sseq <= atomic.LoadUint64(&mb.first.seq) { total += mb.msgs mb.mu.Unlock() continue } // If we are here we need to at least scan the subject fss. // Make sure we have fss loaded. if mb.fssNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() var t uint64 var havePartial bool var updateLLTS bool IntersectStree[SimpleState](mb.fss, sl, func(bsubj []byte, ss *SimpleState) { subj := bytesToString(bsubj) if havePartial { // If we already found a partial then don't do anything else. return } if ss.firstNeedsUpdate || ss.lastNeedsUpdate { mb.recalculateForSubj(subj, ss) } if sseq <= ss.First { t += ss.Msgs } else if sseq <= ss.Last { // We matched but its a partial. havePartial = true } }) // See if we need to scan msgs here. if havePartial { // Make sure we have the cache loaded. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Clear on partial. t = 0 start := sseq if fseq := atomic.LoadUint64(&mb.first.seq); fseq > start { start = fseq } var smv StoreMsg for seq, lseq := start, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). updateLLTS = true continue } if sm, _ := mb.cacheLookup(seq, &smv); sm != nil && isMatch(sm.subj) { t++ updateLLTS = false // cacheLookup already updated it. } } } // If we loaded this block for this operation go ahead and expire it here. if shouldExpire { mb.tryForceExpireCacheLocked() } if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() total += t } return total, validThrough } // If we are here it's better to calculate totals from psim and adjust downward by scanning less blocks. start := uint32(math.MaxUint32) subs := make([]*subscription, 0, sl.Count()) sl.All(&subs) for _, sub := range subs { fs.psim.Match(sub.subject, func(_ []byte, psi *psi) { total += psi.total // Keep track of start index for this subject. if psi.fblk < start { start = psi.fblk } }) } // See if we were asked for all, if so we are done. if sseq <= fs.state.FirstSeq { return total, validThrough } // If we are here we need to calculate partials for the first blocks. firstSubjBlk := fs.bim[start] var firstSubjBlkFound bool // Adjust in case not found. if firstSubjBlk == nil { firstSubjBlkFound = true } // Track how many we need to adjust against the total. var adjust uint64 for i := 0; i <= seqStart; i++ { mb := fs.blks[i] // We can skip blks if we know they are below the first one that has any subject matches. if !firstSubjBlkFound { if firstSubjBlkFound = (mb == firstSubjBlk); !firstSubjBlkFound { continue } } // We need to scan this block. var shouldExpire bool var updateLLTS bool mb.mu.Lock() // Check if we should include all of this block in adjusting. If so work with metadata. if sseq > atomic.LoadUint64(&mb.last.seq) { if isAll { adjust += mb.msgs } else { // We need to adjust for all matches in this block. // Make sure we have fss loaded. This loads whole block now. if mb.fssNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() IntersectStree(mb.fss, sl, func(bsubj []byte, ss *SimpleState) { adjust += ss.Msgs }) } } else { // This is the last block. We need to scan per message here. if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } var last = atomic.LoadUint64(&mb.last.seq) if sseq < last { last = sseq } // We need to walk all messages in this block var smv StoreMsg for seq := atomic.LoadUint64(&mb.first.seq); seq < last; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). updateLLTS = true continue } sm, _ := mb.cacheLookup(seq, &smv) if sm == nil || sm.subj == _EMPTY_ { continue } updateLLTS = false // cacheLookup already updated it. // Check if it matches our filter. if sm.seq < sseq && isMatch(sm.subj) { adjust++ } } } // If we loaded the block try to force expire. if shouldExpire { mb.tryForceExpireCacheLocked() } if updateLLTS { mb.llts = time.Now().UnixNano() } mb.mu.Unlock() } // Make final adjustment. total -= adjust return total, validThrough } // SubjectsTotal return message totals per subject. func (fs *fileStore) SubjectsTotals(filter string) map[string]uint64 { fs.mu.RLock() defer fs.mu.RUnlock() if fs.psim.Size() == 0 { return nil } // Match all if no filter given. if filter == _EMPTY_ { filter = fwcs } fst := make(map[string]uint64) fs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) { fst[string(subj)] = psi.total }) return fst } // RegisterStorageUpdates registers a callback for updates to storage changes. // It will present number of messages and bytes as a signed integer and an // optional sequence number of the message if a single. func (fs *fileStore) RegisterStorageUpdates(cb StorageUpdateHandler) { fs.mu.Lock() fs.scb = cb bsz := fs.state.Bytes fs.mu.Unlock() if cb != nil && bsz > 0 { cb(0, int64(bsz), 0, _EMPTY_) } } // Helper to get hash key for specific message block. // Lock should be held func (fs *fileStore) hashKeyForBlock(index uint32) []byte { return []byte(fmt.Sprintf("%s-%d", fs.cfg.Name, index)) } func (mb *msgBlock) setupWriteCache(buf []byte) { // Make sure we have a cache setup. if mb.cache != nil { return } // Setup simple cache. mb.cache = &cache{buf: buf} // Make sure we set the proper cache offset if we have existing data. var fi os.FileInfo if mb.mfd != nil { fi, _ = mb.mfd.Stat() } else if mb.mfn != _EMPTY_ { fi, _ = os.Stat(mb.mfn) } if fi != nil { mb.cache.off = int(fi.Size()) } mb.llts = time.Now().UnixNano() mb.startCacheExpireTimer() } // This rolls to a new append msg block. // Lock should be held. func (fs *fileStore) newMsgBlockForWrite() (*msgBlock, error) { index := uint32(1) var rbuf []byte if lmb := fs.lmb; lmb != nil { index = lmb.index + 1 // Determine if we can reclaim any resources here. if fs.fip { lmb.mu.Lock() lmb.closeFDsLocked() if lmb.cache != nil { // Reset write timestamp and see if we can expire this cache. rbuf = lmb.tryExpireWriteCache() } lmb.mu.Unlock() } } mb := fs.initMsgBlock(index) // Lock should be held to quiet race detector. mb.mu.Lock() mb.setupWriteCache(rbuf) mb.fss = stree.NewSubjectTree[SimpleState]() // Set cache time to creation time to start. ts := time.Now().UnixNano() mb.llts, mb.lwts = 0, ts // Remember our last sequence number. atomic.StoreUint64(&mb.first.seq, fs.state.LastSeq+1) atomic.StoreUint64(&mb.last.seq, fs.state.LastSeq) mb.mu.Unlock() // Now do local hash. key := sha256.Sum256(fs.hashKeyForBlock(index)) hh, err := highwayhash.New64(key[:]) if err != nil { return nil, fmt.Errorf("could not create hash: %v", err) } mb.hh = hh <-dios mfd, err := os.OpenFile(mb.mfn, os.O_CREATE|os.O_RDWR, defaultFilePerms) dios <- struct{}{} if err != nil { if isPermissionError(err) { return nil, err } mb.dirtyCloseWithRemove(true) return nil, fmt.Errorf("Error creating msg block file: %v", err) } mb.mfd = mfd // Check if encryption is enabled. if fs.prf != nil { if err := fs.genEncryptionKeysForBlock(mb); err != nil { return nil, err } } // If we know we will need this so go ahead and spin up. if !fs.fip { mb.spinUpFlushLoop() } // Add to our list of blocks and mark as last. fs.addMsgBlock(mb) return mb, nil } // Generate the keys for this message block and write them out. func (fs *fileStore) genEncryptionKeysForBlock(mb *msgBlock) error { if mb == nil { return nil } key, bek, seed, encrypted, err := fs.genEncryptionKeys(fmt.Sprintf("%s:%d", fs.cfg.Name, mb.index)) if err != nil { return err } mb.aek, mb.bek, mb.seed, mb.nonce = key, bek, seed, encrypted[:key.NonceSize()] mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) keyFile := filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index)) if _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) { return err } err = fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms) if err != nil { return err } mb.kfn = keyFile return nil } // Stores a raw message with expected sequence number and timestamp. // Lock should be held. func (fs *fileStore) storeRawMsg(subj string, hdr, msg []byte, seq uint64, ts int64) (err error) { if fs.closed { return ErrStoreClosed } // Per subject max check needed. mmp := uint64(fs.cfg.MaxMsgsPer) var psmc uint64 psmax := mmp > 0 && len(subj) > 0 if psmax { if info, ok := fs.psim.Find(stringToBytes(subj)); ok { psmc = info.total } } var fseq uint64 // Check if we are discarding new messages when we reach the limit. if fs.cfg.Discard == DiscardNew { var asl bool if psmax && psmc >= mmp { // If we are instructed to discard new per subject, this is an error. if fs.cfg.DiscardNewPer { return ErrMaxMsgsPerSubject } if fseq, err = fs.firstSeqForSubj(subj); err != nil { return err } asl = true } // If we are discard new and limits policy and clustered, we do the enforcement // above and should not disqualify the message here since it could cause replicas to drift. if fs.cfg.Retention == LimitsPolicy || fs.cfg.Replicas == 1 { if fs.cfg.MaxMsgs > 0 && fs.state.Msgs >= uint64(fs.cfg.MaxMsgs) && !asl { return ErrMaxMsgs } if fs.cfg.MaxBytes > 0 && fs.state.Bytes+fileStoreMsgSize(subj, hdr, msg) >= uint64(fs.cfg.MaxBytes) { if !asl || fs.sizeForSeq(fseq) <= int(fileStoreMsgSize(subj, hdr, msg)) { return ErrMaxBytes } } } } // Check sequence. if seq != fs.state.LastSeq+1 { if seq > 0 { return ErrSequenceMismatch } seq = fs.state.LastSeq + 1 } // Write msg record. n, err := fs.writeMsgRecord(seq, ts, subj, hdr, msg) if err != nil { return err } // Adjust top level tracking of per subject msg counts. if len(subj) > 0 && fs.psim != nil { index := fs.lmb.index if info, ok := fs.psim.Find(stringToBytes(subj)); ok { info.total++ if index > info.lblk { info.lblk = index } } else { fs.psim.Insert(stringToBytes(subj), psi{total: 1, fblk: index, lblk: index}) fs.tsl += len(subj) } } // Adjust first if needed. now := time.Unix(0, ts).UTC() if fs.state.Msgs == 0 { fs.state.FirstSeq = seq fs.state.FirstTime = now } fs.state.Msgs++ fs.state.Bytes += n fs.state.LastSeq = seq fs.state.LastTime = now // Enforce per message limits. // We snapshotted psmc before our actual write, so >= comparison needed. if psmax && psmc >= mmp { // We may have done this above. if fseq == 0 { fseq, _ = fs.firstSeqForSubj(subj) } if ok, _ := fs.removeMsgViaLimits(fseq); ok { // Make sure we are below the limit. if psmc--; psmc >= mmp { bsubj := stringToBytes(subj) for info, ok := fs.psim.Find(bsubj); ok && info.total > mmp; info, ok = fs.psim.Find(bsubj) { if seq, _ := fs.firstSeqForSubj(subj); seq > 0 { if ok, _ := fs.removeMsgViaLimits(seq); !ok { break } } else { break } } } } else if mb := fs.selectMsgBlock(fseq); mb != nil { // If we are here we could not remove fseq from above, so rebuild. var ld *LostStreamData if ld, _, _ = mb.rebuildState(); ld != nil { fs.rebuildStateLocked(ld) } } } // Limits checks and enforcement. // If they do any deletions they will update the // byte count on their own, so no need to compensate. fs.enforceMsgLimit() fs.enforceBytesLimit() // Check if we have and need the age expiration timer running. if fs.ageChk == nil && fs.cfg.MaxAge != 0 { fs.startAgeChk() } return nil } // StoreRawMsg stores a raw message with expected sequence number and timestamp. func (fs *fileStore) StoreRawMsg(subj string, hdr, msg []byte, seq uint64, ts int64) error { fs.mu.Lock() err := fs.storeRawMsg(subj, hdr, msg, seq, ts) cb := fs.scb // Check if first message timestamp requires expiry // sooner than initial replica expiry timer set to MaxAge when initializing. if !fs.receivedAny && fs.cfg.MaxAge != 0 && ts > 0 { fs.receivedAny = true // don't block here by calling expireMsgs directly. // Instead, set short timeout. fs.resetAgeChk(int64(time.Millisecond * 50)) } fs.mu.Unlock() if err == nil && cb != nil { cb(1, int64(fileStoreMsgSize(subj, hdr, msg)), seq, subj) } return err } // Store stores a message. We hold the main filestore lock for any write operation. func (fs *fileStore) StoreMsg(subj string, hdr, msg []byte) (uint64, int64, error) { fs.mu.Lock() seq, ts := fs.state.LastSeq+1, time.Now().UnixNano() err := fs.storeRawMsg(subj, hdr, msg, seq, ts) cb := fs.scb fs.mu.Unlock() if err != nil { seq, ts = 0, 0 } else if cb != nil { cb(1, int64(fileStoreMsgSize(subj, hdr, msg)), seq, subj) } return seq, ts, err } // skipMsg will update this message block for a skipped message. // If we do not have any messages, just update the metadata, otherwise // we will place an empty record marking the sequence as used. The // sequence will be marked erased. // fs lock should be held. func (mb *msgBlock) skipMsg(seq uint64, now time.Time) { if mb == nil { return } var needsRecord bool nowts := now.UnixNano() mb.mu.Lock() // If we are empty can just do meta. if mb.msgs == 0 { atomic.StoreUint64(&mb.last.seq, seq) mb.last.ts = nowts atomic.StoreUint64(&mb.first.seq, seq+1) mb.first.ts = nowts needsRecord = mb == mb.fs.lmb if needsRecord && mb.rbytes > 0 { // We want to make sure since we have no messages // that we write to the beginning since we only need last one. mb.rbytes, mb.cache = 0, &cache{} // If encrypted we need to reset counter since we just keep one. if mb.bek != nil { // Recreate to reset counter. mb.bek, _ = genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) } } } else { needsRecord = true mb.dmap.Insert(seq) } mb.mu.Unlock() if needsRecord { mb.writeMsgRecord(emptyRecordLen, seq|ebit, _EMPTY_, nil, nil, nowts, true) } else { mb.kickFlusher() } } // SkipMsg will use the next sequence number but not store anything. func (fs *fileStore) SkipMsg() uint64 { fs.mu.Lock() defer fs.mu.Unlock() // Grab our current last message block. mb := fs.lmb if mb == nil || mb.msgs > 0 && mb.blkSize()+emptyRecordLen > fs.fcfg.BlockSize { if mb != nil && fs.fcfg.Compression != NoCompression { // We've now reached the end of this message block, if we want // to compress blocks then now's the time to do it. go mb.recompressOnDiskIfNeeded() } var err error if mb, err = fs.newMsgBlockForWrite(); err != nil { return 0 } } // Grab time and last seq. now, seq := time.Now().UTC(), fs.state.LastSeq+1 // Write skip msg. mb.skipMsg(seq, now) // Update fs state. fs.state.LastSeq, fs.state.LastTime = seq, now if fs.state.Msgs == 0 { fs.state.FirstSeq, fs.state.FirstTime = seq, now } if seq == fs.state.FirstSeq { fs.state.FirstSeq, fs.state.FirstTime = seq+1, now } // Mark as dirty for stream state. fs.dirty++ return seq } // Skip multiple msgs. We will determine if we can fit into current lmb or we need to create a new block. func (fs *fileStore) SkipMsgs(seq uint64, num uint64) error { fs.mu.Lock() defer fs.mu.Unlock() // Check sequence matches our last sequence. if seq != fs.state.LastSeq+1 { if seq > 0 { return ErrSequenceMismatch } seq = fs.state.LastSeq + 1 } // Limit number of dmap entries const maxDeletes = 64 * 1024 mb := fs.lmb numDeletes := int(num) if mb != nil { numDeletes += mb.dmap.Size() } if mb == nil || numDeletes > maxDeletes && mb.msgs > 0 || mb.msgs > 0 && mb.blkSize()+emptyRecordLen > fs.fcfg.BlockSize { if mb != nil && fs.fcfg.Compression != NoCompression { // We've now reached the end of this message block, if we want // to compress blocks then now's the time to do it. go mb.recompressOnDiskIfNeeded() } var err error if mb, err = fs.newMsgBlockForWrite(); err != nil { return err } } // Insert into dmap all entries and place last as marker. now := time.Now().UTC() nowts := now.UnixNano() lseq := seq + num - 1 mb.mu.Lock() // If we are empty update meta directly. if mb.msgs == 0 { atomic.StoreUint64(&mb.last.seq, lseq) mb.last.ts = nowts atomic.StoreUint64(&mb.first.seq, lseq+1) mb.first.ts = nowts } else { for ; seq <= lseq; seq++ { mb.dmap.Insert(seq) } } mb.mu.Unlock() // Write out our placeholder. mb.writeMsgRecord(emptyRecordLen, lseq|ebit, _EMPTY_, nil, nil, nowts, true) // Now update FS accounting. // Update fs state. fs.state.LastSeq, fs.state.LastTime = lseq, now if fs.state.Msgs == 0 { fs.state.FirstSeq, fs.state.FirstTime = lseq+1, now } // Mark as dirty for stream state. fs.dirty++ return nil } // Lock should be held. func (fs *fileStore) rebuildFirst() { if len(fs.blks) == 0 { return } fmb := fs.blks[0] if fmb == nil { return } ld, _, _ := fmb.rebuildState() fmb.mu.RLock() isEmpty := fmb.msgs == 0 fmb.mu.RUnlock() if isEmpty { fmb.mu.Lock() fs.removeMsgBlock(fmb) fmb.mu.Unlock() } fs.selectNextFirst() fs.rebuildStateLocked(ld) } // Optimized helper function to return first sequence. // subj will always be publish subject here, meaning non-wildcard. // We assume a fast check that this subj even exists already happened. // Write lock should be held. func (fs *fileStore) firstSeqForSubj(subj string) (uint64, error) { if len(fs.blks) == 0 { return 0, nil } // See if we can optimize where we start. start, stop := fs.blks[0].index, fs.lmb.index if info, ok := fs.psim.Find(stringToBytes(subj)); ok { start, stop = info.fblk, info.lblk } for i := start; i <= stop; i++ { mb := fs.bim[i] if mb == nil { continue } // If we need to load msgs here and we need to walk multiple blocks this // could tie up the upper fs lock, so release while dealing with the block. fs.mu.Unlock() mb.mu.Lock() var shouldExpire bool if mb.fssNotLoaded() { // Make sure we have fss loaded. if err := mb.loadMsgsWithLock(); err != nil { mb.mu.Unlock() // Re-acquire fs lock fs.mu.Lock() return 0, err } shouldExpire = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() bsubj := stringToBytes(subj) if ss, ok := mb.fss.Find(bsubj); ok && ss != nil { // Adjust first if it was not where we thought it should be. if i != start { if info, ok := fs.psim.Find(bsubj); ok { info.fblk = i } } if ss.firstNeedsUpdate || ss.lastNeedsUpdate { mb.recalculateForSubj(subj, ss) } mb.mu.Unlock() // Re-acquire fs lock fs.mu.Lock() return ss.First, nil } // If we did not find it and we loaded this msgBlock try to expire as long as not the last. if shouldExpire { // Expire this cache before moving on. mb.tryForceExpireCacheLocked() } mb.mu.Unlock() // Re-acquire fs lock fs.mu.Lock() } return 0, nil } // Will check the msg limit and drop firstSeq msg if needed. // Lock should be held. func (fs *fileStore) enforceMsgLimit() { if fs.cfg.Discard != DiscardOld { return } if fs.cfg.MaxMsgs <= 0 || fs.state.Msgs <= uint64(fs.cfg.MaxMsgs) { return } for nmsgs := fs.state.Msgs; nmsgs > uint64(fs.cfg.MaxMsgs); nmsgs = fs.state.Msgs { if removed, err := fs.deleteFirstMsg(); err != nil || !removed { fs.rebuildFirst() return } } } // Will check the bytes limit and drop msgs if needed. // Lock should be held. func (fs *fileStore) enforceBytesLimit() { if fs.cfg.Discard != DiscardOld { return } if fs.cfg.MaxBytes <= 0 || fs.state.Bytes <= uint64(fs.cfg.MaxBytes) { return } for bs := fs.state.Bytes; bs > uint64(fs.cfg.MaxBytes); bs = fs.state.Bytes { if removed, err := fs.deleteFirstMsg(); err != nil || !removed { fs.rebuildFirst() return } } } // Will make sure we have limits honored for max msgs per subject on recovery or config update. // We will make sure to go through all msg blocks etc. but in practice this // will most likely only be the last one, so can take a more conservative approach. // Lock should be held. func (fs *fileStore) enforceMsgPerSubjectLimit(fireCallback bool) { start := time.Now() defer func() { if took := time.Since(start); took > time.Minute { fs.warn("enforceMsgPerSubjectLimit took %v", took.Round(time.Millisecond)) } }() maxMsgsPer := uint64(fs.cfg.MaxMsgsPer) // We may want to suppress callbacks from remove during this process // since these should have already been deleted and accounted for. if !fireCallback { cb := fs.scb fs.scb = nil defer func() { fs.scb = cb }() } var numMsgs uint64 // collect all that are not correct. needAttention := make(map[string]*psi) fs.psim.IterFast(func(subj []byte, psi *psi) bool { numMsgs += psi.total if psi.total > maxMsgsPer { needAttention[string(subj)] = psi } return true }) // We had an issue with a use case where psim (and hence fss) were correct but idx was not and was not properly being caught. // So do a quick sanity check here. If we detect a skew do a rebuild then re-check. if numMsgs != fs.state.Msgs { fs.warn("Detected skew in subject-based total (%d) vs raw total (%d), rebuilding", numMsgs, fs.state.Msgs) // Clear any global subject state. fs.psim, fs.tsl = fs.psim.Empty(), 0 for _, mb := range fs.blks { ld, _, err := mb.rebuildState() if err != nil && ld != nil { fs.addLostData(ld) } fs.populateGlobalPerSubjectInfo(mb) } // Rebuild fs state too. fs.rebuildStateLocked(nil) // Need to redo blocks that need attention. needAttention = make(map[string]*psi) fs.psim.IterFast(func(subj []byte, psi *psi) bool { if psi.total > maxMsgsPer { needAttention[string(subj)] = psi } return true }) } // Collect all the msgBlks we alter. blks := make(map[*msgBlock]struct{}) // For re-use below. var sm StoreMsg // Walk all subjects that need attention here. for subj, info := range needAttention { total, start, stop := info.total, info.fblk, info.lblk for i := start; i <= stop; i++ { mb := fs.bim[i] if mb == nil { continue } // Grab the ss entry for this subject in case sparse. mb.mu.Lock() mb.ensurePerSubjectInfoLoaded() ss, ok := mb.fss.Find(stringToBytes(subj)) if ok && ss != nil && (ss.firstNeedsUpdate || ss.lastNeedsUpdate) { mb.recalculateForSubj(subj, ss) } mb.mu.Unlock() if ss == nil { continue } for seq := ss.First; seq <= ss.Last && total > maxMsgsPer; { m, _, err := mb.firstMatching(subj, false, seq, &sm) if err == nil { seq = m.seq + 1 if removed, _ := fs.removeMsgViaLimits(m.seq); removed { total-- blks[mb] = struct{}{} } } else { // On error just do single increment. seq++ } } } } // Expire the cache if we can. for mb := range blks { mb.mu.Lock() if mb.msgs > 0 { mb.tryForceExpireCacheLocked() } mb.mu.Unlock() } } // Lock should be held. func (fs *fileStore) deleteFirstMsg() (bool, error) { return fs.removeMsgViaLimits(fs.state.FirstSeq) } // If we remove via limits that can always be recovered on a restart we // do not force the system to update the index file. // Lock should be held. func (fs *fileStore) removeMsgViaLimits(seq uint64) (bool, error) { return fs.removeMsg(seq, false, true, false) } // RemoveMsg will remove the message from this store. // Will return the number of bytes removed. func (fs *fileStore) RemoveMsg(seq uint64) (bool, error) { return fs.removeMsg(seq, false, false, true) } func (fs *fileStore) EraseMsg(seq uint64) (bool, error) { return fs.removeMsg(seq, true, false, true) } // Convenience function to remove per subject tracking at the filestore level. // Lock should be held. func (fs *fileStore) removePerSubject(subj string) { if len(subj) == 0 || fs.psim == nil { return } // We do not update sense of fblk here but will do so when we resolve during lookup. bsubj := stringToBytes(subj) if info, ok := fs.psim.Find(bsubj); ok { info.total-- if info.total == 1 { info.fblk = info.lblk } else if info.total == 0 { if _, ok = fs.psim.Delete(bsubj); ok { fs.tsl -= len(subj) } } } } // Remove a message, optionally rewriting the mb file. func (fs *fileStore) removeMsg(seq uint64, secure, viaLimits, needFSLock bool) (bool, error) { if seq == 0 { return false, ErrStoreMsgNotFound } fsLock := func() { if needFSLock { fs.mu.Lock() } } fsUnlock := func() { if needFSLock { fs.mu.Unlock() } } fsLock() if fs.closed { fsUnlock() return false, ErrStoreClosed } if !viaLimits && fs.sips > 0 { fsUnlock() return false, ErrStoreSnapshotInProgress } // If in encrypted mode negate secure rewrite here. if secure && fs.prf != nil { secure = false } mb := fs.selectMsgBlock(seq) if mb == nil { var err = ErrStoreEOF if seq <= fs.state.LastSeq { err = ErrStoreMsgNotFound } fsUnlock() return false, err } mb.mu.Lock() // See if we are closed or the sequence number is still relevant or if we know its deleted. if mb.closed || seq < atomic.LoadUint64(&mb.first.seq) || mb.dmap.Exists(seq) { mb.mu.Unlock() fsUnlock() return false, nil } // We used to not have to load in the messages except with callbacks or the filtered subject state (which is now always on). // Now just load regardless. // TODO(dlc) - Figure out a way not to have to load it in, we need subject tracking outside main data block. if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { mb.mu.Unlock() fsUnlock() return false, err } } var smv StoreMsg sm, err := mb.cacheLookup(seq, &smv) if err != nil { mb.mu.Unlock() fsUnlock() // Mimic err behavior from above check to dmap. No error returned if already removed. if err == errDeletedMsg { err = nil } return false, err } // Grab size msz := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg) // Set cache timestamp for last remove. mb.lrts = time.Now().UnixNano() // Global stats if fs.state.Msgs > 0 { fs.state.Msgs-- } if msz < fs.state.Bytes { fs.state.Bytes -= msz } else { fs.state.Bytes = 0 } // Now local mb updates. if mb.msgs > 0 { mb.msgs-- } if msz < mb.bytes { mb.bytes -= msz } else { mb.bytes = 0 } // Allow us to check compaction again. mb.noCompact = false // Mark as dirty for stream state. fs.dirty++ // If we are tracking subjects here make sure we update that accounting. mb.ensurePerSubjectInfoLoaded() // If we are tracking multiple subjects here make sure we update that accounting. mb.removeSeqPerSubject(sm.subj, seq) fs.removePerSubject(sm.subj) if secure { // Grab record info. ri, rl, _, _ := mb.slotInfo(int(seq - mb.cache.fseq)) if err := mb.eraseMsg(seq, int(ri), int(rl)); err != nil { return false, err } } fifo := seq == atomic.LoadUint64(&mb.first.seq) isLastBlock := mb == fs.lmb isEmpty := mb.msgs == 0 if fifo { mb.selectNextFirst() if !isEmpty { // Can update this one in place. if seq == fs.state.FirstSeq { fs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq) // new one. if mb.first.ts == 0 { fs.state.FirstTime = time.Time{} } else { fs.state.FirstTime = time.Unix(0, mb.first.ts).UTC() } } } } else if !isEmpty { // Out of order delete. mb.dmap.Insert(seq) // Make simple check here similar to Compact(). If we can save 50% and over a certain threshold do inline. // All other more thorough cleanup will happen in syncBlocks logic. // Note that we do not have to store empty records for the deleted, so don't use to calculate. // TODO(dlc) - This should not be inline, should kick the sync routine. if !isLastBlock && mb.shouldCompactInline() { mb.compact() } } if secure { if ld, _ := mb.flushPendingMsgsLocked(); ld != nil { // We have the mb lock here, this needs the mb locks so do in its own go routine. go fs.rebuildState(ld) } } // If empty remove this block and check if we need to update first sequence. // We will write a tombstone at the end. var firstSeqNeedsUpdate bool if isEmpty { // This writes tombstone iff mb == lmb, so no need to do below. fs.removeMsgBlock(mb) firstSeqNeedsUpdate = seq == fs.state.FirstSeq } mb.mu.Unlock() // If we emptied the current message block and the seq was state.FirstSeq // then we need to jump message blocks. We will also write the index so // we don't lose track of the first sequence. if firstSeqNeedsUpdate { fs.selectNextFirst() } // Check if we need to write a deleted record tombstone. // This is for user initiated removes or to hold the first seq // when the last block is empty. // If not via limits and not empty (empty writes tombstone above if last) write tombstone. if !viaLimits && !isEmpty && sm != nil { fs.writeTombstone(sm.seq, sm.ts) } if cb := fs.scb; cb != nil { // If we have a callback registered we need to release lock regardless since cb might need it to lookup msg, etc. fs.mu.Unlock() // Storage updates. var subj string if sm != nil { subj = sm.subj } delta := int64(msz) cb(-1, -delta, seq, subj) if !needFSLock { fs.mu.Lock() } } else if needFSLock { // We acquired it so release it. fs.mu.Unlock() } return true, nil } // Tests whether we should try to compact this block while inline removing msgs. // We will want rbytes to be over the minimum and have a 2x potential savings. // Lock should be held. func (mb *msgBlock) shouldCompactInline() bool { return mb.rbytes > compactMinimum && mb.bytes*2 < mb.rbytes } // Tests whether we should try to compact this block while running periodic sync. // We will want rbytes to be over the minimum and have a 2x potential savings. // Ignores 2MB minimum. // Lock should be held. func (mb *msgBlock) shouldCompactSync() bool { return mb.bytes*2 < mb.rbytes && !mb.noCompact } // This will compact and rewrite this block. This version will not process any tombstone cleanup. // Write lock needs to be held. func (mb *msgBlock) compact() { mb.compactWithFloor(0) } // This will compact and rewrite this block. This should only be called when we know we want to rewrite this block. // This should not be called on the lmb since we will prune tail deleted messages which could cause issues with // writing new messages. We will silently bail on any issues with the underlying block and let someone else detect. // if fseq > 0 we will attempt to cleanup stale tombstones. // Write lock needs to be held. func (mb *msgBlock) compactWithFloor(floor uint64) { wasLoaded := mb.cacheAlreadyLoaded() if !wasLoaded { if err := mb.loadMsgsWithLock(); err != nil { return } } buf := mb.cache.buf nbuf := getMsgBlockBuf(len(buf)) // Recycle our nbuf when we are done. defer recycleMsgBlockBuf(nbuf) var le = binary.LittleEndian var firstSet bool fseq := atomic.LoadUint64(&mb.first.seq) isDeleted := func(seq uint64) bool { return seq == 0 || seq&ebit != 0 || mb.dmap.Exists(seq) || seq < fseq } for index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; { if index+msgHdrSize > lbuf { return } hdr := buf[index : index+msgHdrSize] rl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:])) // Clear any headers bit that could be set. rl &^= hbit dlen := int(rl) - msgHdrSize // Do some quick sanity checks here. if dlen < 0 || slen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh { return } // Only need to process non-deleted messages. seq := le.Uint64(hdr[4:]) if !isDeleted(seq) { // Check for tombstones. if seq&tbit != 0 { seq = seq &^ tbit // If this entry is for a lower seq than ours then keep around. // We also check that it is greater than our floor. Floor is zero on normal // calls to compact. if seq < fseq && seq >= floor { nbuf = append(nbuf, buf[index:index+rl]...) } } else { // Normal message here. nbuf = append(nbuf, buf[index:index+rl]...) if !firstSet { firstSet = true atomic.StoreUint64(&mb.first.seq, seq) } } } // Advance to next record. index += rl } // Handle compression if mb.cmp != NoCompression && len(nbuf) > 0 { cbuf, err := mb.cmp.Compress(nbuf) if err != nil { return } meta := &CompressionInfo{ Algorithm: mb.cmp, OriginalSize: uint64(len(nbuf)), } nbuf = append(meta.MarshalMetadata(), cbuf...) } // Check for encryption. if mb.bek != nil && len(nbuf) > 0 { // Recreate to reset counter. rbek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return } rbek.XORKeyStream(nbuf, nbuf) } // Close FDs first. mb.closeFDsLocked() // We will write to a new file and mv/rename it in case of failure. mfn := filepath.Join(mb.fs.fcfg.StoreDir, msgDir, fmt.Sprintf(newScan, mb.index)) <-dios err := os.WriteFile(mfn, nbuf, defaultFilePerms) dios <- struct{}{} if err != nil { os.Remove(mfn) return } if err := os.Rename(mfn, mb.mfn); err != nil { os.Remove(mfn) return } // Make sure to sync mb.needSync = true // Capture the updated rbytes. if rbytes := uint64(len(nbuf)); rbytes == mb.rbytes { // No change, so set our noCompact bool here to avoid attempting to continually compress in syncBlocks. mb.noCompact = true } else { mb.rbytes = rbytes } // Remove any seqs from the beginning of the blk. for seq, nfseq := fseq, atomic.LoadUint64(&mb.first.seq); seq < nfseq; seq++ { mb.dmap.Delete(seq) } // Make sure we clear the cache since no longer valid. mb.clearCacheAndOffset() // If we entered with the msgs loaded make sure to reload them. if wasLoaded { mb.loadMsgsWithLock() } } // Grab info from a slot. // Lock should be held. func (mb *msgBlock) slotInfo(slot int) (uint32, uint32, bool, error) { if mb.cache == nil || slot >= len(mb.cache.idx) { return 0, 0, false, errPartialCache } bi := mb.cache.idx[slot] ri, hashChecked := (bi &^ hbit), (bi&hbit) != 0 // If this is a deleted slot return here. if bi == dbit { return 0, 0, false, errDeletedMsg } // Determine record length var rl uint32 if slot >= len(mb.cache.idx) { rl = mb.cache.lrl } else { // Need to account for dbit markers in idx. // So we will walk until we find valid idx slot to calculate rl. for i := 1; slot+i < len(mb.cache.idx); i++ { ni := mb.cache.idx[slot+i] &^ hbit if ni == dbit { continue } rl = ni - ri break } // check if we had all trailing dbits. // If so use len of cache buf minus ri. if rl == 0 { rl = uint32(len(mb.cache.buf)) - ri } } if rl < msgHdrSize { return 0, 0, false, errBadMsg } return uint32(ri), rl, hashChecked, nil } func (fs *fileStore) isClosed() bool { fs.mu.RLock() closed := fs.closed fs.mu.RUnlock() return closed } // Will spin up our flush loop. func (mb *msgBlock) spinUpFlushLoop() { mb.mu.Lock() defer mb.mu.Unlock() // Are we already running or closed? if mb.flusher || mb.closed { return } mb.flusher = true mb.fch = make(chan struct{}, 1) mb.qch = make(chan struct{}) fch, qch := mb.fch, mb.qch go mb.flushLoop(fch, qch) } // Raw low level kicker for flush loops. func kickFlusher(fch chan struct{}) { if fch != nil { select { case fch <- struct{}{}: default: } } } // Kick flusher for this message block. func (mb *msgBlock) kickFlusher() { mb.mu.RLock() defer mb.mu.RUnlock() kickFlusher(mb.fch) } func (mb *msgBlock) setInFlusher() { mb.mu.Lock() mb.flusher = true mb.mu.Unlock() } func (mb *msgBlock) clearInFlusher() { mb.mu.Lock() mb.flusher = false mb.mu.Unlock() } // flushLoop watches for messages, index info, or recently closed msg block updates. func (mb *msgBlock) flushLoop(fch, qch chan struct{}) { mb.setInFlusher() defer mb.clearInFlusher() for { select { case <-fch: // If we have pending messages process them first. if waiting := mb.pendingWriteSize(); waiting != 0 { ts := 1 * time.Millisecond var waited time.Duration for waiting < coalesceMinimum { time.Sleep(ts) select { case <-qch: return default: } newWaiting := mb.pendingWriteSize() if waited = waited + ts; waited > maxFlushWait || newWaiting <= waiting { break } waiting = newWaiting ts *= 2 } mb.flushPendingMsgs() // Check if we are no longer the last message block. If we are // not we can close FDs and exit. mb.fs.mu.RLock() notLast := mb != mb.fs.lmb mb.fs.mu.RUnlock() if notLast { if err := mb.closeFDs(); err == nil { return } } } case <-qch: return } } } // Lock should be held. func (mb *msgBlock) eraseMsg(seq uint64, ri, rl int) error { var le = binary.LittleEndian var hdr [msgHdrSize]byte le.PutUint32(hdr[0:], uint32(rl)) le.PutUint64(hdr[4:], seq|ebit) le.PutUint64(hdr[12:], 0) le.PutUint16(hdr[20:], 0) // Randomize record data := make([]byte, rl-emptyRecordLen) if n, err := rand.Read(data); err != nil { return err } else if n != len(data) { return fmt.Errorf("not enough overwrite bytes read (%d != %d)", n, len(data)) } // Now write to underlying buffer. var b bytes.Buffer b.Write(hdr[:]) b.Write(data) // Calculate hash. mb.hh.Reset() mb.hh.Write(hdr[4:20]) mb.hh.Write(data) checksum := mb.hh.Sum(nil) // Write to msg record. b.Write(checksum) // Update both cache and disk. nbytes := b.Bytes() // Cache if ri >= mb.cache.off { li := ri - mb.cache.off buf := mb.cache.buf[li : li+rl] copy(buf, nbytes) } // Disk if mb.cache.off+mb.cache.wp > ri { <-dios mfd, err := os.OpenFile(mb.mfn, os.O_RDWR, defaultFilePerms) dios <- struct{}{} if err != nil { return err } defer mfd.Close() if _, err = mfd.WriteAt(nbytes, int64(ri)); err == nil { mfd.Sync() } if err != nil { return err } } return nil } // Truncate this message block to the storedMsg. func (mb *msgBlock) truncate(sm *StoreMsg) (nmsgs, nbytes uint64, err error) { mb.mu.Lock() defer mb.mu.Unlock() // Make sure we are loaded to process messages etc. if err := mb.loadMsgsWithLock(); err != nil { return 0, 0, err } // Calculate new eof using slot info from our new last sm. ri, rl, _, err := mb.slotInfo(int(sm.seq - mb.cache.fseq)) if err != nil { return 0, 0, err } // Calculate new eof. eof := int64(ri + rl) var purged, bytes uint64 checkDmap := mb.dmap.Size() > 0 var smv StoreMsg for seq := atomic.LoadUint64(&mb.last.seq); seq > sm.seq; seq-- { if checkDmap { if mb.dmap.Exists(seq) { // Delete and skip to next. mb.dmap.Delete(seq) checkDmap = !mb.dmap.IsEmpty() continue } } // We should have a valid msg to calculate removal stats. if m, err := mb.cacheLookup(seq, &smv); err == nil { if mb.msgs > 0 { rl := fileStoreMsgSize(m.subj, m.hdr, m.msg) mb.msgs-- if rl > mb.bytes { rl = mb.bytes } mb.bytes -= rl mb.rbytes -= rl // For return accounting. purged++ bytes += uint64(rl) } } } // If the block is compressed then we have to load it into memory // and decompress it, truncate it and then write it back out. // Otherwise, truncate the file itself and close the descriptor. if mb.cmp != NoCompression { buf, err := mb.loadBlock(nil) if err != nil { return 0, 0, fmt.Errorf("failed to load block from disk: %w", err) } if mb.bek != nil && len(buf) > 0 { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return 0, 0, err } mb.bek = bek mb.bek.XORKeyStream(buf, buf) } buf, err = mb.decompressIfNeeded(buf) if err != nil { return 0, 0, fmt.Errorf("failed to decompress block: %w", err) } buf = buf[:eof] copy(mb.lchk[0:], buf[:len(buf)-checksumSize]) buf, err = mb.cmp.Compress(buf) if err != nil { return 0, 0, fmt.Errorf("failed to recompress block: %w", err) } meta := &CompressionInfo{ Algorithm: mb.cmp, OriginalSize: uint64(eof), } buf = append(meta.MarshalMetadata(), buf...) if mb.bek != nil && len(buf) > 0 { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return 0, 0, err } mb.bek = bek mb.bek.XORKeyStream(buf, buf) } n, err := mb.writeAt(buf, 0) if err != nil { return 0, 0, fmt.Errorf("failed to rewrite compressed block: %w", err) } if n != len(buf) { return 0, 0, fmt.Errorf("short write (%d != %d)", n, len(buf)) } mb.mfd.Truncate(int64(len(buf))) mb.mfd.Sync() } else if mb.mfd != nil { mb.mfd.Truncate(eof) mb.mfd.Sync() // Update our checksum. var lchk [8]byte mb.mfd.ReadAt(lchk[:], eof-8) copy(mb.lchk[0:], lchk[:]) } else { return 0, 0, fmt.Errorf("failed to truncate msg block %d, file not open", mb.index) } // Update our last msg. atomic.StoreUint64(&mb.last.seq, sm.seq) mb.last.ts = sm.ts // Clear our cache. mb.clearCacheAndOffset() // Redo per subject info for this block. mb.resetPerSubjectInfo() // Load msgs again. mb.loadMsgsWithLock() return purged, bytes, nil } // Helper to determine if the mb is empty. func (mb *msgBlock) isEmpty() bool { return atomic.LoadUint64(&mb.first.seq) > atomic.LoadUint64(&mb.last.seq) } // Lock should be held. func (mb *msgBlock) selectNextFirst() { var seq uint64 fseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) for seq = fseq + 1; seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // We will move past this so we can delete the entry. mb.dmap.Delete(seq) } else { break } } // Set new first sequence. atomic.StoreUint64(&mb.first.seq, seq) // Check if we are empty.. if seq > lseq { mb.first.ts = 0 return } // Need to get the timestamp. // We will try the cache direct and fallback if needed. var smv StoreMsg sm, _ := mb.cacheLookup(seq, &smv) if sm == nil { // Slow path, need to unlock. mb.mu.Unlock() sm, _, _ = mb.fetchMsg(seq, &smv) mb.mu.Lock() } if sm != nil { mb.first.ts = sm.ts } else { mb.first.ts = 0 } } // Select the next FirstSeq // Lock should be held. func (fs *fileStore) selectNextFirst() { if len(fs.blks) > 0 { mb := fs.blks[0] mb.mu.RLock() fs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq) fs.state.FirstTime = time.Unix(0, mb.first.ts).UTC() mb.mu.RUnlock() } else { // Could not find anything, so treat like purge fs.state.FirstSeq = fs.state.LastSeq + 1 fs.state.FirstTime = time.Time{} } // Mark first as moved. Plays into tombstone cleanup for syncBlocks. fs.firstMoved = true } // Lock should be held. func (mb *msgBlock) resetCacheExpireTimer(td time.Duration) { if td == 0 { td = mb.cexp + 100*time.Millisecond } if mb.ctmr == nil { mb.ctmr = time.AfterFunc(td, mb.expireCache) } else { mb.ctmr.Reset(td) } } // Lock should be held. func (mb *msgBlock) startCacheExpireTimer() { mb.resetCacheExpireTimer(0) } // Used when we load in a message block. // Lock should be held. func (mb *msgBlock) clearCacheAndOffset() { // Reset linear scan tracker. mb.llseq = 0 if mb.cache != nil { mb.cache.off = 0 mb.cache.wp = 0 } mb.clearCache() } // Lock should be held. func (mb *msgBlock) clearCache() { if mb.ctmr != nil { tsla := mb.sinceLastActivity() if mb.fss == nil || tsla > mb.fexp { // Force mb.fss = nil mb.ctmr.Stop() mb.ctmr = nil } else { mb.resetCacheExpireTimer(mb.fexp - tsla) } } if mb.cache == nil { return } buf := mb.cache.buf if mb.cache.off == 0 { mb.cache = nil } else { // Clear msgs and index. mb.cache.buf = nil mb.cache.idx = nil mb.cache.wp = 0 } recycleMsgBlockBuf(buf) } // Called to possibly expire a message block cache. func (mb *msgBlock) expireCache() { mb.mu.Lock() defer mb.mu.Unlock() mb.expireCacheLocked() } func (mb *msgBlock) tryForceExpireCache() { mb.mu.Lock() defer mb.mu.Unlock() mb.tryForceExpireCacheLocked() } // We will attempt to force expire this by temporarily clearing the last load time. func (mb *msgBlock) tryForceExpireCacheLocked() { llts := mb.llts mb.llts = 0 mb.expireCacheLocked() mb.llts = llts } // This is for expiration of the write cache, which will be partial with fip. // So we want to bypass the Pools here. // Lock should be held. func (mb *msgBlock) tryExpireWriteCache() []byte { if mb.cache == nil { return nil } lwts, buf, llts, nra := mb.lwts, mb.cache.buf, mb.llts, mb.cache.nra mb.lwts, mb.cache.nra = 0, true mb.expireCacheLocked() mb.lwts = lwts if mb.cache != nil { mb.cache.nra = nra } // We could check for a certain time since last load, but to be safe just reuse if no loads at all. if llts == 0 && (mb.cache == nil || mb.cache.buf == nil) { // Clear last write time since we now are about to move on to a new lmb. mb.lwts = 0 return buf[:0] } return nil } // Lock should be held. func (mb *msgBlock) expireCacheLocked() { if mb.cache == nil && mb.fss == nil { if mb.ctmr != nil { mb.ctmr.Stop() mb.ctmr = nil } return } // Can't expire if we still have pending. if mb.cache != nil && len(mb.cache.buf)-int(mb.cache.wp) > 0 { mb.resetCacheExpireTimer(mb.cexp) return } // Grab timestamp to compare. tns := time.Now().UnixNano() // For the core buffer of messages, we care about reads and writes, but not removes. bufts := mb.llts if mb.lwts > bufts { bufts = mb.lwts } // Check for activity on the cache that would prevent us from expiring. if tns-bufts <= int64(mb.cexp) { mb.resetCacheExpireTimer(mb.cexp - time.Duration(tns-bufts)) return } // If we are here we will at least expire the core msg buffer. // We need to capture offset in case we do a write next before a full load. if mb.cache != nil { mb.cache.off += len(mb.cache.buf) if !mb.cache.nra { recycleMsgBlockBuf(mb.cache.buf) } mb.cache.buf = nil mb.cache.wp = 0 } // Check if we can clear out our idx unless under force expire. // fss we keep longer and expire under sync timer checks. mb.clearCache() } func (fs *fileStore) startAgeChk() { if fs.ageChk == nil && fs.cfg.MaxAge != 0 { fs.ageChk = time.AfterFunc(fs.cfg.MaxAge, fs.expireMsgs) } } // Lock should be held. func (fs *fileStore) resetAgeChk(delta int64) { if fs.cfg.MaxAge == 0 { return } fireIn := fs.cfg.MaxAge if delta > 0 && time.Duration(delta) < fireIn { if fireIn = time.Duration(delta); fireIn < 250*time.Millisecond { // Only fire at most once every 250ms. // Excessive firing can effect ingest performance. fireIn = time.Second } } if fs.ageChk != nil { fs.ageChk.Reset(fireIn) } else { fs.ageChk = time.AfterFunc(fireIn, fs.expireMsgs) } } // Lock should be held. func (fs *fileStore) cancelAgeChk() { if fs.ageChk != nil { fs.ageChk.Stop() fs.ageChk = nil } } // Will expire msgs that are too old. func (fs *fileStore) expireMsgs() { // We need to delete one by one here and can not optimize for the time being. // Reason is that we need more information to adjust ack pending in consumers. var smv StoreMsg var sm *StoreMsg fs.mu.RLock() maxAge := int64(fs.cfg.MaxAge) minAge := time.Now().UnixNano() - maxAge fs.mu.RUnlock() for sm, _ = fs.msgForSeq(0, &smv); sm != nil && sm.ts <= minAge; sm, _ = fs.msgForSeq(0, &smv) { fs.mu.Lock() fs.removeMsgViaLimits(sm.seq) fs.mu.Unlock() // Recalculate in case we are expiring a bunch. minAge = time.Now().UnixNano() - maxAge } fs.mu.Lock() defer fs.mu.Unlock() // Onky cancel if no message left, not on potential lookup error that would result in sm == nil. if fs.state.Msgs == 0 { fs.cancelAgeChk() } else { if sm == nil { fs.resetAgeChk(0) } else { fs.resetAgeChk(sm.ts - minAge) } } } // Lock should be held. func (fs *fileStore) checkAndFlushAllBlocks() { for _, mb := range fs.blks { if mb.pendingWriteSize() > 0 { // Since fs lock is held need to pull this apart in case we need to rebuild state. mb.mu.Lock() ld, _ := mb.flushPendingMsgsLocked() mb.mu.Unlock() if ld != nil { fs.rebuildStateLocked(ld) } } } } // This will check all the checksums on messages and report back any sequence numbers with errors. func (fs *fileStore) checkMsgs() *LostStreamData { fs.mu.Lock() defer fs.mu.Unlock() fs.checkAndFlushAllBlocks() // Clear any global subject state. fs.psim, fs.tsl = fs.psim.Empty(), 0 for _, mb := range fs.blks { // Make sure encryption loaded if needed for the block. fs.loadEncryptionForMsgBlock(mb) // FIXME(dlc) - check tombstones here too? if ld, _, err := mb.rebuildState(); err != nil && ld != nil { // Rebuild fs state too. fs.rebuildStateLocked(ld) } fs.populateGlobalPerSubjectInfo(mb) } return fs.ld } // Lock should be held. func (mb *msgBlock) enableForWriting(fip bool) error { if mb == nil { return errNoMsgBlk } if mb.mfd != nil { return nil } <-dios mfd, err := os.OpenFile(mb.mfn, os.O_CREATE|os.O_RDWR, defaultFilePerms) dios <- struct{}{} if err != nil { return fmt.Errorf("error opening msg block file [%q]: %v", mb.mfn, err) } mb.mfd = mfd // Spin up our flusher loop if needed. if !fip { mb.spinUpFlushLoop() } return nil } // Helper function to place a delete tombstone. func (mb *msgBlock) writeTombstone(seq uint64, ts int64) error { return mb.writeMsgRecord(emptyRecordLen, seq|tbit, _EMPTY_, nil, nil, ts, true) } // Will write the message record to the underlying message block. // filestore lock will be held. func (mb *msgBlock) writeMsgRecord(rl, seq uint64, subj string, mhdr, msg []byte, ts int64, flush bool) error { mb.mu.Lock() defer mb.mu.Unlock() // Enable for writing if our mfd is not open. if mb.mfd == nil { if err := mb.enableForWriting(flush); err != nil { return err } } // Make sure we have a cache setup. if mb.cache == nil { mb.setupWriteCache(nil) } // Check if we are tracking per subject for our simple state. // Do this before changing the cache that would trigger a flush pending msgs call // if we needed to regenerate the per subject info. // Note that tombstones have no subject so will not trigger here. if len(subj) > 0 && !mb.noTrack { if err := mb.ensurePerSubjectInfoLoaded(); err != nil { return err } // Mark fss activity. mb.lsts = time.Now().UnixNano() if ss, ok := mb.fss.Find(stringToBytes(subj)); ok && ss != nil { ss.Msgs++ ss.Last = seq ss.lastNeedsUpdate = false } else { mb.fss.Insert(stringToBytes(subj), SimpleState{Msgs: 1, First: seq, Last: seq}) } } // Indexing index := len(mb.cache.buf) + int(mb.cache.off) // Formats // Format with no header // total_len(4) sequence(8) timestamp(8) subj_len(2) subj msg hash(8) // With headers, high bit on total length will be set. // total_len(4) sequence(8) timestamp(8) subj_len(2) subj hdr_len(4) hdr msg hash(8) var le = binary.LittleEndian l := uint32(rl) hasHeaders := len(mhdr) > 0 if hasHeaders { l |= hbit } // Reserve space for the header on the underlying buffer. mb.cache.buf = append(mb.cache.buf, make([]byte, msgHdrSize)...) hdr := mb.cache.buf[len(mb.cache.buf)-msgHdrSize : len(mb.cache.buf)] le.PutUint32(hdr[0:], l) le.PutUint64(hdr[4:], seq) le.PutUint64(hdr[12:], uint64(ts)) le.PutUint16(hdr[20:], uint16(len(subj))) // Now write to underlying buffer. mb.cache.buf = append(mb.cache.buf, subj...) if hasHeaders { var hlen [4]byte le.PutUint32(hlen[0:], uint32(len(mhdr))) mb.cache.buf = append(mb.cache.buf, hlen[:]...) mb.cache.buf = append(mb.cache.buf, mhdr...) } mb.cache.buf = append(mb.cache.buf, msg...) // Calculate hash. mb.hh.Reset() mb.hh.Write(hdr[4:20]) mb.hh.Write(stringToBytes(subj)) if hasHeaders { mb.hh.Write(mhdr) } mb.hh.Write(msg) checksum := mb.hh.Sum(mb.lchk[:0:highwayhash.Size64]) copy(mb.lchk[0:], checksum) // Update write through cache. // Write to msg record. mb.cache.buf = append(mb.cache.buf, checksum...) mb.cache.lrl = uint32(rl) // Set cache timestamp for last store. mb.lwts = ts // Only update index and do accounting if not a delete tombstone. if seq&tbit == 0 { // Accounting, do this before stripping ebit, it is ebit aware. mb.updateAccounting(seq, ts, rl) // Strip ebit if set. seq = seq &^ ebit if mb.cache.fseq == 0 { mb.cache.fseq = seq } // Write index mb.cache.idx = append(mb.cache.idx, uint32(index)|hbit) } else { // Make sure to account for tombstones in rbytes. mb.rbytes += rl } fch, werr := mb.fch, mb.werr // If we should be flushing, or had a write error, do so here. if flush || werr != nil { ld, err := mb.flushPendingMsgsLocked() if ld != nil && mb.fs != nil { // We have the mb lock here, this needs the mb locks so do in its own go routine. go mb.fs.rebuildState(ld) } if err != nil { return err } } else { // Kick the flusher here. kickFlusher(fch) } return nil } // How many bytes pending to be written for this message block. func (mb *msgBlock) pendingWriteSize() int { if mb == nil { return 0 } mb.mu.RLock() defer mb.mu.RUnlock() return mb.pendingWriteSizeLocked() } // How many bytes pending to be written for this message block. func (mb *msgBlock) pendingWriteSizeLocked() int { if mb == nil { return 0 } var pending int if !mb.closed && mb.mfd != nil && mb.cache != nil { pending = len(mb.cache.buf) - int(mb.cache.wp) } return pending } // Try to close our FDs if we can. func (mb *msgBlock) closeFDs() error { mb.mu.Lock() defer mb.mu.Unlock() return mb.closeFDsLocked() } func (mb *msgBlock) closeFDsLocked() error { if buf, _ := mb.bytesPending(); len(buf) > 0 { return errPendingData } mb.closeFDsLockedNoCheck() return nil } func (mb *msgBlock) closeFDsLockedNoCheck() { if mb.mfd != nil { mb.mfd.Close() mb.mfd = nil } } // bytesPending returns the buffer to be used for writing to the underlying file. // This marks we are in flush and will return nil if asked again until cleared. // Lock should be held. func (mb *msgBlock) bytesPending() ([]byte, error) { if mb == nil || mb.mfd == nil { return nil, errNoPending } if mb.cache == nil { return nil, errNoCache } if len(mb.cache.buf) <= mb.cache.wp { return nil, errNoPending } buf := mb.cache.buf[mb.cache.wp:] if len(buf) == 0 { return nil, errNoPending } return buf, nil } // Returns the current blkSize including deleted msgs etc. func (mb *msgBlock) blkSize() uint64 { mb.mu.RLock() nb := mb.rbytes mb.mu.RUnlock() return nb } // Update accounting on a write msg. // Lock should be held. func (mb *msgBlock) updateAccounting(seq uint64, ts int64, rl uint64) { isDeleted := seq&ebit != 0 if isDeleted { seq = seq &^ ebit } fseq := atomic.LoadUint64(&mb.first.seq) if (fseq == 0 || mb.first.ts == 0) && seq >= fseq { atomic.StoreUint64(&mb.first.seq, seq) mb.first.ts = ts } // Need atomics here for selectMsgBlock speed. atomic.StoreUint64(&mb.last.seq, seq) mb.last.ts = ts mb.rbytes += rl if !isDeleted { mb.bytes += rl mb.msgs++ } } // Lock should be held. func (fs *fileStore) writeMsgRecord(seq uint64, ts int64, subj string, hdr, msg []byte) (uint64, error) { var err error // Get size for this message. rl := fileStoreMsgSize(subj, hdr, msg) if rl&hbit != 0 { return 0, ErrMsgTooLarge } // Grab our current last message block. mb := fs.lmb // Mark as dirty for stream state. fs.dirty++ if mb == nil || mb.msgs > 0 && mb.blkSize()+rl > fs.fcfg.BlockSize { if mb != nil && fs.fcfg.Compression != NoCompression { // We've now reached the end of this message block, if we want // to compress blocks then now's the time to do it. go mb.recompressOnDiskIfNeeded() } if mb, err = fs.newMsgBlockForWrite(); err != nil { return 0, err } } // Ask msg block to store in write through cache. err = mb.writeMsgRecord(rl, seq, subj, hdr, msg, ts, fs.fip) return rl, err } // For writing tombstones to our lmb. This version will enforce maximum block sizes. // Lock should be held. func (fs *fileStore) writeTombstone(seq uint64, ts int64) error { // Grab our current last message block. lmb := fs.lmb var err error if lmb == nil || lmb.blkSize()+emptyRecordLen > fs.fcfg.BlockSize { if lmb != nil && fs.fcfg.Compression != NoCompression { // We've now reached the end of this message block, if we want // to compress blocks then now's the time to do it. go lmb.recompressOnDiskIfNeeded() } if lmb, err = fs.newMsgBlockForWrite(); err != nil { return err } } return lmb.writeTombstone(seq, ts) } func (mb *msgBlock) recompressOnDiskIfNeeded() error { alg := mb.fs.fcfg.Compression mb.mu.Lock() defer mb.mu.Unlock() origFN := mb.mfn // The original message block on disk. tmpFN := mb.mfn + compressTmpSuffix // The compressed block will be written here. // Open up the file block and read in the entire contents into memory. // One of two things will happen: // 1. The block will be compressed already and have a valid metadata // header, in which case we do nothing. // 2. The block will be uncompressed, in which case we will compress it // and then write it back out to disk, re-encrypting if necessary. <-dios origBuf, err := os.ReadFile(origFN) dios <- struct{}{} if err != nil { return fmt.Errorf("failed to read original block from disk: %w", err) } // If the block is encrypted then we will need to decrypt it before // doing anything. We always encrypt after compressing because then the // compression can be as efficient as possible on the raw data, whereas // the encrypted ciphertext will not compress anywhere near as well. // The block encryption also covers the optional compression metadata. if mb.bek != nil && len(origBuf) > 0 { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return err } mb.bek = bek mb.bek.XORKeyStream(origBuf, origBuf) } meta := &CompressionInfo{} if _, err := meta.UnmarshalMetadata(origBuf); err != nil { // An error is only returned here if there's a problem with parsing // the metadata. If the file has no metadata at all, no error is // returned and the algorithm defaults to no compression. return fmt.Errorf("failed to read existing metadata header: %w", err) } if meta.Algorithm == alg { // The block is already compressed with the chosen algorithm so there // is nothing else to do. This is not a common case, it is here only // to ensure we don't do unnecessary work in case something asked us // to recompress an already compressed block with the same algorithm. return nil } else if alg != NoCompression { // The block is already compressed using some algorithm, so we need // to decompress the block using the existing algorithm before we can // recompress it with the new one. if origBuf, err = meta.Algorithm.Decompress(origBuf); err != nil { return fmt.Errorf("failed to decompress original block: %w", err) } } // Rather than modifying the existing block on disk (which is a dangerous // operation if something goes wrong), create a new temporary file. We will // write out the new block here and then swap the files around afterwards // once everything else has succeeded correctly. <-dios tmpFD, err := os.OpenFile(tmpFN, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, defaultFilePerms) dios <- struct{}{} if err != nil { return fmt.Errorf("failed to create temporary file: %w", err) } errorCleanup := func(err error) error { tmpFD.Close() os.Remove(tmpFN) return err } // The original buffer at this point is uncompressed, so we will now compress // it if needed. Note that if the selected algorithm is NoCompression, the // Compress function will just return the input buffer unmodified. cmpBuf, err := alg.Compress(origBuf) if err != nil { return errorCleanup(fmt.Errorf("failed to compress block: %w", err)) } // We only need to write out the metadata header if compression is enabled. // If we're trying to uncompress the file on disk at this point, don't bother // writing metadata. if alg != NoCompression { meta := &CompressionInfo{ Algorithm: alg, OriginalSize: uint64(len(origBuf)), } cmpBuf = append(meta.MarshalMetadata(), cmpBuf...) } // Re-encrypt the block if necessary. if mb.bek != nil && len(cmpBuf) > 0 { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return errorCleanup(err) } mb.bek = bek mb.bek.XORKeyStream(cmpBuf, cmpBuf) } // Write the new block data (which might be compressed or encrypted) to the // temporary file. if n, err := tmpFD.Write(cmpBuf); err != nil { return errorCleanup(fmt.Errorf("failed to write to temporary file: %w", err)) } else if n != len(cmpBuf) { return errorCleanup(fmt.Errorf("short write to temporary file (%d != %d)", n, len(cmpBuf))) } if err := tmpFD.Sync(); err != nil { return errorCleanup(fmt.Errorf("failed to sync temporary file: %w", err)) } if err := tmpFD.Close(); err != nil { return errorCleanup(fmt.Errorf("failed to close temporary file: %w", err)) } // Now replace the original file with the newly updated temp file. if err := os.Rename(tmpFN, origFN); err != nil { return fmt.Errorf("failed to move temporary file into place: %w", err) } // Since the message block might be retained in memory, make sure the // compression algorithm is up-to-date, since this will be needed when // compacting or truncating. mb.cmp = alg // Also update rbytes mb.rbytes = uint64(len(cmpBuf)) return nil } func (mb *msgBlock) decompressIfNeeded(buf []byte) ([]byte, error) { var meta CompressionInfo if n, err := meta.UnmarshalMetadata(buf); err != nil { // There was a problem parsing the metadata header of the block. // If there's no metadata header, an error isn't returned here, // we will instead just use default values of no compression. return nil, err } else if n == 0 { // There were no metadata bytes, so we assume the block is not // compressed and return it as-is. return buf, nil } else { // Metadata was present so it's quite likely the block contents // are compressed. If by any chance the metadata claims that the // block is uncompressed, then the input slice is just returned // unmodified. return meta.Algorithm.Decompress(buf[n:]) } } // Lock should be held. func (mb *msgBlock) ensureRawBytesLoaded() error { if mb.rbytes > 0 { return nil } f, err := mb.openBlock() if err != nil { return err } defer f.Close() if fi, err := f.Stat(); fi != nil && err == nil { mb.rbytes = uint64(fi.Size()) } else { return err } return nil } // Sync msg and index files as needed. This is called from a timer. func (fs *fileStore) syncBlocks() { fs.mu.Lock() // If closed or a snapshot is in progress bail. if fs.closed || fs.sips > 0 { fs.mu.Unlock() return } blks := append([]*msgBlock(nil), fs.blks...) lmb, firstMoved, firstSeq := fs.lmb, fs.firstMoved, fs.state.FirstSeq // Clear first moved. fs.firstMoved = false fs.mu.Unlock() var markDirty bool for _, mb := range blks { // Do actual sync. Hold lock for consistency. mb.mu.Lock() if mb.closed { mb.mu.Unlock() continue } // See if we can close FDs due to being idle. if mb.mfd != nil && mb.sinceLastWriteActivity() > closeFDsIdle { mb.dirtyCloseWithRemove(false) } // If our first has moved and we are set to noCompact (which is from tombstones), // clear so that we might cleanup tombstones. if firstMoved && mb.noCompact { mb.noCompact = false } // Check if we should compact here as well. // Do not compact last mb. var needsCompact bool if mb != lmb && mb.ensureRawBytesLoaded() == nil && mb.shouldCompactSync() { needsCompact = true markDirty = true } // Check if we need to sync. We will not hold lock during actual sync. needSync := mb.needSync if needSync { // Flush anything that may be pending. mb.flushPendingMsgsLocked() } mb.mu.Unlock() // Check if we should compact here. // Need to hold fs lock in case we reference psim when loading in the mb and we may remove this block if truly empty. if needsCompact { fs.mu.RLock() mb.mu.Lock() mb.compactWithFloor(firstSeq) // If this compact removed all raw bytes due to tombstone cleanup, schedule to remove. shouldRemove := mb.rbytes == 0 mb.mu.Unlock() fs.mu.RUnlock() // Check if we should remove. This will not be common, so we will re-take fs write lock here vs changing // it above which we would prefer to be a readlock such that other lookups can occur while compacting this block. if shouldRemove { fs.mu.Lock() mb.mu.Lock() fs.removeMsgBlock(mb) mb.mu.Unlock() fs.mu.Unlock() needSync = false } } // Check if we need to sync this block. if needSync { mb.mu.Lock() var fd *os.File var didOpen bool if mb.mfd != nil { fd = mb.mfd } else { <-dios fd, _ = os.OpenFile(mb.mfn, os.O_RDWR, defaultFilePerms) dios <- struct{}{} didOpen = true } // If we have an fd. if fd != nil { canClear := fd.Sync() == nil // If we opened the file close the fd. if didOpen { fd.Close() } // Only clear sync flag on success. if canClear { mb.needSync = false } } mb.mu.Unlock() } } fs.mu.Lock() if fs.closed { fs.mu.Unlock() return } fs.setSyncTimer() if markDirty { fs.dirty++ } // Sync state file if we are not running with sync always. if !fs.fcfg.SyncAlways { fn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) <-dios fd, _ := os.OpenFile(fn, os.O_RDWR, defaultFilePerms) dios <- struct{}{} if fd != nil { fd.Sync() fd.Close() } } fs.mu.Unlock() } // Select the message block where this message should be found. // Return nil if not in the set. // Read lock should be held. func (fs *fileStore) selectMsgBlock(seq uint64) *msgBlock { _, mb := fs.selectMsgBlockWithIndex(seq) return mb } // Lock should be held. func (fs *fileStore) selectMsgBlockWithIndex(seq uint64) (int, *msgBlock) { // Check for out of range. if seq < fs.state.FirstSeq || seq > fs.state.LastSeq || fs.state.Msgs == 0 { return -1, nil } const linearThresh = 32 nb := len(fs.blks) - 1 if nb < linearThresh { for i, mb := range fs.blks { if seq <= atomic.LoadUint64(&mb.last.seq) { return i, mb } } return -1, nil } // Do traditional binary search here since we know the blocks are sorted by sequence first and last. for low, high, mid := 0, nb, nb/2; low <= high; mid = (low + high) / 2 { mb := fs.blks[mid] // Right now these atomic loads do not factor in, so fine to leave. Was considering // uplifting these to fs scope to avoid atomic load but not needed. first, last := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) if seq > last { low = mid + 1 } else if seq < first { // A message block's first sequence can change here meaning we could find a gap. // We want to behave like above, which if inclusive (we check at start) should // always return an index and a valid mb. // If we have a gap then our seq would be > fs.blks[mid-1].last.seq if mid == 0 || seq > atomic.LoadUint64(&fs.blks[mid-1].last.seq) { return mid, mb } high = mid - 1 } else { return mid, mb } } return -1, nil } // Select the message block where this message should be found. // Return nil if not in the set. func (fs *fileStore) selectMsgBlockForStart(minTime time.Time) *msgBlock { fs.mu.RLock() defer fs.mu.RUnlock() t := minTime.UnixNano() for _, mb := range fs.blks { mb.mu.RLock() found := t <= mb.last.ts mb.mu.RUnlock() if found { return mb } } return nil } // Index a raw msg buffer. // Lock should be held. func (mb *msgBlock) indexCacheBuf(buf []byte) error { var le = binary.LittleEndian var fseq uint64 var idx []uint32 var index uint32 mbFirstSeq := atomic.LoadUint64(&mb.first.seq) mbLastSeq := atomic.LoadUint64(&mb.last.seq) // Sanity check here since we calculate size to allocate based on this. if mbFirstSeq > (mbLastSeq + 1) { // Purged state first == last + 1 mb.fs.warn("indexCacheBuf corrupt state: mb.first %d mb.last %d", mbFirstSeq, mbLastSeq) // This would cause idxSz to wrap. return errCorruptState } // Capture beginning size of dmap. dms := uint64(mb.dmap.Size()) idxSz := mbLastSeq - mbFirstSeq + 1 if mb.cache == nil { // Approximation, may adjust below. fseq = mbFirstSeq idx = make([]uint32, 0, idxSz) mb.cache = &cache{} } else { fseq = mb.cache.fseq idx = mb.cache.idx if len(idx) == 0 { idx = make([]uint32, 0, idxSz) } index = uint32(len(mb.cache.buf)) buf = append(mb.cache.buf, buf...) } // Create FSS if we should track. var popFss bool if mb.fssNotLoaded() { mb.fss = stree.NewSubjectTree[SimpleState]() popFss = true } // Mark fss activity. mb.lsts = time.Now().UnixNano() lbuf := uint32(len(buf)) var seq uint64 for index < lbuf { if index+msgHdrSize > lbuf { return errCorruptState } hdr := buf[index : index+msgHdrSize] rl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:])) seq = le.Uint64(hdr[4:]) // Clear any headers bit that could be set. rl &^= hbit dlen := int(rl) - msgHdrSize // Do some quick sanity checks here. if dlen < 0 || slen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh { mb.fs.warn("indexCacheBuf corrupt record state: dlen %d slen %d index %d rl %d lbuf %d", dlen, slen, index, rl, lbuf) // This means something is off. // TODO(dlc) - Add into bad list? return errCorruptState } // Check for tombstones which we can skip in terms of indexing. if seq&tbit != 0 { index += rl continue } // Clear any erase bits. erased := seq&ebit != 0 seq = seq &^ ebit // We defer checksum checks to individual msg cache lookups to amortorize costs and // not introduce latency for first message from a newly loaded block. if seq >= mbFirstSeq { // Track that we do not have holes. if slot := int(seq - mbFirstSeq); slot != len(idx) { // If we have a hole fill it. for dseq := mbFirstSeq + uint64(len(idx)); dseq < seq; dseq++ { idx = append(idx, dbit) if dms == 0 { mb.dmap.Insert(dseq) } } } // Add to our index. idx = append(idx, index) mb.cache.lrl = uint32(rl) // Adjust if we guessed wrong. if seq != 0 && seq < fseq { fseq = seq } // Make sure our dmap has this entry if it was erased. if erased && dms == 0 { mb.dmap.Insert(seq) } // Handle FSS inline here. if popFss && slen > 0 && !mb.noTrack && !erased && !mb.dmap.Exists(seq) { bsubj := buf[index+msgHdrSize : index+msgHdrSize+uint32(slen)] if ss, ok := mb.fss.Find(bsubj); ok && ss != nil { ss.Msgs++ ss.Last = seq ss.lastNeedsUpdate = false } else { mb.fss.Insert(bsubj, SimpleState{ Msgs: 1, First: seq, Last: seq, }) } } } index += rl } // Track holes at the end of the block, these would be missed in the // earlier loop if we've ran out of block file to look at, but should // be easily noticed because the seq will be below the last seq from // the index. if seq > 0 && seq < mbLastSeq { for dseq := seq; dseq < mbLastSeq; dseq++ { idx = append(idx, dbit) if dms == 0 { mb.dmap.Insert(dseq) } } } mb.cache.buf = buf mb.cache.idx = idx mb.cache.fseq = fseq mb.cache.wp += int(lbuf) return nil } // flushPendingMsgs writes out any messages for this message block. func (mb *msgBlock) flushPendingMsgs() error { mb.mu.Lock() fsLostData, err := mb.flushPendingMsgsLocked() fs := mb.fs mb.mu.Unlock() // Signals us that we need to rebuild filestore state. if fsLostData != nil && fs != nil { // Rebuild fs state too. fs.rebuildState(fsLostData) } return err } // Write function for actual data. // mb.mfd should not be nil. // Lock should held. func (mb *msgBlock) writeAt(buf []byte, woff int64) (int, error) { // Used to mock write failures. if mb.mockWriteErr { // Reset on trip. mb.mockWriteErr = false return 0, errors.New("mock write error") } <-dios n, err := mb.mfd.WriteAt(buf, woff) dios <- struct{}{} return n, err } // flushPendingMsgsLocked writes out any messages for this message block. // Lock should be held. func (mb *msgBlock) flushPendingMsgsLocked() (*LostStreamData, error) { // Signals us that we need to rebuild filestore state. var fsLostData *LostStreamData if mb.cache == nil || mb.mfd == nil { return nil, nil } buf, err := mb.bytesPending() // If we got an error back return here. if err != nil { // No pending data to be written is not an error. if err == errNoPending || err == errNoCache { err = nil } return nil, err } woff := int64(mb.cache.off + mb.cache.wp) lob := len(buf) // TODO(dlc) - Normally we would not hold the lock across I/O so we can improve performance. // We will hold to stabilize the code base, as we have had a few anomalies with partial cache errors // under heavy load. // Check if we need to encrypt. if mb.bek != nil && lob > 0 { // Need to leave original alone. var dst []byte if lob <= defaultLargeBlockSize { dst = getMsgBlockBuf(lob)[:lob] } else { dst = make([]byte, lob) } mb.bek.XORKeyStream(dst, buf) buf = dst } // Append new data to the message block file. for lbb := lob; lbb > 0; lbb = len(buf) { n, err := mb.writeAt(buf, woff) if err != nil { mb.dirtyCloseWithRemove(false) ld, _, _ := mb.rebuildStateLocked() mb.werr = err return ld, err } // Update our write offset. woff += int64(n) // Partial write. if n != lbb { buf = buf[n:] } else { // Done. break } } // Clear any error. mb.werr = nil // Cache may be gone. if mb.cache == nil || mb.mfd == nil { return fsLostData, mb.werr } // Check if we are in sync always mode. if mb.syncAlways { mb.mfd.Sync() } else { mb.needSync = true } // Check for additional writes while we were writing to the disk. moreBytes := len(mb.cache.buf) - mb.cache.wp - lob // Decide what we want to do with the buffer in hand. If we have load interest // we will hold onto the whole thing, otherwise empty the buffer, possibly reusing it. if ts := time.Now().UnixNano(); ts < mb.llts || (ts-mb.llts) <= int64(mb.cexp) { mb.cache.wp += lob } else { if cap(mb.cache.buf) <= maxBufReuse { buf = mb.cache.buf[:0] } else { recycleMsgBlockBuf(mb.cache.buf) buf = nil } if moreBytes > 0 { nbuf := mb.cache.buf[len(mb.cache.buf)-moreBytes:] if moreBytes > (len(mb.cache.buf)/4*3) && cap(nbuf) <= maxBufReuse { buf = nbuf } else { buf = append(buf, nbuf...) } } // Update our cache offset. mb.cache.off = int(woff) // Reset write pointer. mb.cache.wp = 0 // Place buffer back in the cache structure. mb.cache.buf = buf // Mark fseq to 0 mb.cache.fseq = 0 } return fsLostData, mb.werr } // Lock should be held. func (mb *msgBlock) clearLoading() { mb.loading = false } // Will load msgs from disk. func (mb *msgBlock) loadMsgs() error { // We hold the lock here the whole time by design. mb.mu.Lock() defer mb.mu.Unlock() return mb.loadMsgsWithLock() } // Lock should be held. func (mb *msgBlock) cacheAlreadyLoaded() bool { if mb.cache == nil || mb.cache.off != 0 || mb.cache.fseq == 0 || len(mb.cache.buf) == 0 { return false } numEntries := mb.msgs + uint64(mb.dmap.Size()) + (atomic.LoadUint64(&mb.first.seq) - mb.cache.fseq) return numEntries == uint64(len(mb.cache.idx)) } // Lock should be held. func (mb *msgBlock) cacheNotLoaded() bool { return !mb.cacheAlreadyLoaded() } // Report if our fss is not loaded. // Lock should be held. func (mb *msgBlock) fssNotLoaded() bool { return mb.fss == nil && !mb.noTrack } // Wrap openBlock for the gated semaphore processing. // Lock should be held func (mb *msgBlock) openBlock() (*os.File, error) { // Gate with concurrent IO semaphore. <-dios f, err := os.Open(mb.mfn) dios <- struct{}{} return f, err } // Used to load in the block contents. // Lock should be held and all conditionals satisfied prior. func (mb *msgBlock) loadBlock(buf []byte) ([]byte, error) { var f *os.File // Re-use if we have mfd open. if mb.mfd != nil { f = mb.mfd if n, err := f.Seek(0, 0); n != 0 || err != nil { f = nil mb.closeFDsLockedNoCheck() } } if f == nil { var err error f, err = mb.openBlock() if err != nil { if os.IsNotExist(err) { err = errNoBlkData } return nil, err } defer f.Close() } var sz int if info, err := f.Stat(); err == nil { sz64 := info.Size() if int64(int(sz64)) == sz64 { sz = int(sz64) } else { return nil, errMsgBlkTooBig } } if buf == nil { buf = getMsgBlockBuf(sz) if sz > cap(buf) { // We know we will make a new one so just recycle for now. recycleMsgBlockBuf(buf) buf = nil } } if sz > cap(buf) { buf = make([]byte, sz) } else { buf = buf[:sz] } <-dios n, err := io.ReadFull(f, buf) dios <- struct{}{} // On success capture raw bytes size. if err == nil { mb.rbytes = uint64(n) } return buf[:n], err } // Lock should be held. func (mb *msgBlock) loadMsgsWithLock() error { // Check for encryption, we do not load keys on startup anymore so might need to load them here. if mb.fs != nil && mb.fs.prf != nil && (mb.aek == nil || mb.bek == nil) { if err := mb.fs.loadEncryptionForMsgBlock(mb); err != nil { return err } } // Check to see if we are loading already. if mb.loading { return nil } // Set loading status. mb.loading = true defer mb.clearLoading() var nchecks int checkCache: nchecks++ if nchecks > 8 { return errCorruptState } // Check to see if we have a full cache. if mb.cacheAlreadyLoaded() { return nil } mb.llts = time.Now().UnixNano() // FIXME(dlc) - We could be smarter here. if buf, _ := mb.bytesPending(); len(buf) > 0 { ld, err := mb.flushPendingMsgsLocked() if ld != nil && mb.fs != nil { // We do not know if fs is locked or not at this point. // This should be an exceptional condition so do so in Go routine. go mb.fs.rebuildState(ld) } if err != nil { return err } goto checkCache } // Load in the whole block. // We want to hold the mb lock here to avoid any changes to state. buf, err := mb.loadBlock(nil) if err != nil { mb.fs.warn("loadBlock error: %v", err) if err == errNoBlkData { if ld, _, err := mb.rebuildStateLocked(); err != nil && ld != nil { // Rebuild fs state too. go mb.fs.rebuildState(ld) } } return err } // Reset the cache since we just read everything in. // Make sure this is cleared in case we had a partial when we started. mb.clearCacheAndOffset() // Check if we need to decrypt. if mb.bek != nil && len(buf) > 0 { bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { return err } mb.bek = bek mb.bek.XORKeyStream(buf, buf) } // Check for compression. if buf, err = mb.decompressIfNeeded(buf); err != nil { return err } if err := mb.indexCacheBuf(buf); err != nil { if err == errCorruptState { var ld *LostStreamData if ld, _, err = mb.rebuildStateLocked(); ld != nil { // We do not know if fs is locked or not at this point. // This should be an exceptional condition so do so in Go routine. go mb.fs.rebuildState(ld) } } if err != nil { return err } goto checkCache } if len(buf) > 0 { mb.cloads++ mb.startCacheExpireTimer() } return nil } // Fetch a message from this block, possibly reading in and caching the messages. // We assume the block was selected and is correct, so we do not do range checks. func (mb *msgBlock) fetchMsg(seq uint64, sm *StoreMsg) (*StoreMsg, bool, error) { mb.mu.Lock() defer mb.mu.Unlock() fseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) if seq < fseq || seq > lseq { return nil, false, ErrStoreMsgNotFound } // See if we can short circuit if we already know msg deleted. if mb.dmap.Exists(seq) { // Update for scanning like cacheLookup would have. llseq := mb.llseq if mb.llseq == 0 || seq < mb.llseq || seq == mb.llseq+1 || seq == mb.llseq-1 { mb.llseq = seq } expireOk := (seq == lseq && llseq == seq-1) || (seq == fseq && llseq == seq+1) return nil, expireOk, errDeletedMsg } if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return nil, false, err } } llseq := mb.llseq fsm, err := mb.cacheLookup(seq, sm) if err != nil { return nil, false, err } expireOk := (seq == lseq && llseq == seq-1) || (seq == fseq && llseq == seq+1) return fsm, expireOk, err } var ( errNoCache = errors.New("no message cache") errBadMsg = errors.New("malformed or corrupt message") errDeletedMsg = errors.New("deleted message") errPartialCache = errors.New("partial cache") errNoPending = errors.New("message block does not have pending data") errNotReadable = errors.New("storage directory not readable") errCorruptState = errors.New("corrupt state file") errPriorState = errors.New("prior state file") errPendingData = errors.New("pending data still present") errNoEncryption = errors.New("encryption not enabled") errBadKeySize = errors.New("encryption bad key size") errNoMsgBlk = errors.New("no message block") errMsgBlkTooBig = errors.New("message block size exceeded int capacity") errUnknownCipher = errors.New("unknown cipher") errNoMainKey = errors.New("encrypted store encountered with no main key") errNoBlkData = errors.New("message block data missing") errStateTooBig = errors.New("store state too big for optional write") ) const ( // Used for marking messages that have had their checksums checked. // Used to signal a message record with headers. hbit = 1 << 31 // Used for marking erased messages sequences. ebit = 1 << 63 // Used for marking tombstone sequences. tbit = 1 << 62 // Used to mark an index as deleted and non-existent. dbit = 1 << 30 ) // Will do a lookup from cache. // Lock should be held. func (mb *msgBlock) cacheLookup(seq uint64, sm *StoreMsg) (*StoreMsg, error) { if seq < atomic.LoadUint64(&mb.first.seq) || seq > atomic.LoadUint64(&mb.last.seq) { return nil, ErrStoreMsgNotFound } // The llseq signals us when we can expire a cache at the end of a linear scan. // We want to only update when we know the last reads (multiple consumers) are sequential. // We want to account for forwards and backwards linear scans. if mb.llseq == 0 || seq < mb.llseq || seq == mb.llseq+1 || seq == mb.llseq-1 { mb.llseq = seq } // If we have a delete map check it. if mb.dmap.Exists(seq) { mb.llts = time.Now().UnixNano() return nil, errDeletedMsg } // Detect no cache loaded. if mb.cache == nil || mb.cache.fseq == 0 || len(mb.cache.idx) == 0 || len(mb.cache.buf) == 0 { var reason string if mb.cache == nil { reason = "no cache" } else if mb.cache.fseq == 0 { reason = "fseq is 0" } else if len(mb.cache.idx) == 0 { reason = "no idx present" } else { reason = "cache buf empty" } mb.fs.warn("Cache lookup detected no cache: %s", reason) return nil, errNoCache } // Check partial cache status. if seq < mb.cache.fseq { mb.fs.warn("Cache lookup detected partial cache: seq %d vs cache fseq %d", seq, mb.cache.fseq) return nil, errPartialCache } bi, _, hashChecked, err := mb.slotInfo(int(seq - mb.cache.fseq)) if err != nil { return nil, err } // Update cache activity. mb.llts = time.Now().UnixNano() li := int(bi) - mb.cache.off if li >= len(mb.cache.buf) { return nil, errPartialCache } buf := mb.cache.buf[li:] // We use the high bit to denote we have already checked the checksum. var hh hash.Hash64 if !hashChecked { hh = mb.hh // This will force the hash check in msgFromBuf. } // Parse from the raw buffer. fsm, err := mb.msgFromBuf(buf, sm, hh) if err != nil || fsm == nil { return nil, err } // Deleted messages that are decoded return a 0 for sequence. if fsm.seq == 0 { return nil, errDeletedMsg } if seq != fsm.seq { recycleMsgBlockBuf(mb.cache.buf) mb.cache.buf = nil return nil, fmt.Errorf("sequence numbers for cache load did not match, %d vs %d", seq, fsm.seq) } // Clear the check bit here after we know all is good. if !hashChecked { mb.cache.idx[seq-mb.cache.fseq] = (bi | hbit) } return fsm, nil } // Used when we are checking if discarding a message due to max msgs per subject will give us // enough room for a max bytes condition. // Lock should be already held. func (fs *fileStore) sizeForSeq(seq uint64) int { if seq == 0 { return 0 } var smv StoreMsg if mb := fs.selectMsgBlock(seq); mb != nil { if sm, _, _ := mb.fetchMsg(seq, &smv); sm != nil { return int(fileStoreMsgSize(sm.subj, sm.hdr, sm.msg)) } } return 0 } // Will return message for the given sequence number. func (fs *fileStore) msgForSeq(seq uint64, sm *StoreMsg) (*StoreMsg, error) { // TODO(dlc) - Since Store, Remove, Skip all hold the write lock on fs this will // be stalled. Need another lock if want to happen in parallel. fs.mu.RLock() if fs.closed { fs.mu.RUnlock() return nil, ErrStoreClosed } // Indicates we want first msg. if seq == 0 { seq = fs.state.FirstSeq } // Make sure to snapshot here. mb, lseq := fs.selectMsgBlock(seq), fs.state.LastSeq fs.mu.RUnlock() if mb == nil { var err = ErrStoreEOF if seq <= lseq { err = ErrStoreMsgNotFound } return nil, err } fsm, expireOk, err := mb.fetchMsg(seq, sm) if err != nil { return nil, err } // We detected a linear scan and access to the last message. // If we are not the last message block we can try to expire the cache. if expireOk { mb.tryForceExpireCache() } return fsm, nil } // Internal function to return msg parts from a raw buffer. // Lock should be held. func (mb *msgBlock) msgFromBuf(buf []byte, sm *StoreMsg, hh hash.Hash64) (*StoreMsg, error) { if len(buf) < emptyRecordLen { return nil, errBadMsg } var le = binary.LittleEndian hdr := buf[:msgHdrSize] rl := le.Uint32(hdr[0:]) hasHeaders := rl&hbit != 0 rl &^= hbit // clear header bit dlen := int(rl) - msgHdrSize slen := int(le.Uint16(hdr[20:])) // Simple sanity check. if dlen < 0 || slen > (dlen-recordHashSize) || dlen > int(rl) || int(rl) > len(buf) || rl > rlBadThresh { return nil, errBadMsg } data := buf[msgHdrSize : msgHdrSize+dlen] // Do checksum tests here if requested. if hh != nil { hh.Reset() hh.Write(hdr[4:20]) hh.Write(data[:slen]) if hasHeaders { hh.Write(data[slen+4 : dlen-recordHashSize]) } else { hh.Write(data[slen : dlen-recordHashSize]) } if !bytes.Equal(hh.Sum(nil), data[len(data)-8:]) { return nil, errBadMsg } } seq := le.Uint64(hdr[4:]) if seq&ebit != 0 { seq = 0 } ts := int64(le.Uint64(hdr[12:])) // Create a StoreMsg if needed. if sm == nil { sm = new(StoreMsg) } else { sm.clear() } // To recycle the large blocks we can never pass back a reference, so need to copy for the upper // layers and for us to be safe to expire, and recycle, the large msgBlocks. end := dlen - 8 if hasHeaders { hl := le.Uint32(data[slen:]) bi := slen + 4 li := bi + int(hl) sm.buf = append(sm.buf, data[bi:end]...) li, end = li-bi, end-bi sm.hdr = sm.buf[0:li:li] sm.msg = sm.buf[li:end] } else { sm.buf = append(sm.buf, data[slen:end]...) sm.msg = sm.buf[0 : end-slen] } sm.seq, sm.ts = seq, ts if slen > 0 { // Make a copy since sm.subj lifetime may last longer. sm.subj = string(data[:slen]) } return sm, nil } // LoadMsg will lookup the message by sequence number and return it if found. func (fs *fileStore) LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) { return fs.msgForSeq(seq, sm) } // loadLast will load the last message for a subject. Subject should be non empty and not ">". func (fs *fileStore) loadLast(subj string, sm *StoreMsg) (lsm *StoreMsg, err error) { fs.mu.RLock() defer fs.mu.RUnlock() if fs.closed || fs.lmb == nil { return nil, ErrStoreClosed } if len(fs.blks) == 0 { return nil, ErrStoreMsgNotFound } wc := subjectHasWildcard(subj) var start, stop uint32 // If literal subject check for presence. if wc { start = fs.lmb.index fs.psim.Match(stringToBytes(subj), func(_ []byte, psi *psi) { // Keep track of start and stop indexes for this subject. if psi.fblk < start { start = psi.fblk } if psi.lblk > stop { stop = psi.lblk } }) // None matched. if stop == 0 { return nil, ErrStoreMsgNotFound } // These need to be swapped. start, stop = stop, start } else if info, ok := fs.psim.Find(stringToBytes(subj)); ok { start, stop = info.lblk, info.fblk } else { return nil, ErrStoreMsgNotFound } // Walk blocks backwards. for i := start; i >= stop; i-- { mb := fs.bim[i] if mb == nil { continue } mb.mu.Lock() if err := mb.ensurePerSubjectInfoLoaded(); err != nil { mb.mu.Unlock() return nil, err } // Mark fss activity. mb.lsts = time.Now().UnixNano() var l uint64 // Optimize if subject is not a wildcard. if !wc { if ss, ok := mb.fss.Find(stringToBytes(subj)); ok && ss != nil { l = ss.Last } } if l == 0 { _, _, l = mb.filteredPendingLocked(subj, wc, atomic.LoadUint64(&mb.first.seq)) } if l > 0 { if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { mb.mu.Unlock() return nil, err } } lsm, err = mb.cacheLookup(l, sm) } mb.mu.Unlock() if l > 0 { break } } return lsm, err } // LoadLastMsg will return the last message we have that matches a given subject. // The subject can be a wildcard. func (fs *fileStore) LoadLastMsg(subject string, smv *StoreMsg) (sm *StoreMsg, err error) { if subject == _EMPTY_ || subject == fwcs { sm, err = fs.msgForSeq(fs.lastSeq(), smv) } else { sm, err = fs.loadLast(subject, smv) } if sm == nil || (err != nil && err != ErrStoreClosed) { err = ErrStoreMsgNotFound } return sm, err } // LoadNextMsgMulti will find the next message matching any entry in the sublist. func (fs *fileStore) LoadNextMsgMulti(sl *Sublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { if sl == nil { return fs.LoadNextMsg(_EMPTY_, false, start, smp) } fs.mu.RLock() defer fs.mu.RUnlock() if fs.closed { return nil, 0, ErrStoreClosed } if fs.state.Msgs == 0 || start > fs.state.LastSeq { return nil, fs.state.LastSeq, ErrStoreEOF } if start < fs.state.FirstSeq { start = fs.state.FirstSeq } if bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 { for i := bi; i < len(fs.blks); i++ { mb := fs.blks[i] if sm, expireOk, err := mb.firstMatchingMulti(sl, start, smp); err == nil { if expireOk { mb.tryForceExpireCache() } return sm, sm.seq, nil } else if err != ErrStoreMsgNotFound { return nil, 0, err } else if expireOk { mb.tryForceExpireCache() } } } return nil, fs.state.LastSeq, ErrStoreEOF } func (fs *fileStore) LoadNextMsg(filter string, wc bool, start uint64, sm *StoreMsg) (*StoreMsg, uint64, error) { fs.mu.RLock() defer fs.mu.RUnlock() if fs.closed { return nil, 0, ErrStoreClosed } if fs.state.Msgs == 0 || start > fs.state.LastSeq { return nil, fs.state.LastSeq, ErrStoreEOF } if start < fs.state.FirstSeq { start = fs.state.FirstSeq } // If start is less than or equal to beginning of our stream, meaning our first call, // let's check the psim to see if we can skip ahead. if start <= fs.state.FirstSeq { var ss SimpleState fs.numFilteredPendingNoLast(filter, &ss) // Nothing available. if ss.Msgs == 0 { return nil, fs.state.LastSeq, ErrStoreEOF } // We can skip ahead. if ss.First > start { start = ss.First } } if bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 { for i := bi; i < len(fs.blks); i++ { mb := fs.blks[i] if sm, expireOk, err := mb.firstMatching(filter, wc, start, sm); err == nil { if expireOk { mb.tryForceExpireCache() } return sm, sm.seq, nil } else if err != ErrStoreMsgNotFound { return nil, 0, err } else { // Nothing found in this block. We missed, if first block (bi) check psim. // Similar to above if start <= first seq. // TODO(dlc) - For v2 track these by filter subject since they will represent filtered consumers. // We should not do this at all if we are already on the last block. // Also if we are a wildcard do not check if large subject space. const wcMaxSizeToCheck = 64 * 1024 if i == bi && i < len(fs.blks)-1 && (!wc || fs.psim.Size() < wcMaxSizeToCheck) { nbi, err := fs.checkSkipFirstBlock(filter, wc, bi) // Nothing available. if err == ErrStoreEOF { return nil, fs.state.LastSeq, ErrStoreEOF } // See if we can jump ahead here. // Right now we can only spin on first, so if we have interior sparseness need to favor checking per block fss if loaded. // For v2 will track all blocks that have matches for psim. if nbi > i { i = nbi - 1 // For the iterator condition i++ } } // Check is we can expire. if expireOk { mb.tryForceExpireCache() } } } } return nil, fs.state.LastSeq, ErrStoreEOF } // Will load the next non-deleted msg starting at the start sequence and walking backwards. func (fs *fileStore) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error) { fs.mu.RLock() defer fs.mu.RUnlock() if fs.closed { return nil, ErrStoreClosed } if fs.state.Msgs == 0 || start < fs.state.FirstSeq { return nil, ErrStoreEOF } if start > fs.state.LastSeq { start = fs.state.LastSeq } if smp == nil { smp = new(StoreMsg) } if bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 { for i := bi; i >= 0; i-- { mb := fs.blks[i] mb.mu.Lock() // Need messages loaded from here on out. if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { mb.mu.Unlock() return nil, err } } lseq, fseq := atomic.LoadUint64(&mb.last.seq), atomic.LoadUint64(&mb.first.seq) if start > lseq { start = lseq } for seq := start; seq >= fseq; seq-- { if mb.dmap.Exists(seq) { continue } if sm, err := mb.cacheLookup(seq, smp); err == nil { mb.mu.Unlock() return sm, nil } } mb.mu.Unlock() } } return nil, ErrStoreEOF } // Type returns the type of the underlying store. func (fs *fileStore) Type() StorageType { return FileStorage } // Returns number of subjects in this store. // Lock should be held. func (fs *fileStore) numSubjects() int { return fs.psim.Size() } // numConsumers uses new lock. func (fs *fileStore) numConsumers() int { fs.cmu.RLock() defer fs.cmu.RUnlock() return len(fs.cfs) } // FastState will fill in state with only the following. // Msgs, Bytes, First and Last Sequence and Time and NumDeleted. func (fs *fileStore) FastState(state *StreamState) { fs.mu.RLock() state.Msgs = fs.state.Msgs state.Bytes = fs.state.Bytes state.FirstSeq = fs.state.FirstSeq state.FirstTime = fs.state.FirstTime state.LastSeq = fs.state.LastSeq state.LastTime = fs.state.LastTime // Make sure to reset if being re-used. state.Deleted, state.NumDeleted = nil, 0 if state.LastSeq > state.FirstSeq { state.NumDeleted = int((state.LastSeq - state.FirstSeq + 1) - state.Msgs) if state.NumDeleted < 0 { state.NumDeleted = 0 } } state.Consumers = fs.numConsumers() state.NumSubjects = fs.numSubjects() fs.mu.RUnlock() } // State returns the current state of the stream. func (fs *fileStore) State() StreamState { fs.mu.RLock() state := fs.state state.Consumers = fs.numConsumers() state.NumSubjects = fs.numSubjects() state.Deleted = nil // make sure. if numDeleted := int((state.LastSeq - state.FirstSeq + 1) - state.Msgs); numDeleted > 0 { state.Deleted = make([]uint64, 0, numDeleted) cur := fs.state.FirstSeq for _, mb := range fs.blks { mb.mu.Lock() fseq := atomic.LoadUint64(&mb.first.seq) // Account for messages missing from the head. if fseq > cur { for seq := cur; seq < fseq; seq++ { state.Deleted = append(state.Deleted, seq) } } // Only advance cur if we are increasing. We could have marker blocks with just tombstones. if last := atomic.LoadUint64(&mb.last.seq); last >= cur { cur = last + 1 // Expected next first. } // Add in deleted. mb.dmap.Range(func(seq uint64) bool { state.Deleted = append(state.Deleted, seq) return true }) mb.mu.Unlock() } } fs.mu.RUnlock() state.Lost = fs.lostData() // Can not be guaranteed to be sorted. if len(state.Deleted) > 0 { slices.Sort(state.Deleted) state.NumDeleted = len(state.Deleted) } return state } func (fs *fileStore) Utilization() (total, reported uint64, err error) { fs.mu.RLock() defer fs.mu.RUnlock() for _, mb := range fs.blks { mb.mu.RLock() reported += mb.bytes total += mb.rbytes mb.mu.RUnlock() } return total, reported, nil } func fileStoreMsgSize(subj string, hdr, msg []byte) uint64 { if len(hdr) == 0 { // length of the message record (4bytes) + seq(8) + ts(8) + subj_len(2) + subj + msg + hash(8) return uint64(22 + len(subj) + len(msg) + 8) } // length of the message record (4bytes) + seq(8) + ts(8) + subj_len(2) + subj + hdr_len(4) + hdr + msg + hash(8) return uint64(22 + len(subj) + 4 + len(hdr) + len(msg) + 8) } func fileStoreMsgSizeEstimate(slen, maxPayload int) uint64 { return uint64(emptyRecordLen + slen + 4 + maxPayload) } // Determine time since any last activity, read/load, write or remove. func (mb *msgBlock) sinceLastActivity() time.Duration { if mb.closed { return 0 } last := mb.lwts if mb.lrts > last { last = mb.lrts } if mb.llts > last { last = mb.llts } if mb.lsts > last { last = mb.lsts } return time.Since(time.Unix(0, last).UTC()) } // Determine time since last write or remove of a message. // Read lock should be held. func (mb *msgBlock) sinceLastWriteActivity() time.Duration { if mb.closed { return 0 } last := mb.lwts if mb.lrts > last { last = mb.lrts } return time.Since(time.Unix(0, last).UTC()) } func checkNewHeader(hdr []byte) error { if hdr == nil || len(hdr) < 2 || hdr[0] != magic || (hdr[1] != version && hdr[1] != newVersion) { return errCorruptState } return nil } // readIndexInfo will read in the index information for the message block. func (mb *msgBlock) readIndexInfo() error { ifn := filepath.Join(mb.fs.fcfg.StoreDir, msgDir, fmt.Sprintf(indexScan, mb.index)) buf, err := os.ReadFile(ifn) if err != nil { return err } // Set if first time. if mb.liwsz == 0 { mb.liwsz = int64(len(buf)) } // Decrypt if needed. if mb.aek != nil { buf, err = mb.aek.Open(buf[:0], mb.nonce, buf, nil) if err != nil { return err } } if err := checkNewHeader(buf); err != nil { defer os.Remove(ifn) return fmt.Errorf("bad index file") } bi := hdrLen // Helpers, will set i to -1 on error. readSeq := func() uint64 { if bi < 0 { return 0 } seq, n := binary.Uvarint(buf[bi:]) if n <= 0 { bi = -1 return 0 } bi += n return seq &^ ebit } readCount := readSeq readTimeStamp := func() int64 { if bi < 0 { return 0 } ts, n := binary.Varint(buf[bi:]) if n <= 0 { bi = -1 return -1 } bi += n return ts } mb.msgs = readCount() mb.bytes = readCount() atomic.StoreUint64(&mb.first.seq, readSeq()) mb.first.ts = readTimeStamp() atomic.StoreUint64(&mb.last.seq, readSeq()) mb.last.ts = readTimeStamp() dmapLen := readCount() // Check if this is a short write index file. if bi < 0 || bi+checksumSize > len(buf) { os.Remove(ifn) return fmt.Errorf("short index file") } // Check for consistency if accounting. If something is off bail and we will rebuild. if mb.msgs != (atomic.LoadUint64(&mb.last.seq)-atomic.LoadUint64(&mb.first.seq)+1)-dmapLen { os.Remove(ifn) return fmt.Errorf("accounting inconsistent") } // Checksum copy(mb.lchk[0:], buf[bi:bi+checksumSize]) bi += checksumSize // Now check for presence of a delete map if dmapLen > 0 { // New version is encoded avl seqset. if buf[1] == newVersion { dmap, _, err := avl.Decode(buf[bi:]) if err != nil { return fmt.Errorf("could not decode avl dmap: %v", err) } mb.dmap = *dmap } else { // This is the old version. for i, fseq := 0, atomic.LoadUint64(&mb.first.seq); i < int(dmapLen); i++ { seq := readSeq() if seq == 0 { break } mb.dmap.Insert(seq + fseq) } } } return nil } // Will return total number of cache loads. func (fs *fileStore) cacheLoads() uint64 { var tl uint64 fs.mu.RLock() for _, mb := range fs.blks { tl += mb.cloads } fs.mu.RUnlock() return tl } // Will return total number of cached bytes. func (fs *fileStore) cacheSize() uint64 { var sz uint64 fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() if mb.cache != nil { sz += uint64(len(mb.cache.buf)) } mb.mu.RUnlock() } fs.mu.RUnlock() return sz } // Will return total number of dmapEntries for all msg blocks. func (fs *fileStore) dmapEntries() int { var total int fs.mu.RLock() for _, mb := range fs.blks { total += mb.dmap.Size() } fs.mu.RUnlock() return total } // Fixed helper for iterating. func subjectsEqual(a, b string) bool { return a == b } func subjectsAll(a, b string) bool { return true } func compareFn(subject string) func(string, string) bool { if subject == _EMPTY_ || subject == fwcs { return subjectsAll } if subjectHasWildcard(subject) { return subjectIsSubsetMatch } return subjectsEqual } // PurgeEx will remove messages based on subject filters, sequence and number of messages to keep. // Will return the number of purged messages. func (fs *fileStore) PurgeEx(subject string, sequence, keep uint64) (purged uint64, err error) { if subject == _EMPTY_ || subject == fwcs { if keep == 0 && sequence == 0 { return fs.Purge() } if sequence > 1 { return fs.Compact(sequence) } } eq, wc := compareFn(subject), subjectHasWildcard(subject) var firstSeqNeedsUpdate bool var bytes uint64 // If we have a "keep" designation need to get full filtered state so we know how many to purge. var maxp uint64 if keep > 0 { ss := fs.FilteredState(1, subject) if keep >= ss.Msgs { return 0, nil } maxp = ss.Msgs - keep } var smv StoreMsg var tombs []msgId fs.mu.Lock() // We may remove blocks as we purge, so don't range directly on fs.blks // otherwise we may jump over some (see https://github.com/nats-io/nats-server/issues/3528) for i := 0; i < len(fs.blks); i++ { mb := fs.blks[i] mb.mu.Lock() // If we do not have our fss, try to expire the cache if we have no items in this block. shouldExpire := mb.fssNotLoaded() t, f, l := mb.filteredPendingLocked(subject, wc, atomic.LoadUint64(&mb.first.seq)) if t == 0 { // Expire if we were responsible for loading. if shouldExpire { // Expire this cache before moving on. mb.tryForceExpireCacheLocked() } mb.mu.Unlock() continue } if sequence > 1 && sequence <= l { l = sequence - 1 } if mb.cacheNotLoaded() { mb.loadMsgsWithLock() shouldExpire = true } for seq := f; seq <= l; seq++ { if sm, _ := mb.cacheLookup(seq, &smv); sm != nil && eq(sm.subj, subject) { rl := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg) // Do fast in place remove. // Stats if mb.msgs > 0 { // Msgs fs.state.Msgs-- mb.msgs-- // Bytes, make sure to not go negative. if rl > fs.state.Bytes { rl = fs.state.Bytes } if rl > mb.bytes { rl = mb.bytes } fs.state.Bytes -= rl mb.bytes -= rl // Totals purged++ bytes += rl } // PSIM and FSS updates. mb.removeSeqPerSubject(sm.subj, seq) fs.removePerSubject(sm.subj) // Track tombstones we need to write. tombs = append(tombs, msgId{sm.seq, sm.ts}) // Check for first message. if seq == atomic.LoadUint64(&mb.first.seq) { mb.selectNextFirst() if mb.isEmpty() { fs.removeMsgBlock(mb) i-- // keep flag set, if set previously firstSeqNeedsUpdate = firstSeqNeedsUpdate || seq == fs.state.FirstSeq } else if seq == fs.state.FirstSeq { fs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq) // new one. fs.state.FirstTime = time.Unix(0, mb.first.ts).UTC() } } else { // Out of order delete. mb.dmap.Insert(seq) } if maxp > 0 && purged >= maxp { break } } } // Expire if we were responsible for loading. if shouldExpire { // Expire this cache before moving on. mb.tryForceExpireCacheLocked() } mb.mu.Unlock() // Check if we should break out of top level too. if maxp > 0 && purged >= maxp { break } } if firstSeqNeedsUpdate { fs.selectNextFirst() } fseq := fs.state.FirstSeq // Write any tombstones as needed. for _, tomb := range tombs { if tomb.seq > fseq { fs.writeTombstone(tomb.seq, tomb.ts) } } os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)) fs.dirty++ cb := fs.scb fs.mu.Unlock() if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return purged, nil } // Purge will remove all messages from this store. // Will return the number of purged messages. func (fs *fileStore) Purge() (uint64, error) { return fs.purge(0) } func (fs *fileStore) purge(fseq uint64) (uint64, error) { fs.mu.Lock() if fs.closed { fs.mu.Unlock() return 0, ErrStoreClosed } purged := fs.state.Msgs rbytes := int64(fs.state.Bytes) fs.state.FirstSeq = fs.state.LastSeq + 1 fs.state.FirstTime = time.Time{} fs.state.Bytes = 0 fs.state.Msgs = 0 for _, mb := range fs.blks { mb.dirtyClose() } fs.blks = nil fs.lmb = nil fs.bim = make(map[uint32]*msgBlock) // Clear any per subject tracking. fs.psim, fs.tsl = fs.psim.Empty(), 0 // Mark dirty. fs.dirty++ // Move the msgs directory out of the way, will delete out of band. // FIXME(dlc) - These can error and we need to change api above to propagate? mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) pdir := filepath.Join(fs.fcfg.StoreDir, purgeDir) // If purge directory still exists then we need to wait // in place and remove since rename would fail. if _, err := os.Stat(pdir); err == nil { <-dios os.RemoveAll(pdir) dios <- struct{}{} } <-dios os.Rename(mdir, pdir) dios <- struct{}{} go func() { <-dios os.RemoveAll(pdir) dios <- struct{}{} }() // Create new one. <-dios os.MkdirAll(mdir, defaultDirPerms) dios <- struct{}{} // Make sure we have a lmb to write to. if _, err := fs.newMsgBlockForWrite(); err != nil { fs.mu.Unlock() return purged, err } // Check if we need to set the first seq to a new number. if fseq > fs.state.FirstSeq { fs.state.FirstSeq = fseq fs.state.LastSeq = fseq - 1 } lmb := fs.lmb atomic.StoreUint64(&lmb.first.seq, fs.state.FirstSeq) atomic.StoreUint64(&lmb.last.seq, fs.state.LastSeq) lmb.last.ts = fs.state.LastTime.UnixNano() if lseq := atomic.LoadUint64(&lmb.last.seq); lseq > 1 { // Leave a tombstone so we can remember our starting sequence in case // full state becomes corrupted. fs.writeTombstone(lseq, lmb.last.ts) } cb := fs.scb fs.mu.Unlock() // Force a new index.db to be written. if purged > 0 { fs.forceWriteFullState() } if cb != nil { cb(-int64(purged), -rbytes, 0, _EMPTY_) } return purged, nil } // Compact will remove all messages from this store up to // but not including the seq parameter. // Will return the number of purged messages. func (fs *fileStore) Compact(seq uint64) (uint64, error) { if seq == 0 { return fs.purge(seq) } var purged, bytes uint64 fs.mu.Lock() // Same as purge all. if lseq := fs.state.LastSeq; seq > lseq { fs.mu.Unlock() return fs.purge(seq) } // We have to delete interior messages. smb := fs.selectMsgBlock(seq) if smb == nil { fs.mu.Unlock() return 0, nil } // All msgblocks up to this one can be thrown away. var deleted int for _, mb := range fs.blks { if mb == smb { break } mb.mu.Lock() purged += mb.msgs bytes += mb.bytes // Make sure we do subject cleanup as well. mb.ensurePerSubjectInfoLoaded() mb.fss.IterOrdered(func(bsubj []byte, ss *SimpleState) bool { subj := bytesToString(bsubj) for i := uint64(0); i < ss.Msgs; i++ { fs.removePerSubject(subj) } return true }) // Now close. mb.dirtyCloseWithRemove(true) mb.mu.Unlock() deleted++ } var smv StoreMsg var err error smb.mu.Lock() if atomic.LoadUint64(&smb.first.seq) == seq { fs.state.FirstSeq = atomic.LoadUint64(&smb.first.seq) fs.state.FirstTime = time.Unix(0, smb.first.ts).UTC() goto SKIP } // Make sure we have the messages loaded. if smb.cacheNotLoaded() { if err = smb.loadMsgsWithLock(); err != nil { goto SKIP } } for mseq := atomic.LoadUint64(&smb.first.seq); mseq < seq; mseq++ { sm, err := smb.cacheLookup(mseq, &smv) if err == errDeletedMsg { // Update dmap. if !smb.dmap.IsEmpty() { smb.dmap.Delete(mseq) } } else if sm != nil { sz := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg) if smb.msgs > 0 { smb.msgs-- if sz > smb.bytes { sz = smb.bytes } smb.bytes -= sz bytes += sz purged++ } // Update fss smb.removeSeqPerSubject(sm.subj, mseq) fs.removePerSubject(sm.subj) } } // Check if empty after processing, could happen if tail of messages are all deleted. if isEmpty := smb.msgs == 0; isEmpty { // Only remove if not the last block. if smb != fs.lmb { smb.dirtyCloseWithRemove(true) deleted++ } else { // Make sure to sync changes. smb.needSync = true } // Update fs first here as well. fs.state.FirstSeq = atomic.LoadUint64(&smb.last.seq) + 1 fs.state.FirstTime = time.Time{} } else { // Make sure to sync changes. smb.needSync = true // Update fs first seq and time. atomic.StoreUint64(&smb.first.seq, seq-1) // Just for start condition for selectNextFirst. smb.selectNextFirst() fs.state.FirstSeq = atomic.LoadUint64(&smb.first.seq) fs.state.FirstTime = time.Unix(0, smb.first.ts).UTC() // Check if we should reclaim the head space from this block. // This will be optimistic only, so don't continue if we encounter any errors here. if smb.rbytes > compactMinimum && smb.bytes*2 < smb.rbytes { var moff uint32 moff, _, _, err = smb.slotInfo(int(atomic.LoadUint64(&smb.first.seq) - smb.cache.fseq)) if err != nil || moff >= uint32(len(smb.cache.buf)) { goto SKIP } buf := smb.cache.buf[moff:] // Don't reuse, copy to new recycled buf. nbuf := getMsgBlockBuf(len(buf)) nbuf = append(nbuf, buf...) smb.closeFDsLockedNoCheck() // Check for encryption. if smb.bek != nil && len(nbuf) > 0 { // Recreate to reset counter. bek, err := genBlockEncryptionKey(smb.fs.fcfg.Cipher, smb.seed, smb.nonce) if err != nil { goto SKIP } // For future writes make sure to set smb.bek to keep counter correct. smb.bek = bek smb.bek.XORKeyStream(nbuf, nbuf) } // Recompress if necessary (smb.cmp contains the algorithm used when // the block was loaded from disk, or defaults to NoCompression if not) if nbuf, err = smb.cmp.Compress(nbuf); err != nil { goto SKIP } <-dios err = os.WriteFile(smb.mfn, nbuf, defaultFilePerms) dios <- struct{}{} if err != nil { goto SKIP } // Make sure to remove fss state. smb.fss = nil smb.clearCacheAndOffset() smb.rbytes = uint64(len(nbuf)) } } SKIP: smb.mu.Unlock() if deleted > 0 { // Update block map. if fs.bim != nil { for _, mb := range fs.blks[:deleted] { delete(fs.bim, mb.index) } } // Update blks slice. fs.blks = copyMsgBlocks(fs.blks[deleted:]) if lb := len(fs.blks); lb == 0 { fs.lmb = nil } else { fs.lmb = fs.blks[lb-1] } } // Update top level accounting. if purged > fs.state.Msgs { purged = fs.state.Msgs } fs.state.Msgs -= purged if fs.state.Msgs == 0 { fs.state.FirstSeq = fs.state.LastSeq + 1 fs.state.FirstTime = time.Time{} } if bytes > fs.state.Bytes { bytes = fs.state.Bytes } fs.state.Bytes -= bytes // Any existing state file no longer applicable. We will force write a new one // after we release the lock. os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)) fs.dirty++ cb := fs.scb fs.mu.Unlock() // Force a new index.db to be written. if purged > 0 { fs.forceWriteFullState() } if cb != nil && purged > 0 { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return purged, err } // Will completely reset our store. func (fs *fileStore) reset() error { fs.mu.Lock() if fs.closed { fs.mu.Unlock() return ErrStoreClosed } if fs.sips > 0 { fs.mu.Unlock() return ErrStoreSnapshotInProgress } var purged, bytes uint64 cb := fs.scb for _, mb := range fs.blks { mb.mu.Lock() purged += mb.msgs bytes += mb.bytes mb.dirtyCloseWithRemove(true) mb.mu.Unlock() } // Reset fs.state.FirstSeq = 0 fs.state.FirstTime = time.Time{} fs.state.LastSeq = 0 fs.state.LastTime = time.Now().UTC() // Update msgs and bytes. fs.state.Msgs = 0 fs.state.Bytes = 0 // Reset blocks. fs.blks, fs.lmb = nil, nil // Reset subject mappings. fs.psim, fs.tsl = fs.psim.Empty(), 0 fs.bim = make(map[uint32]*msgBlock) // If we purged anything, make sure we kick flush state loop. if purged > 0 { fs.dirty++ } fs.mu.Unlock() if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return nil } // Return all active tombstones in this msgBlock. func (mb *msgBlock) tombs() []msgId { mb.mu.Lock() defer mb.mu.Unlock() return mb.tombsLocked() } // Return all active tombstones in this msgBlock. // Write lock should be held. func (mb *msgBlock) tombsLocked() []msgId { if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return nil } } var tombs []msgId var le = binary.LittleEndian buf := mb.cache.buf for index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; { if index+msgHdrSize > lbuf { return tombs } hdr := buf[index : index+msgHdrSize] rl, seq := le.Uint32(hdr[0:]), le.Uint64(hdr[4:]) // Clear any headers bit that could be set. rl &^= hbit // Check for tombstones. if seq&tbit != 0 { ts := int64(le.Uint64(hdr[12:])) tombs = append(tombs, msgId{seq &^ tbit, ts}) } // Advance to next record. index += rl } return tombs } // Truncate will truncate a stream store up to seq. Sequence needs to be valid. func (fs *fileStore) Truncate(seq uint64) error { // Check for request to reset. if seq == 0 { return fs.reset() } fs.mu.Lock() if fs.closed { fs.mu.Unlock() return ErrStoreClosed } if fs.sips > 0 { fs.mu.Unlock() return ErrStoreSnapshotInProgress } nlmb := fs.selectMsgBlock(seq) if nlmb == nil { fs.mu.Unlock() return ErrInvalidSequence } lsm, _, _ := nlmb.fetchMsg(seq, nil) if lsm == nil { fs.mu.Unlock() return ErrInvalidSequence } // Set lmb to nlmb and make sure writeable. fs.lmb = nlmb if err := nlmb.enableForWriting(fs.fip); err != nil { fs.mu.Unlock() return err } // Collect all tombstones, we want to put these back so we can survive // a restore without index.db properly. var tombs []msgId tombs = append(tombs, nlmb.tombs()...) var purged, bytes uint64 // Truncate our new last message block. nmsgs, nbytes, err := nlmb.truncate(lsm) if err != nil { fs.mu.Unlock() return fmt.Errorf("nlmb.truncate: %w", err) } // Account for the truncated msgs and bytes. purged += nmsgs bytes += nbytes // Remove any left over msg blocks. getLastMsgBlock := func() *msgBlock { return fs.blks[len(fs.blks)-1] } for mb := getLastMsgBlock(); mb != nlmb; mb = getLastMsgBlock() { mb.mu.Lock() // We do this to load tombs. tombs = append(tombs, mb.tombsLocked()...) purged += mb.msgs bytes += mb.bytes fs.removeMsgBlock(mb) mb.mu.Unlock() } // Reset last. fs.state.LastSeq = lsm.seq fs.state.LastTime = time.Unix(0, lsm.ts).UTC() // Update msgs and bytes. if purged > fs.state.Msgs { purged = fs.state.Msgs } fs.state.Msgs -= purged if bytes > fs.state.Bytes { bytes = fs.state.Bytes } fs.state.Bytes -= bytes // Reset our subject lookup info. fs.resetGlobalPerSubjectInfo() // Always create new write block. fs.newMsgBlockForWrite() // Write any tombstones as needed. for _, tomb := range tombs { if tomb.seq <= lsm.seq { fs.writeTombstone(tomb.seq, tomb.ts) } } // Any existing state file no longer applicable. We will force write a new one // after we release the lock. os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)) fs.dirty++ cb := fs.scb fs.mu.Unlock() // Force a new index.db to be written. if purged > 0 { fs.forceWriteFullState() } if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return nil } func (fs *fileStore) lastSeq() uint64 { fs.mu.RLock() seq := fs.state.LastSeq fs.mu.RUnlock() return seq } // Returns number of msg blks. func (fs *fileStore) numMsgBlocks() int { fs.mu.RLock() defer fs.mu.RUnlock() return len(fs.blks) } // Will add a new msgBlock. // Lock should be held. func (fs *fileStore) addMsgBlock(mb *msgBlock) { fs.blks = append(fs.blks, mb) fs.lmb = mb fs.bim[mb.index] = mb } // Remove from our list of blks. // Both locks should be held. func (fs *fileStore) removeMsgBlockFromList(mb *msgBlock) { // Remove from list. for i, omb := range fs.blks { if mb == omb { fs.dirty++ blks := append(fs.blks[:i], fs.blks[i+1:]...) fs.blks = copyMsgBlocks(blks) if fs.bim != nil { delete(fs.bim, mb.index) } break } } } // Removes the msgBlock // Both locks should be held. func (fs *fileStore) removeMsgBlock(mb *msgBlock) { mb.dirtyCloseWithRemove(true) fs.removeMsgBlockFromList(mb) // Check for us being last message block if mb == fs.lmb { lseq, lts := atomic.LoadUint64(&mb.last.seq), mb.last.ts // Creating a new message write block requires that the lmb lock is not held. mb.mu.Unlock() // Write the tombstone to remember since this was last block. if lmb, _ := fs.newMsgBlockForWrite(); lmb != nil { fs.writeTombstone(lseq, lts) } mb.mu.Lock() } } // Called by purge to simply get rid of the cache and close our fds. // Lock should not be held. func (mb *msgBlock) dirtyClose() { mb.mu.Lock() defer mb.mu.Unlock() mb.dirtyCloseWithRemove(false) } // Should be called with lock held. func (mb *msgBlock) dirtyCloseWithRemove(remove bool) error { if mb == nil { return nil } // Stop cache expiration timer. if mb.ctmr != nil { mb.ctmr.Stop() mb.ctmr = nil } // Close cache mb.clearCacheAndOffset() // Quit our loops. if mb.qch != nil { close(mb.qch) mb.qch = nil } if mb.mfd != nil { mb.mfd.Close() mb.mfd = nil } if remove { // Clear any tracking by subject if we are removing. mb.fss = nil if mb.mfn != _EMPTY_ { err := os.Remove(mb.mfn) if isPermissionError(err) { return err } mb.mfn = _EMPTY_ } if mb.kfn != _EMPTY_ { err := os.Remove(mb.kfn) if isPermissionError(err) { return err } } } return nil } // Remove a seq from the fss and select new first. // Lock should be held. func (mb *msgBlock) removeSeqPerSubject(subj string, seq uint64) { mb.ensurePerSubjectInfoLoaded() if mb.fss == nil { return } bsubj := stringToBytes(subj) ss, ok := mb.fss.Find(bsubj) if !ok || ss == nil { return } if ss.Msgs == 1 { mb.fss.Delete(bsubj) return } ss.Msgs-- // Only one left. if ss.Msgs == 1 { if !ss.lastNeedsUpdate && seq != ss.Last { ss.First = ss.Last ss.firstNeedsUpdate = false return } if !ss.firstNeedsUpdate && seq != ss.First { ss.Last = ss.First ss.lastNeedsUpdate = false return } } // We can lazily calculate the first/last sequence when needed. ss.firstNeedsUpdate = seq == ss.First || ss.firstNeedsUpdate ss.lastNeedsUpdate = seq == ss.Last || ss.lastNeedsUpdate } // Will recalculate the first and/or last sequence for this subject in this block. // Will avoid slower path message lookups and scan the cache directly instead. func (mb *msgBlock) recalculateForSubj(subj string, ss *SimpleState) { // Need to make sure messages are loaded. if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return } } startSlot := int(ss.First - mb.cache.fseq) if startSlot < 0 { startSlot = 0 } if startSlot >= len(mb.cache.idx) { ss.First = ss.Last ss.firstNeedsUpdate = false ss.lastNeedsUpdate = false return } endSlot := int(ss.Last - mb.cache.fseq) if endSlot < 0 { endSlot = 0 } if endSlot >= len(mb.cache.idx) || startSlot > endSlot { return } var le = binary.LittleEndian if ss.firstNeedsUpdate { // Mark first as updated. ss.firstNeedsUpdate = false fseq := ss.First + 1 if mbFseq := atomic.LoadUint64(&mb.first.seq); fseq < mbFseq { fseq = mbFseq } for slot := startSlot; slot < len(mb.cache.idx); slot++ { bi := mb.cache.idx[slot] &^ hbit if bi == dbit { // delete marker so skip. continue } li := int(bi) - mb.cache.off if li >= len(mb.cache.buf) { ss.First = ss.Last // Only need to reset ss.lastNeedsUpdate, ss.firstNeedsUpdate is already reset above. ss.lastNeedsUpdate = false return } buf := mb.cache.buf[li:] hdr := buf[:msgHdrSize] slen := int(le.Uint16(hdr[20:])) if subj == bytesToString(buf[msgHdrSize:msgHdrSize+slen]) { seq := le.Uint64(hdr[4:]) if seq < fseq || seq&ebit != 0 || mb.dmap.Exists(seq) { continue } ss.First = seq if ss.Msgs == 1 { ss.Last = seq ss.lastNeedsUpdate = false return } // Skip the start slot ahead, if we need to recalculate last we can stop early. startSlot = slot break } } } if ss.lastNeedsUpdate { // Mark last as updated. ss.lastNeedsUpdate = false lseq := ss.Last - 1 if mbLseq := atomic.LoadUint64(&mb.last.seq); lseq > mbLseq { lseq = mbLseq } for slot := endSlot; slot >= startSlot; slot-- { bi := mb.cache.idx[slot] &^ hbit if bi == dbit { // delete marker so skip. continue } li := int(bi) - mb.cache.off if li >= len(mb.cache.buf) { // Can't overwrite ss.Last, just skip. return } buf := mb.cache.buf[li:] hdr := buf[:msgHdrSize] slen := int(le.Uint16(hdr[20:])) if subj == bytesToString(buf[msgHdrSize:msgHdrSize+slen]) { seq := le.Uint64(hdr[4:]) if seq > lseq || seq&ebit != 0 || mb.dmap.Exists(seq) { continue } // Sequence should never be lower, but guard against it nonetheless. if seq < ss.First { seq = ss.First } ss.Last = seq if ss.Msgs == 1 { ss.First = seq ss.firstNeedsUpdate = false } return } } } } // Lock should be held. func (fs *fileStore) resetGlobalPerSubjectInfo() { // Clear any global subject state. fs.psim, fs.tsl = fs.psim.Empty(), 0 for _, mb := range fs.blks { fs.populateGlobalPerSubjectInfo(mb) } } // Lock should be held. func (mb *msgBlock) resetPerSubjectInfo() error { mb.fss = nil return mb.generatePerSubjectInfo() } // generatePerSubjectInfo will generate the per subject info via the raw msg block. // Lock should be held. func (mb *msgBlock) generatePerSubjectInfo() error { // Check if this mb is empty. This can happen when its the last one and we are holding onto it for seq and timestamp info. if mb.msgs == 0 { return nil } if mb.cacheNotLoaded() { if err := mb.loadMsgsWithLock(); err != nil { return err } // indexCacheBuf can produce fss now, so if non-nil we are good. if mb.fss != nil { return nil } } // Create new one regardless. mb.fss = stree.NewSubjectTree[SimpleState]() var smv StoreMsg fseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) for seq := fseq; seq <= lseq; seq++ { if mb.dmap.Exists(seq) { // Optimisation to avoid calling cacheLookup which hits time.Now(). // It gets set later on if the fss is non-empty anyway. continue } sm, err := mb.cacheLookup(seq, &smv) if err != nil { // Since we are walking by sequence we can ignore some errors that are benign to rebuilding our state. if err == ErrStoreMsgNotFound || err == errDeletedMsg { continue } if err == errNoCache { return nil } return err } if sm != nil && len(sm.subj) > 0 { if ss, ok := mb.fss.Find(stringToBytes(sm.subj)); ok && ss != nil { ss.Msgs++ ss.Last = seq ss.lastNeedsUpdate = false } else { mb.fss.Insert(stringToBytes(sm.subj), SimpleState{Msgs: 1, First: seq, Last: seq}) } } } if mb.fss.Size() > 0 { // Make sure we run the cache expire timer. mb.llts = time.Now().UnixNano() // Mark fss activity. mb.lsts = time.Now().UnixNano() mb.startCacheExpireTimer() } return nil } // Helper to make sure fss loaded if we are tracking. // Lock should be held func (mb *msgBlock) ensurePerSubjectInfoLoaded() error { if mb.fss != nil || mb.noTrack { if mb.fss != nil { // Mark fss activity. mb.lsts = time.Now().UnixNano() } return nil } if mb.msgs == 0 { mb.fss = stree.NewSubjectTree[SimpleState]() return nil } return mb.generatePerSubjectInfo() } // Called on recovery to populate the global psim state. // Lock should be held. func (fs *fileStore) populateGlobalPerSubjectInfo(mb *msgBlock) { mb.mu.Lock() defer mb.mu.Unlock() if err := mb.ensurePerSubjectInfoLoaded(); err != nil { return } // Now populate psim. mb.fss.IterFast(func(bsubj []byte, ss *SimpleState) bool { if len(bsubj) > 0 { if info, ok := fs.psim.Find(bsubj); ok { info.total += ss.Msgs if mb.index > info.lblk { info.lblk = mb.index } } else { fs.psim.Insert(bsubj, psi{total: ss.Msgs, fblk: mb.index, lblk: mb.index}) fs.tsl += len(bsubj) } } return true }) } // Close the message block. func (mb *msgBlock) close(sync bool) { if mb == nil { return } mb.mu.Lock() defer mb.mu.Unlock() if mb.closed { return } // Stop cache expiration timer. if mb.ctmr != nil { mb.ctmr.Stop() mb.ctmr = nil } // Clear fss. mb.fss = nil // Close cache mb.clearCacheAndOffset() // Quit our loops. if mb.qch != nil { close(mb.qch) mb.qch = nil } if mb.mfd != nil { if sync { mb.mfd.Sync() } mb.mfd.Close() } mb.mfd = nil // Mark as closed. mb.closed = true } func (fs *fileStore) closeAllMsgBlocks(sync bool) { for _, mb := range fs.blks { mb.close(sync) } } func (fs *fileStore) Delete() error { if fs.isClosed() { // Always attempt to remove since we could have been closed beforehand. os.RemoveAll(fs.fcfg.StoreDir) // Since we did remove, if we did have anything remaining make sure to // call into any storage updates that had been registered. fs.mu.Lock() cb, msgs, bytes := fs.scb, int64(fs.state.Msgs), int64(fs.state.Bytes) // Guard against double accounting if called twice. fs.state.Msgs, fs.state.Bytes = 0, 0 fs.mu.Unlock() if msgs > 0 && cb != nil { cb(-msgs, -bytes, 0, _EMPTY_) } return ErrStoreClosed } pdir := filepath.Join(fs.fcfg.StoreDir, purgeDir) // If purge directory still exists then we need to wait // in place and remove since rename would fail. if _, err := os.Stat(pdir); err == nil { os.RemoveAll(pdir) } // Quickly close all blocks and simulate a purge w/o overhead an new write block. fs.mu.Lock() for _, mb := range fs.blks { mb.dirtyClose() } dmsgs := fs.state.Msgs dbytes := int64(fs.state.Bytes) fs.state.Msgs, fs.state.Bytes = 0, 0 fs.blks = nil cb := fs.scb fs.mu.Unlock() if cb != nil { cb(-int64(dmsgs), -dbytes, 0, _EMPTY_) } if err := fs.stop(true, false); err != nil { return err } // Make sure we will not try to recover if killed before removal below completes. if err := os.Remove(filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFile)); err != nil { return err } // Now move into different directory with "." prefix. ndir := filepath.Join(filepath.Dir(fs.fcfg.StoreDir), tsep+filepath.Base(fs.fcfg.StoreDir)) if err := os.Rename(fs.fcfg.StoreDir, ndir); err != nil { return err } // Do this in separate Go routine in case lots of blocks. // Purge above protects us as does the removal of meta artifacts above. go func() { <-dios err := os.RemoveAll(ndir) dios <- struct{}{} if err == nil { return } ttl := time.Now().Add(time.Second) for time.Now().Before(ttl) { time.Sleep(10 * time.Millisecond) <-dios err = os.RemoveAll(ndir) dios <- struct{}{} if err == nil { return } } }() return nil } // Lock should be held. func (fs *fileStore) setSyncTimer() { if fs.syncTmr != nil { fs.syncTmr.Reset(fs.fcfg.SyncInterval) } else { // First time this fires will be between SyncInterval/2 and SyncInterval, // so that different stores are spread out, rather than having many of // them trying to all sync at once, causing blips and contending dios. start := (fs.fcfg.SyncInterval / 2) + (time.Duration(mrand.Int63n(int64(fs.fcfg.SyncInterval / 2)))) fs.syncTmr = time.AfterFunc(start, fs.syncBlocks) } } // Lock should be held. func (fs *fileStore) cancelSyncTimer() { if fs.syncTmr != nil { fs.syncTmr.Stop() fs.syncTmr = nil } } const ( fullStateMagic = uint8(11) fullStateVersion = uint8(1) ) // This go routine periodically writes out our full stream state index. func (fs *fileStore) flushStreamStateLoop(qch, done chan struct{}) { // Signal we are done on exit. defer close(done) // Make sure we do not try to write these out too fast. // Spread these out for large numbers on a server restart. const writeThreshold = 2 * time.Minute writeJitter := time.Duration(mrand.Int63n(int64(30 * time.Second))) t := time.NewTicker(writeThreshold + writeJitter) defer t.Stop() for { select { case <-t.C: err := fs.writeFullState() if isPermissionError(err) && fs.srv != nil { fs.warn("File system permission denied when flushing stream state, disabling JetStream: %v", err) // messages in block cache could be lost in the worst case. // In the clustered mode it is very highly unlikely as a result of replication. fs.srv.DisableJetStream() return } case <-qch: return } } } // Helper since unixnano of zero time undefined. func timestampNormalized(t time.Time) int64 { if t.IsZero() { return 0 } return t.UnixNano() } // writeFullState will proceed to write the full meta state iff not complex and time consuming. // Since this is for quick recovery it is optional and should not block/stall normal operations. func (fs *fileStore) writeFullState() error { return fs._writeFullState(false) } // forceWriteFullState will proceed to write the full meta state. This should only be called by stop() func (fs *fileStore) forceWriteFullState() error { return fs._writeFullState(true) } // This will write the full binary state for the stream. // This plus everything new since last hash will be the total recovered state. // This state dump will have the following. // 1. Stream summary - Msgs, Bytes, First and Last (Sequence and Timestamp) // 2. PSIM - Per Subject Index Map - Tracks first and last blocks with subjects present. // 3. MBs - Index, Bytes, First and Last Sequence and Timestamps, and the deleted map (avl.seqset). // 4. Last block index and hash of record inclusive to this stream state. func (fs *fileStore) _writeFullState(force bool) error { start := time.Now() fs.mu.Lock() if fs.closed || fs.dirty == 0 { fs.mu.Unlock() return nil } // For calculating size and checking time costs for non forced calls. numSubjects := fs.numSubjects() // If we are not being forced to write out our state, check the complexity for time costs as to not // block or stall normal operations. // We will base off of number of subjects and interior deletes. A very large number of msg blocks could also // be used, but for next server version will redo all meta handling to be disk based. So this is temporary. if !force { const numThreshold = 1_000_000 // Calculate interior deletes. var numDeleted int if fs.state.LastSeq > fs.state.FirstSeq { numDeleted = int((fs.state.LastSeq - fs.state.FirstSeq + 1) - fs.state.Msgs) } if numSubjects > numThreshold || numDeleted > numThreshold { fs.mu.Unlock() return errStateTooBig } } // We track this through subsequent runs to get an avg per blk used for subsequent runs. avgDmapLen := fs.adml // If first time through could be 0 if avgDmapLen == 0 && ((fs.state.LastSeq-fs.state.FirstSeq+1)-fs.state.Msgs) > 0 { avgDmapLen = 1024 } // Calculate and estimate of the uper bound on the size to avoid multiple allocations. sz := hdrLen + // Magic and Version (binary.MaxVarintLen64 * 6) + // FS data binary.MaxVarintLen64 + fs.tsl + // NumSubjects + total subject length numSubjects*(binary.MaxVarintLen64*4) + // psi record binary.MaxVarintLen64 + // Num blocks. len(fs.blks)*((binary.MaxVarintLen64*7)+avgDmapLen) + // msg blocks, avgDmapLen is est for dmaps binary.MaxVarintLen64 + 8 + 8 // last index + record checksum + full state checksum // Do 4k on stack if possible. const ssz = 4 * 1024 var buf []byte if sz <= ssz { var _buf [ssz]byte buf, sz = _buf[0:hdrLen:ssz], ssz } else { buf = make([]byte, hdrLen, sz) } buf[0], buf[1] = fullStateMagic, fullStateVersion buf = binary.AppendUvarint(buf, fs.state.Msgs) buf = binary.AppendUvarint(buf, fs.state.Bytes) buf = binary.AppendUvarint(buf, fs.state.FirstSeq) buf = binary.AppendVarint(buf, timestampNormalized(fs.state.FirstTime)) buf = binary.AppendUvarint(buf, fs.state.LastSeq) buf = binary.AppendVarint(buf, timestampNormalized(fs.state.LastTime)) // Do per subject information map if applicable. buf = binary.AppendUvarint(buf, uint64(numSubjects)) if numSubjects > 0 { fs.psim.Match([]byte(fwcs), func(subj []byte, psi *psi) { buf = binary.AppendUvarint(buf, uint64(len(subj))) buf = append(buf, subj...) buf = binary.AppendUvarint(buf, psi.total) buf = binary.AppendUvarint(buf, uint64(psi.fblk)) if psi.total > 1 { buf = binary.AppendUvarint(buf, uint64(psi.lblk)) } }) } // Now walk all blocks and write out first and last and optional dmap encoding. var lbi uint32 var lchk [8]byte nb := len(fs.blks) buf = binary.AppendUvarint(buf, uint64(nb)) // Use basetime to save some space. baseTime := timestampNormalized(fs.state.FirstTime) var scratch [8 * 1024]byte // Track the state as represented by the mbs. var mstate StreamState var dmapTotalLen int for _, mb := range fs.blks { mb.mu.RLock() buf = binary.AppendUvarint(buf, uint64(mb.index)) buf = binary.AppendUvarint(buf, mb.bytes) buf = binary.AppendUvarint(buf, atomic.LoadUint64(&mb.first.seq)) buf = binary.AppendVarint(buf, mb.first.ts-baseTime) buf = binary.AppendUvarint(buf, atomic.LoadUint64(&mb.last.seq)) buf = binary.AppendVarint(buf, mb.last.ts-baseTime) numDeleted := mb.dmap.Size() buf = binary.AppendUvarint(buf, uint64(numDeleted)) if numDeleted > 0 { dmap, _ := mb.dmap.Encode(scratch[:0]) dmapTotalLen += len(dmap) buf = append(buf, dmap...) } // If this is the last one grab the last checksum and the block index, e.g. 22.blk, 22 is the block index. // We use this to quickly open this file on recovery. if mb == fs.lmb { lbi = mb.index mb.ensureLastChecksumLoaded() copy(lchk[0:], mb.lchk[:]) } updateTrackingState(&mstate, mb) mb.mu.RUnlock() } if dmapTotalLen > 0 { fs.adml = dmapTotalLen / len(fs.blks) } // Place block index and hash onto the end. buf = binary.AppendUvarint(buf, uint64(lbi)) buf = append(buf, lchk[:]...) // Encrypt if needed. if fs.prf != nil { if err := fs.setupAEK(); err != nil { fs.mu.Unlock() return err } nonce := make([]byte, fs.aek.NonceSize(), fs.aek.NonceSize()+len(buf)+fs.aek.Overhead()) if n, err := rand.Read(nonce); err != nil { return err } else if n != len(nonce) { return fmt.Errorf("not enough nonce bytes read (%d != %d)", n, len(nonce)) } buf = fs.aek.Seal(nonce, nonce, buf, nil) } fn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) fs.hh.Reset() fs.hh.Write(buf) buf = fs.hh.Sum(buf) // Snapshot prior dirty count. priorDirty := fs.dirty statesEqual := trackingStatesEqual(&fs.state, &mstate) // Release lock. fs.mu.Unlock() // Check consistency here. if !statesEqual { fs.warn("Stream state encountered internal inconsistency on write") // Rebuild our fs state from the mb state. fs.rebuildState(nil) return errCorruptState } if cap(buf) > sz { fs.debug("WriteFullState reallocated from %d to %d", sz, cap(buf)) } // Only warn about construction time since file write not holding any locks. if took := time.Since(start); took > time.Minute { fs.warn("WriteFullState took %v (%d bytes)", took.Round(time.Millisecond), len(buf)) } // Write our update index.db // Protect with dios. <-dios err := os.WriteFile(fn, buf, defaultFilePerms) // if file system is not writable isPermissionError is set to true dios <- struct{}{} if isPermissionError(err) { return err } // Update dirty if successful. if err == nil { fs.mu.Lock() fs.dirty -= priorDirty fs.mu.Unlock() } return nil } // Stop the current filestore. func (fs *fileStore) Stop() error { return fs.stop(false, true) } // Stop the current filestore. func (fs *fileStore) stop(delete, writeState bool) error { fs.mu.Lock() if fs.closed || fs.closing { fs.mu.Unlock() return ErrStoreClosed } // Mark as closing. Do before releasing the lock to writeFullState // so we don't end up with this function running more than once. fs.closing = true if writeState { fs.checkAndFlushAllBlocks() } fs.closeAllMsgBlocks(false) fs.cancelSyncTimer() fs.cancelAgeChk() // Release the state flusher loop. if fs.qch != nil { close(fs.qch) fs.qch = nil } if writeState { // Wait for the state flush loop to exit. fsld := fs.fsld fs.mu.Unlock() <-fsld // Write full state if needed. If not dirty this is a no-op. fs.forceWriteFullState() fs.mu.Lock() } // Mark as closed. Last message block needs to be cleared after // writeFullState has completed. fs.closed = true fs.lmb = nil // We should update the upper usage layer on a stop. cb, bytes := fs.scb, int64(fs.state.Bytes) fs.mu.Unlock() fs.cmu.Lock() var _cfs [256]ConsumerStore cfs := append(_cfs[:0], fs.cfs...) fs.cfs = nil fs.cmu.Unlock() for _, o := range cfs { if delete { o.StreamDelete() } else { o.Stop() } } if bytes > 0 && cb != nil { cb(0, -bytes, 0, _EMPTY_) } return nil } const errFile = "errors.txt" // Stream our snapshot through S2 compression and tar. func (fs *fileStore) streamSnapshot(w io.WriteCloser, includeConsumers bool) { defer w.Close() enc := s2.NewWriter(w) defer enc.Close() tw := tar.NewWriter(enc) defer tw.Close() defer func() { fs.mu.Lock() fs.sips-- fs.mu.Unlock() }() modTime := time.Now().UTC() writeFile := func(name string, buf []byte) error { hdr := &tar.Header{ Name: name, Mode: 0600, ModTime: modTime, Uname: "nats", Gname: "nats", Size: int64(len(buf)), Format: tar.FormatPAX, } if err := tw.WriteHeader(hdr); err != nil { return err } if _, err := tw.Write(buf); err != nil { return err } return nil } writeErr := func(err string) { writeFile(errFile, []byte(err)) } fs.mu.Lock() blks := fs.blks // Grab our general meta data. // We do this now instead of pulling from files since they could be encrypted. meta, err := json.Marshal(fs.cfg) if err != nil { fs.mu.Unlock() writeErr(fmt.Sprintf("Could not gather stream meta file: %v", err)) return } hh := fs.hh hh.Reset() hh.Write(meta) sum := []byte(hex.EncodeToString(fs.hh.Sum(nil))) fs.mu.Unlock() // Meta first. if writeFile(JetStreamMetaFile, meta) != nil { return } if writeFile(JetStreamMetaFileSum, sum) != nil { return } // Can't use join path here, tar only recognizes relative paths with forward slashes. msgPre := msgDir + "/" var bbuf []byte // Now do messages themselves. for _, mb := range blks { if mb.pendingWriteSize() > 0 { mb.flushPendingMsgs() } mb.mu.Lock() // We could stream but don't want to hold the lock and prevent changes, so just read in and // release the lock for now. bbuf, err = mb.loadBlock(bbuf) if err != nil { mb.mu.Unlock() writeErr(fmt.Sprintf("Could not read message block [%d]: %v", mb.index, err)) return } // Check for encryption. if mb.bek != nil && len(bbuf) > 0 { rbek, err := genBlockEncryptionKey(fs.fcfg.Cipher, mb.seed, mb.nonce) if err != nil { mb.mu.Unlock() writeErr(fmt.Sprintf("Could not create encryption key for message block [%d]: %v", mb.index, err)) return } rbek.XORKeyStream(bbuf, bbuf) } // Check for compression. if bbuf, err = mb.decompressIfNeeded(bbuf); err != nil { mb.mu.Unlock() writeErr(fmt.Sprintf("Could not decompress message block [%d]: %v", mb.index, err)) return } mb.mu.Unlock() // Do this one unlocked. if writeFile(msgPre+fmt.Sprintf(blkScan, mb.index), bbuf) != nil { return } } // Do index.db last. We will force a write as well. // Write out full state as well before proceeding. if err := fs.forceWriteFullState(); err == nil { const minLen = 32 sfn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) if buf, err := os.ReadFile(sfn); err == nil && len(buf) >= minLen { if fs.aek != nil { ns := fs.aek.NonceSize() buf, err = fs.aek.Open(nil, buf[:ns], buf[ns:len(buf)-highwayhash.Size64], nil) if err == nil { // Redo hash checksum at end on plaintext. fs.mu.Lock() hh.Reset() hh.Write(buf) buf = fs.hh.Sum(buf) fs.mu.Unlock() } } if err == nil && writeFile(msgPre+streamStreamStateFile, buf) != nil { return } } } // Bail if no consumers requested. if !includeConsumers { return } // Do consumers' state last. fs.cmu.RLock() cfs := fs.cfs fs.cmu.RUnlock() for _, cs := range cfs { o, ok := cs.(*consumerFileStore) if !ok { continue } o.mu.Lock() // Grab our general meta data. // We do this now instead of pulling from files since they could be encrypted. meta, err := json.Marshal(o.cfg) if err != nil { o.mu.Unlock() writeErr(fmt.Sprintf("Could not gather consumer meta file for %q: %v", o.name, err)) return } o.hh.Reset() o.hh.Write(meta) sum := []byte(hex.EncodeToString(o.hh.Sum(nil))) // We can have the running state directly encoded now. state, err := o.encodeState() if err != nil { o.mu.Unlock() writeErr(fmt.Sprintf("Could not encode consumer state for %q: %v", o.name, err)) return } odirPre := filepath.Join(consumerDir, o.name) o.mu.Unlock() // Write all the consumer files. if writeFile(filepath.Join(odirPre, JetStreamMetaFile), meta) != nil { return } if writeFile(filepath.Join(odirPre, JetStreamMetaFileSum), sum) != nil { return } writeFile(filepath.Join(odirPre, consumerState), state) } } // Create a snapshot of this stream and its consumer's state along with messages. func (fs *fileStore) Snapshot(deadline time.Duration, checkMsgs, includeConsumers bool) (*SnapshotResult, error) { fs.mu.Lock() if fs.closed { fs.mu.Unlock() return nil, ErrStoreClosed } // Only allow one at a time. if fs.sips > 0 { fs.mu.Unlock() return nil, ErrStoreSnapshotInProgress } // Mark us as snapshotting fs.sips += 1 fs.mu.Unlock() if checkMsgs { ld := fs.checkMsgs() if ld != nil && len(ld.Msgs) > 0 { return nil, fmt.Errorf("snapshot check detected %d bad messages", len(ld.Msgs)) } } pr, pw := net.Pipe() // Set a write deadline here to protect ourselves. if deadline > 0 { pw.SetWriteDeadline(time.Now().Add(deadline)) } // We can add to our stream while snapshotting but not "user" delete anything. var state StreamState fs.FastState(&state) // Stream in separate Go routine. go fs.streamSnapshot(pw, includeConsumers) return &SnapshotResult{pr, state}, nil } // Helper to return the config. func (fs *fileStore) fileStoreConfig() FileStoreConfig { fs.mu.RLock() defer fs.mu.RUnlock() return fs.fcfg } // Read lock all existing message blocks. // Lock held on entry. func (fs *fileStore) readLockAllMsgBlocks() { for _, mb := range fs.blks { mb.mu.RLock() } } // Read unlock all existing message blocks. // Lock held on entry. func (fs *fileStore) readUnlockAllMsgBlocks() { for _, mb := range fs.blks { mb.mu.RUnlock() } } // Binary encoded state snapshot, >= v2.10 server. func (fs *fileStore) EncodedStreamState(failed uint64) ([]byte, error) { fs.mu.RLock() defer fs.mu.RUnlock() // Calculate deleted. var numDeleted int64 if fs.state.LastSeq > fs.state.FirstSeq { numDeleted = int64(fs.state.LastSeq-fs.state.FirstSeq+1) - int64(fs.state.Msgs) if numDeleted < 0 { numDeleted = 0 } } // Encoded is Msgs, Bytes, FirstSeq, LastSeq, Failed, NumDeleted and optional DeletedBlocks var buf [1024]byte buf[0], buf[1] = streamStateMagic, streamStateVersion n := hdrLen n += binary.PutUvarint(buf[n:], fs.state.Msgs) n += binary.PutUvarint(buf[n:], fs.state.Bytes) n += binary.PutUvarint(buf[n:], fs.state.FirstSeq) n += binary.PutUvarint(buf[n:], fs.state.LastSeq) n += binary.PutUvarint(buf[n:], failed) n += binary.PutUvarint(buf[n:], uint64(numDeleted)) b := buf[0:n] if numDeleted > 0 { var scratch [4 * 1024]byte fs.readLockAllMsgBlocks() defer fs.readUnlockAllMsgBlocks() for _, db := range fs.deleteBlocks() { switch db := db.(type) { case *DeleteRange: first, _, num := db.State() scratch[0] = runLengthMagic i := 1 i += binary.PutUvarint(scratch[i:], first) i += binary.PutUvarint(scratch[i:], num) b = append(b, scratch[0:i]...) case *avl.SequenceSet: buf, err := db.Encode(scratch[:0]) if err != nil { return nil, err } b = append(b, buf...) default: return nil, errors.New("no impl") } } } return b, nil } // We used to be more sophisticated to save memory, but speed is more important. // All blocks should be at least read locked. func (fs *fileStore) deleteBlocks() DeleteBlocks { var dbs DeleteBlocks var prevLast uint64 for _, mb := range fs.blks { // Detect if we have a gap between these blocks. fseq := atomic.LoadUint64(&mb.first.seq) if prevLast > 0 && prevLast+1 != fseq { dbs = append(dbs, &DeleteRange{First: prevLast + 1, Num: fseq - prevLast - 1}) } if mb.dmap.Size() > 0 { dbs = append(dbs, &mb.dmap) } prevLast = atomic.LoadUint64(&mb.last.seq) } return dbs } // SyncDeleted will make sure this stream has same deleted state as dbs. // This will only process deleted state within our current state. func (fs *fileStore) SyncDeleted(dbs DeleteBlocks) { if len(dbs) == 0 { return } fs.mu.Lock() defer fs.mu.Unlock() lseq := fs.state.LastSeq var needsCheck DeleteBlocks fs.readLockAllMsgBlocks() mdbs := fs.deleteBlocks() for i, db := range dbs { first, last, num := db.State() // If the block is same as what we have we can skip. if i < len(mdbs) { eFirst, eLast, eNum := mdbs[i].State() if first == eFirst && last == eLast && num == eNum { continue } } else if first > lseq { // Skip blocks not applicable to our current state. continue } // Need to insert these. needsCheck = append(needsCheck, db) } fs.readUnlockAllMsgBlocks() for _, db := range needsCheck { db.Range(func(dseq uint64) bool { fs.removeMsg(dseq, false, true, false) return true }) } } //////////////////////////////////////////////////////////////////////////////// // Consumers //////////////////////////////////////////////////////////////////////////////// type consumerFileStore struct { mu sync.Mutex fs *fileStore cfg *FileConsumerInfo prf keyGen aek cipher.AEAD name string odir string ifn string hh hash.Hash64 state ConsumerState fch chan struct{} qch chan struct{} flusher bool writing bool dirty bool closed bool } func (fs *fileStore) ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerStore, error) { if fs == nil { return nil, fmt.Errorf("filestore is nil") } if fs.isClosed() { return nil, ErrStoreClosed } if cfg == nil || name == _EMPTY_ { return nil, fmt.Errorf("bad consumer config") } // We now allow overrides from a stream being a filestore type and forcing a consumer to be memory store. if cfg.MemoryStorage { // Create directly here. o := &consumerMemStore{ms: fs, cfg: *cfg} fs.AddConsumer(o) return o, nil } odir := filepath.Join(fs.fcfg.StoreDir, consumerDir, name) if err := os.MkdirAll(odir, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create consumer directory - %v", err) } csi := &FileConsumerInfo{Name: name, Created: time.Now().UTC(), ConsumerConfig: *cfg} o := &consumerFileStore{ fs: fs, cfg: csi, prf: fs.prf, name: name, odir: odir, ifn: filepath.Join(odir, consumerState), } key := sha256.Sum256([]byte(fs.cfg.Name + "/" + name)) hh, err := highwayhash.New64(key[:]) if err != nil { return nil, fmt.Errorf("could not create hash: %v", err) } o.hh = hh // Check for encryption. if o.prf != nil { if ekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey)); err == nil { if len(ekey) < minBlkKeySize { return nil, errBadKeySize } // Recover key encryption key. rb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name)) if err != nil { return nil, err } sc := fs.fcfg.Cipher kek, err := genEncryptionKey(sc, rb) if err != nil { return nil, err } ns := kek.NonceSize() nonce := ekey[:ns] seed, err := kek.Open(nil, nonce, ekey[ns:], nil) if err != nil { // We may be here on a cipher conversion, so attempt to convert. if err = o.convertCipher(); err != nil { return nil, err } } else { o.aek, err = genEncryptionKey(sc, seed) } if err != nil { return nil, err } } } // Track if we are creating the directory so that we can clean up if we encounter an error. var didCreate bool // Write our meta data iff does not exist. meta := filepath.Join(odir, JetStreamMetaFile) if _, err := os.Stat(meta); err != nil && os.IsNotExist(err) { didCreate = true csi.Created = time.Now().UTC() if err := o.writeConsumerMeta(); err != nil { os.RemoveAll(odir) return nil, err } } // If we expect to be encrypted check that what we are restoring is not plaintext. // This can happen on snapshot restores or conversions. if o.prf != nil { keyFile := filepath.Join(odir, JetStreamMetaFileKey) if _, err := os.Stat(keyFile); err != nil && os.IsNotExist(err) { if err := o.writeConsumerMeta(); err != nil { if didCreate { os.RemoveAll(odir) } return nil, err } // Redo the state file as well here if we have one and we can tell it was plaintext. if buf, err := os.ReadFile(o.ifn); err == nil { if _, err := decodeConsumerState(buf); err == nil { state, err := o.encryptState(buf) if err != nil { return nil, err } err = fs.writeFileWithOptionalSync(o.ifn, state, defaultFilePerms) if err != nil { if didCreate { os.RemoveAll(odir) } return nil, err } } } } } // Create channels to control our flush go routine. o.fch = make(chan struct{}, 1) o.qch = make(chan struct{}) go o.flushLoop(o.fch, o.qch) // Make sure to load in our state from disk if needed. o.loadState() // Assign to filestore. fs.AddConsumer(o) return o, nil } func (o *consumerFileStore) convertCipher() error { fs := o.fs odir := filepath.Join(fs.fcfg.StoreDir, consumerDir, o.name) ekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey)) if err != nil { return err } if len(ekey) < minBlkKeySize { return errBadKeySize } // Recover key encryption key. rb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name)) if err != nil { return err } // Do these in reverse since converting. sc := fs.fcfg.Cipher osc := AES if sc == AES { osc = ChaCha } kek, err := genEncryptionKey(osc, rb) if err != nil { return err } ns := kek.NonceSize() nonce := ekey[:ns] seed, err := kek.Open(nil, nonce, ekey[ns:], nil) if err != nil { return err } aek, err := genEncryptionKey(osc, seed) if err != nil { return err } // Now read in and decode our state using the old cipher. buf, err := os.ReadFile(o.ifn) if err != nil { return err } buf, err = aek.Open(nil, buf[:ns], buf[ns:], nil) if err != nil { return err } // Since we are here we recovered our old state. // Now write our meta, which will generate the new keys with the new cipher. if err := o.writeConsumerMeta(); err != nil { return err } // Now write out or state with the new cipher. return o.writeState(buf) } // Kick flusher for this consumer. // Lock should be held. func (o *consumerFileStore) kickFlusher() { if o.fch != nil { select { case o.fch <- struct{}{}: default: } } o.dirty = true } // Set in flusher status func (o *consumerFileStore) setInFlusher() { o.mu.Lock() o.flusher = true o.mu.Unlock() } // Clear in flusher status func (o *consumerFileStore) clearInFlusher() { o.mu.Lock() o.flusher = false o.mu.Unlock() } // Report in flusher status func (o *consumerFileStore) inFlusher() bool { o.mu.Lock() defer o.mu.Unlock() return o.flusher } // flushLoop watches for consumer updates and the quit channel. func (o *consumerFileStore) flushLoop(fch, qch chan struct{}) { o.setInFlusher() defer o.clearInFlusher() // Maintain approximately 10 updates per second per consumer under load. const minTime = 100 * time.Millisecond var lastWrite time.Time var dt *time.Timer setDelayTimer := func(addWait time.Duration) { if dt == nil { dt = time.NewTimer(addWait) return } if !dt.Stop() { select { case <-dt.C: default: } } dt.Reset(addWait) } for { select { case <-fch: if ts := time.Since(lastWrite); ts < minTime { setDelayTimer(minTime - ts) select { case <-dt.C: case <-qch: return } } o.mu.Lock() if o.closed { o.mu.Unlock() return } buf, err := o.encodeState() o.mu.Unlock() if err != nil { return } // TODO(dlc) - if we error should start failing upwards. if err := o.writeState(buf); err == nil { lastWrite = time.Now() } case <-qch: return } } } // SetStarting sets our starting stream sequence. func (o *consumerFileStore) SetStarting(sseq uint64) error { o.mu.Lock() o.state.Delivered.Stream = sseq buf, err := o.encodeState() o.mu.Unlock() if err != nil { return err } return o.writeState(buf) } // HasState returns if this store has a recorded state. func (o *consumerFileStore) HasState() bool { o.mu.Lock() _, err := os.Stat(o.ifn) o.mu.Unlock() return err == nil } // UpdateDelivered is called whenever a new message has been delivered. func (o *consumerFileStore) UpdateDelivered(dseq, sseq, dc uint64, ts int64) error { o.mu.Lock() defer o.mu.Unlock() if dc != 1 && o.cfg.AckPolicy == AckNone { return ErrNoAckPolicy } // On restarts the old leader may get a replay from the raft logs that are old. if dseq <= o.state.AckFloor.Consumer { return nil } // See if we expect an ack for this. if o.cfg.AckPolicy != AckNone { // Need to create pending records here. if o.state.Pending == nil { o.state.Pending = make(map[uint64]*Pending) } var p *Pending // Check for an update to a message already delivered. if sseq <= o.state.Delivered.Stream { if p = o.state.Pending[sseq]; p != nil { // Do not update p.Sequence, that should be the original delivery sequence. p.Timestamp = ts } } else { // Add to pending. o.state.Pending[sseq] = &Pending{dseq, ts} } // Update delivered as needed. if dseq > o.state.Delivered.Consumer { o.state.Delivered.Consumer = dseq } if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq } if dc > 1 { if maxdc := uint64(o.cfg.MaxDeliver); maxdc > 0 && dc > maxdc { // Make sure to remove from pending. delete(o.state.Pending, sseq) } if o.state.Redelivered == nil { o.state.Redelivered = make(map[uint64]uint64) } // Only update if greater then what we already have. if o.state.Redelivered[sseq] < dc-1 { o.state.Redelivered[sseq] = dc - 1 } } } else { // For AckNone just update delivered and ackfloor at the same time. if dseq > o.state.Delivered.Consumer { o.state.Delivered.Consumer = dseq o.state.AckFloor.Consumer = dseq } if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq o.state.AckFloor.Stream = sseq } } // Make sure we flush to disk. o.kickFlusher() return nil } // UpdateAcks is called whenever a consumer with explicit ack or ack all acks a message. func (o *consumerFileStore) UpdateAcks(dseq, sseq uint64) error { o.mu.Lock() defer o.mu.Unlock() if o.cfg.AckPolicy == AckNone { return ErrNoAckPolicy } // On restarts the old leader may get a replay from the raft logs that are old. if dseq <= o.state.AckFloor.Consumer { return nil } // Match leader logic on checking if ack is ahead of delivered. // This could happen on a cooperative takeover with high speed deliveries. if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq + 1 } if len(o.state.Pending) == 0 || o.state.Pending[sseq] == nil { delete(o.state.Redelivered, sseq) return ErrStoreMsgNotFound } // Check for AckAll here. if o.cfg.AckPolicy == AckAll { sgap := sseq - o.state.AckFloor.Stream o.state.AckFloor.Consumer = dseq o.state.AckFloor.Stream = sseq if sgap > uint64(len(o.state.Pending)) { for seq := range o.state.Pending { if seq <= sseq { delete(o.state.Pending, seq) delete(o.state.Redelivered, seq) } } } else { for seq := sseq; seq > sseq-sgap && len(o.state.Pending) > 0; seq-- { delete(o.state.Pending, seq) delete(o.state.Redelivered, seq) } } o.kickFlusher() return nil } // AckExplicit // First delete from our pending state. if p, ok := o.state.Pending[sseq]; ok { delete(o.state.Pending, sseq) if dseq > p.Sequence && p.Sequence > 0 { dseq = p.Sequence // Use the original. } } if len(o.state.Pending) == 0 { o.state.AckFloor.Consumer = o.state.Delivered.Consumer o.state.AckFloor.Stream = o.state.Delivered.Stream } else if dseq == o.state.AckFloor.Consumer+1 { o.state.AckFloor.Consumer = dseq o.state.AckFloor.Stream = sseq if o.state.Delivered.Consumer > dseq { for ss := sseq + 1; ss <= o.state.Delivered.Stream; ss++ { if p, ok := o.state.Pending[ss]; ok { if p.Sequence > 0 { o.state.AckFloor.Consumer = p.Sequence - 1 o.state.AckFloor.Stream = ss - 1 } break } } } } // We do these regardless. delete(o.state.Redelivered, sseq) o.kickFlusher() return nil } const seqsHdrSize = 6*binary.MaxVarintLen64 + hdrLen // Encode our consumer state, version 2. // Lock should be held. func (o *consumerFileStore) EncodedState() ([]byte, error) { o.mu.Lock() defer o.mu.Unlock() return o.encodeState() } func (o *consumerFileStore) encodeState() ([]byte, error) { // Grab reference to state, but make sure we load in if needed, so do not reference o.state directly. state, err := o.stateWithCopyLocked(false) if err != nil { return nil, err } return encodeConsumerState(state), nil } func (o *consumerFileStore) UpdateConfig(cfg *ConsumerConfig) error { o.mu.Lock() defer o.mu.Unlock() // This is mostly unchecked here. We are assuming the upper layers have done sanity checking. csi := o.cfg csi.ConsumerConfig = *cfg return o.writeConsumerMeta() } func (o *consumerFileStore) Update(state *ConsumerState) error { // Sanity checks. if state.AckFloor.Consumer > state.Delivered.Consumer { return fmt.Errorf("bad ack floor for consumer") } if state.AckFloor.Stream > state.Delivered.Stream { return fmt.Errorf("bad ack floor for stream") } // Copy to our state. var pending map[uint64]*Pending var redelivered map[uint64]uint64 if len(state.Pending) > 0 { pending = make(map[uint64]*Pending, len(state.Pending)) for seq, p := range state.Pending { pending[seq] = &Pending{p.Sequence, p.Timestamp} if seq <= state.AckFloor.Stream || seq > state.Delivered.Stream { return fmt.Errorf("bad pending entry, sequence [%d] out of range", seq) } } } if len(state.Redelivered) > 0 { redelivered = make(map[uint64]uint64, len(state.Redelivered)) for seq, dc := range state.Redelivered { redelivered[seq] = dc } } // Replace our state. o.mu.Lock() defer o.mu.Unlock() // Check to see if this is an outdated update. if state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream { return fmt.Errorf("old update ignored") } o.state.Delivered = state.Delivered o.state.AckFloor = state.AckFloor o.state.Pending = pending o.state.Redelivered = redelivered o.kickFlusher() return nil } // Will encrypt the state with our asset key. Will be a no-op if encryption not enabled. // Lock should be held. func (o *consumerFileStore) encryptState(buf []byte) ([]byte, error) { if o.aek == nil { return buf, nil } // TODO(dlc) - Optimize on space usage a bit? nonce := make([]byte, o.aek.NonceSize(), o.aek.NonceSize()+len(buf)+o.aek.Overhead()) if n, err := rand.Read(nonce); err != nil { return nil, err } else if n != len(nonce) { return nil, fmt.Errorf("not enough nonce bytes read (%d != %d)", n, len(nonce)) } return o.aek.Seal(nonce, nonce, buf, nil), nil } // Used to limit number of disk IO calls in flight since they could all be blocking an OS thread. // https://github.com/nats-io/nats-server/issues/2742 var dios chan struct{} // Used to setup our simplistic counting semaphore using buffered channels. // golang.org's semaphore seemed a bit heavy. func init() { // Limit ourselves to a sensible number of blocking I/O calls. Range between // 4-16 concurrent disk I/Os based on CPU cores, or 50% of cores if greater // than 32 cores. mp := runtime.GOMAXPROCS(-1) nIO := min(16, max(4, mp)) if mp > 32 { // If the system has more than 32 cores then limit dios to 50% of cores. nIO = max(16, min(mp, mp/2)) } dios = make(chan struct{}, nIO) // Fill it up to start. for i := 0; i < nIO; i++ { dios <- struct{}{} } } func (o *consumerFileStore) writeState(buf []byte) error { // Check if we have the index file open. o.mu.Lock() if o.writing || len(buf) == 0 { o.mu.Unlock() return nil } // Check on encryption. if o.aek != nil { var err error if buf, err = o.encryptState(buf); err != nil { return err } } o.writing = true o.dirty = false ifn := o.ifn o.mu.Unlock() // Lock not held here but we do limit number of outstanding calls that could block OS threads. err := o.fs.writeFileWithOptionalSync(ifn, buf, defaultFilePerms) o.mu.Lock() if err != nil { o.dirty = true } o.writing = false o.mu.Unlock() return err } // Will upodate the config. Only used when recovering ephemerals. func (o *consumerFileStore) updateConfig(cfg ConsumerConfig) error { o.mu.Lock() defer o.mu.Unlock() o.cfg = &FileConsumerInfo{ConsumerConfig: cfg} return o.writeConsumerMeta() } // Write out the consumer meta data, i.e. state. // Lock should be held. func (cfs *consumerFileStore) writeConsumerMeta() error { meta := filepath.Join(cfs.odir, JetStreamMetaFile) if _, err := os.Stat(meta); err != nil && !os.IsNotExist(err) { return err } if cfs.prf != nil && cfs.aek == nil { fs := cfs.fs key, _, _, encrypted, err := fs.genEncryptionKeys(fs.cfg.Name + tsep + cfs.name) if err != nil { return err } cfs.aek = key keyFile := filepath.Join(cfs.odir, JetStreamMetaFileKey) if _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) { return err } err = cfs.fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms) if err != nil { return err } } b, err := json.Marshal(cfs.cfg) if err != nil { return err } // Encrypt if needed. if cfs.aek != nil { nonce := make([]byte, cfs.aek.NonceSize(), cfs.aek.NonceSize()+len(b)+cfs.aek.Overhead()) if n, err := rand.Read(nonce); err != nil { return err } else if n != len(nonce) { return fmt.Errorf("not enough nonce bytes read (%d != %d)", n, len(nonce)) } b = cfs.aek.Seal(nonce, nonce, b, nil) } err = cfs.fs.writeFileWithOptionalSync(meta, b, defaultFilePerms) if err != nil { return err } cfs.hh.Reset() cfs.hh.Write(b) checksum := hex.EncodeToString(cfs.hh.Sum(nil)) sum := filepath.Join(cfs.odir, JetStreamMetaFileSum) err = cfs.fs.writeFileWithOptionalSync(sum, []byte(checksum), defaultFilePerms) if err != nil { return err } return nil } // Consumer version. func checkConsumerHeader(hdr []byte) (uint8, error) { if hdr == nil || len(hdr) < 2 || hdr[0] != magic { return 0, errCorruptState } version := hdr[1] switch version { case 1, 2: return version, nil } return 0, fmt.Errorf("unsupported version: %d", version) } func (o *consumerFileStore) copyPending() map[uint64]*Pending { pending := make(map[uint64]*Pending, len(o.state.Pending)) for seq, p := range o.state.Pending { pending[seq] = &Pending{p.Sequence, p.Timestamp} } return pending } func (o *consumerFileStore) copyRedelivered() map[uint64]uint64 { redelivered := make(map[uint64]uint64, len(o.state.Redelivered)) for seq, dc := range o.state.Redelivered { redelivered[seq] = dc } return redelivered } // Type returns the type of the underlying store. func (o *consumerFileStore) Type() StorageType { return FileStorage } // State retrieves the state from the state file. // This is not expected to be called in high performance code, only on startup. func (o *consumerFileStore) State() (*ConsumerState, error) { return o.stateWithCopy(true) } // This will not copy pending or redelivered, so should only be done under the // consumer owner's lock. func (o *consumerFileStore) BorrowState() (*ConsumerState, error) { return o.stateWithCopy(false) } func (o *consumerFileStore) stateWithCopy(doCopy bool) (*ConsumerState, error) { o.mu.Lock() defer o.mu.Unlock() return o.stateWithCopyLocked(doCopy) } // Lock should be held. func (o *consumerFileStore) stateWithCopyLocked(doCopy bool) (*ConsumerState, error) { if o.closed { return nil, ErrStoreClosed } state := &ConsumerState{} // See if we have a running state or if we need to read in from disk. if o.state.Delivered.Consumer != 0 || o.state.Delivered.Stream != 0 { state.Delivered = o.state.Delivered state.AckFloor = o.state.AckFloor if len(o.state.Pending) > 0 { if doCopy { state.Pending = o.copyPending() } else { state.Pending = o.state.Pending } } if len(o.state.Redelivered) > 0 { if doCopy { state.Redelivered = o.copyRedelivered() } else { state.Redelivered = o.state.Redelivered } } return state, nil } // Read the state in here from disk.. <-dios buf, err := os.ReadFile(o.ifn) dios <- struct{}{} if err != nil && !os.IsNotExist(err) { return nil, err } if len(buf) == 0 { return state, nil } // Check on encryption. if o.aek != nil { ns := o.aek.NonceSize() buf, err = o.aek.Open(nil, buf[:ns], buf[ns:], nil) if err != nil { return nil, err } } state, err = decodeConsumerState(buf) if err != nil { return nil, err } // Copy this state into our own. o.state.Delivered = state.Delivered o.state.AckFloor = state.AckFloor if len(state.Pending) > 0 { if doCopy { o.state.Pending = make(map[uint64]*Pending, len(state.Pending)) for seq, p := range state.Pending { o.state.Pending[seq] = &Pending{p.Sequence, p.Timestamp} } } else { o.state.Pending = state.Pending } } if len(state.Redelivered) > 0 { if doCopy { o.state.Redelivered = make(map[uint64]uint64, len(state.Redelivered)) for seq, dc := range state.Redelivered { o.state.Redelivered[seq] = dc } } else { o.state.Redelivered = state.Redelivered } } return state, nil } // Lock should be held. Called at startup. func (o *consumerFileStore) loadState() { if _, err := os.Stat(o.ifn); err == nil { // This will load our state in from disk. o.stateWithCopyLocked(false) } } // Decode consumer state. func decodeConsumerState(buf []byte) (*ConsumerState, error) { version, err := checkConsumerHeader(buf) if err != nil { return nil, err } bi := hdrLen // Helpers, will set i to -1 on error. readSeq := func() uint64 { if bi < 0 { return 0 } seq, n := binary.Uvarint(buf[bi:]) if n <= 0 { bi = -1 return 0 } bi += n return seq } readTimeStamp := func() int64 { if bi < 0 { return 0 } ts, n := binary.Varint(buf[bi:]) if n <= 0 { bi = -1 return -1 } bi += n return ts } // Just for clarity below. readLen := readSeq readCount := readSeq state := &ConsumerState{} state.AckFloor.Consumer = readSeq() state.AckFloor.Stream = readSeq() state.Delivered.Consumer = readSeq() state.Delivered.Stream = readSeq() if bi == -1 { return nil, errCorruptState } if version == 1 { // Adjust back. Version 1 also stored delivered as next to be delivered, // so adjust that back down here. if state.AckFloor.Consumer > 1 { state.Delivered.Consumer += state.AckFloor.Consumer - 1 } if state.AckFloor.Stream > 1 { state.Delivered.Stream += state.AckFloor.Stream - 1 } } // Protect ourselves against rolling backwards. const hbit = 1 << 63 if state.AckFloor.Stream&hbit != 0 || state.Delivered.Stream&hbit != 0 { return nil, errCorruptState } // We have additional stuff. if numPending := readLen(); numPending > 0 { mints := readTimeStamp() state.Pending = make(map[uint64]*Pending, numPending) for i := 0; i < int(numPending); i++ { sseq := readSeq() var dseq uint64 if version == 2 { dseq = readSeq() } ts := readTimeStamp() // Check the state machine for corruption, not the value which could be -1. if bi == -1 { return nil, errCorruptState } // Adjust seq back. sseq += state.AckFloor.Stream if sseq == 0 { return nil, errCorruptState } if version == 2 { dseq += state.AckFloor.Consumer } // Adjust the timestamp back. if version == 1 { ts = (ts + mints) * int64(time.Second) } else { ts = (mints - ts) * int64(time.Second) } // Store in pending. state.Pending[sseq] = &Pending{dseq, ts} } } // We have redelivered entries here. if numRedelivered := readLen(); numRedelivered > 0 { state.Redelivered = make(map[uint64]uint64, numRedelivered) for i := 0; i < int(numRedelivered); i++ { if seq, n := readSeq(), readCount(); seq > 0 && n > 0 { // Adjust seq back. seq += state.AckFloor.Stream state.Redelivered[seq] = n } } } return state, nil } // Stop the processing of the consumers's state. func (o *consumerFileStore) Stop() error { o.mu.Lock() if o.closed { o.mu.Unlock() return nil } if o.qch != nil { close(o.qch) o.qch = nil } var err error var buf []byte if o.dirty { // Make sure to write this out.. if buf, err = o.encodeState(); err == nil && len(buf) > 0 { if o.aek != nil { if buf, err = o.encryptState(buf); err != nil { return err } } } } o.odir = _EMPTY_ o.closed = true ifn, fs := o.ifn, o.fs o.mu.Unlock() fs.RemoveConsumer(o) if len(buf) > 0 { o.waitOnFlusher() err = o.fs.writeFileWithOptionalSync(ifn, buf, defaultFilePerms) } return err } func (o *consumerFileStore) waitOnFlusher() { if !o.inFlusher() { return } timeout := time.Now().Add(100 * time.Millisecond) for time.Now().Before(timeout) { if !o.inFlusher() { return } time.Sleep(10 * time.Millisecond) } } // Delete the consumer. func (o *consumerFileStore) Delete() error { return o.delete(false) } func (o *consumerFileStore) StreamDelete() error { return o.delete(true) } func (o *consumerFileStore) delete(streamDeleted bool) error { o.mu.Lock() if o.closed { o.mu.Unlock() return nil } if o.qch != nil { close(o.qch) o.qch = nil } var err error odir := o.odir o.odir = _EMPTY_ o.closed = true fs := o.fs o.mu.Unlock() // If our stream was not deleted this will remove the directories. if odir != _EMPTY_ && !streamDeleted { <-dios err = os.RemoveAll(odir) dios <- struct{}{} } if !streamDeleted { fs.RemoveConsumer(o) } return err } func (fs *fileStore) AddConsumer(o ConsumerStore) error { fs.cmu.Lock() defer fs.cmu.Unlock() fs.cfs = append(fs.cfs, o) return nil } func (fs *fileStore) RemoveConsumer(o ConsumerStore) error { fs.cmu.Lock() defer fs.cmu.Unlock() for i, cfs := range fs.cfs { if o == cfs { fs.cfs = append(fs.cfs[:i], fs.cfs[i+1:]...) break } } return nil } //////////////////////////////////////////////////////////////////////////////// // Templates //////////////////////////////////////////////////////////////////////////////// type templateFileStore struct { dir string hh hash.Hash64 } func newTemplateFileStore(storeDir string) *templateFileStore { tdir := filepath.Join(storeDir, tmplsDir) key := sha256.Sum256([]byte("templates")) hh, err := highwayhash.New64(key[:]) if err != nil { return nil } return &templateFileStore{dir: tdir, hh: hh} } func (ts *templateFileStore) Store(t *streamTemplate) error { dir := filepath.Join(ts.dir, t.Name) if err := os.MkdirAll(dir, defaultDirPerms); err != nil { return fmt.Errorf("could not create templates storage directory for %q- %v", t.Name, err) } meta := filepath.Join(dir, JetStreamMetaFile) if _, err := os.Stat(meta); (err != nil && !os.IsNotExist(err)) || err == nil { return err } t.mu.Lock() b, err := json.Marshal(t) t.mu.Unlock() if err != nil { return err } if err := os.WriteFile(meta, b, defaultFilePerms); err != nil { return err } // FIXME(dlc) - Do checksum ts.hh.Reset() ts.hh.Write(b) checksum := hex.EncodeToString(ts.hh.Sum(nil)) sum := filepath.Join(dir, JetStreamMetaFileSum) if err := os.WriteFile(sum, []byte(checksum), defaultFilePerms); err != nil { return err } return nil } func (ts *templateFileStore) Delete(t *streamTemplate) error { return os.RemoveAll(filepath.Join(ts.dir, t.Name)) } //////////////////////////////////////////////////////////////////////////////// // Compression //////////////////////////////////////////////////////////////////////////////// type CompressionInfo struct { Algorithm StoreCompression OriginalSize uint64 } func (c *CompressionInfo) MarshalMetadata() []byte { b := make([]byte, 14) // 4 + potentially up to 10 for uint64 b[0], b[1], b[2] = 'c', 'm', 'p' b[3] = byte(c.Algorithm) n := binary.PutUvarint(b[4:], c.OriginalSize) return b[:4+n] } func (c *CompressionInfo) UnmarshalMetadata(b []byte) (int, error) { c.Algorithm = NoCompression c.OriginalSize = 0 if len(b) < 5 { // 4 + min 1 for uvarint uint64 return 0, nil } if b[0] != 'c' || b[1] != 'm' || b[2] != 'p' { return 0, nil } var n int c.Algorithm = StoreCompression(b[3]) c.OriginalSize, n = binary.Uvarint(b[4:]) if n <= 0 { return 0, fmt.Errorf("metadata incomplete") } return 4 + n, nil } func (alg StoreCompression) Compress(buf []byte) ([]byte, error) { if len(buf) < checksumSize { return nil, fmt.Errorf("uncompressed buffer is too short") } bodyLen := int64(len(buf) - checksumSize) var output bytes.Buffer var writer io.WriteCloser switch alg { case NoCompression: return buf, nil case S2Compression: writer = s2.NewWriter(&output) default: return nil, fmt.Errorf("compression algorithm not known") } input := bytes.NewReader(buf[:bodyLen]) checksum := buf[bodyLen:] // Compress the block content, but don't compress the checksum. // We will preserve it at the end of the block as-is. if n, err := io.CopyN(writer, input, bodyLen); err != nil { return nil, fmt.Errorf("error writing to compression writer: %w", err) } else if n != bodyLen { return nil, fmt.Errorf("short write on body (%d != %d)", n, bodyLen) } if err := writer.Close(); err != nil { return nil, fmt.Errorf("error closing compression writer: %w", err) } // Now add the checksum back onto the end of the block. if n, err := output.Write(checksum); err != nil { return nil, fmt.Errorf("error writing checksum: %w", err) } else if n != checksumSize { return nil, fmt.Errorf("short write on checksum (%d != %d)", n, checksumSize) } return output.Bytes(), nil } func (alg StoreCompression) Decompress(buf []byte) ([]byte, error) { if len(buf) < checksumSize { return nil, fmt.Errorf("compressed buffer is too short") } bodyLen := int64(len(buf) - checksumSize) input := bytes.NewReader(buf[:bodyLen]) var reader io.ReadCloser switch alg { case NoCompression: return buf, nil case S2Compression: reader = io.NopCloser(s2.NewReader(input)) default: return nil, fmt.Errorf("compression algorithm not known") } // Decompress the block content. The checksum isn't compressed so // we can preserve it from the end of the block as-is. checksum := buf[bodyLen:] output, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("error reading compression reader: %w", err) } output = append(output, checksum...) return output, reader.Close() } // writeFileWithOptionalSync is equivalent to os.WriteFile() but optionally // sets O_SYNC on the open file if SyncAlways is set. The dios semaphore is // handled automatically by this function, so don't wrap calls to it in dios. func (fs *fileStore) writeFileWithOptionalSync(name string, data []byte, perm fs.FileMode) error { if fs.fcfg.SyncAlways { return writeFileWithSync(name, data, perm) } <-dios defer func() { dios <- struct{}{} }() return os.WriteFile(name, data, perm) } func writeFileWithSync(name string, data []byte, perm fs.FileMode) error { <-dios defer func() { dios <- struct{}{} }() flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_SYNC f, err := os.OpenFile(name, flags, perm) if err != nil { return err } if _, err = f.Write(data); err != nil { _ = f.Close() return err } return f.Close() } nats-server-2.10.27/server/filestore_test.go000066400000000000000000007251131477524627100210540ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_store_tests // +build !skip_store_tests package server import ( "archive/tar" "bytes" "crypto/hmac" crand "crypto/rand" "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/fs" "math/bits" "math/rand" "os" "path/filepath" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/nuid" ) func testFileStoreAllPermutations(t *testing.T, fn func(t *testing.T, fcfg FileStoreConfig)) { for _, fcfg := range []FileStoreConfig{ {Cipher: NoCipher, Compression: NoCompression}, {Cipher: NoCipher, Compression: S2Compression}, {Cipher: AES, Compression: NoCompression}, {Cipher: AES, Compression: S2Compression}, {Cipher: ChaCha, Compression: NoCompression}, {Cipher: ChaCha, Compression: S2Compression}, } { subtestName := fmt.Sprintf("%s-%s", fcfg.Cipher, fcfg.Compression) t.Run(subtestName, func(t *testing.T) { fcfg.StoreDir = t.TempDir() fn(t, fcfg) time.Sleep(100 * time.Millisecond) }) } } func prf(fcfg *FileStoreConfig) func(context []byte) ([]byte, error) { if fcfg.Cipher == NoCipher { return nil } return func(context []byte) ([]byte, error) { h := hmac.New(sha256.New, []byte("dlc22")) if _, err := h.Write(context); err != nil { return nil, err } return h.Sum(nil), nil } } func TestFileStoreBasics(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 1; i <= 5; i++ { now := time.Now().UnixNano() if seq, ts, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } else if seq != uint64(i) { t.Fatalf("Expected sequence to be %d, got %d", i, seq) } else if ts < now || ts > now+int64(time.Millisecond) { t.Fatalf("Expected timestamp to be current, got %v", ts-now) } } state := fs.State() if state.Msgs != 5 { t.Fatalf("Expected 5 msgs, got %d", state.Msgs) } expectedSize := 5 * fileStoreMsgSize(subj, nil, msg) if state.Bytes != expectedSize { t.Fatalf("Expected %d bytes, got %d", expectedSize, state.Bytes) } var smv StoreMsg sm, err := fs.LoadMsg(2, &smv) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if sm.subj != subj { t.Fatalf("Subjects don't match, original %q vs %q", subj, sm.subj) } if !bytes.Equal(sm.msg, msg) { t.Fatalf("Msgs don't match, original %q vs %q", msg, sm.msg) } _, err = fs.LoadMsg(3, nil) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } remove := func(seq, expectedMsgs uint64) { t.Helper() removed, err := fs.RemoveMsg(seq) if err != nil { t.Fatalf("Got an error on remove of %d: %v", seq, err) } if !removed { t.Fatalf("Expected remove to return true for %d", seq) } if state := fs.State(); state.Msgs != expectedMsgs { t.Fatalf("Expected %d msgs, got %d", expectedMsgs, state.Msgs) } } // Remove first remove(1, 4) // Remove last remove(5, 3) // Remove a middle remove(3, 2) }) } func TestFileStoreMsgHeaders(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, hdr, msg := "foo", []byte("name:derek"), []byte("Hello World") elen := 22 + len(subj) + 4 + len(hdr) + len(msg) + 8 if sz := int(fileStoreMsgSize(subj, hdr, msg)); sz != elen { t.Fatalf("Wrong size for stored msg with header") } fs.StoreMsg(subj, hdr, msg) var smv StoreMsg sm, err := fs.LoadMsg(1, &smv) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if !bytes.Equal(msg, sm.msg) { t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) } if !bytes.Equal(hdr, sm.hdr) { t.Fatalf("Expected same hdr, got %q vs %q", sm.hdr, hdr) } if removed, _ := fs.EraseMsg(1); !removed { t.Fatalf("Expected erase msg to return success") } }) } func TestFileStoreBasicWriteMsgsAndRestore(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { if _, err := newFileStore(fcfg, StreamConfig{Storage: MemoryStorage}); err == nil { t.Fatalf("Expected an error with wrong type") } if _, err := newFileStore(fcfg, StreamConfig{Storage: FileStorage}); err == nil { t.Fatalf("Expected an error with no name") } created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj := "foo" // Write 100 msgs toStore := uint64(100) for i := uint64(1); i <= toStore; i++ { msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) if seq, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } else if seq != uint64(i) { t.Fatalf("Expected sequence to be %d, got %d", i, seq) } } state := fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } msg22 := []byte(fmt.Sprintf("[%08d] Hello World!", 22)) expectedSize := toStore * fileStoreMsgSize(subj, nil, msg22) if state.Bytes != expectedSize { t.Fatalf("Expected %d bytes, got %d", expectedSize, state.Bytes) } // Stop will flush to disk. fs.Stop() // Make sure Store call after does not work. if _, _, err := fs.StoreMsg(subj, nil, []byte("no work")); err == nil { t.Fatalf("Expected an error for StoreMsg call after Stop, got none") } // Restart fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != expectedSize { t.Fatalf("Expected %d bytes, got %d", expectedSize, state.Bytes) } // Now write 100 more msgs for i := uint64(101); i <= toStore*2; i++ { msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) if seq, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } else if seq != uint64(i) { t.Fatalf("Expected sequence to be %d, got %d", i, seq) } } state = fs.State() if state.Msgs != toStore*2 { t.Fatalf("Expected %d msgs, got %d", toStore*2, state.Msgs) } // Now cycle again and make sure that last batch was stored. // Stop will flush to disk. fs.Stop() // Restart fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() if state.Msgs != toStore*2 { t.Fatalf("Expected %d msgs, got %d", toStore*2, state.Msgs) } if state.Bytes != expectedSize*2 { t.Fatalf("Expected %d bytes, got %d", expectedSize*2, state.Bytes) } fs.Purge() fs.Stop() // Restart fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() if state.Msgs != 0 { t.Fatalf("Expected %d msgs, got %d", 0, state.Msgs) } if state.Bytes != 0 { t.Fatalf("Expected %d bytes, got %d", 0, state.Bytes) } seq, _, err := fs.StoreMsg(subj, nil, []byte("Hello")) if err != nil { t.Fatalf("Unexpected error: %v", err) } fs.RemoveMsg(seq) fs.Stop() // Restart fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() require_Equal(t, state.FirstSeq, seq+1) }) } func TestFileStoreSelectNextFirst(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 256 fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() numMsgs := 10 subj, msg := "zzz", []byte("Hello World") for i := 0; i < numMsgs; i++ { fs.StoreMsg(subj, nil, msg) } if state := fs.State(); state.Msgs != uint64(numMsgs) { t.Fatalf("Expected %d msgs, got %d", numMsgs, state.Msgs) } // Note the 256 block size is tied to the msg size below to give us 5 messages per block. if fmb := fs.selectMsgBlock(1); fmb.msgs != 5 { t.Fatalf("Expected 5 messages per block, but got %d", fmb.msgs) } // Delete 2-7, this will cross message blocks. for i := 2; i <= 7; i++ { fs.RemoveMsg(uint64(i)) } if state := fs.State(); state.Msgs != 4 || state.FirstSeq != 1 { t.Fatalf("Expected 4 msgs, first seq of 11, got msgs of %d and first seq of %d", state.Msgs, state.FirstSeq) } // Now close the gap which will force the system to jump underlying message blocks to find the right sequence. fs.RemoveMsg(1) if state := fs.State(); state.Msgs != 3 || state.FirstSeq != 8 { t.Fatalf("Expected 3 msgs, first seq of 8, got msgs of %d and first seq of %d", state.Msgs, state.FirstSeq) } }) } func TestFileStoreSkipMsg(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 256 created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() numSkips := 10 for i := 0; i < numSkips; i++ { fs.SkipMsg() } state := fs.State() if state.Msgs != 0 { t.Fatalf("Expected %d msgs, got %d", 0, state.Msgs) } if state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips) { t.Fatalf("Expected first to be %d and last to be %d. got first %d and last %d", numSkips+1, numSkips, state.FirstSeq, state.LastSeq) } fs.StoreMsg("zzz", nil, []byte("Hello World!")) fs.SkipMsg() fs.SkipMsg() fs.StoreMsg("zzz", nil, []byte("Hello World!")) fs.SkipMsg() state = fs.State() if state.Msgs != 2 { t.Fatalf("Expected %d msgs, got %d", 2, state.Msgs) } if state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips+5) { t.Fatalf("Expected first to be %d and last to be %d. got first %d and last %d", numSkips+1, numSkips+5, state.FirstSeq, state.LastSeq) } // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() if state.Msgs != 2 { t.Fatalf("Expected %d msgs, got %d", 2, state.Msgs) } if state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips+5) { t.Fatalf("Expected first to be %d and last to be %d. got first %d and last %d", numSkips+1, numSkips+5, state.FirstSeq, state.LastSeq) } var smv StoreMsg sm, err := fs.LoadMsg(11, &smv) if err != nil { t.Fatalf("Unexpected error looking up seq 11: %v", err) } if sm.subj != "zzz" || string(sm.msg) != "Hello World!" { t.Fatalf("Message did not match") } fs.SkipMsg() nseq, _, err := fs.StoreMsg("AAA", nil, []byte("Skip?")) if err != nil { t.Fatalf("Unexpected error looking up seq 11: %v", err) } if nseq != 17 { t.Fatalf("Expected seq of %d but got %d", 17, nseq) } // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() sm, err = fs.LoadMsg(nseq, &smv) if err != nil { t.Fatalf("Unexpected error looking up seq %d: %v", nseq, err) } if sm.subj != "AAA" || string(sm.msg) != "Skip?" { t.Fatalf("Message did not match") } }) } func TestFileStoreWriteExpireWrite(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cexp := 10 * time.Millisecond fcfg.CacheExpire = cexp created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() toSend := 10 for i := 0; i < toSend; i++ { fs.StoreMsg("zzz", nil, []byte("Hello World!")) } // Wait for write cache portion to go to zero. checkFor(t, time.Second, 20*time.Millisecond, func() error { if csz := fs.cacheSize(); csz != 0 { return fmt.Errorf("cache size not 0, got %s", friendlyBytes(int64(csz))) } return nil }) for i := 0; i < toSend; i++ { fs.StoreMsg("zzz", nil, []byte("Hello World! - 22")) } if state := fs.State(); state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d msgs, got %d", toSend*2, state.Msgs) } // Make sure we recover same state. fs.Stop() fcfg.CacheExpire = 0 fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d msgs, got %d", toSend*2, state.Msgs) } // Now load them in and check. var smv StoreMsg for i := 1; i <= toSend*2; i++ { sm, err := fs.LoadMsg(uint64(i), &smv) if err != nil { t.Fatalf("Unexpected error looking up seq %d: %v", i, err) } str := "Hello World!" if i > toSend { str = "Hello World! - 22" } if sm.subj != "zzz" || string(sm.msg) != str { t.Fatalf("Message did not match") } } }) } func TestFileStoreMsgLimit(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxMsgs: 10}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != 10 { t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) } if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } state = fs.State() if state.Msgs != 10 { t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) } if state.LastSeq != 11 { t.Fatalf("Expected the last sequence to be 11 now, but got %d", state.LastSeq) } if state.FirstSeq != 2 { t.Fatalf("Expected the first sequence to be 2 now, but got %d", state.FirstSeq) } // Make sure we can not lookup seq 1. if _, err := fs.LoadMsg(1, nil); err == nil { t.Fatalf("Expected error looking up seq 1 but got none") } }) } func TestFileStoreMsgLimitBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxMsgs: 1}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") fs.StoreMsg(subj, nil, msg) fs.StoreMsg(subj, nil, msg) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxMsgs: 1}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fs.StoreMsg(subj, nil, msg) }) } func TestFileStoreBytesLimit(t *testing.T) { subj, msg := "foo", make([]byte, 512) storedMsgSize := fileStoreMsgSize(subj, nil, msg) toStore := uint64(1024) maxBytes := storedMsgSize * toStore testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxBytes: int64(maxBytes)}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := uint64(0); i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } // Now send 10 more and check that bytes limit enforced. for i := 0; i < 10; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } } state = fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } if state.FirstSeq != 11 { t.Fatalf("Expected first sequence to be 11, got %d", state.FirstSeq) } if state.LastSeq != toStore+10 { t.Fatalf("Expected last sequence to be %d, got %d", toStore+10, state.LastSeq) } }) } // https://github.com/nats-io/nats-server/issues/4771 func TestFileStoreBytesLimitWithDiscardNew(t *testing.T) { subj, msg := "tiny", make([]byte, 7) storedMsgSize := fileStoreMsgSize(subj, nil, msg) toStore := uint64(2) maxBytes := 100 testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Storage: FileStorage, MaxBytes: int64(maxBytes), Discard: DiscardNew} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 10; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) if i < int(toStore) { if err != nil { t.Fatalf("Error storing msg: %v", err) } } else if !errors.Is(err, ErrMaxBytes) { t.Fatalf("Storing msg should result in: %v", ErrMaxBytes) } } state := fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } }) } func TestFileStoreAgeLimit(t *testing.T) { maxAge := 1 * time.Second testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { if fcfg.Compression != NoCompression { // TODO(nat): This test fails at the moment with compression enabled // because it takes longer to compress the blocks, by which time the // messages have expired. Need to think about a balanced age so that // the test doesn't take too long in non-compressed cases. t.SkipNow() } fcfg.BlockSize = 256 fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxAge: maxAge}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Store some messages. Does not really matter how many. subj, msg := "foo", []byte("Hello World") toStore := 500 for i := 0; i < toStore; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } checkExpired := func(t *testing.T) { t.Helper() checkFor(t, 5*time.Second, maxAge, func() error { state = fs.State() if state.Msgs != 0 { return fmt.Errorf("Expected no msgs, got %d", state.Msgs) } if state.Bytes != 0 { return fmt.Errorf("Expected no bytes, got %d", state.Bytes) } return nil }) } // Let them expire checkExpired(t) // Now add some more and make sure that timer will fire again. for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state = fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } fs.RemoveMsg(502) fs.RemoveMsg(602) fs.RemoveMsg(702) fs.RemoveMsg(802) // We will measure the time to make sure expires works with interior deletes. start := time.Now() checkExpired(t) if elapsed := time.Since(start); elapsed > 5*time.Second { t.Fatalf("Took too long to expire: %v", elapsed) } }) } func TestFileStoreTimeStamps(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() last := time.Now().UnixNano() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { time.Sleep(5 * time.Millisecond) fs.StoreMsg(subj, nil, msg) } var smv StoreMsg for seq := uint64(1); seq <= 10; seq++ { sm, err := fs.LoadMsg(seq, &smv) if err != nil { t.Fatalf("Unexpected error looking up msg [%d]: %v", seq, err) } // These should be different if sm.ts <= last { t.Fatalf("Expected different timestamps, got last %v vs %v", last, sm.ts) } last = sm.ts } }) } func TestFileStorePurge(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { blkSize := uint64(64 * 1024) fcfg.BlockSize = blkSize created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", make([]byte, 8*1024) storedMsgSize := fileStoreMsgSize(subj, nil, msg) toStore := uint64(1024) for i := uint64(0); i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } expectedBlocks := int(storedMsgSize * toStore / blkSize) if numBlocks := fs.numMsgBlocks(); numBlocks <= expectedBlocks { t.Fatalf("Expected to have more then %d msg blocks, got %d", blkSize, numBlocks) } fs.Purge() if numBlocks := fs.numMsgBlocks(); numBlocks != 1 { t.Fatalf("Expected to have exactly 1 empty msg block, got %d", numBlocks) } checkPurgeState := func(stored uint64) { t.Helper() state = fs.State() if state.Msgs != 0 { t.Fatalf("Expected 0 msgs after purge, got %d", state.Msgs) } if state.Bytes != 0 { t.Fatalf("Expected 0 bytes after purge, got %d", state.Bytes) } if state.LastSeq != stored { t.Fatalf("Expected LastSeq to be %d., got %d", toStore, state.LastSeq) } if state.FirstSeq != stored+1 { t.Fatalf("Expected FirstSeq to be %d., got %d", toStore+1, state.FirstSeq) } } checkPurgeState(toStore) // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if numBlocks := fs.numMsgBlocks(); numBlocks != 1 { t.Fatalf("Expected to have exactly 1 empty msg block, got %d", numBlocks) } checkPurgeState(toStore) // Now make sure we clean up any dangling purged messages. for i := uint64(0); i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state = fs.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } // We will simulate crashing before the purge directory is cleared. mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) pdir := filepath.Join(fs.fcfg.StoreDir, "ptest") os.Rename(mdir, pdir) os.MkdirAll(mdir, 0755) fs.Purge() checkPurgeState(toStore * 2) // Make sure we recover same state. fs.Stop() purgeDir := filepath.Join(fs.fcfg.StoreDir, purgeDir) os.Rename(pdir, purgeDir) fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if numBlocks := fs.numMsgBlocks(); numBlocks != 1 { t.Fatalf("Expected to have exactly 1 empty msg block, got %d", numBlocks) } checkPurgeState(toStore * 2) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if _, err := os.Stat(purgeDir); err == nil { return fmt.Errorf("purge directory still present") } return nil }) }) } func TestFileStoreCompact(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 350 created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } if state := fs.State(); state.Msgs != 10 { t.Fatalf("Expected 10 msgs, got %d", state.Msgs) } n, err := fs.Compact(6) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n != 5 { t.Fatalf("Expected to have purged 5 msgs, got %d", n) } state := fs.State() if state.Msgs != 5 { t.Fatalf("Expected 5 msgs, got %d", state.Msgs) } if state.FirstSeq != 6 { t.Fatalf("Expected first seq of 6, got %d", state.FirstSeq) } // Now test that compact will also reset first if seq > last n, err = fs.Compact(100) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n != 5 { t.Fatalf("Expected to have purged 5 msgs, got %d", n) } if state = fs.State(); state.FirstSeq != 100 { t.Fatalf("Expected first seq of 100, got %d", state.FirstSeq) } fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state = fs.State(); state.FirstSeq != 100 { t.Fatalf("Expected first seq of 100, got %d", state.FirstSeq) } }) } func TestFileStoreCompactLastPlusOne(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 8192 fcfg.AsyncFlush = true fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", make([]byte, 10_000) for i := 0; i < 10_000; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } // The performance of this test is quite terrible with compression // if we have AsyncFlush = false, so we'll batch flushes instead. fs.mu.Lock() fs.checkAndFlushAllBlocks() fs.mu.Unlock() if state := fs.State(); state.Msgs != 10_000 { t.Fatalf("Expected 1000000 msgs, got %d", state.Msgs) } if _, err := fs.Compact(10_001); err != nil { t.Fatalf("Unexpected error: %v", err) } state := fs.State() if state.Msgs != 0 { t.Fatalf("Expected no message but got %d", state.Msgs) } fs.StoreMsg(subj, nil, msg) state = fs.State() if state.Msgs != 1 { t.Fatalf("Expected one message but got %d", state.Msgs) } }) } func TestFileStoreCompactMsgCountBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } if state := fs.State(); state.Msgs != 10 { t.Fatalf("Expected 10 msgs, got %d", state.Msgs) } // Now delete 2,3,4. fs.EraseMsg(2) fs.EraseMsg(3) fs.EraseMsg(4) // Also delete 7,8, and 9. fs.RemoveMsg(7) fs.RemoveMsg(8) fs.RemoveMsg(9) n, err := fs.Compact(6) if err != nil { t.Fatalf("Unexpected error: %v", err) } // 1 & 5 if n != 2 { t.Fatalf("Expected to have deleted 2 msgs, got %d", n) } if state := fs.State(); state.Msgs != 2 { t.Fatalf("Expected to have 2 remaining, got %d", state.Msgs) } }) } func TestFileStoreCompactPerf(t *testing.T) { t.SkipNow() testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 8192 fcfg.AsyncFlush = true fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 100_000; i++ { fs.StoreMsg(subj, nil, msg) } if state := fs.State(); state.Msgs != 100_000 { t.Fatalf("Expected 1000000 msgs, got %d", state.Msgs) } start := time.Now() n, err := fs.Compact(90_001) if err != nil { t.Fatalf("Unexpected error: %v", err) } t.Logf("Took %v to compact\n", time.Since(start)) if n != 90_000 { t.Fatalf("Expected to have purged 90_000 msgs, got %d", n) } state := fs.State() if state.Msgs != 10_000 { t.Fatalf("Expected 10_000 msgs, got %d", state.Msgs) } if state.FirstSeq != 90_001 { t.Fatalf("Expected first seq of 90_001, got %d", state.FirstSeq) } }) } func TestFileStoreStreamTruncate(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 350 cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() tseq := uint64(50) subj, toStore := "foo", uint64(100) for i := uint64(1); i < tseq; i++ { _, _, err := fs.StoreMsg(subj, nil, []byte("ok")) require_NoError(t, err) } subj = "bar" for i := tseq; i <= toStore; i++ { _, _, err := fs.StoreMsg(subj, nil, []byte("ok")) require_NoError(t, err) } if state := fs.State(); state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } // Check that sequence has to be interior. if err := fs.Truncate(toStore + 1); err != ErrInvalidSequence { t.Fatalf("Expected err of '%v', got '%v'", ErrInvalidSequence, err) } if err := fs.Truncate(tseq); err != nil { t.Fatalf("Unexpected error: %v", err) } if state := fs.State(); state.Msgs != tseq { t.Fatalf("Expected %d msgs, got %d", tseq, state.Msgs) } // Now make sure we report properly if we have some deleted interior messages. fs.RemoveMsg(10) fs.RemoveMsg(20) fs.RemoveMsg(30) fs.RemoveMsg(40) tseq = uint64(25) if err := fs.Truncate(tseq); err != nil { t.Fatalf("Unexpected error: %v", err) } state := fs.State() if state.Msgs != tseq-2 { t.Fatalf("Expected %d msgs, got %d", tseq-2, state.Msgs) } expected := []uint64{10, 20} if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } before := state // Make sure we can recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); !reflect.DeepEqual(state, before) { t.Fatalf("Expected state of %+v, got %+v", before, state) } mb := fs.getFirstBlock() require_True(t, mb != nil) require_NoError(t, mb.loadMsgs()) // Also make sure we can recover properly with no index.db present. // We want to make sure we preserve tombstones from any blocks being deleted. fs.Stop() os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); !reflect.DeepEqual(state, before) { t.Fatalf("Expected state of %+v, got %+v without index.db state", before, state) } }) } func TestFileStoreRemovePartialRecovery(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } // Remove half for i := 1; i <= toStore/2; i++ { fs.RemoveMsg(uint64(i)) } state = fs.State() if state.Msgs != uint64(toStore/2) { t.Fatalf("Expected %d msgs, got %d", toStore/2, state.Msgs) } // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state2 := fs.State() if !reflect.DeepEqual(state2, state) { t.Fatalf("Expected recovered state to be the same, got %+v vs %+v\n", state2, state) } }) } func TestFileStoreRemoveOutOfOrderRecovery(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } // Remove evens for i := 2; i <= toStore; i += 2 { if removed, _ := fs.RemoveMsg(uint64(i)); !removed { t.Fatalf("Expected remove to return true") } } state = fs.State() if state.Msgs != uint64(toStore/2) { t.Fatalf("Expected %d msgs, got %d", toStore/2, state.Msgs) } var smv StoreMsg if _, err := fs.LoadMsg(1, &smv); err != nil { t.Fatalf("Expected to retrieve seq 1") } for i := 2; i <= toStore; i += 2 { if _, err := fs.LoadMsg(uint64(i), &smv); err == nil { t.Fatalf("Expected error looking up seq %d that should be deleted", i) } } // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state2 := fs.State() if !reflect.DeepEqual(state2, state) { t.Fatalf("Expected recovered states to be the same, got %+v vs %+v\n", state, state2) } if _, err := fs.LoadMsg(1, &smv); err != nil { t.Fatalf("Expected to retrieve seq 1") } for i := 2; i <= toStore; i += 2 { if _, err := fs.LoadMsg(uint64(i), nil); err == nil { t.Fatalf("Expected error looking up seq %d that should be deleted", i) } } }) } func TestFileStoreAgeLimitRecovery(t *testing.T) { maxAge := 1 * time.Second testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.CacheExpire = 1 * time.Millisecond cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage, MaxAge: maxAge} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Store some messages. Does not really matter how many. subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } fs.Stop() time.Sleep(maxAge) fcfg.CacheExpire = 0 fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure they expire. checkFor(t, time.Second, 2*maxAge, func() error { t.Helper() state = fs.State() if state.Msgs != 0 { return fmt.Errorf("Expected no msgs, got %d", state.Msgs) } if state.Bytes != 0 { return fmt.Errorf("Expected no bytes, got %d", state.Bytes) } return nil }) }) } func TestFileStoreBitRot(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Store some messages. Does not really matter how many. subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if ld := fs.checkMsgs(); ld != nil && len(ld.Msgs) > 0 { t.Fatalf("Expected to have no corrupt msgs, got %d", len(ld.Msgs)) } for i := 0; i < 10; i++ { // Now twiddle some bits. fs.mu.Lock() lmb := fs.lmb contents, err := os.ReadFile(lmb.mfn) require_NoError(t, err) require_True(t, len(contents) > 0) var index int for { index = rand.Intn(len(contents)) // Reverse one byte anywhere. b := contents[index] contents[index] = bits.Reverse8(b) if b != contents[index] { break } } os.WriteFile(lmb.mfn, contents, 0644) fs.mu.Unlock() ld := fs.checkMsgs() if len(ld.Msgs) > 0 { break } // If our bitrot caused us to not be able to recover any messages we can break as well. if state := fs.State(); state.Msgs == 0 { break } // Fail the test if we have tried the 10 times and still did not // get any corruption report. if i == 9 { t.Fatalf("Expected to have corrupt msgs got none: changed [%d]", index) } } // Make sure we can restore. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // checkMsgs will repair the underlying store, so checkMsgs should be clean now. if ld := fs.checkMsgs(); ld != nil { // If we have no msgs left this will report the head msgs as lost again. if state := fs.State(); state.Msgs > 0 { t.Fatalf("Expected no errors restoring checked and fixed filestore, got %+v", ld) } } }) } func TestFileStoreEraseMsg(t *testing.T) { // Just do no encryption, etc. fcfg := FileStoreConfig{StoreDir: t.TempDir()} fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") fs.StoreMsg(subj, nil, msg) fs.StoreMsg(subj, nil, msg) // To keep block from being deleted. var smv StoreMsg sm, err := fs.LoadMsg(1, &smv) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if !bytes.Equal(msg, sm.msg) { t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) } if removed, _ := fs.EraseMsg(1); !removed { t.Fatalf("Expected erase msg to return success") } if sm2, _ := fs.msgForSeq(1, nil); sm2 != nil { t.Fatalf("Expected msg to be erased") } fs.checkAndFlushAllBlocks() // Now look on disk as well. rl := fileStoreMsgSize(subj, nil, msg) buf := make([]byte, rl) fp, err := os.Open(filepath.Join(fcfg.StoreDir, msgDir, fmt.Sprintf(blkScan, 1))) if err != nil { t.Fatalf("Error opening msg block file: %v", err) } defer fp.Close() fp.ReadAt(buf, 0) fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() mb.mu.Lock() sm, err = mb.msgFromBuf(buf, nil, nil) mb.mu.Unlock() if err != nil { t.Fatalf("error reading message from block: %v", err) } if sm.subj == subj { t.Fatalf("Expected the subjects to be different") } if sm.seq != 0 && sm.seq&ebit == 0 { t.Fatalf("Expected seq to be 0, marking as deleted, got %d", sm.seq) } if sm.ts != 0 { t.Fatalf("Expected timestamp to be 0, got %d", sm.ts) } if bytes.Equal(sm.msg, msg) { t.Fatalf("Expected message body to be randomized") } } func TestFileStoreEraseAndNoIndexRecovery(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } // Erase the even messages. for i := 2; i <= toStore; i += 2 { if removed, _ := fs.EraseMsg(uint64(i)); !removed { t.Fatalf("Expected erase msg to return true") } } state = fs.State() if state.Msgs != uint64(toStore/2) { t.Fatalf("Expected %d msgs, got %d", toStore/2, state.Msgs) } // Stop and remove the optional index file. fs.Stop() ifn := filepath.Join(fcfg.StoreDir, msgDir, fmt.Sprintf(indexScan, 1)) os.Remove(ifn) fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() if state.Msgs != uint64(toStore/2) { t.Fatalf("Expected %d msgs, got %d", toStore/2, state.Msgs) } for i := 2; i <= toStore; i += 2 { if _, err := fs.LoadMsg(uint64(i), nil); err == nil { t.Fatalf("Expected error looking up seq %d that should be erased", i) } } }) } func TestFileStoreMeta(t *testing.T) { // Just do no encryption, etc. fcfg := FileStoreConfig{StoreDir: t.TempDir()} mconfig := StreamConfig{Name: "ZZ-22-33", Storage: FileStorage, Subjects: []string{"foo.*"}, Replicas: 22} fs, err := newFileStore(fcfg, mconfig) require_NoError(t, err) defer fs.Stop() metafile := filepath.Join(fcfg.StoreDir, JetStreamMetaFile) metasum := filepath.Join(fcfg.StoreDir, JetStreamMetaFileSum) // Test to make sure meta file and checksum are present. if _, err := os.Stat(metafile); os.IsNotExist(err) { t.Fatalf("Expected metafile %q to exist", metafile) } if _, err := os.Stat(metasum); os.IsNotExist(err) { t.Fatalf("Expected metafile's checksum %q to exist", metasum) } buf, err := os.ReadFile(metafile) if err != nil { t.Fatalf("Error reading metafile: %v", err) } var mconfig2 StreamConfig if err := json.Unmarshal(buf, &mconfig2); err != nil { t.Fatalf("Error unmarshalling: %v", err) } if !reflect.DeepEqual(mconfig, mconfig2) { t.Fatalf("Stream configs not equal, got %+v vs %+v", mconfig2, mconfig) } checksum, err := os.ReadFile(metasum) if err != nil { t.Fatalf("Error reading metafile checksum: %v", err) } fs.mu.Lock() fs.hh.Reset() fs.hh.Write(buf) mychecksum := hex.EncodeToString(fs.hh.Sum(nil)) fs.mu.Unlock() if mychecksum != string(checksum) { t.Fatalf("Checksums do not match, got %q vs %q", mychecksum, checksum) } // Now create a consumer. Same deal for them. oconfig := ConsumerConfig{ DeliverSubject: "d", FilterSubject: "foo", AckPolicy: AckAll, } oname := "obs22" obs, err := fs.ConsumerStore(oname, &oconfig) if err != nil { t.Fatalf("Unexpected error: %v", err) } ometafile := filepath.Join(fcfg.StoreDir, consumerDir, oname, JetStreamMetaFile) ometasum := filepath.Join(fcfg.StoreDir, consumerDir, oname, JetStreamMetaFileSum) // Test to make sure meta file and checksum are present. if _, err := os.Stat(ometafile); os.IsNotExist(err) { t.Fatalf("Expected consumer metafile %q to exist", ometafile) } if _, err := os.Stat(ometasum); os.IsNotExist(err) { t.Fatalf("Expected consumer metafile's checksum %q to exist", ometasum) } buf, err = os.ReadFile(ometafile) if err != nil { t.Fatalf("Error reading consumer metafile: %v", err) } var oconfig2 ConsumerConfig if err := json.Unmarshal(buf, &oconfig2); err != nil { t.Fatalf("Error unmarshalling: %v", err) } // Since we set name we will get that back now. oconfig.Name = oname if !reflect.DeepEqual(oconfig2, oconfig) { t.Fatalf("Consumer configs not equal, got %+v vs %+v", oconfig2, oconfig) } checksum, err = os.ReadFile(ometasum) if err != nil { t.Fatalf("Error reading consumer metafile checksum: %v", err) } hh := obs.(*consumerFileStore).hh hh.Reset() hh.Write(buf) mychecksum = hex.EncodeToString(hh.Sum(nil)) if mychecksum != string(checksum) { t.Fatalf("Checksums do not match, got %q vs %q", mychecksum, checksum) } } func TestFileStoreWriteAndReadSameBlock(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World!") for i := uint64(1); i <= 10; i++ { fs.StoreMsg(subj, nil, msg) if _, err := fs.LoadMsg(i, nil); err != nil { t.Fatalf("Error loading %d: %v", i, err) } } }) } func TestFileStoreAndRetrieveMultiBlock(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { subj, msg := "foo", []byte("Hello World!") storedMsgSize := fileStoreMsgSize(subj, nil, msg) fcfg.BlockSize = 4 * storedMsgSize created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 20; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != 20 { t.Fatalf("Expected 20 msgs, got %d", state.Msgs) } fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() var smv StoreMsg for i := uint64(1); i <= 20; i++ { if _, err := fs.LoadMsg(i, &smv); err != nil { t.Fatalf("Error loading %d: %v", i, err) } } }) } func TestFileStoreCollapseDmap(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { subj, msg := "foo", []byte("Hello World!") storedMsgSize := fileStoreMsgSize(subj, nil, msg) fcfg.BlockSize = 4 * storedMsgSize fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } state := fs.State() if state.Msgs != 10 { t.Fatalf("Expected 10 msgs, got %d", state.Msgs) } checkDmapTotal := func(total int) { t.Helper() if nde := fs.dmapEntries(); nde != total { t.Fatalf("Expecting only %d entries, got %d", total, nde) } } checkFirstSeq := func(seq uint64) { t.Helper() state := fs.State() if state.FirstSeq != seq { t.Fatalf("Expected first seq to be %d, got %d", seq, state.FirstSeq) } } // Now remove some out of order, forming gaps and entries in dmaps. fs.RemoveMsg(2) checkFirstSeq(1) fs.RemoveMsg(4) checkFirstSeq(1) fs.RemoveMsg(8) checkFirstSeq(1) state = fs.State() if state.Msgs != 7 { t.Fatalf("Expected 7 msgs, got %d", state.Msgs) } checkDmapTotal(3) // Close gaps.. fs.RemoveMsg(1) checkDmapTotal(2) checkFirstSeq(3) fs.RemoveMsg(3) checkDmapTotal(1) checkFirstSeq(5) fs.RemoveMsg(5) checkDmapTotal(1) checkFirstSeq(6) fs.RemoveMsg(7) checkDmapTotal(2) fs.RemoveMsg(6) checkDmapTotal(0) }) } func TestFileStoreReadCache(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.CacheExpire = 100 * time.Millisecond subj, msg := "foo.bar", make([]byte, 1024) storedMsgSize := fileStoreMsgSize(subj, nil, msg) fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() toStore := 500 totalBytes := uint64(toStore) * storedMsgSize for i := 0; i < toStore; i++ { fs.StoreMsg(subj, nil, msg) } // Wait for cache to go to zero. checkFor(t, time.Second, 10*time.Millisecond, func() error { if csz := fs.cacheSize(); csz != 0 { return fmt.Errorf("cache size not 0, got %s", friendlyBytes(int64(csz))) } return nil }) fs.LoadMsg(1, nil) if csz := fs.cacheSize(); csz != totalBytes { t.Fatalf("Expected all messages to be cached, got %d vs %d", csz, totalBytes) } // Should expire and be removed. checkFor(t, time.Second, 10*time.Millisecond, func() error { if csz := fs.cacheSize(); csz != 0 { return fmt.Errorf("cache size not 0, got %s", friendlyBytes(int64(csz))) } return nil }) if cls := fs.cacheLoads(); cls != 1 { t.Fatalf("Expected only 1 cache load, got %d", cls) } // Now make sure we do not reload cache if there is activity. fs.LoadMsg(1, nil) timeout := time.Now().Add(250 * time.Millisecond) for time.Now().Before(timeout) { if cls := fs.cacheLoads(); cls != 2 { t.Fatalf("cache loads not 2, got %d", cls) } time.Sleep(5 * time.Millisecond) fs.LoadMsg(1, nil) // register activity. } }) } func TestFileStorePartialCacheExpiration(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cexp := 10 * time.Millisecond fcfg.CacheExpire = cexp fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, []byte("msg1")) // Should expire and be removed. time.Sleep(2 * cexp) fs.StoreMsg("bar", nil, []byte("msg2")) // Again wait for cache to expire. time.Sleep(2 * cexp) if _, err := fs.LoadMsg(1, nil); err != nil { t.Fatalf("Error loading message 1: %v", err) } }) } func TestFileStorePartialIndexes(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cexp := 10 * time.Millisecond fcfg.CacheExpire = cexp fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() toSend := 5 for i := 0; i < toSend; i++ { fs.StoreMsg("foo", nil, []byte("ok-1")) } // Now wait til the cache expires, including the index. fs.mu.Lock() mb := fs.blks[0] fs.mu.Unlock() // Force idx to expire by resetting last remove ts. mb.mu.Lock() mb.llts = mb.llts - int64(defaultCacheBufferExpiration*2) mb.mu.Unlock() checkFor(t, time.Second, 10*time.Millisecond, func() error { mb.mu.Lock() defer mb.mu.Unlock() if mb.cache == nil || len(mb.cache.idx) == 0 { return nil } return fmt.Errorf("Index not empty") }) // Create a partial cache by adding more msgs. for i := 0; i < toSend; i++ { fs.StoreMsg("foo", nil, []byte("ok-2")) } // If we now load in a message in second half if we do not // detect idx is a partial correctly this will panic. if _, err := fs.LoadMsg(8, nil); err != nil { t.Fatalf("Error loading %d: %v", 1, err) } }) } func TestFileStoreSnapshot(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { subj, msg := "foo", []byte("Hello Snappy!") scfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, scfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() toSend := 2233 for i := 0; i < toSend; i++ { fs.StoreMsg(subj, nil, msg) } // Create a few consumers. o1, err := fs.ConsumerStore("o22", &ConsumerConfig{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } o2, err := fs.ConsumerStore("o33", &ConsumerConfig{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } state := &ConsumerState{} state.Delivered.Consumer = 100 state.Delivered.Stream = 100 state.AckFloor.Consumer = 22 state.AckFloor.Stream = 22 if err := o1.Update(state); err != nil { t.Fatalf("Unexpected error updating state: %v", err) } state.AckFloor.Consumer = 33 state.AckFloor.Stream = 33 if err := o2.Update(state); err != nil { t.Fatalf("Unexpected error updating state: %v", err) } snapshot := func() []byte { t.Helper() r, err := fs.Snapshot(5*time.Second, true, true) if err != nil { t.Fatalf("Error creating snapshot") } snapshot, err := io.ReadAll(r.Reader) if err != nil { t.Fatalf("Error reading snapshot") } return snapshot } // This will unzip the snapshot and create a new filestore that will recover the state. // We will compare the states for this vs the original one. verifySnapshot := func(snap []byte) { t.Helper() r := bytes.NewReader(snap) tr := tar.NewReader(s2.NewReader(r)) rstoreDir := t.TempDir() for { hdr, err := tr.Next() if err == io.EOF { break // End of archive } if err != nil { t.Fatalf("Error getting next entry from snapshot: %v", err) } fpath := filepath.Join(rstoreDir, filepath.Clean(hdr.Name)) pdir := filepath.Dir(fpath) os.MkdirAll(pdir, 0755) fd, err := os.OpenFile(fpath, os.O_CREATE|os.O_RDWR, 0600) if err != nil { t.Fatalf("Error opening file[%s]: %v", fpath, err) } if _, err := io.Copy(fd, tr); err != nil { t.Fatalf("Error writing file[%s]: %v", fpath, err) } fd.Close() } fcfg.StoreDir = rstoreDir fsr, err := newFileStoreWithCreated(fcfg, scfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fsr.Stop() state := fs.State() rstate := fsr.State() // FIXME(dlc) // Right now the upper layers in JetStream recover the consumers and do not expect // the lower layers to do that. So for now blank that out of our original state. // Will have more exhaustive tests in jetstream_test.go. state.Consumers = 0 // Just check the state. if !reflect.DeepEqual(rstate, state) { t.Fatalf("Restored state does not match:\n%+v\n\n%+v", rstate, state) } } // Simple case first. snap := snapshot() verifySnapshot(snap) // Remove first 100 messages. for i := 1; i <= 100; i++ { fs.RemoveMsg(uint64(i)) } snap = snapshot() verifySnapshot(snap) // Now sporadic messages inside the stream. total := int64(toSend - 100) // Delete 50 random messages. for i := 0; i < 50; i++ { seq := uint64(rand.Int63n(total) + 101) fs.RemoveMsg(seq) } snap = snapshot() verifySnapshot(snap) // Make sure compaction works with snapshots. fs.mu.RLock() for _, mb := range fs.blks { // Should not call compact on last msg block. if mb != fs.lmb { mb.mu.Lock() mb.compact() mb.mu.Unlock() } } fs.mu.RUnlock() snap = snapshot() verifySnapshot(snap) // Now check to make sure that we get the correct error when trying to delete or erase // a message when a snapshot is in progress and that closing the reader releases that condition. sr, err := fs.Snapshot(5*time.Second, false, true) if err != nil { t.Fatalf("Error creating snapshot") } if _, err := fs.RemoveMsg(122); err != ErrStoreSnapshotInProgress { t.Fatalf("Did not get the correct error on remove during snapshot: %v", err) } if _, err := fs.EraseMsg(122); err != ErrStoreSnapshotInProgress { t.Fatalf("Did not get the correct error on remove during snapshot: %v", err) } // Now make sure we can do these when we close the reader and release the snapshot condition. sr.Reader.Close() checkFor(t, time.Second, 10*time.Millisecond, func() error { if _, err := fs.RemoveMsg(122); err != nil { return fmt.Errorf("Got an error on remove after snapshot: %v", err) } return nil }) // Make sure if we do not read properly then it will close the writer and report an error. sr, err = fs.Snapshot(25*time.Millisecond, false, false) if err != nil { t.Fatalf("Error creating snapshot") } // Cause snapshot to timeout. time.Sleep(50 * time.Millisecond) // Read should fail var buf [32]byte if _, err := sr.Reader.Read(buf[:]); err != io.EOF { t.Fatalf("Expected read to produce an error, got none") } }) } func TestFileStoreConsumer(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() o, err := fs.ConsumerStore("obs22", &ConsumerConfig{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if state, err := o.State(); err != nil || state.Delivered.Consumer != 0 { t.Fatalf("Unexpected state or error: %v", err) } state := &ConsumerState{} updateAndCheck := func() { t.Helper() if err := o.Update(state); err != nil { t.Fatalf("Unexpected error updating state: %v", err) } s2, err := o.State() if err != nil { t.Fatalf("Unexpected error getting state: %v", err) } if !reflect.DeepEqual(state, s2) { t.Fatalf("State is not the same: wanted %+v got %+v", state, s2) } } shouldFail := func() { t.Helper() if err := o.Update(state); err == nil { t.Fatalf("Expected an error and got none") } } state.Delivered.Consumer = 1 state.Delivered.Stream = 22 updateAndCheck() state.Delivered.Consumer = 100 state.Delivered.Stream = 122 state.AckFloor.Consumer = 50 state.AckFloor.Stream = 123 // This should fail, bad state. shouldFail() // So should this. state.AckFloor.Consumer = 200 state.AckFloor.Stream = 100 shouldFail() // Should succeed state.AckFloor.Consumer = 50 state.AckFloor.Stream = 72 updateAndCheck() tn := time.Now().UnixNano() // We should sanity check pending here as well, so will check if a pending value is below // ack floor or above delivered. state.Pending = map[uint64]*Pending{70: {70, tn}} shouldFail() state.Pending = map[uint64]*Pending{140: {140, tn}} shouldFail() state.Pending = map[uint64]*Pending{72: {72, tn}} // exact on floor should fail shouldFail() // Put timestamps a second apart. // We will downsample to second resolution to save space. So setup our times // to reflect that. ago := time.Now().Add(-30 * time.Second).Truncate(time.Second) nt := func() *Pending { ago = ago.Add(time.Second) return &Pending{0, ago.UnixNano()} } // Should succeed. state.Pending = map[uint64]*Pending{75: nt(), 80: nt(), 83: nt(), 90: nt(), 111: nt()} updateAndCheck() // Now do redlivery, but first with no pending. state.Pending = nil state.Redelivered = map[uint64]uint64{22: 3, 44: 8} updateAndCheck() // All together. state.Pending = map[uint64]*Pending{75: nt(), 80: nt(), 83: nt(), 90: nt(), 111: nt()} updateAndCheck() // Large one state.Delivered.Consumer = 10000 state.Delivered.Stream = 10000 state.AckFloor.Consumer = 100 state.AckFloor.Stream = 100 // Generate 8k pending. state.Pending = make(map[uint64]*Pending) for len(state.Pending) < 8192 { seq := uint64(rand.Intn(9890) + 101) if _, ok := state.Pending[seq]; !ok { state.Pending[seq] = nt() } } updateAndCheck() state.Pending = nil state.AckFloor.Consumer = 10000 state.AckFloor.Stream = 10000 updateAndCheck() }) } func TestFileStoreConsumerEncodeDecodeRedelivered(t *testing.T) { state := &ConsumerState{} state.Delivered.Consumer = 100 state.Delivered.Stream = 100 state.AckFloor.Consumer = 50 state.AckFloor.Stream = 50 state.Redelivered = map[uint64]uint64{122: 3, 144: 8} buf := encodeConsumerState(state) rstate, err := decodeConsumerState(buf) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !reflect.DeepEqual(state, rstate) { t.Fatalf("States do not match: %+v vs %+v", state, rstate) } } func TestFileStoreConsumerEncodeDecodePendingBelowStreamAckFloor(t *testing.T) { state := &ConsumerState{} state.Delivered.Consumer = 1192 state.Delivered.Stream = 10185 state.AckFloor.Consumer = 1189 state.AckFloor.Stream = 10815 now := time.Now().Round(time.Second).Add(-10 * time.Second).UnixNano() state.Pending = map[uint64]*Pending{ 10782: {1190, now}, 10810: {1191, now + int64(time.Second)}, 10815: {1192, now + int64(2*time.Second)}, } buf := encodeConsumerState(state) rstate, err := decodeConsumerState(buf) if err != nil { t.Fatalf("Unexpected error: %v", err) } if len(rstate.Pending) != 3 { t.Fatalf("Invalid pending: %v", rstate.Pending) } for k, v := range state.Pending { rv, ok := rstate.Pending[k] if !ok { t.Fatalf("Did not find sseq=%v", k) } if !reflect.DeepEqual(v, rv) { t.Fatalf("Pending for sseq=%v should be %+v, got %+v", k, v, rv) } } state.Pending, rstate.Pending = nil, nil if !reflect.DeepEqual(*state, *rstate) { t.Fatalf("States do not match: %+v vs %+v", state, rstate) } } func TestFileStoreWriteFailures(t *testing.T) { // This test should be run inside an environment where this directory // has a limited size. // E.g. Docker // docker run -ti --tmpfs /jswf_test:rw,size=32k --rm -v ~/Development/go/src:/go/src -w /go/src/github.com/nats-io/nats-server/ golang:1.21 /bin/bash tdir := filepath.Join("/", "jswf_test") if stat, err := os.Stat(tdir); err != nil || !stat.IsDir() { t.SkipNow() } storeDir := filepath.Join(tdir, JetStreamStoreDir) os.MkdirAll(storeDir, 0755) testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.StoreDir = storeDir cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello Write Failures!") var lseq uint64 // msz about will be ~54 bytes, so if limit is 32k trying to send 1000 will fail at some point. for i := 1; i <= 1000; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { lseq = uint64(i) break } } if lseq == 0 { t.Fatalf("Expected to get a failure but did not") } state := fs.State() if state.LastSeq != lseq-1 { t.Fatalf("Expected last seq to be %d, got %d\n", lseq-1, state.LastSeq) } if state.Msgs != lseq-1 { t.Fatalf("Expected total msgs to be %d, got %d\n", lseq-1, state.Msgs) } if _, err := fs.LoadMsg(lseq, nil); err == nil { t.Fatalf("Expected error loading seq that failed, got none") } // Loading should still work. if _, err := fs.LoadMsg(1, nil); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state2 := fs.State() // Ignore lost state. state.Lost, state2.Lost = nil, nil if !reflect.DeepEqual(state2, state) { t.Fatalf("Expected recovered state to be the same\n%+v\nvs\n%+v\n", state2, state) } // We should still fail here. for i := 1; i <= 100; i++ { _, _, err = fs.StoreMsg(subj, nil, msg) if err != nil { break } } require_Error(t, err) lseq = fs.State().LastSeq + 1 // Purge should help. if _, err := fs.Purge(); err != nil { t.Fatalf("Unexpected error: %v", err) } // Wait for purge to complete its out of band processing. time.Sleep(50 * time.Millisecond) // Check we will fail again in same spot. for i := 1; i <= 1000; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { if i != int(lseq) { t.Fatalf("Expected to fail after purge about the same spot, wanted %d got %d", lseq, i) } break } } }) } func TestFileStorePerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.AsyncFlush = true subj, msg := "foo", make([]byte, 1024-33) for i := 0; i < len(msg); i++ { msg[i] = 'D' } storedMsgSize := fileStoreMsgSize(subj, nil, msg) // 5GB toStore := 5 * 1024 * 1024 * 1024 / storedMsgSize fmt.Printf("storing %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() start := time.Now() for i := 0; i < int(toStore); i++ { fs.StoreMsg(subj, nil, msg) } fs.Stop() tt := time.Since(start) fmt.Printf("time to store is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) fmt.Printf("Filesystem cache flush, paused 5 seconds.\n\n") time.Sleep(5 * time.Second) fmt.Printf("Restoring..\n") start = time.Now() fcfg.AsyncFlush = false fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fmt.Printf("time to restore is %v\n\n", time.Since(start)) fmt.Printf("LOAD: reading %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) var smv StoreMsg start = time.Now() for i := uint64(1); i <= toStore; i++ { if _, err := fs.LoadMsg(i, &smv); err != nil { t.Fatalf("Error loading %d: %v", i, err) } } tt = time.Since(start) fmt.Printf("time to read all back messages is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) // Do again to test skip for hash.. fmt.Printf("\nSKIP CHECKSUM: reading %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) start = time.Now() for i := uint64(1); i <= toStore; i++ { if _, err := fs.LoadMsg(i, &smv); err != nil { t.Fatalf("Error loading %d: %v", i, err) } } tt = time.Since(start) fmt.Printf("time to read all back messages is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fmt.Printf("\nremoving [in order] %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) start = time.Now() // For reverse order. //for i := toStore; i > 0; i-- { for i := uint64(1); i <= toStore; i++ { fs.RemoveMsg(i) } fs.Stop() tt = time.Since(start) fmt.Printf("time to remove all messages is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state := fs.State() if state.Msgs != 0 { t.Fatalf("Expected no msgs, got %d", state.Msgs) } if state.Bytes != 0 { t.Fatalf("Expected no bytes, got %d", state.Bytes) } }) } func TestFileStoreReadBackMsgPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() subj := "foo" msg := []byte("ABCDEFGH") // Smaller shows problems more. storedMsgSize := fileStoreMsgSize(subj, nil, msg) // Make sure we store 2 blocks. toStore := defaultLargeBlockSize * 2 / storedMsgSize testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fmt.Printf("storing %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() start := time.Now() for i := 0; i < int(toStore); i++ { fs.StoreMsg(subj, nil, msg) } tt := time.Since(start) fmt.Printf("time to store is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) // We should not have cached here with no reads. // Pick something towards end of the block. index := defaultLargeBlockSize/storedMsgSize - 22 start = time.Now() fs.LoadMsg(index, nil) fmt.Printf("Time to load first msg [%d] = %v\n", index, time.Since(start)) start = time.Now() fs.LoadMsg(index+2, nil) fmt.Printf("Time to load second msg [%d] = %v\n", index+2, time.Since(start)) }) } // This test is testing an upper level stream with a message or byte limit. // Even though this is 1, any limit would trigger same behavior once the limit was reached // and we were adding and removing. func TestFileStoreStoreLimitRemovePerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() subj, msg := "foo", make([]byte, 1024-33) for i := 0; i < len(msg); i++ { msg[i] = 'D' } storedMsgSize := fileStoreMsgSize(subj, nil, msg) // 1GB toStore := 1 * 1024 * 1024 * 1024 / storedMsgSize testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fs.RegisterStorageUpdates(func(md, bd int64, seq uint64, subj string) {}) fmt.Printf("storing and removing (limit 1) %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) start := time.Now() for i := 0; i < int(toStore); i++ { seq, _, err := fs.StoreMsg(subj, nil, msg) if err != nil { t.Fatalf("Unexpected error storing message: %v", err) } if i > 0 { fs.RemoveMsg(seq - 1) } } fs.Stop() tt := time.Since(start) fmt.Printf("time to store and remove all messages is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) }) } func TestFileStorePubPerfWithSmallBlkSize(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() subj, msg := "foo", make([]byte, 1024-33) for i := 0; i < len(msg); i++ { msg[i] = 'D' } storedMsgSize := fileStoreMsgSize(subj, nil, msg) // 1GB toStore := 1 * 1024 * 1024 * 1024 / storedMsgSize testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fmt.Printf("storing %d msgs of %s each, totalling %s\n", toStore, friendlyBytes(int64(storedMsgSize)), friendlyBytes(int64(toStore*storedMsgSize)), ) fcfg.BlockSize = FileStoreMinBlkSize fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() start := time.Now() for i := 0; i < int(toStore); i++ { fs.StoreMsg(subj, nil, msg) } fs.Stop() tt := time.Since(start) fmt.Printf("time to store is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) fmt.Printf("%s per sec\n", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds()))) }) } // Saw this manifest from a restart test with max delivered set for JetStream. func TestFileStoreConsumerRedeliveredLost(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() cfg := &ConsumerConfig{AckPolicy: AckExplicit} o, err := fs.ConsumerStore("o22", cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } restartConsumer := func() { t.Helper() o.Stop() time.Sleep(20 * time.Millisecond) // Wait for all things to settle. o, err = fs.ConsumerStore("o22", cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we recovered Redelivered. state, err := o.State() if err != nil { t.Fatalf("Unexpected error: %v", err) } if state == nil { t.Fatalf("Did not recover state") } if len(state.Redelivered) == 0 { t.Fatalf("Did not recover redelivered") } } ts := time.Now().UnixNano() o.UpdateDelivered(1, 1, 1, ts) o.UpdateDelivered(2, 1, 2, ts) o.UpdateDelivered(3, 1, 3, ts) o.UpdateDelivered(4, 1, 4, ts) o.UpdateDelivered(5, 2, 1, ts) restartConsumer() o.UpdateDelivered(6, 2, 2, ts) o.UpdateDelivered(7, 3, 1, ts) restartConsumer() if state, _ := o.State(); len(state.Pending) != 3 { t.Fatalf("Did not recover pending correctly") } o.UpdateAcks(7, 3) o.UpdateAcks(6, 2) restartConsumer() o.UpdateAcks(4, 1) state, _ := o.State() if len(state.Pending) != 0 { t.Fatalf("Did not clear pending correctly") } if len(state.Redelivered) != 0 { t.Fatalf("Did not clear redelivered correctly") } }) } func TestFileStoreConsumerFlusher(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() o, err := fs.ConsumerStore("o22", &ConsumerConfig{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Get the underlying impl. oc := o.(*consumerFileStore) // Wait for flusher to be running. checkFor(t, time.Second, 20*time.Millisecond, func() error { if !oc.inFlusher() { return fmt.Errorf("Flusher not running") } return nil }) // Stop and make sure the flusher goes away o.Stop() // Wait for flusher to stop. checkFor(t, time.Second, 20*time.Millisecond, func() error { if oc.inFlusher() { return fmt.Errorf("Flusher still running") } return nil }) }) } func TestFileStoreConsumerDeliveredUpdates(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Simple consumer, no ack policy configured. o, err := fs.ConsumerStore("o22", &ConsumerConfig{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.Stop() testDelivered := func(dseq, sseq uint64) { t.Helper() ts := time.Now().UnixNano() if err := o.UpdateDelivered(dseq, sseq, 1, ts); err != nil { t.Fatalf("Unexpected error: %v", err) } state, err := o.State() if err != nil { t.Fatalf("Error getting state: %v", err) } if state == nil { t.Fatalf("No state available") } expected := SequencePair{dseq, sseq} if state.Delivered != expected { t.Fatalf("Unexpected state, wanted %+v, got %+v", expected, state.Delivered) } if state.AckFloor != expected { t.Fatalf("Unexpected ack floor state, wanted %+v, got %+v", expected, state.AckFloor) } if len(state.Pending) != 0 { t.Fatalf("Did not expect any pending, got %d pending", len(state.Pending)) } } testDelivered(1, 100) testDelivered(2, 110) testDelivered(5, 130) // If we try to do an ack this should err since we are not configured with ack policy. if err := o.UpdateAcks(1, 100); err != ErrNoAckPolicy { t.Fatalf("Expected a no ack policy error on update acks, got %v", err) } // Also if we do an update with a delivery count of anything but 1 here should also give same error. ts := time.Now().UnixNano() if err := o.UpdateDelivered(5, 130, 2, ts); err != ErrNoAckPolicy { t.Fatalf("Expected a no ack policy error on update delivered with dc > 1, got %v", err) } }) } func TestFileStoreConsumerDeliveredAndAckUpdates(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Simple consumer, no ack policy configured. o, err := fs.ConsumerStore("o22", &ConsumerConfig{AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.Stop() // Track pending. var pending int testDelivered := func(dseq, sseq uint64) { t.Helper() ts := time.Now().Round(time.Second).UnixNano() if err := o.UpdateDelivered(dseq, sseq, 1, ts); err != nil { t.Fatalf("Unexpected error: %v", err) } pending++ state, err := o.State() if err != nil { t.Fatalf("Error getting state: %v", err) } if state == nil { t.Fatalf("No state available") } expected := SequencePair{dseq, sseq} if state.Delivered != expected { t.Fatalf("Unexpected delivered state, wanted %+v, got %+v", expected, state.Delivered) } if len(state.Pending) != pending { t.Fatalf("Expected %d pending, got %d pending", pending, len(state.Pending)) } } testDelivered(1, 100) testDelivered(2, 110) testDelivered(3, 130) testDelivered(4, 150) testDelivered(5, 165) testBadAck := func(dseq, sseq uint64) { t.Helper() if err := o.UpdateAcks(dseq, sseq); err == nil { t.Fatalf("Expected error but got none") } } testBadAck(3, 101) testBadAck(1, 1) testAck := func(dseq, sseq, dflr, sflr uint64) { t.Helper() if err := o.UpdateAcks(dseq, sseq); err != nil { t.Fatalf("Unexpected error: %v", err) } pending-- state, err := o.State() if err != nil { t.Fatalf("Error getting state: %v", err) } if state == nil { t.Fatalf("No state available") } if len(state.Pending) != pending { t.Fatalf("Expected %d pending, got %d pending", pending, len(state.Pending)) } eflr := SequencePair{dflr, sflr} if state.AckFloor != eflr { t.Fatalf("Unexpected ack floor state, wanted %+v, got %+v", eflr, state.AckFloor) } } testAck(1, 100, 1, 109) testAck(3, 130, 1, 109) testAck(2, 110, 3, 149) // We do not track explicit state on previous stream floors, so we take last known -1 testAck(5, 165, 3, 149) testAck(4, 150, 5, 165) testDelivered(6, 170) testDelivered(7, 171) testDelivered(8, 172) testDelivered(9, 173) testDelivered(10, 200) testAck(7, 171, 5, 165) testAck(8, 172, 5, 165) state, err := o.State() if err != nil { t.Fatalf("Unexpected error getting state: %v", err) } o.Stop() o, err = fs.ConsumerStore("o22", &ConsumerConfig{AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.Stop() nstate, err := o.State() if err != nil { t.Fatalf("Unexpected error getting state: %v", err) } if !reflect.DeepEqual(nstate, state) { t.Fatalf("States don't match! NEW %+v OLD %+v", nstate, state) } }) } func TestFileStoreStreamStateDeleted(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, toStore := "foo", uint64(10) for i := uint64(1); i <= toStore; i++ { msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } } state := fs.State() if len(state.Deleted) != 0 { t.Fatalf("Expected deleted to be empty") } // Now remove some interior messages. var expected []uint64 for seq := uint64(2); seq < toStore; seq += 2 { fs.RemoveMsg(seq) expected = append(expected, seq) } state = fs.State() if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } // Now fill the gap by deleting 1 and 3 fs.RemoveMsg(1) fs.RemoveMsg(3) expected = expected[2:] state = fs.State() if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } if state.FirstSeq != 5 { t.Fatalf("Expected first seq to be 5, got %d", state.FirstSeq) } fs.Purge() if state = fs.State(); len(state.Deleted) != 0 { t.Fatalf("Expected no deleted after purge, got %+v\n", state.Deleted) } }) } // We have reports that sometimes under load a stream could complain about a storage directory // not being empty. func TestFileStoreStreamDeleteDirNotEmpty(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, toStore := "foo", uint64(10) for i := uint64(1); i <= toStore; i++ { msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } } ready := make(chan bool) go func() { g := filepath.Join(fcfg.StoreDir, "g") ready <- true for i := 0; i < 100; i++ { os.WriteFile(g, []byte("OK"), defaultFilePerms) } }() <-ready if err := fs.Delete(); err != nil { t.Fatalf("Delete returned an error: %v", err) } }) } func TestFileStoreConsumerPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() o, err := fs.ConsumerStore("o22", &ConsumerConfig{AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Get the underlying impl. oc := o.(*consumerFileStore) // Wait for flusher to br running checkFor(t, time.Second, 20*time.Millisecond, func() error { if !oc.inFlusher() { return fmt.Errorf("not in flusher") } return nil }) // Stop flusher for this benchmark since we will invoke directly. oc.mu.Lock() qch := oc.qch oc.qch = nil oc.mu.Unlock() close(qch) checkFor(t, time.Second, 20*time.Millisecond, func() error { if oc.inFlusher() { return fmt.Errorf("still in flusher") } return nil }) toStore := uint64(1_000_000) start := time.Now() ts := start.UnixNano() for i := uint64(1); i <= toStore; i++ { if err := o.UpdateDelivered(i, i, 1, ts); err != nil { t.Fatalf("Unexpected error: %v", err) } } tt := time.Since(start) fmt.Printf("time to update %d is %v\n", toStore, tt) fmt.Printf("%.0f updates/sec\n", float64(toStore)/tt.Seconds()) start = time.Now() oc.mu.Lock() buf, err := oc.encodeState() oc.mu.Unlock() if err != nil { t.Fatalf("Error encoding state: %v", err) } fmt.Printf("time to encode %d bytes is %v\n", len(buf), time.Since(start)) start = time.Now() oc.writeState(buf) fmt.Printf("time to write is %v\n", time.Since(start)) }) } // Reported by Ivan. func TestFileStoreStreamDeleteCacheBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.CacheExpire = 50 * time.Millisecond fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := fs.EraseMsg(1); err != nil { t.Fatalf("Got an error on remove of %d: %v", 1, err) } time.Sleep(100 * time.Millisecond) if _, err := fs.LoadMsg(2, nil); err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } }) } // rip func TestFileStoreStreamFailToRollBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 512 fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage, MaxBytes: 300}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure we properly roll underlying blocks. n, msg := 200, bytes.Repeat([]byte("ABC"), 33) // ~100bytes for i := 0; i < n; i++ { if _, _, err := fs.StoreMsg("zzz", nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Grab some info for introspection. fs.mu.RLock() numBlks := len(fs.blks) var index uint32 var blkSize int64 if numBlks > 0 { mb := fs.blks[0] mb.mu.RLock() index = mb.index if fi, _ := os.Stat(mb.mfn); fi != nil { blkSize = fi.Size() } mb.mu.RUnlock() } fs.mu.RUnlock() if numBlks != 1 { t.Fatalf("Expected only one block, got %d", numBlks) } if index < 60 { t.Fatalf("Expected a block index > 60, got %d", index) } if blkSize > 512 { t.Fatalf("Expected block to be <= 512, got %d", blkSize) } }) } // We had a case where a consumer state had a redelivered record that had seq of 0. // This was causing the server to panic. func TestFileStoreBadConsumerState(t *testing.T) { bs := []byte("\x16\x02\x01\x01\x03\x02\x01\x98\xf4\x8a\x8a\f\x01\x03\x86\xfa\n\x01\x00\x01") if cs, err := decodeConsumerState(bs); err != nil || cs == nil { t.Fatalf("Expected to not throw error, got %v and %+v", err, cs) } } func TestFileStoreExpireMsgsOnStart(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 8 * 1024 ttl := 250 * time.Millisecond cfg := StreamConfig{Name: "ORDERS", Subjects: []string{"orders.*"}, Storage: FileStorage, MaxAge: ttl} var fs *fileStore startFS := func() *fileStore { t.Helper() fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) return fs } newFS := func() *fileStore { t.Helper() if fs != nil { fs.Stop() fs = nil } removeDir(t, fcfg.StoreDir) return startFS() } restartFS := func(delay time.Duration) *fileStore { if fs != nil { fs.Stop() fs = nil time.Sleep(delay) } fs = startFS() return fs } fs = newFS() defer fs.Stop() msg := bytes.Repeat([]byte("ABC"), 33) // ~100bytes loadMsgs := func(n int) { t.Helper() for i := 1; i <= n; i++ { if _, _, err := fs.StoreMsg(fmt.Sprintf("orders.%d", i%10), nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } } checkState := func(msgs, first, last uint64) { t.Helper() if fs == nil { t.Fatalf("No fs") return } state := fs.State() if state.Msgs != msgs { t.Fatalf("Expected %d msgs, got %d", msgs, state.Msgs) } if state.FirstSeq != first { t.Fatalf("Expected %d as first, got %d", first, state.FirstSeq) } if state.LastSeq != last { t.Fatalf("Expected %d as last, got %d", last, state.LastSeq) } } checkNumBlks := func(expected int) { t.Helper() fs.mu.RLock() n := len(fs.blks) fs.mu.RUnlock() if n != expected { t.Fatalf("Expected %d msg blks, got %d", expected, n) } } // Check the filtered subject state and make sure that is tracked properly. checkFiltered := func(subject string, ss SimpleState) { t.Helper() fss := fs.FilteredState(1, subject) if fss != ss { t.Fatalf("Expected FilteredState of %+v, got %+v", ss, fss) } } // Make sure state on disk matches (e.g. writeIndexInfo properly called) checkBlkState := func(index int) { t.Helper() fs.mu.RLock() if index >= len(fs.blks) { t.Fatalf("Out of range, wanted %d but only %d blks", index, len(fs.blks)) } fs.mu.RUnlock() } lastSeqForBlk := func() uint64 { t.Helper() fs.mu.RLock() defer fs.mu.RUnlock() if len(fs.blks) == 0 { t.Fatalf("No blocks?") } mb := fs.blks[0] mb.mu.RLock() defer mb.mu.RUnlock() return mb.last.seq } // Actual testing here. loadMsgs(500) restartFS(ttl + 100*time.Millisecond) checkState(0, 501, 500) // We actually hold onto the last one now to remember our starting sequence. checkNumBlks(1) // Now check partial expires and the fss tracking state. // Small numbers is to keep them in one block. fs = newFS() loadMsgs(10) time.Sleep(100 * time.Millisecond) loadMsgs(10) checkFiltered("orders.*", SimpleState{Msgs: 20, First: 1, Last: 20}) restartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half checkState(10, 11, 20) checkNumBlks(1) checkFiltered("orders.*", SimpleState{Msgs: 10, First: 11, Last: 20}) checkFiltered("orders.5", SimpleState{Msgs: 1, First: 15, Last: 15}) checkBlkState(0) fs = newFS() loadMsgs(5) time.Sleep(100 * time.Millisecond) loadMsgs(15) restartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half checkState(15, 6, 20) checkFiltered("orders.*", SimpleState{Msgs: 15, First: 6, Last: 20}) checkFiltered("orders.5", SimpleState{Msgs: 2, First: 10, Last: 20}) // Now we want to test that if the end of a msg block is all deletes msgs that we do the right thing. fs = newFS() loadMsgs(150) time.Sleep(100 * time.Millisecond) loadMsgs(100) checkNumBlks(5) // Now delete 10 messages from the end of the first block which we will expire on restart. // We will expire up to seq 100, so delete 91-100. lseq := lastSeqForBlk() for seq := lseq; seq > lseq-10; seq-- { removed, err := fs.RemoveMsg(seq) if err != nil || !removed { t.Fatalf("Error removing message: %v", err) } } restartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half checkState(100, 151, 250) checkNumBlks(3) // We should only have 3 blks left. checkBlkState(0) // Now make sure that we properly clean up any internal dmap entries (sparse) when expiring. fs = newFS() loadMsgs(10) // Remove some in sparse fashion, adding to dmap. fs.RemoveMsg(2) fs.RemoveMsg(4) fs.RemoveMsg(6) time.Sleep(100 * time.Millisecond) loadMsgs(10) restartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half checkState(10, 11, 20) checkNumBlks(1) checkBlkState(0) // Make sure expiring a block with tail deleted messages removes the message block etc. fs = newFS() loadMsgs(7) time.Sleep(100 * time.Millisecond) loadMsgs(3) fs.RemoveMsg(8) fs.RemoveMsg(9) fs.RemoveMsg(10) restartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) checkState(0, 11, 10) fs.Stop() // Not for start per se but since we have all the test tooling here check that Compact() does right thing as well. fs = newFS() defer fs.Stop() loadMsgs(100) checkFiltered("orders.*", SimpleState{Msgs: 100, First: 1, Last: 100}) checkFiltered("orders.5", SimpleState{Msgs: 10, First: 5, Last: 95}) // Check that Compact keeps fss updated, does dmap etc. fs.Compact(51) checkFiltered("orders.*", SimpleState{Msgs: 50, First: 51, Last: 100}) checkFiltered("orders.5", SimpleState{Msgs: 5, First: 55, Last: 95}) checkBlkState(0) }) } func TestFileStoreSparseCompaction(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1024 * 1024 cfg := StreamConfig{Name: "KV", Subjects: []string{"kv.>"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("ABC"), 33) // ~100bytes loadMsgs := func(n int) { t.Helper() for i := 1; i <= n; i++ { if _, _, err := fs.StoreMsg(fmt.Sprintf("kv.%d", i%10), nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } } checkState := func(msgs, first, last uint64) { t.Helper() if fs == nil { t.Fatalf("No fs") return } state := fs.State() if state.Msgs != msgs { t.Fatalf("Expected %d msgs, got %d", msgs, state.Msgs) } if state.FirstSeq != first { t.Fatalf("Expected %d as first, got %d", first, state.FirstSeq) } if state.LastSeq != last { t.Fatalf("Expected %d as last, got %d", last, state.LastSeq) } } deleteMsgs := func(seqs ...uint64) { t.Helper() for _, seq := range seqs { removed, err := fs.RemoveMsg(seq) if err != nil || !removed { t.Fatalf("Got an error on remove of %d: %v", seq, err) } } } eraseMsgs := func(seqs ...uint64) { t.Helper() for _, seq := range seqs { removed, err := fs.EraseMsg(seq) if err != nil || !removed { t.Fatalf("Got an error on erase of %d: %v", seq, err) } } } compact := func() { t.Helper() var ssb, ssa StreamState fs.FastState(&ssb) tb, ub, _ := fs.Utilization() fs.mu.RLock() if len(fs.blks) == 0 { t.Fatalf("No blocks?") } mb := fs.blks[0] fs.mu.RUnlock() mb.mu.Lock() mb.compact() mb.mu.Unlock() fs.FastState(&ssa) if !reflect.DeepEqual(ssb, ssa) { t.Fatalf("States do not match; %+v vs %+v", ssb, ssa) } ta, ua, _ := fs.Utilization() if ub != ua { t.Fatalf("Expected used to be the same, got %d vs %d", ub, ua) } if ta >= tb { t.Fatalf("Expected total after to be less then before, got %d vs %d", tb, ta) } } // Actual testing here. loadMsgs(1000) checkState(1000, 1, 1000) // Now delete a few messages. deleteMsgs(1) compact() deleteMsgs(1000, 999, 998, 997) compact() eraseMsgs(500, 502, 504, 506, 508, 510) compact() }) } func TestFileStoreSparseCompactionWithInteriorDeletes(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "KV", Subjects: []string{"kv.>"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 1; i <= 1000; i++ { if _, _, err := fs.StoreMsg(fmt.Sprintf("kv.%d", i%10), nil, []byte("OK")); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Now do interior deletes. for _, seq := range []uint64{500, 600, 700, 800} { removed, err := fs.RemoveMsg(seq) if err != nil || !removed { t.Fatalf("Got an error on remove of %d: %v", seq, err) } } _, err = fs.LoadMsg(900, nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do compact by hand, make sure we can still access msgs past the interior deletes. fs.mu.RLock() lmb := fs.lmb lmb.dirtyCloseWithRemove(false) lmb.compact() fs.mu.RUnlock() if _, err = fs.LoadMsg(900, nil); err != nil { t.Fatalf("Unexpected error: %v", err) } }) } // When messages span multiple blocks and we want to purge but keep some amount, say 1, we would remove all. // This is because we would not break out of iterator across more message blocks. // Issue #2622 func TestFileStorePurgeExKeepOneBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 128 cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fill := bytes.Repeat([]byte("X"), 128) fs.StoreMsg("A", nil, []byte("META")) fs.StoreMsg("B", nil, fill) fs.StoreMsg("A", nil, []byte("META")) fs.StoreMsg("B", nil, fill) if fss := fs.FilteredState(1, "A"); fss.Msgs != 2 { t.Fatalf("Expected to find 2 `A` msgs, got %d", fss.Msgs) } n, err := fs.PurgeEx("A", 0, 1) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n != 1 { t.Fatalf("Expected PurgeEx to remove 1 `A` msgs, got %d", n) } if fss := fs.FilteredState(1, "A"); fss.Msgs != 1 { t.Fatalf("Expected to find 1 `A` msgs, got %d", fss.Msgs) } }) } func TestFileStoreFilteredPendingBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, []byte("msg")) fs.StoreMsg("bar", nil, []byte("msg")) fs.StoreMsg("baz", nil, []byte("msg")) fs.mu.Lock() mb := fs.lmb fs.mu.Unlock() total, f, l := mb.filteredPending("foo", false, 3) if total != 0 { t.Fatalf("Expected total of 0 but got %d", total) } if f != 0 || l != 0 { t.Fatalf("Expected first and last to be 0 as well, but got %d %d", f, l) } }) } // Test to optimize the selectMsgBlock with lots of blocks. func TestFileStoreFetchPerf(t *testing.T) { // Comment out to run. t.SkipNow() testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 8192 fcfg.AsyncFlush = true cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Will create 25k msg blocks. n, subj, msg := 100_000, "zzz", bytes.Repeat([]byte("ABC"), 600) for i := 0; i < n; i++ { if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Time how long it takes us to load all messages. var smv StoreMsg now := time.Now() for i := 0; i < n; i++ { _, err := fs.LoadMsg(uint64(i), &smv) if err != nil { t.Fatalf("Unexpected error looking up seq %d: %v", i, err) } } fmt.Printf("Elapsed to load all messages is %v\n", time.Since(now)) }) } // For things like raft log when we compact and have a message block that could reclaim > 50% of space for block we want to do that. // https://github.com/nats-io/nats-server/issues/2936 func TestFileStoreCompactReclaimHeadSpace(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 4 * 1024 * 1024 cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Create random bytes for payload to test for corruption vs repeated. msg := make([]byte, 64*1024) crand.Read(msg) // This gives us ~63 msgs in first and ~37 in second. n, subj := 100, "z" for i := 0; i < n; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } checkNumBlocks := func(n int) { t.Helper() fs.mu.RLock() defer fs.mu.RUnlock() if len(fs.blks) != n { t.Fatalf("Expected to have %d blocks, got %d", n, len(fs.blks)) } } getBlock := func(index int) *msgBlock { t.Helper() fs.mu.RLock() defer fs.mu.RUnlock() return fs.blks[index] } // Check that we did right thing and actually reclaimed since > 50% checkBlock := func(mb *msgBlock) { t.Helper() mb.mu.RLock() nbytes, rbytes, mfn := mb.bytes, mb.rbytes, mb.mfn fseq, lseq := mb.first.seq, mb.last.seq mb.mu.RUnlock() // Check that sizes match as long as we are not doing compression. if fcfg.Compression == NoCompression { // Check rbytes then the actual file as well. if nbytes != rbytes { t.Fatalf("Expected to reclaim and have bytes == rbytes, got %d vs %d", nbytes, rbytes) } file, err := os.Open(mfn) require_NoError(t, err) defer file.Close() fi, err := file.Stat() require_NoError(t, err) if rbytes != uint64(fi.Size()) { t.Fatalf("Expected to rbytes == fi.Size, got %d vs %d", rbytes, fi.Size()) } } // Make sure we can pull messages and that they are ok. var smv StoreMsg sm, err := fs.LoadMsg(fseq, &smv) require_NoError(t, err) if !bytes.Equal(sm.msg, msg) { t.Fatalf("Msgs don't match, original %q vs %q", msg, sm.msg) } sm, err = fs.LoadMsg(lseq, &smv) require_NoError(t, err) if !bytes.Equal(sm.msg, msg) { t.Fatalf("Msgs don't match, original %q vs %q", msg, sm.msg) } } checkNumBlocks(2) _, err = fs.Compact(33) require_NoError(t, err) checkNumBlocks(2) checkBlock(getBlock(0)) checkBlock(getBlock(1)) _, err = fs.Compact(85) require_NoError(t, err) checkNumBlocks(1) checkBlock(getBlock(0)) // Make sure we can write. _, _, err = fs.StoreMsg(subj, nil, msg) require_NoError(t, err) checkNumBlocks(1) checkBlock(getBlock(0)) // Stop and start again. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() checkNumBlocks(1) checkBlock(getBlock(0)) // Make sure we can write. _, _, err = fs.StoreMsg(subj, nil, msg) require_NoError(t, err) }) } func TestFileStoreRememberLastMsgTime(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { var fs *fileStore cfg := StreamConfig{Name: "TEST", Storage: FileStorage, MaxAge: 1 * time.Second} getFS := func() *fileStore { t.Helper() fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) return fs } restartFS := func() { t.Helper() fs.Stop() fs = getFS() } msg := bytes.Repeat([]byte("X"), 2*1024*1024) // Get first one. fs = getFS() defer fs.Stop() seq, ts, err := fs.StoreMsg("foo", nil, msg) require_NoError(t, err) // We will test that last msg time survives from delete, purge and expires after restart. removed, err := fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, removed) lt := time.Unix(0, ts).UTC() require_True(t, lt == fs.State().LastTime) // Restart restartFS() // Test that last time survived. require_True(t, lt == fs.State().LastTime) seq, ts, err = fs.StoreMsg("foo", nil, msg) require_NoError(t, err) var smv StoreMsg _, err = fs.LoadMsg(seq, &smv) require_NoError(t, err) fs.Purge() // Restart restartFS() lt = time.Unix(0, ts).UTC() require_True(t, lt == fs.State().LastTime) _, _, err = fs.StoreMsg("foo", nil, msg) require_NoError(t, err) seq, ts, err = fs.StoreMsg("foo", nil, msg) require_NoError(t, err) require_True(t, seq == 4) // Wait til messages expire. checkFor(t, 5*time.Second, time.Second, func() error { state := fs.State() if state.Msgs == 0 { return nil } return fmt.Errorf("Still has %d msgs", state.Msgs) }) // Restart restartFS() lt = time.Unix(0, ts).UTC() require_True(t, lt == fs.State().LastTime) // Now make sure we retain the true last seq. _, _, err = fs.StoreMsg("foo", nil, msg) require_NoError(t, err) seq, ts, err = fs.StoreMsg("foo", nil, msg) require_NoError(t, err) require_True(t, seq == 6) removed, err = fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, removed) removed, err = fs.RemoveMsg(seq - 1) require_NoError(t, err) require_True(t, removed) // Restart restartFS() lt = time.Unix(0, ts).UTC() require_True(t, lt == fs.State().LastTime) require_True(t, seq == 6) }) } func (fs *fileStore) getFirstBlock() *msgBlock { fs.mu.RLock() defer fs.mu.RUnlock() if len(fs.blks) == 0 { return nil } return fs.blks[0] } func TestFileStoreRebuildStateDmapAccountingBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1024 * 1024 fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 100; i++ { _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } // Delete 2-40. for i := 2; i <= 40; i++ { _, err := fs.RemoveMsg(uint64(i)) require_NoError(t, err) } mb := fs.getFirstBlock() require_True(t, mb != nil) check := func() { t.Helper() mb.mu.RLock() defer mb.mu.RUnlock() dmapLen := uint64(mb.dmap.Size()) if mb.msgs != (mb.last.seq-mb.first.seq+1)-dmapLen { t.Fatalf("Consistency check failed: %d != %d -> last %d first %d len(dmap) %d", mb.msgs, (mb.last.seq-mb.first.seq+1)-dmapLen, mb.last.seq, mb.first.seq, dmapLen) } } check() mb.mu.Lock() mb.compact() mb.mu.Unlock() // Now delete first. _, err = fs.RemoveMsg(1) require_NoError(t, err) mb.mu.Lock() _, _, err = mb.rebuildStateLocked() mb.mu.Unlock() require_NoError(t, err) check() }) } func TestFileStorePurgeExWithSubject(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1000 cfg := StreamConfig{Name: "TEST", Subjects: []string{"foo.>"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() payload := make([]byte, 20) _, _, err = fs.StoreMsg("foo.0", nil, payload) require_NoError(t, err) total := 200 for i := 0; i < total; i++ { _, _, err = fs.StoreMsg("foo.1", nil, payload) require_NoError(t, err) } _, _, err = fs.StoreMsg("foo.2", nil, []byte("xxxxxx")) require_NoError(t, err) // This should purge all "foo.1" p, err := fs.PurgeEx("foo.1", 1, 0) require_NoError(t, err) require_Equal(t, p, uint64(total)) state := fs.State() require_Equal(t, state.Msgs, 2) require_Equal(t, state.FirstSeq, 1) // Make sure we can recover same state. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() before := state if state := fs.State(); !reflect.DeepEqual(state, before) { t.Fatalf("Expected state of %+v, got %+v", before, state) } // Also make sure we can recover properly with no index.db present. // We want to make sure we preserve any tombstones from the subject based purge. fs.Stop() os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); !reflect.DeepEqual(state, before) { t.Fatalf("Expected state of %+v, got %+v without index.db state", before, state) } }) } // When the N.idx file is shorter than the previous write we could fail to recover the idx properly. // For instance, with encryption and an expiring stream that has no messages, when a restart happens the decrypt will fail // since their are extra bytes, and this could lead to a stream sequence reset to zero. // // NOTE: We do not use idx files anymore, but keeping test. func TestFileStoreShortIndexWriteBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { // Encrypted mode shows, but could effect non-encrypted mode. cfg := StreamConfig{Name: "TEST", Storage: FileStorage, MaxAge: time.Second} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 100; i++ { _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } // Wait til messages all go away. checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { if state := fs.State(); state.Msgs != 0 { return fmt.Errorf("Expected no msgs, got %d", state.Msgs) } return nil }) if state := fs.State(); state.FirstSeq != 101 { t.Fatalf("Expected first sequence of 101 vs %d", state.FirstSeq) } // Now restart.. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); state.FirstSeq != 101 || state.LastSeq != 100 { t.Fatalf("Expected first sequence of 101 vs %d", state.FirstSeq) } }) } func TestFileStoreDoubleCompactWithWriteInBetweenEncryptedBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("ouch") for i := 0; i < 5; i++ { fs.StoreMsg(subj, nil, msg) } _, err = fs.Compact(5) require_NoError(t, err) if state := fs.State(); state.LastSeq != 5 { t.Fatalf("Expected last sequence to be 5 but got %d", state.LastSeq) } for i := 0; i < 5; i++ { fs.StoreMsg(subj, nil, msg) } _, err = fs.Compact(10) require_NoError(t, err) if state := fs.State(); state.LastSeq != 10 { t.Fatalf("Expected last sequence to be 10 but got %d", state.LastSeq) } }) } // When we kept the empty block for tracking sequence, we needed to reset the bek // counter when encrypted for subsequent writes to be correct. The bek in place could // possibly still have a non-zero counter from previous writes. // Happens when all messages expire and the are flushed and then subsequent writes occur. func TestFileStoreEncryptedKeepIndexNeedBekResetBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { ttl := 1 * time.Second cfg := StreamConfig{Name: "zzz", Storage: FileStorage, MaxAge: ttl} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("ouch") for i := 0; i < 5; i++ { fs.StoreMsg(subj, nil, msg) } // Want to go to 0. // This will leave the marker. checkFor(t, 5*time.Second, ttl, func() error { if state := fs.State(); state.Msgs != 0 { return fmt.Errorf("Expected no msgs, got %d", state.Msgs) } return nil }) // Now write additional messages. for i := 0; i < 5; i++ { fs.StoreMsg(subj, nil, msg) } // Make sure the buffer is cleared. fs.mu.RLock() mb := fs.lmb fs.mu.RUnlock() mb.mu.Lock() mb.clearCacheAndOffset() mb.mu.Unlock() // Now make sure we can read. var smv StoreMsg _, err = fs.LoadMsg(10, &smv) require_NoError(t, err) }) } func (fs *fileStore) reportMeta() (hasPSIM, hasAnyFSS bool) { fs.mu.RLock() defer fs.mu.RUnlock() hasPSIM = fs.psim != nil for _, mb := range fs.blks { mb.mu.RLock() hasAnyFSS = hasAnyFSS || mb.fss != nil mb.mu.RUnlock() if hasAnyFSS { break } } return hasPSIM, hasAnyFSS } func TestFileStoreExpireSubjectMeta(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1024 fcfg.CacheExpire = 500 * time.Millisecond fcfg.SubjectStateExpire = time.Second cfg := StreamConfig{Name: "zzz", Subjects: []string{"kv.>"}, Storage: FileStorage, MaxMsgsPer: 1} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() ns := 100 for i := 1; i <= ns; i++ { subj := fmt.Sprintf("kv.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) require_NoError(t, err) } // Test that on restart we do not have extensize metadata but do have correct number of subjects/keys. // Only thing really needed for store state / stream info. fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() var ss StreamState fs.FastState(&ss) if ss.NumSubjects != ns { t.Fatalf("Expected NumSubjects of %d, got %d", ns, ss.NumSubjects) } // Make sure we clear mb fss meta checkFor(t, fcfg.SubjectStateExpire*2, 500*time.Millisecond, func() error { if _, hasAnyFSS := fs.reportMeta(); hasAnyFSS { return fmt.Errorf("Still have mb fss state") } return nil }) // LoadLast, which is what KV uses, should load meta and succeed. _, err = fs.LoadLastMsg("kv.22", nil) require_NoError(t, err) // Make sure we clear mb fss meta checkFor(t, fcfg.SubjectStateExpire*2, 500*time.Millisecond, func() error { if _, hasAnyFSS := fs.reportMeta(); hasAnyFSS { return fmt.Errorf("Still have mb fss state") } return nil }) }) } func TestFileStoreMaxMsgsPerSubject(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 128 fcfg.CacheExpire = time.Second cfg := StreamConfig{Name: "zzz", Subjects: []string{"kv.>"}, Storage: FileStorage, MaxMsgsPer: 1} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() ns := 100 for i := 1; i <= ns; i++ { subj := fmt.Sprintf("kv.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) require_NoError(t, err) } for i := 1; i <= ns; i++ { subj := fmt.Sprintf("kv.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) require_NoError(t, err) } if state := fs.State(); state.Msgs != 100 || state.FirstSeq != 101 || state.LastSeq != 200 || len(state.Deleted) != 0 { t.Fatalf("Bad state: %+v", state) } if nb := fs.numMsgBlocks(); nb != 34 { t.Fatalf("Expected 34 blocks, got %d", nb) } }) } // Testing the case in https://github.com/nats-io/nats-server/issues/4247 func TestFileStoreMaxMsgsAndMaxMsgsPerSubject(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 128 fcfg.CacheExpire = time.Second cfg := StreamConfig{ Name: "zzz", Subjects: []string{"kv.>"}, Storage: FileStorage, Discard: DiscardNew, MaxMsgs: 100, // Total stream policy DiscardNewPer: true, MaxMsgsPer: 1, // Per-subject policy } fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 1; i <= 101; i++ { subj := fmt.Sprintf("kv.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) if i == 101 { // The 101th iteration should fail because MaxMsgs is set to // 100 and the policy is DiscardNew. require_Error(t, err) } else { require_NoError(t, err) } } for i := 1; i <= 100; i++ { subj := fmt.Sprintf("kv.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) // All of these iterations should fail because MaxMsgsPer is set // to 1 and DiscardNewPer is set to true, forcing us to reject // cases where there is already a message on this subject. require_Error(t, err) } if state := fs.State(); state.Msgs != 100 || state.FirstSeq != 1 || state.LastSeq != 100 || len(state.Deleted) != 0 { // There should be 100 messages exactly, as the 101st subject // should have been rejected in the first loop, and any duplicates // on the other subjects should have been rejected in the second loop. t.Fatalf("Bad state: %+v", state) } }) } func TestFileStoreSubjectStateCacheExpiration(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 32 fcfg.CacheExpire = time.Second fcfg.SubjectStateExpire = time.Second cfg := StreamConfig{Name: "zzz", Subjects: []string{"kv.>"}, Storage: FileStorage, MaxMsgsPer: 2} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 1; i <= 100; i++ { subj := fmt.Sprintf("kv.foo.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) require_NoError(t, err) } for i := 1; i <= 100; i++ { subj := fmt.Sprintf("kv.bar.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("value")) require_NoError(t, err) } // Make sure we clear mb fss meta before asking for SubjectState. checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { if _, hasAnyFSS := fs.reportMeta(); hasAnyFSS { return fmt.Errorf("Still have mb fss state") } return nil }) if fss := fs.SubjectsState("kv.bar.>"); len(fss) != 100 { t.Fatalf("Expected 100 entries but got %d", len(fss)) } fss := fs.SubjectsState("kv.bar.99") if len(fss) != 1 { t.Fatalf("Expected 1 entry but got %d", len(fss)) } expected := SimpleState{Msgs: 1, First: 199, Last: 199} if ss := fss["kv.bar.99"]; ss != expected { t.Fatalf("Bad subject state, expected %+v but got %+v", expected, ss) } // Now add one to end and check as well for non-wildcard. _, _, err = fs.StoreMsg("kv.foo.1", nil, []byte("value22")) require_NoError(t, err) if state := fs.State(); state.Msgs != 201 { t.Fatalf("Expected 201 msgs but got %+v", state) } fss = fs.SubjectsState("kv.foo.1") if len(fss) != 1 { t.Fatalf("Expected 1 entry but got %d", len(fss)) } expected = SimpleState{Msgs: 2, First: 1, Last: 201} if ss := fss["kv.foo.1"]; ss != expected { t.Fatalf("Bad subject state, expected %+v but got %+v", expected, ss) } }) } func TestFileStoreEncrypted(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("aes ftw") for i := 0; i < 50; i++ { fs.StoreMsg(subj, nil, msg) } o, err := fs.ConsumerStore("o22", &ConsumerConfig{}) require_NoError(t, err) state := &ConsumerState{} state.Delivered.Consumer = 22 state.Delivered.Stream = 22 state.AckFloor.Consumer = 11 state.AckFloor.Stream = 11 err = o.Update(state) require_NoError(t, err) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Now make sure we can read. var smv StoreMsg sm, err := fs.LoadMsg(10, &smv) require_NoError(t, err) require_True(t, string(sm.msg) == "aes ftw") o, err = fs.ConsumerStore("o22", &ConsumerConfig{}) require_NoError(t, err) rstate, err := o.State() require_NoError(t, err) if rstate.Delivered != state.Delivered || rstate.AckFloor != state.AckFloor { t.Fatalf("Bad recovered consumer state, expected %+v got %+v", state, rstate) } }) } // Make sure we do not go through block loads when we know no subjects will exists, e.g. raft. func TestFileStoreNoFSSWhenNoSubjects(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { created := time.Now() fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() n, msg := 100, []byte("raft state") for i := 0; i < n; i++ { _, _, err := fs.StoreMsg(_EMPTY_, nil, msg) require_NoError(t, err) } state := fs.State() require_True(t, state.Msgs == uint64(n)) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure we did not load the block trying to generate fss. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() mb.mu.Lock() defer mb.mu.Unlock() if mb.cloads > 0 { t.Fatalf("Expected no cache loads but got %d", mb.cloads) } if mb.fss != nil { t.Fatalf("Expected fss to be nil") } }) } func TestFileStoreNoFSSBugAfterRemoveFirst(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 8 * 1024 * 1024 fcfg.CacheExpire = 200 * time.Millisecond cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.bar.*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() n, msg := 100, bytes.Repeat([]byte("ZZZ"), 33) // ~100bytes for i := 0; i < n; i++ { subj := fmt.Sprintf("foo.bar.%d", i) _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } state := fs.State() require_True(t, state.Msgs == uint64(n)) // Let fss expire. time.Sleep(250 * time.Millisecond) _, err = fs.RemoveMsg(1) require_NoError(t, err) sm, _, err := fs.LoadNextMsg("foo.>", true, 1, nil) require_NoError(t, err) require_True(t, sm.subj == "foo.bar.1") // Make sure mb.fss does not have the entry for foo.bar.0 fs.mu.Lock() mb := fs.blks[0] fs.mu.Unlock() mb.mu.RLock() ss, ok := mb.fss.Find([]byte("foo.bar.0")) mb.mu.RUnlock() if ok && ss != nil { t.Fatalf("Expected no state for %q, but got %+v\n", "foo.bar.0", ss) } }) } // NOTE: We do not use fss files anymore, but leaving test in place. func TestFileStoreNoFSSAfterRecover(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() n, msg := 100, []byte("no fss for you!") for i := 0; i < n; i++ { _, _, err := fs.StoreMsg(_EMPTY_, nil, msg) require_NoError(t, err) } state := fs.State() require_True(t, state.Msgs == uint64(n)) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure we did not load the block trying to generate fss. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() mb.mu.Lock() defer mb.mu.Unlock() if mb.fss != nil { t.Fatalf("Expected no fss post recover") } }) } func TestFileStoreFSSCloseAndKeepOnExpireOnRecoverBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { ttl := 100 * time.Millisecond cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage, MaxAge: ttl} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) fs.Stop() time.Sleep(2 * ttl) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if state := fs.State(); state.NumSubjects != 0 { t.Fatalf("Expected no subjects with no messages, got %d", state.NumSubjects) } }) } func TestFileStoreExpireOnRecoverSubjectAccounting(t *testing.T) { const msgLen = 19 msg := bytes.Repeat([]byte("A"), msgLen) testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 100 ttl := 200 * time.Millisecond cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxAge: ttl} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // These are in first block. fs.StoreMsg("A", nil, msg) fs.StoreMsg("B", nil, msg) time.Sleep(ttl / 2) // This one in 2nd block. fs.StoreMsg("C", nil, msg) fs.Stop() time.Sleep(ttl/2 + 10*time.Millisecond) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure we take into account PSIM when throwing a whole block away. if state := fs.State(); state.NumSubjects != 1 { t.Fatalf("Expected 1 subject, got %d", state.NumSubjects) } }) } func TestFileStoreFSSExpireNumPendingBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cexp := 100 * time.Millisecond fcfg.CacheExpire = cexp cfg := StreamConfig{Name: "zzz", Subjects: []string{"KV.>"}, MaxMsgsPer: 1, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Let FSS meta expire. time.Sleep(2 * cexp) _, _, err = fs.StoreMsg("KV.X", nil, []byte("Y")) require_NoError(t, err) if fss := fs.FilteredState(1, "KV.X"); fss.Msgs != 1 { t.Fatalf("Expected only 1 msg, got %d", fss.Msgs) } }) } // https://github.com/nats-io/nats-server/issues/3484 func TestFileStoreFilteredFirstMatchingBug(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.>"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() _, _, err = fs.StoreMsg("foo.foo", nil, []byte("A")) require_NoError(t, err) _, _, err = fs.StoreMsg("foo.foo", nil, []byte("B")) require_NoError(t, err) _, _, err = fs.StoreMsg("foo.foo", nil, []byte("C")) require_NoError(t, err) fs.mu.RLock() mb := fs.lmb fs.mu.RUnlock() mb.mu.Lock() // Simulate swapping out the fss state and reading it back in with only one subject // present in the block. if mb.fss != nil { mb.fss = nil } // Now load info back in. mb.generatePerSubjectInfo() mb.mu.Unlock() // Now add in a different subject. _, _, err = fs.StoreMsg("foo.bar", nil, []byte("X")) require_NoError(t, err) // Now see if a filtered load would incorrectly succeed. sm, _, err := fs.LoadNextMsg("foo.foo", false, 4, nil) if err == nil || sm != nil { t.Fatalf("Loaded filtered message with wrong subject, wanted %q got %q", "foo.foo", sm.subj) } }) } func TestFileStoreOutOfSpaceRebuildState(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() _, _, err = fs.StoreMsg("foo", nil, []byte("A")) require_NoError(t, err) _, _, err = fs.StoreMsg("bar", nil, []byte("B")) require_NoError(t, err) // Grab state. state := fs.State() ss := fs.SubjectsState(">") // Set mock out of space error to trip. fs.mu.RLock() mb := fs.lmb fs.mu.RUnlock() mb.mu.Lock() mb.mockWriteErr = true mb.mu.Unlock() _, _, err = fs.StoreMsg("baz", nil, []byte("C")) require_Error(t, err, errors.New("mock write error")) nstate := fs.State() nss := fs.SubjectsState(">") if !reflect.DeepEqual(state, nstate) { t.Fatalf("State expected to be\n %+v\nvs\n %+v", state, nstate) } if !reflect.DeepEqual(ss, nss) { t.Fatalf("Subject state expected to be\n %+v\nvs\n %+v", ss, nss) } }) } func TestFileStoreRebuildStateProperlyWithMaxMsgsPerSubject(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 4096 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo", "bar", "baz"}, Storage: FileStorage, MaxMsgsPer: 1} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Send one to baz at beginning. _, _, err = fs.StoreMsg("baz", nil, nil) require_NoError(t, err) ns := 1000 for i := 1; i <= ns; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) _, _, err = fs.StoreMsg("bar", nil, nil) require_NoError(t, err) } var ss StreamState fs.FastState(&ss) if ss.NumSubjects != 3 { t.Fatalf("Expected NumSubjects of 3, got %d", ss.NumSubjects) } if ss.Msgs != 3 { t.Fatalf("Expected NumMsgs of 3, got %d", ss.Msgs) } }) } func TestFileStoreUpdateMaxMsgsPerSubject(t *testing.T) { cfg := StreamConfig{ Name: "TEST", Storage: FileStorage, Subjects: []string{"foo"}, MaxMsgsPer: 10, } testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure this is honored on an update. cfg.MaxMsgsPer = 50 err = fs.UpdateConfig(&cfg) require_NoError(t, err) numStored := 22 for i := 0; i < numStored; i++ { _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } ss := fs.SubjectsState("foo")["foo"] if ss.Msgs != uint64(numStored) { t.Fatalf("Expected to have %d stored, got %d", numStored, ss.Msgs) } // Now make sure we trunk if setting to lower value. cfg.MaxMsgsPer = 10 err = fs.UpdateConfig(&cfg) require_NoError(t, err) ss = fs.SubjectsState("foo")["foo"] if ss.Msgs != 10 { t.Fatalf("Expected to have %d stored, got %d", 10, ss.Msgs) } }) } func TestFileStoreBadFirstAndFailedExpireAfterRestart(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 256 ttl := time.Second cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage, MaxAge: ttl} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // With block size of 256 and subject and message below, seq 8 starts new block. // Will double check and fail test if not the case since test depends on this. subj, msg := "foo", []byte("ZZ") // These are all instant and will expire after 1 sec. start := time.Now() for i := 0; i < 7; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } // Put two more after a delay. time.Sleep(1500 * time.Millisecond) seq, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) _, _, err = fs.StoreMsg(subj, nil, msg) require_NoError(t, err) // Make sure that sequence 8 is first in second block, and break test if that is not true. fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.RLock() first := lmb.first.seq lmb.mu.RUnlock() require_True(t, first == 8) // Instantly remove first one from second block. // On restart this will trigger expire on recover which will set fs.FirstSeq to the deleted one. fs.RemoveMsg(seq) // Stop the filstore and wait til first block expires. fs.Stop() time.Sleep(ttl - time.Since(start) + (time.Second)) fs, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Check that state is correct for first message which should be 9 and have a proper timestamp. var state StreamState fs.FastState(&state) ts := state.FirstTime require_True(t, state.Msgs == 1) require_True(t, state.FirstSeq == 9) require_True(t, !state.FirstTime.IsZero()) // Wait and make sure expire timer is still working properly. time.Sleep(2 * ttl) fs.FastState(&state) require_Equal(t, state.Msgs, 0) require_Equal(t, state.FirstSeq, 10) require_Equal(t, state.LastSeq, 9) require_Equal(t, state.LastTime, ts) }) } func TestFileStoreCompactAllWithDanglingLMB(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("ZZ") for i := 0; i < 100; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } fs.RemoveMsg(100) purged, err := fs.Compact(100) require_NoError(t, err) require_True(t, purged == 99) _, _, err = fs.StoreMsg(subj, nil, msg) require_NoError(t, err) }) } func TestFileStoreStateWithBlkFirstDeleted(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 4096 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 500 for i := 0; i < toStore; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } // Delete some messages from the beginning of an interior block. fs.mu.RLock() require_True(t, len(fs.blks) > 2) fseq := fs.blks[1].first.seq fs.mu.RUnlock() // Now start from first seq of second blk and delete 10 msgs for seq := fseq; seq < fseq+10; seq++ { removed, err := fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, removed) } // This bug was in normal detailed state. But check fast state too. var fstate StreamState fs.FastState(&fstate) require_True(t, fstate.NumDeleted == 10) state := fs.State() require_True(t, state.NumDeleted == 10) }) } func TestFileStoreMsgBlkFailOnKernelFaultLostDataReporting(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 4096 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 500 for i := 0; i < toStore; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } // We want to make sure all of the scenarios report lost data properly. // Will run 3 scenarios, 1st block, last block, interior block. // The new system does not detect byzantine behavior by default on creating the store. // A LoadMsg() of checkMsgs() call will be needed now. // First block fs.mu.RLock() require_True(t, len(fs.blks) > 0) mfn := fs.blks[0].mfn fs.mu.RUnlock() fs.Stop() require_NoError(t, os.Remove(mfn)) // Restart. fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() _, err = fs.LoadMsg(1, nil) require_Error(t, err, errNoBlkData) // Load will rebuild fs itself async.. checkFor(t, time.Second, 50*time.Millisecond, func() error { if state := fs.State(); state.Lost != nil { return nil } return errors.New("no ld yet") }) state := fs.State() require_True(t, state.FirstSeq == 94) require_True(t, state.Lost != nil) require_True(t, len(state.Lost.Msgs) == 93) // Last block fs.mu.RLock() require_True(t, len(fs.blks) > 0) require_True(t, fs.lmb != nil) mfn = fs.lmb.mfn fs.mu.RUnlock() fs.Stop() require_NoError(t, os.Remove(mfn)) // Restart. fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() require_True(t, state.FirstSeq == 94) require_True(t, state.LastSeq == 500) // Make sure we do not lose last seq. require_True(t, state.NumDeleted == 35) // These are interiors require_True(t, state.Lost != nil) require_True(t, len(state.Lost.Msgs) == 35) // Interior block. fs.mu.RLock() require_True(t, len(fs.blks) > 3) mfn = fs.blks[len(fs.blks)-3].mfn fs.mu.RUnlock() fs.Stop() require_NoError(t, os.Remove(mfn)) // Restart. fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Need checkMsgs to catch interior one. require_True(t, fs.checkMsgs() != nil) state = fs.State() require_True(t, state.FirstSeq == 94) require_True(t, state.LastSeq == 500) // Make sure we do not lose last seq. require_True(t, state.NumDeleted == 128) require_True(t, state.Lost != nil) require_True(t, len(state.Lost.Msgs) == 93) }) } func TestFileStoreAllFilteredStateWithDeleted(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1024 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 100; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } remove := func(seqs ...uint64) { for _, seq := range seqs { ok, err := fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, ok) } } checkFilteredState := func(start, msgs, first, last int) { fss := fs.FilteredState(uint64(start), _EMPTY_) if fss.Msgs != uint64(msgs) { t.Fatalf("Expected %d msgs, got %d", msgs, fss.Msgs) } if fss.First != uint64(first) { t.Fatalf("Expected %d to be first, got %d", first, fss.First) } if fss.Last != uint64(last) { t.Fatalf("Expected %d to be last, got %d", last, fss.Last) } } checkFilteredState(1, 100, 1, 100) remove(2) checkFilteredState(2, 98, 3, 100) remove(3, 4, 5) checkFilteredState(2, 95, 6, 100) checkFilteredState(6, 95, 6, 100) remove(8, 10, 12, 14, 16, 18) checkFilteredState(7, 88, 7, 100) // Now check when purged that we return first and last sequences properly. fs.Purge() checkFilteredState(0, 0, 101, 100) }) } func TestFileStoreStreamTruncateResetMultiBlock(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 128 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 1000; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } fs.syncBlocks() require_True(t, fs.numMsgBlocks() == 500) // Reset everything require_NoError(t, fs.Truncate(0)) require_True(t, fs.numMsgBlocks() == 0) state := fs.State() require_Equal(t, state.Msgs, 0) require_Equal(t, state.Bytes, 0) require_Equal(t, state.FirstSeq, 0) require_Equal(t, state.LastSeq, 0) require_Equal(t, state.NumSubjects, 0) require_Equal(t, state.NumDeleted, 0) for i := 0; i < 1000; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } fs.syncBlocks() state = fs.State() require_Equal(t, state.Msgs, 1000) require_Equal(t, state.Bytes, 44000) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 1000) require_Equal(t, state.NumSubjects, 1) require_Equal(t, state.NumDeleted, 0) }) } func TestFileStoreStreamCompactMultiBlockSubjectInfo(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 128 cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) _, _, err := fs.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } require_True(t, fs.numMsgBlocks() == 500) // Compact such that we know we throw blocks away from the beginning. deleted, err := fs.Compact(501) require_NoError(t, err) require_True(t, deleted == 500) require_True(t, fs.numMsgBlocks() == 250) // Make sure we adjusted for subjects etc. state := fs.State() require_True(t, state.NumSubjects == 500) }) } func TestFileStoreSubjectsTotals(t *testing.T) { // No need for all permutations here. storeDir := t.TempDir() fcfg := FileStoreConfig{StoreDir: storeDir} fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fmap := make(map[int]int) bmap := make(map[int]int) var m map[int]int var ft string for i := 0; i < 10_000; i++ { // Flip coin for prefix if rand.Intn(2) == 0 { ft, m = "foo", fmap } else { ft, m = "bar", bmap } dt := rand.Intn(100) subj := fmt.Sprintf("%s.%d", ft, dt) m[dt]++ _, _, err := fs.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } // Now test SubjectsTotal for dt, total := range fmap { subj := fmt.Sprintf("foo.%d", dt) m := fs.SubjectsTotals(subj) if m[subj] != uint64(total) { t.Fatalf("Expected %q to have %d total, got %d", subj, total, m[subj]) } } // Check fmap. if st := fs.SubjectsTotals("foo.*"); len(st) != len(fmap) { t.Fatalf("Expected %d subjects for %q, got %d", len(fmap), "foo.*", len(st)) } else { expected := 0 for _, n := range fmap { expected += n } received := uint64(0) for _, n := range st { received += n } if received != uint64(expected) { t.Fatalf("Expected %d total but got %d", expected, received) } } // Check bmap. if st := fs.SubjectsTotals("bar.*"); len(st) != len(bmap) { t.Fatalf("Expected %d subjects for %q, got %d", len(bmap), "bar.*", len(st)) } else { expected := 0 for _, n := range bmap { expected += n } received := uint64(0) for _, n := range st { received += n } if received != uint64(expected) { t.Fatalf("Expected %d total but got %d", expected, received) } } // All with pwc match. if st, expected := fs.SubjectsTotals("*.*"), len(bmap)+len(fmap); len(st) != expected { t.Fatalf("Expected %d subjects for %q, got %d", expected, "*.*", len(st)) } } func TestFileStoreConsumerStoreEncodeAfterRestart(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() o, err := fs.ConsumerStore("o22", &ConsumerConfig{AckPolicy: AckExplicit}) require_NoError(t, err) state := &ConsumerState{} state.Delivered.Consumer = 22 state.Delivered.Stream = 22 state.AckFloor.Consumer = 11 state.AckFloor.Stream = 11 err = o.Update(state) require_NoError(t, err) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: "zzz", Storage: FileStorage}, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() o, err = fs.ConsumerStore("o22", &ConsumerConfig{AckPolicy: AckExplicit}) require_NoError(t, err) if o.(*consumerFileStore).state.Delivered != state.Delivered { t.Fatalf("Consumer state is wrong %+v vs %+v", o.(*consumerFileStore).state, state) } if o.(*consumerFileStore).state.AckFloor != state.AckFloor { t.Fatalf("Consumer state is wrong %+v vs %+v", o.(*consumerFileStore).state, state) } }) } func TestFileStoreNumPendingLargeNumBlks(t *testing.T) { // No need for all permutations here. storeDir := t.TempDir() fcfg := FileStoreConfig{ StoreDir: storeDir, BlockSize: 128, // Small on purpose to create alot of blks. } fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"zzz"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "zzz", bytes.Repeat([]byte("X"), 100) numMsgs := 10_000 for i := 0; i < numMsgs; i++ { fs.StoreMsg(subj, nil, msg) } start := time.Now() total, _ := fs.NumPending(4000, "zzz", false) require_LessThan(t, time.Since(start), 15*time.Millisecond) require_Equal(t, total, 6001) start = time.Now() total, _ = fs.NumPending(6000, "zzz", false) require_LessThan(t, time.Since(start), 25*time.Millisecond) require_Equal(t, total, 4001) // Now delete a message in first half and second half. fs.RemoveMsg(1000) fs.RemoveMsg(9000) start = time.Now() total, _ = fs.NumPending(4000, "zzz", false) require_LessThan(t, time.Since(start), 50*time.Millisecond) require_Equal(t, total, 6000) start = time.Now() total, _ = fs.NumPending(6000, "zzz", false) require_LessThan(t, time.Since(start), 50*time.Millisecond) require_Equal(t, total, 4000) } func TestFileStoreSkipMsgAndNumBlocks(t *testing.T) { // No need for all permutations here. storeDir := t.TempDir() fcfg := FileStoreConfig{ StoreDir: storeDir, BlockSize: 128, // Small on purpose to create alot of blks. } fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"zzz"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "zzz", bytes.Repeat([]byte("X"), 100) numMsgs := 10_000 fs.StoreMsg(subj, nil, msg) for i := 0; i < numMsgs; i++ { fs.SkipMsg() } fs.StoreMsg(subj, nil, msg) require_True(t, fs.numMsgBlocks() == 2) } func TestFileStoreRestoreEncryptedWithNoKeyFuncFails(t *testing.T) { // No need for all permutations here. fcfg := FileStoreConfig{StoreDir: t.TempDir(), Cipher: AES} cfg := StreamConfig{Name: "zzz", Subjects: []string{"zzz"}, Storage: FileStorage} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() subj, msg := "zzz", bytes.Repeat([]byte("X"), 100) numMsgs := 100 for i := 0; i < numMsgs; i++ { fs.StoreMsg(subj, nil, msg) } fs.Stop() // Make sure if we try to restore with no prf (key) that it fails. _, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), nil, nil) require_Error(t, err, errNoMainKey) } func TestFileStoreInitialFirstSeq(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Storage: FileStorage, FirstSeq: 1000} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() seq, _, err := fs.StoreMsg("A", nil, []byte("OK")) require_NoError(t, err) if seq != 1000 { t.Fatalf("Message should have been sequence 1000 but was %d", seq) } seq, _, err = fs.StoreMsg("B", nil, []byte("OK")) require_NoError(t, err) if seq != 1001 { t.Fatalf("Message should have been sequence 1001 but was %d", seq) } var state StreamState fs.FastState(&state) switch { case state.Msgs != 2: t.Fatalf("Expected 2 messages, got %d", state.Msgs) case state.FirstSeq != 1000: t.Fatalf("Expected first seq 1000, got %d", state.FirstSeq) case state.LastSeq != 1001: t.Fatalf("Expected last seq 1001, got %d", state.LastSeq) } }) } func TestFileStoreRecaluclateFirstForSubjBug(t *testing.T) { fs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, nil) // 1 fs.StoreMsg("bar", nil, nil) // 2 fs.StoreMsg("foo", nil, nil) // 3 // Now remove first 2.. fs.RemoveMsg(1) fs.RemoveMsg(2) // Now grab first (and only) block. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() // Since we lazy update the first, simulate that we have not updated it as of yet. ss := &SimpleState{Msgs: 1, First: 1, Last: 3, firstNeedsUpdate: true} mb.mu.Lock() defer mb.mu.Unlock() // Flush the cache. mb.clearCacheAndOffset() // Now call with start sequence of 1, the old one // This will panic without the fix. mb.recalculateForSubj("foo", ss) // Make sure it was update properly. require_True(t, *ss == SimpleState{Msgs: 1, First: 3, Last: 3, firstNeedsUpdate: false}) } func TestFileStoreKeepWithDeletedMsgsBug(t *testing.T) { fs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("A"), 19) for i := 0; i < 5; i++ { fs.StoreMsg("A", nil, msg) fs.StoreMsg("B", nil, msg) } n, err := fs.PurgeEx("A", 0, 0) require_NoError(t, err) require_True(t, n == 5) // Purge with keep. n, err = fs.PurgeEx(_EMPTY_, 0, 2) require_NoError(t, err) require_True(t, n == 3) } func TestFileStoreRestartWithExpireAndLockingBug(t *testing.T) { sd := t.TempDir() scfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} fs, err := newFileStore(FileStoreConfig{StoreDir: sd}, scfg) require_NoError(t, err) defer fs.Stop() // 20 total msg := []byte("HELLO WORLD") for i := 0; i < 10; i++ { fs.StoreMsg("A", nil, msg) fs.StoreMsg("B", nil, msg) } fs.Stop() // Now change config underneath of so we will do expires at startup. scfg.MaxMsgs = 15 scfg.MaxMsgsPer = 2 newCfg := FileStreamInfo{Created: fs.cfg.Created, StreamConfig: scfg} // Replace fs.cfg = newCfg require_NoError(t, fs.writeStreamMeta()) fs, err = newFileStore(FileStoreConfig{StoreDir: sd}, scfg) require_NoError(t, err) defer fs.Stop() } // Test that loads from lmb under lots of writes do not return errPartialCache. func TestFileStoreErrPartialLoad(t *testing.T) { fs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() put := func(num int) { for i := 0; i < num; i++ { fs.StoreMsg("Z", nil, []byte("ZZZZZZZZZZZZZ")) } } put(100) // Dump cache of lmb. clearCache := func() { fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.Lock() lmb.clearCache() lmb.mu.Unlock() } clearCache() qch := make(chan struct{}) defer close(qch) for i := 0; i < 10; i++ { go func() { for { select { case <-qch: return default: put(5) } } }() } time.Sleep(100 * time.Millisecond) var smv StoreMsg for i := 0; i < 10_000; i++ { fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.Lock() first, last := fs.lmb.first.seq, fs.lmb.last.seq if i%100 == 0 { lmb.clearCache() } lmb.mu.Unlock() if spread := int(last - first); spread > 0 { seq := first + uint64(rand.Intn(spread)) _, err = fs.LoadMsg(seq, &smv) require_NoError(t, err) } } } func TestFileStoreErrPartialLoadOnSyncClose(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 500}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}, ) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 10 msgs per blk. msgLen := 19 msg := bytes.Repeat([]byte("A"), msgLen) // Load up half the block. for _, subj := range []string{"A", "B", "C", "D", "E"} { fs.StoreMsg(subj, nil, msg) } // Now simulate the sync timer closing the last block. fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() require_True(t, lmb != nil) lmb.mu.Lock() lmb.expireCacheLocked() lmb.dirtyCloseWithRemove(false) lmb.mu.Unlock() fs.StoreMsg("Z", nil, msg) _, err = fs.LoadMsg(1, nil) require_NoError(t, err) } func TestFileStoreSyncIntervals(t *testing.T) { fcfg := FileStoreConfig{StoreDir: t.TempDir(), SyncInterval: 250 * time.Millisecond} fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() checkSyncFlag := func(expected bool) { fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.RLock() syncNeeded := lmb.needSync lmb.mu.RUnlock() if syncNeeded != expected { t.Fatalf("Expected needSync to be %v", expected) } } checkSyncFlag(false) fs.StoreMsg("Z", nil, []byte("hello")) checkSyncFlag(true) time.Sleep(400 * time.Millisecond) checkSyncFlag(false) fs.Stop() // Now check always fcfg.SyncInterval = 10 * time.Second fcfg.SyncAlways = true fs, err = newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() checkSyncFlag(false) fs.StoreMsg("Z", nil, []byte("hello")) checkSyncFlag(false) } // https://github.com/nats-io/nats-server/issues/4529 // Run this wuth --race and you will see the unlocked access that probably caused this. func TestFileStoreRecalcFirstSequenceBug(t *testing.T) { fcfg := FileStoreConfig{StoreDir: t.TempDir()} fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"*"}, MaxMsgsPer: 2, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("A"), 22) for _, subj := range []string{"A", "A", "B", "B"} { fs.StoreMsg(subj, nil, msg) } // Make sure the buffer is cleared. clearLMBCache := func() { fs.mu.RLock() mb := fs.lmb fs.mu.RUnlock() mb.mu.Lock() mb.clearCacheAndOffset() mb.mu.Unlock() } clearLMBCache() // Do first here. fs.StoreMsg("A", nil, msg) var wg sync.WaitGroup start := make(chan bool) wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 1_000; i++ { fs.LoadLastMsg("A", nil) clearLMBCache() } }() wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 1_000; i++ { fs.StoreMsg("A", nil, msg) } }() close(start) wg.Wait() } /////////////////////////////////////////////////////////////////////////// // New WAL based architecture tests /////////////////////////////////////////////////////////////////////////// func TestFileStoreFullStateBasics(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 100 cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. subj, msgLen, recLen := "A", 19, uint64(50) msgA := bytes.Repeat([]byte("A"), msgLen) msgZ := bytes.Repeat([]byte("Z"), msgLen) // Send 2 msgs and stop, check for presence of our full state file. fs.StoreMsg(subj, nil, msgA) fs.StoreMsg(subj, nil, msgZ) require_True(t, fs.numMsgBlocks() == 1) // Make sure there is a full state file after we do a stop. fs.Stop() sfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile) if _, err := os.Stat(sfile); err != nil { t.Fatalf("Expected stream state file but got %v", err) } // Read it in and make sure len > 0. buf, err := os.ReadFile(sfile) require_NoError(t, err) require_True(t, len(buf) > 0) // Now make sure we recover properly. fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Make sure there are no old idx or fss files. matches, err := filepath.Glob(filepath.Join(fcfg.StoreDir, msgDir, "%d.fss")) require_NoError(t, err) require_True(t, len(matches) == 0) matches, err = filepath.Glob(filepath.Join(fcfg.StoreDir, msgDir, "%d.idx")) require_NoError(t, err) require_True(t, len(matches) == 0) state := fs.State() require_Equal(t, state.Msgs, 2) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 2) // Now make sure we can read in values. var smv StoreMsg sm, err := fs.LoadMsg(1, &smv) require_NoError(t, err) require_True(t, bytes.Equal(sm.msg, msgA)) sm, err = fs.LoadMsg(2, &smv) require_NoError(t, err) require_True(t, bytes.Equal(sm.msg, msgZ)) // Now add in 1 more here to split the lmb. fs.StoreMsg(subj, nil, msgZ) // Now stop the filestore and replace the old stream state and make sure we recover correctly. fs.Stop() // Regrab the stream state buf, err = os.ReadFile(sfile) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Add in one more. fs.StoreMsg(subj, nil, msgZ) fs.Stop() // Put old stream state back with only 3. err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() require_Equal(t, state.Msgs, 4) require_Equal(t, state.Bytes, 4*recLen) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 4) require_Equal(t, fs.numMsgBlocks(), 2) // Make sure we are tracking subjects correctly. fs.mu.RLock() info, _ := fs.psim.Find(stringToBytes(subj)) psi := *info fs.mu.RUnlock() require_Equal(t, psi.total, 4) require_Equal(t, psi.fblk, 1) require_Equal(t, psi.lblk, 2) // Store 1 more fs.StoreMsg(subj, nil, msgA) fs.Stop() // Put old stream state back with only 3. err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() state = fs.State() require_Equal(t, state.Msgs, 5) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 5) require_Equal(t, fs.numMsgBlocks(), 3) // Make sure we are tracking subjects correctly. fs.mu.RLock() info, _ = fs.psim.Find(stringToBytes(subj)) psi = *info fs.mu.RUnlock() require_Equal(t, psi.total, 5) require_Equal(t, psi.fblk, 1) require_Equal(t, psi.lblk, 3) }) } func TestFileStoreFullStatePurge(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 132 // Leave room for tombstones. cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. subj, msg := "A", bytes.Repeat([]byte("A"), 19) // Should be 2 per block, so 5 blocks. for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } n, err := fs.Purge() require_NoError(t, err) require_Equal(t, n, 10) state := fs.State() fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } // Add in more 10 more total, some B some C. for i := 0; i < 5; i++ { fs.StoreMsg("B", nil, msg) fs.StoreMsg("C", nil, msg) } n, err = fs.PurgeEx("B", 0, 0) require_NoError(t, err) require_Equal(t, n, 5) state = fs.State() fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } // Purge with keep. n, err = fs.PurgeEx(_EMPTY_, 0, 2) require_NoError(t, err) require_Equal(t, n, 3) state = fs.State() // Do some quick checks here, keep had a bug. require_Equal(t, state.Msgs, 2) require_Equal(t, state.FirstSeq, 18) require_Equal(t, state.LastSeq, 20) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } // Make sure we can survive a purge with no full stream state and have the correct first sequence. // This used to be provided by the idx file and is now tombstones and the full stream state snapshot. n, err = fs.Purge() require_NoError(t, err) require_Equal(t, n, 2) state = fs.State() fs.Stop() sfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile) os.Remove(sfile) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } }) } func TestFileStoreFullStateTestUserRemoveWAL(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 132 // Leave room for tombstones. cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. msgLen := 19 msgA := bytes.Repeat([]byte("A"), msgLen) msgZ := bytes.Repeat([]byte("Z"), msgLen) // Store 2 msgs and delete first. fs.StoreMsg("A", nil, msgA) fs.StoreMsg("Z", nil, msgZ) fs.RemoveMsg(1) // Check we can load things properly since the block will have a tombstone now for seq 1. sm, err := fs.LoadMsg(2, nil) require_NoError(t, err) require_True(t, bytes.Equal(sm.msg, msgZ)) require_Equal(t, fs.numMsgBlocks(), 1) state := fs.State() fs.Stop() // Grab the state from this stop. sfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile) buf, err := os.ReadFile(sfile) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Check we can load things properly since the block will have a tombstone now for seq 1. _, err = fs.LoadMsg(2, nil) require_NoError(t, err) _, err = fs.LoadMsg(1, nil) require_Error(t, err, ErrStoreMsgNotFound) if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state does not match:\n%+v\n%+v", state, newState) } require_True(t, !state.FirstTime.IsZero()) // Store 2 more msgs and delete 2 & 4. fs.StoreMsg("A", nil, msgA) fs.StoreMsg("Z", nil, msgZ) fs.RemoveMsg(2) fs.RemoveMsg(4) state = fs.State() require_Equal(t, len(state.Deleted), state.NumDeleted) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state does not match:\n%+v\n%+v", state, newState) } require_True(t, !state.FirstTime.IsZero()) // Now close again and put back old stream state. // This will test that we can remember user deletes by placing tombstones in the lmb/wal. fs.Stop() err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() newState := fs.State() // We will properly detect lost data for sequence #2 here. require_True(t, newState.Lost != nil) require_Equal(t, len(newState.Lost.Msgs), 1) require_Equal(t, newState.Lost.Msgs[0], 2) // Clear for deep equal compare below. newState.Lost = nil if !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state does not match:\n%+v\n%+v", state, newState) } require_True(t, !state.FirstTime.IsZero()) }) } func TestFileStoreFullStateTestSysRemovals(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 100 cfg := StreamConfig{ Name: "zzz", Subjects: []string{"*"}, MaxMsgs: 10, MaxMsgsPer: 1, Storage: FileStorage, } created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. msgLen := 19 msg := bytes.Repeat([]byte("A"), msgLen) for _, subj := range []string{"A", "B", "A", "B"} { fs.StoreMsg(subj, nil, msg) } state := fs.State() require_Equal(t, state.Msgs, 2) require_Equal(t, state.FirstSeq, 3) require_Equal(t, state.LastSeq, 4) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } for _, subj := range []string{"C", "D", "E", "F", "G", "H", "I", "J"} { fs.StoreMsg(subj, nil, msg) } state = fs.State() require_Equal(t, state.Msgs, 10) require_Equal(t, state.FirstSeq, 3) require_Equal(t, state.LastSeq, 12) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } // Goes over limit fs.StoreMsg("ZZZ", nil, msg) state = fs.State() require_Equal(t, state.Msgs, 10) require_Equal(t, state.FirstSeq, 4) require_Equal(t, state.LastSeq, 13) fs.Stop() fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state after purge does not match:\n%+v\n%+v", state, newState) } }) } func TestFileStoreSelectBlockWithFirstSeqRemovals(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 100 cfg := StreamConfig{ Name: "zzz", Subjects: []string{"*"}, MaxMsgsPer: 1, Storage: FileStorage, } fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. msgLen := 19 msg := bytes.Repeat([]byte("A"), msgLen) subjects := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+@$^" // We need over 32 blocks to kick in binary search. So 32*2+1 (65) msgs to get 33 blocks. for i := 0; i < 32*2+1; i++ { subj := string(subjects[i]) fs.StoreMsg(subj, nil, msg) } require_Equal(t, fs.numMsgBlocks(), 33) // Now we want to delete the first msg of each block to move the first sequence. // Want to do this via system removes, not user initiated moves. for i := 0; i < len(subjects); i += 2 { subj := string(subjects[i]) fs.StoreMsg(subj, nil, msg) } var ss StreamState fs.FastState(&ss) // We want to make sure that select always returns an index and a non-nil mb. for seq := ss.FirstSeq; seq <= ss.LastSeq; seq++ { fs.mu.RLock() index, mb := fs.selectMsgBlockWithIndex(seq) fs.mu.RUnlock() require_True(t, index >= 0) require_True(t, mb != nil) require_Equal(t, (seq-1)/2, uint64(index)) } }) } func TestFileStoreMsgBlockHolesAndIndexing(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() // Grab the message block by hand and manipulate at that level. mb := fs.getFirstBlock() writeMsg := func(subj string, seq uint64) { rl := fileStoreMsgSize(subj, nil, []byte(subj)) require_NoError(t, mb.writeMsgRecord(rl, seq, subj, nil, []byte(subj), time.Now().UnixNano(), true)) fs.rebuildState(nil) } readMsg := func(seq uint64, expectedSubj string) { // Clear cache so we load back in from disk and need to properly process any holes. ld, tombs, err := mb.rebuildState() require_NoError(t, err) require_Equal(t, ld, nil) require_Equal(t, len(tombs), 0) fs.rebuildState(nil) sm, _, err := mb.fetchMsg(seq, nil) require_NoError(t, err) require_Equal(t, sm.subj, expectedSubj) require_True(t, bytes.Equal(sm.buf[:len(expectedSubj)], []byte(expectedSubj))) } writeMsg("A", 2) require_Equal(t, mb.first.seq, 2) require_Equal(t, mb.last.seq, 2) writeMsg("B", 4) require_Equal(t, mb.first.seq, 2) require_Equal(t, mb.last.seq, 4) writeMsg("C", 12) readMsg(4, "B") require_True(t, mb.dmap.Exists(3)) readMsg(12, "C") readMsg(2, "A") // Check that we get deleted for the right ones etc. checkDeleted := func(seq uint64) { _, _, err = mb.fetchMsg(seq, nil) require_Error(t, err, ErrStoreMsgNotFound, errDeletedMsg) mb.mu.RLock() shouldExist, exists := seq >= mb.first.seq, mb.dmap.Exists(seq) mb.mu.RUnlock() if shouldExist { require_True(t, exists) } } checkDeleted(1) checkDeleted(3) for seq := 5; seq < 12; seq++ { checkDeleted(uint64(seq)) } } func TestFileStoreMsgBlockCompactionAndHoles(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("Z"), 1024) for _, subj := range []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"} { fs.StoreMsg(subj, nil, msg) } // Leave first one but delete the rest. for seq := uint64(2); seq < 10; seq++ { fs.RemoveMsg(seq) } require_Equal(t, fs.numMsgBlocks(), 1) mb := fs.getFirstBlock() require_NotNil(t, mb) _, ub, _ := fs.Utilization() // Do compaction, should remove all excess now. mb.mu.Lock() mb.compact() mb.mu.Unlock() ta, ua, _ := fs.Utilization() require_Equal(t, ub, ua) require_Equal(t, ta, ua) } func TestFileStoreRemoveLastNoDoubleTombstones(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("A", nil, []byte("hello")) fs.mu.Lock() fs.removeMsgViaLimits(1) fs.mu.Unlock() require_Equal(t, fs.numMsgBlocks(), 1) mb := fs.getFirstBlock() require_NotNil(t, mb) mb.loadMsgs() rbytes, _, err := fs.Utilization() require_NoError(t, err) require_Equal(t, rbytes, emptyRecordLen) } func TestFileStoreFullStateMultiBlockPastWAL(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 100 cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. msgLen := 19 msgA := bytes.Repeat([]byte("A"), msgLen) msgZ := bytes.Repeat([]byte("Z"), msgLen) // Store 2 msgs fs.StoreMsg("A", nil, msgA) fs.StoreMsg("B", nil, msgZ) require_Equal(t, fs.numMsgBlocks(), 1) fs.Stop() // Grab the state from this stop. sfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile) buf, err := os.ReadFile(sfile) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Store 6 more msgs. fs.StoreMsg("C", nil, msgA) fs.StoreMsg("D", nil, msgZ) fs.StoreMsg("E", nil, msgA) fs.StoreMsg("F", nil, msgZ) fs.StoreMsg("G", nil, msgA) fs.StoreMsg("H", nil, msgZ) require_Equal(t, fs.numMsgBlocks(), 4) state := fs.State() fs.Stop() // Put back old stream state. // This will test that we properly walk multiple blocks past where we snapshotted state. err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state does not match:\n%+v\n%+v", state, newState) } require_True(t, !state.FirstTime.IsZero()) }) } // This tests we can successfully recover without having to rebuild the whole stream from a mid block index.db marker // when the updated block has a removed entry. // Make sure this does not cause a recover of the full state. func TestFileStoreFullStateMidBlockPastWAL(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxMsgsPer: 1} created := time.Now() fs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk. msg := bytes.Repeat([]byte("Z"), 19) // Store 5 msgs fs.StoreMsg("A", nil, msg) fs.StoreMsg("B", nil, msg) fs.StoreMsg("C", nil, msg) fs.StoreMsg("D", nil, msg) fs.StoreMsg("E", nil, msg) require_Equal(t, fs.numMsgBlocks(), 1) fs.Stop() // Grab the state from this stop. sfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile) buf, err := os.ReadFile(sfile) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // Store 5 more messages, then remove seq 2, "B". fs.StoreMsg("F", nil, msg) fs.StoreMsg("G", nil, msg) fs.StoreMsg("H", nil, msg) fs.StoreMsg("I", nil, msg) fs.StoreMsg("J", nil, msg) fs.RemoveMsg(2) require_Equal(t, fs.numMsgBlocks(), 1) state := fs.State() fs.Stop() // Put back old stream state. // This will test that we properly walk multiple blocks past where we snapshotted state. err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) fs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() if newState := fs.State(); !reflect.DeepEqual(state, newState) { t.Fatalf("Restore state does not match:\n%+v\n%+v", state, newState) } // Check that index.db is still there. If we recover by raw data on a corrupt state we delete this. _, err = os.Stat(sfile) require_NoError(t, err) }) } func TestFileStoreCompactingBlocksOnSync(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 1000 // 20 msgs per block. fcfg.SyncInterval = 100 * time.Millisecond cfg := StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage, MaxMsgsPer: 1} fs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 20 msgs per blk. msg := bytes.Repeat([]byte("Z"), 19) subjects := "ABCDEFGHIJKLMNOPQRST" for _, subj := range subjects { fs.StoreMsg(string(subj), nil, msg) } require_Equal(t, fs.numMsgBlocks(), 1) total, reported, err := fs.Utilization() require_NoError(t, err) require_Equal(t, total, reported) // Now start removing, since we are small this should not kick in any inline logic. // Remove all interior messages, leave 1 and 20. So write B-S for i := 1; i < 19; i++ { fs.StoreMsg(string(subjects[i]), nil, msg) } require_Equal(t, fs.numMsgBlocks(), 2) blkUtil := func() (uint64, uint64) { fs.mu.RLock() fmb := fs.blks[0] fs.mu.RUnlock() fmb.mu.RLock() defer fmb.mu.RUnlock() return fmb.rbytes, fmb.bytes } total, reported = blkUtil() require_Equal(t, reported, 100) // Raw bytes will be 1000, but due to compression could be less. if fcfg.Compression != NoCompression { require_True(t, total > reported) } else { require_Equal(t, total, 1000) } // Make sure the sync interval when kicked in compacts down to rbytes == 100. checkFor(t, time.Second, 100*time.Millisecond, func() error { if total, reported := blkUtil(); total <= reported { return nil } return fmt.Errorf("Not compacted yet, raw %v vs reported %v", friendlyBytes(total), friendlyBytes(reported)) }) }) } // Make sure a call to Compact() updates PSIM correctly. func TestFileStoreCompactAndPSIMWhenDeletingBlocks(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 512}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "A", bytes.Repeat([]byte("ABC"), 33) // ~100bytes // Add in 10 As for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } require_Equal(t, fs.numMsgBlocks(), 4) // Should leave 1. n, err := fs.Compact(10) require_NoError(t, err) require_Equal(t, n, 9) require_Equal(t, fs.numMsgBlocks(), 1) fs.mu.RLock() info, _ := fs.psim.Find(stringToBytes(subj)) psi := *info fs.mu.RUnlock() require_Equal(t, psi.total, 1) require_Equal(t, psi.fblk, psi.lblk) } func TestFileStoreTrackSubjLenForPSIM(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Place 1000 msgs with varying subjects. // Make sure we track the subject length properly. smap := make(map[string]int, 1000) buf := make([]byte, 10) for i := 0; i < 1000; i++ { var b strings.Builder // 1-6 tokens. numTokens := rand.Intn(6) + 1 for i := 0; i < numTokens; i++ { tlen := rand.Intn(4) + 2 tok := buf[:tlen] crand.Read(tok) b.WriteString(hex.EncodeToString(tok)) if i != numTokens-1 { b.WriteString(".") } } subj := b.String() // Avoid dupes since will cause check to fail after we delete messages. if _, ok := smap[subj]; ok { continue } smap[subj] = len(subj) fs.StoreMsg(subj, nil, nil) } check := func() { t.Helper() var total int for _, slen := range smap { total += slen } fs.mu.RLock() tsl := fs.tsl fs.mu.RUnlock() require_Equal(t, tsl, total) } check() // Delete ~half var smv StoreMsg for i := 0; i < 500; i++ { seq := uint64(rand.Intn(1000) + 1) sm, err := fs.LoadMsg(seq, &smv) if err != nil { continue } fs.RemoveMsg(seq) delete(smap, sm.subj) } check() // Make sure we can recover same after restart. fs.Stop() fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() check() // Drain the rest through purge. fs.Purge() smap = nil check() } // This was used to make sure our estimate was correct, but not needed normally. func TestFileStoreLargeFullStatePSIM(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() buf := make([]byte, 20) for i := 0; i < 100_000; i++ { var b strings.Builder // 1-6 tokens. numTokens := rand.Intn(6) + 1 for i := 0; i < numTokens; i++ { tlen := rand.Intn(8) + 2 tok := buf[:tlen] crand.Read(tok) b.WriteString(hex.EncodeToString(tok)) if i != numTokens-1 { b.WriteString(".") } } subj := b.String() fs.StoreMsg(subj, nil, nil) } fs.Stop() } func TestFileStoreLargeFullStateMetaCleanup(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "foo.bar.baz", bytes.Repeat([]byte("ABC"), 33) // ~100bytes for i := 0; i < 1000; i++ { fs.StoreMsg(subj, nil, nil) } fs.Stop() mdir := filepath.Join(sd, msgDir) idxFile := filepath.Join(mdir, "1.idx") fssFile := filepath.Join(mdir, "1.fss") require_NoError(t, os.WriteFile(idxFile, msg, defaultFilePerms)) require_NoError(t, os.WriteFile(fssFile, msg, defaultFilePerms)) fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() checkFor(t, time.Second, 50*time.Millisecond, func() error { if _, err := os.Stat(idxFile); err == nil { return errors.New("idx file still exists") } if _, err := os.Stat(fssFile); err == nil { return errors.New("fss file still exists") } return nil }) } func TestFileStoreIndexDBExistsAfterShutdown(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj := "foo.bar.baz" for i := 0; i < 1000; i++ { fs.StoreMsg(subj, nil, nil) } idxFile := filepath.Join(sd, msgDir, streamStreamStateFile) fs.mu.Lock() fs.dirty = 1 if err := os.Remove(idxFile); err != nil && !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } fs.mu.Unlock() fs.Stop() checkFor(t, time.Second, 50*time.Millisecond, func() error { if _, err := os.Stat(idxFile); err != nil { return fmt.Errorf("%q doesn't exist", idxFile) } return nil }) } // https://github.com/nats-io/nats-server/issues/4842 func TestFileStoreSubjectCorruption(t *testing.T) { sd, blkSize := t.TempDir(), uint64(2*1024*1024) fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: blkSize}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() numSubjects := 100 msgs := [][]byte{bytes.Repeat([]byte("ABC"), 333), bytes.Repeat([]byte("ABC"), 888), bytes.Repeat([]byte("ABC"), 555)} for i := 0; i < 10_000; i++ { subj := fmt.Sprintf("foo.%d", rand.Intn(numSubjects)+1) msg := msgs[rand.Intn(len(msgs))] fs.StoreMsg(subj, nil, msg) } fs.Stop() require_NoError(t, os.Remove(filepath.Join(sd, msgDir, streamStreamStateFile))) fs, err = newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: blkSize}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() for subj := range fs.SubjectsTotals(">") { var n int _, err := fmt.Sscanf(subj, "foo.%d", &n) require_NoError(t, err) } } // Since 2.10 we no longer have fss, and the approach for calculating NumPending would branch // based on the old fss metadata being present. This meant that calculating NumPending in >= 2.10.x // would load all blocks to complete. This test makes sure we do not do that anymore. func TestFileStoreNumPendingLastBySubject(t *testing.T) { sd, blkSize := t.TempDir(), uint64(1024) fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: blkSize}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() numSubjects := 20 msg := bytes.Repeat([]byte("ABC"), 25) for i := 1; i <= 1000; i++ { subj := fmt.Sprintf("foo.%d.%d", rand.Intn(numSubjects)+1, i) fs.StoreMsg(subj, nil, msg) } // Each block has ~8 msgs. require_True(t, fs.numMsgBlocks() > 100) calcCacheLoads := func() (cloads uint64) { fs.mu.RLock() defer fs.mu.RUnlock() for _, mb := range fs.blks { mb.mu.RLock() cloads += mb.cloads mb.mu.RUnlock() } return cloads } total, _ := fs.NumPending(0, "foo.*.*", true) require_Equal(t, total, 1000) // Make sure no blocks were loaded to calculate this as a new consumer. require_Equal(t, calcCacheLoads(), 0) checkResult := func(sseq, np uint64, filter string) { t.Helper() var checkTotal uint64 var smv StoreMsg for seq := sseq; seq <= 1000; seq++ { sm, err := fs.LoadMsg(seq, &smv) require_NoError(t, err) if subjectIsSubsetMatch(sm.subj, filter) { checkTotal++ } } require_Equal(t, np, checkTotal) } // Make sure partials work properly. for _, filter := range []string{"foo.10.*", "*.22.*", "*.*.222", "foo.5.999", "*.2.*"} { sseq := uint64(rand.Intn(250) + 200) // Between 200-450 total, _ = fs.NumPending(sseq, filter, true) checkResult(sseq, total, filter) } } // We had a bug that could cause internal memory corruption of the psim keys in memory // which could have been written to disk via index.db. func TestFileStoreCorruptPSIMOnDisk(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo.bar", nil, []byte("ABC")) fs.StoreMsg("foo.baz", nil, []byte("XYZ")) // Force bad subject. fs.mu.Lock() psi, _ := fs.psim.Find(stringToBytes("foo.bar")) bad := make([]byte, 7) crand.Read(bad) fs.psim.Insert(bad, *psi) fs.psim.Delete(stringToBytes("foo.bar")) fs.dirty++ fs.mu.Unlock() // Restart fs.Stop() fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() sm, err := fs.LoadLastMsg("foo.bar", nil) require_NoError(t, err) require_True(t, bytes.Equal(sm.msg, []byte("ABC"))) sm, err = fs.LoadLastMsg("foo.baz", nil) require_NoError(t, err) require_True(t, bytes.Equal(sm.msg, []byte("XYZ"))) } func TestFileStorePurgeExBufPool(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 1024}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("ABC"), 33) // ~100bytes for i := 0; i < 1000; i++ { fs.StoreMsg("foo.foo", nil, msg) fs.StoreMsg("foo.bar", nil, msg) } p, err := fs.PurgeEx("foo.bar", 1, 0) require_NoError(t, err) require_Equal(t, p, 1000) // Now make sure we do not have all of the msg blocks cache's loaded. var loaded int fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() if mb.cacheAlreadyLoaded() { loaded++ } mb.mu.RUnlock() } fs.mu.RUnlock() require_Equal(t, loaded, 1) } func TestFileStoreFSSMeta(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 100, CacheExpire: 200 * time.Millisecond, SubjectStateExpire: time.Second}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. So 2 msgs per blk with subject len of 1, e.g. "A" or "Z". msg := bytes.Repeat([]byte("Z"), 19) // Should leave us with |A-Z| |Z-Z| |Z-Z| |Z-A| fs.StoreMsg("A", nil, msg) for i := 0; i < 6; i++ { fs.StoreMsg("Z", nil, msg) } fs.StoreMsg("A", nil, msg) // Let cache's expire before PurgeEx which will load them back in. time.Sleep(250 * time.Millisecond) p, err := fs.PurgeEx("A", 1, 0) require_NoError(t, err) require_Equal(t, p, 2) // Make sure cache is not loaded. var stillHasCache bool fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() stillHasCache = stillHasCache || mb.cacheAlreadyLoaded() mb.mu.RUnlock() } fs.mu.RUnlock() require_False(t, stillHasCache) // Let fss expire via SubjectStateExpire. time.Sleep(1500 * time.Millisecond) var noFSS bool fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() noFSS = noFSS || mb.fssNotLoaded() mb.mu.RUnlock() } fs.mu.RUnlock() require_True(t, noFSS) } func TestFileStoreExpireCacheOnLinearWalk(t *testing.T) { sd := t.TempDir() expire := 250 * time.Millisecond fs, err := newFileStore( FileStoreConfig{StoreDir: sd, CacheExpire: expire}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // This yields an internal record length of 50 bytes. subj, msg := "Z", bytes.Repeat([]byte("Z"), 19) // Store 10 messages, so 5 blocks. for i := 0; i < 10; i++ { fs.StoreMsg(subj, nil, msg) } // Let them all expire. This way we load as we walk and can test that we expire all blocks without // needing to worry about last write times blocking forced expiration. time.Sleep(expire) checkNoCache := func() { t.Helper() fs.mu.RLock() var stillHasCache bool for _, mb := range fs.blks { mb.mu.RLock() stillHasCache = stillHasCache || mb.cacheAlreadyLoaded() mb.mu.RUnlock() } fs.mu.RUnlock() require_False(t, stillHasCache) } // Walk forward. var smv StoreMsg for seq := uint64(1); seq <= 10; seq++ { _, err := fs.LoadMsg(seq, &smv) require_NoError(t, err) } checkNoCache() // No test walking backwards. We have this scenario when we search for starting points for sourced streams. // Noticed some memory bloat when we have to search many blocks looking for a source that may be closer to the // beginning of the stream (infrequently updated sourced stream). for seq := uint64(10); seq >= 1; seq-- { _, err := fs.LoadMsg(seq, &smv) require_NoError(t, err) } checkNoCache() // Now make sure still expires properly on linear scans with deleted msgs. // We want to make sure we track linear updates even if message deleted. _, err = fs.RemoveMsg(2) require_NoError(t, err) _, err = fs.RemoveMsg(9) require_NoError(t, err) // Walk forward. for seq := uint64(1); seq <= 10; seq++ { _, err := fs.LoadMsg(seq, &smv) if seq == 2 || seq == 9 { require_Error(t, err, errDeletedMsg) } else { require_NoError(t, err) } } checkNoCache() } func TestFileStoreSkipMsgs(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Test on empty FS first. // Make sure wrong starting sequence fails. err = fs.SkipMsgs(10, 100) require_Error(t, err, ErrSequenceMismatch) err = fs.SkipMsgs(1, 100) require_NoError(t, err) state := fs.State() require_Equal(t, state.FirstSeq, 101) require_Equal(t, state.LastSeq, 100) require_Equal(t, fs.numMsgBlocks(), 1) // Now add alot. err = fs.SkipMsgs(101, 100_000) require_NoError(t, err) state = fs.State() require_Equal(t, state.FirstSeq, 100_101) require_Equal(t, state.LastSeq, 100_100) require_Equal(t, fs.numMsgBlocks(), 1) // Now add in a message, and then skip to check dmap. fs, err = newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, nil) err = fs.SkipMsgs(2, 10) require_NoError(t, err) state = fs.State() require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 11) require_Equal(t, state.Msgs, 1) require_Equal(t, state.NumDeleted, 10) require_Equal(t, len(state.Deleted), 10) // Check Fast State too. state.Deleted = nil fs.FastState(&state) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 11) require_Equal(t, state.Msgs, 1) require_Equal(t, state.NumDeleted, 10) } func TestFileStoreOptimizeFirstLoadNextMsgWithSequenceZero(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 4096}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("ZZZ"), 33) // ~100bytes for i := 0; i < 5000; i++ { fs.StoreMsg("foo.A", nil, msg) } // This will create alot of blocks, ~167. // Just used to check that we do not load these in when searching. // Now add in 10 for foo.bar at the end. for i := 0; i < 10; i++ { fs.StoreMsg("foo.B", nil, msg) } // The bug would not be visible on running server per se since we would have had fss loaded // and that sticks around a bit longer, we would use that to skip over the early blocks. So stop // and restart the filestore. fs.Stop() fs, err = newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 4096}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Now fetch the next message for foo.B but set starting sequence to 0. _, nseq, err := fs.LoadNextMsg("foo.B", false, 0, nil) require_NoError(t, err) require_Equal(t, nseq, 5001) // Now check how many blks are loaded, should be only 1. require_Equal(t, fs.cacheLoads(), 1) } func TestFileStoreWriteFullStateHighSubjectCardinality(t *testing.T) { t.Skip() sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 4096}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte{1, 2, 3} for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("subj_%d", i) _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } start := time.Now() require_NoError(t, fs.writeFullState()) t.Logf("Took %s to writeFullState", time.Since(start)) } func TestFileStoreEraseMsgWithDbitSlots(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, []byte("abd")) for i := 0; i < 10; i++ { fs.SkipMsg() } fs.StoreMsg("foo", nil, []byte("abd")) // Now grab that first block and compact away the skips which will // introduce dbits into our idx. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() // Compact. mb.mu.Lock() mb.compact() mb.mu.Unlock() removed, err := fs.EraseMsg(1) require_NoError(t, err) require_True(t, removed) } func TestFileStoreEraseMsgWithAllTrailingDbitSlots(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.StoreMsg("foo", nil, []byte("abc")) fs.StoreMsg("foo", nil, []byte("abcdefg")) for i := 0; i < 10; i++ { fs.SkipMsg() } // Now grab that first block and compact away the skips which will // introduce dbits into our idx. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() // Compact. mb.mu.Lock() mb.compact() mb.mu.Unlock() removed, err := fs.EraseMsg(2) require_NoError(t, err) require_True(t, removed) } // https://github.com/nats-io/nats-server/issues/5236 // Unclear how the sequences get off here, this is just forcing the situation reported. func TestFileStoreMsgBlockFirstAndLastSeqCorrupt(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } fs.Purge() fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() mb.mu.Lock() mb.tryForceExpireCacheLocked() atomic.StoreUint64(&mb.last.seq, 9) mb.mu.Unlock() // We should rebuild here and return no error. require_NoError(t, mb.loadMsgs()) mb.mu.RLock() fseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq) mb.mu.RUnlock() require_Equal(t, fseq, 11) require_Equal(t, lseq, 10) } func TestFileStoreWriteFullStateAfterPurgeEx(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } fs.RemoveMsg(8) fs.RemoveMsg(9) fs.RemoveMsg(10) n, err := fs.PurgeEx(">", 8, 0) require_NoError(t, err) require_Equal(t, n, 7) var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 11) require_Equal(t, ss.LastSeq, 10) // Make sure this does not reset our state due to skew with msg blocks. fs.writeFullState() fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 11) require_Equal(t, ss.LastSeq, 10) } func TestFileStoreFSSExpire(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 8192, CacheExpire: 1 * time.Second, SubjectStateExpire: time.Second}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, MaxMsgsPer: 1, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 1000; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } // Flush fss by hand, cache should be flushed as well. fs.mu.RLock() for _, mb := range fs.blks { mb.mu.Lock() mb.fss = nil mb.mu.Unlock() } fs.mu.RUnlock() fs.StoreMsg("foo.11", nil, msg) time.Sleep(900 * time.Millisecond) // This should keep fss alive in the first block.. // As well as cache itself due to remove activity. fs.StoreMsg("foo.22", nil, msg) time.Sleep(300 * time.Millisecond) // Check that fss and the cache are still loaded. fs.mu.RLock() mb := fs.blks[0] fs.mu.RUnlock() mb.mu.RLock() cache, fss := mb.cache, mb.fss mb.mu.RUnlock() require_True(t, fss != nil) require_True(t, cache != nil) } func TestFileStoreFSSExpireNumPending(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 8192, CacheExpire: 1 * time.Second, SubjectStateExpire: 2 * time.Second}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, MaxMsgsPer: 1, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 100_000; i++ { fs.StoreMsg(fmt.Sprintf("foo.A.%d", i), nil, msg) fs.StoreMsg(fmt.Sprintf("foo.B.%d", i), nil, msg) } // Flush fss by hand, cache should be flushed as well. fs.mu.RLock() for _, mb := range fs.blks { mb.mu.Lock() mb.fss = nil mb.mu.Unlock() } fs.mu.RUnlock() nb := fs.numMsgBlocks() // Now execute NumPending() such that we load lots of blocks and make sure fss do not expire. start := time.Now() n, _ := fs.NumPending(100_000, "foo.A.*", false) elapsed := time.Since(start) require_Equal(t, n, 50_000) // Make sure we did not force expire the fss. We would have loaded first half of blocks. var noFss bool last := nb/2 - 1 fs.mu.RLock() for i, mb := range fs.blks { mb.mu.RLock() noFss = mb.fss == nil mb.mu.RUnlock() if noFss || i == last { break } } fs.mu.RUnlock() require_False(t, noFss) // Run again, make sure faster. This is consequence of fss being loaded now. start = time.Now() fs.NumPending(100_000, "foo.A.*", false) require_True(t, elapsed > 2*time.Since(start)) // Now do with start past the mid-point. start = time.Now() fs.NumPending(150_000, "foo.B.*", false) elapsed = time.Since(start) time.Sleep(time.Second) start = time.Now() fs.NumPending(150_000, "foo.B.*", false) require_True(t, elapsed > time.Since(start)) // Sleep enough so that all mb.fss should expire, which is 2s above. time.Sleep(4 * time.Second) fs.mu.RLock() for i, mb := range fs.blks { mb.mu.RLock() fss := mb.fss mb.mu.RUnlock() if fss != nil { fs.mu.RUnlock() t.Fatalf("Detected loaded fss for mb %d (size %d)", i, fss.Size()) } } fs.mu.RUnlock() } // We want to ensure that recovery of deleted messages survives no index.db and compactions. func TestFileStoreRecoverWithRemovesAndNoIndexDB(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 250}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } fs.RemoveMsg(1) fs.RemoveMsg(2) fs.RemoveMsg(8) var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 3) require_Equal(t, ss.LastSeq, 10) require_Equal(t, ss.Msgs, 7) // Compact last block. fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.Lock() lmb.compact() lmb.mu.Unlock() // Stop but remove index.db sfile := filepath.Join(sd, msgDir, streamStreamStateFile) fs.Stop() os.Remove(sfile) fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 3) require_Equal(t, ss.LastSeq, 10) require_Equal(t, ss.Msgs, 7) } func TestFileStoreReloadAndLoseLastSequence(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() for i := 0; i < 22; i++ { fs.SkipMsg() } // Restart 5 times. for i := 0; i < 5; i++ { fs.Stop() fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 23) require_Equal(t, ss.LastSeq, 22) } } func TestFileStoreReloadAndLoseLastSequenceWithSkipMsgs(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Make sure same works with SkipMsgs which can kick in from delete blocks to replicas. require_NoError(t, fs.SkipMsgs(0, 22)) // Restart 5 times. for i := 0; i < 5; i++ { fs.Stop() fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 23) require_Equal(t, ss.LastSeq, 22) } } func TestFileStoreLoadLastWildcard(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 512}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") fs.StoreMsg("foo.22.baz", nil, msg) fs.StoreMsg("foo.22.bar", nil, msg) for i := 0; i < 1000; i++ { fs.StoreMsg("foo.11.foo", nil, msg) } // Make sure we remove fss since that would mask the problem that we walk // all the blocks. fs.mu.RLock() for _, mb := range fs.blks { mb.mu.Lock() mb.fss = nil mb.mu.Unlock() } fs.mu.RUnlock() // Attempt to load the last msg using a wildcarded subject. sm, err := fs.LoadLastMsg("foo.22.*", nil) require_NoError(t, err) require_Equal(t, sm.seq, 2) // Make sure that we took advantage of psim meta data and only load one block. var cloads uint64 fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() cloads += mb.cloads mb.mu.RUnlock() } fs.mu.RUnlock() require_Equal(t, cloads, 1) } func TestFileStoreLoadLastWildcardWithPresenceMultipleBlocks(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 64}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Make sure we have "foo.222.bar" in multiple blocks to show bug. fs.StoreMsg("foo.22.bar", nil, []byte("hello")) fs.StoreMsg("foo.22.baz", nil, []byte("ok")) fs.StoreMsg("foo.22.baz", nil, []byte("ok")) fs.StoreMsg("foo.22.bar", nil, []byte("hello22")) require_True(t, fs.numMsgBlocks() > 1) sm, err := fs.LoadLastMsg("foo.*.bar", nil) require_NoError(t, err) require_Equal(t, "hello22", string(sm.msg)) } // We want to make sure that we update psim correctly on a miss. func TestFileStoreFilteredPendingPSIMFirstBlockUpdate(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 512}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // When PSIM detects msgs == 1 will catch up, so msgs needs to be > 1. // Then create a huge block gap. msg := []byte("hello") fs.StoreMsg("foo.baz", nil, msg) for i := 0; i < 1000; i++ { fs.StoreMsg("foo.foo", nil, msg) } // Bookend with 2 more foo.baz fs.StoreMsg("foo.baz", nil, msg) fs.StoreMsg("foo.baz", nil, msg) // Now remove first one. removed, err := fs.RemoveMsg(1) require_NoError(t, err) require_True(t, removed) // 84 blocks. require_Equal(t, fs.numMsgBlocks(), 84) fs.mu.RLock() psi, ok := fs.psim.Find([]byte("foo.baz")) fs.mu.RUnlock() require_True(t, ok) require_Equal(t, psi.total, 2) require_Equal(t, psi.fblk, 1) require_Equal(t, psi.lblk, 84) // No make sure that a call to numFilterPending which will initially walk all blocks if starting from seq 1 updates psi. var ss SimpleState fs.mu.RLock() fs.numFilteredPending("foo.baz", &ss) fs.mu.RUnlock() require_Equal(t, ss.Msgs, 2) require_Equal(t, ss.First, 1002) require_Equal(t, ss.Last, 1003) // Check psi was updated. This is done in separate go routine to acquire // the write lock now. checkFor(t, time.Second, 100*time.Millisecond, func() error { fs.mu.RLock() psi, ok = fs.psim.Find([]byte("foo.baz")) total, fblk, lblk := psi.total, psi.fblk, psi.lblk fs.mu.RUnlock() require_True(t, ok) require_Equal(t, total, 2) require_Equal(t, lblk, 84) if fblk != 84 { return fmt.Errorf("fblk should be 84, still %d", fblk) } return nil }) } func TestFileStoreWildcardFilteredPendingPSIMFirstBlockUpdate(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 512}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // When PSIM detects msgs == 1 will catch up, so msgs needs to be > 1. // Then create a huge block gap. msg := []byte("hello") fs.StoreMsg("foo.22.baz", nil, msg) fs.StoreMsg("foo.22.bar", nil, msg) for i := 0; i < 1000; i++ { fs.StoreMsg("foo.1.foo", nil, msg) } // Bookend with 3 more, two foo.baz and two foo.bar. fs.StoreMsg("foo.22.baz", nil, msg) fs.StoreMsg("foo.22.baz", nil, msg) fs.StoreMsg("foo.22.bar", nil, msg) fs.StoreMsg("foo.22.bar", nil, msg) // Now remove first one for foo.bar and foo.baz. removed, err := fs.RemoveMsg(1) require_NoError(t, err) require_True(t, removed) removed, err = fs.RemoveMsg(2) require_NoError(t, err) require_True(t, removed) // 92 blocks. require_Equal(t, fs.numMsgBlocks(), 92) fs.mu.RLock() psi, ok := fs.psim.Find([]byte("foo.22.baz")) total, fblk, lblk := psi.total, psi.fblk, psi.lblk fs.mu.RUnlock() require_True(t, ok) require_Equal(t, total, 2) require_Equal(t, fblk, 1) require_Equal(t, lblk, 92) fs.mu.RLock() psi, ok = fs.psim.Find([]byte("foo.22.bar")) total, fblk, lblk = psi.total, psi.fblk, psi.lblk fs.mu.RUnlock() require_True(t, ok) require_Equal(t, total, 2) require_Equal(t, fblk, 1) require_Equal(t, lblk, 92) // No make sure that a call to numFilterPending which will initially walk all blocks if starting from seq 1 updates psi. var ss SimpleState fs.mu.RLock() fs.numFilteredPending("foo.22.*", &ss) fs.mu.RUnlock() require_Equal(t, ss.Msgs, 4) require_Equal(t, ss.First, 1003) require_Equal(t, ss.Last, 1006) // Check both psi were updated. checkFor(t, time.Second, 100*time.Millisecond, func() error { fs.mu.RLock() psi, ok = fs.psim.Find([]byte("foo.22.baz")) total, fblk, lblk = psi.total, psi.fblk, psi.lblk fs.mu.RUnlock() require_True(t, ok) require_Equal(t, total, 2) require_Equal(t, lblk, 92) if fblk != 92 { return fmt.Errorf("fblk should be 92, still %d", fblk) } return nil }) checkFor(t, time.Second, 100*time.Millisecond, func() error { fs.mu.RLock() psi, ok = fs.psim.Find([]byte("foo.22.bar")) total, fblk, lblk = psi.total, psi.fblk, psi.lblk fs.mu.RUnlock() require_True(t, ok) require_Equal(t, total, 2) require_Equal(t, fblk, 92) if fblk != 92 { return fmt.Errorf("fblk should be 92, still %d", fblk) } return nil }) } // Make sure if we only miss by one for fblk that we still update it. func TestFileStoreFilteredPendingPSIMFirstBlockUpdateNextBlock(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 128}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") // Create 4 blocks, each block holds 2 msgs for i := 0; i < 4; i++ { fs.StoreMsg("foo.22.bar", nil, msg) fs.StoreMsg("foo.22.baz", nil, msg) } require_Equal(t, fs.numMsgBlocks(), 4) fetch := func(subj string) *psi { t.Helper() fs.mu.RLock() var info psi psi, ok := fs.psim.Find([]byte(subj)) if ok && psi != nil { info = *psi } fs.mu.RUnlock() require_True(t, ok) return &info } psi := fetch("foo.22.bar") require_Equal(t, psi.total, 4) require_Equal(t, psi.fblk, 1) require_Equal(t, psi.lblk, 4) // Now remove first instance of "foo.22.bar" removed, err := fs.RemoveMsg(1) require_NoError(t, err) require_True(t, removed) // Call into numFilterePending(), we want to make sure it updates fblk. var ss SimpleState fs.mu.Lock() fs.numFilteredPending("foo.22.bar", &ss) fs.mu.Unlock() require_Equal(t, ss.Msgs, 3) require_Equal(t, ss.First, 3) require_Equal(t, ss.Last, 7) // Now make sure that we properly updated the psim entry. checkFor(t, time.Second, 100*time.Millisecond, func() error { psi = fetch("foo.22.bar") require_Equal(t, psi.total, 3) require_Equal(t, psi.lblk, 4) if psi.fblk != 2 { return fmt.Errorf("fblk should be 2, still %d", psi.fblk) } return nil }) // Now make sure wildcard calls into also update blks. // First remove first "foo.22.baz" which will remove first block. removed, err = fs.RemoveMsg(2) require_NoError(t, err) require_True(t, removed) // Make sure 3 blks left. require_Equal(t, fs.numMsgBlocks(), 3) psi = fetch("foo.22.baz") require_Equal(t, psi.total, 3) require_Equal(t, psi.fblk, 1) require_Equal(t, psi.lblk, 4) // Now call wildcard version of numFilteredPending to make sure it clears. fs.mu.Lock() fs.numFilteredPending("foo.*.baz", &ss) fs.mu.Unlock() require_Equal(t, ss.Msgs, 3) require_Equal(t, ss.First, 4) require_Equal(t, ss.Last, 8) checkFor(t, time.Second, 100*time.Millisecond, func() error { psi = fetch("foo.22.baz") require_Equal(t, psi.total, 3) require_Equal(t, psi.lblk, 4) if psi.fblk != 2 { return fmt.Errorf("fblk should be 2, still %d", psi.fblk) } return nil }) } func TestFileStoreLargeSparseMsgsDoNotLoadAfterLast(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 128}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") // Create 2 blocks with each, each block holds 2 msgs for i := 0; i < 2; i++ { fs.StoreMsg("foo.22.bar", nil, msg) fs.StoreMsg("foo.22.baz", nil, msg) } // Now create 8 more blocks with just baz. So no matches for these 8 blocks // for "foo.22.bar". for i := 0; i < 8; i++ { fs.StoreMsg("foo.22.baz", nil, msg) fs.StoreMsg("foo.22.baz", nil, msg) } require_Equal(t, fs.numMsgBlocks(), 10) // Remove all blk cache and fss. fs.mu.RLock() for _, mb := range fs.blks { mb.mu.Lock() mb.fss, mb.cache = nil, nil mb.mu.Unlock() } fs.mu.RUnlock() // "foo.22.bar" is at sequence 1 and 3. // Make sure if we do a LoadNextMsg() starting at 4 that we do not load // all the tail blocks. _, _, err = fs.LoadNextMsg("foo.*.bar", true, 4, nil) require_Error(t, err, ErrStoreEOF) // Now make sure we did not load fss and cache. var loaded int fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() if mb.cache != nil || mb.fss != nil { loaded++ } mb.mu.RUnlock() } fs.mu.RUnlock() // We will load first block for starting seq 4, but no others should have loaded. require_Equal(t, loaded, 1) } func TestFileStoreCheckSkipFirstBlockBug(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 128}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") fs.StoreMsg("foo.BB.bar", nil, msg) fs.StoreMsg("foo.BB.bar", nil, msg) fs.StoreMsg("foo.AA.bar", nil, msg) for i := 0; i < 5; i++ { fs.StoreMsg("foo.BB.bar", nil, msg) } fs.StoreMsg("foo.AA.bar", nil, msg) fs.StoreMsg("foo.AA.bar", nil, msg) // Should have created 4 blocks. // BB BB | AA BB | BB BB | BB BB | AA AA require_Equal(t, fs.numMsgBlocks(), 5) fs.RemoveMsg(3) fs.RemoveMsg(4) // Second block should be gone now. // BB BB | -- -- | BB BB | BB BB | AA AA require_Equal(t, fs.numMsgBlocks(), 4) _, _, err = fs.LoadNextMsg("foo.AA.bar", false, 4, nil) require_NoError(t, err) } // https://github.com/nats-io/nats-server/issues/5702 func TestFileStoreTombstoneRbytes(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Block can hold 24 msgs. // So will fill one block and half of the other msg := []byte("hello") for i := 0; i < 34; i++ { fs.StoreMsg("foo.22", nil, msg) } require_True(t, fs.numMsgBlocks() > 1) // Now delete second half of first block which will place tombstones in second blk. for seq := 11; seq <= 24; seq++ { fs.RemoveMsg(uint64(seq)) } // Now check that rbytes has been properly accounted for in second block. fs.mu.RLock() blk := fs.blks[1] fs.mu.RUnlock() blk.mu.RLock() bytes, rbytes := blk.bytes, blk.rbytes blk.mu.RUnlock() require_True(t, rbytes > bytes) } func TestFileStoreMsgBlockShouldCompact(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // 127 fit into a block. msg := bytes.Repeat([]byte("Z"), 64*1024) for i := 0; i < 190; i++ { fs.StoreMsg("foo.22", nil, msg) } require_True(t, fs.numMsgBlocks() > 1) // Now delete second half of first block which will place tombstones in second blk. for seq := 64; seq <= 127; seq++ { fs.RemoveMsg(uint64(seq)) } fs.mu.RLock() fblk := fs.blks[0] sblk := fs.blks[1] fs.mu.RUnlock() fblk.mu.RLock() bytes, rbytes := fblk.bytes, fblk.rbytes shouldCompact := fblk.shouldCompactInline() fblk.mu.RUnlock() // Should have tripped compaction already. require_Equal(t, bytes, rbytes) require_False(t, shouldCompact) sblk.mu.RLock() shouldCompact = sblk.shouldCompactInline() sblk.mu.RUnlock() require_False(t, shouldCompact) } func TestFileStoreCheckSkipFirstBlockNotLoadOldBlocks(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 128}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") fs.StoreMsg("foo.BB.bar", nil, msg) fs.StoreMsg("foo.AA.bar", nil, msg) for i := 0; i < 6; i++ { fs.StoreMsg("foo.BB.bar", nil, msg) } fs.StoreMsg("foo.AA.bar", nil, msg) // Sequence 9 fs.StoreMsg("foo.AA.bar", nil, msg) // Sequence 10 for i := 0; i < 4; i++ { fs.StoreMsg("foo.BB.bar", nil, msg) } // Should have created 7 blocks. // BB AA | BB BB | BB BB | BB BB | AA AA | BB BB | BB BB require_Equal(t, fs.numMsgBlocks(), 7) fs.RemoveMsg(1) fs.RemoveMsg(2) // First block should be gone now. // -- -- | BB BB | BB BB | BB BB | AA AA | BB BB | BB BB require_Equal(t, fs.numMsgBlocks(), 6) // Remove all blk cache and fss. fs.mu.RLock() for _, mb := range fs.blks { mb.mu.Lock() mb.fss, mb.cache = nil, nil mb.mu.Unlock() } fs.mu.RUnlock() // But this means that the psim still points fblk to block 1. // So when we try to load AA from near the end (last AA sequence), it will not find anything and will then // check if we can skip ahead, but in the process reload blocks 2, 3, 4 amd 5.. // This can trigger for an up to date consumer near the end of the stream that gets a new pull request that will pop it out of msgWait // and it will call LoadNextMsg() like we do here with starting sequence of 11. _, _, err = fs.LoadNextMsg("foo.AA.bar", false, 11, nil) require_Error(t, err, ErrStoreEOF) // Now make sure we did not load fss and cache. var loaded int fs.mu.RLock() for _, mb := range fs.blks { mb.mu.RLock() if mb.cache != nil || mb.fss != nil { loaded++ } mb.mu.RUnlock() } fs.mu.RUnlock() // We will load last block for starting seq 9, but no others should have loaded. require_Equal(t, loaded, 1) } func TestFileStoreSyncCompressOnlyIfDirty(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 256, SyncInterval: 250 * time.Millisecond}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") // 6 msgs per block. // Fill 2 blocks. for i := 0; i < 12; i++ { fs.StoreMsg("foo.BB", nil, msg) } // Create third block with just one message in it. fs.StoreMsg("foo.BB", nil, msg) // Should have created 3 blocks. require_Equal(t, fs.numMsgBlocks(), 3) // Now delete a bunch that will will fill up 3 block with tombstones. for _, seq := range []uint64{2, 3, 4, 5, 8, 9, 10, 11} { _, err = fs.RemoveMsg(seq) require_NoError(t, err) } // Now make sure we add 4/5th block so syncBlocks will try to compact. for i := 0; i < 6; i++ { fs.StoreMsg("foo.BB", nil, msg) } require_Equal(t, fs.numMsgBlocks(), 5) // All should have compact set. fs.mu.Lock() // Only check first 3 blocks. for i := 0; i < 3; i++ { mb := fs.blks[i] mb.mu.Lock() shouldCompact := mb.shouldCompactSync() mb.mu.Unlock() if !shouldCompact { fs.mu.Unlock() t.Fatalf("Expected should compact to be true for %d, got false", mb.getIndex()) } } fs.mu.Unlock() // Let sync run. time.Sleep(300 * time.Millisecond) // We want to make sure the last block, which is filled with tombstones and is not compactable, returns false now. fs.mu.Lock() for _, mb := range fs.blks { mb.mu.Lock() shouldCompact := mb.shouldCompactSync() mb.mu.Unlock() if shouldCompact { fs.mu.Unlock() t.Fatalf("Expected should compact to be false for %d, got true", mb.getIndex()) } } fs.mu.Unlock() // Now remove some from block 3 and verify that compact is not suppressed. _, err = fs.RemoveMsg(13) require_NoError(t, err) fs.mu.Lock() mb := fs.blks[2] // block 3. mb.mu.Lock() noCompact := mb.noCompact mb.mu.Unlock() fs.mu.Unlock() // Verify that since we deleted a message we should be considered for compaction again in syncBlocks(). require_False(t, noCompact) } // This test is for deleted interior message tracking after compaction from limits based deletes, meaning no tombstones. // Bug was that dmap would not be properly be hydrated after the compact from rebuild. But we did so in populateGlobalInfo. // So this is just to fix a bug in rebuildState tracking gaps after a compact. func TestFileStoreDmapBlockRecoverAfterCompact(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 256}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage, MaxMsgsPer: 1}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") // 6 msgs per block. // Fill the first block. for i := 1; i <= 6; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } require_Equal(t, fs.numMsgBlocks(), 1) // Now create holes in the first block via the max msgs per subject of 1. for i := 2; i < 6; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } require_Equal(t, fs.numMsgBlocks(), 2) // Compact and rebuild the first blk. Do not have it call indexCacheBuf which will fix it up. mb := fs.getFirstBlock() mb.mu.Lock() mb.compact() // Empty out dmap state. mb.dmap.Empty() ld, tombs, err := mb.rebuildStateLocked() dmap := mb.dmap.Clone() mb.mu.Unlock() require_NoError(t, err) require_Equal(t, ld, nil) require_Equal(t, len(tombs), 0) require_Equal(t, dmap.Size(), 4) } func TestFileStoreRestoreIndexWithMatchButLeftOverBlocks(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 256}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage, MaxMsgsPer: 1}) require_NoError(t, err) defer fs.Stop() msg := []byte("hello") // 6 msgs per block. // Fill the first 2 blocks. for i := 1; i <= 12; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } require_Equal(t, fs.numMsgBlocks(), 2) // We will now stop which will create the index.db file which will // match the last record exactly. sfile := filepath.Join(sd, msgDir, streamStreamStateFile) fs.Stop() // Grab it since we will put it back. buf, err := os.ReadFile(sfile) require_NoError(t, err) require_True(t, len(buf) > 0) // Now do an additional block, but with the MaxMsgsPer this will remove the first block, // but leave the second so on recovery will match the checksum for the last msg in second block. fs, err = newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 256}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage, MaxMsgsPer: 1}) require_NoError(t, err) defer fs.Stop() for i := 1; i <= 6; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } // Grab correct state, we will use it to make sure we do the right thing. var state StreamState fs.FastState(&state) require_Equal(t, state.Msgs, 12) require_Equal(t, state.FirstSeq, 7) require_Equal(t, state.LastSeq, 18) // This will be block 2 and 3. require_Equal(t, fs.numMsgBlocks(), 2) fs.Stop() // Put old stream state back. require_NoError(t, os.WriteFile(sfile, buf, defaultFilePerms)) fs, err = newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 256}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage, MaxMsgsPer: 1}) require_NoError(t, err) defer fs.Stop() fs.FastState(&state) require_Equal(t, state.Msgs, 12) require_Equal(t, state.FirstSeq, 7) require_Equal(t, state.LastSeq, 18) } func TestFileStoreRestoreDeleteTombstonesExceedingMaxBlkSize(t *testing.T) { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fcfg.BlockSize = 256 fs, err := newFileStoreWithCreated( fcfg, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}, time.Now(), prf(&fcfg), nil, ) require_NoError(t, err) defer fs.Stop() n, err := fs.PurgeEx(_EMPTY_, 1_000_000_000, 0) require_NoError(t, err) require_Equal(t, n, 0) msg := []byte("hello") // 6 msgs per block with blk size 256. for i := 1; i <= 10_000; i++ { fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) } // Now delete msgs which will write tombstones. for seq := uint64(1_000_000_001); seq < 1_000_000_101; seq++ { removed, err := fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, removed) } // Check last block and make sure the tombstones did not exceed blk size maximum. // Check to make sure no blocks exceed blk size. fs.mu.RLock() blks := append([]*msgBlock(nil), fs.blks...) lmb := fs.lmb fs.mu.RUnlock() var emptyBlks []*msgBlock for _, mb := range blks { mb.mu.RLock() bytes, rbytes := mb.bytes, mb.rbytes mb.mu.RUnlock() require_True(t, bytes < 256) require_True(t, rbytes < 256) if bytes == 0 && mb != lmb { emptyBlks = append(emptyBlks, mb) } } // Check each block such that it signals it can be compacted but if we attempt compact here nothing should change. for _, mb := range emptyBlks { mb.mu.Lock() mb.ensureRawBytesLoaded() bytes, rbytes, shouldCompact := mb.bytes, mb.rbytes, mb.shouldCompactSync() // Do the compact and make sure nothing changed. mb.compact() nbytes, nrbytes := mb.bytes, mb.rbytes mb.mu.Unlock() require_True(t, shouldCompact) require_Equal(t, bytes, nbytes) require_Equal(t, rbytes, nrbytes) } // Now remove first msg which will invalidate the tombstones since they will be < first sequence. removed, err := fs.RemoveMsg(1_000_000_000) require_NoError(t, err) require_True(t, removed) // Now simulate a syncBlocks call and make sure it cleans up the tombstones that are no longer relevant. fs.syncBlocks() for _, mb := range emptyBlks { mb.mu.Lock() mb.ensureRawBytesLoaded() index, bytes, rbytes := mb.index, mb.bytes, mb.rbytes mb.mu.Unlock() require_Equal(t, bytes, 0) require_Equal(t, rbytes, 0) // Also make sure we removed these blks all together. fs.mu.RLock() imb := fs.bim[index] fs.mu.RUnlock() require_True(t, imb == nil) } }) } /////////////////////////////////////////////////////////////////////////// // Benchmarks /////////////////////////////////////////////////////////////////////////// func Benchmark_FileStoreSelectMsgBlock(b *testing.B) { // We use small block size to create lots of blocks for this test. fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 128}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() subj, msg := "A", bytes.Repeat([]byte("ABC"), 33) // ~100bytes // Add in a bunch of blocks. for i := 0; i < 1000; i++ { fs.StoreMsg(subj, nil, msg) } if fs.numMsgBlocks() < 1000 { b.Fatalf("Expected at least 1000 blocks, got %d", fs.numMsgBlocks()) } fs.mu.RLock() defer fs.mu.RUnlock() b.ResetTimer() for i := 0; i < b.N; i++ { _, mb := fs.selectMsgBlockWithIndex(1) if mb == nil { b.Fatalf("Expected a non-nil mb") } } b.StopTimer() } func Benchmark_FileStoreLoadNextMsgSameFilterAsStream(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 100_000; i++ { subj := fmt.Sprintf("foo.%d", rand.Intn(1024)) fs.StoreMsg(subj, nil, msg) } b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Needs to start at ~1 to show slowdown. _, _, err := fs.LoadNextMsg("foo.*", true, 10, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgLiteralSubject(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 100_000; i++ { subj := fmt.Sprintf("foo.%d", rand.Intn(1024)) fs.StoreMsg(subj, nil, msg) } // This is the one we will try to match. fs.StoreMsg("foo.2222", nil, msg) // So not last and we think we are done linear scan. fs.StoreMsg("foo.3333", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { _, _, err := fs.LoadNextMsg("foo.2222", false, 10, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgNoMsgsFirstSeq(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 1_000_000; i++ { fs.StoreMsg("foo.bar", nil, msg) } b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // This should error with EOF _, _, err := fs.LoadNextMsg("foo.baz", false, 1, &smv) if err != ErrStoreEOF { b.Fatalf("Wrong error, expected EOF got %v", err) } } } func Benchmark_FileStoreLoadNextMsgNoMsgsNotFirstSeq(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 1_000_000; i++ { fs.StoreMsg("foo.bar", nil, msg) } b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // This should error with EOF // Make sure the sequence is not first seq of 1. _, _, err := fs.LoadNextMsg("foo.baz", false, 10, &smv) if err != ErrStoreEOF { b.Fatalf("Wrong error, expected EOF got %v", err) } } } func Benchmark_FileStoreLoadNextMsgVerySparseMsgsFirstSeq(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 1_000_000; i++ { fs.StoreMsg("foo.bar", nil, msg) } // Make last msg one that would match. fs.StoreMsg("foo.baz", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { _, _, err := fs.LoadNextMsg("foo.baz", false, 1, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgVerySparseMsgsNotFirstSeq(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Add in a bunch of msgs for i := 0; i < 1_000_000; i++ { fs.StoreMsg("foo.bar", nil, msg) } // Make last msg one that would match. fs.StoreMsg("foo.baz", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Make sure not first seq. _, _, err := fs.LoadNextMsg("foo.baz", false, 10, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgVerySparseMsgsInBetween(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Make first msg one that would match as well. fs.StoreMsg("foo.baz", nil, msg) // Add in a bunch of msgs for i := 0; i < 1_000_000; i++ { fs.StoreMsg("foo.bar", nil, msg) } // Make last msg one that would match as well. fs.StoreMsg("foo.baz", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Make sure not first seq. _, _, err := fs.LoadNextMsg("foo.baz", false, 2, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgVerySparseMsgsInBetweenWithWildcard(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Make first msg one that would match as well. fs.StoreMsg("foo.1.baz", nil, msg) // Add in a bunch of msgs. // We need to make sure we have a range of subjects that could kick in a linear scan. for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("foo.%d.bar", rand.Intn(100_000)+2) fs.StoreMsg(subj, nil, msg) } // Make last msg one that would match as well. fs.StoreMsg("foo.1.baz", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Make sure not first seq. _, _, err := fs.LoadNextMsg("foo.*.baz", true, 2, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextManySubjectsWithWildcardNearLastBlock(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Make first msg one that would match as well. fs.StoreMsg("foo.1.baz", nil, msg) // Add in a bunch of msgs. // We need to make sure we have a range of subjects that could kick in a linear scan. for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("foo.%d.bar", rand.Intn(100_000)+2) fs.StoreMsg(subj, nil, msg) } // Make last msg one that would match as well. fs.StoreMsg("foo.1.baz", nil, msg) b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Make sure not first seq. _, _, err := fs.LoadNextMsg("foo.*.baz", true, 999_990, &smv) require_NoError(b, err) } } func Benchmark_FileStoreLoadNextMsgVerySparseMsgsLargeTail(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() // Small om purpose. msg := []byte("ok") // Make first msg one that would match as well. fs.StoreMsg("foo.1.baz", nil, msg) // Add in a bunch of msgs. // We need to make sure we have a range of subjects that could kick in a linear scan. for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("foo.%d.bar", rand.Intn(64_000)+2) fs.StoreMsg(subj, nil, msg) } b.ResetTimer() var smv StoreMsg for i := 0; i < b.N; i++ { // Make sure not first seq. _, _, err := fs.LoadNextMsg("foo.*.baz", true, 2, &smv) require_Error(b, err, ErrStoreEOF) } } func Benchmark_FileStoreCreateConsumerStores(b *testing.B) { for _, syncAlways := range []bool{true, false} { b.Run(fmt.Sprintf("%v", syncAlways), func(b *testing.B) { fs, err := newFileStore( FileStoreConfig{StoreDir: b.TempDir(), SyncAlways: syncAlways}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: FileStorage}) require_NoError(b, err) defer fs.Stop() oconfig := ConsumerConfig{ DeliverSubject: "d", FilterSubject: "foo", AckPolicy: AckAll, } b.ResetTimer() for i := 0; i < b.N; i++ { oname := fmt.Sprintf("obs22_%d", i) ofs, err := fs.ConsumerStore(oname, &oconfig) require_NoError(b, err) require_NoError(b, ofs.Stop()) } }) } } func TestFileStoreWriteFullStateDetectCorruptState(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { _, _, err = fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) require_NoError(t, err) } // Simulate a change in a message block not being reflected in the fs. mb := fs.selectMsgBlock(2) mb.mu.Lock() mb.msgs-- mb.mu.Unlock() var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 1) require_Equal(t, ss.LastSeq, 10) require_Equal(t, ss.Msgs, 10) // Make sure we detect the corrupt state and rebuild. err = fs.writeFullState() require_Error(t, err, errCorruptState) fs.FastState(&ss) require_Equal(t, ss.FirstSeq, 1) require_Equal(t, ss.LastSeq, 10) require_Equal(t, ss.Msgs, 9) } func TestFileStoreRecoverFullStateDetectCorruptState(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { _, _, err = fs.StoreMsg(fmt.Sprintf("foo.%d", i), nil, msg) require_NoError(t, err) } err = fs.writeFullState() require_NoError(t, err) sfile := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) buf, err := os.ReadFile(sfile) require_NoError(t, err) // Update to an incorrect message count. binary.PutUvarint(buf[2:], 0) // Just append a corrected checksum to the end to make it pass the checks. fs.hh.Reset() fs.hh.Write(buf) buf = fs.hh.Sum(buf) err = os.WriteFile(sfile, buf, defaultFilePerms) require_NoError(t, err) err = fs.recoverFullState() require_Error(t, err, errCorruptState) } func TestFileStoreNumPendingMulti(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"ev.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() totalMsgs := 100_000 totalSubjects := 10_000 numFiltered := 5000 startSeq := uint64(5_000 + rand.Intn(90_000)) subjects := make([]string, 0, totalSubjects) for i := 0; i < totalSubjects; i++ { subjects = append(subjects, fmt.Sprintf("ev.%s", nuid.Next())) } // Put in 100k msgs with random subjects. msg := bytes.Repeat([]byte("ZZZ"), 333) for i := 0; i < totalMsgs; i++ { _, _, err = fs.StoreMsg(subjects[rand.Intn(totalSubjects)], nil, msg) require_NoError(t, err) } // Now we want to do a calculate NumPendingMulti. filters := NewSublistNoCache() for filters.Count() < uint32(numFiltered) { filter := subjects[rand.Intn(totalSubjects)] if !filters.HasInterest(filter) { filters.Insert(&subscription{subject: []byte(filter)}) } } // Use new function. total, _ := fs.NumPendingMulti(startSeq, filters, false) // Check our results. var checkTotal uint64 var smv StoreMsg for seq := startSeq; seq <= uint64(totalMsgs); seq++ { sm, err := fs.LoadMsg(seq, &smv) require_NoError(t, err) if filters.HasInterest(sm.subj) { checkTotal++ } } require_Equal(t, total, checkTotal) } func TestFileStoreStoreRawMessageThrowsPermissionErrorIfFSModeReadOnly(t *testing.T) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"ev.1"}, Storage: FileStorage, MaxAge: 500 * time.Millisecond} fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, cfg) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("Z"), 1024) directory := fs.fcfg.StoreDir ORIGINAL_FILE_MODE, _ := os.Stat(directory) READONLY_MODE := os.FileMode(0o555) changeDirectoryPermission(directory, READONLY_MODE) require_NoError(t, err) totalMsgs := 10000 i := 0 for i = 0; i < totalMsgs; i++ { _, _, err = fs.StoreMsg("ev.1", nil, msg) if err != nil { break } } changeDirectoryPermission(directory, ORIGINAL_FILE_MODE.Mode()) require_Error(t, err, os.ErrPermission) } func TestFileStoreWriteFullStateThrowsPermissionErrorIfFSModeReadOnly(t *testing.T) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"ev.1"}, Storage: FileStorage, MaxAge: 500 * time.Millisecond} fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, cfg) require_NoError(t, err) defer fs.Stop() msg := bytes.Repeat([]byte("Z"), 1024) directory := fs.fcfg.StoreDir ORIGINAL_FILE_MODE, _ := os.Stat(directory) READONLY_MODE := os.FileMode(0o555) require_NoError(t, err) totalMsgs := 10000 i := 0 for i = 0; i < totalMsgs; i++ { _, _, err = fs.StoreMsg("ev.1", nil, msg) if err != nil { break } } changeDirectoryPermission(directory, READONLY_MODE) err = fs.writeFullState() changeDirectoryPermission(directory, ORIGINAL_FILE_MODE.Mode()) require_Error(t, err, os.ErrPermission) } func changeDirectoryPermission(directory string, mode fs.FileMode) error { err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { if err != nil { return fmt.Errorf("error accessing path %q: %w", path, err) } // Check if the path is a directory or file and set permissions accordingly if info.IsDir() { err = os.Chmod(path, mode) if err != nil { return fmt.Errorf("error changing directory permissions for %q: %w", path, err) } } else { err = os.Chmod(path, mode) if err != nil { return fmt.Errorf("error changing file permissions for %q: %w", path, err) } } return nil }) return err } func TestFileStoreLeftoverSkipMsgInDmap(t *testing.T) { storeDir := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "zzz", Subjects: []string{"test.*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() getLmbState := func(fs *fileStore) (uint64, uint64, int) { fs.mu.RLock() lmb := fs.lmb fs.mu.RUnlock() lmb.mu.RLock() fseq := atomic.LoadUint64(&lmb.first.seq) lseq := atomic.LoadUint64(&lmb.last.seq) dmaps := lmb.dmap.Size() lmb.mu.RUnlock() return fseq, lseq, dmaps } // Only skip a message. fs.SkipMsg() // Confirm state. state := fs.State() require_Equal(t, state.FirstSeq, 2) require_Equal(t, state.LastSeq, 1) require_Equal(t, state.NumDeleted, 0) fseq, lseq, dmaps := getLmbState(fs) require_Equal(t, fseq, 2) require_Equal(t, lseq, 1) require_Len(t, dmaps, 0) // Stop without writing index.db so we recover based on just the blk file. require_NoError(t, fs.stop(false, false)) fs, err = newFileStore( FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "zzz", Subjects: []string{"test.*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() // Confirm the skipped message is not included in the deletes. state = fs.State() require_Equal(t, state.FirstSeq, 2) require_Equal(t, state.LastSeq, 1) require_Equal(t, state.NumDeleted, 0) fseq, lseq, dmaps = getLmbState(fs) require_Equal(t, fseq, 2) require_Equal(t, lseq, 1) require_Len(t, dmaps, 0) } nats-server-2.10.27/server/fuzz.go000066400000000000000000000023431477524627100170100ustar00rootroot00000000000000// Copyright 2020-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build gofuzz // +build gofuzz package server var defaultFuzzServerOptions = Options{ Host: "127.0.0.1", Trace: true, Debug: true, DisableShortFirstPing: true, NoLog: true, NoSigs: true, } func dummyFuzzClient() *client { return &client{srv: New(&defaultFuzzServerOptions), msubs: -1, mpay: MAX_PAYLOAD_SIZE, mcl: MAX_CONTROL_LINE_SIZE} } func FuzzClient(data []byte) int { if len(data) < 100 { return -1 } c := dummyFuzzClient() err := c.parse(data[:50]) if err != nil { return 0 } err = c.parse(data[50:]) if err != nil { return 0 } return 1 } nats-server-2.10.27/server/gateway.go000066400000000000000000003047151477524627100174630ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" "crypto/sha256" "crypto/tls" "encoding/json" "fmt" "math/rand" "net" "net/url" "slices" "strconv" "sync" "sync/atomic" "time" ) const ( defaultSolicitGatewaysDelay = time.Second defaultGatewayConnectDelay = time.Second defaultGatewayReconnectDelay = time.Second defaultGatewayRecentSubExpiration = 2 * time.Second defaultGatewayMaxRUnsubBeforeSwitch = 1000 oldGWReplyPrefix = "$GR." oldGWReplyPrefixLen = len(oldGWReplyPrefix) oldGWReplyStart = oldGWReplyPrefixLen + 5 // len of prefix above + len of hash (4) + "." // The new prefix is "_GR_..." where is 6 characters // hash of origin cluster name and is 6 characters hash of origin server pub key. gwReplyPrefix = "_GR_." gwReplyPrefixLen = len(gwReplyPrefix) gwHashLen = 6 gwClusterOffset = gwReplyPrefixLen gwServerOffset = gwClusterOffset + gwHashLen + 1 gwSubjectOffset = gwServerOffset + gwHashLen + 1 // Gateway connections send PINGs regardless of traffic. The interval is // either Options.PingInterval or this value, whichever is the smallest. gwMaxPingInterval = 15 * time.Second ) var ( gatewayConnectDelay = defaultGatewayConnectDelay gatewayReconnectDelay = defaultGatewayReconnectDelay gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch gatewaySolicitDelay = int64(defaultSolicitGatewaysDelay) gatewayMaxPingInterval = gwMaxPingInterval ) // Warning when user configures gateway TLS insecure const gatewayTLSInsecureWarning = "TLS certificate chain and hostname of solicited gateways will not be verified. DO NOT USE IN PRODUCTION!" // SetGatewaysSolicitDelay sets the initial delay before gateways // connections are initiated. // Used by tests. func SetGatewaysSolicitDelay(delay time.Duration) { atomic.StoreInt64(&gatewaySolicitDelay, int64(delay)) } // ResetGatewaysSolicitDelay resets the initial delay before gateways // connections are initiated to its default values. // Used by tests. func ResetGatewaysSolicitDelay() { atomic.StoreInt64(&gatewaySolicitDelay, int64(defaultSolicitGatewaysDelay)) } const ( gatewayCmdGossip byte = 1 gatewayCmdAllSubsStart byte = 2 gatewayCmdAllSubsComplete byte = 3 ) // GatewayInterestMode represents an account interest mode for a gateway connection type GatewayInterestMode byte // GatewayInterestMode values const ( // optimistic is the default mode where a cluster will send // to a gateway unless it is been told that there is no interest // (this is for plain subscribers only). Optimistic GatewayInterestMode = iota // transitioning is when a gateway has to send too many // no interest on subjects to the remote and decides that it is // now time to move to modeInterestOnly (this is on a per account // basis). Transitioning // interestOnly means that a cluster sends all it subscriptions // interest to the gateway, which in return does not send a message // unless it knows that there is explicit interest. InterestOnly ) func (im GatewayInterestMode) String() string { switch im { case Optimistic: return "Optimistic" case InterestOnly: return "Interest-Only" case Transitioning: return "Transitioning" default: return "Unknown" } } var gwDoNotForceInterestOnlyMode bool // GatewayDoNotForceInterestOnlyMode is used ONLY in tests. // DO NOT USE in normal code or if you embed the NATS Server. func GatewayDoNotForceInterestOnlyMode(doNotForce bool) { gwDoNotForceInterestOnlyMode = doNotForce } type srvGateway struct { totalQSubs int64 //total number of queue subs in all remote gateways (used with atomic operations) sync.RWMutex enabled bool // Immutable, true if both a name and port are configured name string // Name of the Gateway on this server out map[string]*client // outbound gateways outo []*client // outbound gateways maintained in an order suitable for sending msgs (currently based on RTT) in map[uint64]*client // inbound gateways remotes map[string]*gatewayCfg // Config of remote gateways URLs refCountedUrlSet // Set of all Gateway URLs in the cluster URL string // This server gateway URL (after possible random port is resolved) info *Info // Gateway Info protocol infoJSON []byte // Marshal'ed Info protocol runknown bool // Rejects unknown (not configured) gateway connections replyPfx []byte // Will be "$GNR.<1:reserved>.<8:cluster hash>.<8:server hash>." // For backward compatibility oldReplyPfx []byte oldHash []byte // We maintain the interest of subjects and queues per account. // For a given account, entries in the map could be something like this: // foo.bar {n: 3} // 3 subs on foo.bar // foo.> {n: 6} // 6 subs on foo.> // foo bar {n: 1, q: true} // 1 qsub on foo, queue bar // foo baz {n: 3, q: true} // 3 qsubs on foo, queue baz pasi struct { // Protect map since accessed from different go-routine and avoid // possible race resulting in RS+ being sent before RS- resulting // in incorrect interest suppression. // Will use while sending QSubs (on GW connection accept) and when // switching to the send-all-subs mode. sync.Mutex m map[string]map[string]*sitally } // This is to track recent subscriptions for a given account rsubs sync.Map resolver netResolver // Used to resolve host name before calling net.Dial() sqbsz int // Max buffer size to send queue subs protocol. Used for testing. recSubExp time.Duration // For how long do we check if there is a subscription match for a message with reply // These are used for routing of mapped replies. sIDHash []byte // Server ID hash (6 bytes) routesIDByHash sync.Map // Route's server ID is hashed (6 bytes) and stored in this map. // If a server has its own configuration in the "Gateways" remotes configuration // we will keep track of the URLs that are defined in the config so they can // be reported in monitoring. ownCfgURLs []string } // Subject interest tally. Also indicates if the key in the map is a // queue or not. type sitally struct { n int32 // number of subscriptions directly matching q bool // indicate that this is a queue } type gatewayCfg struct { sync.RWMutex *RemoteGatewayOpts hash []byte oldHash []byte urls map[string]*url.URL connAttempts int tlsName string implicit bool varzUpdateURLs bool // Tells monitoring code to update URLs when varz is inspected. } // Struct for client's gateway related fields type gateway struct { name string cfg *gatewayCfg connectURL *url.URL // Needed when sending CONNECT after receiving INFO from remote outsim *sync.Map // Per-account subject interest (or no-interest) (outbound conn) insim map[string]*insie // Per-account subject no-interest sent or modeInterestOnly mode (inbound conn) // This is an outbound GW connection outbound bool // Set/check in readLoop without lock. This is to know that an inbound has sent the CONNECT protocol first connected bool // Set to true if outbound is to a server that only knows about $GR, not $GNR useOldPrefix bool // If true, it indicates that the inbound side will switch any account to // interest-only mode "immediately", so the outbound should disregard // the optimistic mode when checking for interest. interestOnlyMode bool } // Outbound subject interest entry. type outsie struct { sync.RWMutex // Indicate that all subs should be stored. This is // set to true when receiving the command from the // remote that we are about to receive all its subs. mode GatewayInterestMode // If not nil, used for no-interest for plain subs. // If a subject is present in this map, it means that // the remote is not interested in that subject. // When we have received the command that says that // the remote has sent all its subs, this is set to nil. ni map[string]struct{} // Contains queue subscriptions when in optimistic mode, // and all subs when pk is > 0. sl *Sublist // Number of queue subs qsubs int } // Inbound subject interest entry. // If `ni` is not nil, it stores the subjects for which an // RS- was sent to the remote gateway. When a subscription // is created, this is used to know if we need to send // an RS+ to clear the no-interest in the remote. // When an account is switched to modeInterestOnly (we send // all subs of an account to the remote), then `ni` is nil and // when all subs have been sent, mode is set to modeInterestOnly type insie struct { ni map[string]struct{} // Record if RS- was sent for given subject mode GatewayInterestMode } type gwReplyMap struct { ms string exp int64 } type gwReplyMapping struct { // Indicate if we should check the map or not. Since checking the map is done // when processing inbound messages and requires the lock we want to // check only when needed. This is set/get using atomic, so needs to // be memory aligned. check int32 // To keep track of gateway replies mapping mapping map[string]*gwReplyMap } // Returns the corresponding gw routed subject, and `true` to indicate that a // mapping was found. If no entry is found, the passed subject is returned // as-is and `false` is returned to indicate that no mapping was found. // Caller is responsible to ensure the locking. func (g *gwReplyMapping) get(subject []byte) ([]byte, bool) { rm, ok := g.mapping[string(subject)] if !ok { return subject, false } subj := []byte(rm.ms) return subj, true } // clone returns a deep copy of the RemoteGatewayOpts object func (r *RemoteGatewayOpts) clone() *RemoteGatewayOpts { if r == nil { return nil } clone := &RemoteGatewayOpts{ Name: r.Name, URLs: deepCopyURLs(r.URLs), } if r.TLSConfig != nil { clone.TLSConfig = r.TLSConfig.Clone() clone.TLSTimeout = r.TLSTimeout } return clone } // Ensure that gateway is properly configured. func validateGatewayOptions(o *Options) error { if o.Gateway.Name == "" && o.Gateway.Port == 0 { return nil } if o.Gateway.Name == "" { return fmt.Errorf("gateway has no name") } if o.Gateway.Port == 0 { return fmt.Errorf("gateway %q has no port specified (select -1 for random port)", o.Gateway.Name) } for i, g := range o.Gateway.Gateways { if g.Name == "" { return fmt.Errorf("gateway in the list %d has no name", i) } if len(g.URLs) == 0 { return fmt.Errorf("gateway %q has no URL", g.Name) } } if err := validatePinnedCerts(o.Gateway.TLSPinnedCerts); err != nil { return fmt.Errorf("gateway %q: %v", o.Gateway.Name, err) } return nil } // Computes a hash of 6 characters for the name. // This will be used for routing of replies. func getGWHash(name string) []byte { return []byte(getHashSize(name, gwHashLen)) } func getOldHash(name string) []byte { sha := sha256.New() sha.Write([]byte(name)) fullHash := []byte(fmt.Sprintf("%x", sha.Sum(nil))) return fullHash[:4] } // Initialize the s.gateway structure. We do this even if the server // does not have a gateway configured. In some part of the code, the // server will check the number of outbound gateways, etc.. and so // we don't have to check if s.gateway is nil or not. func (s *Server) newGateway(opts *Options) error { gateway := &srvGateway{ name: opts.Gateway.Name, out: make(map[string]*client), outo: make([]*client, 0, 4), in: make(map[uint64]*client), remotes: make(map[string]*gatewayCfg), URLs: make(refCountedUrlSet), resolver: opts.Gateway.resolver, runknown: opts.Gateway.RejectUnknown, oldHash: getOldHash(opts.Gateway.Name), } gateway.Lock() defer gateway.Unlock() gateway.sIDHash = getGWHash(s.info.ID) clusterHash := getGWHash(opts.Gateway.Name) prefix := make([]byte, 0, gwSubjectOffset) prefix = append(prefix, gwReplyPrefix...) prefix = append(prefix, clusterHash...) prefix = append(prefix, '.') prefix = append(prefix, gateway.sIDHash...) prefix = append(prefix, '.') gateway.replyPfx = prefix prefix = make([]byte, 0, oldGWReplyStart) prefix = append(prefix, oldGWReplyPrefix...) prefix = append(prefix, gateway.oldHash...) prefix = append(prefix, '.') gateway.oldReplyPfx = prefix gateway.pasi.m = make(map[string]map[string]*sitally) if gateway.resolver == nil { gateway.resolver = netResolver(net.DefaultResolver) } // Create remote gateways for _, rgo := range opts.Gateway.Gateways { // Ignore if there is a remote gateway with our name. if rgo.Name == gateway.name { gateway.ownCfgURLs = getURLsAsString(rgo.URLs) continue } cfg := &gatewayCfg{ RemoteGatewayOpts: rgo.clone(), hash: getGWHash(rgo.Name), oldHash: getOldHash(rgo.Name), urls: make(map[string]*url.URL, len(rgo.URLs)), } if opts.Gateway.TLSConfig != nil && cfg.TLSConfig == nil { cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() } if cfg.TLSTimeout == 0 { cfg.TLSTimeout = opts.Gateway.TLSTimeout } for _, u := range rgo.URLs { // For TLS, look for a hostname that we can use for TLSConfig.ServerName cfg.saveTLSHostname(u) cfg.urls[u.Host] = u } gateway.remotes[cfg.Name] = cfg } gateway.sqbsz = opts.Gateway.sendQSubsBufSize if gateway.sqbsz == 0 { gateway.sqbsz = maxBufSize } gateway.recSubExp = defaultGatewayRecentSubExpiration gateway.enabled = opts.Gateway.Name != "" && opts.Gateway.Port != 0 s.gateway = gateway return nil } // Update remote gateways TLS configurations after a config reload. func (g *srvGateway) updateRemotesTLSConfig(opts *Options) { g.Lock() defer g.Unlock() for _, ro := range opts.Gateway.Gateways { if ro.Name == g.name { continue } if cfg, ok := g.remotes[ro.Name]; ok { cfg.Lock() // If TLS config is in remote, use that one, otherwise, // use the TLS config from the main block. if ro.TLSConfig != nil { cfg.TLSConfig = ro.TLSConfig.Clone() } else if opts.Gateway.TLSConfig != nil { cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() } // Ensure that OCSP callbacks are always setup after a reload if needed. mustStaple := opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways if mustStaple && opts.Gateway.TLSConfig != nil { clientCB := opts.Gateway.TLSConfig.GetClientCertificate verifyCB := opts.Gateway.TLSConfig.VerifyConnection if mustStaple && cfg.TLSConfig != nil { if clientCB != nil && cfg.TLSConfig.GetClientCertificate == nil { cfg.TLSConfig.GetClientCertificate = clientCB } if verifyCB != nil && cfg.TLSConfig.VerifyConnection == nil { cfg.TLSConfig.VerifyConnection = verifyCB } } } cfg.Unlock() } } } // Returns if this server rejects connections from gateways that are not // explicitly configured. func (g *srvGateway) rejectUnknown() bool { g.RLock() reject := g.runknown g.RUnlock() return reject } // Starts the gateways accept loop and solicit explicit gateways // after an initial delay. This delay is meant to give a chance to // the cluster to form and this server gathers gateway URLs for this // cluster in order to send that as part of the connect/info process. func (s *Server) startGateways() { s.startGatewayAcceptLoop() // Delay start of creation of gateways to give a chance // to the local cluster to form. s.startGoRoutine(func() { defer s.grWG.Done() dur := s.getOpts().gatewaysSolicitDelay if dur == 0 { dur = time.Duration(atomic.LoadInt64(&gatewaySolicitDelay)) } select { case <-time.After(dur): s.solicitGateways() case <-s.quitCh: return } }) } // This starts the gateway accept loop in a go routine, unless it // is detected that the server has already been shutdown. func (s *Server) startGatewayAcceptLoop() { if s.isShuttingDown() { return } // Snapshot server options. opts := s.getOpts() port := opts.Gateway.Port if port == -1 { port = 0 } s.mu.Lock() hp := net.JoinHostPort(opts.Gateway.Host, strconv.Itoa(port)) l, e := natsListen("tcp", hp) s.gatewayListenerErr = e if e != nil { s.mu.Unlock() s.Fatalf("Error listening on gateway port: %d - %v", opts.Gateway.Port, e) return } s.Noticef("Gateway name is %s", s.getGatewayName()) s.Noticef("Listening for gateways connections on %s", net.JoinHostPort(opts.Gateway.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) tlsReq := opts.Gateway.TLSConfig != nil authRequired := opts.Gateway.Username != "" info := &Info{ ID: s.info.ID, Name: opts.ServerName, Version: s.info.Version, AuthRequired: authRequired, TLSRequired: tlsReq, TLSVerify: tlsReq, MaxPayload: s.info.MaxPayload, Gateway: opts.Gateway.Name, GatewayNRP: true, Headers: s.supportsHeaders(), } // Unless in some tests we want to keep the old behavior, we are now // (since v2.9.0) indicate that this server will switch all accounts // to InterestOnly mode when accepting an inbound or when a new // account is fetched. if !gwDoNotForceInterestOnlyMode { info.GatewayIOM = true } // If we have selected a random port... if port == 0 { // Write resolved port back to options. opts.Gateway.Port = l.Addr().(*net.TCPAddr).Port } // Possibly override Host/Port based on Gateway.Advertise if err := s.setGatewayInfoHostPort(info, opts); err != nil { s.Fatalf("Error setting gateway INFO with Gateway.Advertise value of %s, err=%v", opts.Gateway.Advertise, err) l.Close() s.mu.Unlock() return } // Setup state that can enable shutdown s.gatewayListener = l // Warn if insecure is configured in the main Gateway configuration // or any of the RemoteGateway's. This means that we need to check // remotes even if TLS would not be configured for the accept. warn := tlsReq && opts.Gateway.TLSConfig.InsecureSkipVerify if !warn { for _, g := range opts.Gateway.Gateways { if g.TLSConfig != nil && g.TLSConfig.InsecureSkipVerify { warn = true break } } } if warn { s.Warnf(gatewayTLSInsecureWarning) } go s.acceptConnections(l, "Gateway", func(conn net.Conn) { s.createGateway(nil, nil, conn) }, nil) s.mu.Unlock() } // Similar to setInfoHostPortAndGenerateJSON, but for gatewayInfo. func (s *Server) setGatewayInfoHostPort(info *Info, o *Options) error { gw := s.gateway gw.Lock() defer gw.Unlock() gw.URLs.removeUrl(gw.URL) if o.Gateway.Advertise != "" { advHost, advPort, err := parseHostPort(o.Gateway.Advertise, o.Gateway.Port) if err != nil { return err } info.Host = advHost info.Port = advPort } else { info.Host = o.Gateway.Host info.Port = o.Gateway.Port // If the host is "0.0.0.0" or "::" we need to resolve to a public IP. // This will return at most 1 IP. hostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(info.Host, false) if err != nil { return err } if hostIsIPAny { if len(ips) == 0 { // TODO(ik): Should we fail here (prevent starting)? If not, we // are going to "advertise" the 0.0.0.0: url, which means // that remote are going to try to connect to 0.0.0.0:, // which means a connect to loopback address, which is going // to fail with either TLS error, conn refused if the remote // is using different gateway port than this one, or error // saying that it tried to connect to itself. s.Errorf("Could not find any non-local IP for gateway %q with listen specification %q", gw.name, info.Host) } else { // Take the first from the list... info.Host = ips[0] } } } gw.URL = net.JoinHostPort(info.Host, strconv.Itoa(info.Port)) if o.Gateway.Advertise != "" { s.Noticef("Advertise address for gateway %q is set to %s", gw.name, gw.URL) } else { s.Noticef("Address for gateway %q is %s", gw.name, gw.URL) } gw.URLs[gw.URL]++ gw.info = info info.GatewayURL = gw.URL // (re)generate the gatewayInfoJSON byte array gw.generateInfoJSON() return nil } // Generates the Gateway INFO protocol. // The gateway lock is held on entry func (g *srvGateway) generateInfoJSON() { // We could be here when processing a route INFO that has a gateway URL, // but this server is not configured for gateways, so simply ignore here. // The configuration mismatch is reported somewhere else. if !g.enabled || g.info == nil { return } g.info.GatewayURLs = g.URLs.getAsStringSlice() b, err := json.Marshal(g.info) if err != nil { panic(err) } g.infoJSON = []byte(fmt.Sprintf(InfoProto, b)) } // Goes through the list of registered gateways and try to connect to those. // The list (remotes) is initially containing the explicit remote gateways, // but the list is augmented with any implicit (discovered) gateway. Therefore, // this function only solicit explicit ones. func (s *Server) solicitGateways() { gw := s.gateway gw.RLock() defer gw.RUnlock() for _, cfg := range gw.remotes { // Since we delay the creation of gateways, it is // possible that server starts to receive inbound from // other clusters and in turn create outbounds. So here // we create only the ones that are configured. if !cfg.isImplicit() { cfg := cfg // Create new instance for the goroutine. s.startGoRoutine(func() { s.solicitGateway(cfg, true) s.grWG.Done() }) } } } // Reconnect to the gateway after a little wait period. For explicit // gateways, we also wait for the default reconnect time. func (s *Server) reconnectGateway(cfg *gatewayCfg) { defer s.grWG.Done() delay := time.Duration(rand.Intn(100)) * time.Millisecond if !cfg.isImplicit() { delay += gatewayReconnectDelay } select { case <-time.After(delay): case <-s.quitCh: return } s.solicitGateway(cfg, false) } // This function will loop trying to connect to any URL attached // to the given Gateway. It will return once a connection has been created. func (s *Server) solicitGateway(cfg *gatewayCfg, firstConnect bool) { var ( opts = s.getOpts() isImplicit = cfg.isImplicit() attempts int typeStr string ) if isImplicit { typeStr = "implicit" } else { typeStr = "explicit" } const connFmt = "Connecting to %s gateway %q (%s) at %s (attempt %v)" const connErrFmt = "Error connecting to %s gateway %q (%s) at %s (attempt %v): %v" for s.isRunning() { urls := cfg.getURLs() if len(urls) == 0 { break } attempts++ report := s.shouldReportConnectErr(firstConnect, attempts) // Iteration is random for _, u := range urls { address, err := s.getRandomIP(s.gateway.resolver, u.Host, nil) if err != nil { s.Errorf("Error getting IP for %s gateway %q (%s): %v", typeStr, cfg.Name, u.Host, err) continue } if report { s.Noticef(connFmt, typeStr, cfg.Name, u.Host, address, attempts) } else { s.Debugf(connFmt, typeStr, cfg.Name, u.Host, address, attempts) } conn, err := natsDialTimeout("tcp", address, DEFAULT_ROUTE_DIAL) if err == nil { // We could connect, create the gateway connection and return. s.createGateway(cfg, u, conn) return } if report { s.Errorf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err) } else { s.Debugf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err) } // Break this loop if server is being shutdown... if !s.isRunning() { break } } if isImplicit { if opts.Gateway.ConnectRetries == 0 || attempts > opts.Gateway.ConnectRetries { s.gateway.Lock() // We could have just accepted an inbound for this remote gateway. // So if there is an inbound, let's try again to connect. if s.gateway.hasInbound(cfg.Name) { s.gateway.Unlock() continue } delete(s.gateway.remotes, cfg.Name) s.gateway.Unlock() return } } select { case <-s.quitCh: return case <-time.After(gatewayConnectDelay): continue } } } // Returns true if there is an inbound for the given `name`. // Lock held on entry. func (g *srvGateway) hasInbound(name string) bool { for _, ig := range g.in { ig.mu.Lock() igname := ig.gw.name ig.mu.Unlock() if igname == name { return true } } return false } // Called when a gateway connection is either accepted or solicited. // If accepted, the gateway is marked as inbound. // If solicited, the gateway is marked as outbound. func (s *Server) createGateway(cfg *gatewayCfg, url *url.URL, conn net.Conn) { // Snapshot server options. opts := s.getOpts() now := time.Now() c := &client{srv: s, nc: conn, start: now, last: now, kind: GATEWAY} // Are we creating the gateway based on the configuration solicit := cfg != nil var tlsRequired bool s.gateway.RLock() infoJSON := s.gateway.infoJSON s.gateway.RUnlock() // Perform some initialization under the client lock c.mu.Lock() c.initClient() c.gw = &gateway{} if solicit { // This is an outbound gateway connection cfg.RLock() tlsRequired = cfg.TLSConfig != nil cfgName := cfg.Name cfg.RUnlock() c.gw.outbound = true c.gw.name = cfgName c.gw.cfg = cfg cfg.bumpConnAttempts() // Since we are delaying the connect until after receiving // the remote's INFO protocol, save the URL we need to connect to. c.gw.connectURL = url c.Noticef("Creating outbound gateway connection to %q", cfgName) } else { c.flags.set(expectConnect) // Inbound gateway connection c.Noticef("Processing inbound gateway connection") // Check if TLS is required for inbound GW connections. tlsRequired = opts.Gateway.TLSConfig != nil // We expect a CONNECT from the accepted connection. c.setAuthTimer(secondsToDuration(opts.Gateway.AuthTimeout)) } // Check for TLS if tlsRequired { var tlsConfig *tls.Config var tlsName string var timeout float64 if solicit { var ( mustStaple = opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways clientCB func(*tls.CertificateRequestInfo) (*tls.Certificate, error) verifyCB func(tls.ConnectionState) error ) // Snapshot callbacks for OCSP outside an ongoing reload which might be happening. if mustStaple { s.reloadMu.RLock() s.optsMu.RLock() clientCB = s.opts.Gateway.TLSConfig.GetClientCertificate verifyCB = s.opts.Gateway.TLSConfig.VerifyConnection s.optsMu.RUnlock() s.reloadMu.RUnlock() } cfg.RLock() tlsName = cfg.tlsName tlsConfig = cfg.TLSConfig.Clone() timeout = cfg.TLSTimeout // Ensure that OCSP callbacks are always setup on gateway reconnect when OCSP policy is set to always. if mustStaple { if clientCB != nil && tlsConfig.GetClientCertificate == nil { tlsConfig.GetClientCertificate = clientCB } if verifyCB != nil && tlsConfig.VerifyConnection == nil { tlsConfig.VerifyConnection = verifyCB } } cfg.RUnlock() } else { tlsConfig = opts.Gateway.TLSConfig timeout = opts.Gateway.TLSTimeout } // Perform (either server or client side) TLS handshake. if resetTLSName, err := c.doTLSHandshake("gateway", solicit, url, tlsConfig, tlsName, timeout, opts.Gateway.TLSPinnedCerts); err != nil { if resetTLSName { cfg.Lock() cfg.tlsName = _EMPTY_ cfg.Unlock() } c.mu.Unlock() return } } // Do final client initialization c.in.pacache = make(map[string]*perAccountCache) if solicit { // This is an outbound gateway connection c.gw.outsim = &sync.Map{} } else { // Inbound gateway connection c.gw.insim = make(map[string]*insie) } // Register in temp map for now until gateway properly registered // in out or in gateways. if !s.addToTempClients(c.cid, c) { c.mu.Unlock() c.closeConnection(ServerShutdown) return } // Only send if we accept a connection. Will send CONNECT+INFO as an // outbound only after processing peer's INFO protocol. if !solicit { c.enqueueProto(infoJSON) } // Spin up the read loop. s.startGoRoutine(func() { c.readLoop(nil) }) // Spin up the write loop. s.startGoRoutine(func() { c.writeLoop() }) if tlsRequired { c.Debugf("TLS handshake complete") cs := c.nc.(*tls.Conn).ConnectionState() c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tlsCipher(cs.CipherSuite)) } // For outbound, we can't set the normal ping timer yet since the other // side would fail with a parse error should it receive anything but the // CONNECT protocol as the first protocol. We still want to make sure // that the connection is not stale until the first INFO from the remote // is received. if solicit { c.watchForStaleConnection(adjustPingInterval(GATEWAY, opts.PingInterval), opts.MaxPingsOut) } c.mu.Unlock() // Announce ourselves again to new connections. if solicit && s.EventsEnabled() { s.sendStatszUpdate() } } // Builds and sends the CONNECT protocol for a gateway. // Client lock held on entry. func (c *client) sendGatewayConnect(opts *Options) { // FIXME: This can race with updateRemotesTLSConfig tlsRequired := c.gw.cfg.TLSConfig != nil url := c.gw.connectURL c.gw.connectURL = nil var user, pass string if userInfo := url.User; userInfo != nil { user = userInfo.Username() pass, _ = userInfo.Password() } else if opts != nil { user = opts.Gateway.Username pass = opts.Gateway.Password } cinfo := connectInfo{ Verbose: false, Pedantic: false, User: user, Pass: pass, TLS: tlsRequired, Name: c.srv.info.ID, Gateway: c.srv.gateway.name, } b, err := json.Marshal(cinfo) if err != nil { panic(err) } c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) } // Process the CONNECT protocol from a gateway connection. // Returns an error to the connection if the CONNECT is not from a gateway // (for instance a client or route connecting to the gateway port), or // if the destination does not match the gateway name of this server. // // func (c *client) processGatewayConnect(arg []byte) error { connect := &connectInfo{} if err := json.Unmarshal(arg, connect); err != nil { return err } // Coming from a client or a route, reject if connect.Gateway == "" { c.sendErrAndErr(ErrClientOrRouteConnectedToGatewayPort.Error()) c.closeConnection(WrongPort) return ErrClientOrRouteConnectedToGatewayPort } c.mu.Lock() s := c.srv c.mu.Unlock() // If we reject unknown gateways, make sure we have it configured, // otherwise return an error. if s.gateway.rejectUnknown() && s.getRemoteGateway(connect.Gateway) == nil { c.Errorf("Rejecting connection from gateway %q", connect.Gateway) c.sendErr(fmt.Sprintf("Connection to gateway %q rejected", s.getGatewayName())) c.closeConnection(WrongGateway) return ErrWrongGateway } c.mu.Lock() c.gw.connected = true // Set the Ping timer after sending connect and info. c.setFirstPingTimer() c.mu.Unlock() return nil } // Process the INFO protocol from a gateway connection. // // If the gateway connection is an outbound (this server initiated the connection), // this function checks that the incoming INFO contains the Gateway field. If empty, // it means that this is a response from an older server or that this server connected // to the wrong port. // The outbound gateway may also receive a gossip INFO protocol from the remote gateway, // indicating other gateways that the remote knows about. This server will try to connect // to those gateways (if not explicitly configured or already implicitly connected). // In both cases (explicit or implicit), the local cluster is notified about the existence // of this new gateway. This allows servers in the cluster to ensure that they have an // outbound connection to this gateway. // // For an inbound gateway, the gateway is simply registered and the info protocol // is saved to be used after processing the CONNECT. // // func (c *client) processGatewayInfo(info *Info) { var ( gwName string cfg *gatewayCfg ) c.mu.Lock() s := c.srv cid := c.cid // Check if this is the first INFO. (this call sets the flag if not already set). isFirstINFO := c.flags.setIfNotSet(infoReceived) isOutbound := c.gw.outbound if isOutbound { gwName = c.gw.name cfg = c.gw.cfg } else if isFirstINFO { c.gw.name = info.Gateway } if isFirstINFO { c.opts.Name = info.ID } c.mu.Unlock() // For an outbound connection... if isOutbound { // Check content of INFO for fields indicating that it comes from a gateway. // If we incorrectly connect to the wrong port (client or route), we won't // have the Gateway field set. if info.Gateway == "" { c.sendErrAndErr(fmt.Sprintf("Attempt to connect to gateway %q using wrong port", gwName)) c.closeConnection(WrongPort) return } // Check that the gateway name we got is what we expect if info.Gateway != gwName { // Unless this is the very first INFO, it may be ok if this is // a gossip request to connect to other gateways. if !isFirstINFO && info.GatewayCmd == gatewayCmdGossip { // If we are configured to reject unknown, do not attempt to // connect to one that we don't have configured. if s.gateway.rejectUnknown() && s.getRemoteGateway(info.Gateway) == nil { return } s.processImplicitGateway(info) return } // Otherwise, this is a failure... // We are reporting this error in the log... c.Errorf("Failing connection to gateway %q, remote gateway name is %q", gwName, info.Gateway) // ...and sending this back to the remote so that the error // makes more sense in the remote server's log. c.sendErr(fmt.Sprintf("Connection from %q rejected, wanted to connect to %q, this is %q", s.getGatewayName(), gwName, info.Gateway)) c.closeConnection(WrongGateway) return } // Check for duplicate server name with servers in our cluster if s.isDuplicateServerName(info.Name) { c.Errorf("Remote server has a duplicate name: %q", info.Name) c.closeConnection(DuplicateServerName) return } // Possibly add URLs that we get from the INFO protocol. if len(info.GatewayURLs) > 0 { cfg.updateURLs(info.GatewayURLs) } // If this is the first INFO, send our connect if isFirstINFO { s.gateway.RLock() infoJSON := s.gateway.infoJSON s.gateway.RUnlock() supportsHeaders := s.supportsHeaders() opts := s.getOpts() // Note, if we want to support NKeys, then we would get the nonce // from this INFO protocol and can sign it in the CONNECT we are // going to send now. c.mu.Lock() c.gw.interestOnlyMode = info.GatewayIOM c.sendGatewayConnect(opts) c.Debugf("Gateway connect protocol sent to %q", gwName) // Send INFO too c.enqueueProto(infoJSON) c.gw.useOldPrefix = !info.GatewayNRP c.headers = supportsHeaders && info.Headers c.mu.Unlock() // Register as an outbound gateway.. if we had a protocol to ack our connect, // then we should do that when process that ack. if s.registerOutboundGatewayConnection(gwName, c) { c.Noticef("Outbound gateway connection to %q (%s) registered", gwName, info.ID) // Now that the outbound gateway is registered, we can remove from temp map. s.removeFromTempClients(cid) // Set the Ping timer after sending connect and info. c.mu.Lock() c.setFirstPingTimer() c.mu.Unlock() } else { // There was a bug that would cause a connection to possibly // be called twice resulting in reconnection of twice the // same outbound connection. The issue is fixed, but adding // defensive code above that if we did not register this connection // because we already have an outbound for this name, then // close this connection (and make sure it does not try to reconnect) c.mu.Lock() c.flags.set(noReconnect) c.mu.Unlock() c.closeConnection(WrongGateway) return } } else if info.GatewayCmd > 0 { switch info.GatewayCmd { case gatewayCmdAllSubsStart: c.gatewayAllSubsReceiveStart(info) return case gatewayCmdAllSubsComplete: c.gatewayAllSubsReceiveComplete(info) return default: s.Warnf("Received unknown command %v from gateway %q", info.GatewayCmd, gwName) return } } // Flood local cluster with information about this gateway. // Servers in this cluster will ensure that they have (or otherwise create) // an outbound connection to this gateway. s.forwardNewGatewayToLocalCluster(info) } else if isFirstINFO { // This is the first INFO of an inbound connection... // Check for duplicate server name with servers in our cluster if s.isDuplicateServerName(info.Name) { c.Errorf("Remote server has a duplicate name: %q", info.Name) c.closeConnection(DuplicateServerName) return } s.registerInboundGatewayConnection(cid, c) c.Noticef("Inbound gateway connection from %q (%s) registered", info.Gateway, info.ID) // Now that it is registered, we can remove from temp map. s.removeFromTempClients(cid) // Send our QSubs. s.sendQueueSubsToGateway(c) // Initiate outbound connection. This function will behave correctly if // we have already one. s.processImplicitGateway(info) // Send back to the server that initiated this gateway connection the // list of all remote gateways known on this server. s.gossipGatewaysToInboundGateway(info.Gateway, c) // Now make sure if we have any knowledge of connected leafnodes that we resend the // connect events to switch those accounts into interest only mode. s.mu.Lock() s.ensureGWsInterestOnlyForLeafNodes() s.mu.Unlock() js := s.js.Load() // If running in some tests, maintain the original behavior. if gwDoNotForceInterestOnlyMode && js != nil { // Switch JetStream accounts to interest-only mode. var accounts []string js.mu.Lock() if len(js.accounts) > 0 { accounts = make([]string, 0, len(js.accounts)) for accName := range js.accounts { accounts = append(accounts, accName) } } js.mu.Unlock() for _, accName := range accounts { if acc, err := s.LookupAccount(accName); err == nil && acc != nil { if acc.JetStreamEnabled() { s.switchAccountToInterestMode(acc.GetName()) } } } } else if !gwDoNotForceInterestOnlyMode { // Starting 2.9.0, we are phasing out the optimistic mode, so change // all accounts to interest-only mode, unless instructed not to do so // in some tests. s.accounts.Range(func(_, v any) bool { acc := v.(*Account) s.switchAccountToInterestMode(acc.GetName()) return true }) } } } // Sends to the given inbound gateway connection a gossip INFO protocol // for each gateway known by this server. This allows for a "full mesh" // of gateways. func (s *Server) gossipGatewaysToInboundGateway(gwName string, c *client) { gw := s.gateway gw.RLock() defer gw.RUnlock() for gwCfgName, cfg := range gw.remotes { // Skip the gateway that we just created if gwCfgName == gwName { continue } info := Info{ ID: s.info.ID, GatewayCmd: gatewayCmdGossip, } urls := cfg.getURLsAsStrings() if len(urls) > 0 { info.Gateway = gwCfgName info.GatewayURLs = urls b, _ := json.Marshal(&info) c.mu.Lock() c.enqueueProto([]byte(fmt.Sprintf(InfoProto, b))) c.mu.Unlock() } } } // Sends the INFO protocol of a gateway to all routes known by this server. func (s *Server) forwardNewGatewayToLocalCluster(oinfo *Info) { // Need to protect s.routes here, so use server's lock s.mu.Lock() defer s.mu.Unlock() // We don't really need the ID to be set, but, we need to make sure // that it is not set to the server ID so that if we were to connect // to an older server that does not expect a "gateway" INFO, it // would think that it needs to create an implicit route (since info.ID // would not match the route's remoteID), but will fail to do so because // the sent protocol will not have host/port defined. info := &Info{ ID: "GW" + s.info.ID, Name: s.getOpts().ServerName, Gateway: oinfo.Gateway, GatewayURLs: oinfo.GatewayURLs, GatewayCmd: gatewayCmdGossip, } b, _ := json.Marshal(info) infoJSON := []byte(fmt.Sprintf(InfoProto, b)) s.forEachRemote(func(r *client) { r.mu.Lock() r.enqueueProto(infoJSON) r.mu.Unlock() }) } // Sends queue subscriptions interest to remote gateway. // This is sent from the inbound side, that is, the side that receives // messages from the remote's outbound connection. This side is // the one sending the subscription interest. func (s *Server) sendQueueSubsToGateway(c *client) { s.sendSubsToGateway(c, _EMPTY_) } // Sends all subscriptions for the given account to the remove gateway // This is sent from the inbound side, that is, the side that receives // messages from the remote's outbound connection. This side is // the one sending the subscription interest. func (s *Server) sendAccountSubsToGateway(c *client, accName string) { s.sendSubsToGateway(c, accName) } func gwBuildSubProto(buf *bytes.Buffer, accName string, acc map[string]*sitally, doQueues bool) { for saq, si := range acc { if doQueues && si.q || !doQueues && !si.q { buf.Write(rSubBytes) buf.WriteString(accName) buf.WriteByte(' ') // For queue subs (si.q is true), saq will be // subject + ' ' + queue, for plain subs, this is // just the subject. buf.WriteString(saq) if doQueues { buf.WriteString(" 1") } buf.WriteString(CR_LF) } } } // Sends subscriptions to remote gateway. func (s *Server) sendSubsToGateway(c *client, accountName string) { var ( bufa = [32 * 1024]byte{} bbuf = bytes.NewBuffer(bufa[:0]) ) gw := s.gateway // This needs to run under this lock for the whole duration gw.pasi.Lock() defer gw.pasi.Unlock() // If account is specified... if accountName != _EMPTY_ { // Simply send all plain subs (no queues) for this specific account gwBuildSubProto(bbuf, accountName, gw.pasi.m[accountName], false) // Instruct to send all subs (RS+/-) for this account from now on. c.mu.Lock() e := c.gw.insim[accountName] if e == nil { e = &insie{} c.gw.insim[accountName] = e } e.mode = InterestOnly c.mu.Unlock() } else { // Send queues for all accounts for accName, acc := range gw.pasi.m { gwBuildSubProto(bbuf, accName, acc, true) } } buf := bbuf.Bytes() // Nothing to send. if len(buf) == 0 { return } if len(buf) > cap(bufa) { s.Debugf("Sending subscriptions to %q, buffer size: %v", c.gw.name, len(buf)) } // Send c.mu.Lock() c.enqueueProto(buf) c.Debugf("Sent queue subscriptions to gateway") c.mu.Unlock() } // This is invoked when getting an INFO protocol for gateway on the ROUTER port. // This function will then execute appropriate function based on the command // contained in the protocol. // func (s *Server) processGatewayInfoFromRoute(info *Info, routeSrvID string) { switch info.GatewayCmd { case gatewayCmdGossip: s.processImplicitGateway(info) default: s.Errorf("Unknown command %d from server %v", info.GatewayCmd, routeSrvID) } } // Sends INFO protocols to the given route connection for each known Gateway. // These will be processed by the route and delegated to the gateway code to // invoke processImplicitGateway. func (s *Server) sendGatewayConfigsToRoute(route *client) { gw := s.gateway gw.RLock() // Send only to gateways for which we have actual outbound connection to. if len(gw.out) == 0 { gw.RUnlock() return } // Collect gateway configs for which we have an outbound connection. gwCfgsa := [16]*gatewayCfg{} gwCfgs := gwCfgsa[:0] for _, c := range gw.out { c.mu.Lock() if c.gw.cfg != nil { gwCfgs = append(gwCfgs, c.gw.cfg) } c.mu.Unlock() } gw.RUnlock() if len(gwCfgs) == 0 { return } // Check forwardNewGatewayToLocalCluster() as to why we set ID this way. info := Info{ ID: "GW" + s.info.ID, GatewayCmd: gatewayCmdGossip, } for _, cfg := range gwCfgs { urls := cfg.getURLsAsStrings() if len(urls) > 0 { info.Gateway = cfg.Name info.GatewayURLs = urls b, _ := json.Marshal(&info) route.mu.Lock() route.enqueueProto([]byte(fmt.Sprintf(InfoProto, b))) route.mu.Unlock() } } } // Initiates a gateway connection using the info contained in the INFO protocol. // If a gateway with the same name is already registered (either because explicitly // configured, or already implicitly connected), this function will augmment the // remote URLs with URLs present in the info protocol and return. // Otherwise, this function will register this remote (to prevent multiple connections // to the same remote) and call solicitGateway (which will run in a different go-routine). func (s *Server) processImplicitGateway(info *Info) { s.gateway.Lock() defer s.gateway.Unlock() // Name of the gateway to connect to is the Info.Gateway field. gwName := info.Gateway // If this is our name, bail. if gwName == s.gateway.name { return } // Check if we already have this config, and if so, we are done cfg := s.gateway.remotes[gwName] if cfg != nil { // However, possibly augment the list of URLs with the given // info.GatewayURLs content. cfg.Lock() cfg.addURLs(info.GatewayURLs) cfg.Unlock() return } opts := s.getOpts() cfg = &gatewayCfg{ RemoteGatewayOpts: &RemoteGatewayOpts{Name: gwName}, hash: getGWHash(gwName), oldHash: getOldHash(gwName), urls: make(map[string]*url.URL, len(info.GatewayURLs)), implicit: true, } if opts.Gateway.TLSConfig != nil { cfg.TLSConfig = opts.Gateway.TLSConfig.Clone() cfg.TLSTimeout = opts.Gateway.TLSTimeout } // Since we know we don't have URLs (no config, so just based on what we // get from INFO), directly call addURLs(). We don't need locking since // we just created that structure and no one else has access to it yet. cfg.addURLs(info.GatewayURLs) // If there is no URL, we can't proceed. if len(cfg.urls) == 0 { return } s.gateway.remotes[gwName] = cfg s.startGoRoutine(func() { s.solicitGateway(cfg, true) s.grWG.Done() }) } // NumOutboundGateways is public here mostly for testing. func (s *Server) NumOutboundGateways() int { return s.numOutboundGateways() } // Returns the number of outbound gateway connections func (s *Server) numOutboundGateways() int { s.gateway.RLock() n := len(s.gateway.out) s.gateway.RUnlock() return n } // Returns the number of inbound gateway connections func (s *Server) numInboundGateways() int { s.gateway.RLock() n := len(s.gateway.in) s.gateway.RUnlock() return n } // Returns the remoteGateway (if any) that has the given `name` func (s *Server) getRemoteGateway(name string) *gatewayCfg { s.gateway.RLock() cfg := s.gateway.remotes[name] s.gateway.RUnlock() return cfg } // Used in tests func (g *gatewayCfg) bumpConnAttempts() { g.Lock() g.connAttempts++ g.Unlock() } // Used in tests func (g *gatewayCfg) getConnAttempts() int { g.Lock() ca := g.connAttempts g.Unlock() return ca } // Used in tests func (g *gatewayCfg) resetConnAttempts() { g.Lock() g.connAttempts = 0 g.Unlock() } // Returns if this remote gateway is implicit or not. func (g *gatewayCfg) isImplicit() bool { g.RLock() ii := g.implicit g.RUnlock() return ii } // getURLs returns an array of URLs in random order suitable for // an iteration to try to connect. func (g *gatewayCfg) getURLs() []*url.URL { g.RLock() a := make([]*url.URL, 0, len(g.urls)) for _, u := range g.urls { a = append(a, u) } g.RUnlock() // Map iteration is random, but not that good with small maps. rand.Shuffle(len(a), func(i, j int) { a[i], a[j] = a[j], a[i] }) return a } // Similar to getURLs but returns the urls as an array of strings. func (g *gatewayCfg) getURLsAsStrings() []string { g.RLock() a := make([]string, 0, len(g.urls)) for _, u := range g.urls { a = append(a, u.Host) } g.RUnlock() return a } // updateURLs creates the urls map with the content of the config's URLs array // and the given array that we get from the INFO protocol. func (g *gatewayCfg) updateURLs(infoURLs []string) { g.Lock() // Clear the map... g.urls = make(map[string]*url.URL, len(g.URLs)+len(infoURLs)) // Add the urls from the config URLs array. for _, u := range g.URLs { g.urls[u.Host] = u } // Then add the ones from the infoURLs array we got. g.addURLs(infoURLs) // The call above will set varzUpdateURLs only when finding ULRs in infoURLs // that are not present in the config. That does not cover the case where // previously "discovered" URLs are now gone. We could check "before" size // of g.urls and if bigger than current size, set the boolean to true. // Not worth it... simply set this to true to allow a refresh of gateway // URLs in varz. g.varzUpdateURLs = true g.Unlock() } // Saves the hostname of the given URL (if not already done). // This may be used as the ServerName of the TLSConfig when initiating a // TLS connection. // Write lock held on entry. func (g *gatewayCfg) saveTLSHostname(u *url.URL) { if g.TLSConfig != nil && g.tlsName == "" && net.ParseIP(u.Hostname()) == nil { g.tlsName = u.Hostname() } } // add URLs from the given array to the urls map only if not already present. // remoteGateway write lock is assumed to be held on entry. // Write lock is held on entry. func (g *gatewayCfg) addURLs(infoURLs []string) { var scheme string if g.TLSConfig != nil { scheme = "tls" } else { scheme = "nats" } for _, iu := range infoURLs { if _, present := g.urls[iu]; !present { // Urls in Info.GatewayURLs come without scheme. Add it to parse // the url (otherwise it fails). if u, err := url.Parse(fmt.Sprintf("%s://%s", scheme, iu)); err == nil { // Also, if a tlsName has not been set yet and we are dealing // with a hostname and not a bare IP, save the hostname. g.saveTLSHostname(u) // Use u.Host for the key. g.urls[u.Host] = u // Signal that we have updated the list. Used by monitoring code. g.varzUpdateURLs = true } } } } // Adds this URL to the set of Gateway URLs. // Returns true if the URL has been added, false otherwise. // Server lock held on entry func (s *Server) addGatewayURL(urlStr string) bool { s.gateway.Lock() added := s.gateway.URLs.addUrl(urlStr) if added { s.gateway.generateInfoJSON() } s.gateway.Unlock() return added } // Removes this URL from the set of gateway URLs. // Returns true if the URL has been removed, false otherwise. // Server lock held on entry func (s *Server) removeGatewayURL(urlStr string) bool { if s.isShuttingDown() { return false } s.gateway.Lock() removed := s.gateway.URLs.removeUrl(urlStr) if removed { s.gateway.generateInfoJSON() } s.gateway.Unlock() return removed } // Sends a Gateway's INFO to all inbound GW connections. // Server lock is held on entry func (s *Server) sendAsyncGatewayInfo() { s.gateway.RLock() for _, ig := range s.gateway.in { ig.mu.Lock() ig.enqueueProto(s.gateway.infoJSON) ig.mu.Unlock() } s.gateway.RUnlock() } // This returns the URL of the Gateway listen spec, or empty string // if the server has no gateway configured. func (s *Server) getGatewayURL() string { s.gateway.RLock() url := s.gateway.URL s.gateway.RUnlock() return url } // Returns this server gateway name. // Same than calling s.gateway.getName() func (s *Server) getGatewayName() string { // This is immutable return s.gateway.name } // All gateway connections (outbound and inbound) are put in the given map. func (s *Server) getAllGatewayConnections(conns map[uint64]*client) { gw := s.gateway gw.RLock() for _, c := range gw.out { c.mu.Lock() cid := c.cid c.mu.Unlock() conns[cid] = c } for cid, c := range gw.in { conns[cid] = c } gw.RUnlock() } // Register the given gateway connection (*client) in the inbound gateways // map. The key is the connection ID (like for clients and routes). func (s *Server) registerInboundGatewayConnection(cid uint64, gwc *client) { s.gateway.Lock() s.gateway.in[cid] = gwc s.gateway.Unlock() } // Register the given gateway connection (*client) in the outbound gateways // map with the given name as the key. func (s *Server) registerOutboundGatewayConnection(name string, gwc *client) bool { s.gateway.Lock() if _, exist := s.gateway.out[name]; exist { s.gateway.Unlock() return false } s.gateway.out[name] = gwc s.gateway.outo = append(s.gateway.outo, gwc) s.gateway.orderOutboundConnectionsLocked() s.gateway.Unlock() return true } // Returns the outbound gateway connection (*client) with the given name, // or nil if not found func (s *Server) getOutboundGatewayConnection(name string) *client { s.gateway.RLock() gwc := s.gateway.out[name] s.gateway.RUnlock() return gwc } // Returns all outbound gateway connections in the provided array. // The order of the gateways is suited for the sending of a message. // Current ordering is based on individual gateway's RTT value. func (s *Server) getOutboundGatewayConnections(a *[]*client) { s.gateway.RLock() for i := 0; i < len(s.gateway.outo); i++ { *a = append(*a, s.gateway.outo[i]) } s.gateway.RUnlock() } // Orders the array of outbound connections. // Current ordering is by lowest RTT. // Gateway write lock is held on entry func (g *srvGateway) orderOutboundConnectionsLocked() { // Order the gateways by lowest RTT slices.SortFunc(g.outo, func(i, j *client) int { return cmp.Compare(i.getRTTValue(), j.getRTTValue()) }) } // Orders the array of outbound connections. // Current ordering is by lowest RTT. func (g *srvGateway) orderOutboundConnections() { g.Lock() g.orderOutboundConnectionsLocked() g.Unlock() } // Returns all inbound gateway connections in the provided array func (s *Server) getInboundGatewayConnections(a *[]*client) { s.gateway.RLock() for _, gwc := range s.gateway.in { *a = append(*a, gwc) } s.gateway.RUnlock() } // This is invoked when a gateway connection is closed and the server // is removing this connection from its state. func (s *Server) removeRemoteGatewayConnection(c *client) { c.mu.Lock() cid := c.cid isOutbound := c.gw.outbound gwName := c.gw.name if isOutbound && c.gw.outsim != nil { // We do this to allow the GC to release this connection. // Since the map is used by the rest of the code without client lock, // we can't simply set it to nil, instead, just make sure we empty it. c.gw.outsim.Range(func(k, _ any) bool { c.gw.outsim.Delete(k) return true }) } c.mu.Unlock() gw := s.gateway gw.Lock() if isOutbound { delete(gw.out, gwName) louto := len(gw.outo) reorder := false for i := 0; i < len(gw.outo); i++ { if gw.outo[i] == c { // If last, simply remove and no need to reorder if i != louto-1 { gw.outo[i] = gw.outo[louto-1] reorder = true } gw.outo = gw.outo[:louto-1] } } if reorder { gw.orderOutboundConnectionsLocked() } } else { delete(gw.in, cid) } gw.Unlock() s.removeFromTempClients(cid) if isOutbound { // Update number of totalQSubs for this gateway qSubsRemoved := int64(0) c.mu.Lock() for _, sub := range c.subs { if sub.queue != nil { qSubsRemoved++ } } c.subs = nil c.mu.Unlock() // Update total count of qsubs in remote gateways. atomic.AddInt64(&c.srv.gateway.totalQSubs, -qSubsRemoved) } else { var subsa [1024]*subscription var subs = subsa[:0] // For inbound GW connection, if we have subs, those are // local subs on "_R_." subjects. c.mu.Lock() for _, sub := range c.subs { subs = append(subs, sub) } c.subs = nil c.mu.Unlock() for _, sub := range subs { c.removeReplySub(sub) } } } // GatewayAddr returns the net.Addr object for the gateway listener. func (s *Server) GatewayAddr() *net.TCPAddr { s.mu.Lock() defer s.mu.Unlock() if s.gatewayListener == nil { return nil } return s.gatewayListener.Addr().(*net.TCPAddr) } // A- protocol received from the remote after sending messages // on an account that it has no interest in. Mark this account // with a "no interest" marker to prevent further messages send. // func (c *client) processGatewayAccountUnsub(accName string) { // Just to indicate activity around "subscriptions" events. c.in.subs++ // This account may have an entry because of queue subs. // If that's the case, we can reset the no-interest map, // but not set the entry to nil. setToNil := true if ei, ok := c.gw.outsim.Load(accName); ei != nil { e := ei.(*outsie) e.Lock() // Reset the no-interest map if we have queue subs // and don't set the entry to nil. if e.qsubs > 0 { e.ni = make(map[string]struct{}) setToNil = false } e.Unlock() } else if ok { // Already set to nil, so skip setToNil = false } if setToNil { c.gw.outsim.Store(accName, nil) } } // A+ protocol received from remote gateway if it had previously // sent an A-. Clear the "no interest" marker for this account. // func (c *client) processGatewayAccountSub(accName string) error { // Just to indicate activity around "subscriptions" events. c.in.subs++ // If this account has an entry because of queue subs, we // can't delete the entry. remove := true if ei, ok := c.gw.outsim.Load(accName); ei != nil { e := ei.(*outsie) e.Lock() if e.qsubs > 0 { remove = false } e.Unlock() } else if !ok { // There is no entry, so skip remove = false } if remove { c.gw.outsim.Delete(accName) } return nil } // RS- protocol received from the remote after sending messages // on a subject that it has no interest in (but knows about the // account). Mark this subject with a "no interest" marker to // prevent further messages being sent. // If in modeInterestOnly or for a queue sub, remove from // the sublist if present. // func (c *client) processGatewayRUnsub(arg []byte) error { _, accName, subject, queue, err := c.parseUnsubProto(arg, true, false) if err != nil { return fmt.Errorf("processGatewaySubjectUnsub %s", err.Error()) } var ( e *outsie useSl bool newe bool callUpdate bool srv *Server sub *subscription ) // Possibly execute this on exit after all locks have been released. // If callUpdate is true, srv and sub will be not nil. defer func() { if callUpdate { srv.updateInterestForAccountOnGateway(accName, sub, -1) } }() c.mu.Lock() if c.gw.outsim == nil { c.Errorf("Received RS- from gateway on inbound connection") c.mu.Unlock() c.closeConnection(ProtocolViolation) return nil } defer c.mu.Unlock() // If closed, c.subs map will be nil, so bail out. if c.isClosed() { return nil } ei, _ := c.gw.outsim.Load(accName) if ei != nil { e = ei.(*outsie) e.Lock() defer e.Unlock() // If there is an entry, for plain sub we need // to know if we should store the sub useSl = queue != nil || e.mode != Optimistic } else if queue != nil { // should not even happen... c.Debugf("Received RS- without prior RS+ for subject %q, queue %q", subject, queue) return nil } else { // Plain sub, assume optimistic sends, create entry. e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()} newe = true } // This is when a sub or queue sub is supposed to be in // the sublist. Look for it and remove. if useSl { var ok bool key := arg // m[string()] does not cause mem allocation sub, ok = c.subs[string(key)] // if RS- for a sub that we don't have, just ignore. if !ok { return nil } if e.sl.Remove(sub) == nil { delete(c.subs, bytesToString(key)) if queue != nil { e.qsubs-- atomic.AddInt64(&c.srv.gateway.totalQSubs, -1) } // If last, we can remove the whole entry only // when in optimistic mode and there is no element // in the `ni` map. if e.sl.Count() == 0 && e.mode == Optimistic && len(e.ni) == 0 { c.gw.outsim.Delete(accName) } } // We are going to call updateInterestForAccountOnGateway on exit. srv = c.srv callUpdate = true } else { e.ni[string(subject)] = struct{}{} if newe { c.gw.outsim.Store(accName, e) } } return nil } // For plain subs, RS+ protocol received from remote gateway if it // had previously sent a RS-. Clear the "no interest" marker for // this subject (under this account). // For queue subs, or if in modeInterestOnly, register interest // from remote gateway. // func (c *client) processGatewayRSub(arg []byte) error { // Indicate activity. c.in.subs++ var ( queue []byte qw int32 ) args := splitArg(arg) switch len(args) { case 2: case 4: queue = args[2] qw = int32(parseSize(args[3])) default: return fmt.Errorf("processGatewaySubjectSub Parse Error: '%s'", arg) } accName := args[0] subject := args[1] var ( e *outsie useSl bool newe bool callUpdate bool srv *Server sub *subscription ) // Possibly execute this on exit after all locks have been released. // If callUpdate is true, srv and sub will be not nil. defer func() { if callUpdate { srv.updateInterestForAccountOnGateway(string(accName), sub, 1) } }() c.mu.Lock() if c.gw.outsim == nil { c.Errorf("Received RS+ from gateway on inbound connection") c.mu.Unlock() c.closeConnection(ProtocolViolation) return nil } defer c.mu.Unlock() // If closed, c.subs map will be nil, so bail out. if c.isClosed() { return nil } ei, _ := c.gw.outsim.Load(bytesToString(accName)) // We should always have an existing entry for plain subs because // in optimistic mode we would have received RS- first, and // in full knowledge, we are receiving RS+ for an account after // getting many RS- from the remote.. if ei != nil { e = ei.(*outsie) e.Lock() defer e.Unlock() useSl = queue != nil || e.mode != Optimistic } else if queue == nil { return nil } else { e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()} newe = true useSl = true } if useSl { var key []byte // We store remote subs by account/subject[/queue]. // For queue, remove the trailing weight if queue != nil { key = arg[:len(arg)-len(args[3])-1] } else { key = arg } // If RS+ for a sub that we already have, ignore. // (m[string()] does not allocate memory) if _, ok := c.subs[string(key)]; ok { return nil } // new subscription. copy subject (and queue) to // not reference the underlying possibly big buffer. var csubject []byte var cqueue []byte if queue != nil { // make single allocation and use different slices // to point to subject and queue name. cbuf := make([]byte, len(subject)+1+len(queue)) copy(cbuf, key[len(accName)+1:]) csubject = cbuf[:len(subject)] cqueue = cbuf[len(subject)+1:] } else { csubject = make([]byte, len(subject)) copy(csubject, subject) } sub = &subscription{client: c, subject: csubject, queue: cqueue, qw: qw} // If no error inserting in sublist... if e.sl.Insert(sub) == nil { c.subs[string(key)] = sub if queue != nil { e.qsubs++ atomic.AddInt64(&c.srv.gateway.totalQSubs, 1) } if newe { c.gw.outsim.Store(string(accName), e) } } // We are going to call updateInterestForAccountOnGateway on exit. srv = c.srv callUpdate = true } else { subj := bytesToString(subject) // If this is an RS+ for a wc subject, then // remove from the no interest map all subjects // that are a subset of this wc subject. if subjectHasWildcard(subj) { for k := range e.ni { if subjectIsSubsetMatch(k, subj) { delete(e.ni, k) } } } else { delete(e.ni, subj) } } return nil } // Returns true if this gateway has possible interest in the // given account/subject (which means, it does not have a registered // no-interest on the account and/or subject) and the sublist result // for queue subscriptions. // func (c *client) gatewayInterest(acc string, subj []byte) (bool, *SublistResult) { ei, accountInMap := c.gw.outsim.Load(acc) // If there is an entry for this account and ei is nil, // it means that the remote is not interested at all in // this account and we could not possibly have queue subs. if accountInMap && ei == nil { return false, nil } // Assume interest if account not in map, unless we support // only interest-only mode. psi := !accountInMap && !c.gw.interestOnlyMode var r *SublistResult if accountInMap { // If in map, check for subs interest with sublist. e := ei.(*outsie) e.RLock() // Unless each side has agreed on interest-only mode, // we may be in transition to modeInterestOnly // but until e.ni is nil, use it to know if we // should suppress interest or not. if !c.gw.interestOnlyMode && e.ni != nil { if _, inMap := e.ni[string(subj)]; !inMap { psi = true } } // If we are in modeInterestOnly (e.ni will be nil) // or if we have queue subs, we also need to check sl.Match. if e.ni == nil || e.qsubs > 0 { r = e.sl.MatchBytes(subj) if len(r.psubs) > 0 { psi = true } } e.RUnlock() // Since callers may just check if the sublist result is nil or not, // make sure that if what is returned by sl.Match() is the emptyResult, then // we return nil to the caller. if r == emptyResult { r = nil } } return psi, r } // switchAccountToInterestMode will switch an account over to interestMode. // Lock should NOT be held. func (s *Server) switchAccountToInterestMode(accName string) { gwsa := [16]*client{} gws := gwsa[:0] s.getInboundGatewayConnections(&gws) for _, gin := range gws { var e *insie var ok bool gin.mu.Lock() if e, ok = gin.gw.insim[accName]; !ok || e == nil { e = &insie{} gin.gw.insim[accName] = e } // Do it only if we are in Optimistic mode if e.mode == Optimistic { gin.gatewaySwitchAccountToSendAllSubs(e, accName) } gin.mu.Unlock() } } // This is invoked when registering (or unregistering) the first // (or last) subscription on a given account/subject. For each // GWs inbound connections, we will check if we need to send an RS+ or A+ // protocol. func (s *Server) maybeSendSubOrUnsubToGateways(accName string, sub *subscription, added bool) { if sub.queue != nil { return } gwsa := [16]*client{} gws := gwsa[:0] s.getInboundGatewayConnections(&gws) if len(gws) == 0 { return } var ( rsProtoa [512]byte rsProto []byte accProtoa [256]byte accProto []byte proto []byte subject = bytesToString(sub.subject) hasWC = subjectHasWildcard(subject) ) for _, c := range gws { proto = nil c.mu.Lock() e, inMap := c.gw.insim[accName] // If there is a inbound subject interest entry... if e != nil { sendProto := false // In optimistic mode, we care only about possibly sending RS+ (or A+) // so if e.ni is not nil we do things only when adding a new subscription. if e.ni != nil && added { // For wildcard subjects, we will remove from our no-interest // map, all subjects that are a subset of this wc subject, but we // still send the wc subject and let the remote do its own cleanup. if hasWC { for enis := range e.ni { if subjectIsSubsetMatch(enis, subject) { delete(e.ni, enis) sendProto = true } } } else if _, noInterest := e.ni[subject]; noInterest { delete(e.ni, subject) sendProto = true } } else if e.mode == InterestOnly { // We are in the mode where we always send RS+/- protocols. sendProto = true } if sendProto { if rsProto == nil { // Construct the RS+/- only once proto = rsProtoa[:0] if added { proto = append(proto, rSubBytes...) } else { proto = append(proto, rUnsubBytes...) } proto = append(proto, accName...) proto = append(proto, ' ') proto = append(proto, sub.subject...) proto = append(proto, CR_LF...) rsProto = proto } else { // Point to the already constructed RS+/- proto = rsProto } } } else if added && inMap { // Here, we have a `nil` entry for this account in // the map, which means that we have previously sent // an A-. We have a new subscription, so we need to // send an A+ and delete the entry from the map so // that we do this only once. delete(c.gw.insim, accName) if accProto == nil { // Construct the A+ only once proto = accProtoa[:0] proto = append(proto, aSubBytes...) proto = append(proto, accName...) proto = append(proto, CR_LF...) accProto = proto } else { // Point to the already constructed A+ proto = accProto } } if proto != nil { c.enqueueProto(proto) if c.trace { c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) } } c.mu.Unlock() } } // This is invoked when the first (or last) queue subscription on a // given subject/group is registered (or unregistered). Sent to all // inbound gateways. func (s *Server) sendQueueSubOrUnsubToGateways(accName string, qsub *subscription, added bool) { if qsub.queue == nil { return } gwsa := [16]*client{} gws := gwsa[:0] s.getInboundGatewayConnections(&gws) if len(gws) == 0 { return } var protoa [512]byte var proto []byte for _, c := range gws { if proto == nil { proto = protoa[:0] if added { proto = append(proto, rSubBytes...) } else { proto = append(proto, rUnsubBytes...) } proto = append(proto, accName...) proto = append(proto, ' ') proto = append(proto, qsub.subject...) proto = append(proto, ' ') proto = append(proto, qsub.queue...) if added { // For now, just use 1 for the weight proto = append(proto, ' ', '1') } proto = append(proto, CR_LF...) } c.mu.Lock() // If we add a queue sub, and we had previously sent an A-, // we don't need to send an A+ here, but we need to clear // the fact that we did sent the A- so that we don't send // an A+ when we will get the first non-queue sub registered. if added { if ei, ok := c.gw.insim[accName]; ok && ei == nil { delete(c.gw.insim, accName) } } c.enqueueProto(proto) if c.trace { c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) } c.mu.Unlock() } } // This is invoked when a subscription (plain or queue) is // added/removed locally or in our cluster. We use ref counting // to know when to update the inbound gateways. // func (s *Server) gatewayUpdateSubInterest(accName string, sub *subscription, change int32) { if sub.si { return } var ( keya [1024]byte key = keya[:0] entry *sitally isNew bool ) s.gateway.pasi.Lock() defer s.gateway.pasi.Unlock() accMap := s.gateway.pasi.m // First see if we have the account st := accMap[accName] if st == nil { // Ignore remove of something we don't have if change < 0 { return } st = make(map[string]*sitally) accMap[accName] = st isNew = true } // Lookup: build the key as subject[+' '+queue] key = append(key, sub.subject...) if sub.queue != nil { key = append(key, ' ') key = append(key, sub.queue...) } if !isNew { entry = st[string(key)] } first := false last := false if entry == nil { // Ignore remove of something we don't have if change < 0 { return } entry = &sitally{n: 1, q: sub.queue != nil} st[string(key)] = entry first = true } else { entry.n += change if entry.n <= 0 { delete(st, bytesToString(key)) last = true if len(st) == 0 { delete(accMap, accName) } } } if sub.client != nil { rsubs := &s.gateway.rsubs acc := sub.client.acc sli, _ := rsubs.Load(acc) if change > 0 { var sl *Sublist if sli == nil { sl = NewSublistNoCache() rsubs.Store(acc, sl) } else { sl = sli.(*Sublist) } sl.Insert(sub) time.AfterFunc(s.gateway.recSubExp, func() { sl.Remove(sub) }) } else if sli != nil { sl := sli.(*Sublist) sl.Remove(sub) if sl.Count() == 0 { rsubs.Delete(acc) } } } if first || last { if entry.q { s.sendQueueSubOrUnsubToGateways(accName, sub, first) } else { s.maybeSendSubOrUnsubToGateways(accName, sub, first) } } } // Returns true if the given subject is a GW routed reply subject, // that is, starts with $GNR and is long enough to contain cluster/server hash // and subject. func isGWRoutedReply(subj []byte) bool { return len(subj) > gwSubjectOffset && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix } // Same than isGWRoutedReply but accepts the old prefix $GR and returns // a boolean indicating if this is the old prefix func isGWRoutedSubjectAndIsOldPrefix(subj []byte) (bool, bool) { if isGWRoutedReply(subj) { return true, false } if len(subj) > oldGWReplyStart && bytesToString(subj[:oldGWReplyPrefixLen]) == oldGWReplyPrefix { return true, true } return false, false } // Returns true if subject starts with "$GNR.". This is to check that // clients can't publish on this subject. func hasGWRoutedReplyPrefix(subj []byte) bool { return len(subj) > gwReplyPrefixLen && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix } // Evaluates if the given reply should be mapped or not. func (g *srvGateway) shouldMapReplyForGatewaySend(acc *Account, reply []byte) bool { // If for this account there is a recent matching subscription interest // then we will map. sli, _ := g.rsubs.Load(acc) if sli == nil { return false } sl := sli.(*Sublist) if sl.Count() > 0 { if sl.HasInterest(string(reply)) { return true } } return false } var subPool = &sync.Pool{ New: func() any { return &subscription{} }, } // May send a message to all outbound gateways. It is possible // that the message is not sent to a given gateway if for instance // it is known that this gateway has no interest in the account or // subject, etc.. // When invoked from a LEAF connection, `checkLeafQF` should be passed as `true` // so that we skip any queue subscription interest that is not part of the // `c.pa.queues` filter (similar to what we do in `processMsgResults`). However, // when processing service imports, then this boolean should be passes as `false`, // regardless if it is a LEAF connection or not. // func (c *client) sendMsgToGateways(acc *Account, msg, subject, reply []byte, qgroups [][]byte, checkLeafQF bool) bool { // We had some times when we were sending across a GW with no subject, and the other side would break // due to parser error. These need to be fixed upstream but also double check here. if len(subject) == 0 { return false } gwsa := [16]*client{} gws := gwsa[:0] // This is in fast path, so avoid calling functions when possible. // Get the outbound connections in place instead of calling // getOutboundGatewayConnections(). srv := c.srv gw := srv.gateway gw.RLock() for i := 0; i < len(gw.outo); i++ { gws = append(gws, gw.outo[i]) } thisClusterReplyPrefix := gw.replyPfx thisClusterOldReplyPrefix := gw.oldReplyPfx gw.RUnlock() if len(gws) == 0 { return false } var ( queuesa = [512]byte{} queues = queuesa[:0] accName = acc.Name mreplya [256]byte mreply []byte dstHash []byte checkReply = len(reply) > 0 didDeliver bool prodIsMQTT = c.isMqtt() dlvMsgs int64 ) // Get a subscription from the pool sub := subPool.Get().(*subscription) // Check if the subject is on the reply prefix, if so, we // need to send that message directly to the origin cluster. directSend, old := isGWRoutedSubjectAndIsOldPrefix(subject) if directSend { if old { dstHash = subject[oldGWReplyPrefixLen : oldGWReplyStart-1] } else { dstHash = subject[gwClusterOffset : gwClusterOffset+gwHashLen] } } for i := 0; i < len(gws); i++ { gwc := gws[i] if directSend { gwc.mu.Lock() var ok bool if gwc.gw.cfg != nil { if old { ok = bytes.Equal(dstHash, gwc.gw.cfg.oldHash) } else { ok = bytes.Equal(dstHash, gwc.gw.cfg.hash) } } gwc.mu.Unlock() if !ok { continue } } else { // Plain sub interest and queue sub results for this account/subject psi, qr := gwc.gatewayInterest(accName, subject) if !psi && qr == nil { continue } queues = queuesa[:0] if qr != nil { for i := 0; i < len(qr.qsubs); i++ { qsubs := qr.qsubs[i] if len(qsubs) > 0 { queue := qsubs[0].queue if checkLeafQF { // Skip any queue that is not in the leaf's queue filter. skip := true for _, qn := range c.pa.queues { if bytes.Equal(queue, qn) { skip = false break } } if skip { continue } // Now we still need to check that it was not delivered // locally by checking the given `qgroups`. } add := true for _, qn := range qgroups { if bytes.Equal(queue, qn) { add = false break } } if add { qgroups = append(qgroups, queue) queues = append(queues, queue...) queues = append(queues, ' ') } } } } if !psi && len(queues) == 0 { continue } } if checkReply { // Check/map only once checkReply = false // Assume we will use original mreply = reply // Decide if we should map. if gw.shouldMapReplyForGatewaySend(acc, reply) { mreply = mreplya[:0] gwc.mu.Lock() useOldPrefix := gwc.gw.useOldPrefix gwc.mu.Unlock() if useOldPrefix { mreply = append(mreply, thisClusterOldReplyPrefix...) } else { mreply = append(mreply, thisClusterReplyPrefix...) } mreply = append(mreply, reply...) } } // Setup the message header. // Make sure we are an 'R' proto by default c.msgb[0] = 'R' mh := c.msgb[:msgHeadProtoLen] mh = append(mh, accName...) mh = append(mh, ' ') mh = append(mh, subject...) mh = append(mh, ' ') if len(queues) > 0 { if len(reply) > 0 { mh = append(mh, "+ "...) // Signal that there is a reply. mh = append(mh, mreply...) mh = append(mh, ' ') } else { mh = append(mh, "| "...) // Only queues } mh = append(mh, queues...) } else if len(reply) > 0 { mh = append(mh, mreply...) mh = append(mh, ' ') } // Headers hasHeader := c.pa.hdr > 0 canReceiveHeader := gwc.headers if hasHeader { if canReceiveHeader { mh[0] = 'H' mh = append(mh, c.pa.hdb...) mh = append(mh, ' ') mh = append(mh, c.pa.szb...) } else { // If we are here we need to truncate the payload size nsz := strconv.Itoa(c.pa.size - c.pa.hdr) mh = append(mh, nsz...) } } else { mh = append(mh, c.pa.szb...) } mh = append(mh, CR_LF...) // We reuse the subscription object that we pass to deliverMsg. // So set/reset important fields. sub.nm, sub.max = 0, 0 sub.client = gwc sub.subject = subject if c.deliverMsg(prodIsMQTT, sub, acc, subject, mreply, mh, msg, false) { // We don't count internal deliveries so count only if sub.icb is nil if sub.icb == nil { dlvMsgs++ } didDeliver = true } } if dlvMsgs > 0 { totalBytes := dlvMsgs * int64(len(msg)) // For non MQTT producers, remove the CR_LF * number of messages if !prodIsMQTT { totalBytes -= dlvMsgs * int64(LEN_CR_LF) } if acc != nil { atomic.AddInt64(&acc.outMsgs, dlvMsgs) atomic.AddInt64(&acc.outBytes, totalBytes) } atomic.AddInt64(&srv.outMsgs, dlvMsgs) atomic.AddInt64(&srv.outBytes, totalBytes) } // Done with subscription, put back to pool. We don't need // to reset content since we explicitly set when using it. // However, make sure to not hold a reference to a connection. sub.client = nil subPool.Put(sub) return didDeliver } // Possibly sends an A- to the remote gateway `c`. // Invoked when processing an inbound message and the account is not found. // A check under a lock that protects processing of SUBs and UNSUBs is // done to make sure that we don't send the A- if a subscription has just // been created at the same time, which would otherwise results in the // remote never sending messages on this account until a new subscription // is created. func (s *Server) gatewayHandleAccountNoInterest(c *client, accName []byte) { // Check and possibly send the A- under this lock. s.gateway.pasi.Lock() defer s.gateway.pasi.Unlock() si, inMap := s.gateway.pasi.m[string(accName)] if inMap && si != nil && len(si) > 0 { return } c.sendAccountUnsubToGateway(accName) } // Helper that sends an A- to this remote gateway if not already done. // This function should not be invoked directly but instead be invoked // by functions holding the gateway.pasi's Lock. func (c *client) sendAccountUnsubToGateway(accName []byte) { // Check if we have sent the A- or not. c.mu.Lock() e, sent := c.gw.insim[string(accName)] if e != nil || !sent { // Add a nil value to indicate that we have sent an A- // so that we know to send A+ when needed. c.gw.insim[string(accName)] = nil var protoa [256]byte proto := protoa[:0] proto = append(proto, aUnsubBytes...) proto = append(proto, accName...) proto = append(proto, CR_LF...) c.enqueueProto(proto) if c.trace { c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) } } c.mu.Unlock() } // Possibly sends an A- for this account or RS- for this subject. // Invoked when processing an inbound message and the account is found // but there is no interest on this subject. // A test is done under a lock that protects processing of SUBs and UNSUBs // and if there is no subscription at this time, we send an A-. If there // is at least a subscription, but no interest on this subject, we send // an RS- for this subject (if not already done). func (s *Server) gatewayHandleSubjectNoInterest(c *client, acc *Account, accName, subject []byte) { s.gateway.pasi.Lock() defer s.gateway.pasi.Unlock() // If there is no subscription for this account, we would normally // send an A-, however, if this account has the internal subscription // for service reply, send a specific RS- for the subject instead. // Need to grab the lock here since sublist can change during reload. acc.mu.RLock() hasSubs := acc.sl.Count() > 0 || acc.siReply != nil acc.mu.RUnlock() // If there is at least a subscription, possibly send RS- if hasSubs { sendProto := false c.mu.Lock() // Send an RS- protocol if not already done and only if // not in the modeInterestOnly. e := c.gw.insim[string(accName)] if e == nil { e = &insie{ni: make(map[string]struct{})} e.ni[string(subject)] = struct{}{} c.gw.insim[string(accName)] = e sendProto = true } else if e.ni != nil { // If we are not in modeInterestOnly, check if we // have already sent an RS- if _, alreadySent := e.ni[string(subject)]; !alreadySent { // TODO(ik): pick some threshold as to when // we need to switch mode if len(e.ni) >= gatewayMaxRUnsubBeforeSwitch { // If too many RS-, switch to all-subs-mode. c.gatewaySwitchAccountToSendAllSubs(e, string(accName)) } else { e.ni[string(subject)] = struct{}{} sendProto = true } } } if sendProto { var ( protoa = [512]byte{} proto = protoa[:0] ) proto = append(proto, rUnsubBytes...) proto = append(proto, accName...) proto = append(proto, ' ') proto = append(proto, subject...) proto = append(proto, CR_LF...) c.enqueueProto(proto) if c.trace { c.traceOutOp("", proto[:len(proto)-LEN_CR_LF]) } } c.mu.Unlock() } else { // There is not a single subscription, send an A- (if not already done). c.sendAccountUnsubToGateway([]byte(acc.Name)) } } // Returns the cluster hash from the gateway reply prefix func (g *srvGateway) getClusterHash() []byte { g.RLock() clusterHash := g.replyPfx[gwClusterOffset : gwClusterOffset+gwHashLen] g.RUnlock() return clusterHash } // Store this route in map with the key being the remote server's name hash // and the remote server's ID hash used by gateway replies mapping routing. func (s *Server) storeRouteByHash(srvIDHash string, c *client) { if !s.gateway.enabled { return } s.gateway.routesIDByHash.Store(srvIDHash, c) } // Remove the route with the given keys from the map. func (s *Server) removeRouteByHash(srvIDHash string) { if !s.gateway.enabled { return } s.gateway.routesIDByHash.Delete(srvIDHash) } // Returns the route with given hash or nil if not found. // This is for gateways only. func (s *Server) getRouteByHash(hash, accName []byte) (*client, bool) { id := bytesToString(hash) var perAccount bool if v, ok := s.accRouteByHash.Load(bytesToString(accName)); ok { if v == nil { id += bytesToString(accName) perAccount = true } else { id += strconv.Itoa(v.(int)) } } if v, ok := s.gateway.routesIDByHash.Load(id); ok { return v.(*client), perAccount } else if !perAccount { // Check if we have a "no pool" connection at index 0. if v, ok := s.gateway.routesIDByHash.Load(bytesToString(hash) + "0"); ok { if r := v.(*client); r != nil { r.mu.Lock() noPool := r.route.noPool r.mu.Unlock() if noPool { return r, false } } } } return nil, perAccount } // Returns the subject from the routed reply func getSubjectFromGWRoutedReply(reply []byte, isOldPrefix bool) []byte { if isOldPrefix { return reply[oldGWReplyStart:] } return reply[gwSubjectOffset:] } // This should be invoked only from processInboundGatewayMsg() or // processInboundRoutedMsg() and is checking if the subject // (c.pa.subject) has the _GR_ prefix. If so, this is processed // as a GW reply and `true` is returned to indicate to the caller // that it should stop processing. // If gateway is not enabled on this server or if the subject // does not start with _GR_, `false` is returned and caller should // process message as usual. func (c *client) handleGatewayReply(msg []byte) (processed bool) { // Do not handle GW prefixed messages if this server does not have // gateway enabled or if the subject does not start with the previx. if !c.srv.gateway.enabled { return false } isGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(c.pa.subject) if !isGWPrefix { return false } // Save original subject (in case we have to forward) orgSubject := c.pa.subject var clusterHash []byte var srvHash []byte var subject []byte if oldPrefix { clusterHash = c.pa.subject[oldGWReplyPrefixLen : oldGWReplyStart-1] // Check if this reply is intended for our cluster. if !bytes.Equal(clusterHash, c.srv.gateway.oldHash) { // We could report, for now, just drop. return true } subject = c.pa.subject[oldGWReplyStart:] } else { clusterHash = c.pa.subject[gwClusterOffset : gwClusterOffset+gwHashLen] // Check if this reply is intended for our cluster. if !bytes.Equal(clusterHash, c.srv.gateway.getClusterHash()) { // We could report, for now, just drop. return true } srvHash = c.pa.subject[gwServerOffset : gwServerOffset+gwHashLen] subject = c.pa.subject[gwSubjectOffset:] } var route *client var perAccount bool // If the origin is not this server, get the route this should be sent to. if c.kind == GATEWAY && srvHash != nil && !bytes.Equal(srvHash, c.srv.gateway.sIDHash) { route, perAccount = c.srv.getRouteByHash(srvHash, c.pa.account) // This will be possibly nil, and in this case we will try to process // the interest from this server. } // Adjust the subject c.pa.subject = subject // Use a stack buffer to rewrite c.pa.cache since we only need it for // getAccAndResultFromCache() var _pacache [256]byte pacache := _pacache[:0] // For routes that are dedicated to an account, do not put the account // name in the pacache. if c.kind == GATEWAY || (c.kind == ROUTER && c.route != nil && len(c.route.accName) == 0) { pacache = append(pacache, c.pa.account...) pacache = append(pacache, ' ') } pacache = append(pacache, c.pa.subject...) c.pa.pacache = pacache acc, r := c.getAccAndResultFromCache() if acc == nil { typeConn := "routed" if c.kind == GATEWAY { typeConn = "gateway" } c.Debugf("Unknown account %q for %s message on subject: %q", c.pa.account, typeConn, c.pa.subject) if c.kind == GATEWAY { c.srv.gatewayHandleAccountNoInterest(c, c.pa.account) } return true } // If route is nil, we will process the incoming message locally. if route == nil { // Check if this is a service reply subject (_R_) isServiceReply := isServiceReply(c.pa.subject) var queues [][]byte if len(r.psubs)+len(r.qsubs) > 0 { flags := pmrCollectQueueNames | pmrIgnoreEmptyQueueFilter // If this message came from a ROUTE, allow to pick queue subs // only if the message was directly sent by the "gateway" server // in our cluster that received it. if c.kind == ROUTER { flags |= pmrAllowSendFromRouteToRoute } _, queues = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flags) } // Since this was a reply that made it to the origin cluster, // we now need to send the message with the real subject to // gateways in case they have interest on that reply subject. if !isServiceReply { c.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, queues, false) } } else if c.kind == GATEWAY { // Only if we are a gateway connection should we try to route // to the server where the request originated. var bufa [256]byte var buf = bufa[:0] buf = append(buf, msgHeadProto...) if !perAccount { buf = append(buf, acc.Name...) buf = append(buf, ' ') } buf = append(buf, orgSubject...) buf = append(buf, ' ') if len(c.pa.reply) > 0 { buf = append(buf, c.pa.reply...) buf = append(buf, ' ') } szb := c.pa.szb if c.pa.hdr >= 0 { if route.headers { buf[0] = 'H' buf = append(buf, c.pa.hdb...) buf = append(buf, ' ') } else { szb = []byte(strconv.Itoa(c.pa.size - c.pa.hdr)) msg = msg[c.pa.hdr:] } } buf = append(buf, szb...) mhEnd := len(buf) buf = append(buf, _CRLF_...) buf = append(buf, msg...) route.mu.Lock() route.enqueueProto(buf) if route.trace { route.traceOutOp("", buf[:mhEnd]) } route.mu.Unlock() } return true } // Process a message coming from a remote gateway. Send to any sub/qsub // in our cluster that is matching. When receiving a message for an // account or subject for which there is no interest in this cluster // an A-/RS- protocol may be send back. // func (c *client) processInboundGatewayMsg(msg []byte) { // Update statistics c.in.msgs++ // The msg includes the CR_LF, so pull back out for accounting. c.in.bytes += int32(len(msg) - LEN_CR_LF) if c.opts.Verbose { c.sendOK() } // Mostly under testing scenarios. if c.srv == nil { return } // If the subject (c.pa.subject) has the gateway prefix, this function will // handle it. if c.handleGatewayReply(msg) { // We are done here. return } acc, r := c.getAccAndResultFromCache() if acc == nil { c.Debugf("Unknown account %q for gateway message on subject: %q", c.pa.account, c.pa.subject) c.srv.gatewayHandleAccountNoInterest(c, c.pa.account) return } // Check if this is a service reply subject (_R_) noInterest := len(r.psubs) == 0 checkNoInterest := true if acc.NumServiceImports() > 0 { if isServiceReply(c.pa.subject) { checkNoInterest = false } else { // We need to eliminate the subject interest from the service imports here to // make sure we send the proper no interest if the service import is the only interest. noInterest = true for _, sub := range r.psubs { // sub.si indicates that this is a subscription for service import, and is immutable. // So sub.si is false, then this is a subscription for something else, so there is // actually proper interest. if !sub.si { noInterest = false break } } } } if checkNoInterest && noInterest { // If there is no interest on plain subs, possibly send an RS-, // even if there is qsubs interest. c.srv.gatewayHandleSubjectNoInterest(c, acc, c.pa.account, c.pa.subject) // If there is also no queue filter, then no point in continuing // (even if r.qsubs i > 0). if len(c.pa.queues) == 0 { return } } c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag) } // Indicates that the remote which we are sending messages to // has decided to send us all its subs interest so that we // stop doing optimistic sends. // func (c *client) gatewayAllSubsReceiveStart(info *Info) { account := getAccountFromGatewayCommand(c, info, "start") if account == "" { return } c.Debugf("Gateway %q: switching account %q to %s mode", info.Gateway, account, InterestOnly) // Since the remote would send us this start command // only after sending us too many RS- for this account, // we should always have an entry here. // TODO(ik): Should we close connection with protocol violation // error if that happens? ei, _ := c.gw.outsim.Load(account) if ei != nil { e := ei.(*outsie) e.Lock() e.mode = Transitioning e.Unlock() } else { e := &outsie{sl: NewSublistWithCache()} e.mode = Transitioning c.mu.Lock() c.gw.outsim.Store(account, e) c.mu.Unlock() } } // Indicates that the remote has finished sending all its // subscriptions and we should now not send unless we know // there is explicit interest. // func (c *client) gatewayAllSubsReceiveComplete(info *Info) { account := getAccountFromGatewayCommand(c, info, "complete") if account == _EMPTY_ { return } // Done receiving all subs from remote. Set the `ni` // map to nil so that gatewayInterest() no longer // uses it. ei, _ := c.gw.outsim.Load(account) if ei != nil { e := ei.(*outsie) // Needs locking here since `ni` is checked by // many go-routines calling gatewayInterest() e.Lock() e.ni = nil e.mode = InterestOnly e.Unlock() c.Debugf("Gateway %q: switching account %q to %s mode complete", info.Gateway, account, InterestOnly) } } // small helper to get the account name from the INFO command. func getAccountFromGatewayCommand(c *client, info *Info, cmd string) string { if info.GatewayCmdPayload == nil { c.sendErrAndErr(fmt.Sprintf("Account absent from receive-all-subscriptions-%s command", cmd)) c.closeConnection(ProtocolViolation) return _EMPTY_ } return string(info.GatewayCmdPayload) } // Switch to send-all-subs mode for the given gateway and account. // This is invoked when processing an inbound message and we // reach a point where we had to send a lot of RS- for this // account. We will send an INFO protocol to indicate that we // start sending all our subs (for this account), followed by // all subs (RS+) and finally an INFO to indicate the end of it. // The remote will then send messages only if it finds explicit // interest in the sublist created based on all RS+ that we just // sent. // The client's lock is held on entry. // func (c *client) gatewaySwitchAccountToSendAllSubs(e *insie, accName string) { // Set this map to nil so that the no-interest is no longer checked. e.ni = nil // Switch mode to transitioning to prevent switchAccountToInterestMode // to possibly call this function multiple times. e.mode = Transitioning s := c.srv remoteGWName := c.gw.name c.Debugf("Gateway %q: switching account %q to %s mode", remoteGWName, accName, InterestOnly) // Function that will create an INFO protocol // and set proper command. sendCmd := func(cmd byte, useLock bool) { // Use bare server info and simply set the // gateway name and command info := Info{ Gateway: s.gateway.name, GatewayCmd: cmd, GatewayCmdPayload: stringToBytes(accName), } b, _ := json.Marshal(&info) infoJSON := []byte(fmt.Sprintf(InfoProto, b)) if useLock { c.mu.Lock() } c.enqueueProto(infoJSON) if useLock { c.mu.Unlock() } } // Send the start command. When remote receives this, // it may continue to send optimistic messages, but // it will start to register RS+/RS- in sublist instead // of noInterest map. sendCmd(gatewayCmdAllSubsStart, false) // Execute this in separate go-routine as to not block // the readLoop (which may cause the otherside to close // the connection due to slow consumer) s.startGoRoutine(func() { defer s.grWG.Done() s.sendAccountSubsToGateway(c, accName) // Send the complete command. When the remote receives // this, it will not send a message unless it has a // matching sub from us. sendCmd(gatewayCmdAllSubsComplete, true) c.Debugf("Gateway %q: switching account %q to %s mode complete", remoteGWName, accName, InterestOnly) }) } // Keeps track of the routed reply to be used when/if application sends back a // message on the reply without the prefix. // If `client` is not nil, it will be stored in the client gwReplyMapping structure, // and client lock is held on entry. // If `client` is nil, the mapping is stored in the client's account's gwReplyMapping // structure. Account lock will be explicitly acquired. // This is a server receiver because we use a timer interval that is avail in // Server.gateway object. func (s *Server) trackGWReply(c *client, acc *Account, reply, routedReply []byte) { var l sync.Locker var g *gwReplyMapping if acc != nil { acc.mu.Lock() defer acc.mu.Unlock() g = &acc.gwReplyMapping l = &acc.mu } else { g = &c.gwReplyMapping l = &c.mu } ttl := s.gateway.recSubExp wasEmpty := len(g.mapping) == 0 if g.mapping == nil { g.mapping = make(map[string]*gwReplyMap) } // The reason we pass both `reply` and `routedReply`, is that in some cases, // `routedReply` may have a deliver subject appended, something look like: // "_GR_.xxx.yyy.$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0@$MQTT.msgs.foo" // but `reply` has already been cleaned up (delivery subject removed from tail): // "$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0" // So we will use that knowledge so we don't have to make any cleaning here. routedReply = routedReply[:gwSubjectOffset+len(reply)] // We need to make a copy so that we don't reference the underlying // read buffer. ms := string(routedReply) grm := &gwReplyMap{ms: ms, exp: time.Now().Add(ttl).UnixNano()} // If we are here with the same key but different mapped replies // (say $GNR._.A.srv1.bar and then $GNR._.B.srv2.bar), we need to // store it otherwise we would take the risk of the reply not // making it back. g.mapping[ms[gwSubjectOffset:]] = grm if wasEmpty { atomic.StoreInt32(&g.check, 1) s.gwrm.m.Store(g, l) if atomic.CompareAndSwapInt32(&s.gwrm.w, 0, 1) { select { case s.gwrm.ch <- ttl: default: } } } } // Starts a long lived go routine that is responsible to // remove GW reply mapping that have expired. func (s *Server) startGWReplyMapExpiration() { s.mu.Lock() s.gwrm.ch = make(chan time.Duration, 1) s.mu.Unlock() s.startGoRoutine(func() { defer s.grWG.Done() t := time.NewTimer(time.Hour) var ttl time.Duration for { select { case <-t.C: if ttl == 0 { t.Reset(time.Hour) continue } now := time.Now().UnixNano() mapEmpty := true s.gwrm.m.Range(func(k, v any) bool { g := k.(*gwReplyMapping) l := v.(sync.Locker) l.Lock() for k, grm := range g.mapping { if grm.exp <= now { delete(g.mapping, k) if len(g.mapping) == 0 { atomic.StoreInt32(&g.check, 0) s.gwrm.m.Delete(g) } } } l.Unlock() mapEmpty = false return true }) if mapEmpty && atomic.CompareAndSwapInt32(&s.gwrm.w, 1, 0) { ttl = 0 t.Reset(time.Hour) } else { t.Reset(ttl) } case cttl := <-s.gwrm.ch: ttl = cttl if !t.Stop() { select { case <-t.C: default: } } t.Reset(ttl) case <-s.quitCh: return } } }) } nats-server-2.10.27/server/gateway_test.go000066400000000000000000006557761477524627100205420ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "context" "crypto/tls" "encoding/json" "fmt" "net" "net/url" "os" "runtime" "strconv" "strings" "sync" "sync/atomic" "testing" "time" . "github.com/nats-io/nats-server/v2/internal/ocsp" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats.go" "golang.org/x/crypto/ocsp" ) func init() { gatewayConnectDelay = 15 * time.Millisecond gatewayReconnectDelay = 15 * time.Millisecond } // Wait for the expected number of outbound gateways, or fails. func waitForOutboundGateways(t *testing.T, s *Server, expected int, timeout time.Duration) { t.Helper() if timeout < 2*time.Second { timeout = 2 * time.Second } checkFor(t, timeout, 15*time.Millisecond, func() error { if n := s.numOutboundGateways(); n != expected { return fmt.Errorf("Expected %v outbound gateway(s), got %v", expected, n) } return nil }) } // Wait for the expected number of inbound gateways, or fails. func waitForInboundGateways(t *testing.T, s *Server, expected int, timeout time.Duration) { t.Helper() if timeout < 2*time.Second { timeout = 2 * time.Second } checkFor(t, timeout, 15*time.Millisecond, func() error { if n := s.numInboundGateways(); n != expected { return fmt.Errorf("Expected %v inbound gateway(s), got %v", expected, n) } return nil }) } func waitForGatewayFailedConnect(t *testing.T, s *Server, gwName string, expectFailure bool, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { var c int cfg := s.getRemoteGateway(gwName) if cfg != nil { c = cfg.getConnAttempts() } if expectFailure && c <= 1 { return fmt.Errorf("Expected several attempts to connect, got %v", c) } else if !expectFailure && c > 1 { return fmt.Errorf("Expected single attempt to connect, got %v", c) } return nil }) } func checkForRegisteredQSubInterest(t *testing.T, s *Server, gwName, acc, subj string, expected int, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { count := 0 c := s.getOutboundGatewayConnection(gwName) ei, _ := c.gw.outsim.Load(acc) if ei != nil { sl := ei.(*outsie).sl r := sl.Match(subj) for _, qsubs := range r.qsubs { count += len(qsubs) } } if count == expected { return nil } return fmt.Errorf("Expected %v qsubs in sublist, got %v", expected, count) }) } func checkForSubjectNoInterest(t *testing.T, c *client, account, subject string, expectNoInterest bool, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { ei, _ := c.gw.outsim.Load(account) if ei == nil { return fmt.Errorf("Did not receive subject no-interest") } e := ei.(*outsie) e.RLock() _, inMap := e.ni[subject] e.RUnlock() if expectNoInterest { if inMap { return nil } return fmt.Errorf("Did not receive subject no-interest on %q", subject) } if inMap { return fmt.Errorf("No-interest on subject %q was not cleared", subject) } return nil }) } func checkForAccountNoInterest(t *testing.T, c *client, account string, expectedNoInterest bool, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { ei, ok := c.gw.outsim.Load(account) if !ok && expectedNoInterest { return fmt.Errorf("No-interest for account %q not yet registered", account) } else if ok && !expectedNoInterest { return fmt.Errorf("Account %q should not have a no-interest", account) } if ei != nil { return fmt.Errorf("Account %q should have a global no-interest, not subject no-interest", account) } return nil }) } func checkGWInterestOnlyMode(t *testing.T, s *Server, outboundGWName, accName string) { t.Helper() checkGWInterestOnlyModeOrNotPresent(t, s, outboundGWName, accName, false) } func checkGWInterestOnlyModeOrNotPresent(t *testing.T, s *Server, outboundGWName, accName string, notPresentOk bool) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { gwc := s.getOutboundGatewayConnection(outboundGWName) if gwc == nil { return fmt.Errorf("No outbound gateway connection %q for server %v", outboundGWName, s) } gwc.mu.Lock() defer gwc.mu.Unlock() out, ok := gwc.gw.outsim.Load(accName) if !ok { if notPresentOk { return nil } else { return fmt.Errorf("Server %v - outbound gateway connection %q: no account %q found in map", s, outboundGWName, accName) } } if out == nil { return fmt.Errorf("Server %v - outbound gateway connection %q: interest map not found for account %q", s, outboundGWName, accName) } e := out.(*outsie) e.RLock() defer e.RUnlock() if e.mode != InterestOnly { return fmt.Errorf( "Server %v - outbound gateway connection %q: account %q mode shoule be InterestOnly but is %v", s, outboundGWName, accName, e.mode) } return nil }) } func checkGWInterestOnlyModeInterestOn(t *testing.T, s *Server, outboundGWName, accName, subject string) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { c := s.getOutboundGatewayConnection(outboundGWName) outsiei, _ := c.gw.outsim.Load(accName) if outsiei == nil { return fmt.Errorf("Server %s - outbound gateway connection %q: no map entry found for account %q", s, outboundGWName, accName) } outsie := outsiei.(*outsie) r := outsie.sl.Match(subject) if len(r.psubs) == 0 { return fmt.Errorf("Server %s - outbound gateway connection %q - account %q: no subject interest for %q", s, outboundGWName, accName, subject) } return nil }) } func waitCh(t *testing.T, ch chan bool, errTxt string) { t.Helper() select { case <-ch: return case <-time.After(5 * time.Second): t.Fatal(errTxt) } } var noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {} func natsConnect(t testing.TB, url string, options ...nats.Option) *nats.Conn { t.Helper() opts := nats.GetDefaultOptions() for _, opt := range options { if err := opt(&opts); err != nil { t.Fatalf("Error applying client option: %v", err) } } nc, err := nats.Connect(url, options...) if err != nil { t.Fatalf("Error on connect: %v", err) } if opts.AsyncErrorCB == nil { // Set this up to not pollute the logs when running tests. nc.SetErrorHandler(noOpErrHandler) } return nc } func natsSub(t *testing.T, nc *nats.Conn, subj string, cb nats.MsgHandler) *nats.Subscription { t.Helper() sub, err := nc.Subscribe(subj, cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } return sub } func natsSubSync(t *testing.T, nc *nats.Conn, subj string) *nats.Subscription { t.Helper() sub, err := nc.SubscribeSync(subj) if err != nil { t.Fatalf("Error on subscribe: %v", err) } return sub } func natsNexMsg(t *testing.T, sub *nats.Subscription, timeout time.Duration) *nats.Msg { t.Helper() msg, err := sub.NextMsg(timeout) if err != nil { t.Fatalf("Failed getting next message: %v", err) } return msg } func natsQueueSub(t *testing.T, nc *nats.Conn, subj, queue string, cb nats.MsgHandler) *nats.Subscription { t.Helper() sub, err := nc.QueueSubscribe(subj, queue, cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } return sub } func natsQueueSubSync(t *testing.T, nc *nats.Conn, subj, queue string) *nats.Subscription { t.Helper() sub, err := nc.QueueSubscribeSync(subj, queue) if err != nil { t.Fatalf("Error on subscribe: %v", err) } return sub } func natsFlush(t *testing.T, nc *nats.Conn) { t.Helper() if err := nc.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } } func natsPub(t testing.TB, nc *nats.Conn, subj string, payload []byte) { t.Helper() if err := nc.Publish(subj, payload); err != nil { t.Fatalf("Error on publish: %v", err) } } func natsPubReq(t *testing.T, nc *nats.Conn, subj, reply string, payload []byte) { t.Helper() if err := nc.PublishRequest(subj, reply, payload); err != nil { t.Fatalf("Error on publish: %v", err) } } func natsUnsub(t *testing.T, sub *nats.Subscription) { t.Helper() if err := sub.Unsubscribe(); err != nil { t.Fatalf("Error on unsubscribe: %v", err) } } func testDefaultOptionsForGateway(name string) *Options { o := DefaultOptions() o.NoSystemAccount = true o.Cluster.Name = name o.Gateway.Name = name o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 o.gatewaysSolicitDelay = 15 * time.Millisecond return o } func runGatewayServer(o *Options) *Server { s := RunServer(o) s.SetLogger(&DummyLogger{}, true, true) return s } func testGatewayOptionsFromToWithServers(t *testing.T, org, dst string, servers ...*Server) *Options { t.Helper() o := testDefaultOptionsForGateway(org) gw := &RemoteGatewayOpts{Name: dst} for _, s := range servers { us := fmt.Sprintf("nats://127.0.0.1:%d", s.GatewayAddr().Port) u, err := url.Parse(us) if err != nil { t.Fatalf("Error parsing url: %v", err) } gw.URLs = append(gw.URLs, u) } o.Gateway.Gateways = append(o.Gateway.Gateways, gw) return o } func testAddGatewayURLs(t *testing.T, o *Options, dst string, urls []string) { t.Helper() gw := &RemoteGatewayOpts{Name: dst} for _, us := range urls { u, err := url.Parse(us) if err != nil { t.Fatalf("Error parsing url: %v", err) } gw.URLs = append(gw.URLs, u) } o.Gateway.Gateways = append(o.Gateway.Gateways, gw) } func testGatewayOptionsFromToWithURLs(t *testing.T, org, dst string, urls []string) *Options { o := testDefaultOptionsForGateway(org) testAddGatewayURLs(t, o, dst, urls) return o } func testGatewayOptionsWithTLS(t *testing.T, name string) *Options { t.Helper() o := testDefaultOptionsForGateway(name) var ( tc = &TLSConfigOpts{} err error ) if name == "A" { tc.CertFile = "../test/configs/certs/srva-cert.pem" tc.KeyFile = "../test/configs/certs/srva-key.pem" } else { tc.CertFile = "../test/configs/certs/srvb-cert.pem" tc.KeyFile = "../test/configs/certs/srvb-key.pem" } tc.CaFile = "../test/configs/certs/ca.pem" o.Gateway.TLSConfig, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } o.Gateway.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert o.Gateway.TLSConfig.RootCAs = o.Gateway.TLSConfig.ClientCAs o.Gateway.TLSTimeout = 2.0 return o } func testGatewayOptionsFromToWithTLS(t *testing.T, org, dst string, urls []string) *Options { o := testGatewayOptionsWithTLS(t, org) testAddGatewayURLs(t, o, dst, urls) return o } func TestGatewayBasic(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.ConnectRetries = 0 s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) // and an outbound too waitForOutboundGateways(t, s2, 1, time.Second) // Stop s2 server s2.Shutdown() // gateway should go away waitForOutboundGateways(t, s1, 0, time.Second) waitForInboundGateways(t, s1, 0, time.Second) // Restart server s2 = runGatewayServer(o2) defer s2.Shutdown() // gateway should reconnect waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForInboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) // Shutdown s1, remove the gateway from A to B and restart. s1.Shutdown() // When s2 detects the connection is closed, it will attempt // to reconnect once (even if the route is implicit). // We need to wait more than a dial timeout to make sure // s1 does not restart too quickly and s2 can actually reconnect. time.Sleep(DEFAULT_ROUTE_DIAL + 250*time.Millisecond) // Restart s1 without gateway to B. o1.Gateway.Gateways = nil s1 = runGatewayServer(o1) defer s1.Shutdown() // s1 should not have any outbound nor inbound waitForOutboundGateways(t, s1, 0, 2*time.Second) waitForInboundGateways(t, s1, 0, 2*time.Second) // Same for s2 waitForOutboundGateways(t, s2, 0, 2*time.Second) waitForInboundGateways(t, s2, 0, 2*time.Second) // Verify that s2 no longer has A gateway in its list checkFor(t, time.Second, 15*time.Millisecond, func() error { if s2.getRemoteGateway("A") != nil { return fmt.Errorf("Gateway A should have been removed from s2") } return nil }) } func TestGatewayIgnoreSelfReference(t *testing.T) { o := testDefaultOptionsForGateway("A") // To create a reference to itself before running the server // it means that we have to assign an explicit port o.Gateway.Port = 5222 o.gatewaysSolicitDelay = 0 u, _ := url.Parse(fmt.Sprintf("nats://%s:%d", o.Gateway.Host, o.Gateway.Port)) cfg := &RemoteGatewayOpts{ Name: "A", URLs: []*url.URL{u}, } o.Gateway.Gateways = append(o.Gateway.Gateways, cfg) o.NoSystemAccount = true s := runGatewayServer(o) defer s.Shutdown() // Wait a bit to make sure that there is no attempt to connect. time.Sleep(20 * time.Millisecond) // No outbound connection expected, and no attempt to connect. if s.getRemoteGateway("A") != nil { t.Fatalf("Should not have a remote gateway config for A") } if s.getOutboundGatewayConnection("A") != nil { t.Fatalf("Should not have a gateway connection to A") } s.Shutdown() // Now try with config files and include s1, _ := RunServerWithConfig("configs/gwa.conf") defer s1.Shutdown() s2, _ := RunServerWithConfig("configs/gwb.conf") defer s2.Shutdown() waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForInboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) if s1.getRemoteGateway("A") != nil { t.Fatalf("Should not have a remote gateway config for A") } if s1.getOutboundGatewayConnection("A") != nil { t.Fatalf("Should not have a gateway connection to A") } if s2.getRemoteGateway("B") != nil { t.Fatalf("Should not have a remote gateway config for B") } if s2.getOutboundGatewayConnection("B") != nil { t.Fatalf("Should not have a gateway connection to B") } } func TestGatewayHeaderInfo(t *testing.T) { o := testDefaultOptionsForGateway("A") s := runGatewayServer(o) defer s.Shutdown() gwconn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.Gateway.Host, o.Gateway.Port)) if err != nil { t.Fatalf("Error dialing server: %v\n", err) } defer gwconn.Close() client := bufio.NewReaderSize(gwconn, maxBufSize) l, err := client.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } var info serverInfo if err = json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if !info.Headers { t.Fatalf("Expected by default for header support to be enabled") } s.Shutdown() gwconn.Close() // Now turn headers off. o.NoHeaderSupport = true s = runGatewayServer(o) defer s.Shutdown() gwconn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", o.Gateway.Host, o.Gateway.Port)) if err != nil { t.Fatalf("Error dialing server: %v\n", err) } defer gwconn.Close() client = bufio.NewReaderSize(gwconn, maxBufSize) l, err = client.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } if err = json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Headers { t.Fatalf("Expected header support to be disabled") } } func TestGatewayHeaderSupport(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.ConnectRetries = 0 s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // and an inbound too waitForInboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) // and an outbound too waitForOutboundGateways(t, s2, 1, time.Second) c, cr, _ := newClientForServer(s1) defer c.close() connect := "CONNECT {\"headers\":true}" subOp := "SUB foo 1" pingOp := "PING\r\n" cmd := strings.Join([]string{connect, subOp, pingOp}, "\r\n") c.parseAsync(cmd) if _, err := cr.ReadString('\n'); err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } // Wait for interest to be registered on s2 checkGWInterestOnlyModeInterestOn(t, s2, "A", globalAccountName, "foo") b, _, _ := newClientForServer(s2) defer b.close() pubOp := "HPUB foo 12 14\r\nName:Derek\r\nOK\r\n" cmd = strings.Join([]string{connect, pubOp}, "\r\n") b.parseAsync(cmd) l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := hmsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a match for %q", l) } matches := am[0] if len(matches) != 7 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 7) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[HDR_INDEX] != "12" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[HDR_INDEX]) } if matches[TLEN_INDEX] != "14" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[TLEN_INDEX]) } checkPayload(cr, []byte("Name:Derek\r\nOK\r\n"), t) } func TestGatewayHeaderDeliverStrippedMsg(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.ConnectRetries = 0 s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) o1.NoHeaderSupport = true s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // and an inbound too waitForInboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) // and an outbound too waitForOutboundGateways(t, s2, 1, time.Second) c, cr, _ := newClientForServer(s1) defer c.close() connect := "CONNECT {\"headers\":true}" subOp := "SUB foo 1" pingOp := "PING\r\n" cmd := strings.Join([]string{connect, subOp, pingOp}, "\r\n") c.parseAsync(cmd) if _, err := cr.ReadString('\n'); err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } // Wait for interest to be registered on s2 checkGWInterestOnlyModeInterestOn(t, s2, "A", globalAccountName, "foo") b, _, _ := newClientForServer(s2) defer b.close() pubOp := "HPUB foo 12 14\r\nName:Derek\r\nOK\r\n" cmd = strings.Join([]string{connect, pubOp}, "\r\n") b.parseAsync(cmd) l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Error receiving msg from server: %v\n", err) } am := smsgPat.FindAllStringSubmatch(l, -1) if len(am) == 0 { t.Fatalf("Did not get a correct match for %q", l) } matches := am[0] if len(matches) != 6 { t.Fatalf("Did not get correct # matches: %d vs %d\n", len(matches), 6) } if matches[SUB_INDEX] != "foo" { t.Fatalf("Did not get correct subject: '%s'\n", matches[SUB_INDEX]) } if matches[SID_INDEX] != "1" { t.Fatalf("Did not get correct sid: '%s'\n", matches[SID_INDEX]) } if matches[LEN_INDEX] != "2" { t.Fatalf("Did not get correct msg length: '%s'\n", matches[LEN_INDEX]) } checkPayload(cr, []byte("OK\r\n"), t) if cr.Buffered() != 0 { t.Fatalf("Expected no extra bytes to be buffered, got %d", cr.Buffered()) } } func TestGatewaySolicitDelay(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) // Set the solicit delay to 0. This tests that server will use its // default value, currently set at 1 sec. o1.gatewaysSolicitDelay = 0 start := time.Now() s1 := runGatewayServer(o1) defer s1.Shutdown() // After 500ms, check outbound gateway. Should not be there. time.Sleep(500 * time.Millisecond) if time.Since(start) < defaultSolicitGatewaysDelay { if s1.numOutboundGateways() > 0 { t.Fatalf("The outbound gateway was initiated sooner than expected (%v)", time.Since(start)) } } // Ultimately, s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, 2*time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, 2*time.Second) s1.Shutdown() // Make sure that server can be shutdown while waiting // for that initial solicit delay o1.gatewaysSolicitDelay = 2 * time.Second s1 = runGatewayServer(o1) start = time.Now() s1.Shutdown() if dur := time.Since(start); dur >= 2*time.Second { t.Fatalf("Looks like shutdown was delayed: %v", dur) } } func TestGatewaySolicitDelayWithImplicitOutbounds(t *testing.T) { // Cause a situation where A connects to B, and because of // delay of solicit gateways set on B, we want to make sure // that B does not end-up with 2 connections to A. o2 := testDefaultOptionsForGateway("B") o2.gatewaysSolicitDelay = 500 * time.Millisecond s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound and inbound gateway to s2. waitForOutboundGateways(t, s1, 1, 2*time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, 2*time.Second) // Wait for more than s2 solicit delay time.Sleep(750 * time.Millisecond) // The way we store outbound (map key'ed by gw name), we would // not know if we had created 2 (since the newer would replace // the older in the map). But if a second connection was made, // then s1 would have 2 inbounds. So check it has only 1. waitForInboundGateways(t, s1, 1, time.Second) } type slowResolver struct { inLookupCh chan struct{} releaseCh chan struct{} } func (r *slowResolver) LookupHost(ctx context.Context, h string) ([]string, error) { if r.inLookupCh != nil { select { case r.inLookupCh <- struct{}{}: default: } <-r.releaseCh } else { time.Sleep(500 * time.Millisecond) } return []string{h}, nil } func TestGatewaySolicitShutdown(t *testing.T) { var urls []string for i := 0; i < 5; i++ { u := fmt.Sprintf("nats://localhost:%d", 1234+i) urls = append(urls, u) } o1 := testGatewayOptionsFromToWithURLs(t, "A", "B", urls) o1.Gateway.resolver = &slowResolver{} s1 := runGatewayServer(o1) defer s1.Shutdown() time.Sleep(o1.gatewaysSolicitDelay + 10*time.Millisecond) start := time.Now() s1.Shutdown() if dur := time.Since(start); dur > 1200*time.Millisecond { t.Fatalf("Took too long to shutdown: %v", dur) } } func testFatalErrorOnStart(t *testing.T, o *Options, errTxt string) { t.Helper() s := New(o) defer s.Shutdown() l := &captureFatalLogger{fatalCh: make(chan string, 1)} s.SetLogger(l, false, false) // This does not block s.Start() select { case e := <-l.fatalCh: if !strings.Contains(e, errTxt) { t.Fatalf("Error should contain %q, got %s", errTxt, e) } case <-time.After(time.Second): t.Fatal("Should have got a fatal error") } s.Shutdown() s.WaitForShutdown() } func TestGatewayListenError(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testDefaultOptionsForGateway("A") o1.Gateway.Port = s2.GatewayAddr().Port testFatalErrorOnStart(t, o1, "listening on") } func TestGatewayWithListenToAny(t *testing.T) { confB1 := createConfFile(t, []byte(` listen: "127.0.0.1:-1" cluster { listen: "127.0.0.1:-1" } gateway { name: "B" listen: "0.0.0.0:-1" } `)) sb1, ob1 := RunServerWithConfig(confB1) defer sb1.Shutdown() confB2 := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" cluster { listen: "127.0.0.1:-1" routes: ["%s"] } gateway { name: "B" listen: "0.0.0.0:-1" } `, fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)))) sb2, ob2 := RunServerWithConfig(confB2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) confA := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" cluster { listen: "127.0.0.1:-1" } gateway { name: "A" listen: "0.0.0.0:-1" gateways [ { name: "B" urls: ["%s", "%s"] } ] } `, fmt.Sprintf("nats://127.0.0.1:%d", ob1.Gateway.Port), fmt.Sprintf("nats://127.0.0.1:%d", ob2.Gateway.Port)))) oa := LoadConfig(confA) oa.gatewaysSolicitDelay = 15 * time.Millisecond sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, time.Second) waitForInboundGateways(t, sa, 2, time.Second) checkAll := func(t *testing.T) { t.Helper() checkURL := func(t *testing.T, s *Server) { t.Helper() url := s.getGatewayURL() if strings.HasPrefix(url, "0.0.0.0") { t.Fatalf("URL still references 0.0.0.0") } s.gateway.RLock() for url := range s.gateway.URLs { if strings.HasPrefix(url, "0.0.0.0") { s.gateway.RUnlock() t.Fatalf("URL still references 0.0.0.0") } } s.gateway.RUnlock() var cfg *gatewayCfg if s.getGatewayName() == "A" { cfg = s.getRemoteGateway("B") } else { cfg = s.getRemoteGateway("A") } urls := cfg.getURLs() for _, url := range urls { if strings.HasPrefix(url.Host, "0.0.0.0") { t.Fatalf("URL still references 0.0.0.0") } } } checkURL(t, sb1) checkURL(t, sb2) checkURL(t, sa) } checkAll(t) // Perform a reload and ensure that nothing has changed servers := []*Server{sb1, sb2, sa} for _, s := range servers { if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } checkAll(t) } } func TestGatewayAdvertise(t *testing.T) { o3 := testDefaultOptionsForGateway("C") s3 := runGatewayServer(o3) defer s3.Shutdown() o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) // Set the advertise so that this points to C o1.Gateway.Advertise = fmt.Sprintf("127.0.0.1:%d", s3.GatewayAddr().Port) s1 := runGatewayServer(o1) defer s1.Shutdown() // We should have outbound from s1 to s2 waitForOutboundGateways(t, s1, 1, time.Second) // But no inbound from s2 waitForInboundGateways(t, s1, 0, time.Second) // And since B tries to connect to A but reaches C, it should fail to connect, // and without connect retries, stop trying. So no outbound for s2, and no // inbound/outbound for s3. waitForInboundGateways(t, s2, 1, time.Second) waitForOutboundGateways(t, s2, 0, time.Second) waitForInboundGateways(t, s3, 0, time.Second) waitForOutboundGateways(t, s3, 0, time.Second) } func TestGatewayAdvertiseErr(t *testing.T) { o1 := testDefaultOptionsForGateway("A") o1.Gateway.Advertise = "wrong:address" testFatalErrorOnStart(t, o1, "Gateway.Advertise") } func TestGatewayAuth(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.Username = "me" o2.Gateway.Password = "pwd" s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithURLs(t, "A", "B", []string{fmt.Sprintf("nats://me:pwd@127.0.0.1:%d", s2.GatewayAddr().Port)}) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) s2.Shutdown() s1.Shutdown() o2.Gateway.Username = "me" o2.Gateway.Password = "wrong" s2 = runGatewayServer(o2) defer s2.Shutdown() s1 = runGatewayServer(o1) defer s1.Shutdown() // Connection should fail... waitForGatewayFailedConnect(t, s1, "B", true, 2*time.Second) s2.Shutdown() s1.Shutdown() o2.Gateway.Username = "wrong" o2.Gateway.Password = "pwd" s2 = runGatewayServer(o2) defer s2.Shutdown() s1 = runGatewayServer(o1) defer s1.Shutdown() // Connection should fail... waitForGatewayFailedConnect(t, s1, "B", true, 2*time.Second) } func TestGatewayTLS(t *testing.T) { o2 := testGatewayOptionsWithTLS(t, "B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithTLS(t, "A", "B", []string{fmt.Sprintf("nats://127.0.0.1:%d", s2.GatewayAddr().Port)}) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) // and vice-versa waitForOutboundGateways(t, s2, 1, time.Second) waitForInboundGateways(t, s1, 1, time.Second) // Stop s2 server s2.Shutdown() // gateway should go away waitForOutboundGateways(t, s1, 0, time.Second) waitForInboundGateways(t, s1, 0, time.Second) waitForOutboundGateways(t, s2, 0, time.Second) waitForInboundGateways(t, s2, 0, time.Second) // Restart server s2 = runGatewayServer(o2) defer s2.Shutdown() // gateway should reconnect waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForInboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) s1.Shutdown() // Wait for s2 to lose connections to s1. waitForOutboundGateways(t, s2, 0, 2*time.Second) waitForInboundGateways(t, s2, 0, 2*time.Second) // Make an explicit TLS config for remote gateway config "B" // on cluster A. o1.Gateway.Gateways[0].TLSConfig = o1.Gateway.TLSConfig.Clone() u, _ := url.Parse(fmt.Sprintf("tls://localhost:%d", s2.GatewayAddr().Port)) o1.Gateway.Gateways[0].URLs = []*url.URL{u} // Make the TLSTimeout so small that it should fail to connect. smallTimeout := 0.00000001 o1.Gateway.Gateways[0].TLSTimeout = smallTimeout s1 = runGatewayServer(o1) defer s1.Shutdown() // Check that s1 reports connection failures waitForGatewayFailedConnect(t, s1, "B", true, 2*time.Second) // Check that TLSConfig from s1's remote "B" is based on // what we have configured. cfg := s1.getRemoteGateway("B") cfg.RLock() tlsName := cfg.tlsName timeout := cfg.TLSTimeout cfg.RUnlock() if tlsName != "localhost" { t.Fatalf("Expected server name to be localhost, got %v", tlsName) } if timeout != smallTimeout { t.Fatalf("Expected tls timeout to be %v, got %v", smallTimeout, timeout) } s1.Shutdown() // Wait for s2 to lose connections to s1. waitForOutboundGateways(t, s2, 0, 2*time.Second) waitForInboundGateways(t, s2, 0, 2*time.Second) // Remove explicit TLSTimeout from gateway "B" and check that // we use the A's spec one. o1.Gateway.Gateways[0].TLSTimeout = 0 s1 = runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForInboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) cfg = s1.getRemoteGateway("B") cfg.RLock() timeout = cfg.TLSTimeout cfg.RUnlock() if timeout != o1.Gateway.TLSTimeout { t.Fatalf("Expected tls timeout to be %v, got %v", o1.Gateway.TLSTimeout, timeout) } } func TestGatewayTLSErrors(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithTLS(t, "A", "B", []string{fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)}) s1 := runGatewayServer(o1) defer s1.Shutdown() // Expect s1 to have a failed to connect count > 0 waitForGatewayFailedConnect(t, s1, "B", true, 2*time.Second) } func TestGatewayServerNameInTLSConfig(t *testing.T) { o2 := testDefaultOptionsForGateway("B") var ( tc = &TLSConfigOpts{} err error ) tc.CertFile = "../test/configs/certs/server-noip.pem" tc.KeyFile = "../test/configs/certs/server-key-noip.pem" tc.CaFile = "../test/configs/certs/ca.pem" o2.Gateway.TLSConfig, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } o2.Gateway.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert o2.Gateway.TLSConfig.RootCAs = o2.Gateway.TLSConfig.ClientCAs o2.Gateway.TLSTimeout = 2.0 s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithTLS(t, "A", "B", []string{fmt.Sprintf("nats://127.0.0.1:%d", s2.GatewayAddr().Port)}) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should fail to connect since we don't have proper expected hostname. waitForGatewayFailedConnect(t, s1, "B", true, 2*time.Second) // Now set server name, and it should work. s1.Shutdown() o1.Gateway.TLSConfig.ServerName = "localhost" s1 = runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, 2*time.Second) } func TestGatewayWrongDestination(t *testing.T) { // Start a server with a gateway named "C" o2 := testDefaultOptionsForGateway("C") s2 := runGatewayServer(o2) defer s2.Shutdown() // Configure a gateway to "B", but since we are connecting to "C"... o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // we should not be able to connect. waitForGatewayFailedConnect(t, s1, "B", true, time.Second) // Shutdown s2 and fix the gateway name. // s1 should then connect ok and failed connect should be cleared. s2.Shutdown() // Reset the conn attempts cfg := s1.getRemoteGateway("B") cfg.resetConnAttempts() o2.Gateway.Name = "B" o2.Cluster.Name = "B" s2 = runGatewayServer(o2) defer s2.Shutdown() // At some point, the number of failed connect count should be reset to 0. waitForGatewayFailedConnect(t, s1, "B", false, 2*time.Second) } func TestGatewayConnectToWrongPort(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() // Configure a gateway to "B", but connect to the wrong port urls := []string{fmt.Sprintf("nats://127.0.0.1:%d", s2.Addr().(*net.TCPAddr).Port)} o1 := testGatewayOptionsFromToWithURLs(t, "A", "B", urls) s1 := runGatewayServer(o1) defer s1.Shutdown() // we should not be able to connect. waitForGatewayFailedConnect(t, s1, "B", true, time.Second) s1.Shutdown() // Repeat with route port urls = []string{fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)} o1 = testGatewayOptionsFromToWithURLs(t, "A", "B", urls) s1 = runGatewayServer(o1) defer s1.Shutdown() // we should not be able to connect. waitForGatewayFailedConnect(t, s1, "B", true, time.Second) s1.Shutdown() // Now have a client connect to s2's gateway port. nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", s2.GatewayAddr().Port)) if err == nil { nc.Close() t.Fatal("Expected error, got none") } } func TestGatewayCreateImplicit(t *testing.T) { // Create a regular cluster of 2 servers o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o3 := testDefaultOptionsForGateway("B") o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)) s3 := runGatewayServer(o3) defer s3.Shutdown() checkClusterFormed(t, s2, s3) // Now start s1 that creates a Gateway connection to s2 or s3 o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2, s3) s1 := runGatewayServer(o1) defer s1.Shutdown() // We should have an outbound gateway connection on ALL servers. waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForOutboundGateways(t, s3, 1, 2*time.Second) // Server s1 must have 2 inbound ones waitForInboundGateways(t, s1, 2, 2*time.Second) // However, s1 may have created the outbound to s2 or s3. It is possible that // either s2 or s3 does not an inbound connection. s2Inbound := s2.numInboundGateways() s3Inbound := s3.numInboundGateways() if (s2Inbound == 1 && s3Inbound != 0) || (s3Inbound == 1 && s2Inbound != 0) { t.Fatalf("Unexpected inbound for s2/s3: %v/%v", s2Inbound, s3Inbound) } } func TestGatewayCreateImplicitOnNewRoute(t *testing.T) { // Start with only 2 clusters of 1 server each o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() // Now start s1 that creates a Gateway connection to s2 o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // Check outbounds waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) // Now add a server to cluster B o3 := testDefaultOptionsForGateway("B") o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)) s3 := runGatewayServer(o3) defer s3.Shutdown() // Wait for cluster between s2/s3 to form checkClusterFormed(t, s2, s3) // s3 should have been notified about existence of A and create its gateway to A. waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForOutboundGateways(t, s3, 1, 2*time.Second) } func TestGatewayImplicitReconnect(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.ConnectRetries = 5 s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // s1 should have an outbound gateway to s2. waitForOutboundGateways(t, s1, 1, time.Second) // s2 should have an inbound gateway waitForInboundGateways(t, s2, 1, time.Second) // It will have also created an implicit outbound connection to s1. // We need to wait for that implicit outbound connection to be made // to show that it will try to reconnect when we stop/restart s1 // (without config to connect to B). waitForOutboundGateways(t, s2, 1, time.Second) // Shutdown s1, remove the gateway from A to B and restart. s1.Shutdown() o1.Gateway.Gateways = o1.Gateway.Gateways[:0] s1 = runGatewayServer(o1) defer s1.Shutdown() // s1 should have both outbound and inbound to s2 waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s1, 1, 2*time.Second) // Same for s2 waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) // Verify that s2 still has "A" in its gateway config if s2.getRemoteGateway("A") == nil { t.Fatal("Gateway A should be in s2") } } func TestGatewayImplicitReconnectRace(t *testing.T) { ob := testDefaultOptionsForGateway("B") resolver := &slowResolver{ inLookupCh: make(chan struct{}, 1), releaseCh: make(chan struct{}), } ob.Gateway.resolver = resolver sb := runGatewayServer(ob) defer sb.Shutdown() oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa1 := runGatewayServer(oa1) defer sa1.Shutdown() // Wait for the proper connections waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // On sb, change the URL to sa1 so that it is a name, instead of an IP, // so that we hit the slow resolver. cfg := sb.getRemoteGateway("A") cfg.updateURLs([]string{fmt.Sprintf("localhost:%d", sa1.GatewayAddr().Port)}) // Shutdown sa1 now... sa1.Shutdown() // Wait to be notified that B has detected the connection close // and it is trying to resolve the host during the reconnect. <-resolver.inLookupCh // Start a new "A" server (sa2). oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() // Make sure we have our outbound to sb registered on sa2 and inbound // from sa2 on sb before releasing the resolver. waitForOutboundGateways(t, sa2, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) // Now release the resolver and ensure we have all connections. close(resolver.releaseCh) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sa2, 1, 2*time.Second) } type gwReconnAttemptLogger struct { DummyLogger errCh chan string } func (l *gwReconnAttemptLogger) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, `Error connecting to implicit gateway "A"`) { select { case l.errCh <- msg: default: } } } func TestGatewayImplicitReconnectHonorConnectRetries(t *testing.T) { ob := testDefaultOptionsForGateway("B") ob.ReconnectErrorReports = 1 ob.Gateway.ConnectRetries = 2 sb := runGatewayServer(ob) defer sb.Shutdown() l := &gwReconnAttemptLogger{errCh: make(chan string, 3)} sb.SetLogger(l, true, false) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() // Wait for the proper connections waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Now have C connect to B. oc := testGatewayOptionsFromToWithServers(t, "C", "B", sb) sc := runGatewayServer(oc) defer sc.Shutdown() // Wait for the proper connections waitForOutboundGateways(t, sa, 2, time.Second) waitForOutboundGateways(t, sb, 2, time.Second) waitForOutboundGateways(t, sc, 2, time.Second) waitForInboundGateways(t, sa, 2, time.Second) waitForInboundGateways(t, sb, 2, time.Second) waitForInboundGateways(t, sc, 2, time.Second) // Shutdown sa now... sa.Shutdown() // B will try to reconnect to A 3 times (we stop after attempts > ConnectRetries) timeout := time.NewTimer(time.Second) for i := 0; i < 3; i++ { select { case <-l.errCh: // OK case <-timeout.C: t.Fatal("Did not get debug trace about reconnect") } } // If we get 1 more, we have an issue! select { case e := <-l.errCh: t.Fatalf("Should not have attempted to reconnect: %q", e) case <-time.After(250 * time.Millisecond): // OK! } waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) waitForOutboundGateways(t, sc, 1, 2*time.Second) waitForInboundGateways(t, sc, 1, 2*time.Second) } func TestGatewayURLsFromClusterSentInINFO(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o3 := testDefaultOptionsForGateway("B") o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)) s3 := runGatewayServer(o3) defer s3.Shutdown() checkClusterFormed(t, s2, s3) // Now start s1 that creates a Gateway connection to s2 o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // Make sure we have proper outbound/inbound waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) waitForOutboundGateways(t, s3, 1, time.Second) // Although s1 connected to s2 and knew only about s2, it should have // received the list of gateway URLs in the B cluster. So if we shutdown // server s2, it should be able to reconnect to s3. s2.Shutdown() // Wait for s3 to register that there s2 is gone. checkNumRoutes(t, s3, 0) // s1 should have reconnected to s3 because it learned about it // when connecting earlier to s2. waitForOutboundGateways(t, s1, 1, 2*time.Second) // Also make sure that the gateway's urls map has 2 urls. gw := s1.getRemoteGateway("B") if gw == nil { t.Fatal("Did not find gateway B") } gw.RLock() l := len(gw.urls) gw.RUnlock() if l != 2 { t.Fatalf("S1 should have 2 urls, got %v", l) } } func TestGatewayUseUpdatedURLs(t *testing.T) { // For this test, we have cluster B with an explicit gateway to cluster A // on a given URL. Then we create cluster A with a gateway to B with server B's // GW url, and we expect server B to ultimately create an outbound GW connection // to server A (with the URL it will get from server A connecting to it). ob := testGatewayOptionsFromToWithURLs(t, "B", "A", []string{"nats://127.0.0.1:1234"}) sb := runGatewayServer(ob) defer sb.Shutdown() // Add a delay before starting server A to make sure that server B start // initiating the connection to A on inexistant server at :1234. time.Sleep(100 * time.Millisecond) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() // sa should have no problem creating outbound connection to sb waitForOutboundGateways(t, sa, 1, time.Second) // Make sure that since sb learns about sa's GW URL, it can successfully // connect to it. waitForOutboundGateways(t, sb, 1, 3*time.Second) waitForInboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) } func TestGatewayAutoDiscovery(t *testing.T) { o4 := testDefaultOptionsForGateway("D") s4 := runGatewayServer(o4) defer s4.Shutdown() o3 := testGatewayOptionsFromToWithServers(t, "C", "D", s4) s3 := runGatewayServer(o3) defer s3.Shutdown() o2 := testGatewayOptionsFromToWithServers(t, "B", "C", s3) s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() // Each server should have 3 outbound gateway connections. waitForOutboundGateways(t, s1, 3, 2*time.Second) waitForOutboundGateways(t, s2, 3, 2*time.Second) waitForOutboundGateways(t, s3, 3, 2*time.Second) waitForOutboundGateways(t, s4, 3, 2*time.Second) s1.Shutdown() s2.Shutdown() s3.Shutdown() s4.Shutdown() o2 = testDefaultOptionsForGateway("B") s2 = runGatewayServer(o2) defer s2.Shutdown() o4 = testGatewayOptionsFromToWithServers(t, "D", "B", s2) s4 = runGatewayServer(o4) defer s4.Shutdown() o3 = testGatewayOptionsFromToWithServers(t, "C", "B", s2) s3 = runGatewayServer(o3) defer s3.Shutdown() o1 = testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 = runGatewayServer(o1) defer s1.Shutdown() // Each server should have 3 outbound gateway connections. waitForOutboundGateways(t, s1, 3, 2*time.Second) waitForOutboundGateways(t, s2, 3, 2*time.Second) waitForOutboundGateways(t, s3, 3, 2*time.Second) waitForOutboundGateways(t, s4, 3, 2*time.Second) s1.Shutdown() s2.Shutdown() s3.Shutdown() s4.Shutdown() o1 = testDefaultOptionsForGateway("A") s1 = runGatewayServer(o1) defer s1.Shutdown() o2 = testDefaultOptionsForGateway("A") o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s1.ClusterAddr().Port)) s2 = runGatewayServer(o2) defer s2.Shutdown() o3 = testDefaultOptionsForGateway("A") o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s1.ClusterAddr().Port)) s3 = runGatewayServer(o3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) o4 = testGatewayOptionsFromToWithServers(t, "B", "A", s1) s4 = runGatewayServer(o4) defer s4.Shutdown() waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) waitForOutboundGateways(t, s3, 1, 2*time.Second) waitForOutboundGateways(t, s4, 1, 2*time.Second) waitForInboundGateways(t, s4, 3, 2*time.Second) o5 := testGatewayOptionsFromToWithServers(t, "C", "B", s4) s5 := runGatewayServer(o5) defer s5.Shutdown() waitForOutboundGateways(t, s1, 2, 2*time.Second) waitForOutboundGateways(t, s2, 2, 2*time.Second) waitForOutboundGateways(t, s3, 2, 2*time.Second) waitForOutboundGateways(t, s4, 2, 2*time.Second) waitForInboundGateways(t, s4, 4, 2*time.Second) waitForOutboundGateways(t, s5, 2, 2*time.Second) waitForInboundGateways(t, s5, 4, 2*time.Second) } func TestGatewayRejectUnknown(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() // Create a gateway from A to B, but configure B to reject non configured ones. o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) o1.Gateway.RejectUnknown = true s1 := runGatewayServer(o1) defer s1.Shutdown() // Wait for outbound/inbound to be created. waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) waitForInboundGateways(t, s1, 1, time.Second) waitForInboundGateways(t, s2, 1, time.Second) // Create gateway C to B. B will tell C to connect to A, // which A should reject. o3 := testGatewayOptionsFromToWithServers(t, "C", "B", s2) s3 := runGatewayServer(o3) defer s3.Shutdown() // s3 should have outbound to B, but not to A waitForOutboundGateways(t, s3, 1, time.Second) // s2 should have 2 inbounds (one from s1 one from s3) waitForInboundGateways(t, s2, 2, time.Second) // s1 should have single outbound/inbound with s2. waitForOutboundGateways(t, s1, 1, time.Second) waitForInboundGateways(t, s1, 1, time.Second) // It should not have a registered remote gateway with C (s3) if s1.getOutboundGatewayConnection("C") != nil { t.Fatalf("A should not have outbound gateway to C") } if s1.getRemoteGateway("C") != nil { t.Fatalf("A should not have a registered remote gateway to C") } // Restart s1 and this time, B will tell A to connect to C. // But A will not even attempt that since it does not have // C configured. s1.Shutdown() waitForOutboundGateways(t, s2, 1, time.Second) waitForInboundGateways(t, s2, 1, time.Second) s1 = runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s2, 2, time.Second) waitForInboundGateways(t, s2, 2, time.Second) waitForOutboundGateways(t, s1, 1, time.Second) waitForInboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s3, 1, time.Second) waitForInboundGateways(t, s3, 1, time.Second) // It should not have a registered remote gateway with C (s3) if s1.getOutboundGatewayConnection("C") != nil { t.Fatalf("A should not have outbound gateway to C") } if s1.getRemoteGateway("C") != nil { t.Fatalf("A should not have a registered remote gateway to C") } } func TestGatewayNoReconnectOnClose(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) // Shutdown s1, and check that there is no attempt to reconnect. s1.Shutdown() time.Sleep(250 * time.Millisecond) waitForOutboundGateways(t, s1, 0, time.Second) waitForOutboundGateways(t, s2, 0, time.Second) waitForInboundGateways(t, s2, 0, time.Second) } func TestGatewayDontSendSubInterest(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) s2Url := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) subnc := natsConnect(t, s2Url) defer subnc.Close() natsSub(t, subnc, "foo", func(_ *nats.Msg) {}) natsFlush(t, subnc) checkExpectedSubs(t, 1, s2) // Subscription should not be sent to s1 checkExpectedSubs(t, 0, s1) // Restart s1 s1.Shutdown() s1 = runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) checkExpectedSubs(t, 1, s2) checkExpectedSubs(t, 0, s1) } func setAccountUserPassInOptions(o *Options, accName, username, password string) { acc := NewAccount(accName) o.Accounts = append(o.Accounts, acc) o.Users = append(o.Users, &User{Username: username, Password: password, Account: acc}) } func TestGatewayAccountInterest(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) o2 := testDefaultOptionsForGateway("B") // Add users to cause s2 to require auth. Will add an account with user later. o2.Users = append([]*User(nil), &User{Username: "test", Password: "pwd"}) s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) setAccountUserPassInOptions(o1, "$foo", "ivan", "password") s1 := runGatewayServer(o1) defer s1.Shutdown() // Make this server initiate connection to A, so it is faster // when restarting it at the end of this test. o3 := testGatewayOptionsFromToWithServers(t, "C", "A", s1) setAccountUserPassInOptions(o3, "$foo", "ivan", "password") s3 := runGatewayServer(o3) defer s3.Shutdown() waitForOutboundGateways(t, s1, 2, time.Second) waitForOutboundGateways(t, s2, 2, time.Second) waitForOutboundGateways(t, s3, 2, time.Second) s1Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o1.Port) nc := natsConnect(t, s1Url) defer nc.Close() natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) // On first send, the message should be sent. checkCount := func(t *testing.T, c *client, expected int) { t.Helper() c.mu.Lock() out := c.outMsgs c.mu.Unlock() if int(out) != expected { t.Fatalf("Expected %d message(s) to be sent over, got %v", expected, out) } } gwcb := s1.getOutboundGatewayConnection("B") checkCount(t, gwcb, 1) gwcc := s1.getOutboundGatewayConnection("C") checkCount(t, gwcc, 1) // S2 and S3 should have sent a protocol indicating no account interest. checkForAccountNoInterest(t, gwcb, "$foo", true, 2*time.Second) checkForAccountNoInterest(t, gwcc, "$foo", true, 2*time.Second) // Second send should not go to B nor C. natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 1) checkCount(t, gwcc, 1) // Add account to S2 and a client, this should clear the no-interest // for that account. s2FooAcc, err := s2.RegisterAccount("$foo") if err != nil { t.Fatalf("Error registering account: %v", err) } s2.mu.Lock() s2.users["ivan"] = &User{Account: s2FooAcc, Username: "ivan", Password: "password"} s2.mu.Unlock() s2Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o2.Port) ncS2 := natsConnect(t, s2Url) defer ncS2.Close() // Any subscription should cause s2 to send an A+ natsSubSync(t, ncS2, "asub") // Wait for the A+ checkForAccountNoInterest(t, gwcb, "$foo", false, 2*time.Second) // Now publish a message that should go to B natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 2) // Still won't go to C since there is no sub interest checkCount(t, gwcc, 1) // We should have received a subject no interest for foo checkForSubjectNoInterest(t, gwcb, "$foo", "foo", true, 2*time.Second) // Now if we close the client, which removed the sole subscription, // and publish to a new subject, we should then get an A- ncS2.Close() // Wait a bit... time.Sleep(20 * time.Millisecond) // Publish on new subject natsPub(t, nc, "bar", []byte("hello")) natsFlush(t, nc) // It should go out to B... checkCount(t, gwcb, 3) // But then we should get a A- checkForAccountNoInterest(t, gwcb, "$foo", true, 2*time.Second) // Restart C and that should reset the no-interest s3.Shutdown() s3 = runGatewayServer(o3) defer s3.Shutdown() waitForOutboundGateways(t, s1, 2, 2*time.Second) waitForOutboundGateways(t, s2, 2, 2*time.Second) waitForOutboundGateways(t, s3, 2, 2*time.Second) // First refresh gwcc gwcc = s1.getOutboundGatewayConnection("C") // Verify that it's count is 0 checkCount(t, gwcc, 0) // Publish and now... natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) // it should not go to B (no sub interest) checkCount(t, gwcb, 3) // but will go to C checkCount(t, gwcc, 1) } func TestGatewayAccountUnsub(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Connect on B ncb := natsConnect(t, fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port)) defer ncb.Close() // Create subscription natsSub(t, ncb, "foo", func(m *nats.Msg) { ncb.Publish(m.Reply, []byte("reply")) }) natsFlush(t, ncb) // Connect on A nca := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer nca.Close() // Send a request if _, err := nca.Request("foo", []byte("req"), time.Second); err != nil { t.Fatalf("Error getting reply: %v", err) } // Now close connection on B ncb.Close() // Publish lots of messages on "foo" from A. // We should receive an A- shortly and the number // of outbound messages from A to B should not be // close to the number of messages sent here. total := 5000 for i := 0; i < total; i++ { natsPub(t, nca, "foo", []byte("hello")) // Try to slow down things a bit to give a chance // to srvB to send the A- and to srvA to be able // to process it, which will then suppress the sends. if i%100 == 0 { natsFlush(t, nca) } } natsFlush(t, nca) c := sa.getOutboundGatewayConnection("B") c.mu.Lock() out := c.outMsgs c.mu.Unlock() if out >= int64(80*total)/100 { t.Fatalf("Unexpected number of messages sent from A to B: %v", out) } } func TestGatewaySubjectInterest(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) o1 := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(o1, "$foo", "ivan", "password") s1 := runGatewayServer(o1) defer s1.Shutdown() o2 := testGatewayOptionsFromToWithServers(t, "B", "A", s1) setAccountUserPassInOptions(o2, "$foo", "ivan", "password") s2 := runGatewayServer(o2) defer s2.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) // We will create a subscription that we are not testing so // that we don't get an A- in this test. s2Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o2.Port) ncb := natsConnect(t, s2Url) defer ncb.Close() natsSubSync(t, ncb, "not.used") checkExpectedSubs(t, 1, s2) s1Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o1.Port) nc := natsConnect(t, s1Url) defer nc.Close() natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) // On first send, the message should be sent. checkCount := func(t *testing.T, c *client, expected int) { t.Helper() c.mu.Lock() out := c.outMsgs c.mu.Unlock() if int(out) != expected { t.Fatalf("Expected %d message(s) to be sent over, got %v", expected, out) } } gwcb := s1.getOutboundGatewayConnection("B") checkCount(t, gwcb, 1) // S2 should have sent a protocol indicating no subject interest. checkNoInterest := func(t *testing.T, subject string, expectedNoInterest bool) { t.Helper() checkForSubjectNoInterest(t, gwcb, "$foo", subject, expectedNoInterest, 2*time.Second) } checkNoInterest(t, "foo", true) // Second send should not go through to B natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 1) // Now create subscription interest on B (s2) ch := make(chan bool, 1) sub := natsSub(t, ncb, "foo", func(_ *nats.Msg) { ch <- true }) natsFlush(t, ncb) checkExpectedSubs(t, 2, s2) checkExpectedSubs(t, 0, s1) // This should clear the no interest for this subject checkNoInterest(t, "foo", false) // Third send should go to B natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 2) // Make sure message is received waitCh(t, ch, "Did not get our message") // Now unsubscribe, there won't be an UNSUB sent to the gateway. natsUnsub(t, sub) natsFlush(t, ncb) checkExpectedSubs(t, 1, s2) checkExpectedSubs(t, 0, s1) // So now sending a message should go over, but then we should get an RS- natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 3) checkNoInterest(t, "foo", true) // Send one more time and now it should not go to B natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 3) // Send on bar, message should go over. natsPub(t, nc, "bar", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 4) // But now we should have receives an RS- on bar. checkNoInterest(t, "bar", true) // Check that wildcards are supported. Create a subscription on '*' on B. // This should clear the no-interest on both "foo" and "bar" natsSub(t, ncb, "*", func(_ *nats.Msg) {}) natsFlush(t, ncb) checkExpectedSubs(t, 2, s2) checkExpectedSubs(t, 0, s1) checkNoInterest(t, "foo", false) checkNoInterest(t, "bar", false) // Publish on message on foo and one on bar and they should go. natsPub(t, nc, "foo", []byte("hello")) natsPub(t, nc, "bar", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 6) // Restart B and that should clear everything on A ncb.Close() s2.Shutdown() s2 = runGatewayServer(o2) defer s2.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) ncb = natsConnect(t, s2Url) defer ncb.Close() natsSubSync(t, ncb, "not.used") checkExpectedSubs(t, 1, s2) gwcb = s1.getOutboundGatewayConnection("B") checkCount(t, gwcb, 0) natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 1) checkNoInterest(t, "foo", true) natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 1) // Add a node to B cluster and subscribe there. // We want to ensure that the no-interest is cleared // when s2 receives remote SUB from s2bis o2bis := testGatewayOptionsFromToWithServers(t, "B", "A", s1) setAccountUserPassInOptions(o2bis, "$foo", "ivan", "password") o2bis.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)) s2bis := runGatewayServer(o2bis) defer s2bis.Shutdown() checkClusterFormed(t, s2, s2bis) // Make sure all outbound gateway connections are setup waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) waitForOutboundGateways(t, s2bis, 1, time.Second) // A should have 2 inbound waitForInboundGateways(t, s1, 2, time.Second) // Create sub on s2bis ncb2bis := natsConnect(t, fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o2bis.Port)) defer ncb2bis.Close() natsSub(t, ncb2bis, "foo", func(_ *nats.Msg) {}) natsFlush(t, ncb2bis) // Wait for subscriptions to be registered locally on s2bis and remotely on s2 checkExpectedSubs(t, 2, s2, s2bis) // Check that subject no-interest on A was cleared. checkNoInterest(t, "foo", false) // Now publish. Remember, s1 has outbound gateway to s2, and s2 does not // have a local subscription and has previously sent a no-interest on "foo". // We check that this has been cleared due to the interest on s2bis. natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 2) } func TestGatewayDoesntSendBackToItself(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) s2Url := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) nc2 := natsConnect(t, s2Url) defer nc2.Close() count := int32(0) cb := func(_ *nats.Msg) { atomic.AddInt32(&count, 1) } natsSub(t, nc2, "foo", cb) natsFlush(t, nc2) s1Url := fmt.Sprintf("nats://127.0.0.1:%d", o1.Port) nc1 := natsConnect(t, s1Url) defer nc1.Close() natsSub(t, nc1, "foo", cb) natsFlush(t, nc1) // Now send 1 message. If there is a cycle, after few ms we // should have tons of messages... natsPub(t, nc1, "foo", []byte("cycle")) natsFlush(t, nc1) time.Sleep(100 * time.Millisecond) if c := atomic.LoadInt32(&count); c != 2 { t.Fatalf("Expected only 2 messages, got %v", c) } } func TestGatewayOrderedOutbounds(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() o3 := testGatewayOptionsFromToWithServers(t, "C", "B", s2) s3 := runGatewayServer(o3) defer s3.Shutdown() waitForOutboundGateways(t, s1, 2, time.Second) waitForOutboundGateways(t, s2, 2, time.Second) waitForOutboundGateways(t, s3, 2, time.Second) gws := make([]*client, 0, 2) s2.getOutboundGatewayConnections(&gws) // RTTs are expected to be initially 0. So update RTT of first // in the array so that its value is no longer 0, this should // cause order to be flipped. c := gws[0] c.mu.Lock() c.sendPing() c.mu.Unlock() // Wait a tiny but time.Sleep(15 * time.Millisecond) // Get the ordering again. gws = gws[:0] s2.getOutboundGatewayConnections(&gws) // Verify order is correct. fRTT := gws[0].getRTTValue() sRTT := gws[1].getRTTValue() if fRTT > sRTT { t.Fatalf("Wrong ordering: %v, %v", fRTT, sRTT) } // What is the first in the array? gws[0].mu.Lock() gwName := gws[0].gw.name gws[0].mu.Unlock() if gwName == "A" { s1.Shutdown() } else { s3.Shutdown() } waitForOutboundGateways(t, s2, 1, time.Second) gws = gws[:0] s2.getOutboundGatewayConnections(&gws) if len(gws) != 1 { t.Fatalf("Expected size of outo to be 1, got %v", len(gws)) } gws[0].mu.Lock() name := gws[0].gw.name gws[0].mu.Unlock() if gwName == name { t.Fatalf("Gateway %q should have been removed", gwName) } // Stop the remaining gateway if gwName == "A" { s3.Shutdown() } else { s1.Shutdown() } waitForOutboundGateways(t, s2, 0, time.Second) gws = gws[:0] s2.getOutboundGatewayConnections(&gws) if len(gws) != 0 { t.Fatalf("Expected size of outo to be 0, got %v", len(gws)) } } func TestGatewayQueueSub(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) sBUrl := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) ncB := natsConnect(t, sBUrl) defer ncB.Close() count2 := int32(0) cb2 := func(_ *nats.Msg) { atomic.AddInt32(&count2, 1) } qsubOnB := natsQueueSub(t, ncB, "foo", "bar", cb2) natsFlush(t, ncB) sAUrl := fmt.Sprintf("nats://127.0.0.1:%d", o1.Port) ncA := natsConnect(t, sAUrl) defer ncA.Close() count1 := int32(0) cb1 := func(_ *nats.Msg) { atomic.AddInt32(&count1, 1) } qsubOnA := natsQueueSub(t, ncA, "foo", "bar", cb1) natsFlush(t, ncA) // Make sure subs are registered on each server checkExpectedSubs(t, 1, s1, s2) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 1, time.Second) checkForRegisteredQSubInterest(t, s2, "A", globalAccountName, "foo", 1, time.Second) total := 100 send := func(t *testing.T, nc *nats.Conn) { t.Helper() for i := 0; i < total; i++ { // Alternate with adding a reply if i%2 == 0 { natsPubReq(t, nc, "foo", "reply", []byte("msg")) } else { natsPub(t, nc, "foo", []byte("msg")) } } natsFlush(t, nc) } // Send from client connecting to S1 (cluster A) send(t, ncA) check := func(t *testing.T, count *int32, expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if n := int(atomic.LoadInt32(count)); n != expected { return fmt.Errorf("Expected to get %v messages, got %v", expected, n) } return nil }) } // Check that all messages stay on S1 (cluster A) check(t, &count1, total) check(t, &count2, 0) // Now send from the other side send(t, ncB) check(t, &count1, total) check(t, &count2, total) // Reset counters atomic.StoreInt32(&count1, 0) atomic.StoreInt32(&count2, 0) // Add different queue group and make sure that messages are received count3 := int32(0) cb3 := func(_ *nats.Msg) { atomic.AddInt32(&count3, 1) } batQSub := natsQueueSub(t, ncB, "foo", "bat", cb3) natsFlush(t, ncB) checkExpectedSubs(t, 2, s2) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 2, time.Second) send(t, ncA) check(t, &count1, total) check(t, &count2, 0) check(t, &count3, total) // Reset counters atomic.StoreInt32(&count1, 0) atomic.StoreInt32(&count2, 0) atomic.StoreInt32(&count3, 0) natsUnsub(t, batQSub) natsFlush(t, ncB) checkExpectedSubs(t, 1, s2) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 1, time.Second) // Stop qsub on A, and send messages to A, they should // be routed to B. qsubOnA.Unsubscribe() checkExpectedSubs(t, 0, s1) send(t, ncA) check(t, &count1, 0) check(t, &count2, total) // Reset counters atomic.StoreInt32(&count1, 0) atomic.StoreInt32(&count2, 0) // Create a C gateway now o3 := testGatewayOptionsFromToWithServers(t, "C", "B", s2) s3 := runGatewayServer(o3) defer s3.Shutdown() waitForOutboundGateways(t, s1, 2, time.Second) waitForOutboundGateways(t, s2, 2, time.Second) waitForOutboundGateways(t, s3, 2, time.Second) waitForInboundGateways(t, s1, 2, time.Second) waitForInboundGateways(t, s2, 2, time.Second) waitForInboundGateways(t, s3, 2, time.Second) // Create another qsub "bar" sCUrl := fmt.Sprintf("nats://127.0.0.1:%d", o3.Port) ncC := natsConnect(t, sCUrl) defer ncC.Close() // Associate this with count1 (since A qsub is no longer running) natsQueueSub(t, ncC, "foo", "bar", cb1) natsFlush(t, ncC) checkExpectedSubs(t, 1, s3) checkForRegisteredQSubInterest(t, s1, "C", globalAccountName, "foo", 1, time.Second) checkForRegisteredQSubInterest(t, s2, "C", globalAccountName, "foo", 1, time.Second) // Artificially bump the RTT from A to C so that // the code should favor sending to B. gwcC := s1.getOutboundGatewayConnection("C") gwcC.mu.Lock() gwcC.rtt = 10 * time.Second gwcC.mu.Unlock() s1.gateway.orderOutboundConnections() send(t, ncA) check(t, &count1, 0) check(t, &count2, total) // Add a new group on s3 that should receive all messages natsQueueSub(t, ncC, "foo", "baz", cb3) natsFlush(t, ncC) checkExpectedSubs(t, 2, s3) checkForRegisteredQSubInterest(t, s1, "C", globalAccountName, "foo", 2, time.Second) checkForRegisteredQSubInterest(t, s2, "C", globalAccountName, "foo", 2, time.Second) // Reset counters atomic.StoreInt32(&count1, 0) atomic.StoreInt32(&count2, 0) // Make the RTTs equal gwcC.mu.Lock() gwcC.rtt = time.Second gwcC.mu.Unlock() gwcB := s1.getOutboundGatewayConnection("B") gwcB.mu.Lock() gwcB.rtt = time.Second gwcB.mu.Unlock() s1.gateway.Lock() s1.gateway.orderOutboundConnectionsLocked() destName := s1.gateway.outo[0].gw.name s1.gateway.Unlock() send(t, ncA) // Group baz should receive all messages check(t, &count3, total) // Ordering is normally re-evaluated when processing PONGs, // but rest of the time order will remain the same. // Since RTT are equal, messages will go to the first // GW in the array. if destName == "B" { check(t, &count2, total) } else if destName == "C" && int(atomic.LoadInt32(&count2)) != total { check(t, &count1, total) } // Unsubscribe qsub on B and C should receive // all messages on count1 and count3. qsubOnB.Unsubscribe() checkExpectedSubs(t, 0, s2) // gwcB should have the qsubs interest map empty now. checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { ei, _ := gwcB.gw.outsim.Load(globalAccountName) if ei != nil { sl := ei.(*outsie).sl if sl.Count() == 0 { return nil } } return fmt.Errorf("Qsub interest for account should have been removed") }) // Reset counters atomic.StoreInt32(&count1, 0) atomic.StoreInt32(&count2, 0) atomic.StoreInt32(&count3, 0) send(t, ncA) check(t, &count1, total) check(t, &count3, total) } func TestGatewayTotalQSubs(t *testing.T) { ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() ob2 := testDefaultOptionsForGateway("B") ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) sb2 := runGatewayServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) sb1URL := fmt.Sprintf("nats://%s:%d", ob1.Host, ob1.Port) ncb1 := natsConnect(t, sb1URL, nats.ReconnectWait(50*time.Millisecond)) defer ncb1.Close() sb2URL := fmt.Sprintf("nats://%s:%d", ob2.Host, ob2.Port) ncb2 := natsConnect(t, sb2URL, nats.ReconnectWait(50*time.Millisecond)) defer ncb2.Close() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa, 2, 2*time.Second) waitForInboundGateways(t, sb1, 1, 2*time.Second) checkTotalQSubs := func(t *testing.T, s *Server, expected int) { t.Helper() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := int(atomic.LoadInt64(&s.gateway.totalQSubs)); n != expected { return fmt.Errorf("Expected TotalQSubs to be %v, got %v", expected, n) } return nil }) } cb := func(_ *nats.Msg) {} natsQueueSub(t, ncb1, "foo", "bar", cb) checkTotalQSubs(t, sa, 1) qsub2 := natsQueueSub(t, ncb1, "foo", "baz", cb) checkTotalQSubs(t, sa, 2) qsub3 := natsQueueSub(t, ncb1, "foo", "baz", cb) checkTotalQSubs(t, sa, 2) // Shutdown sb1, there should be a failover from clients // to sb2. sb2 will then send the queue subs to sa. sb1.Shutdown() checkClientsCount(t, sb2, 2) checkExpectedSubs(t, 3, sb2) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sb2, 1, 2*time.Second) // When sb1 is shutdown, the total qsubs on sa should fall // down to 0, but will be updated as soon as sa and sb2 // connect to each other. So instead we will verify by // making sure that the count is 2 instead of 4 if there // was a bug. // (note that there are 2 qsubs on same group, so only // 1 RS+ would have been sent for that group) checkTotalQSubs(t, sa, 2) // Restart sb1 sb1 = runGatewayServer(ob1) defer sb1.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa, 2, 2*time.Second) waitForInboundGateways(t, sb1, 0, 2*time.Second) waitForInboundGateways(t, sb2, 1, 2*time.Second) // Now start unsubscribing. Start with one of the duplicate // and check that count stays same. natsUnsub(t, qsub3) checkTotalQSubs(t, sa, 2) // Now the other, which would cause an RS- natsUnsub(t, qsub2) checkTotalQSubs(t, sa, 1) // Now test that if connections are closed, things are updated // properly. ncb1.Close() ncb2.Close() checkTotalQSubs(t, sa, 0) } func TestGatewaySendQSubsOnGatewayConnect(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() s2Url := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) subnc := natsConnect(t, s2Url) defer subnc.Close() ch := make(chan bool, 1) cb := func(_ *nats.Msg) { ch <- true } natsQueueSub(t, subnc, "foo", "bar", cb) natsFlush(t, subnc) // Now start s1 that creates a gateway to s2 o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 1, time.Second) // Publish from s1, message should be received on s2. pubnc := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", o1.Port)) defer pubnc.Close() // Publish 1 message natsPub(t, pubnc, "foo", []byte("hello")) waitCh(t, ch, "Did not get out message") pubnc.Close() s1.Shutdown() s1 = runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 1, time.Second) pubnc = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", o1.Port)) defer pubnc.Close() // Publish 1 message natsPub(t, pubnc, "foo", []byte("hello")) waitCh(t, ch, "Did not get out message") } func TestGatewaySendRemoteQSubs(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() ob2 := testDefaultOptionsForGateway("B") ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", ob1.Cluster.Host, ob1.Cluster.Port)) sb2 := runGatewayServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) sbURL := fmt.Sprintf("nats://127.0.0.1:%d", ob2.Port) subnc := natsConnect(t, sbURL) defer subnc.Close() ch := make(chan bool, 1) cb := func(_ *nats.Msg) { ch <- true } qsub1 := natsQueueSub(t, subnc, "foo", "bar", cb) qsub2 := natsQueueSub(t, subnc, "foo", "bar", cb) natsFlush(t, subnc) // There will be 2 local qsubs on the sb2 server where the client is connected checkExpectedSubs(t, 2, sb2) // But only 1 remote on sb1 checkExpectedSubs(t, 1, sb1) // Now start s1 that creates a gateway to sb1 (the one that does not have the local QSub) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, time.Second) checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "foo", 1, time.Second) // Publish from s1, message should be received on s2. saURL := fmt.Sprintf("nats://127.0.0.1:%d", oa.Port) pubnc := natsConnect(t, saURL) defer pubnc.Close() // Publish 1 message natsPub(t, pubnc, "foo", []byte("hello")) natsFlush(t, pubnc) waitCh(t, ch, "Did not get out message") // Note that since cluster B has no plain sub, an "RS- $G foo" will have been sent. // Wait for the no interest to be received by A checkFor(t, time.Second, 15*time.Millisecond, func() error { gw := sa.getOutboundGatewayConnection("B").gw ei, _ := gw.outsim.Load(globalAccountName) if ei != nil { e := ei.(*outsie) e.RLock() defer e.RUnlock() if _, inMap := e.ni["foo"]; inMap { return nil } } return fmt.Errorf("No-interest still not registered") }) // Unsubscribe 1 qsub natsUnsub(t, qsub1) natsFlush(t, subnc) // There should be only 1 local qsub on sb2 now, and the remote should still exist on sb1 checkExpectedSubs(t, 1, sb1, sb2) // Publish 1 message natsPub(t, pubnc, "foo", []byte("hello")) natsFlush(t, pubnc) waitCh(t, ch, "Did not get out message") // Unsubscribe the remaining natsUnsub(t, qsub2) natsFlush(t, subnc) // No more subs now on both sb1 and sb2 checkExpectedSubs(t, 0, sb1, sb2) // Server sb1 should not have qsub in its sub interest map checkFor(t, time.Second, 15*time.Millisecond, func() error { var entry *sitally var err error sb1.gateway.pasi.Lock() asim := sb1.gateway.pasi.m[globalAccountName] if asim != nil { entry = asim["foo bar"] } if entry != nil { err = fmt.Errorf("Map should not have an entry, got %#v", entry) } sb1.gateway.pasi.Unlock() return err }) // Let's wait for A to receive the unsubscribe checkFor(t, time.Second, 15*time.Millisecond, func() error { gw := sa.getOutboundGatewayConnection("B").gw ei, _ := gw.outsim.Load(globalAccountName) if ei != nil { sl := ei.(*outsie).sl if sl.Count() == 0 { return nil } } return fmt.Errorf("Interest still present") }) // Now send a message, it won't be sent because A received an RS- // on the first published message since there was no plain sub interest. natsPub(t, pubnc, "foo", []byte("hello")) natsFlush(t, pubnc) // Get the gateway connection from A (sa) to B (sb1) gw := sa.getOutboundGatewayConnection("B") gw.mu.Lock() out := gw.outMsgs gw.mu.Unlock() if out != 2 { t.Fatalf("Expected 2 out messages, got %v", out) } // Restart A pubnc.Close() sa.Shutdown() sa = runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) // Check qsubs interest should be empty checkFor(t, time.Second, 15*time.Millisecond, func() error { gw := sa.getOutboundGatewayConnection("B").gw if ei, _ := gw.outsim.Load(globalAccountName); ei == nil { return nil } return fmt.Errorf("Interest still present") }) } func TestGatewayComplexSetup(t *testing.T) { doLog := false // This test will have the following setup: // --- means route connection // === means gateway connection // [o] is outbound // [i] is inbound // Each server as an outbound connection to the other cluster. // It may have 0 or more inbound connection(s). // // Cluster A Cluster B // sa1 [o]===========>[i] // | [i]<===========[o] // | sb1 ------- sb2 // | [i] [o] // sa2 [o]=============^ || // [i]<========================|| ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() if doLog { sb1.SetLogger(logger.NewTestLogger("[B1] - ", true), true, true) } oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa1 := runGatewayServer(oa1) defer sa1.Shutdown() if doLog { sa1.SetLogger(logger.NewTestLogger("[A1] - ", true), true, true) } waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sa1.ClusterAddr().Port)) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() if doLog { sa2.SetLogger(logger.NewTestLogger("[A2] - ", true), true, true) } checkClusterFormed(t, sa1, sa2) waitForOutboundGateways(t, sa2, 1, time.Second) waitForInboundGateways(t, sb1, 2, time.Second) ob2 := testGatewayOptionsFromToWithServers(t, "B", "A", sa2) ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) var sb2 *Server for { sb2 = runGatewayServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sb2, 1, time.Second) waitForInboundGateways(t, sb2, 0, time.Second) // For this test, we want the outbound to be to sa2, so if we don't have that, // restart sb2 until we get lucky. time.Sleep(100 * time.Millisecond) if sa2.numInboundGateways() == 0 { sb2.Shutdown() sb2 = nil } else { break } } if doLog { sb2.SetLogger(logger.NewTestLogger("[B2] - ", true), true, true) } ch := make(chan bool, 1) cb := func(_ *nats.Msg) { ch <- true } // Create a subscription on sa1 and sa2. ncsa1 := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", oa1.Port)) defer ncsa1.Close() sub1 := natsSub(t, ncsa1, "foo", cb) natsFlush(t, ncsa1) ncsa2 := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", oa2.Port)) defer ncsa2.Close() sub2 := natsSub(t, ncsa2, "foo", cb) natsFlush(t, ncsa2) // sa1 will have 1 local, one remote (from sa2), same for sa2. checkExpectedSubs(t, 2, sa1, sa2) // Connect to sb2 and send 1 message ncsb2 := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob2.Port)) defer ncsb2.Close() natsPub(t, ncsb2, "foo", []byte("hello")) natsFlush(t, ncsb2) for i := 0; i < 2; i++ { waitCh(t, ch, "Did not get our message") } // Unsubscribe sub2, and send 1, should still get it. natsUnsub(t, sub2) natsFlush(t, ncsa2) natsPub(t, ncsb2, "foo", []byte("hello")) natsFlush(t, ncsb2) waitCh(t, ch, "Did not get our message") // Unsubscribe sub1, all server's sublist should be empty sub1.Unsubscribe() natsFlush(t, ncsa1) checkExpectedSubs(t, 0, sa1, sa2, sb1, sb2) // Create queue subs total := 100 c1 := int32(0) c2 := int32(0) c3 := int32(0) tc := int32(0) natsQueueSub(t, ncsa1, "foo", "bar", func(_ *nats.Msg) { atomic.AddInt32(&c1, 1) if c := atomic.AddInt32(&tc, 1); int(c) == total { ch <- true } }) natsFlush(t, ncsa1) natsQueueSub(t, ncsa2, "foo", "bar", func(_ *nats.Msg) { atomic.AddInt32(&c2, 1) if c := atomic.AddInt32(&tc, 1); int(c) == total { ch <- true } }) natsFlush(t, ncsa2) checkExpectedSubs(t, 2, sa1, sa2) qsubOnB2 := natsQueueSub(t, ncsb2, "foo", "bar", func(_ *nats.Msg) { atomic.AddInt32(&c3, 1) if c := atomic.AddInt32(&tc, 1); int(c) == total { ch <- true } }) natsFlush(t, ncsb2) checkExpectedSubs(t, 1, sb2) checkForRegisteredQSubInterest(t, sb1, "A", globalAccountName, "foo", 1, time.Second) // Publish all messages. The queue sub on cluster B should receive all // messages. for i := 0; i < total; i++ { natsPub(t, ncsb2, "foo", []byte("msg")) } natsFlush(t, ncsb2) waitCh(t, ch, "Did not get all our queue messages") if n := int(atomic.LoadInt32(&c1)); n != 0 { t.Fatalf("No message should have been received by qsub1, got %v", n) } if n := int(atomic.LoadInt32(&c2)); n != 0 { t.Fatalf("No message should have been received by qsub2, got %v", n) } if n := int(atomic.LoadInt32(&c3)); n != total { t.Fatalf("All messages should have been delivered to qsub on B, got %v", n) } // Reset counters atomic.StoreInt32(&c1, 0) atomic.StoreInt32(&c2, 0) atomic.StoreInt32(&c3, 0) atomic.StoreInt32(&tc, 0) // Now send from cluster A, messages should be distributed to qsubs on A. for i := 0; i < total; i++ { natsPub(t, ncsa1, "foo", []byte("msg")) } natsFlush(t, ncsa1) expectedLow := int(float32(total/2) * 0.6) expectedHigh := int(float32(total/2) * 1.4) checkCount := func(t *testing.T, count *int32) { t.Helper() c := int(atomic.LoadInt32(count)) if c < expectedLow || c > expectedHigh { t.Fatalf("Expected value to be between %v/%v, got %v", expectedLow, expectedHigh, c) } } waitCh(t, ch, "Did not get all our queue messages") checkCount(t, &c1) checkCount(t, &c2) // Now unsubscribe sub on B and reset counters natsUnsub(t, qsubOnB2) checkExpectedSubs(t, 0, sb2) atomic.StoreInt32(&c1, 0) atomic.StoreInt32(&c2, 0) atomic.StoreInt32(&c3, 0) atomic.StoreInt32(&tc, 0) // Publish from cluster B, messages should be delivered to cluster A. for i := 0; i < total; i++ { natsPub(t, ncsb2, "foo", []byte("msg")) } natsFlush(t, ncsb2) waitCh(t, ch, "Did not get all our queue messages") if n := int(atomic.LoadInt32(&c3)); n != 0 { t.Fatalf("There should not have been messages on unsubscribed sub, got %v", n) } checkCount(t, &c1) checkCount(t, &c2) } func TestGatewayMsgSentOnlyOnce(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) s2Url := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) nc2 := natsConnect(t, s2Url) defer nc2.Close() s1Url := fmt.Sprintf("nats://127.0.0.1:%d", o1.Port) nc1 := natsConnect(t, s1Url) defer nc1.Close() ch := make(chan bool, 1) count := int32(0) expected := int32(4) cb := func(_ *nats.Msg) { if c := atomic.AddInt32(&count, 1); c == expected { ch <- true } } // On s1, create 2 plain subs, 2 queue members for group // "bar" and 1 for group "baz". natsSub(t, nc1, ">", cb) natsSub(t, nc1, "foo", cb) natsQueueSub(t, nc1, "foo", "bar", cb) natsQueueSub(t, nc1, "foo", "bar", cb) natsQueueSub(t, nc1, "foo", "baz", cb) natsFlush(t, nc1) // Ensure subs registered in S1 checkExpectedSubs(t, 5, s1) // Also need to wait for qsubs to be registered on s2. checkForRegisteredQSubInterest(t, s2, "A", globalAccountName, "foo", 2, time.Second) // From s2, send 1 message, s1 should receive 1 only, // and total we should get the callback notified 4 times. natsPub(t, nc2, "foo", []byte("hello")) natsFlush(t, nc2) waitCh(t, ch, "Did not get our messages") // Verifiy that count is still 4 if c := atomic.LoadInt32(&count); c != expected { t.Fatalf("Expected %v messages, got %v", expected, c) } // Check s2 outbound connection stats. It should say that it // sent only 1 message. c := s2.getOutboundGatewayConnection("A") if c == nil { t.Fatalf("S2 outbound gateway not found") } c.mu.Lock() out := c.outMsgs c.mu.Unlock() if out != 1 { t.Fatalf("Expected s2's outbound gateway to have sent a single message, got %v", out) } // Now check s1's inbound gateway s1.gateway.RLock() c = nil for _, ci := range s1.gateway.in { c = ci break } s1.gateway.RUnlock() if c == nil { t.Fatalf("S1 inbound gateway not found") } if in := atomic.LoadInt64(&c.inMsgs); in != 1 { t.Fatalf("Expected s1's inbound gateway to have received a single message, got %v", in) } } type checkErrorLogger struct { DummyLogger checkErrorStr string gotError bool } func (l *checkErrorLogger) Errorf(format string, args ...any) { l.DummyLogger.Errorf(format, args...) l.Lock() if strings.Contains(l.Msg, l.checkErrorStr) { l.gotError = true } l.Unlock() } func TestGatewayRoutedServerWithoutGatewayConfigured(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) o3 := DefaultOptions() o3.NoSystemAccount = true o3.Cluster.Name = "B" o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s2.ClusterAddr().Port)) s3 := New(o3) defer s3.Shutdown() l := &checkErrorLogger{checkErrorStr: "not configured"} s3.SetLogger(l, true, true) wg := sync.WaitGroup{} wg.Add(1) go func() { s3.Start() wg.Done() }() checkClusterFormed(t, s2, s3) // Check that server s3 does not panic when being notified // about the A gateway, but report an error. deadline := time.Now().Add(2 * time.Second) gotIt := false for time.Now().Before(deadline) { l.Lock() gotIt = l.gotError l.Unlock() if gotIt { break } time.Sleep(15 * time.Millisecond) } if !gotIt { t.Fatalf("Should have reported error about gateway not configured") } s3.Shutdown() wg.Wait() } func TestGatewaySendsToNonLocalSubs(t *testing.T) { ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa1 := runGatewayServer(oa1) defer sa1.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sa1.ClusterAddr().Port)) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() checkClusterFormed(t, sa1, sa2) waitForOutboundGateways(t, sa2, 1, time.Second) waitForInboundGateways(t, sb1, 2, time.Second) ch := make(chan bool, 1) // Create an interest of sa2 ncSub := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", oa2.Port)) defer ncSub.Close() natsSub(t, ncSub, "foo", func(_ *nats.Msg) { ch <- true }) natsFlush(t, ncSub) checkExpectedSubs(t, 1, sa1, sa2) // Produce a message from sb1, make sure it can be received. ncPub := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob1.Port)) defer ncPub.Close() natsPub(t, ncPub, "foo", []byte("hello")) waitCh(t, ch, "Did not get our message") ncSub.Close() ncPub.Close() checkExpectedSubs(t, 0, sa1, sa2) // Now create sb2 that has a route to sb1 and gateway connects to sa2. ob2 := testGatewayOptionsFromToWithServers(t, "B", "A", sa2) ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) sb2 := runGatewayServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sb2, 1, time.Second) ncSub = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", oa1.Port)) defer ncSub.Close() natsSub(t, ncSub, "foo", func(_ *nats.Msg) { ch <- true }) natsFlush(t, ncSub) checkExpectedSubs(t, 1, sa1, sa2) ncPub = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob2.Port)) defer ncPub.Close() natsPub(t, ncPub, "foo", []byte("hello")) waitCh(t, ch, "Did not get our message") } func TestGatewayUnknownGatewayCommand(t *testing.T) { o1 := testDefaultOptionsForGateway("A") s1 := runGatewayServer(o1) defer s1.Shutdown() l := &checkErrorLogger{checkErrorStr: "Unknown command"} s1.SetLogger(l, true, true) o2 := testDefaultOptionsForGateway("A") o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", s1.ClusterAddr().Port)) s2 := runGatewayServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) var route *client s2.mu.Lock() if r := getFirstRoute(s2); r != nil { route = r } s2.mu.Unlock() route.mu.Lock() info := &Info{ Gateway: "B", GatewayCmd: 255, } b, _ := json.Marshal(info) route.enqueueProto([]byte(fmt.Sprintf(InfoProto, b))) route.mu.Unlock() checkFor(t, time.Second, 15*time.Millisecond, func() error { l.Lock() gotIt := l.gotError l.Unlock() if gotIt { return nil } return fmt.Errorf("Did not get expected error") }) } func TestGatewayRandomIP(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithURLs(t, "A", "B", []string{ "nats://noport", fmt.Sprintf("nats://localhost:%d", sb.GatewayAddr().Port), }) // Create a dummy resolver that returns error since we // don't provide any IP. The code should then use the configured // url (localhost:port) and try with that, which in this case // should work. oa.Gateway.resolver = &myDummyDNSResolver{} sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) } func TestGatewaySendQSubsBufSize(t *testing.T) { for _, test := range []struct { name string bufSize int }{ { name: "Bufsize 45, more than one at a time", bufSize: 45, }, { name: "Bufsize 15, one at a time", bufSize: 15, }, { name: "Bufsize 0, default to maxBufSize, all at once", bufSize: 0, }, } { t.Run(test.name, func(t *testing.T) { o2 := testDefaultOptionsForGateway("B") o2.Gateway.sendQSubsBufSize = test.bufSize s2 := runGatewayServer(o2) defer s2.Shutdown() s2Url := fmt.Sprintf("nats://%s:%d", o2.Host, o2.Port) nc := natsConnect(t, s2Url) defer nc.Close() natsQueueSub(t, nc, "foo", "bar", func(_ *nats.Msg) {}) natsQueueSub(t, nc, "foo", "baz", func(_ *nats.Msg) {}) natsQueueSub(t, nc, "foo", "bat", func(_ *nats.Msg) {}) natsQueueSub(t, nc, "foo", "bax", func(_ *nats.Msg) {}) checkExpectedSubs(t, 4, s2) o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) checkForRegisteredQSubInterest(t, s1, "B", globalAccountName, "foo", 4, time.Second) // Make sure we have the 4 we expected c := s1.getOutboundGatewayConnection("B") ei, _ := c.gw.outsim.Load(globalAccountName) if ei == nil { t.Fatalf("No interest found") } sl := ei.(*outsie).sl r := sl.Match("foo") if len(r.qsubs) != 4 { t.Fatalf("Expected 4 groups, got %v", len(r.qsubs)) } var gotBar, gotBaz, gotBat, gotBax bool for _, qs := range r.qsubs { if len(qs) != 1 { t.Fatalf("Unexpected number of subs for group %s: %v", qs[0].queue, len(qs)) } q := qs[0].queue switch string(q) { case "bar": gotBar = true case "baz": gotBaz = true case "bat": gotBat = true case "bax": gotBax = true default: t.Fatalf("Unexpected group name: %s", q) } } if !gotBar || !gotBaz || !gotBat || !gotBax { t.Fatalf("Did not get all we wanted: bar=%v baz=%v bat=%v bax=%v", gotBar, gotBaz, gotBat, gotBax) } nc.Close() s1.Shutdown() s2.Shutdown() waitForOutboundGateways(t, s1, 0, time.Second) waitForOutboundGateways(t, s2, 0, time.Second) }) } } func TestGatewayRaceBetweenPubAndSub(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) s2Url := fmt.Sprintf("nats://127.0.0.1:%d", o2.Port) nc2 := natsConnect(t, s2Url) defer nc2.Close() s1Url := fmt.Sprintf("nats://127.0.0.1:%d", o1.Port) var ncaa [5]*nats.Conn var nca = ncaa[:0] for i := 0; i < 5; i++ { nc := natsConnect(t, s1Url) defer nc.Close() nca = append(nca, nc) } ch := make(chan bool, 1) wg := sync.WaitGroup{} wg.Add(5) for _, nc := range nca { nc := nc go func(n *nats.Conn) { defer wg.Done() for { n.Publish("foo", []byte("hello")) select { case <-ch: return default: } } }(nc) } time.Sleep(100 * time.Millisecond) natsQueueSub(t, nc2, "foo", "bar", func(m *nats.Msg) { natsUnsub(t, m.Sub) close(ch) }) wg.Wait() } // Returns the first (if any) of the inbound connections for this name. func getInboundGatewayConnection(s *Server, name string) *client { var gwsa [4]*client var gws = gwsa[:0] s.getInboundGatewayConnections(&gws) for _, gw := range gws { gw.mu.Lock() ok := gw.gw.name == name gw.mu.Unlock() if ok { return gw } } return nil } func TestGatewaySendAllSubs(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) gatewayMaxRUnsubBeforeSwitch = 100 defer func() { gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch }() ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() oc := testGatewayOptionsFromToWithServers(t, "C", "B", sb) sc := runGatewayServer(oc) defer sc.Shutdown() waitForOutboundGateways(t, sa, 2, time.Second) waitForOutboundGateways(t, sb, 2, time.Second) waitForOutboundGateways(t, sc, 2, time.Second) waitForInboundGateways(t, sa, 2, time.Second) waitForInboundGateways(t, sb, 2, time.Second) waitForInboundGateways(t, sc, 2, time.Second) // On A, create a sub to register some interest aURL := fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port) ncA := natsConnect(t, aURL) defer ncA.Close() natsSub(t, ncA, "sub.on.a.*", func(m *nats.Msg) {}) natsFlush(t, ncA) checkExpectedSubs(t, 1, sa) // On C, have some sub activity while it receives // unwanted messages and switches to interestOnly mode. cURL := fmt.Sprintf("nats://%s:%d", oc.Host, oc.Port) ncC := natsConnect(t, cURL) defer ncC.Close() wg := sync.WaitGroup{} wg.Add(2) done := make(chan bool) consCount := 0 accsCount := 0 go func() { defer wg.Done() for i := 0; ; i++ { // Create subs and qsubs on same subject natsSub(t, ncC, fmt.Sprintf("foo.%d", i+1), func(_ *nats.Msg) {}) natsQueueSub(t, ncC, fmt.Sprintf("foo.%d", i+1), fmt.Sprintf("bar.%d", i+1), func(_ *nats.Msg) {}) // Create psubs and qsubs on unique subjects natsSub(t, ncC, fmt.Sprintf("foox.%d", i+1), func(_ *nats.Msg) {}) natsQueueSub(t, ncC, fmt.Sprintf("fooy.%d", i+1), fmt.Sprintf("bar.%d", i+1), func(_ *nats.Msg) {}) consCount += 4 // Register account sc.RegisterAccount(fmt.Sprintf("acc.%d", i+1)) accsCount++ select { case <-done: return case <-time.After(15 * time.Millisecond): } } }() // From B publish on subjects for which C has an interest bURL := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) ncB := natsConnect(t, bURL) defer ncB.Close() go func() { defer wg.Done() time.Sleep(10 * time.Millisecond) for { for i := 0; i < 10; i++ { natsPub(t, ncB, fmt.Sprintf("foo.%d", i+1), []byte("hello")) } select { case <-done: return case <-time.After(5 * time.Millisecond): } } }() // From B, send a lot of messages that A is interested in, // but not C. // TODO(ik): May need to change that if we change the threshold // for when the switch happens. total := 300 for i := 0; i < total; i++ { if err := ncB.Publish(fmt.Sprintf("sub.on.a.%d", i), []byte("hi")); err != nil { t.Fatalf("Error waiting for reply: %v", err) } } close(done) // Normally, C would receive a message for each req inbox and // would send and RS- on that to B, making both have an unbounded // growth of the no-interest map. But after a certain amount // of RS-, C will send all its sub for the given account and // instruct B to send only if there is explicit interest. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { // Check C inbound connection from B c := getInboundGatewayConnection(sc, "B") c.mu.Lock() var switchedMode bool e := c.gw.insim[globalAccountName] if e != nil { switchedMode = e.ni == nil && e.mode == InterestOnly } c.mu.Unlock() if !switchedMode { return fmt.Errorf("C has still not switched mode") } return nil }) checkGWInterestOnlyMode(t, sb, "C", globalAccountName) wg.Wait() // Check consCount and accsCount on C checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { sc.gateway.pasi.Lock() scount := len(sc.gateway.pasi.m[globalAccountName]) sc.gateway.pasi.Unlock() if scount != consCount { return fmt.Errorf("Expected %v consumers for global account, got %v", consCount, scount) } acount := sc.numAccounts() if acount != accsCount+1 { return fmt.Errorf("Expected %v accounts, got %v", accsCount+1, acount) } return nil }) // Also, after all that, if a sub is created on C, it should // be sent to B (but not A). Check that this is the case. // So first send from A on the subject that we are going to // use for this new sub. natsPub(t, ncA, "newsub", []byte("hello")) natsFlush(t, ncA) aOutboundToC := sa.getOutboundGatewayConnection("C") checkForSubjectNoInterest(t, aOutboundToC, globalAccountName, "newsub", true, 2*time.Second) newSubSub := natsSub(t, ncC, "newsub", func(_ *nats.Msg) {}) natsFlush(t, ncC) checkExpectedSubs(t, consCount+1) checkFor(t, time.Second, 15*time.Millisecond, func() error { c := sb.getOutboundGatewayConnection("C") ei, _ := c.gw.outsim.Load(globalAccountName) if ei != nil { sl := ei.(*outsie).sl r := sl.Match("newsub") if len(r.psubs) == 1 { return nil } } return fmt.Errorf("Newsub not registered on B") }) checkForSubjectNoInterest(t, aOutboundToC, globalAccountName, "newsub", false, 2*time.Second) natsUnsub(t, newSubSub) natsFlush(t, ncC) checkExpectedSubs(t, consCount) checkFor(t, time.Second, 15*time.Millisecond, func() error { c := sb.getOutboundGatewayConnection("C") ei, _ := c.gw.outsim.Load(globalAccountName) if ei != nil { sl := ei.(*outsie).sl r := sl.Match("newsub") if len(r.psubs) == 0 { return nil } } return fmt.Errorf("Newsub still registered on B") }) } func TestGatewaySendAllSubsBadProtocol(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // For this test, make sure to use inbound from A so // A will reconnect when we send bad proto that // causes connection to be closed. c := getInboundGatewayConnection(sa, "B") // Mock an invalid protocol (account name missing) info := &Info{ Gateway: "B", GatewayCmd: gatewayCmdAllSubsStart, } b, _ := json.Marshal(info) c.mu.Lock() c.enqueueProto([]byte(fmt.Sprintf("INFO %s\r\n", b))) c.mu.Unlock() orgConn := c checkFor(t, 3*time.Second, 100*time.Millisecond, func() error { curConn := getInboundGatewayConnection(sa, "B") if orgConn == curConn { return fmt.Errorf("Not reconnected") } return nil }) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) // Refresh c = nil checkFor(t, 3*time.Second, 15*time.Millisecond, func() error { c = getInboundGatewayConnection(sa, "B") if c == nil { return fmt.Errorf("Did not reconnect") } return nil }) // Do correct start info.GatewayCmdPayload = []byte(globalAccountName) b, _ = json.Marshal(info) c.mu.Lock() c.enqueueProto([]byte(fmt.Sprintf("INFO %s\r\n", b))) c.mu.Unlock() // But incorrect end. info.GatewayCmd = gatewayCmdAllSubsComplete info.GatewayCmdPayload = nil b, _ = json.Marshal(info) c.mu.Lock() c.enqueueProto([]byte(fmt.Sprintf("INFO %s\r\n", b))) c.mu.Unlock() orgConn = c checkFor(t, 3*time.Second, 100*time.Millisecond, func() error { curConn := getInboundGatewayConnection(sa, "B") if orgConn == curConn { return fmt.Errorf("Not reconnected") } return nil }) } func TestGatewayRaceOnClose(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) bURL := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) ncB := natsConnect(t, bURL, nats.NoReconnect()) defer ncB.Close() wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() cb := func(_ *nats.Msg) {} for { // Expect failure at one point and just return. qsub, err := ncB.QueueSubscribe("foo", "bar", cb) if err != nil { return } if err := qsub.Unsubscribe(); err != nil { return } } }() // Wait a bit and kill B time.Sleep(200 * time.Millisecond) sb.Shutdown() wg.Wait() } // Similar to TestNewRoutesServiceImport but with 2 GW servers instead // of a cluster of 2 servers. func TestGatewayServiceImport(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) oa := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(oa, "$foo", "clientA", "password") setAccountUserPassInOptions(oa, "$bar", "yyyyyyy", "password") sa := runGatewayServer(oa) defer sa.Shutdown() ob := testGatewayOptionsFromToWithServers(t, "B", "A", sa) setAccountUserPassInOptions(ob, "$foo", "clientBFoo", "password") setAccountUserPassInOptions(ob, "$bar", "clientB", "password") sb := runGatewayServer(ob) defer sb.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Get accounts fooA, _ := sa.LookupAccount("$foo") barA, _ := sa.LookupAccount("$bar") fooB, _ := sb.LookupAccount("$foo") barB, _ := sb.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooA.AddServiceExport("test.request", nil) fooB.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Same on A. if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. aURL := fmt.Sprintf("nats://clientA:password@127.0.0.1:%d", oa.Port) clientA := natsConnect(t, aURL) defer clientA.Close() subA := natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. bURL := fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob.Port) clientB := natsConnect(t, bURL) defer clientB.Close() subB := natsSubSync(t, clientB, "reply") natsFlush(t, clientB) for i := 1; i <= 2; i++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err := subA.NextMsg(time.Second) if err != nil { t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } // Check for duplicate message if msg, err := subA.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } expected := int64(i * 2) vz, _ := sa.Varz(nil) if vz.OutMsgs != expected { t.Fatalf("Expected %d outMsgs for A, got %v", expected, vz.OutMsgs) } // For B, we expect it to send to gateway on the two subjects: test.request // and foo.request then send the reply to the client and optimistically // to the other gateway. if i == 1 { expected = 4 } else { // The second time, one of the accounts will be suppressed and the reply going // back so we should only get 2 more messages. expected = 6 } vz, _ = sb.Varz(nil) if vz.OutMsgs != expected { t.Fatalf("Expected %d outMsgs for B, got %v", expected, vz.OutMsgs) } } checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 1 { return fmt.Errorf("Expected one sub to be left on fooA, but got %d", ts) } return nil }) // Speed up exiration err := fooA.SetServiceExportResponseThreshold("test.request", 50*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request: %v", err) } } // These reply subjects will be dangling off of $foo account on serverA. // Remove our service endpoint and wait for the dangling replies to go to zero. natsUnsub(t, subA) natsFlush(t, clientA) checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 0 { return fmt.Errorf("Number of subs is %d, should be zero", ts) } return nil }) // Repeat similar test but without the small TTL and verify // that if B is shutdown, the dangling subs for replies are // cleared from the account sublist. err = fooA.SetServiceExportResponseThreshold("test.request", 10*time.Second) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } subA = natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request: %v", err) } } // Shutdown B clientB.Close() sb.Shutdown() // Close our last sub natsUnsub(t, subA) natsFlush(t, clientA) // Verify that they are gone before the 10 sec TTL checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 0 { return fmt.Errorf("Number of subs is %d, should be zero", ts) } return nil }) // Check that this all work in interest-only mode sb = runGatewayServer(ob) defer sb.Shutdown() fooB, _ = sb.LookupAccount("$foo") barB, _ = sb.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooB.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) // We need at least a subscription on A otherwise when publishing // to subjects with no interest we would simply get an A- natsSubSync(t, clientA, "not.used") // Create a client on B that will use account $foo bURL = fmt.Sprintf("nats://clientBFoo:password@127.0.0.1:%d", ob.Port) clientB = natsConnect(t, bURL) defer clientB.Close() // First flood with subjects that remote gw is not interested // so we switch to interest-only. for i := 0; i < 1100; i++ { natsPub(t, clientB, fmt.Sprintf("no.interest.%d", i), []byte("hello")) } natsFlush(t, clientB) checkGWInterestOnlyMode(t, sb, "A", "$foo") // Go back to clientB on $bar. clientB.Close() bURL = fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob.Port) clientB = natsConnect(t, bURL) defer clientB.Close() subA = natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) subB = natsSubSync(t, clientB, "reply") natsFlush(t, clientB) // Sine it is interest-only, B should receive an interest // on $foo test.request checkGWInterestOnlyModeInterestOn(t, sb, "A", "$foo", "test.request") // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err := subA.NextMsg(time.Second) if err != nil { t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } // Check for duplicate message if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } } func TestGatewayServiceImportWithQueue(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) oa := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(oa, "$foo", "clientA", "password") setAccountUserPassInOptions(oa, "$bar", "yyyyyyy", "password") sa := runGatewayServer(oa) defer sa.Shutdown() ob := testGatewayOptionsFromToWithServers(t, "B", "A", sa) setAccountUserPassInOptions(ob, "$foo", "clientBFoo", "password") setAccountUserPassInOptions(ob, "$bar", "clientB", "password") sb := runGatewayServer(ob) defer sb.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Get accounts fooA, _ := sa.LookupAccount("$foo") barA, _ := sa.LookupAccount("$bar") fooB, _ := sb.LookupAccount("$foo") barB, _ := sb.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooA.AddServiceExport("test.request", nil) fooB.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Same on A. if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. aURL := fmt.Sprintf("nats://clientA:password@127.0.0.1:%d", oa.Port) clientA := natsConnect(t, aURL) defer clientA.Close() subA := natsQueueSubSync(t, clientA, "test.request", "queue") natsFlush(t, clientA) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. bURL := fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob.Port) clientB := natsConnect(t, bURL) defer clientB.Close() subB := natsQueueSubSync(t, clientB, "reply", "queue2") natsFlush(t, clientB) // Wait for queue interest on test.request from A to be registered // on server B. checkForRegisteredQSubInterest(t, sb, "A", "$foo", "test.request", 1, time.Second) for i := 0; i < 2; i++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err := subA.NextMsg(time.Second) if err != nil { t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } // Check for duplicate message if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } expected := int64((i + 1) * 2) vz, _ := sa.Varz(nil) if vz.OutMsgs != expected { t.Fatalf("Expected %d outMsgs for A, got %v", expected, vz.OutMsgs) } // For B, we expect it to send to gateway on the two subjects: test.request // and foo.request then send the reply to the client and optimistically // to the other gateway. if i == 0 { expected = 4 } else { // The second time, one of the accounts will be suppressed and the reply going // back so we should get only 2 more messages. expected = 6 } vz, _ = sb.Varz(nil) if vz.OutMsgs != expected { t.Fatalf("Expected %d outMsgs for B, got %v", expected, vz.OutMsgs) } } checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 1 { return fmt.Errorf("Expected one sub to be left on fooA, but got %d", ts) } return nil }) // Speed up exiration err := fooA.SetServiceExportResponseThreshold("test.request", 10*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request: %v", err) } } // These reply subjects will be dangling off of $foo account on serverA. // Remove our service endpoint and wait for the dangling replies to go to zero. natsUnsub(t, subA) natsFlush(t, clientA) checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 0 { return fmt.Errorf("Number of subs is %d, should be zero", ts) } return nil }) checkForRegisteredQSubInterest(t, sb, "A", "$foo", "test.request", 0, time.Second) // Repeat similar test but without the small TTL and verify // that if B is shutdown, the dangling subs for replies are // cleared from the account sublist. err = fooA.SetServiceExportResponseThreshold("test.request", 10*time.Second) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } subA = natsQueueSubSync(t, clientA, "test.request", "queue") natsFlush(t, clientA) checkForRegisteredQSubInterest(t, sb, "A", "$foo", "test.request", 1, time.Second) // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request %d: %v", i+1, err) } } // Shutdown B clientB.Close() sb.Shutdown() // Close our last sub natsUnsub(t, subA) natsFlush(t, clientA) // Verify that they are gone before the 10 sec TTL checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 0 { return fmt.Errorf("Number of subs is %d, should be zero", ts) } return nil }) // Check that this all work in interest-only mode sb = runGatewayServer(ob) defer sb.Shutdown() fooB, _ = sb.LookupAccount("$foo") barB, _ = sb.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooB.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) // We need at least a subscription on A otherwise when publishing // to subjects with no interest we would simply get an A- natsSubSync(t, clientA, "not.used") // Create a client on B that will use account $foo bURL = fmt.Sprintf("nats://clientBFoo:password@127.0.0.1:%d", ob.Port) clientB = natsConnect(t, bURL) defer clientB.Close() // First flood with subjects that remote gw is not interested // so we switch to interest-only. for i := 0; i < 1100; i++ { natsPub(t, clientB, fmt.Sprintf("no.interest.%d", i), []byte("hello")) } natsFlush(t, clientB) checkGWInterestOnlyMode(t, sb, "A", "$foo") // Go back to clientB on $bar. clientB.Close() bURL = fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob.Port) clientB = natsConnect(t, bURL) defer clientB.Close() subA = natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) subB = natsSubSync(t, clientB, "reply") natsFlush(t, clientB) // Sine it is interest-only, B should receive an interest // on $foo test.request checkGWInterestOnlyModeInterestOn(t, sb, "A", "$foo", "test.request") // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err := subA.NextMsg(time.Second) if err != nil { t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } // Check for duplicate message if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } } func ensureGWConnectTo(t *testing.T, s *Server, remoteGWName string, remoteGWServer *Server) { t.Helper() var good bool for i := 0; !good && (i < 3); i++ { checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if s.numOutboundGateways() == 0 { return fmt.Errorf("Still no gw outbound connection") } return nil }) ogc := s.getOutboundGatewayConnection(remoteGWName) ogc.mu.Lock() name := ogc.opts.Name nc := ogc.nc ogc.mu.Unlock() if name != remoteGWServer.ID() { rg := s.getRemoteGateway(remoteGWName) goodURL := remoteGWServer.getGatewayURL() rg.Lock() for u := range rg.urls { if u != goodURL { delete(rg.urls, u) } } rg.Unlock() if nc != nil { nc.Close() } } else { good = true } } if !good { t.Fatalf("Could not ensure that server connects to remote gateway %q at URL %q", remoteGWName, remoteGWServer.getGatewayURL()) } } func TestGatewayServiceImportComplexSetup(t *testing.T) { // This test will have following setup: // // |- responder (subs to "$foo test.request") // | (sends to "$foo _R_.xxxx") // route v // [A1]<----------------->[A2] // ^ |^ | // |gw| \______gw________ gw| // | v \ v // [B1]<----------------->[B2] // ^ route // | // |_ requestor (sends "$bar foo.request reply") // // Setup first A1 and B1 to ensure that they have GWs // connections as described above. oa1 := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(oa1, "$foo", "clientA", "password") setAccountUserPassInOptions(oa1, "$bar", "yyyyyyy", "password") sa1 := runGatewayServer(oa1) defer sa1.Shutdown() ob1 := testGatewayOptionsFromToWithServers(t, "B", "A", sa1) setAccountUserPassInOptions(ob1, "$foo", "xxxxxxx", "password") setAccountUserPassInOptions(ob1, "$bar", "clientB", "password") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) ob2 := testGatewayOptionsFromToWithServers(t, "B", "A", sa1) ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) setAccountUserPassInOptions(ob2, "$foo", "clientBFoo", "password") setAccountUserPassInOptions(ob2, "$bar", "clientB", "password") ob2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap sb2 := runGatewayServer(ob2) defer sb2.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa1, 2, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sb2, 0, time.Second) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb2) oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sa1.ClusterAddr().Port)) setAccountUserPassInOptions(oa2, "$foo", "clientA", "password") setAccountUserPassInOptions(oa2, "$bar", "yyyyyyy", "password") oa2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap sa2 := runGatewayServer(oa2) defer sa2.Shutdown() ensureGWConnectTo(t, sa2, "B", sb2) checkClusterFormed(t, sa1, sa2) checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, time.Second) waitForOutboundGateways(t, sa2, 1, 2*time.Second) waitForInboundGateways(t, sa1, 2, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa2, 0, time.Second) // Verification that we have what we wanted c := sa2.getOutboundGatewayConnection("B") if c == nil || c.opts.Name != sb2.ID() { t.Fatalf("A2 does not have outbound to B2") } c = getInboundGatewayConnection(sa2, "B") if c != nil { t.Fatalf("Bad setup") } c = sb2.getOutboundGatewayConnection("A") if c == nil || c.opts.Name != sa1.ID() { t.Fatalf("B2 does not have outbound to A1") } c = getInboundGatewayConnection(sb2, "A") if c == nil || c.opts.Name != sa2.ID() { t.Fatalf("Bad setup") } // Ok, so now that we have proper setup, do actual test! // Get accounts fooA1, _ := sa1.LookupAccount("$foo") barA1, _ := sa1.LookupAccount("$bar") fooA2, _ := sa2.LookupAccount("$foo") barA2, _ := sa2.LookupAccount("$bar") fooB1, _ := sb1.LookupAccount("$foo") barB1, _ := sb1.LookupAccount("$bar") fooB2, _ := sb2.LookupAccount("$foo") barB2, _ := sb2.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooA1.AddServiceExport("test.request", nil) fooA2.AddServiceExport("test.request", nil) fooB1.AddServiceExport("test.request", nil) fooB2.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB1.AddServiceImport(fooB1, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } if err := barB2.AddServiceImport(fooB2, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Same on A. if err := barA1.AddServiceImport(fooA1, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } if err := barA2.AddServiceImport(fooA2, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to A2 and be the service endpoint and responder. a2URL := fmt.Sprintf("nats://clientA:password@127.0.0.1:%d", oa2.Port) clientA := natsConnect(t, a2URL) defer clientA.Close() subA := natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) // Now setup client B on B1 who will do a sub from account $bar // that should map account $foo's foo subject. b1URL := fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob1.Port) clientB := natsConnect(t, b1URL) defer clientB.Close() subB := natsSubSync(t, clientB, "reply") natsFlush(t, clientB) var msg *nats.Msg var err error for attempts := 1; attempts <= 2; attempts++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err = subA.NextMsg(time.Second) if err != nil { if attempts == 1 { // Since we are in interestOnly mode, it is possible // that server B did not receive the subscription // interest yet, so try again. continue } t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } } // Make sure we don't receive a second copy if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Received unexpected message: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Make sure we don't receive a second copy if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Received unexpected message: %v", msg) } checkSubs := func(t *testing.T, acc *Account, srvName string, expected int) { t.Helper() checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := acc.TotalSubs(); ts != expected { return fmt.Errorf("Number of subs is %d on acc=%s srv=%s, should be %v", ts, acc.Name, srvName, expected) } return nil }) } checkSubs(t, fooA1, "A1", 1) checkSubs(t, barA1, "A1", 1) checkSubs(t, fooA2, "A2", 1) checkSubs(t, barA2, "A2", 1) checkSubs(t, fooB1, "B1", 1) checkSubs(t, barB1, "B1", 2) checkSubs(t, fooB2, "B2", 1) checkSubs(t, barB2, "B2", 2) // Speed up exiration err = fooA2.SetServiceExportResponseThreshold("test.request", 10*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } err = fooB1.SetServiceExportResponseThreshold("test.request", 10*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request: %v", err) } } // Unsubsribe all and ensure counts go to 0. natsUnsub(t, subA) natsFlush(t, clientA) natsUnsub(t, subB) natsFlush(t, clientB) // We should expire because ttl. checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if nr := len(fooA1.exports.responses); nr != 0 { return fmt.Errorf("Number of responses is %d", nr) } return nil }) checkSubs(t, fooA1, "A1", 0) checkSubs(t, fooA2, "A2", 0) checkSubs(t, fooB1, "B1", 1) checkSubs(t, fooB2, "B2", 1) checkSubs(t, barA1, "A1", 1) checkSubs(t, barA2, "A2", 1) checkSubs(t, barB1, "B1", 1) checkSubs(t, barB2, "B2", 1) // Check that this all work in interest-only mode. // We need at least a subscription on B2 otherwise when publishing // to subjects with no interest we would simply get an A- b2URL := fmt.Sprintf("nats://clientBFoo:password@127.0.0.1:%d", ob2.Port) clientB2 := natsConnect(t, b2URL) defer clientB2.Close() natsSubSync(t, clientB2, "not.used") natsFlush(t, clientB2) // Make A2 flood B2 with subjects that B2 is not interested in. for i := 0; i < 1100; i++ { natsPub(t, clientA, fmt.Sprintf("no.interest.%d", i), []byte("hello")) } natsFlush(t, clientA) // Wait for B2 to switch to interest-only checkGWInterestOnlyMode(t, sa2, "B", "$foo") subA = natsSubSync(t, clientA, "test.request") natsFlush(t, clientA) subB = natsSubSync(t, clientB, "reply") natsFlush(t, clientB) for attempts := 1; attempts <= 2; attempts++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "foo.request", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err = subA.NextMsg(time.Second) if err != nil { if attempts == 1 { // Since we are in interestOnly mode, it is possible // that server B did not receive the subscription // interest yet, so try again. continue } t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "test.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } } // Check for duplicate message if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } } func TestGatewayServiceExportWithWildcards(t *testing.T) { // This test will have following setup: // // |- responder // | // route v // [A1]<----------------->[A2] // ^ |^ | // |gw| \______gw________ gw| // | v \ v // [B1]<----------------->[B2] // ^ route // | // |_ requestor // for _, test := range []struct { name string public bool }{ { name: "public", public: true, }, { name: "private", public: false, }, } { t.Run(test.name, func(t *testing.T) { // Setup first A1 and B1 to ensure that they have GWs // connections as described above. oa1 := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(oa1, "$foo", "clientA", "password") setAccountUserPassInOptions(oa1, "$bar", "yyyyyyy", "password") sa1 := runGatewayServer(oa1) defer sa1.Shutdown() ob1 := testGatewayOptionsFromToWithServers(t, "B", "A", sa1) setAccountUserPassInOptions(ob1, "$foo", "xxxxxxx", "password") setAccountUserPassInOptions(ob1, "$bar", "clientB", "password") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) ob2 := testGatewayOptionsFromToWithServers(t, "B", "A", sa1) ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) setAccountUserPassInOptions(ob2, "$foo", "clientBFoo", "password") setAccountUserPassInOptions(ob2, "$bar", "clientB", "password") ob2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap sb2 := runGatewayServer(ob2) defer sb2.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa1, 2, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sb2, 0, time.Second) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb2) oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sa1.ClusterAddr().Port)) setAccountUserPassInOptions(oa2, "$foo", "clientA", "password") setAccountUserPassInOptions(oa2, "$bar", "yyyyyyy", "password") oa2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap sa2 := runGatewayServer(oa2) defer sa2.Shutdown() ensureGWConnectTo(t, sa2, "B", sb2) checkClusterFormed(t, sa1, sa2) checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sa1, 1, time.Second) waitForOutboundGateways(t, sb1, 1, time.Second) waitForOutboundGateways(t, sb2, 1, time.Second) waitForOutboundGateways(t, sa2, 1, 2*time.Second) waitForInboundGateways(t, sa1, 2, time.Second) waitForInboundGateways(t, sb1, 1, time.Second) waitForInboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa2, 0, time.Second) // Verification that we have what we wanted c := sa2.getOutboundGatewayConnection("B") if c == nil || c.opts.Name != sb2.ID() { t.Fatalf("A2 does not have outbound to B2") } c = getInboundGatewayConnection(sa2, "B") if c != nil { t.Fatalf("Bad setup") } c = sb2.getOutboundGatewayConnection("A") if c == nil || c.opts.Name != sa1.ID() { t.Fatalf("B2 does not have outbound to A1") } c = getInboundGatewayConnection(sb2, "A") if c == nil || c.opts.Name != sa2.ID() { t.Fatalf("Bad setup") } // Ok, so now that we have proper setup, do actual test! // Get accounts fooA1, _ := sa1.LookupAccount("$foo") barA1, _ := sa1.LookupAccount("$bar") fooA2, _ := sa2.LookupAccount("$foo") barA2, _ := sa2.LookupAccount("$bar") fooB1, _ := sb1.LookupAccount("$foo") barB1, _ := sb1.LookupAccount("$bar") fooB2, _ := sb2.LookupAccount("$foo") barB2, _ := sb2.LookupAccount("$bar") var accs []*Account // Add in the service export for the requests. if !test.public { accs = []*Account{barA1} } fooA1.AddServiceExport("ngs.update.*", accs) if !test.public { accs = []*Account{barA2} } fooA2.AddServiceExport("ngs.update.*", accs) if !test.public { accs = []*Account{barB1} } fooB1.AddServiceExport("ngs.update.*", accs) if !test.public { accs = []*Account{barB2} } fooB2.AddServiceExport("ngs.update.*", accs) // Add import abilities to server B's bar account from foo. if err := barB1.AddServiceImport(fooB1, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } if err := barB2.AddServiceImport(fooB2, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Same on A. if err := barA1.AddServiceImport(fooA1, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } if err := barA2.AddServiceImport(fooA2, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to A2 and be the service endpoint and responder. a2URL := fmt.Sprintf("nats://clientA:password@127.0.0.1:%d", oa2.Port) clientA := natsConnect(t, a2URL) defer clientA.Close() subA := natsSubSync(t, clientA, "ngs.update.$bar") natsFlush(t, clientA) // Now setup client B on B1 who will do a sub from account $bar // that should map account $foo's foo subject. b1URL := fmt.Sprintf("nats://clientB:password@127.0.0.1:%d", ob1.Port) clientB := natsConnect(t, b1URL) defer clientB.Close() subB := natsSubSync(t, clientB, "reply") natsFlush(t, clientB) var msg *nats.Msg var err error for attempts := 1; attempts <= 2; attempts++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "ngs.update", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err = subA.NextMsg(time.Second) if err != nil { if attempts == 1 { // Since we are in interestOnly mode, it is possible // that server B did not receive the subscription // interest yet, so try again. continue } t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "ngs.update.$bar" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } } // Make sure we don't receive a second copy if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Received unexpected message: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Make sure we don't receive a second copy if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Received unexpected message: %v", msg) } checkSubs := func(t *testing.T, acc *Account, srvName string, expected int) { t.Helper() checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if ts := acc.TotalSubs(); ts != expected { return fmt.Errorf("Number of subs is %d on acc=%s srv=%s, should be %v", ts, acc.Name, srvName, expected) } return nil }) } checkSubs(t, fooA1, "A1", 1) checkSubs(t, barA1, "A1", 1) checkSubs(t, fooA2, "A2", 1) checkSubs(t, barA2, "A2", 1) checkSubs(t, fooB1, "B1", 1) checkSubs(t, barB1, "B1", 2) checkSubs(t, fooB2, "B2", 1) checkSubs(t, barB2, "B2", 2) // Speed up exiration err = fooA1.SetServiceExportResponseThreshold("ngs.update.*", 10*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } err = fooB1.SetServiceExportResponseThreshold("ngs.update.*", 10*time.Millisecond) if err != nil { t.Fatalf("Error setting response threshold: %v", err) } // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { natsPubReq(t, clientB, "ngs.update", "reply", []byte("hi")) } natsFlush(t, clientB) // Consume the requests, but don't reply to them... for i := 0; i < 100; i++ { if _, err := subA.NextMsg(time.Second); err != nil { t.Fatalf("subA did not receive request: %v", err) } } // Unsubsribe all and ensure counts go to 0. natsUnsub(t, subA) natsFlush(t, clientA) natsUnsub(t, subB) natsFlush(t, clientB) // We should expire because ttl. checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if nr := len(fooA1.exports.responses); nr != 0 { return fmt.Errorf("Number of responses is %d", nr) } return nil }) checkSubs(t, fooA1, "A1", 0) checkSubs(t, fooA2, "A2", 0) checkSubs(t, fooB1, "B1", 1) checkSubs(t, fooB2, "B2", 1) checkSubs(t, barA1, "A1", 1) checkSubs(t, barA2, "A2", 1) checkSubs(t, barB1, "B1", 1) checkSubs(t, barB2, "B2", 1) // Check that this all work in interest-only mode. // We need at least a subscription on B2 otherwise when publishing // to subjects with no interest we would simply get an A- b2URL := fmt.Sprintf("nats://clientBFoo:password@127.0.0.1:%d", ob2.Port) clientB2 := natsConnect(t, b2URL) defer clientB2.Close() natsSubSync(t, clientB2, "not.used") natsFlush(t, clientB2) // Make A2 flood B2 with subjects that B2 is not interested in. for i := 0; i < 1100; i++ { natsPub(t, clientA, fmt.Sprintf("no.interest.%d", i), []byte("hello")) } natsFlush(t, clientA) // Wait for B2 to switch to interest-only checkGWInterestOnlyMode(t, sa2, "B", "$foo") subA = natsSubSync(t, clientA, "ngs.update.*") natsFlush(t, clientA) subB = natsSubSync(t, clientB, "reply") natsFlush(t, clientB) for attempts := 1; attempts <= 2; attempts++ { // Send the request from clientB on foo.request, natsPubReq(t, clientB, "ngs.update", "reply", []byte("hi")) natsFlush(t, clientB) // Expect the request on A msg, err = subA.NextMsg(time.Second) if err != nil { if attempts == 1 { // Since we are in interestOnly mode, it is possible // that server B did not receive the subscription // interest yet, so try again. continue } t.Fatalf("subA failed to get request: %v", err) } if msg.Subject != "ngs.update.$bar" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } } // Check for duplicate message if msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientA, msg.Reply, []byte("ok")) natsFlush(t, clientA) msg, err = subB.NextMsg(time.Second) if err != nil { t.Fatalf("subB failed to get reply: %v", err) } if msg.Subject != "reply" || string(msg.Data) != "ok" { t.Fatalf("Unexpected message: %v", msg) } // Check for duplicate message if msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } }) } } // NOTE: if this fails for you and says only has <10 outbound, make sure ulimit for open files > 256. func TestGatewayMemUsage(t *testing.T) { // Try to clean up. runtime.GC() var m runtime.MemStats runtime.ReadMemStats(&m) pta := m.TotalAlloc o := testDefaultOptionsForGateway("A") s := runGatewayServer(o) defer s.Shutdown() var servers []*Server servers = append(servers, s) numServers := 10 for i := 0; i < numServers; i++ { rn := fmt.Sprintf("RG_%d", i+1) o := testGatewayOptionsFromToWithServers(t, rn, "A", s) s := runGatewayServer(o) defer s.Shutdown() servers = append(servers, s) } // Each server should have an outbound for _, s := range servers { waitForOutboundGateways(t, s, numServers, 2*time.Second) } // The first started server should have numServers inbounds (since // they all connect to it). waitForInboundGateways(t, s, numServers, 2*time.Second) // Calculate in MB what we are using now. const max = 50 * 1024 * 1024 // 50MB runtime.ReadMemStats(&m) used := m.TotalAlloc - pta if used > max { t.Fatalf("Cluster using too much memory, expect < 50MB, got %dMB", used/(1024*1024)) } for _, s := range servers { s.Shutdown() } } func TestGatewayMapReplyOnlyForRecentSub(t *testing.T) { o2 := testDefaultOptionsForGateway("B") s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) // Change s1's recent sub expiration default value s1.mu.Lock() s1.gateway.pasi.Lock() s1.gateway.recSubExp = 100 * time.Millisecond s1.gateway.pasi.Unlock() s1.mu.Unlock() // Setup a replier on s2 nc2 := natsConnect(t, fmt.Sprintf("nats://%s:%d", o2.Host, o2.Port)) defer nc2.Close() errCh := make(chan error, 1) natsSub(t, nc2, "foo", func(m *nats.Msg) { // Send reply regardless.. nc2.Publish(m.Reply, []byte("reply")) // Check that reply given to application is not mapped. if !strings.HasPrefix(m.Reply, nats.InboxPrefix) { errCh <- fmt.Errorf("Reply expected to have normal inbox, got %v", m.Reply) return } errCh <- nil }) natsFlush(t, nc2) checkExpectedSubs(t, 1, s2) // Create requestor on s1 nc1 := natsConnect(t, fmt.Sprintf("nats://%s:%d", o1.Host, o1.Port)) defer nc1.Close() // Send first request, reply should be mapped nc1.Request("foo", []byte("msg1"), time.Second) // Wait more than the recent sub expiration (that we have set to 100ms) time.Sleep(200 * time.Millisecond) // Send second request (reply should not be mapped) nc1.Request("foo", []byte("msg2"), time.Second) select { case e := <-errCh: if e != nil { t.Fatal(e.Error()) } case <-time.After(time.Second): t.Fatalf("Did not get replies") } } type delayedWriteConn struct { sync.Mutex net.Conn bytes [][]byte delay bool wg sync.WaitGroup } func (c *delayedWriteConn) Write(b []byte) (int, error) { c.Lock() defer c.Unlock() if c.delay || len(c.bytes) > 0 { c.bytes = append(c.bytes, append([]byte(nil), b...)) c.wg.Add(1) go func() { defer c.wg.Done() c.Lock() defer c.Unlock() if c.delay { c.Unlock() time.Sleep(100 * time.Millisecond) c.Lock() } if len(c.bytes) > 0 { b = c.bytes[0] c.bytes = c.bytes[1:] c.Conn.Write(b) } }() return len(b), nil } return c.Conn.Write(b) } // This test uses a single account and makes sure that when // a reply subject is prefixed with $GR it comes back to // the origin cluster and delivered to proper reply subject // there, but also to subscribers on that reply subject // on the other cluster. func TestGatewaySendReplyAcrossGateways(t *testing.T) { for _, test := range []struct { name string poolSize int peracc bool }{ {"no pooling", -1, false}, {"pooling", 5, false}, {"per account", 0, true}, } { t.Run(test.name, func(t *testing.T) { ob := testDefaultOptionsForGateway("B") ob.Accounts = []*Account{NewAccount("ACC")} ob.Users = []*User{{Username: "user", Password: "pwd", Account: ob.Accounts[0]}} sb := runGatewayServer(ob) defer sb.Shutdown() oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa1.Cluster.PoolSize = test.poolSize if test.peracc { oa1.Cluster.PinnedAccounts = []string{"ACC"} } oa1.Accounts = []*Account{NewAccount("ACC")} oa1.Users = []*User{{Username: "user", Password: "pwd", Account: oa1.Accounts[0]}} sa1 := runGatewayServer(oa1) defer sa1.Shutdown() waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) waitForOutboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) // Now start another server in cluster "A". This will allow us // to test the reply from cluster "B" coming back directly to // the server where the request originates, and indirectly through // route. oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa2.Cluster.PoolSize = test.poolSize if test.peracc { oa2.Cluster.PinnedAccounts = []string{"ACC"} } oa2.Accounts = []*Account{NewAccount("ACC")} oa2.Users = []*User{{Username: "user", Password: "pwd", Account: oa2.Accounts[0]}} oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa1.Cluster.Host, oa1.Cluster.Port)) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() waitForOutboundGateways(t, sa2, 1, time.Second) waitForInboundGateways(t, sb, 2, time.Second) checkClusterFormed(t, sa1, sa2) replySubj := "bar" // Setup a responder on sb ncb := natsConnect(t, fmt.Sprintf("nats://user:pwd@%s:%d", ob.Host, ob.Port)) defer ncb.Close() natsSub(t, ncb, "foo", func(m *nats.Msg) { m.Respond([]byte("reply")) }) // Set a subscription on the reply subject on sb subSB := natsSubSync(t, ncb, replySubj) natsFlush(t, ncb) checkExpectedSubs(t, 2, sb) testReqReply := func(t *testing.T, host string, port int, createSubOnA bool) { t.Helper() nca := natsConnect(t, fmt.Sprintf("nats://user:pwd@%s:%d", host, port)) defer nca.Close() if createSubOnA { subSA := natsSubSync(t, nca, replySubj) natsPubReq(t, nca, "foo", replySubj, []byte("hello")) natsNexMsg(t, subSA, time.Second) // Check for duplicates if _, err := subSA.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Received duplicate message on subSA!") } } else { natsPubReq(t, nca, "foo", replySubj, []byte("hello")) } natsNexMsg(t, subSB, time.Second) // Check for duplicates if _, err := subSB.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Received duplicate message on subSB!") } } // Create requestor on sa1 to check for direct reply from GW: testReqReply(t, oa1.Host, oa1.Port, true) // Wait for subscription to be gone... checkExpectedSubs(t, 0, sa1) // Now create requestor on sa2, it will receive reply through sa1. testReqReply(t, oa2.Host, oa2.Port, true) checkExpectedSubs(t, 0, sa1) checkExpectedSubs(t, 0, sa2) // Now issue requests but without any interest in the requestor's // origin cluster and make sure the other cluster gets the reply. testReqReply(t, oa1.Host, oa1.Port, false) testReqReply(t, oa2.Host, oa2.Port, false) // There is a possible race between sa2 sending the RS+ for the // subscription on the reply subject, and the GW reply making it // to sa1 before the RS+ is processed there. // We are going to force this race by making the route connection // block as needed. acc, _ := sa2.LookupAccount("ACC") acc.mu.RLock() api := acc.routePoolIdx acc.mu.RUnlock() var route *client sa2.mu.Lock() if test.peracc { if conns, ok := sa2.accRoutes["ACC"]; ok { for _, r := range conns { route = r break } } } else if test.poolSize > 0 { sa2.forEachRoute(func(r *client) { r.mu.Lock() if r.route.poolIdx == api { route = r } r.mu.Unlock() }) } else if r := getFirstRoute(sa2); r != nil { route = r } sa2.mu.Unlock() route.mu.Lock() routeConn := &delayedWriteConn{ Conn: route.nc, wg: sync.WaitGroup{}, } route.nc = routeConn route.mu.Unlock() delayRoute := func() { routeConn.Lock() routeConn.delay = true routeConn.Unlock() } stopDelayRoute := func() { routeConn.Lock() routeConn.delay = false wg := &routeConn.wg routeConn.Unlock() wg.Wait() } delayRoute() testReqReply(t, oa2.Host, oa2.Port, true) stopDelayRoute() // Same test but now we have a local interest on the reply subject // on sa1 to make sure that interest there does not prevent sending // the RMSG to sa2, which is the origin of the request. checkExpectedSubs(t, 0, sa1) checkExpectedSubs(t, 0, sa2) nca1 := natsConnect(t, fmt.Sprintf("nats://user:pwd@%s:%d", oa1.Host, oa1.Port)) defer nca1.Close() subSA1 := natsSubSync(t, nca1, replySubj) natsFlush(t, nca1) checkExpectedSubs(t, 1, sa1) checkExpectedSubs(t, 1, sa2) delayRoute() testReqReply(t, oa2.Host, oa2.Port, true) stopDelayRoute() natsNexMsg(t, subSA1, time.Second) }) } } // This test will have a requestor on cluster A and responder // on cluster B, but when the responder sends the response, // it will also have a reply subject to receive a response // for the response. func TestGatewayPingPongReplyAcrossGateways(t *testing.T) { ob := testDefaultOptionsForGateway("B") ob.Accounts = []*Account{NewAccount("ACC")} ob.Users = []*User{{Username: "user", Password: "pwd", Account: ob.Accounts[0]}} sb := runGatewayServer(ob) defer sb.Shutdown() oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa1.Accounts = []*Account{NewAccount("ACC")} oa1.Users = []*User{{Username: "user", Password: "pwd", Account: oa1.Accounts[0]}} sa1 := runGatewayServer(oa1) defer sa1.Shutdown() waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) waitForOutboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) // Now start another server in cluster "A". This will allow us // to test the reply from cluster "B" coming back directly to // the server where the request originates, and indirectly through // route. oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa2.Accounts = []*Account{NewAccount("ACC")} oa2.Users = []*User{{Username: "user", Password: "pwd", Account: oa2.Accounts[0]}} oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa1.Cluster.Host, oa1.Cluster.Port)) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() waitForOutboundGateways(t, sa2, 1, time.Second) waitForInboundGateways(t, sb, 2, time.Second) checkClusterFormed(t, sa1, sa2) // Setup a responder on sb ncb := natsConnect(t, fmt.Sprintf("nats://user:pwd@%s:%d", ob.Host, ob.Port)) defer ncb.Close() sbReplySubj := "sbreply" subSB := natsSubSync(t, ncb, sbReplySubj) natsSub(t, ncb, "foo", func(m *nats.Msg) { ncb.PublishRequest(m.Reply, sbReplySubj, []byte("sb reply")) }) natsFlush(t, ncb) checkExpectedSubs(t, 2, sb) testReqReply := func(t *testing.T, host string, port int) { t.Helper() nca := natsConnect(t, fmt.Sprintf("nats://user:pwd@%s:%d", host, port)) defer nca.Close() msg, err := nca.Request("foo", []byte("sa request"), time.Second) if err != nil { t.Fatalf("Did not get response: %v", err) } // Check response from sb, it should have content "sb reply" and // reply subject should not have GW prefix if string(msg.Data) != "sb reply" || msg.Reply != sbReplySubj { t.Fatalf("Unexpected message from sb: %+v", msg) } // Now send our own reply: nca.Publish(msg.Reply, []byte("sa reply")) // And make sure that subS2 receives it... msg = natsNexMsg(t, subSB, time.Second) if string(msg.Data) != "sa reply" || msg.Reply != _EMPTY_ { t.Fatalf("Unexpected message from sa: %v", msg) } } // Create requestor on sa1 to check for direct reply from GW: testReqReply(t, oa1.Host, oa1.Port) // Now from sa2 to see reply coming from route (sa1) testReqReply(t, oa2.Host, oa2.Port) } // Similar to TestGatewaySendReplyAcrossGateways, but this time // with service import. func TestGatewaySendReplyAcrossGatewaysServiceImport(t *testing.T) { ob := testDefaultOptionsForGateway("B") setAccountUserPassInOptions(ob, "$foo", "clientBFoo", "password") setAccountUserPassInOptions(ob, "$bar", "clientBBar", "password") sb := runGatewayServer(ob) defer sb.Shutdown() oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa1.Cluster.PoolSize = 1 setAccountUserPassInOptions(oa1, "$foo", "clientAFoo", "password") setAccountUserPassInOptions(oa1, "$bar", "clientABar", "password") sa1 := runGatewayServer(oa1) defer sa1.Shutdown() waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) waitForOutboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sa1, 1, time.Second) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa2.Cluster.PoolSize = 1 setAccountUserPassInOptions(oa2, "$foo", "clientAFoo", "password") setAccountUserPassInOptions(oa2, "$bar", "clientABar", "password") oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa1.Cluster.Host, oa1.Cluster.Port)) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() oa3 := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa3.Cluster.PoolSize = 1 setAccountUserPassInOptions(oa3, "$foo", "clientAFoo", "password") setAccountUserPassInOptions(oa3, "$bar", "clientABar", "password") oa3.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa1.Cluster.Host, oa1.Cluster.Port)) sa3 := runGatewayServer(oa3) defer sa3.Shutdown() waitForOutboundGateways(t, sa2, 1, time.Second) waitForOutboundGateways(t, sa3, 1, time.Second) waitForInboundGateways(t, sb, 3, time.Second) checkClusterFormed(t, sa1, sa2, sa3) // Setup account on B fooB, _ := sb.LookupAccount("$foo") // Add in the service export for the requests. Make it public. fooB.AddServiceExport("foo.request", nil) // Setup accounts on sa1, sa2 and sa3 setupAccsOnA := func(s *Server) { // Get accounts fooA, _ := s.LookupAccount("$foo") barA, _ := s.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooA.AddServiceExport("foo.request", nil) // Add import abilities to server A's bar account from foo. if err := barA.AddServiceImport(fooA, "bar.request", "foo.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } } setupAccsOnA(sa1) setupAccsOnA(sa2) setupAccsOnA(sa3) // clientB will be connected to sb and be the service endpoint and responder. bURL := fmt.Sprintf("nats://clientBFoo:password@127.0.0.1:%d", ob.Port) clientBFoo := natsConnect(t, bURL) defer clientBFoo.Close() subBFoo := natsSubSync(t, clientBFoo, "foo.request") natsFlush(t, clientBFoo) // Create another client on B for account $bar that will listen to // the reply subject. bURL = fmt.Sprintf("nats://clientBBar:password@127.0.0.1:%d", ob.Port) clientBBar := natsConnect(t, bURL) defer clientBBar.Close() replySubj := "reply" subBReply := natsSubSync(t, clientBBar, replySubj) natsFlush(t, clientBBar) testServiceImport := func(t *testing.T, host string, port int) { t.Helper() bURL := fmt.Sprintf("nats://clientABar:password@%s:%d", host, port) clientABar := natsConnect(t, bURL) defer clientABar.Close() subAReply := natsSubSync(t, clientABar, replySubj) natsFlush(t, clientABar) // Send the request from clientA on bar.request, which // will be translated to foo.request and sent over. natsPubReq(t, clientABar, "bar.request", replySubj, []byte("hi")) natsFlush(t, clientABar) // Expect the request to be received on subAFoo msg, err := subBFoo.NextMsg(time.Second) if err != nil { t.Fatalf("subBFoo failed to get request: %v", err) } if msg.Subject != "foo.request" || string(msg.Data) != "hi" { t.Fatalf("Unexpected message: %v", msg) } if msg.Reply == replySubj { t.Fatalf("Expected randomized reply, but got original") } // Check for duplicate message if msg, err := subBFoo.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected msg: %v", msg) } // Send reply natsPub(t, clientBFoo, msg.Reply, []byte("ok-42")) natsFlush(t, clientBFoo) // Now check that the subscription on the reply receives the message... checkReply := func(t *testing.T, sub *nats.Subscription) { t.Helper() msg, err = sub.NextMsg(time.Second) if err != nil { t.Fatalf("sub failed to get reply: %v", err) } if msg.Subject != replySubj || string(msg.Data) != "ok-42" { t.Fatalf("Unexpected message: %v", msg) } } // Check subscription on A (where the request originated) checkReply(t, subAReply) // And the subscription on B (where the responder is located) checkReply(t, subBReply) } // We check the service import with GW working ok with either // direct connection between the responder's server to the // requestor's server and also through routes. testServiceImport(t, oa1.Host, oa1.Port) testServiceImport(t, oa2.Host, oa2.Port) // sa1 is the one receiving the reply from GW between B and A. // Check that the server routes directly to the server // with the interest. checkRoute := func(t *testing.T, s *Server, expected int64) { t.Helper() s.mu.Lock() defer s.mu.Unlock() s.forEachRoute(func(r *client) { r.mu.Lock() if r.route.remoteID != sa1.ID() { r.mu.Unlock() return } inMsgs := atomic.LoadInt64(&r.inMsgs) r.mu.Unlock() if inMsgs != expected { t.Fatalf("Expected %v incoming msgs, got %v", expected, inMsgs) } }) } // Wait a bit to make sure that we don't have a loop that // cause messages to be routed more than needed. time.Sleep(100 * time.Millisecond) checkRoute(t, sa2, 1) checkRoute(t, sa3, 0) testServiceImport(t, oa3.Host, oa3.Port) // Wait a bit to make sure that we don't have a loop that // cause messages to be routed more than needed. time.Sleep(100 * time.Millisecond) checkRoute(t, sa2, 1) checkRoute(t, sa3, 1) } func TestGatewayClientsDontReceiveMsgsOnGWPrefix(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Setup a responder on sb ncb := natsConnect(t, fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port)) defer ncb.Close() natsSub(t, ncb, "foo", func(m *nats.Msg) { if strings.HasPrefix(m.Reply, gwReplyPrefix) { m.Respond([]byte(fmt.Sprintf("-ERR: received request with mapped reply subject %q", m.Reply))) } else { m.Respond([]byte("+OK: reply")) } }) // And create a sub on ">" that should not get the $GR reply. subSB := natsSubSync(t, ncb, ">") natsFlush(t, ncb) checkExpectedSubs(t, 2, sb) nca := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer nca.Close() msg, err := nca.Request("foo", []byte("request"), time.Second) if err != nil { t.Fatalf("Did not get response: %v", err) } if string(msg.Data) != "+OK: reply" { t.Fatalf("Error from responder: %q", msg.Data) } // subSB would have also received the request, so drop that one. msg = natsNexMsg(t, subSB, time.Second) if string(msg.Data) != "request" { t.Fatalf("Wrong request: %q", msg.Data) } // Once sa gets the direct reply, it should resend the reply // with normal subject. So subSB should get the message with // a subject that does not start with $GNR prefix. msg = natsNexMsg(t, subSB, time.Second) if string(msg.Data) != "+OK: reply" || strings.HasPrefix(msg.Subject, gwReplyPrefix) { t.Fatalf("Unexpected message from sa: %v", msg) } // Check no more message... if m, err := subSB.NextMsg(100 * time.Millisecond); m != nil || err == nil { t.Fatalf("Expected only 1 message, got %+v", m) } } func TestGatewayNoAccInterestThenQSubThenRegularSub(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Connect on A and send a message ncA := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer ncA.Close() natsPub(t, ncA, "foo", []byte("hello")) natsFlush(t, ncA) // expect an A- on return gwb := sa.getOutboundGatewayConnection("B") checkForAccountNoInterest(t, gwb, globalAccountName, true, time.Second) // Create a connection o B, and create a queue sub first ncB := natsConnect(t, fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port)) defer ncB.Close() qsub := natsQueueSubSync(t, ncB, "bar", "queue") natsFlush(t, ncB) // A should have received a queue interest checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "bar", 1, time.Second) // Now on B, create a regular sub sub := natsSubSync(t, ncB, "baz") natsFlush(t, ncB) // From A now, produce a message on each subject and // expect both subs to receive their message. msgForQSub := []byte("msg_qsub") natsPub(t, ncA, "bar", msgForQSub) natsFlush(t, ncA) if msg := natsNexMsg(t, qsub, time.Second); !bytes.Equal(msgForQSub, msg.Data) { t.Fatalf("Expected msg for queue sub to be %q, got %q", msgForQSub, msg.Data) } // Publish for the regular sub msgForSub := []byte("msg_sub") natsPub(t, ncA, "baz", msgForSub) natsFlush(t, ncA) if msg := natsNexMsg(t, sub, time.Second); !bytes.Equal(msgForSub, msg.Data) { t.Fatalf("Expected msg for sub to be %q, got %q", msgForSub, msg.Data) } } // Similar to TestGatewayNoAccInterestThenQSubThenRegularSub but simulate // older incorrect behavior. func TestGatewayHandleUnexpectedASubUnsub(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Connect on A and send a message ncA := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer ncA.Close() natsPub(t, ncA, "foo", []byte("hello")) natsFlush(t, ncA) // expect an A- on return gwb := sa.getOutboundGatewayConnection("B") checkForAccountNoInterest(t, gwb, globalAccountName, true, time.Second) // Create a connection o B, and create a queue sub first ncB := natsConnect(t, fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port)) defer ncB.Close() qsub := natsQueueSubSync(t, ncB, "bar", "queue") natsFlush(t, ncB) // A should have received a queue interest checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "bar", 1, time.Second) // Now on B, create a regular sub sub := natsSubSync(t, ncB, "baz") natsFlush(t, ncB) // and reproduce old, wrong, behavior that would have resulted in sending an A- gwA := getInboundGatewayConnection(sb, "A") gwA.mu.Lock() gwA.enqueueProto([]byte("A- $G\r\n")) gwA.mu.Unlock() // From A now, produce a message on each subject and // expect both subs to receive their message. msgForQSub := []byte("msg_qsub") natsPub(t, ncA, "bar", msgForQSub) natsFlush(t, ncA) if msg := natsNexMsg(t, qsub, time.Second); !bytes.Equal(msgForQSub, msg.Data) { t.Fatalf("Expected msg for queue sub to be %q, got %q", msgForQSub, msg.Data) } // Publish for the regular sub msgForSub := []byte("msg_sub") natsPub(t, ncA, "baz", msgForSub) natsFlush(t, ncA) if msg := natsNexMsg(t, sub, time.Second); !bytes.Equal(msgForSub, msg.Data) { t.Fatalf("Expected msg for sub to be %q, got %q", msgForSub, msg.Data) } // Remove all subs on B. qsub.Unsubscribe() sub.Unsubscribe() ncB.Flush() // Produce a message from A expect A- natsPub(t, ncA, "foo", []byte("hello")) natsFlush(t, ncA) // expect an A- on return checkForAccountNoInterest(t, gwb, globalAccountName, true, time.Second) // Simulate B sending another A-, on A account no interest should remain same. gwA.mu.Lock() gwA.enqueueProto([]byte("A- $G\r\n")) gwA.mu.Unlock() checkForAccountNoInterest(t, gwb, globalAccountName, true, time.Second) // Create a queue sub on B qsub = natsQueueSubSync(t, ncB, "bar", "queue") natsFlush(t, ncB) checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "bar", 1, time.Second) // Make B send an A+ and verify that we sitll have the registered qsub interest gwA.mu.Lock() gwA.enqueueProto([]byte("A+ $G\r\n")) gwA.mu.Unlock() // Give a chance to A to possibly misbehave when receiving this proto time.Sleep(250 * time.Millisecond) // Now check interest is still there checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "bar", 1, time.Second) qsub.Unsubscribe() natsFlush(t, ncB) checkForRegisteredQSubInterest(t, sa, "B", globalAccountName, "bar", 0, time.Second) // Send A-, server A should set entry to nil gwA.mu.Lock() gwA.enqueueProto([]byte("A- $G\r\n")) gwA.mu.Unlock() checkForAccountNoInterest(t, gwb, globalAccountName, true, time.Second) // Send A+ and entry should be removed since there is no longer reason to // keep the entry. gwA.mu.Lock() gwA.enqueueProto([]byte("A+ $G\r\n")) gwA.mu.Unlock() checkForAccountNoInterest(t, gwb, globalAccountName, false, time.Second) // Last A+ should not change because account already removed from map. gwA.mu.Lock() gwA.enqueueProto([]byte("A+ $G\r\n")) gwA.mu.Unlock() checkForAccountNoInterest(t, gwb, globalAccountName, false, time.Second) } type captureGWInterestSwitchLogger struct { DummyLogger imss []string } func (l *captureGWInterestSwitchLogger) Debugf(format string, args ...any) { l.Lock() msg := fmt.Sprintf(format, args...) if strings.Contains(msg, fmt.Sprintf("switching account %q to %s mode", globalAccountName, InterestOnly)) || strings.Contains(msg, fmt.Sprintf("switching account %q to %s mode complete", globalAccountName, InterestOnly)) { l.imss = append(l.imss, msg) } l.Unlock() } func TestGatewayLogAccountInterestModeSwitch(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() logB := &captureGWInterestSwitchLogger{} sb.SetLogger(logB, true, true) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() logA := &captureGWInterestSwitchLogger{} sa.SetLogger(logA, true, true) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) ncB := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob.Port)) defer ncB.Close() natsSubSync(t, ncB, "foo") natsFlush(t, ncB) ncA := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", oa.Port)) defer ncA.Close() for i := 0; i < gatewayMaxRUnsubBeforeSwitch+10; i++ { subj := fmt.Sprintf("bar.%d", i) natsPub(t, ncA, subj, []byte("hello")) } natsFlush(t, ncA) gwA := getInboundGatewayConnection(sb, "A") checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { mode := Optimistic gwA.mu.Lock() e := gwA.gw.insim[globalAccountName] if e != nil { mode = e.mode } gwA.mu.Unlock() if mode != InterestOnly { return fmt.Errorf("not switched yet") } return nil }) checkGWInterestOnlyMode(t, sa, "B", globalAccountName) checkLog := func(t *testing.T, l *captureGWInterestSwitchLogger) { t.Helper() l.Lock() logs := append([]string(nil), l.imss...) l.Unlock() if len(logs) != 2 { t.Fatalf("Expected 2 logs about switching to interest-only, got %v", logs) } if !strings.Contains(logs[0], "switching account") { t.Fatalf("First log statement should have been about switching, got %v", logs[0]) } if !strings.Contains(logs[1], "complete") { t.Fatalf("Second log statement should have been about having switched, got %v", logs[1]) } } checkLog(t, logB) checkLog(t, logA) // Clear log of server B logB.Lock() logB.imss = nil logB.Unlock() // Force a switch on B to inbound gateway from A and make sure that it is // a no-op since this gateway connection has already been switched. sb.switchAccountToInterestMode(globalAccountName) logB.Lock() didSwitch := len(logB.imss) > 0 logB.Unlock() if didSwitch { t.Fatalf("Attempted to switch while it was already in interest mode only") } } func TestGatewayAccountInterestModeSwitchOnlyOncePerAccount(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() logB := &captureGWInterestSwitchLogger{} sb.SetLogger(logB, true, true) nc := natsConnect(t, sb.ClientURL()) defer nc.Close() natsSubSync(t, nc, "foo") natsQueueSubSync(t, nc, "bar", "baz") oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) wg := sync.WaitGroup{} total := 20 wg.Add(total) for i := 0; i < total; i++ { go func() { sb.switchAccountToInterestMode(globalAccountName) wg.Done() }() } wg.Wait() time.Sleep(50 * time.Millisecond) logB.Lock() nl := len(logB.imss) logB.Unlock() // There should be a trace for switching and when switch is complete if nl != 2 { t.Fatalf("Attempted to switch account too many times, number lines=%v", nl) } } func TestGatewaySingleOutbound(t *testing.T) { l, err := natsListen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error on listen: %v", err) } defer l.Close() port := l.Addr().(*net.TCPAddr).Port oa := testGatewayOptionsFromToWithTLS(t, "A", "B", []string{fmt.Sprintf("nats://127.0.0.1:%d", port)}) oa.Gateway.TLSTimeout = 0.1 sa := runGatewayServer(oa) defer sa.Shutdown() // Wait a bit for reconnections time.Sleep(500 * time.Millisecond) // Now prepare gateway B to take place of the bare listener. ob := testGatewayOptionsWithTLS(t, "B") // There is a risk that when stopping the listener and starting // the actual server, that port is being reused by some other process. ob.Gateway.Port = port l.Close() sb := runGatewayServer(ob) defer sb.Shutdown() // To make sure that we don't fail, bump the TLSTimeout now. cfg := sa.getRemoteGateway("B") cfg.Lock() cfg.TLSTimeout = 2.0 cfg.Unlock() waitForOutboundGateways(t, sa, 1, time.Second) sa.gateway.Lock() lm := len(sa.gateway.out) sa.gateway.Unlock() if lm != 1 { t.Fatalf("Expected 1 outbound, got %v", lm) } } func TestGatewayReplyMapTracking(t *testing.T) { // Increase the recSubExp value on servers so we have time // to check the replies mapping structures. subExp := 400 * time.Millisecond setRecSub := func(s *Server) { s.gateway.pasi.Lock() s.gateway.recSubExp = subExp s.gateway.pasi.Unlock() } ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() setRecSub(sb) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() setRecSub(sa) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) ncb := natsConnect(t, sb.ClientURL()) defer ncb.Close() count := 0 total := 100 ch := make(chan bool, 1) natsSub(t, ncb, "foo", func(m *nats.Msg) { m.Respond([]byte("reply")) if count++; count == total { ch <- true } }) natsFlush(t, ncb) var bc *client sb.mu.Lock() for _, c := range sb.clients { bc = c break } sb.mu.Unlock() nca := natsConnect(t, sa.ClientURL()) defer nca.Close() replySub := natsSubSync(t, nca, "bar.>") for i := 0; i < total; i++ { nca.PublishRequest("foo", fmt.Sprintf("bar.%d", i), []byte("request")) } waitCh(t, ch, "Did not receive all requests") check := func(t *testing.T, expectedIndicator int32, expectLenMap int, expectedSrvMapEmpty bool) { t.Helper() bc.mu.Lock() mapIndicator := atomic.LoadInt32(&bc.gwReplyMapping.check) var lenMap int if bc.gwReplyMapping.mapping != nil { lenMap = len(bc.gwReplyMapping.mapping) } bc.mu.Unlock() if mapIndicator != expectedIndicator { t.Fatalf("Client should map indicator should be %v, got %v", expectedIndicator, mapIndicator) } if lenMap != expectLenMap { t.Fatalf("Client map should have %v entries, got %v", expectLenMap, lenMap) } srvMapEmpty := true sb.gwrm.m.Range(func(_, _ any) bool { srvMapEmpty = false return false }) if srvMapEmpty != expectedSrvMapEmpty { t.Fatalf("Expected server map to be empty=%v, got %v", expectedSrvMapEmpty, srvMapEmpty) } } // Check that indicator is set and that there "total" entries in the map // and that srv map is not empty check(t, 1, total, false) // Receive all replies for i := 0; i < total; i++ { natsNexMsg(t, replySub, time.Second) } // Wait until entries expire time.Sleep(2*subExp + 100*time.Millisecond) // Now check again. check(t, 0, 0, true) } func TestGatewayNoAccountUnsubWhenServiceReplyInUse(t *testing.T) { oa := testDefaultOptionsForGateway("A") setAccountUserPassInOptions(oa, "$foo", "clientFoo", "password") setAccountUserPassInOptions(oa, "$bar", "clientBar", "password") sa := runGatewayServer(oa) defer sa.Shutdown() ob := testGatewayOptionsFromToWithServers(t, "B", "A", sa) setAccountUserPassInOptions(ob, "$foo", "clientFoo", "password") setAccountUserPassInOptions(ob, "$bar", "clientBar", "password") sb := runGatewayServer(ob) defer sb.Shutdown() waitForOutboundGateways(t, sa, 1, time.Second) waitForOutboundGateways(t, sb, 1, time.Second) waitForInboundGateways(t, sa, 1, time.Second) waitForInboundGateways(t, sb, 1, time.Second) // Get accounts fooA, _ := sa.LookupAccount("$foo") barA, _ := sa.LookupAccount("$bar") fooB, _ := sb.LookupAccount("$foo") barB, _ := sb.LookupAccount("$bar") // Add in the service export for the requests. Make it public. fooA.AddServiceExport("test.request", nil) fooB.AddServiceExport("test.request", nil) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Same on A. if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. aURL := fmt.Sprintf("nats://clientFoo:password@127.0.0.1:%d", oa.Port) clientA := natsConnect(t, aURL) defer clientA.Close() natsSub(t, clientA, "test.request", func(m *nats.Msg) { m.Respond([]byte("reply")) }) natsFlush(t, clientA) // Now setup client B on srvB who will send the requests. bURL := fmt.Sprintf("nats://clientBar:password@127.0.0.1:%d", ob.Port) clientB := natsConnect(t, bURL) defer clientB.Close() if _, err := clientB.Request("foo.request", []byte("request"), time.Second); err != nil { t.Fatalf("Did not get the reply: %v", err) } quitCh := make(chan bool, 1) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { select { case <-quitCh: return default: clientA.Publish("any.subject", []byte("any message")) time.Sleep(time.Millisecond) } } }() for i := 0; i < 1000; i++ { if _, err := clientB.Request("foo.request", []byte("request"), time.Second); err != nil { t.Fatalf("Did not get the reply: %v", err) } } close(quitCh) wg.Wait() } func TestGatewayCloseTLSConnection(t *testing.T) { oa := testGatewayOptionsWithTLS(t, "A") oa.DisableShortFirstPing = true oa.Gateway.TLSConfig.ClientAuth = tls.NoClientCert oa.Gateway.TLSTimeout = 100 sa := runGatewayServer(oa) defer sa.Shutdown() ob1 := testGatewayOptionsFromToWithTLS(t, "B", "A", []string{fmt.Sprintf("nats://127.0.0.1:%d", sa.GatewayAddr().Port)}) sb1 := runGatewayServer(ob1) defer sb1.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForInboundGateways(t, sb1, 1, 2*time.Second) endpoint := fmt.Sprintf("%s:%d", oa.Gateway.Host, oa.Gateway.Port) conn, err := net.DialTimeout("tcp", endpoint, 2*time.Second) if err != nil { t.Fatalf("Unexpected error on dial: %v", err) } defer conn.Close() tlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true}) defer tlsConn.Close() if err := tlsConn.Handshake(); err != nil { t.Fatalf("Unexpected error during handshake: %v", err) } connectOp := []byte("CONNECT {\"name\":\"serverID\",\"verbose\":false,\"pedantic\":false,\"tls_required\":true,\"gateway\":\"B\"}\r\n") if _, err := tlsConn.Write(connectOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } infoOp := []byte("INFO {\"server_id\":\"serverID\",\"tls_required\":true,\"gateway\":\"B\",\"gateway_nrp\":true}\r\n") if _, err := tlsConn.Write(infoOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } if _, err := tlsConn.Write([]byte("PING\r\n")); err != nil { t.Fatalf("Unexpected error writing PING: %v", err) } // Get gw connection var gw *client checkFor(t, time.Second, 15*time.Millisecond, func() error { sa.gateway.RLock() for _, g := range sa.gateway.in { g.mu.Lock() if g.opts.Name == "serverID" { gw = g } g.mu.Unlock() break } sa.gateway.RUnlock() if gw == nil { return fmt.Errorf("No gw registered yet") } return nil }) // Fill the buffer. We want to timeout on write so that nc.Close() // would block due to a write that cannot complete. buf := make([]byte, 64*1024) done := false for !done { gw.nc.SetWriteDeadline(time.Now().Add(time.Second)) if _, err := gw.nc.Write(buf); err != nil { done = true } gw.nc.SetWriteDeadline(time.Time{}) } ch := make(chan bool) go func() { select { case <-ch: return case <-time.After(3 * time.Second): fmt.Println("!!!! closeConnection is blocked, test will hang !!!") return } }() // Close the gateway gw.closeConnection(SlowConsumerWriteDeadline) ch <- true } func TestGatewayNoCrashOnInvalidSubject(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) ncB := natsConnect(t, sb.ClientURL()) defer ncB.Close() natsSubSync(t, ncB, "foo") natsFlush(t, ncB) ncA := natsConnect(t, sa.ClientURL()) defer ncA.Close() // Send on an invalid subject. Since there is interest on B, // we will receive an RS- instead of A- natsPub(t, ncA, "bar..baz", []byte("bad subject")) natsFlush(t, ncA) // Now create on B a sub on a wildcard subject sub := natsSubSync(t, ncB, "bar.*") natsFlush(t, ncB) // Server should not have crashed... natsPub(t, ncA, "bar.baz", []byte("valid subject")) if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Error getting message: %v", err) } } func TestGatewayUpdateURLsFromRemoteCluster(t *testing.T) { ob1 := testDefaultOptionsForGateway("B") sb1 := RunServer(ob1) defer sb1.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa := RunServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) // Add a server to cluster B. ob2 := testDefaultOptionsForGateway("B") ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) sb2 := RunServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa, 2, 2*time.Second) pmap := make(map[int]string) pmap[ob1.Gateway.Port] = "B1" pmap[ob2.Gateway.Port] = "B2" checkURLs := func(eurls map[string]string) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { rg := sa.getRemoteGateway("B") urls := rg.getURLsAsStrings() for _, u := range urls { if _, ok := eurls[u]; !ok { _, sport, _ := net.SplitHostPort(u) port, _ := strconv.Atoi(sport) return fmt.Errorf("URL %q (%s) should not be in the list of urls (%q)", u, pmap[port], eurls) } } return nil }) } expected := make(map[string]string) expected[fmt.Sprintf("127.0.0.1:%d", ob1.Gateway.Port)] = "B1" expected[fmt.Sprintf("127.0.0.1:%d", ob2.Gateway.Port)] = "B2" checkURLs(expected) // Add another in cluster B ob3 := testDefaultOptionsForGateway("B") ob3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) sb3 := RunServer(ob3) defer sb3.Shutdown() checkClusterFormed(t, sb1, sb2, sb3) waitForOutboundGateways(t, sb3, 1, 2*time.Second) waitForInboundGateways(t, sa, 3, 2*time.Second) pmap[ob3.Gateway.Port] = "B3" expected = make(map[string]string) expected[fmt.Sprintf("127.0.0.1:%d", ob1.Gateway.Port)] = "B1" expected[fmt.Sprintf("127.0.0.1:%d", ob2.Gateway.Port)] = "B2" expected[fmt.Sprintf("127.0.0.1:%d", ob3.Gateway.Port)] = "B3" checkURLs(expected) // Now stop server SB2, which should cause SA to remove it from its list. sb2.Shutdown() expected = make(map[string]string) expected[fmt.Sprintf("127.0.0.1:%d", ob1.Gateway.Port)] = "B1" expected[fmt.Sprintf("127.0.0.1:%d", ob3.Gateway.Port)] = "B3" checkURLs(expected) } type capturePingConn struct { net.Conn ch chan struct{} } func (c *capturePingConn) Write(b []byte) (int, error) { if bytes.Contains(b, []byte(pingProto)) { select { case c.ch <- struct{}{}: default: } } return c.Conn.Write(b) } func TestGatewayPings(t *testing.T) { gatewayMaxPingInterval = 50 * time.Millisecond defer func() { gatewayMaxPingInterval = gwMaxPingInterval }() ob := testDefaultOptionsForGateway("B") sb := RunServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := RunServer(oa) defer sa.Shutdown() waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) c := sa.getOutboundGatewayConnection("B") ch := make(chan struct{}, 1) c.mu.Lock() c.nc = &capturePingConn{c.nc, ch} c.mu.Unlock() for i := 0; i < 5; i++ { select { case <-ch: case <-time.After(250 * time.Millisecond): t.Fatalf("Did not send PING") } } } func TestGatewayTLSConfigReload(t *testing.T) { template := ` listen: 127.0.0.1:-1 gateway { name: "A" listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" %s timeout: 2 } } ` confA := createConfFile(t, []byte(fmt.Sprintf(template, ""))) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() optsB := testGatewayOptionsFromToWithTLS(t, "B", "A", []string{fmt.Sprintf("nats://127.0.0.1:%d", optsA.Gateway.Port)}) srvB := runGatewayServer(optsB) defer srvB.Shutdown() waitForGatewayFailedConnect(t, srvB, "A", true, time.Second) reloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file: "../test/configs/certs/ca.pem"`)) waitForInboundGateways(t, srvA, 1, time.Second) waitForOutboundGateways(t, srvA, 1, time.Second) waitForInboundGateways(t, srvB, 1, time.Second) waitForOutboundGateways(t, srvB, 1, time.Second) } func TestGatewayTLSConfigReloadForRemote(t *testing.T) { SetGatewaysSolicitDelay(5 * time.Millisecond) defer ResetGatewaysSolicitDelay() optsA := testGatewayOptionsWithTLS(t, "A") srvA := runGatewayServer(optsA) defer srvA.Shutdown() template := ` listen: 127.0.0.1:-1 gateway { name: "B" listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 } gateways [ { name: "A" url: "nats://127.0.0.1:%d" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" %s timeout: 2 } } ] } ` confB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.Gateway.Port, ""))) srvB, _ := RunServerWithConfig(confB) defer srvB.Shutdown() waitForGatewayFailedConnect(t, srvB, "A", true, time.Second) reloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.Gateway.Port, `ca_file: "../test/configs/certs/ca.pem"`)) waitForInboundGateways(t, srvA, 1, time.Second) waitForOutboundGateways(t, srvA, 1, time.Second) waitForInboundGateways(t, srvB, 1, time.Second) waitForOutboundGateways(t, srvB, 1, time.Second) } func TestGatewayAuthDiscovered(t *testing.T) { SetGatewaysSolicitDelay(5 * time.Millisecond) defer ResetGatewaysSolicitDelay() confA := createConfFile(t, []byte(` listen: 127.0.0.1:-1 gateway { name: "A" listen: 127.0.0.1:-1 authorization: { user: gwuser, password: changeme } } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() confB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 gateway { name: "B" listen: 127.0.0.1:-1 authorization: { user: gwuser, password: changeme } gateways: [ { name: A, url: nats://gwuser:changeme@127.0.0.1:%d } ] } `, optsA.Gateway.Port))) srvB, _ := RunServerWithConfig(confB) defer srvB.Shutdown() waitForInboundGateways(t, srvA, 1, time.Second) waitForOutboundGateways(t, srvA, 1, time.Second) waitForInboundGateways(t, srvB, 1, time.Second) waitForOutboundGateways(t, srvB, 1, time.Second) } func TestGatewayTLSCertificateImplicitAllowPass(t *testing.T) { testGatewayTLSCertificateImplicitAllow(t, true) } func TestGatewayTLSCertificateImplicitAllowFail(t *testing.T) { testGatewayTLSCertificateImplicitAllow(t, false) } func testGatewayTLSCertificateImplicitAllow(t *testing.T, pass bool) { // Base config for the servers cfg := createTempFile(t, "cfg") cfg.WriteString(fmt.Sprintf(` gateway { tls { cert_file = "../test/configs/certs/tlsauth/server.pem" key_file = "../test/configs/certs/tlsauth/server-key.pem" ca_file = "../test/configs/certs/tlsauth/ca.pem" verify_cert_and_check_known_urls = true insecure = %t timeout = 1 } } `, !pass)) // set insecure to skip verification on the outgoing end if err := cfg.Sync(); err != nil { t.Fatal(err) } cfg.Close() optsA := LoadConfig(cfg.Name()) optsB := LoadConfig(cfg.Name()) urlA := "nats://localhost:9995" urlB := "nats://localhost:9996" if !pass { urlA = "nats://127.0.0.1:9995" urlB = "nats://127.0.0.1:9996" } gwA, err := url.Parse(urlA) if err != nil { t.Fatal(err) } gwB, err := url.Parse(urlB) if err != nil { t.Fatal(err) } optsA.Host = "127.0.0.1" optsA.Port = -1 optsA.Gateway.Name = "A" optsA.Gateway.Port = 9995 optsA.Gateway.resolver = &localhostResolver{} optsB.Host = "127.0.0.1" optsB.Port = -1 optsB.Gateway.Name = "B" optsB.Gateway.Port = 9996 optsB.Gateway.resolver = &localhostResolver{} gateways := make([]*RemoteGatewayOpts, 2) gateways[0] = &RemoteGatewayOpts{ Name: optsA.Gateway.Name, URLs: []*url.URL{gwA}, } gateways[1] = &RemoteGatewayOpts{ Name: optsB.Gateway.Name, URLs: []*url.URL{gwB}, } optsA.Gateway.Gateways = gateways optsB.Gateway.Gateways = gateways SetGatewaysSolicitDelay(100 * time.Millisecond) defer ResetGatewaysSolicitDelay() srvA := RunServer(optsA) defer srvA.Shutdown() srvB := RunServer(optsB) defer srvB.Shutdown() if pass { waitForOutboundGateways(t, srvA, 1, 5*time.Second) waitForOutboundGateways(t, srvB, 1, 5*time.Second) } else { time.Sleep(1 * time.Second) // the fail case uses the IP, so a short wait is sufficient checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if srvA.NumOutboundGateways() != 0 || srvB.NumOutboundGateways() != 0 { return fmt.Errorf("No outbound gateway connection expected") } return nil }) } } func TestGatewayURLsNotRemovedOnDuplicateRoute(t *testing.T) { // For this test, we need to have servers in cluster B creating routes // to each other to help produce the "duplicate route" situation, so // we are forced to use deterministic ports. getEphemeralPort := func() int { t.Helper() l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error getting a port: %v", err) } p := l.Addr().(*net.TCPAddr).Port l.Close() return p } p1 := getEphemeralPort() p2 := getEphemeralPort() routeURLs := fmt.Sprintf("nats://127.0.0.1:%d,nats://127.0.0.1:%d", p1, p2) ob1 := testDefaultOptionsForGateway("B") ob1.Cluster.Port = p1 ob1.Routes = RoutesFromStr(routeURLs) sb1 := RunServer(ob1) defer sb1.Shutdown() ob2 := testDefaultOptionsForGateway("B") ob2.Cluster.Port = p2 ob2.Routes = RoutesFromStr(routeURLs) sb2 := RunServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) sa := RunServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 2, 2*time.Second) checkURLs := func(s *Server) { t.Helper() s.mu.Lock() urls := s.gateway.URLs.getAsStringSlice() s.mu.Unlock() if len(urls) != 2 { t.Fatalf("Expected 2 urls, got %v", urls) } } checkURLs(sb1) checkURLs(sb2) // As for sa, we should have both sb1 and sb2 urls in its outbound urls map c := sa.getOutboundGatewayConnection("B") if c == nil { t.Fatal("No outound connection found!") } c.mu.Lock() urls := c.gw.cfg.urls c.mu.Unlock() if len(urls) != 2 { t.Fatalf("Expected 2 urls to B, got %v", urls) } } func TestGatewayDuplicateServerName(t *testing.T) { // We will have 2 servers per cluster names "nats1" and "nats2", and have // the servers in the second cluster with the same name, but we will make // sure to connect "A/nats1" to "B/nats2" and "A/nats2" to "B/nats1" and // verify that we still discover the duplicate names. ob1 := testDefaultOptionsForGateway("B") ob1.ServerName = "nats1" sb1 := RunServer(ob1) defer sb1.Shutdown() ob2 := testDefaultOptionsForGateway("B") ob2.ServerName = "nats2" ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) sb2 := RunServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb2) oa1.ServerName = "nats1" // Needed later in the test oa1.Gateway.RejectUnknown = true sa1 := RunServer(oa1) defer sa1.Shutdown() sa1l := &captureErrorLogger{errCh: make(chan string, 100)} sa1.SetLogger(sa1l, false, false) oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) oa2.ServerName = "nats2" // Needed later in the test oa2.Gateway.RejectUnknown = true oa2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", oa1.Cluster.Port)) sa2 := RunServer(oa2) defer sa2.Shutdown() sa2l := &captureErrorLogger{errCh: make(chan string, 100)} sa2.SetLogger(sa2l, false, false) checkClusterFormed(t, sa1, sa2) checkForDupError := func(errCh chan string) { t.Helper() timeout := time.NewTimer(time.Second) for done := false; !done; { select { case err := <-errCh: if strings.Contains(err, "server has a duplicate name") { done = true } case <-timeout.C: t.Fatal("Did not get error about servers in super-cluster with same name") } } } // Since only servers from "A" have configured outbound to // cluster "B", only servers on "A" are expected to report error. for _, errCh := range []chan string{sa1l.errCh, sa2l.errCh} { checkForDupError(errCh) } // So now we are going to fix names and wait for the super cluster to form. sa2.Shutdown() sa1.Shutdown() // Drain the error channels for _, errCh := range []chan string{sa1l.errCh, sa2l.errCh} { for done := false; !done; { select { case <-errCh: default: done = true } } } oa1.ServerName = "a_nats1" oa2.ServerName = "a_nats2" sa1 = RunServer(oa1) defer sa1.Shutdown() sa2 = RunServer(oa2) defer sa2.Shutdown() checkClusterFormed(t, sa1, sa2) waitForOutboundGateways(t, sa1, 1, 2*time.Second) waitForOutboundGateways(t, sa2, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) // Now add a server on cluster B (that does not have outbound // gateway connections explicitly defined) and use the name // of one of the cluster A's server. We should get an error. ob3 := testDefaultOptionsForGateway("B") ob3.ServerName = "a_nats2" ob3.Accounts = []*Account{NewAccount("sys")} ob3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob2.Cluster.Port)) sb3 := RunServer(ob3) defer sb3.Shutdown() sb3l := &captureErrorLogger{errCh: make(chan string, 100)} sb3.SetLogger(sb3l, false, false) checkClusterFormed(t, sb1, sb2, sb3) // It should report the error when trying to create the GW connection checkForDupError(sb3l.errCh) // Stop this node sb3.Shutdown() checkClusterFormed(t, sb1, sb2) // Now create a GW "C" with a server that uses the same name than one of // the server on "A", say "a_nats2". // This server will connect to "B", and "B" will gossip "A" back to "C" // and "C" will then try to connect to "A", but "A" rejects unknown, so // connection will be refused. However, we want to make sure that the // duplicate server name is still detected. oc := testGatewayOptionsFromToWithServers(t, "C", "B", sb1) oc.ServerName = "a_nats2" oc.Accounts = []*Account{NewAccount("sys")} sc := RunServer(oc) defer sc.Shutdown() scl := &captureErrorLogger{errCh: make(chan string, 100)} sc.SetLogger(scl, false, false) // It should report the error when trying to create the GW connection // to cluster "A" checkForDupError(scl.errCh) } func TestGatewayNoPanicOnStartupWithMonitoring(t *testing.T) { o := testDefaultOptionsForGateway("B") o.HTTPHost = "127.0.0.1" o.HTTPPort = 8888 s, err := NewServer(o) require_NoError(t, err) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() time.Sleep(50 * time.Millisecond) s.Start() s.WaitForShutdown() }() for { g, err := s.Gatewayz(nil) if err != nil { continue } if g.Port != 0 && g.Port != s.GatewayAddr().Port { t.Fatalf("Unexpected port: %v vs %v", g.Port, s.GatewayAddr().Port) } break } s.Shutdown() wg.Wait() } func TestGatewaySwitchToInterestOnlyModeImmediately(t *testing.T) { o2 := testDefaultOptionsForGateway("B") // Add users to cause s2 to require auth. Will add an account with user later. o2.Users = append([]*User(nil), &User{Username: "test", Password: "pwd"}) s2 := runGatewayServer(o2) defer s2.Shutdown() o1 := testGatewayOptionsFromToWithServers(t, "A", "B", s2) setAccountUserPassInOptions(o1, "$foo", "ivan", "password") s1 := runGatewayServer(o1) defer s1.Shutdown() waitForOutboundGateways(t, s1, 1, time.Second) waitForOutboundGateways(t, s2, 1, time.Second) s1Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o1.Port) nc := natsConnect(t, s1Url) defer nc.Close() natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount := func(t *testing.T, c *client, expected int) { t.Helper() c.mu.Lock() out := c.outMsgs c.mu.Unlock() if int(out) != expected { t.Fatalf("Expected %d message(s) to be sent over, got %v", expected, out) } } // No message should be sent gwcb := s1.getOutboundGatewayConnection("B") checkCount(t, gwcb, 0) // Check that we are in interest-only mode, but in this case, since s2 does // have the account, we should have the account not even present in the map. checkGWInterestOnlyModeOrNotPresent(t, s1, "B", "$foo", true) // Add account to S2 and a client. s2FooAcc, err := s2.RegisterAccount("$foo") if err != nil { t.Fatalf("Error registering account: %v", err) } s2.mu.Lock() s2.users["ivan"] = &User{Account: s2FooAcc, Username: "ivan", Password: "password"} s2.mu.Unlock() s2Url := fmt.Sprintf("nats://ivan:password@127.0.0.1:%d", o2.Port) ncS2 := natsConnect(t, s2Url) defer ncS2.Close() natsSubSync(t, ncS2, "asub") // This time we will have the account in the map and it will be interest-only checkGWInterestOnlyMode(t, s1, "B", "$foo") // Now publish a message, still should not go because the sub is on "asub" natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 0) natsSubSync(t, ncS2, "foo") natsFlush(t, ncS2) checkGWInterestOnlyModeInterestOn(t, s1, "B", "$foo", "foo") // Publish on foo natsPub(t, nc, "foo", []byte("hello")) natsFlush(t, nc) checkCount(t, gwcb, 1) } func TestGatewaySlowConsumer(t *testing.T) { gatewayMaxPingInterval = 50 * time.Millisecond defer func() { gatewayMaxPingInterval = gwMaxPingInterval }() ob := testDefaultOptionsForGateway("B") sb := RunServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) sa := RunServer(oa) defer sa.Shutdown() waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) c := sa.getOutboundGatewayConnection("B") c.mu.Lock() c.out.wdl = time.Nanosecond c.mu.Unlock() <-time.After(250 * time.Millisecond) got := sa.NumSlowConsumersGateways() expected := uint64(1) if got != 1 { t.Errorf("got: %d, expected: %d", got, expected) } got = sb.NumSlowConsumersGateways() expected = 0 if got != expected { t.Errorf("got: %d, expected: %d", got, expected) } } // https://github.com/nats-io/nats-server/issues/5187 func TestGatewayConnectEvents(t *testing.T) { checkEvents := func(t *testing.T, name string, queue bool) { t.Run(name, func(t *testing.T) { ca := createClusterEx(t, true, 5*time.Millisecond, true, "A", 2) defer shutdownCluster(ca) cb := createClusterEx(t, true, 5*time.Millisecond, true, "B", 2, ca) defer shutdownCluster(cb) sysA, err := nats.Connect(ca.randomServer().ClientURL(), nats.UserInfo("sys", "pass")) require_NoError(t, err) defer sysA.Close() var sub1 *nats.Subscription if queue { sub1, err = sysA.QueueSubscribeSync("$SYS.ACCOUNT.FOO.CONNECT", "myqueue") } else { sub1, err = sysA.SubscribeSync("$SYS.ACCOUNT.FOO.CONNECT") } require_NoError(t, err) cA, err := nats.Connect(ca.randomServer().ClientURL(), nats.UserInfo("foo", "pass")) require_NoError(t, err) defer cA.Close() msg, err := sub1.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, msg.Subject, "$SYS.ACCOUNT.FOO.CONNECT") cB, err := nats.Connect(cb.randomServer().ClientURL(), nats.UserInfo("foo", "pass")) require_NoError(t, err) defer cB.Close() msg, err = sub1.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, msg.Subject, "$SYS.ACCOUNT.FOO.CONNECT") }) } checkEvents(t, "Unqueued", false) checkEvents(t, "Queued", true) } func disconnectInboundGateways(s *Server) { s.gateway.RLock() in := s.gateway.in s.gateway.RUnlock() s.gateway.RLock() for _, client := range in { s.gateway.RUnlock() client.closeConnection(ClientClosed) s.gateway.RLock() } s.gateway.RUnlock() } type testMissingOCSPStapleLogger struct { DummyLogger ch chan string } func (l *testMissingOCSPStapleLogger) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "peer missing OCSP Staple") { select { case l.ch <- msg: default: } } } func TestGatewayOCSPMissingPeerStapleIssue(t *testing.T) { const ( caCert = "../test/configs/certs/ocsp/ca-cert.pem" caKey = "../test/configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponderCustomTimeout(t, caCert, caKey, 10*time.Minute) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) // Node A SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) // Node B SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) // Node C SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) // Node A rotated certs SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "../test/configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" ocsp { mode = always } system_account = sys accounts { sys { users = [{ user: sys, pass: sys }]} guest { users = [{ user: guest, pass: guest }]} } no_auth_user = guest store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "../test/configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "../test/configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "../test/configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // Gateway B connects to Gateway A. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" ocsp { mode = always } system_account = sys accounts { sys { users = [{ user: sys, pass: sys }]} guest { users = [{ user: guest, pass: guest }]} } no_auth_user = guest store_dir: '%s' gateway { name: B host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A" url: "nats://127.0.0.1:%d" }] tls { cert_file: "../test/configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "../test/configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "../test/configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() // Wait for connectivity between A and B. waitForOutboundGateways(t, srvB, 1, 5*time.Second) // Gateway C also connects to Gateway A. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" ocsp { mode = always } system_account = sys accounts { sys { users = [{ user: sys, pass: sys }]} guest { users = [{ user: guest, pass: guest }]} } no_auth_user = guest store_dir: '%s' gateway { name: C host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{name: "A", url: "nats://127.0.0.1:%d" }] tls { cert_file: "../test/configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "../test/configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "../test/configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() //////////////////////////////////////////////////////////////////////////// // // // A and B are connected at this point and A is starting with certs that // // will be rotated. // // //////////////////////////////////////////////////////////////////////////// cB, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port), nats.ErrorHandler(noOpErrHandler), ) require_NoError(t, err) defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsC.Port), nats.ErrorHandler(noOpErrHandler), ) require_NoError(t, err) defer cC.Close() _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) require_NoError(t, err) cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) require_NoError(t, err) cB.Flush() waitForOutboundGateways(t, srvB, 1, 10*time.Second) waitForOutboundGateways(t, srvC, 2, 10*time.Second) ///////////////////////////////////////////////////////////////////////////////// // // // Switch all the certs from server A, all OCSP monitors should be restarted // // so it should have new staples. // // // ///////////////////////////////////////////////////////////////////////////////// srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" ocsp { mode = always } system_account = sys accounts { sys { users = [{ user: sys, pass: sys }]} guest { users = [{ user: guest, pass: guest }]} } no_auth_user = guest store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "../test/configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "../test/configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "../test/configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } waitForOutboundGateways(t, srvA, 2, 5*time.Second) waitForOutboundGateways(t, srvB, 2, 5*time.Second) waitForOutboundGateways(t, srvC, 2, 5*time.Second) // Now clients connect to C can communicate with B and A. _, err = cC.Request("foo", nil, 2*time.Second) require_NoError(t, err) _, err = cC.Request("bar", nil, 2*time.Second) require_NoError(t, err) // Reload and disconnect very fast trying to produce the race. ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) defer cancel() // Swap logger from server to capture the missing peer log. lA := &testMissingOCSPStapleLogger{ch: make(chan string, 30)} srvA.SetLogger(lA, false, false) lB := &testMissingOCSPStapleLogger{ch: make(chan string, 30)} srvB.SetLogger(lB, false, false) lC := &testMissingOCSPStapleLogger{ch: make(chan string, 30)} srvC.SetLogger(lC, false, false) // Start with a reload from the last server that connected directly to A. err = srvC.Reload() require_NoError(t, err) // Stress reconnections and reloading servers without getting // missing OCSP peer staple errors. var wg sync.WaitGroup wg.Add(1) go func() { for range time.NewTicker(500 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } disconnectInboundGateways(srvA) } }() wg.Add(1) go func() { for range time.NewTicker(500 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } disconnectInboundGateways(srvB) } }() wg.Add(1) go func() { for range time.NewTicker(500 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } disconnectInboundGateways(srvC) } }() wg.Add(1) go func() { for range time.NewTicker(700 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } srvC.Reload() } }() wg.Add(1) go func() { for range time.NewTicker(800 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } srvB.Reload() } }() wg.Add(1) go func() { for range time.NewTicker(900 * time.Millisecond).C { select { case <-ctx.Done(): wg.Done() return default: } srvA.Reload() } }() select { case <-ctx.Done(): case msg := <-lA.ch: t.Fatalf("Server A: Got OCSP Staple error: %v", msg) case msg := <-lB.ch: t.Fatalf("Server B: Got OCSP Staple error: %v", msg) case msg := <-lC.ch: t.Fatalf("Server C: Got OCSP Staple error: %v", msg) } waitForOutboundGateways(t, srvA, 2, 5*time.Second) waitForOutboundGateways(t, srvB, 2, 5*time.Second) waitForOutboundGateways(t, srvC, 2, 5*time.Second) wg.Wait() } func TestGatewayOutboundDetectsStaleConnectionIfNoInfo(t *testing.T) { l, err := net.Listen("tcp", "127.0.0.1:0") require_NoError(t, err) defer l.Close() ch := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() c, err := l.Accept() if err != nil { return } defer c.Close() <-ch }() url := fmt.Sprintf("nats://%s", l.Addr()) o := testGatewayOptionsFromToWithURLs(t, "A", "B", []string{url}) o.gatewaysSolicitDelay = time.Millisecond o.DisableShortFirstPing = false o.PingInterval = 50 * time.Millisecond o.MaxPingsOut = 3 o.NoLog = false s, err := NewServer(o) require_NoError(t, err) defer s.Shutdown() log := &captureDebugLogger{dbgCh: make(chan string, 100)} s.SetLogger(log, true, false) s.Start() timeout := time.NewTimer(time.Second) defer timeout.Stop() for done := false; !done; { select { case dbg := <-log.dbgCh: // The server should not send PING because the accept side expects // the CONNECT as the first protocol (otherwise it would be a parse // error if that were to happen). if strings.Contains(dbg, "Ping Timer") { t.Fatalf("The server should not have sent a ping, got %q", dbg) } // However, it should detect at one point that the connection is // stale and close it. if strings.Contains(dbg, "Stale") { done = true } case <-timeout.C: t.Fatalf("Did not capture the stale connection condition") } } s.Shutdown() close(ch) wg.Wait() s.WaitForShutdown() } nats-server-2.10.27/server/gsl/000077500000000000000000000000001477524627100162465ustar00rootroot00000000000000nats-server-2.10.27/server/gsl/gsl.go000066400000000000000000000272371477524627100173750ustar00rootroot00000000000000// Copyright 2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gsl import ( "errors" "sync" "github.com/nats-io/nats-server/v2/server/stree" ) // Sublist is a routing mechanism to handle subject distribution and // provides a facility to match subjects from published messages to // interested subscribers. Subscribers can have wildcard subjects to // match multiple published subjects. // Common byte variables for wildcards and token separator. const ( pwc = '*' pwcs = "*" fwc = '>' fwcs = ">" tsep = "." btsep = '.' _EMPTY_ = "" ) // Sublist related errors var ( ErrInvalidSubject = errors.New("gsl: invalid subject") ErrNotFound = errors.New("gsl: no matches found") ErrNilChan = errors.New("gsl: nil channel") ErrAlreadyRegistered = errors.New("gsl: notification already registered") ) // A GenericSublist stores and efficiently retrieves subscriptions. type GenericSublist[T comparable] struct { sync.RWMutex root *level[T] count uint32 } // A node contains subscriptions and a pointer to the next level. type node[T comparable] struct { next *level[T] subs map[T]string // value -> subject } // A level represents a group of nodes and special pointers to // wildcard nodes. type level[T comparable] struct { nodes map[string]*node[T] pwc, fwc *node[T] } // Create a new default node. func newNode[T comparable]() *node[T] { return &node[T]{subs: make(map[T]string)} } // Create a new default level. func newLevel[T comparable]() *level[T] { return &level[T]{nodes: make(map[string]*node[T])} } // NewSublist will create a default sublist with caching enabled per the flag. func NewSublist[T comparable]() *GenericSublist[T] { return &GenericSublist[T]{root: newLevel[T]()} } // Insert adds a subscription into the sublist func (s *GenericSublist[T]) Insert(subject string, value T) error { tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) s.Lock() var sfwc bool var n *node[T] l := s.root for _, t := range tokens { lt := len(t) if lt == 0 || sfwc { s.Unlock() return ErrInvalidSubject } if lt > 1 { n = l.nodes[t] } else { switch t[0] { case pwc: n = l.pwc case fwc: n = l.fwc sfwc = true default: n = l.nodes[t] } } if n == nil { n = newNode[T]() if lt > 1 { l.nodes[t] = n } else { switch t[0] { case pwc: l.pwc = n case fwc: l.fwc = n default: l.nodes[t] = n } } } if n.next == nil { n.next = newLevel[T]() } l = n.next } n.subs[value] = subject s.count++ s.Unlock() return nil } // Match will match all entries to the literal subject. // It will return a set of results for both normal and queue subscribers. func (s *GenericSublist[T]) Match(subject string, cb func(T)) { s.match(subject, cb, true) } // MatchBytes will match all entries to the literal subject. // It will return a set of results for both normal and queue subscribers. func (s *GenericSublist[T]) MatchBytes(subject []byte, cb func(T)) { s.match(string(subject), cb, true) } // HasInterest will return whether or not there is any interest in the subject. // In cases where more detail is not required, this may be faster than Match. func (s *GenericSublist[T]) HasInterest(subject string) bool { return s.hasInterest(subject, true, nil) } // NumInterest will return the number of subs interested in the subject. // In cases where more detail is not required, this may be faster than Match. func (s *GenericSublist[T]) NumInterest(subject string) (np int) { s.hasInterest(subject, true, &np) return } func (s *GenericSublist[T]) match(subject string, cb func(T), doLock bool) { tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { if i-start == 0 { return } tokens = append(tokens, subject[start:i]) start = i + 1 } } if start >= len(subject) { return } tokens = append(tokens, subject[start:]) if doLock { s.RLock() defer s.RUnlock() } matchLevel(s.root, tokens, cb) } func (s *GenericSublist[T]) hasInterest(subject string, doLock bool, np *int) bool { tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { if i-start == 0 { return false } tokens = append(tokens, subject[start:i]) start = i + 1 } } if start >= len(subject) { return false } tokens = append(tokens, subject[start:]) if doLock { s.RLock() defer s.RUnlock() } return matchLevelForAny(s.root, tokens, np) } func matchLevelForAny[T comparable](l *level[T], toks []string, np *int) bool { var pwc, n *node[T] for i, t := range toks { if l == nil { return false } if l.fwc != nil { if np != nil { *np += len(l.fwc.subs) } return true } if pwc = l.pwc; pwc != nil { if match := matchLevelForAny(pwc.next, toks[i+1:], np); match { return true } } n = l.nodes[t] if n != nil { l = n.next } else { l = nil } } if n != nil { if np != nil { *np += len(n.subs) } return len(n.subs) > 0 } if pwc != nil { if np != nil { *np += len(pwc.subs) } return len(pwc.subs) > 0 } return false } // callbacksForResults will make the necessary callbacks for each // result in this node. func callbacksForResults[T comparable](n *node[T], cb func(T)) { for sub := range n.subs { cb(sub) } } // matchLevel is used to recursively descend into the trie. func matchLevel[T comparable](l *level[T], toks []string, cb func(T)) { var pwc, n *node[T] for i, t := range toks { if l == nil { return } if l.fwc != nil { callbacksForResults(l.fwc, cb) } if pwc = l.pwc; pwc != nil { matchLevel(pwc.next, toks[i+1:], cb) } n = l.nodes[t] if n != nil { l = n.next } else { l = nil } } if n != nil { callbacksForResults(n, cb) } if pwc != nil { callbacksForResults(pwc, cb) } } // lnt is used to track descent into levels for a removal for pruning. type lnt[T comparable] struct { l *level[T] n *node[T] t string } // Raw low level remove, can do batches with lock held outside. func (s *GenericSublist[T]) remove(subject string, value T, shouldLock bool) error { tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) if shouldLock { s.Lock() defer s.Unlock() } var sfwc bool var n *node[T] l := s.root // Track levels for pruning var lnts [32]lnt[T] levels := lnts[:0] for _, t := range tokens { lt := len(t) if lt == 0 || sfwc { return ErrInvalidSubject } if l == nil { return ErrNotFound } if lt > 1 { n = l.nodes[t] } else { switch t[0] { case pwc: n = l.pwc case fwc: n = l.fwc sfwc = true default: n = l.nodes[t] } } if n != nil { levels = append(levels, lnt[T]{l, n, t}) l = n.next } else { l = nil } } if !s.removeFromNode(n, value) { return ErrNotFound } s.count-- for i := len(levels) - 1; i >= 0; i-- { l, n, t := levels[i].l, levels[i].n, levels[i].t if n.isEmpty() { l.pruneNode(n, t) } } return nil } // Remove will remove a subscription. func (s *GenericSublist[T]) Remove(subject string, value T) error { return s.remove(subject, value, true) } // pruneNode is used to prune an empty node from the tree. func (l *level[T]) pruneNode(n *node[T], t string) { if n == nil { return } if n == l.fwc { l.fwc = nil } else if n == l.pwc { l.pwc = nil } else { delete(l.nodes, t) } } // isEmpty will test if the node has any entries. Used // in pruning. func (n *node[T]) isEmpty() bool { return len(n.subs) == 0 && (n.next == nil || n.next.numNodes() == 0) } // Return the number of nodes for the given level. func (l *level[T]) numNodes() int { num := len(l.nodes) if l.pwc != nil { num++ } if l.fwc != nil { num++ } return num } // Remove the sub for the given node. func (s *GenericSublist[T]) removeFromNode(n *node[T], value T) (found bool) { if n == nil { return false } if _, found = n.subs[value]; found { delete(n.subs, value) } return found } // Count returns the number of subscriptions. func (s *GenericSublist[T]) Count() uint32 { s.RLock() defer s.RUnlock() return s.count } // numLevels will return the maximum number of levels // contained in the Sublist tree. func (s *GenericSublist[T]) numLevels() int { return visitLevel(s.root, 0) } // visitLevel is used to descend the Sublist tree structure // recursively. func visitLevel[T comparable](l *level[T], depth int) int { if l == nil || l.numNodes() == 0 { return depth } depth++ maxDepth := depth for _, n := range l.nodes { if n == nil { continue } newDepth := visitLevel(n.next, depth) if newDepth > maxDepth { maxDepth = newDepth } } if l.pwc != nil { pwcDepth := visitLevel(l.pwc.next, depth) if pwcDepth > maxDepth { maxDepth = pwcDepth } } if l.fwc != nil { fwcDepth := visitLevel(l.fwc.next, depth) if fwcDepth > maxDepth { maxDepth = fwcDepth } } return maxDepth } // IntersectStree will match all items in the given subject tree that // have interest expressed in the given sublist. The callback will only be called // once for each subject, regardless of overlapping subscriptions in the sublist. func IntersectStree[T1 any, T2 comparable](st *stree.SubjectTree[T1], sl *GenericSublist[T2], cb func(subj []byte, entry *T1)) { var _subj [255]byte intersectStree(st, sl.root, _subj[:0], cb) } func intersectStree[T1 any, T2 comparable](st *stree.SubjectTree[T1], r *level[T2], subj []byte, cb func(subj []byte, entry *T1)) { if r.numNodes() == 0 { // For wildcards we can't avoid Match, but if it's a literal subject at // this point, using Find is considerably cheaper. if subjectHasWildcard(string(subj)) { st.Match(subj, cb) } else if e, ok := st.Find(subj); ok { cb(subj, e) } return } nsubj := subj if len(nsubj) > 0 { nsubj = append(subj, '.') } switch { case r.fwc != nil: // We've reached a full wildcard, do a FWC match on the stree at this point // and don't keep iterating downward. nsubj := append(nsubj, '>') st.Match(nsubj, cb) case r.pwc != nil: // We've found a partial wildcard. We'll keep iterating downwards, but first // check whether there's interest at this level (without triggering dupes) and // match if so. nsubj := append(nsubj, '*') if len(r.pwc.subs) > 0 && r.pwc.next != nil && r.pwc.next.numNodes() > 0 { st.Match(nsubj, cb) } intersectStree(st, r.pwc.next, nsubj, cb) case r.numNodes() > 0: // Normal node with subject literals, keep iterating. for t, n := range r.nodes { nsubj := append(nsubj, t...) intersectStree(st, n.next, nsubj, cb) } } } // Determine if a subject has any wildcard tokens. func subjectHasWildcard(subject string) bool { // This one exits earlier then !subjectIsLiteral(subject) for i, c := range subject { if c == pwc || c == fwc { if (i == 0 || subject[i-1] == btsep) && (i+1 == len(subject) || subject[i+1] == btsep) { return true } } } return false } nats-server-2.10.27/server/gsl/gsl_test.go000066400000000000000000000343501477524627100204260ustar00rootroot00000000000000// Copyright 2016-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package gsl import ( "strings" "testing" "github.com/nats-io/nats-server/v2/server/stree" ) func TestGenericSublistInit(t *testing.T) { s := NewSublist[struct{}]() require_Equal(t, s.count, 0) require_Equal(t, s.Count(), s.count) } func TestGenericSublistInsertCount(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("foo", struct{}{})) require_NoError(t, s.Insert("bar", struct{}{})) require_NoError(t, s.Insert("foo.bar", struct{}{})) require_Equal(t, s.Count(), 3) } func TestGenericSublistSimple(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("foo", struct{}{})) require_Matches(t, s, "foo", 1) } func TestGenericSublistSimpleMultiTokens(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("foo.bar.baz", struct{}{})) require_Matches(t, s, "foo.bar.baz", 1) } func TestGenericSublistPartialWildcard(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("a.b.c", struct{}{})) require_NoError(t, s.Insert("a.*.c", struct{}{})) require_Matches(t, s, "a.b.c", 2) } func TestGenericSublistPartialWildcardAtEnd(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("a.b.c", struct{}{})) require_NoError(t, s.Insert("a.b.*", struct{}{})) require_Matches(t, s, "a.b.c", 2) } func TestGenericSublistFullWildcard(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("a.b.c", struct{}{})) require_NoError(t, s.Insert("a.>", struct{}{})) require_Matches(t, s, "a.b.c", 2) require_Matches(t, s, "a.>", 1) } func TestGenericSublistRemove(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("a.b.c.d", struct{}{})) require_Equal(t, s.Count(), 1) require_Matches(t, s, "a.b.c.d", 1) require_NoError(t, s.Remove("a.b.c.d", struct{}{})) require_Equal(t, s.Count(), 0) require_Matches(t, s, "a.b.c.d", 0) } func TestGenericSublistRemoveWildcard(t *testing.T) { s := NewSublist[int]() require_NoError(t, s.Insert("a.b.c.d", 11)) require_NoError(t, s.Insert("a.b.*.d", 22)) require_NoError(t, s.Insert("a.b.>", 33)) require_Equal(t, s.Count(), 3) require_Matches(t, s, "a.b.c.d", 3) require_NoError(t, s.Remove("a.b.*.d", 22)) require_Equal(t, s.Count(), 2) require_Matches(t, s, "a.b.c.d", 2) require_NoError(t, s.Remove("a.b.>", 33)) require_Equal(t, s.Count(), 1) require_Matches(t, s, "a.b.c.d", 1) require_NoError(t, s.Remove("a.b.c.d", 11)) require_Equal(t, s.Count(), 0) require_Matches(t, s, "a.b.c.d", 0) } func TestGenericSublistRemoveCleanup(t *testing.T) { s := NewSublist[struct{}]() require_Equal(t, s.numLevels(), 0) require_NoError(t, s.Insert("a.b.c.d.e.f", struct{}{})) require_Equal(t, s.numLevels(), 6) require_NoError(t, s.Remove("a.b.c.d.e.f", struct{}{})) require_Equal(t, s.numLevels(), 0) } func TestGenericSublistRemoveCleanupWildcards(t *testing.T) { s := NewSublist[struct{}]() require_Equal(t, s.numLevels(), 0) require_NoError(t, s.Insert("a.b.*.d.e.>", struct{}{})) require_Equal(t, s.numLevels(), 6) require_NoError(t, s.Remove("a.b.*.d.e.>", struct{}{})) require_Equal(t, s.numLevels(), 0) } func TestGenericSublistInvalidSubjectsInsert(t *testing.T) { s := NewSublist[struct{}]() // Insert, or subscriptions, can have wildcards, but not empty tokens, // and can not have a FWC that is not the terminal token. require_Error(t, s.Insert(".foo", struct{}{}), ErrInvalidSubject) require_Error(t, s.Insert("foo.", struct{}{}), ErrInvalidSubject) require_Error(t, s.Insert("foo..bar", struct{}{}), ErrInvalidSubject) require_Error(t, s.Insert("foo.bar..baz", struct{}{}), ErrInvalidSubject) require_Error(t, s.Insert("foo.>.baz", struct{}{}), ErrInvalidSubject) } func TestGenericSublistBadSubjectOnRemove(t *testing.T) { s := NewSublist[struct{}]() require_Error(t, s.Insert("a.b..d", struct{}{}), ErrInvalidSubject) require_Error(t, s.Remove("a.b..d", struct{}{}), ErrInvalidSubject) require_Error(t, s.Remove("a.>.b", struct{}{}), ErrInvalidSubject) } func TestGenericSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert("foo", struct{}{})) require_Matches(t, s, "foo", 1) require_Matches(t, s, "foo.bar", 0) } func TestGenericSublistInsertWithWildcardsAsLiterals(t *testing.T) { s := NewSublist[int]() for i, subject := range []string{"foo.*-", "foo.>-"} { require_NoError(t, s.Insert(subject, i)) require_Matches(t, s, "foo.bar", 0) require_Matches(t, s, subject, 1) } } func TestGenericSublistRemoveWithWildcardsAsLiterals(t *testing.T) { s := NewSublist[int]() for i, subject := range []string{"foo.*-", "foo.>-"} { require_NoError(t, s.Insert(subject, i)) require_Matches(t, s, "foo.bar", 0) require_Matches(t, s, subject, 1) require_Error(t, s.Remove("foo.bar", i), ErrNotFound) require_Equal(t, s.Count(), 1) require_NoError(t, s.Remove(subject, i)) require_Equal(t, s.Count(), 0) } } func TestGenericSublistMatchWithEmptyTokens(t *testing.T) { s := NewSublist[struct{}]() require_NoError(t, s.Insert(">", struct{}{})) for _, subject := range []string{".foo", "..foo", "foo..", "foo.", "foo..bar", "foo...bar"} { t.Run(subject, func(t *testing.T) { require_Matches(t, s, subject, 0) }) } } func TestGenericSublistHasInterest(t *testing.T) { s := NewSublist[int]() require_NoError(t, s.Insert("foo", 11)) // Expect to find that "foo" matches but "bar" doesn't. // At this point nothing should be in the cache. require_True(t, s.HasInterest("foo")) require_False(t, s.HasInterest("bar")) // Call Match on a subject we know there is no match. require_Matches(t, s, "bar", 0) require_False(t, s.HasInterest("bar")) // Remove fooSub and check interest again require_NoError(t, s.Remove("foo", 11)) require_False(t, s.HasInterest("foo")) // Try with some wildcards require_NoError(t, s.Insert("foo.*", 22)) require_False(t, s.HasInterest("foo")) require_True(t, s.HasInterest("foo.bar")) require_False(t, s.HasInterest("foo.bar.baz")) // Remove sub, there should be no interest require_NoError(t, s.Remove("foo.*", 22)) require_False(t, s.HasInterest("foo")) require_False(t, s.HasInterest("foo.bar")) require_False(t, s.HasInterest("foo.bar.baz")) require_NoError(t, s.Insert("foo.>", 33)) require_False(t, s.HasInterest("foo")) require_True(t, s.HasInterest("foo.bar")) require_True(t, s.HasInterest("foo.bar.baz")) require_NoError(t, s.Remove("foo.>", 33)) require_False(t, s.HasInterest("foo")) require_False(t, s.HasInterest("foo.bar")) require_False(t, s.HasInterest("foo.bar.baz")) require_NoError(t, s.Insert("*.>", 44)) require_False(t, s.HasInterest("foo")) require_True(t, s.HasInterest("foo.bar")) require_True(t, s.HasInterest("foo.baz")) require_NoError(t, s.Remove("*.>", 44)) require_NoError(t, s.Insert("*.bar", 55)) require_False(t, s.HasInterest("foo")) require_True(t, s.HasInterest("foo.bar")) require_False(t, s.HasInterest("foo.baz")) require_NoError(t, s.Remove("*.bar", 55)) require_NoError(t, s.Insert("*", 66)) require_True(t, s.HasInterest("foo")) require_False(t, s.HasInterest("foo.bar")) require_NoError(t, s.Remove("*", 66)) } func TestGenericSublistNumInterest(t *testing.T) { s := NewSublist[int]() require_NoError(t, s.Insert("foo", 11)) require_NumInterest := func(t *testing.T, subj string, wnp int) { t.Helper() require_Matches(t, s, subj, wnp) require_Equal(t, s.NumInterest(subj), wnp) } // Expect to find that "foo" matches but "bar" doesn't. // At this point nothing should be in the cache. require_NumInterest(t, "foo", 1) require_NumInterest(t, "bar", 0) // Remove fooSub and check interest again require_NoError(t, s.Remove("foo", 11)) require_NumInterest(t, "foo", 0) // Try with some wildcards require_NoError(t, s.Insert("foo.*", 22)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 1) require_NumInterest(t, "foo.bar.baz", 0) // Remove sub, there should be no interest require_NoError(t, s.Remove("foo.*", 22)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 0) require_NumInterest(t, "foo.bar.baz", 0) require_NoError(t, s.Insert("foo.>", 33)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 1) require_NumInterest(t, "foo.bar.baz", 1) require_NoError(t, s.Remove("foo.>", 33)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 0) require_NumInterest(t, "foo.bar.baz", 0) require_NoError(t, s.Insert("*.>", 44)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 1) require_NumInterest(t, "foo.bar.baz", 1) require_NoError(t, s.Remove("*.>", 44)) require_NoError(t, s.Insert("*.bar", 55)) require_NumInterest(t, "foo", 0) require_NumInterest(t, "foo.bar", 1) require_NumInterest(t, "foo.bar.baz", 0) require_NoError(t, s.Remove("*.bar", 55)) require_NoError(t, s.Insert("*", 66)) require_NumInterest(t, "foo", 1) require_NumInterest(t, "foo.bar", 0) require_NoError(t, s.Remove("*", 66)) } func TestGenericSublistInterestBasedIntersection(t *testing.T) { st := stree.NewSubjectTree[struct{}]() st.Insert([]byte("one.two.three.four"), struct{}{}) st.Insert([]byte("one.two.three.five"), struct{}{}) st.Insert([]byte("one.two.six"), struct{}{}) st.Insert([]byte("one.two.seven"), struct{}{}) st.Insert([]byte("eight.nine"), struct{}{}) require_NoDuplicates := func(t *testing.T, got map[string]int) { for _, c := range got { require_Equal(t, c, 1) } } t.Run("Literals", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one.two.six", 11)) require_NoError(t, sl.Insert("eight.nine", 22)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWC", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one.two.*.*", 11)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWCOverlapping", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one.two.*.four", 11)) require_NoError(t, sl.Insert("one.two.*.*", 22)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWCAll", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("*.*", 11)) require_NoError(t, sl.Insert("*.*.*", 22)) require_NoError(t, sl.Insert("*.*.*.*", 33)) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) require_True(t, sl.HasInterest("foo.bar.baz.qux")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 5) require_NoDuplicates(t, got) }) t.Run("FWC", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one.>", 11)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 4) require_NoDuplicates(t, got) }) t.Run("FWCOverlapping", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one.two.three.four", 11)) require_NoError(t, sl.Insert("one.>", 22)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 4) require_NoDuplicates(t, got) }) t.Run("FWCAll", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert(">", 11)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 5) require_NoDuplicates(t, got) }) t.Run("NoMatch", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one", 11)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 0) }) t.Run("NoMatches", func(t *testing.T) { got := map[string]int{} sl := NewSublist[int]() require_NoError(t, sl.Insert("one", 11)) require_NoError(t, sl.Insert("eight", 22)) require_NoError(t, sl.Insert("ten", 33)) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 0) }) } // --- TEST HELPERS --- func require_Matches[T comparable](t *testing.T, s *GenericSublist[T], sub string, c int) { t.Helper() matches := 0 s.Match(sub, func(_ T) { matches++ }) require_Equal(t, matches, c) } func require_True(t testing.TB, b bool) { t.Helper() if !b { t.Fatalf("require true, but got false") } } func require_False(t testing.TB, b bool) { t.Helper() if b { t.Fatalf("require false, but got true") } } func require_NoError(t testing.TB, err error) { t.Helper() if err != nil { t.Fatalf("require no error, but got: %v", err) } } func require_Error(t testing.TB, err error, expected ...error) { t.Helper() if err == nil { t.Fatalf("require error, but got none") } if len(expected) == 0 { return } // Try to strip nats prefix from Go library if present. const natsErrPre = "nats: " eStr := err.Error() if strings.HasPrefix(eStr, natsErrPre) { eStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1) } for _, e := range expected { if err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) { return } } t.Fatalf("Expected one of %v, got '%v'", expected, err) } func require_Equal[T comparable](t testing.TB, a, b T) { t.Helper() if a != b { t.Fatalf("require %T equal, but got: %v != %v", a, a, b) } } func require_Len(t testing.TB, a, b int) { t.Helper() if a != b { t.Fatalf("require len, but got: %v != %v", a, b) } } nats-server-2.10.27/server/ipqueue.go000066400000000000000000000142151477524627100174700ustar00rootroot00000000000000// Copyright 2021-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "sync" "sync/atomic" ) const ipQueueDefaultMaxRecycleSize = 4 * 1024 // This is a generic intra-process queue. type ipQueue[T any] struct { inprogress int64 sync.Mutex ch chan struct{} elts []T pos int pool *sync.Pool mrs int name string m *sync.Map } type ipQueueOpts struct { maxRecycleSize int } type ipQueueOpt func(*ipQueueOpts) // This option allows to set the maximum recycle size when attempting // to put back a slice to the pool. func ipQueue_MaxRecycleSize(max int) ipQueueOpt { return func(o *ipQueueOpts) { o.maxRecycleSize = max } } func newIPQueue[T any](s *Server, name string, opts ...ipQueueOpt) *ipQueue[T] { qo := ipQueueOpts{maxRecycleSize: ipQueueDefaultMaxRecycleSize} for _, o := range opts { o(&qo) } q := &ipQueue[T]{ ch: make(chan struct{}, 1), mrs: qo.maxRecycleSize, pool: &sync.Pool{}, name: name, m: &s.ipQueues, } s.ipQueues.Store(name, q) return q } // Add the element `e` to the queue, notifying the queue channel's `ch` if the // entry is the first to be added, and returns the length of the queue after // this element is added. func (q *ipQueue[T]) push(e T) int { var signal bool q.Lock() l := len(q.elts) - q.pos if l == 0 { signal = true eltsi := q.pool.Get() if eltsi != nil { // Reason we use pointer to slice instead of slice is explained // here: https://staticcheck.io/docs/checks#SA6002 q.elts = (*(eltsi.(*[]T)))[:0] } if cap(q.elts) == 0 { q.elts = make([]T, 0, 32) } } q.elts = append(q.elts, e) l++ q.Unlock() if signal { select { case q.ch <- struct{}{}: default: } } return l } // Returns the whole list of elements currently present in the queue, // emptying the queue. This should be called after receiving a notification // from the queue's `ch` notification channel that indicates that there // is something in the queue. // However, in cases where `drain()` may be called from another go // routine, it is possible that a routine is notified that there is // something, but by the time it calls `pop()`, the drain() would have // emptied the queue. So the caller should never assume that pop() will // return a slice of 1 or more, it could return `nil`. func (q *ipQueue[T]) pop() []T { if q == nil { return nil } var elts []T q.Lock() if q.pos == 0 { elts = q.elts } else { elts = q.elts[q.pos:] } q.elts, q.pos = nil, 0 atomic.AddInt64(&q.inprogress, int64(len(elts))) q.Unlock() return elts } func (q *ipQueue[T]) resetAndReturnToPool(elts *[]T) { (*elts) = (*elts)[:0] q.pool.Put(elts) } // Returns the first element from the queue, if any. See comment above // regarding calling after being notified that there is something and // the use of drain(). In short, the caller should always check the // boolean return value to ensure that the value is genuine and not a // default empty value. func (q *ipQueue[T]) popOne() (T, bool) { q.Lock() l := len(q.elts) - q.pos if l < 1 { q.Unlock() var empty T return empty, false } e := q.elts[q.pos] q.pos++ l-- if l > 0 { // We need to re-signal select { case q.ch <- struct{}{}: default: } } else { // We have just emptied the queue, so we can recycle now. q.resetAndReturnToPool(&q.elts) q.elts, q.pos = nil, 0 } q.Unlock() return e, true } // After a pop(), the slice can be recycled for the next push() when // a first element is added to the queue. // This will also decrement the "in progress" count with the length // of the slice. // Reason we use pointer to slice instead of slice is explained // here: https://staticcheck.io/docs/checks#SA6002 func (q *ipQueue[T]) recycle(elts *[]T) { // If invoked with a nil list, nothing to do. if elts == nil || *elts == nil { return } // Update the in progress count. if len(*elts) > 0 { if atomic.AddInt64(&q.inprogress, int64(-(len(*elts)))) < 0 { atomic.StoreInt64(&q.inprogress, 0) } } // We also don't want to recycle huge slices, so check against the max. // q.mrs is normally immutable but can be changed, in a safe way, in some tests. if cap(*elts) > q.mrs { return } q.resetAndReturnToPool(elts) } // Returns the current length of the queue. func (q *ipQueue[T]) len() int { q.Lock() l := len(q.elts) - q.pos q.Unlock() return l } // Empty the queue and consumes the notification signal if present. // Returns the number of items that were drained from the queue. // Note that this could cause a reader go routine that has been // notified that there is something in the queue (reading from queue's `ch`) // may then get nothing if `drain()` is invoked before the `pop()` or `popOne()`. func (q *ipQueue[T]) drain() int { if q == nil { return 0 } q.Lock() olen := len(q.elts) if q.elts != nil { q.resetAndReturnToPool(&q.elts) q.elts, q.pos = nil, 0 } // Consume the signal if it was present to reduce the chance of a reader // routine to be think that there is something in the queue... select { case <-q.ch: default: } q.Unlock() return olen } // Since the length of the queue goes to 0 after a pop(), it is good to // have an insight on how many elements are yet to be processed after a pop(). // For that reason, the queue maintains a count of elements returned through // the pop() API. When the caller will call q.recycle(), this count will // be reduced by the size of the slice returned by pop(). func (q *ipQueue[T]) inProgress() int64 { return atomic.LoadInt64(&q.inprogress) } // Remove this queue from the server's map of ipQueues. // All ipQueue operations (such as push/pop/etc..) are still possible. func (q *ipQueue[T]) unregister() { if q == nil { return } q.m.Delete(q.name) } nats-server-2.10.27/server/ipqueue_test.go000066400000000000000000000220751477524627100205320ustar00rootroot00000000000000// Copyright 2021-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "sync" "testing" "time" ) func TestIPQueueBasic(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") // Check that pool has been created if q.pool == nil { t.Fatal("Expected pool to have been created") } // Check for the default mrs if q.mrs != ipQueueDefaultMaxRecycleSize { t.Fatalf("Expected default max recycle size to be %v, got %v", ipQueueDefaultMaxRecycleSize, q.mrs) } select { case <-q.ch: t.Fatalf("Should not have been notified") default: // OK! } if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } // Try to change the max recycle size q2 := newIPQueue[int](s, "test2", ipQueue_MaxRecycleSize(10)) if q2.mrs != 10 { t.Fatalf("Expected max recycle size to be 10, got %v", q2.mrs) } // Check that those 2 queues are registered var gotFirst bool var gotSecond bool s.ipQueues.Range(func(k, v any) bool { switch k.(string) { case "test": gotFirst = true case "test2": gotSecond = true default: t.Fatalf("Unknown queue: %q", k.(string)) } return true }) if !gotFirst { t.Fatalf("Did not find queue %q", "test") } if !gotSecond { t.Fatalf("Did not find queue %q", "test2") } // Unregister them q.unregister() q2.unregister() // They should have been removed from the map s.ipQueues.Range(func(k, v any) bool { t.Fatalf("Got queue %q", k.(string)) return false }) // But verify that we can still push/pop q.push(1) elts := q.pop() if len(elts) != 1 { t.Fatalf("Should have gotten 1 element, got %v", len(elts)) } q2.push(2) if e, ok := q2.popOne(); !ok || e != 2 { t.Fatalf("popOne failed: %+v", e) } } func TestIPQueuePush(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") q.push(1) if l := q.len(); l != 1 { t.Fatalf("Expected len to be 1, got %v", l) } select { case <-q.ch: // OK default: t.Fatalf("Should have been notified of addition") } // Push a new element, we should not be notified. q.push(2) if l := q.len(); l != 2 { t.Fatalf("Expected len to be 2, got %v", l) } select { case <-q.ch: t.Fatalf("Should not have been notified of addition") default: // OK } } func TestIPQueuePop(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") q.push(1) <-q.ch elts := q.pop() if l := len(elts); l != 1 { t.Fatalf("Expected 1 elt, got %v", l) } if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } // The channel notification should be empty select { case <-q.ch: t.Fatalf("Should not have been notified of addition") default: // OK } // Since pop() brings the number of pending to 0, we keep track of the // number of "in progress" elements. Check that the value is 1 here. if n := q.inProgress(); n != 1 { t.Fatalf("Expected count to be 1, got %v", n) } // Recycling will bring it down to 0. q.recycle(&elts) if n := q.inProgress(); n != 0 { t.Fatalf("Expected count to be 0, got %v", n) } // If we call pop() now, we should get an empty list. if elts = q.pop(); elts != nil { t.Fatalf("Expected nil, got %v", elts) } // The in progress count should still be 0 if n := q.inProgress(); n != 0 { t.Fatalf("Expected count to be 0, got %v", n) } } func TestIPQueuePopOne(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") q.push(1) <-q.ch e, ok := q.popOne() if !ok { t.Fatal("Got nil") } if i := e; i != 1 { t.Fatalf("Expected 1, got %v", i) } if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } // That does not affect the number of notProcessed if n := q.inProgress(); n != 0 { t.Fatalf("Expected count to be 0, got %v", n) } select { case <-q.ch: t.Fatalf("Should not have been notified of addition") default: // OK } q.push(2) q.push(3) e, ok = q.popOne() if !ok { t.Fatal("Got nil") } if i := e; i != 2 { t.Fatalf("Expected 2, got %v", i) } if l := q.len(); l != 1 { t.Fatalf("Expected len to be 1, got %v", l) } select { case <-q.ch: // OK default: t.Fatalf("Should have been notified that there is more") } e, ok = q.popOne() if !ok { t.Fatal("Got nil") } if i := e; i != 3 { t.Fatalf("Expected 3, got %v", i) } if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } select { case <-q.ch: t.Fatalf("Should not have been notified that there is more") default: // OK } // Calling it again now that we know there is nothing, we // should get nil. if e, ok = q.popOne(); ok { t.Fatalf("Expected nil, got %v", e) } q = newIPQueue[int](s, "test2") q.push(1) q.push(2) // Capture current capacity q.Lock() c := cap(q.elts) q.Unlock() e, ok = q.popOne() if !ok || e != 1 { t.Fatalf("Invalid value: %v", e) } if l := q.len(); l != 1 { t.Fatalf("Expected len to be 1, got %v", l) } values := q.pop() if len(values) != 1 || values[0] != 2 { t.Fatalf("Unexpected values: %v", values) } if cap(values) != c-1 { t.Fatalf("Unexpected capacity: %v vs %v", cap(values), c-1) } if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } // Just make sure that this is ok... q.recycle(&values) } func TestIPQueueMultiProducers(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") wg := sync.WaitGroup{} wg.Add(3) send := func(start, end int) { defer wg.Done() for i := start; i <= end; i++ { q.push(i) } } go send(1, 100) go send(101, 200) go send(201, 300) tm := time.NewTimer(2 * time.Second) m := make(map[int]struct{}) for done := false; !done; { select { case <-q.ch: values := q.pop() for _, v := range values { m[v] = struct{}{} } q.recycle(&values) if n := q.inProgress(); n != 0 { t.Fatalf("Expected count to be 0, got %v", n) } done = len(m) == 300 case <-tm.C: t.Fatalf("Did not receive all elements: %v", m) } } wg.Wait() } func TestIPQueueRecycle(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") total := 1000 for iter := 0; iter < 5; iter++ { var sz int for i := 0; i < total; i++ { sz = q.push(i) } if sz != total { t.Fatalf("Expected size to be %v, got %v", total, sz) } values := q.pop() preRecycleCap := cap(values) q.recycle(&values) sz = q.push(1001) if sz != 1 { t.Fatalf("Expected size to be %v, got %v", 1, sz) } values = q.pop() if l := len(values); l != 1 { t.Fatalf("Len should be 1, got %v", l) } if c := cap(values); c == preRecycleCap { break } else if iter == 4 { // We can't fail the test since there is no guarantee that the slice // is still present in the pool when we do a Get(), but let's log that // recycling did not occur even after all iterations.. t.Logf("Seem like the previous slice was not recycled, old cap=%v new cap=%v", preRecycleCap, c) } } q = newIPQueue[int](s, "test2", ipQueue_MaxRecycleSize(10)) for i := 0; i < 100; i++ { q.push(i) } values := q.pop() preRecycleCap := cap(values) q.recycle(&values) q.push(1001) values = q.pop() if l := len(values); l != 1 { t.Fatalf("Len should be 1, got %v", l) } // This time, we should not have recycled it, so the new cap should // be 1 for the new element added. In case Go creates a slice of // cap more than 1 in some future release, just check that the // cap is lower than the pre recycle cap. if c := cap(values); c >= preRecycleCap { t.Fatalf("The slice should not have been put back in the pool, got cap of %v", c) } // Also check that if we mistakenly pop a queue that was not // notified (pop() will return nil), and we try to recycle, // recycle() will ignore the call. values = q.pop() q.recycle(&values) q.push(1002) q.Lock() recycled := &q.elts == &values q.Unlock() if recycled { t.Fatalf("Unexpected recycled slice") } // Check that we don't crash when recycling a nil or empty slice values = q.pop() q.recycle(&values) q.recycle(nil) } func TestIPQueueDrain(t *testing.T) { s := &Server{} q := newIPQueue[int](s, "test") for iter, recycled := 0, false; iter < 5 && !recycled; iter++ { for i := 0; i < 100; i++ { q.push(i + 1) } q.drain() // Try to get something from the pool right away s := q.pool.Get() recycled := s != nil if !recycled { // We can't fail the test, since we have no guarantee it will be recycled // especially when running with `-race` flag... if iter == 4 { t.Log("nothing was recycled") } } // Check that we have consumed the signal... select { case <-q.ch: t.Fatal("Signal should have been consumed by drain") default: // OK! } // Check len if l := q.len(); l != 0 { t.Fatalf("Expected len to be 0, got %v", l) } if recycled { break } } } nats-server-2.10.27/server/jetstream.go000066400000000000000000002441771477524627100200250ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "crypto/hmac" "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" "fmt" "math" "os" "path/filepath" "runtime/debug" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/minio/highwayhash" "github.com/nats-io/nats-server/v2/server/sysmem" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) // JetStreamConfig determines this server's configuration. // MaxMemory and MaxStore are in bytes. type JetStreamConfig struct { MaxMemory int64 `json:"max_memory"` MaxStore int64 `json:"max_storage"` StoreDir string `json:"store_dir,omitempty"` SyncInterval time.Duration `json:"sync_interval,omitempty"` SyncAlways bool `json:"sync_always,omitempty"` Domain string `json:"domain,omitempty"` CompressOK bool `json:"compress_ok,omitempty"` UniqueTag string `json:"unique_tag,omitempty"` } // Statistics about JetStream for this server. type JetStreamStats struct { Memory uint64 `json:"memory"` Store uint64 `json:"storage"` ReservedMemory uint64 `json:"reserved_memory"` ReservedStore uint64 `json:"reserved_storage"` Accounts int `json:"accounts"` HAAssets int `json:"ha_assets"` API JetStreamAPIStats `json:"api"` } type JetStreamAccountLimits struct { MaxMemory int64 `json:"max_memory"` MaxStore int64 `json:"max_storage"` MaxStreams int `json:"max_streams"` MaxConsumers int `json:"max_consumers"` MaxAckPending int `json:"max_ack_pending"` MemoryMaxStreamBytes int64 `json:"memory_max_stream_bytes"` StoreMaxStreamBytes int64 `json:"storage_max_stream_bytes"` MaxBytesRequired bool `json:"max_bytes_required"` } type JetStreamTier struct { Memory uint64 `json:"memory"` Store uint64 `json:"storage"` ReservedMemory uint64 `json:"reserved_memory"` ReservedStore uint64 `json:"reserved_storage"` Streams int `json:"streams"` Consumers int `json:"consumers"` Limits JetStreamAccountLimits `json:"limits"` } // JetStreamAccountStats returns current statistics about the account's JetStream usage. type JetStreamAccountStats struct { JetStreamTier // in case tiers are used, reflects totals with limits not set Domain string `json:"domain,omitempty"` API JetStreamAPIStats `json:"api"` Tiers map[string]JetStreamTier `json:"tiers,omitempty"` // indexed by tier name } type JetStreamAPIStats struct { Total uint64 `json:"total"` Errors uint64 `json:"errors"` Inflight uint64 `json:"inflight,omitempty"` } // This is for internal accounting for JetStream for this server. type jetStream struct { // These are here first because of atomics on 32bit systems. apiInflight int64 apiTotal int64 apiErrors int64 memReserved int64 storeReserved int64 memUsed int64 storeUsed int64 queueLimit int64 clustered int32 mu sync.RWMutex srv *Server config JetStreamConfig cluster *jetStreamCluster accounts map[string]*jsAccount apiSubs *Sublist started time.Time // System level request to purge a stream move accountPurge *subscription // Some bools regarding general state. metaRecovering bool standAlone bool oos bool shuttingDown bool // Atomic versions disabled atomic.Bool } type remoteUsage struct { tiers map[string]*jsaUsage // indexed by tier name api uint64 err uint64 } type jsaStorage struct { total jsaUsage local jsaUsage } // This represents a jetstream enabled account. // Worth noting that we include the jetstream pointer, this is because // in general we want to be very efficient when receiving messages on // an internal sub for a stream, so we will direct link to the stream // and walk backwards as needed vs multiple hash lookups and locks, etc. type jsAccount struct { mu sync.RWMutex js *jetStream account *Account storeDir string inflight sync.Map streams map[string]*stream templates map[string]*streamTemplate store TemplateStore // From server sendq *ipQueue[*pubMsg] // For limiting only running one checkAndSync at a time. sync atomic.Bool // Usage/limits related fields that will be protected by usageMu usageMu sync.RWMutex limits map[string]JetStreamAccountLimits // indexed by tierName usage map[string]*jsaStorage // indexed by tierName rusage map[string]*remoteUsage // indexed by node id apiTotal uint64 apiErrors uint64 usageApi uint64 usageErr uint64 updatesPub string updatesSub *subscription lupdate time.Time utimer *time.Timer } // Track general usage for this account. type jsaUsage struct { mem int64 store int64 } // EnableJetStream will enable JetStream support on this server with the given configuration. // A nil configuration will dynamically choose the limits and temporary file storage directory. func (s *Server) EnableJetStream(config *JetStreamConfig) error { if s.JetStreamEnabled() { return fmt.Errorf("jetstream already enabled") } s.Noticef("Starting JetStream") if config == nil || config.MaxMemory <= 0 || config.MaxStore <= 0 { var storeDir, domain, uniqueTag string var maxStore, maxMem int64 if config != nil { storeDir, domain, uniqueTag = config.StoreDir, config.Domain, config.UniqueTag maxStore, maxMem = config.MaxStore, config.MaxMemory } config = s.dynJetStreamConfig(storeDir, maxStore, maxMem) if maxMem > 0 { config.MaxMemory = maxMem } if domain != _EMPTY_ { config.Domain = domain } if uniqueTag != _EMPTY_ { config.UniqueTag = uniqueTag } s.Debugf("JetStream creating dynamic configuration - %s memory, %s disk", friendlyBytes(config.MaxMemory), friendlyBytes(config.MaxStore)) } else if config.StoreDir != _EMPTY_ { config.StoreDir = filepath.Join(config.StoreDir, JetStreamStoreDir) } cfg := *config if cfg.StoreDir == _EMPTY_ { cfg.StoreDir = filepath.Join(os.TempDir(), JetStreamStoreDir) s.Warnf("Temporary storage directory used, data could be lost on system reboot") } // We will consistently place the 'jetstream' directory under the storedir that was handed to us. Prior to 2.2.3 though // we could have a directory on disk without the 'jetstream' directory. This will check and fix if needed. if err := s.checkStoreDir(&cfg); err != nil { return err } return s.enableJetStream(cfg) } // Function signature to generate a key encryption key. type keyGen func(context []byte) ([]byte, error) // Return a key generation function or nil if encryption not enabled. // keyGen defined in filestore.go - keyGen func(iv, context []byte) []byte func (s *Server) jsKeyGen(jsKey, info string) keyGen { if ek := jsKey; ek != _EMPTY_ { return func(context []byte) ([]byte, error) { h := hmac.New(sha256.New, []byte(ek)) if _, err := h.Write([]byte(info)); err != nil { return nil, err } if _, err := h.Write(context); err != nil { return nil, err } return h.Sum(nil), nil } } return nil } // Decode the encrypted metafile. func (s *Server) decryptMeta(sc StoreCipher, ekey, buf []byte, acc, context string) ([]byte, bool, error) { if len(ekey) < minMetaKeySize { return nil, false, errBadKeySize } var osc StoreCipher switch sc { case AES: osc = ChaCha case ChaCha: osc = AES } type prfWithCipher struct { keyGen StoreCipher } var prfs []prfWithCipher if prf := s.jsKeyGen(s.getOpts().JetStreamKey, acc); prf == nil { return nil, false, errNoEncryption } else { // First of all, try our current encryption keys with both // store cipher algorithms. prfs = append(prfs, prfWithCipher{prf, sc}) prfs = append(prfs, prfWithCipher{prf, osc}) } if prf := s.jsKeyGen(s.getOpts().JetStreamOldKey, acc); prf != nil { // Then, if we have an old encryption key, try with also with // both store cipher algorithms. prfs = append(prfs, prfWithCipher{prf, sc}) prfs = append(prfs, prfWithCipher{prf, osc}) } for i, prf := range prfs { rb, err := prf.keyGen([]byte(context)) if err != nil { continue } kek, err := genEncryptionKey(prf.StoreCipher, rb) if err != nil { continue } ns := kek.NonceSize() seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil) if err != nil { continue } aek, err := genEncryptionKey(prf.StoreCipher, seed) if err != nil { continue } if aek.NonceSize() != kek.NonceSize() { continue } plain, err := aek.Open(nil, buf[:ns], buf[ns:], nil) if err != nil { continue } return plain, i > 0, nil } return nil, false, fmt.Errorf("unable to recover keys") } // Check to make sure directory has the jetstream directory. // We will have it properly configured here now regardless, so need to look inside. func (s *Server) checkStoreDir(cfg *JetStreamConfig) error { fis, _ := os.ReadDir(cfg.StoreDir) // If we have nothing underneath us, could be just starting new, but if we see this we can check. if len(fis) != 0 { return nil } // Let's check the directory above. If it has us 'jetstream' but also other stuff that we can // identify as accounts then we can fix. fis, _ = os.ReadDir(filepath.Dir(cfg.StoreDir)) // If just one that is us 'jetstream' and all is ok. if len(fis) == 1 { return nil } haveJetstreamDir := false for _, fi := range fis { if fi.Name() == JetStreamStoreDir { haveJetstreamDir = true break } } for _, fi := range fis { // Skip the 'jetstream' directory. if fi.Name() == JetStreamStoreDir { continue } // Let's see if this is an account. if accName := fi.Name(); accName != _EMPTY_ { _, ok := s.accounts.Load(accName) if !ok && s.AccountResolver() != nil && nkeys.IsValidPublicAccountKey(accName) { // Account is not local but matches the NKEY account public key, // this is enough indication to move this directory, no need to // fetch the account. ok = true } // If this seems to be an account go ahead and move the directory. This will include all assets // like streams and consumers. if ok { if !haveJetstreamDir { err := os.Mkdir(filepath.Join(filepath.Dir(cfg.StoreDir), JetStreamStoreDir), defaultDirPerms) if err != nil { return err } haveJetstreamDir = true } old := filepath.Join(filepath.Dir(cfg.StoreDir), fi.Name()) new := filepath.Join(cfg.StoreDir, fi.Name()) s.Noticef("JetStream relocated account %q to %q", old, new) if err := os.Rename(old, new); err != nil { return err } } } } return nil } // enableJetStream will start up the JetStream subsystem. func (s *Server) enableJetStream(cfg JetStreamConfig) error { js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache()} s.gcbMu.Lock() if s.gcbOutMax = s.getOpts().JetStreamMaxCatchup; s.gcbOutMax == 0 { s.gcbOutMax = defaultMaxTotalCatchupOutBytes } s.gcbMu.Unlock() // TODO: Not currently reloadable. atomic.StoreInt64(&js.queueLimit, s.getOpts().JetStreamRequestQueueLimit) s.js.Store(js) // FIXME(dlc) - Allow memory only operation? if stat, err := os.Stat(cfg.StoreDir); os.IsNotExist(err) { if err := os.MkdirAll(cfg.StoreDir, defaultDirPerms); err != nil { return fmt.Errorf("could not create storage directory - %v", err) } } else { // Make sure its a directory and that we can write to it. if stat == nil || !stat.IsDir() { return fmt.Errorf("storage directory is not a directory") } tmpfile, err := os.CreateTemp(cfg.StoreDir, "_test_") if err != nil { return fmt.Errorf("storage directory is not writable") } tmpfile.Close() os.Remove(tmpfile.Name()) } // JetStream is an internal service so we need to make sure we have a system account. // This system account will export the JetStream service endpoints. if s.SystemAccount() == nil { s.SetDefaultSystemAccount() } opts := s.getOpts() if !opts.DisableJetStreamBanner { s.Noticef(" _ ___ _____ ___ _____ ___ ___ _ __ __") s.Noticef(" _ | | __|_ _/ __|_ _| _ \\ __| /_\\ | \\/ |") s.Noticef("| || | _| | | \\__ \\ | | | / _| / _ \\| |\\/| |") s.Noticef(" \\__/|___| |_| |___/ |_| |_|_\\___/_/ \\_\\_| |_|") s.Noticef("") s.Noticef(" https://docs.nats.io/jetstream") s.Noticef("") } s.Noticef("---------------- JETSTREAM ----------------") s.Noticef(" Max Memory: %s", friendlyBytes(cfg.MaxMemory)) s.Noticef(" Max Storage: %s", friendlyBytes(cfg.MaxStore)) s.Noticef(" Store Directory: \"%s\"", cfg.StoreDir) if cfg.Domain != _EMPTY_ { s.Noticef(" Domain: %s", cfg.Domain) } if ek := opts.JetStreamKey; ek != _EMPTY_ { s.Noticef(" Encryption: %s", opts.JetStreamCipher) } s.Noticef("-------------------------------------------") // Setup our internal subscriptions. if err := s.setJetStreamExportSubs(); err != nil { return fmt.Errorf("setting up internal jetstream subscriptions failed: %v", err) } // Setup our internal system exports. s.Debugf(" Exports:") s.Debugf(" %s", jsAllAPI) s.setupJetStreamExports() standAlone, canExtend := s.standAloneMode(), s.canExtendOtherDomain() if standAlone && canExtend && s.getOpts().JetStreamExtHint != jsWillExtend { canExtend = false s.Noticef("Standalone server started in clustered mode do not support extending domains") s.Noticef(`Manually disable standalone mode by setting the JetStream Option "extension_hint: %s"`, jsWillExtend) } // Indicate if we will be standalone for checking resource reservations, etc. js.setJetStreamStandAlone(standAlone && !canExtend) // Enable accounts and restore state before starting clustering. if err := s.enableJetStreamAccounts(); err != nil { return err } // If we are in clustered mode go ahead and start the meta controller. if !standAlone || canExtend { if err := s.enableJetStreamClustering(); err != nil { return err } // Set our atomic bool to clustered. s.jsClustered.Store(true) } // Mark when we are up and running. js.setStarted() return nil } const jsNoExtend = "no_extend" const jsWillExtend = "will_extend" // This will check if we have a solicited leafnode that shares the system account // and extension is not manually disabled func (s *Server) canExtendOtherDomain() bool { opts := s.getOpts() sysAcc := s.SystemAccount().GetName() for _, r := range opts.LeafNode.Remotes { if r.LocalAccount == sysAcc { for _, denySub := range r.DenyImports { if subjectIsSubsetMatch(denySub, raftAllSubj) { return false } } return true } } return false } func (s *Server) updateJetStreamInfoStatus(enabled bool) { s.mu.Lock() s.info.JetStream = enabled s.mu.Unlock() } // restartJetStream will try to re-enable JetStream during a reload if it had been disabled during runtime. func (s *Server) restartJetStream() error { opts := s.getOpts() cfg := JetStreamConfig{ StoreDir: opts.StoreDir, SyncInterval: opts.SyncInterval, SyncAlways: opts.SyncAlways, MaxMemory: opts.JetStreamMaxMemory, MaxStore: opts.JetStreamMaxStore, Domain: opts.JetStreamDomain, } s.Noticef("Restarting JetStream") err := s.EnableJetStream(&cfg) if err != nil { s.Warnf("Can't start JetStream: %v", err) return s.DisableJetStream() } s.updateJetStreamInfoStatus(true) return nil } // checkJetStreamExports will check if we have the JS exports setup // on the system account, and if not go ahead and set them up. func (s *Server) checkJetStreamExports() { if sacc := s.SystemAccount(); sacc != nil { sacc.mu.RLock() se := sacc.getServiceExport(jsAllAPI) sacc.mu.RUnlock() if se == nil { s.setupJetStreamExports() } } } func (s *Server) setupJetStreamExports() { // Setup our internal system export. if err := s.SystemAccount().AddServiceExport(jsAllAPI, nil); err != nil { s.Warnf("Error setting up jetstream service exports: %v", err) } } func (s *Server) jetStreamOOSPending() (wasPending bool) { if js := s.getJetStream(); js != nil { js.mu.Lock() wasPending = js.oos js.oos = true js.mu.Unlock() } return wasPending } func (s *Server) setJetStreamDisabled() { if js := s.getJetStream(); js != nil { js.disabled.Store(true) } } func (s *Server) handleOutOfSpace(mset *stream) { if s.JetStreamEnabled() && !s.jetStreamOOSPending() { var stream string if mset != nil { stream = mset.name() s.Errorf("JetStream out of %s resources, will be DISABLED", mset.Store().Type()) } else { s.Errorf("JetStream out of resources, will be DISABLED") } go s.DisableJetStream() adv := &JSServerOutOfSpaceAdvisory{ TypedEvent: TypedEvent{ Type: JSServerOutOfStorageAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Server: s.Name(), ServerID: s.ID(), Stream: stream, Cluster: s.cachedClusterName(), Domain: s.getOpts().JetStreamDomain, } s.publishAdvisory(nil, JSAdvisoryServerOutOfStorage, adv) } } // DisableJetStream will turn off JetStream and signals in clustered mode // to have the metacontroller remove us from the peer list. func (s *Server) DisableJetStream() error { if !s.JetStreamEnabled() { return nil } s.setJetStreamDisabled() if s.JetStreamIsClustered() { isLeader := s.JetStreamIsLeader() js, cc := s.getJetStreamCluster() if js == nil { s.shutdownJetStream() return nil } js.mu.RLock() meta := cc.meta js.mu.RUnlock() if meta != nil { if isLeader { s.Warnf("JetStream initiating meta leader transfer") meta.StepDown() select { case <-s.quitCh: return nil case <-time.After(2 * time.Second): } if !s.JetStreamIsCurrent() { s.Warnf("JetStream timeout waiting for meta leader transfer") } } meta.Delete() } } // Update our info status. s.updateJetStreamInfoStatus(false) // Normal shutdown. s.shutdownJetStream() // Shut down the RAFT groups. s.shutdownRaftNodes() return nil } func (s *Server) enableJetStreamAccounts() error { // If we have no configured accounts setup then setup imports on global account. if s.globalAccountOnly() { gacc := s.GlobalAccount() gacc.mu.Lock() if len(gacc.jsLimits) == 0 { gacc.jsLimits = defaultJSAccountTiers } gacc.mu.Unlock() if err := s.configJetStream(gacc); err != nil { return err } } else if err := s.configAllJetStreamAccounts(); err != nil { return fmt.Errorf("Error enabling jetstream on configured accounts: %v", err) } return nil } // enableAllJetStreamServiceImportsAndMappings turns on all service imports and mappings for jetstream for this account. func (a *Account) enableAllJetStreamServiceImportsAndMappings() error { a.mu.RLock() s := a.srv a.mu.RUnlock() if s == nil { return fmt.Errorf("jetstream account not registered") } if !a.serviceImportExists(jsAllAPI) { // Capture si so we can turn on implicit sharing with JetStream layer. // Make sure to set "to" otherwise will incur performance slow down. si, err := a.addServiceImport(s.SystemAccount(), jsAllAPI, jsAllAPI, nil) if err != nil { return fmt.Errorf("Error setting up jetstream service imports for account: %v", err) } a.mu.Lock() si.share = true a.mu.Unlock() } // Check if we have a Domain specified. // If so add in a subject mapping that will allow local connected clients to reach us here as well. if opts := s.getOpts(); opts.JetStreamDomain != _EMPTY_ { mappings := generateJSMappingTable(opts.JetStreamDomain) a.mu.RLock() for _, m := range a.mappings { delete(mappings, m.src) } a.mu.RUnlock() for src, dest := range mappings { if err := a.AddMapping(src, dest); err != nil { s.Errorf("Error adding JetStream domain mapping: %v", err) } } } return nil } // enableJetStreamInfoServiceImportOnly will enable the single service import responder. // Should we do them all regardless? func (a *Account) enableJetStreamInfoServiceImportOnly() error { // Check if this import would be overshadowed. This can happen when accounts // are importing from another account for JS access. if a.serviceImportShadowed(JSApiAccountInfo) { return nil } return a.enableAllJetStreamServiceImportsAndMappings() } func (s *Server) configJetStream(acc *Account) error { if acc == nil { return nil } acc.mu.RLock() jsLimits := acc.jsLimits acc.mu.RUnlock() if jsLimits != nil { // Check if already enabled. This can be during a reload. if acc.JetStreamEnabled() { if err := acc.enableAllJetStreamServiceImportsAndMappings(); err != nil { return err } if err := acc.UpdateJetStreamLimits(jsLimits); err != nil { return err } } else { if err := acc.EnableJetStream(jsLimits); err != nil { return err } if s.gateway.enabled { s.switchAccountToInterestMode(acc.GetName()) } } } else if acc != s.SystemAccount() { if acc.JetStreamEnabled() { acc.DisableJetStream() } // We will setup basic service imports to respond to // requests if JS is enabled for this account. if err := acc.enableJetStreamInfoServiceImportOnly(); err != nil { return err } } return nil } // configAllJetStreamAccounts walk all configured accounts and turn on jetstream if requested. func (s *Server) configAllJetStreamAccounts() error { // Check to see if system account has been enabled. We could arrive here via reload and // a non-default system account. s.checkJetStreamExports() // Bail if server not enabled. If it was enabled and a reload turns it off // that will be handled elsewhere. js := s.getJetStream() if js == nil { return nil } // Snapshot into our own list. Might not be needed. s.mu.RLock() if s.sys != nil { // clustered stream removal will perform this cleanup as well // this is mainly for initial cleanup saccName := s.sys.account.Name accStoreDirs, _ := os.ReadDir(js.config.StoreDir) for _, acc := range accStoreDirs { if accName := acc.Name(); accName != saccName { // no op if not empty accDir := filepath.Join(js.config.StoreDir, accName) os.Remove(filepath.Join(accDir, streamsDir)) os.Remove(accDir) } } } var jsAccounts []*Account s.accounts.Range(func(k, v any) bool { jsAccounts = append(jsAccounts, v.(*Account)) return true }) accounts := &s.accounts s.mu.RUnlock() // Process any jetstream enabled accounts here. These will be accounts we are // already aware of at startup etc. for _, acc := range jsAccounts { if err := s.configJetStream(acc); err != nil { return err } } // Now walk all the storage we have and resolve any accounts that we did not process already. // This is important in resolver/operator models. fis, _ := os.ReadDir(js.config.StoreDir) for _, fi := range fis { if accName := fi.Name(); accName != _EMPTY_ { // Only load up ones not already loaded since they are processed above. if _, ok := accounts.Load(accName); !ok { if acc, err := s.lookupAccount(accName); err != nil && acc != nil { if err := s.configJetStream(acc); err != nil { return err } } } } } return nil } // Mark our started time. func (js *jetStream) setStarted() { js.mu.Lock() defer js.mu.Unlock() js.started = time.Now() } func (js *jetStream) isEnabled() bool { if js == nil { return false } return !js.disabled.Load() } // Mark that we will be in standlone mode. func (js *jetStream) setJetStreamStandAlone(isStandAlone bool) { if js == nil { return } js.mu.Lock() defer js.mu.Unlock() if js.standAlone = isStandAlone; js.standAlone { // Update our server atomic. js.srv.isMetaLeader.Store(true) js.accountPurge, _ = js.srv.systemSubscribe(JSApiAccountPurge, _EMPTY_, false, nil, js.srv.jsLeaderAccountPurgeRequest) } else if js.accountPurge != nil { js.srv.sysUnsubscribe(js.accountPurge) } } // JetStreamEnabled reports if jetstream is enabled for this server. func (s *Server) JetStreamEnabled() bool { return s.getJetStream().isEnabled() } // JetStreamEnabledForDomain will report if any servers have JetStream enabled within this domain. func (s *Server) JetStreamEnabledForDomain() bool { if s.JetStreamEnabled() { return true } var jsFound bool // If we are here we do not have JetStream enabled for ourselves, but we need to check all connected servers. // TODO(dlc) - Could optimize and memoize this. s.nodeToInfo.Range(func(k, v any) bool { // This should not be dependent on online status, so only check js. if v.(nodeInfo).js { jsFound = true return false } return true }) return jsFound } // Will signal that all pull requests for consumers on this server are now invalid. func (s *Server) signalPullConsumers() { js := s.getJetStream() if js == nil { return } js.mu.RLock() defer js.mu.RUnlock() // In case we have stale pending requests. const hdr = "NATS/1.0 409 Server Shutdown\r\n" + JSPullRequestPendingMsgs + ": %d\r\n" + JSPullRequestPendingBytes + ": %d\r\n\r\n" var didSend bool for _, jsa := range js.accounts { jsa.mu.RLock() for _, stream := range jsa.streams { stream.mu.RLock() for _, o := range stream.consumers { o.mu.RLock() // Only signal on R1. if o.cfg.Replicas <= 1 { for reply, wr := range o.pendingRequests() { shdr := fmt.Sprintf(hdr, wr.n, wr.b) o.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, []byte(shdr), nil, nil, 0)) didSend = true } } o.mu.RUnlock() } stream.mu.RUnlock() } jsa.mu.RUnlock() } // Give time for migration information to make it out of our server. if didSend { time.Sleep(50 * time.Millisecond) } } // Helper for determining if we are shutting down. func (js *jetStream) isShuttingDown() bool { js.mu.RLock() defer js.mu.RUnlock() return js.shuttingDown } // Shutdown jetstream for this server. func (s *Server) shutdownJetStream() { js := s.getJetStream() if js == nil { return } s.Noticef("Initiating JetStream Shutdown...") defer s.Noticef("JetStream Shutdown") // If we have folks blocked on sync requests, unblock. // Send 1 is enough, but use select in case they were all present. select { case s.syncOutSem <- struct{}{}: default: } var _a [512]*Account accounts := _a[:0] js.mu.Lock() // Collect accounts. for _, jsa := range js.accounts { if a := jsa.acc(); a != nil { accounts = append(accounts, a) } } accPurgeSub := js.accountPurge js.accountPurge = nil // Signal we are shutting down. js.shuttingDown = true js.mu.Unlock() if accPurgeSub != nil { s.sysUnsubscribe(accPurgeSub) } for _, a := range accounts { a.removeJetStream() } s.js.Store(nil) js.mu.Lock() js.accounts = nil var qch chan struct{} if cc := js.cluster; cc != nil { if cc.qch != nil { qch = cc.qch cc.qch = nil } js.stopUpdatesSub() if cc.c != nil { cc.c.closeConnection(ClientClosed) cc.c = nil } cc.meta = nil // Set our atomic bool to false. s.jsClustered.Store(false) } js.mu.Unlock() // If we were clustered signal the monitor cluster go routine. // We will wait for a bit for it to close. // Do this without the lock. if qch != nil { select { case qch <- struct{}{}: select { case <-qch: case <-time.After(2 * time.Second): s.Warnf("Did not receive signal for successful shutdown of cluster routine") } default: } } } // JetStreamConfig will return the current config. Useful if the system // created a dynamic configuration. A copy is returned. func (s *Server) JetStreamConfig() *JetStreamConfig { var c *JetStreamConfig if js := s.getJetStream(); js != nil { copy := js.config c = &(copy) } return c } // StoreDir returns the current JetStream directory. func (s *Server) StoreDir() string { js := s.getJetStream() if js == nil { return _EMPTY_ } return js.config.StoreDir } // JetStreamNumAccounts returns the number of enabled accounts this server is tracking. func (s *Server) JetStreamNumAccounts() int { js := s.getJetStream() if js == nil { return 0 } js.mu.Lock() defer js.mu.Unlock() return len(js.accounts) } // JetStreamReservedResources returns the reserved resources if JetStream is enabled. func (s *Server) JetStreamReservedResources() (int64, int64, error) { js := s.getJetStream() if js == nil { return -1, -1, NewJSNotEnabledForAccountError() } js.mu.RLock() defer js.mu.RUnlock() return js.memReserved, js.storeReserved, nil } func (s *Server) getJetStream() *jetStream { return s.js.Load() } func (a *Account) assignJetStreamLimits(limits map[string]JetStreamAccountLimits) { a.mu.Lock() a.jsLimits = limits a.mu.Unlock() } // EnableJetStream will enable JetStream on this account with the defined limits. // This is a helper for JetStreamEnableAccount. func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits) error { a.mu.RLock() s := a.srv a.mu.RUnlock() if s == nil { return fmt.Errorf("jetstream account not registered") } if s.SystemAccount() == a { return fmt.Errorf("jetstream can not be enabled on the system account") } s.mu.RLock() if s.sys == nil { s.mu.RUnlock() return ErrServerNotRunning } sendq := s.sys.sendq s.mu.RUnlock() // No limits means we dynamically set up limits. // We also place limits here so we know that the account is configured for JetStream. if len(limits) == 0 { limits = defaultJSAccountTiers } a.assignJetStreamLimits(limits) js := s.getJetStream() if js == nil { return NewJSNotEnabledError() } js.mu.Lock() if jsa, ok := js.accounts[a.Name]; ok { a.mu.Lock() a.js = jsa a.mu.Unlock() js.mu.Unlock() return a.enableAllJetStreamServiceImportsAndMappings() } // Check the limits against existing reservations. if err := js.sufficientResources(limits); err != nil { js.mu.Unlock() return err } sysNode := s.Node() jsa := &jsAccount{js: js, account: a, limits: limits, streams: make(map[string]*stream), sendq: sendq, usage: make(map[string]*jsaStorage)} jsa.storeDir = filepath.Join(js.config.StoreDir, a.Name) // A single server does not need to do the account updates at this point. if js.cluster != nil || !s.standAloneMode() { jsa.usageMu.Lock() jsa.utimer = time.AfterFunc(usageTick, jsa.sendClusterUsageUpdateTimer) // Cluster mode updates to resource usage. System internal prevents echos. jsa.updatesPub = fmt.Sprintf(jsaUpdatesPubT, a.Name, sysNode) jsa.updatesSub, _ = s.sysSubscribe(fmt.Sprintf(jsaUpdatesSubT, a.Name), jsa.remoteUpdateUsage) jsa.usageMu.Unlock() } js.accounts[a.Name] = jsa // Stamp inside account as well. Needs to be done under js's lock. a.mu.Lock() a.js = jsa a.mu.Unlock() js.mu.Unlock() // Create the proper imports here. if err := a.enableAllJetStreamServiceImportsAndMappings(); err != nil { return err } s.Debugf("Enabled JetStream for account %q", a.Name) if l, ok := limits[_EMPTY_]; ok { s.Debugf(" Max Memory: %s", friendlyBytes(l.MaxMemory)) s.Debugf(" Max Storage: %s", friendlyBytes(l.MaxStore)) } else { for t, l := range limits { s.Debugf(" Tier: %s", t) s.Debugf(" Max Memory: %s", friendlyBytes(l.MaxMemory)) s.Debugf(" Max Storage: %s", friendlyBytes(l.MaxStore)) } } // Clean up any old snapshots that were orphaned while staging. os.RemoveAll(filepath.Join(js.config.StoreDir, snapStagingDir)) sdir := filepath.Join(jsa.storeDir, streamsDir) if _, err := os.Stat(sdir); os.IsNotExist(err) { if err := os.MkdirAll(sdir, defaultDirPerms); err != nil { return fmt.Errorf("could not create storage streams directory - %v", err) } // Just need to make sure we can write to the directory. // Remove the directory will create later if needed. os.RemoveAll(sdir) // when empty remove parent directory, which may have been created as well os.Remove(jsa.storeDir) } else { // Restore any state here. s.Debugf("Recovering JetStream state for account %q", a.Name) } // Check templates first since messsage sets will need proper ownership. // FIXME(dlc) - Make this consistent. tdir := filepath.Join(jsa.storeDir, tmplsDir) if stat, err := os.Stat(tdir); err == nil && stat.IsDir() { key := sha256.Sum256([]byte("templates")) hh, err := highwayhash.New64(key[:]) if err != nil { return err } fis, _ := os.ReadDir(tdir) for _, fi := range fis { metafile := filepath.Join(tdir, fi.Name(), JetStreamMetaFile) metasum := filepath.Join(tdir, fi.Name(), JetStreamMetaFileSum) buf, err := os.ReadFile(metafile) if err != nil { s.Warnf(" Error reading StreamTemplate metafile %q: %v", metasum, err) continue } if _, err := os.Stat(metasum); os.IsNotExist(err) { s.Warnf(" Missing StreamTemplate checksum for %q", metasum) continue } sum, err := os.ReadFile(metasum) if err != nil { s.Warnf(" Error reading StreamTemplate checksum %q: %v", metasum, err) continue } hh.Reset() hh.Write(buf) checksum := hex.EncodeToString(hh.Sum(nil)) if checksum != string(sum) { s.Warnf(" StreamTemplate checksums do not match %q vs %q", sum, checksum) continue } var cfg StreamTemplateConfig if err := json.Unmarshal(buf, &cfg); err != nil { s.Warnf(" Error unmarshalling StreamTemplate metafile: %v", err) continue } cfg.Config.Name = _EMPTY_ if _, err := a.addStreamTemplate(&cfg); err != nil { s.Warnf(" Error recreating StreamTemplate %q: %v", cfg.Name, err) continue } } } // Collect consumers, do after all streams. type ce struct { mset *stream odir string } var consumers []*ce // Collect any interest policy streams to check for // https://github.com/nats-io/nats-server/issues/3612 var ipstreams []*stream // Remember if we should be encrypted and what cipher we think we should use. encrypted := s.getOpts().JetStreamKey != _EMPTY_ plaintext := true sc := s.getOpts().JetStreamCipher // Now recover the streams. fis, _ := os.ReadDir(sdir) for _, fi := range fis { mdir := filepath.Join(sdir, fi.Name()) // Check for partially deleted streams. They are marked with "." prefix. if strings.HasPrefix(fi.Name(), tsep) { go os.RemoveAll(mdir) continue } key := sha256.Sum256([]byte(fi.Name())) hh, err := highwayhash.New64(key[:]) if err != nil { return err } metafile := filepath.Join(mdir, JetStreamMetaFile) metasum := filepath.Join(mdir, JetStreamMetaFileSum) if _, err := os.Stat(metafile); os.IsNotExist(err) { s.Warnf(" Missing stream metafile for %q", metafile) continue } buf, err := os.ReadFile(metafile) if err != nil { s.Warnf(" Error reading metafile %q: %v", metafile, err) continue } if _, err := os.Stat(metasum); os.IsNotExist(err) { s.Warnf(" Missing stream checksum file %q", metasum) continue } sum, err := os.ReadFile(metasum) if err != nil { s.Warnf(" Error reading Stream metafile checksum %q: %v", metasum, err) continue } hh.Write(buf) checksum := hex.EncodeToString(hh.Sum(nil)) if checksum != string(sum) { s.Warnf(" Stream metafile %q: checksums do not match %q vs %q", metafile, sum, checksum) continue } // Track if we are converting ciphers. var convertingCiphers bool // Check if we are encrypted. keyFile := filepath.Join(mdir, JetStreamMetaFileKey) keyBuf, err := os.ReadFile(keyFile) if err == nil { s.Debugf(" Stream metafile is encrypted, reading encrypted keyfile") if len(keyBuf) < minMetaKeySize { s.Warnf(" Bad stream encryption key length of %d", len(keyBuf)) continue } // Decode the buffer before proceeding. var nbuf []byte nbuf, convertingCiphers, err = s.decryptMeta(sc, keyBuf, buf, a.Name, fi.Name()) if err != nil { s.Warnf(" Error decrypting our stream metafile: %v", err) continue } buf = nbuf plaintext = false } var cfg FileStreamInfo if err := json.Unmarshal(buf, &cfg); err != nil { s.Warnf(" Error unmarshalling stream metafile %q: %v", metafile, err) continue } if cfg.Template != _EMPTY_ { if err := jsa.addStreamNameToTemplate(cfg.Template, cfg.Name); err != nil { s.Warnf(" Error adding stream %q to template %q: %v", cfg.Name, cfg.Template, err) } } // We had a bug that set a default de dupe window on mirror, despite that being not a valid config fixCfgMirrorWithDedupWindow(&cfg.StreamConfig) // We had a bug that could allow subjects in that had prefix or suffix spaces. We check for that here // and will patch them on the fly for now. We will warn about them. var hadSubjErr bool for i, subj := range cfg.StreamConfig.Subjects { if !IsValidSubject(subj) { s.Warnf(" Detected bad subject %q while adding stream %q, will attempt to repair", subj, cfg.Name) if nsubj := strings.TrimSpace(subj); IsValidSubject(nsubj) { s.Warnf(" Bad subject %q repaired to %q", subj, nsubj) cfg.StreamConfig.Subjects[i] = nsubj } else { s.Warnf(" Error recreating stream %q: %v", cfg.Name, "invalid subject") hadSubjErr = true break } } } if hadSubjErr { continue } // The other possible bug is assigning subjects to mirrors, so check for that and patch as well. if cfg.StreamConfig.Mirror != nil && len(cfg.StreamConfig.Subjects) > 0 { s.Warnf(" Detected subjects on a mirrored stream %q, will remove", cfg.Name) cfg.StreamConfig.Subjects = nil } s.Noticef(" Starting restore for stream '%s > %s'", a.Name, cfg.StreamConfig.Name) rt := time.Now() // Log if we are converting from plaintext to encrypted. if encrypted { if plaintext { s.Noticef(" Encrypting stream '%s > %s'", a.Name, cfg.StreamConfig.Name) } else if convertingCiphers { s.Noticef(" Converting to %s for stream '%s > %s'", sc, a.Name, cfg.StreamConfig.Name) // Remove the key file to have system regenerate with the new cipher. os.Remove(keyFile) } } // Add in the stream. mset, err := a.addStream(&cfg.StreamConfig) if err != nil { s.Warnf(" Error recreating stream %q: %v", cfg.Name, err) // If we removed a keyfile from above make sure to put it back. if convertingCiphers { err := os.WriteFile(keyFile, keyBuf, defaultFilePerms) if err != nil { s.Warnf(" Error replacing meta keyfile for stream %q: %v", cfg.Name, err) } } continue } if !cfg.Created.IsZero() { mset.setCreatedTime(cfg.Created) } state := mset.state() s.Noticef(" Restored %s messages for stream '%s > %s' in %v", comma(int64(state.Msgs)), mset.accName(), mset.name(), time.Since(rt).Round(time.Millisecond)) // Collect to check for dangling messages. // TODO(dlc) - Can be removed eventually. if cfg.StreamConfig.Retention == InterestPolicy { ipstreams = append(ipstreams, mset) } // Now do the consumers. odir := filepath.Join(sdir, fi.Name(), consumerDir) consumers = append(consumers, &ce{mset, odir}) } for _, e := range consumers { ofis, _ := os.ReadDir(e.odir) if len(ofis) > 0 { s.Noticef(" Recovering %d consumers for stream - '%s > %s'", len(ofis), e.mset.accName(), e.mset.name()) } for _, ofi := range ofis { metafile := filepath.Join(e.odir, ofi.Name(), JetStreamMetaFile) metasum := filepath.Join(e.odir, ofi.Name(), JetStreamMetaFileSum) if _, err := os.Stat(metafile); os.IsNotExist(err) { s.Warnf(" Missing consumer metafile %q", metafile) continue } buf, err := os.ReadFile(metafile) if err != nil { s.Warnf(" Error reading consumer metafile %q: %v", metafile, err) continue } if _, err := os.Stat(metasum); os.IsNotExist(err) { s.Warnf(" Missing consumer checksum for %q", metasum) continue } // Check if we are encrypted. if key, err := os.ReadFile(filepath.Join(e.odir, ofi.Name(), JetStreamMetaFileKey)); err == nil { s.Debugf(" Consumer metafile is encrypted, reading encrypted keyfile") // Decode the buffer before proceeding. ctxName := e.mset.name() + tsep + ofi.Name() nbuf, _, err := s.decryptMeta(sc, key, buf, a.Name, ctxName) if err != nil { s.Warnf(" Error decrypting our consumer metafile: %v", err) continue } buf = nbuf } var cfg FileConsumerInfo if err := json.Unmarshal(buf, &cfg); err != nil { s.Warnf(" Error unmarshalling consumer metafile %q: %v", metafile, err) continue } isEphemeral := !isDurableConsumer(&cfg.ConsumerConfig) if isEphemeral { // This is an ephermal consumer and this could fail on restart until // the consumer can reconnect. We will create it as a durable and switch it. cfg.ConsumerConfig.Durable = ofi.Name() } obs, err := e.mset.addConsumerWithAssignment(&cfg.ConsumerConfig, _EMPTY_, nil, true, ActionCreateOrUpdate) if err != nil { s.Warnf(" Error adding consumer %q: %v", cfg.Name, err) continue } if isEphemeral { obs.switchToEphemeral() } if !cfg.Created.IsZero() { obs.setCreatedTime(cfg.Created) } if err != nil { s.Warnf(" Error restoring consumer %q state: %v", cfg.Name, err) } } } // Make sure to cleanup any old remaining snapshots. os.RemoveAll(filepath.Join(jsa.storeDir, snapsDir)) // Check interest policy streams for auto cleanup. for _, mset := range ipstreams { mset.checkForOrphanMsgs() mset.checkConsumerReplication() } s.Debugf("JetStream state for account %q recovered", a.Name) return nil } // Return whether we require MaxBytes to be set and if > 0 an upper limit for stream size exists // Both limits are independent of each other. func (a *Account) maxBytesLimits(cfg *StreamConfig) (bool, int64) { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa == nil { return false, 0 } jsa.usageMu.RLock() var replicas int if cfg != nil { replicas = cfg.Replicas } selectedLimits, _, ok := jsa.selectLimits(replicas) jsa.usageMu.RUnlock() if !ok { return false, 0 } maxStreamBytes := int64(0) if cfg.Storage == MemoryStorage { maxStreamBytes = selectedLimits.MemoryMaxStreamBytes } else { maxStreamBytes = selectedLimits.StoreMaxStreamBytes } return selectedLimits.MaxBytesRequired, maxStreamBytes } // NumStreams will return how many streams we have. func (a *Account) numStreams() int { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa == nil { return 0 } jsa.mu.Lock() n := len(jsa.streams) jsa.mu.Unlock() return n } // Streams will return all known streams. func (a *Account) streams() []*stream { return a.filteredStreams(_EMPTY_) } func (a *Account) filteredStreams(filter string) []*stream { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa == nil { return nil } jsa.mu.RLock() defer jsa.mu.RUnlock() var msets []*stream for _, mset := range jsa.streams { if filter != _EMPTY_ { mset.cfgMu.RLock() for _, subj := range mset.cfg.Subjects { if SubjectsCollide(filter, subj) { msets = append(msets, mset) break } } mset.cfgMu.RUnlock() } else { msets = append(msets, mset) } } return msets } // lookupStream will lookup a stream by name. func (a *Account) lookupStream(name string) (*stream, error) { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa == nil { return nil, NewJSNotEnabledForAccountError() } jsa.mu.RLock() defer jsa.mu.RUnlock() mset, ok := jsa.streams[name] if !ok { return nil, NewJSStreamNotFoundError() } return mset, nil } // UpdateJetStreamLimits will update the account limits for a JetStream enabled account. func (a *Account) UpdateJetStreamLimits(limits map[string]JetStreamAccountLimits) error { a.mu.RLock() s, jsa := a.srv, a.js a.mu.RUnlock() if s == nil { return fmt.Errorf("jetstream account not registered") } js := s.getJetStream() if js == nil { return NewJSNotEnabledError() } if jsa == nil { return NewJSNotEnabledForAccountError() } if len(limits) == 0 { limits = defaultJSAccountTiers } // Calculate the delta between what we have and what we want. jsa.usageMu.RLock() dl := diffCheckedLimits(jsa.limits, limits) jsa.usageMu.RUnlock() js.mu.Lock() // Check the limits against existing reservations. if err := js.sufficientResources(dl); err != nil { js.mu.Unlock() return err } js.mu.Unlock() // Update jsa.usageMu.Lock() jsa.limits = limits jsa.usageMu.Unlock() return nil } func diffCheckedLimits(a, b map[string]JetStreamAccountLimits) map[string]JetStreamAccountLimits { diff := map[string]JetStreamAccountLimits{} for t, la := range a { // in a, not in b will return 0 lb := b[t] diff[t] = JetStreamAccountLimits{ MaxMemory: lb.MaxMemory - la.MaxMemory, MaxStore: lb.MaxStore - la.MaxStore, } } for t, lb := range b { if la, ok := a[t]; !ok { // only in b not in a. (in a and b already covered) diff[t] = JetStreamAccountLimits{ MaxMemory: lb.MaxMemory - la.MaxMemory, MaxStore: lb.MaxStore - la.MaxStore, } } } return diff } // Return reserved bytes for memory and store for this account on this server. // Lock should be held. func (jsa *jsAccount) reservedStorage(tier string) (mem, store uint64) { for _, mset := range jsa.streams { cfg := &mset.cfg if tier == _EMPTY_ || tier == tierName(cfg.Replicas) && cfg.MaxBytes > 0 { switch cfg.Storage { case FileStorage: store += uint64(cfg.MaxBytes) case MemoryStorage: mem += uint64(cfg.MaxBytes) } } } return mem, store } // Return reserved bytes for memory and store for this account in clustered mode. // js lock should be held. func reservedStorage(sas map[string]*streamAssignment, tier string) (mem, store uint64) { for _, sa := range sas { cfg := sa.Config if tier == _EMPTY_ || tier == tierName(cfg.Replicas) && cfg.MaxBytes > 0 { switch cfg.Storage { case FileStorage: store += uint64(cfg.MaxBytes) case MemoryStorage: mem += uint64(cfg.MaxBytes) } } } return mem, store } // JetStreamUsage reports on JetStream usage and limits for an account. func (a *Account) JetStreamUsage() JetStreamAccountStats { a.mu.RLock() jsa, aname := a.js, a.Name accJsLimits := a.jsLimits a.mu.RUnlock() var stats JetStreamAccountStats if jsa != nil { js := jsa.js js.mu.RLock() cc := js.cluster singleServer := cc == nil jsa.mu.RLock() jsa.usageMu.RLock() stats.Memory, stats.Store = jsa.storageTotals() stats.Domain = js.config.Domain stats.API = JetStreamAPIStats{ Total: jsa.apiTotal, Errors: jsa.apiErrors, } if singleServer { stats.ReservedMemory, stats.ReservedStore = jsa.reservedStorage(_EMPTY_) } else { stats.ReservedMemory, stats.ReservedStore = reservedStorage(cc.streams[aname], _EMPTY_) } l, defaultTier := jsa.limits[_EMPTY_] if defaultTier { stats.Limits = l } else { skipped := 0 stats.Tiers = make(map[string]JetStreamTier) for t, total := range jsa.usage { if _, ok := jsa.limits[t]; !ok && (*total) == (jsaStorage{}) { // skip tiers not present that don't contain a count // In case this shows an empty stream, that tier will be added when iterating over streams skipped++ } else { tier := JetStreamTier{ Memory: uint64(total.total.mem), Store: uint64(total.total.store), Limits: jsa.limits[t], } if singleServer { tier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t) } else { tier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t) } stats.Tiers[t] = tier } } if len(accJsLimits) != len(jsa.usage)-skipped { // insert unused limits for t, lim := range accJsLimits { if _, ok := stats.Tiers[t]; !ok { tier := JetStreamTier{Limits: lim} if singleServer { tier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t) } else { tier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t) } stats.Tiers[t] = tier } } } } jsa.usageMu.RUnlock() // Clustered if cc := js.cluster; cc != nil { sas := cc.streams[aname] if defaultTier { stats.Streams = len(sas) stats.ReservedMemory, stats.ReservedStore = reservedStorage(sas, _EMPTY_) } for _, sa := range sas { if defaultTier { stats.Consumers += len(sa.consumers) } else { stats.Streams++ streamTier := tierName(sa.Config.Replicas) su, ok := stats.Tiers[streamTier] if !ok { su = JetStreamTier{} } su.Streams++ stats.Tiers[streamTier] = su // Now consumers, check each since could be different tiers. for _, ca := range sa.consumers { stats.Consumers++ consumerTier := tierName(ca.Config.replicas(sa.Config)) cu, ok := stats.Tiers[consumerTier] if !ok { cu = JetStreamTier{} } cu.Consumers++ stats.Tiers[consumerTier] = cu } } } } else { if defaultTier { stats.Streams = len(jsa.streams) } for _, mset := range jsa.streams { consCount := mset.numConsumers() stats.Consumers += consCount if !defaultTier { u, ok := stats.Tiers[mset.tier] if !ok { u = JetStreamTier{} } u.Streams++ stats.Streams++ u.Consumers += consCount stats.Tiers[mset.tier] = u } } } jsa.mu.RUnlock() js.mu.RUnlock() } return stats } // DisableJetStream will disable JetStream for this account. func (a *Account) DisableJetStream() error { return a.removeJetStream() } // removeJetStream is called when JetStream has been disabled for this account. func (a *Account) removeJetStream() error { a.mu.Lock() s := a.srv a.js = nil a.mu.Unlock() if s == nil { return fmt.Errorf("jetstream account not registered") } js := s.getJetStream() if js == nil { return NewJSNotEnabledForAccountError() } return js.disableJetStream(js.lookupAccount(a)) } // Disable JetStream for the account. func (js *jetStream) disableJetStream(jsa *jsAccount) error { if jsa == nil || jsa.account == nil { return NewJSNotEnabledForAccountError() } js.mu.Lock() delete(js.accounts, jsa.account.Name) js.mu.Unlock() jsa.delete() return nil } // jetStreamConfigured reports whether the account has JetStream configured, regardless of this // servers JetStream status. func (a *Account) jetStreamConfigured() bool { if a == nil { return false } a.mu.RLock() defer a.mu.RUnlock() return len(a.jsLimits) > 0 } // JetStreamEnabled is a helper to determine if jetstream is enabled for an account. func (a *Account) JetStreamEnabled() bool { if a == nil { return false } a.mu.RLock() enabled := a.js != nil a.mu.RUnlock() return enabled } func (jsa *jsAccount) remoteUpdateUsage(sub *subscription, c *client, _ *Account, subject, _ string, msg []byte) { // jsa.js.srv is immutable and guaranteed to no be nil, so no lock needed. s := jsa.js.srv jsa.usageMu.Lock() defer jsa.usageMu.Unlock() if len(msg) < minUsageUpdateLen { s.Warnf("Ignoring remote usage update with size too short") return } var rnode string if li := strings.LastIndexByte(subject, btsep); li > 0 && li < len(subject) { rnode = subject[li+1:] } if rnode == _EMPTY_ { s.Warnf("Received remote usage update with no remote node") return } rUsage, ok := jsa.rusage[rnode] if !ok { if jsa.rusage == nil { jsa.rusage = make(map[string]*remoteUsage) } rUsage = &remoteUsage{tiers: make(map[string]*jsaUsage)} jsa.rusage[rnode] = rUsage } updateTotal := func(tierName string, memUsed, storeUsed int64) { total, ok := jsa.usage[tierName] if !ok { total = &jsaStorage{} jsa.usage[tierName] = total } // Update the usage for this remote. if usage := rUsage.tiers[tierName]; usage != nil { // Decrement our old values. total.total.mem -= usage.mem total.total.store -= usage.store usage.mem, usage.store = memUsed, storeUsed } else { rUsage.tiers[tierName] = &jsaUsage{memUsed, storeUsed} } total.total.mem += memUsed total.total.store += storeUsed } var le = binary.LittleEndian apiTotal, apiErrors := le.Uint64(msg[16:]), le.Uint64(msg[24:]) memUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:])) // We later extended the data structure to support multiple tiers var excessRecordCnt uint32 var tierName string if len(msg) >= usageMultiTiersLen { excessRecordCnt = le.Uint32(msg[minUsageUpdateLen:]) length := le.Uint64(msg[minUsageUpdateLen+4:]) // Need to protect past this point in case this is wrong. if uint64(len(msg)) < usageMultiTiersLen+length { s.Warnf("Received corrupt remote usage update") return } tierName = string(msg[usageMultiTiersLen : usageMultiTiersLen+length]) msg = msg[usageMultiTiersLen+length:] } updateTotal(tierName, memUsed, storeUsed) for ; excessRecordCnt > 0 && len(msg) >= usageRecordLen; excessRecordCnt-- { memUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:])) length := le.Uint64(msg[16:]) if uint64(len(msg)) < usageRecordLen+length { s.Warnf("Received corrupt remote usage update on excess record") return } tierName = string(msg[usageRecordLen : usageRecordLen+length]) msg = msg[usageRecordLen+length:] updateTotal(tierName, memUsed, storeUsed) } jsa.apiTotal -= rUsage.api jsa.apiErrors -= rUsage.err rUsage.api = apiTotal rUsage.err = apiErrors jsa.apiTotal += apiTotal jsa.apiErrors += apiErrors } // When we detect a skew of some sort this will verify the usage reporting is correct. // No locks should be held. func (jsa *jsAccount) checkAndSyncUsage(tierName string, storeType StorageType) { // This will run in a separate go routine, so check that we are only running once. if !jsa.sync.CompareAndSwap(false, true) { return } defer jsa.sync.Store(false) // Hold the account read lock and the usage lock while we calculate. // We scope by tier and storage type, but if R3 File has 200 streams etc. could // show a pause. I did test with > 100 non-active streams and was 80-200ns or so. // Should be rare this gets called as well. jsa.mu.RLock() defer jsa.mu.RUnlock() js := jsa.js if js == nil { return } s := js.srv // We need to collect the stream stores before we acquire the usage lock since in storeUpdates the // stream lock could be held if deletion are inline with storing a new message, e.g. via limits. var stores []StreamStore for _, mset := range jsa.streams { mset.mu.RLock() if mset.tier == tierName && mset.stype == storeType && mset.store != nil { stores = append(stores, mset.store) } mset.mu.RUnlock() } // Now range and qualify, hold usage lock to prevent updates. jsa.usageMu.Lock() defer jsa.usageMu.Unlock() usage, ok := jsa.usage[tierName] if !ok { return } // Collect current total for all stream stores that matched. var total int64 var state StreamState for _, store := range stores { store.FastState(&state) total += int64(state.Bytes) } var needClusterUpdate bool // If we do not match on our calculations compute delta and adjust. if storeType == MemoryStorage { if total != usage.local.mem { s.Warnf("MemStore usage drift of %v vs %v detected for account %q", friendlyBytes(total), friendlyBytes(usage.local.mem), jsa.account.GetName()) delta := total - usage.local.mem usage.local.mem += delta usage.total.mem += delta atomic.AddInt64(&js.memUsed, delta) needClusterUpdate = true } } else { if total != usage.local.store { s.Warnf("FileStore usage drift of %v vs %v detected for account %q", friendlyBytes(total), friendlyBytes(usage.local.store), jsa.account.GetName()) delta := total - usage.local.store usage.local.store += delta usage.total.store += delta atomic.AddInt64(&js.storeUsed, delta) needClusterUpdate = true } } // Publish our local updates if in clustered mode. if needClusterUpdate && js.isClusteredNoLock() { jsa.sendClusterUsageUpdate() } } // Updates accounting on in use memory and storage. This is called from locally // by the lower storage layers. func (jsa *jsAccount) updateUsage(tierName string, storeType StorageType, delta int64) { // jsa.js is immutable and cannot be nil, so ok w/o lock. js := jsa.js // updateUsage() may be invoked under the mset's lock, so we can't get // the js' lock to check if clustered. So use this function that make // use of an atomic to do the check without having data race reports. isClustered := js.isClusteredNoLock() var needsCheck bool jsa.usageMu.Lock() s, ok := jsa.usage[tierName] if !ok { s = &jsaStorage{} jsa.usage[tierName] = s } if storeType == MemoryStorage { s.local.mem += delta s.total.mem += delta atomic.AddInt64(&js.memUsed, delta) needsCheck = s.local.mem < 0 } else { s.local.store += delta s.total.store += delta atomic.AddInt64(&js.storeUsed, delta) needsCheck = s.local.store < 0 } // Publish our local updates if in clustered mode. if isClustered { jsa.sendClusterUsageUpdate() } jsa.usageMu.Unlock() if needsCheck { // We could be holding the stream lock from up in the stack, and this // will want the jsa lock, which would violate locking order. // So do this in a Go routine. The function will check if it is already running. go jsa.checkAndSyncUsage(tierName, storeType) } } var usageTick = 1500 * time.Millisecond func (jsa *jsAccount) sendClusterUsageUpdateTimer() { jsa.usageMu.Lock() defer jsa.usageMu.Unlock() jsa.sendClusterUsageUpdate() if jsa.utimer != nil { jsa.utimer.Reset(usageTick) } } // For usage fields. const ( minUsageUpdateLen = 32 stackUsageUpdate = 72 usageRecordLen = 24 usageMultiTiersLen = 44 apiStatsAndNumTiers = 20 minUsageUpdateWindow = 250 * time.Millisecond ) // Send updates to our account usage for this server. // jsa.usageMu lock should be held. func (jsa *jsAccount) sendClusterUsageUpdate() { // These values are absolute so we can limit send rates. now := time.Now() if now.Sub(jsa.lupdate) < minUsageUpdateWindow { return } jsa.lupdate = now lenUsage := len(jsa.usage) if lenUsage == 0 { return } // every base record contains mem/store/len(tier) as well as the tier name l := usageRecordLen * lenUsage for tier := range jsa.usage { l += len(tier) } // first record contains api/usage errors as well as count for extra base records l += apiStatsAndNumTiers var raw [stackUsageUpdate]byte var b []byte if l > stackUsageUpdate { b = make([]byte, l) } else { b = raw[:l] } var i int var le = binary.LittleEndian for tier, usage := range jsa.usage { le.PutUint64(b[i+0:], uint64(usage.local.mem)) le.PutUint64(b[i+8:], uint64(usage.local.store)) if i == 0 { le.PutUint64(b[16:], jsa.usageApi) le.PutUint64(b[24:], jsa.usageErr) le.PutUint32(b[32:], uint32(len(jsa.usage)-1)) le.PutUint64(b[36:], uint64(len(tier))) copy(b[usageMultiTiersLen:], tier) i = usageMultiTiersLen + len(tier) } else { le.PutUint64(b[i+16:], uint64(len(tier))) copy(b[i+usageRecordLen:], tier) i += usageRecordLen + len(tier) } } jsa.sendq.push(newPubMsg(nil, jsa.updatesPub, _EMPTY_, nil, nil, b, noCompression, false, false)) } func (js *jetStream) wouldExceedLimits(storeType StorageType, sz int) bool { var ( total *int64 max int64 ) if storeType == MemoryStorage { total, max = &js.memUsed, js.config.MaxMemory } else { total, max = &js.storeUsed, js.config.MaxStore } return (atomic.LoadInt64(total) + int64(sz)) > max } func (js *jetStream) limitsExceeded(storeType StorageType) bool { return js.wouldExceedLimits(storeType, 0) } func tierName(replicas int) string { // TODO (mh) this is where we could select based off a placement tag as well "qos:tier" if replicas == 0 { replicas = 1 } return fmt.Sprintf("R%d", replicas) } func isSameTier(cfgA, cfgB *StreamConfig) bool { // TODO (mh) this is where we could select based off a placement tag as well "qos:tier" return cfgA.Replicas == cfgB.Replicas } func (jsa *jsAccount) jetStreamAndClustered() (*jetStream, bool) { jsa.mu.RLock() js := jsa.js jsa.mu.RUnlock() return js, js.isClustered() } // jsa.usageMu read lock should be held. func (jsa *jsAccount) selectLimits(replicas int) (JetStreamAccountLimits, string, bool) { if selectedLimits, ok := jsa.limits[_EMPTY_]; ok { return selectedLimits, _EMPTY_, true } tier := tierName(replicas) if selectedLimits, ok := jsa.limits[tier]; ok { return selectedLimits, tier, true } return JetStreamAccountLimits{}, _EMPTY_, false } // Lock should be held. func (jsa *jsAccount) countStreams(tier string, cfg *StreamConfig) (streams int) { for _, sa := range jsa.streams { // Don't count the stream toward the limit if it already exists. if (tier == _EMPTY_ || isSameTier(&sa.cfg, cfg)) && sa.cfg.Name != cfg.Name { streams++ } } return streams } // jsa.usageMu read lock (at least) should be held. func (jsa *jsAccount) storageTotals() (uint64, uint64) { mem := uint64(0) store := uint64(0) for _, sa := range jsa.usage { mem += uint64(sa.total.mem) store += uint64(sa.total.store) } return mem, store } func (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, replicas int) (bool, *ApiError) { return jsa.wouldExceedLimits(storeType, tierName, replicas, _EMPTY_, nil, nil) } func (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string, replicas int, subj string, hdr, msg []byte) (bool, *ApiError) { jsa.usageMu.RLock() defer jsa.usageMu.RUnlock() selectedLimits, ok := jsa.limits[tierName] if !ok { return true, NewJSNoLimitsError() } inUse := jsa.usage[tierName] if inUse == nil { // Imply totals of 0 return false, nil } r := int64(replicas) // Make sure replicas is correct. if r < 1 { r = 1 } // This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication. lr := r if tierName == _EMPTY_ { lr = 1 } // Since tiers are flat we need to scale limit up by replicas when checking. if storeType == MemoryStorage { totalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r) if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*lr { return true, nil } if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr { return true, nil } } else { totalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r) if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*lr { return true, nil } if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr { return true, nil } } return false, nil } // Check account limits. // Read Lock should be held func (js *jetStream) checkAccountLimits(selected *JetStreamAccountLimits, config *StreamConfig, currentRes int64) error { return js.checkLimits(selected, config, false, currentRes, 0) } // Check account and server limits. // Read Lock should be held func (js *jetStream) checkAllLimits(selected *JetStreamAccountLimits, config *StreamConfig, currentRes, maxBytesOffset int64) error { return js.checkLimits(selected, config, true, currentRes, maxBytesOffset) } // Check if a new proposed msg set while exceed our account limits. // Lock should be held. func (js *jetStream) checkLimits(selected *JetStreamAccountLimits, config *StreamConfig, checkServer bool, currentRes, maxBytesOffset int64) error { // Check MaxConsumers if config.MaxConsumers > 0 && selected.MaxConsumers > 0 && config.MaxConsumers > selected.MaxConsumers { return NewJSMaximumConsumersLimitError() } // stream limit is checked separately on stream create only! // Check storage, memory or disk. return js.checkBytesLimits(selected, config.MaxBytes, config.Storage, checkServer, currentRes, maxBytesOffset) } // Check if additional bytes will exceed our account limits and optionally the server itself. // Read Lock should be held. func (js *jetStream) checkBytesLimits(selectedLimits *JetStreamAccountLimits, addBytes int64, storage StorageType, checkServer bool, currentRes, maxBytesOffset int64) error { if addBytes < 0 { addBytes = 1 } totalBytes := addBytes + maxBytesOffset switch storage { case MemoryStorage: // Account limits defined. if selectedLimits.MaxMemory >= 0 && currentRes+totalBytes > selectedLimits.MaxMemory { return NewJSMemoryResourcesExceededError() } // Check if this server can handle request. if checkServer && js.memReserved+totalBytes > js.config.MaxMemory { return NewJSMemoryResourcesExceededError() } case FileStorage: // Account limits defined. if selectedLimits.MaxStore >= 0 && currentRes+totalBytes > selectedLimits.MaxStore { return NewJSStorageResourcesExceededError() } // Check if this server can handle request. if checkServer && js.storeReserved+totalBytes > js.config.MaxStore { return NewJSStorageResourcesExceededError() } } return nil } func (jsa *jsAccount) acc() *Account { return jsa.account } // Delete the JetStream resources. func (jsa *jsAccount) delete() { var streams []*stream var ts []string jsa.mu.Lock() // The update timer and subs need to be protected by usageMu lock jsa.usageMu.Lock() if jsa.utimer != nil { jsa.utimer.Stop() jsa.utimer = nil } if jsa.updatesSub != nil && jsa.js.srv != nil { s := jsa.js.srv s.sysUnsubscribe(jsa.updatesSub) jsa.updatesSub = nil } jsa.usageMu.Unlock() for _, ms := range jsa.streams { streams = append(streams, ms) } acc := jsa.account for _, t := range jsa.templates { ts = append(ts, t.Name) } jsa.templates = nil jsa.mu.Unlock() for _, mset := range streams { mset.stop(false, false) } for _, t := range ts { acc.deleteStreamTemplate(t) } } // Lookup the jetstream account for a given account. func (js *jetStream) lookupAccount(a *Account) *jsAccount { if a == nil { return nil } js.mu.RLock() jsa := js.accounts[a.Name] js.mu.RUnlock() return jsa } // Report on JetStream stats and usage for this server. func (js *jetStream) usageStats() *JetStreamStats { var stats JetStreamStats js.mu.RLock() stats.Accounts = len(js.accounts) stats.ReservedMemory = uint64(js.memReserved) stats.ReservedStore = uint64(js.storeReserved) s := js.srv js.mu.RUnlock() stats.API.Total = uint64(atomic.LoadInt64(&js.apiTotal)) stats.API.Errors = uint64(atomic.LoadInt64(&js.apiErrors)) stats.API.Inflight = uint64(atomic.LoadInt64(&js.apiInflight)) // Make sure we do not report negative. used := atomic.LoadInt64(&js.memUsed) if used < 0 { used = 0 } stats.Memory = uint64(used) used = atomic.LoadInt64(&js.storeUsed) if used < 0 { used = 0 } stats.Store = uint64(used) stats.HAAssets = s.numRaftNodes() return &stats } // Check to see if we have enough system resources for this account. // Lock should be held. func (js *jetStream) sufficientResources(limits map[string]JetStreamAccountLimits) error { // If we are clustered we do not really know how many resources will be ultimately available. // This needs to be handled out of band. // If we are a single server, we can make decisions here. if limits == nil || !js.standAlone { return nil } totalMaxBytes := func(limits map[string]JetStreamAccountLimits) (int64, int64) { totalMaxMemory := int64(0) totalMaxStore := int64(0) for _, l := range limits { if l.MaxMemory > 0 { totalMaxMemory += l.MaxMemory } if l.MaxStore > 0 { totalMaxStore += l.MaxStore } } return totalMaxMemory, totalMaxStore } totalMaxMemory, totalMaxStore := totalMaxBytes(limits) // Reserved is now specific to the MaxBytes for streams. if js.memReserved+totalMaxMemory > js.config.MaxMemory { return NewJSMemoryResourcesExceededError() } if js.storeReserved+totalMaxStore > js.config.MaxStore { return NewJSStorageResourcesExceededError() } // Since we know if we are here we are single server mode, check the account reservations. var storeReserved, memReserved int64 for _, jsa := range js.accounts { if jsa.account.IsExpired() { continue } jsa.usageMu.RLock() maxMemory, maxStore := totalMaxBytes(jsa.limits) jsa.usageMu.RUnlock() memReserved += maxMemory storeReserved += maxStore } if memReserved+totalMaxMemory > js.config.MaxMemory { return NewJSMemoryResourcesExceededError() } if storeReserved+totalMaxStore > js.config.MaxStore { return NewJSStorageResourcesExceededError() } return nil } // This will reserve the stream resources requested. // This will spin off off of MaxBytes. func (js *jetStream) reserveStreamResources(cfg *StreamConfig) { if cfg == nil || cfg.MaxBytes <= 0 { return } js.mu.Lock() switch cfg.Storage { case MemoryStorage: js.memReserved += cfg.MaxBytes case FileStorage: js.storeReserved += cfg.MaxBytes } s, clustered := js.srv, !js.standAlone js.mu.Unlock() // If clustered send an update to the system immediately. if clustered { s.sendStatszUpdate() } } // Release reserved resources held by a stream. func (js *jetStream) releaseStreamResources(cfg *StreamConfig) { if cfg == nil || cfg.MaxBytes <= 0 { return } js.mu.Lock() switch cfg.Storage { case MemoryStorage: js.memReserved -= cfg.MaxBytes case FileStorage: js.storeReserved -= cfg.MaxBytes } s, clustered := js.srv, !js.standAlone js.mu.Unlock() // If clustered send an update to the system immediately. if clustered { s.sendStatszUpdate() } } const ( // JetStreamStoreDir is the prefix we use. JetStreamStoreDir = "jetstream" // JetStreamMaxStoreDefault is the default disk storage limit. 1TB JetStreamMaxStoreDefault = 1024 * 1024 * 1024 * 1024 // JetStreamMaxMemDefault is only used when we can't determine system memory. 256MB JetStreamMaxMemDefault = 1024 * 1024 * 256 // snapshot staging for restores. snapStagingDir = ".snap-staging" ) // Dynamically create a config with a tmp based directory (repeatable) and 75% of system memory. func (s *Server) dynJetStreamConfig(storeDir string, maxStore, maxMem int64) *JetStreamConfig { jsc := &JetStreamConfig{} if storeDir != _EMPTY_ { jsc.StoreDir = filepath.Join(storeDir, JetStreamStoreDir) } else { // Create one in tmp directory, but make it consistent for restarts. jsc.StoreDir = filepath.Join(os.TempDir(), "nats", JetStreamStoreDir) s.Warnf("Temporary storage directory used, data could be lost on system reboot") } opts := s.getOpts() // Sync options. jsc.SyncInterval = opts.SyncInterval jsc.SyncAlways = opts.SyncAlways if opts.maxStoreSet && maxStore >= 0 { jsc.MaxStore = maxStore } else { jsc.MaxStore = diskAvailable(jsc.StoreDir) } if opts.maxMemSet && maxMem >= 0 { jsc.MaxMemory = maxMem } else { // Estimate to 75% of total memory if we can determine system memory. if sysMem := sysmem.Memory(); sysMem > 0 { // Check if we have been limited with GOMEMLIMIT and if lower use that value. if gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 && gml < sysMem { s.Debugf("JetStream detected GOMEMLIMIT of %v", friendlyBytes(gml)) sysMem = gml } jsc.MaxMemory = sysMem / 4 * 3 } else { jsc.MaxMemory = JetStreamMaxMemDefault } } return jsc } // Helper function. func (a *Account) checkForJetStream() (*Server, *jsAccount, error) { a.mu.RLock() s := a.srv jsa := a.js a.mu.RUnlock() if s == nil || jsa == nil { return nil, nil, NewJSNotEnabledForAccountError() } return s, jsa, nil } // StreamTemplateConfig allows a configuration to auto-create streams based on this template when a message // is received that matches. Each new stream will use the config as the template config to create them. type StreamTemplateConfig struct { Name string `json:"name"` Config *StreamConfig `json:"config"` MaxStreams uint32 `json:"max_streams"` } // StreamTemplateInfo type StreamTemplateInfo struct { Config *StreamTemplateConfig `json:"config"` Streams []string `json:"streams"` } // streamTemplate type streamTemplate struct { mu sync.Mutex tc *client jsa *jsAccount *StreamTemplateConfig streams []string } func (t *StreamTemplateConfig) deepCopy() *StreamTemplateConfig { copy := *t cfg := *t.Config copy.Config = &cfg return © } // addStreamTemplate will add a stream template to this account that allows auto-creation of streams. func (a *Account) addStreamTemplate(tc *StreamTemplateConfig) (*streamTemplate, error) { s, jsa, err := a.checkForJetStream() if err != nil { return nil, err } if tc.Config.Name != "" { return nil, fmt.Errorf("template config name should be empty") } if len(tc.Name) > JSMaxNameLen { return nil, fmt.Errorf("template name is too long, maximum allowed is %d", JSMaxNameLen) } // FIXME(dlc) - Hacky tcopy := tc.deepCopy() tcopy.Config.Name = "_" cfg, apiErr := s.checkStreamCfg(tcopy.Config, a) if apiErr != nil { return nil, apiErr } tcopy.Config = &cfg t := &streamTemplate{ StreamTemplateConfig: tcopy, tc: s.createInternalJetStreamClient(), jsa: jsa, } t.tc.registerWithAccount(a) jsa.mu.Lock() if jsa.templates == nil { jsa.templates = make(map[string]*streamTemplate) // Create the appropriate store if cfg.Storage == FileStorage { jsa.store = newTemplateFileStore(jsa.storeDir) } else { jsa.store = newTemplateMemStore() } } else if _, ok := jsa.templates[tcopy.Name]; ok { jsa.mu.Unlock() return nil, fmt.Errorf("template with name %q already exists", tcopy.Name) } jsa.templates[tcopy.Name] = t jsa.mu.Unlock() // FIXME(dlc) - we can not overlap subjects between templates. Need to have test. // Setup the internal subscriptions to trap the messages. if err := t.createTemplateSubscriptions(); err != nil { return nil, err } if err := jsa.store.Store(t); err != nil { t.delete() return nil, err } return t, nil } func (t *streamTemplate) createTemplateSubscriptions() error { if t == nil { return fmt.Errorf("no template") } if t.tc == nil { return fmt.Errorf("template not enabled") } c := t.tc if !c.srv.EventsEnabled() { return ErrNoSysAccount } sid := 1 for _, subject := range t.Config.Subjects { // Now create the subscription if _, err := c.processSub([]byte(subject), nil, []byte(strconv.Itoa(sid)), t.processInboundTemplateMsg, false); err != nil { c.acc.deleteStreamTemplate(t.Name) return err } sid++ } return nil } func (t *streamTemplate) processInboundTemplateMsg(_ *subscription, pc *client, acc *Account, subject, reply string, msg []byte) { if t == nil || t.jsa == nil { return } jsa := t.jsa cn := canonicalName(subject) jsa.mu.Lock() // If we already are registered then we can just return here. if _, ok := jsa.streams[cn]; ok { jsa.mu.Unlock() return } jsa.mu.Unlock() // Check if we are at the maximum and grab some variables. t.mu.Lock() c := t.tc cfg := *t.Config cfg.Template = t.Name atLimit := len(t.streams) >= int(t.MaxStreams) if !atLimit { t.streams = append(t.streams, cn) } t.mu.Unlock() if atLimit { c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q, at limit", acc.Name, subject) return } // We need to create the stream here. // Change the config from the template and only use literal subject. cfg.Name = cn cfg.Subjects = []string{subject} mset, err := acc.addStream(&cfg) if err != nil { acc.validateStreams(t) c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q: %v", acc.Name, subject, err) return } // Process this message directly by invoking mset. mset.processInboundJetStreamMsg(nil, pc, acc, subject, reply, msg) } // lookupStreamTemplate looks up the names stream template. func (a *Account) lookupStreamTemplate(name string) (*streamTemplate, error) { _, jsa, err := a.checkForJetStream() if err != nil { return nil, err } jsa.mu.Lock() defer jsa.mu.Unlock() if jsa.templates == nil { return nil, fmt.Errorf("template not found") } t, ok := jsa.templates[name] if !ok { return nil, fmt.Errorf("template not found") } return t, nil } // This function will check all named streams and make sure they are valid. func (a *Account) validateStreams(t *streamTemplate) { t.mu.Lock() var vstreams []string for _, sname := range t.streams { if _, err := a.lookupStream(sname); err == nil { vstreams = append(vstreams, sname) } } t.streams = vstreams t.mu.Unlock() } func (t *streamTemplate) delete() error { if t == nil { return fmt.Errorf("nil stream template") } t.mu.Lock() jsa := t.jsa c := t.tc t.tc = nil defer func() { if c != nil { c.closeConnection(ClientClosed) } }() t.mu.Unlock() if jsa == nil { return NewJSNotEnabledForAccountError() } jsa.mu.Lock() if jsa.templates == nil { jsa.mu.Unlock() return fmt.Errorf("template not found") } if _, ok := jsa.templates[t.Name]; !ok { jsa.mu.Unlock() return fmt.Errorf("template not found") } delete(jsa.templates, t.Name) acc := jsa.account jsa.mu.Unlock() // Remove streams associated with this template. var streams []*stream t.mu.Lock() for _, name := range t.streams { if mset, err := acc.lookupStream(name); err == nil { streams = append(streams, mset) } } t.mu.Unlock() if jsa.store != nil { if err := jsa.store.Delete(t); err != nil { return fmt.Errorf("error deleting template from store: %v", err) } } var lastErr error for _, mset := range streams { if err := mset.delete(); err != nil { lastErr = err } } return lastErr } func (a *Account) deleteStreamTemplate(name string) error { t, err := a.lookupStreamTemplate(name) if err != nil { return NewJSStreamTemplateNotFoundError() } return t.delete() } func (a *Account) templates() []*streamTemplate { var ts []*streamTemplate _, jsa, err := a.checkForJetStream() if err != nil { return nil } jsa.mu.Lock() for _, t := range jsa.templates { // FIXME(dlc) - Copy? ts = append(ts, t) } jsa.mu.Unlock() return ts } // Will add a stream to a template, this is for recovery. func (jsa *jsAccount) addStreamNameToTemplate(tname, mname string) error { if jsa.templates == nil { return fmt.Errorf("template not found") } t, ok := jsa.templates[tname] if !ok { return fmt.Errorf("template not found") } // We found template. t.mu.Lock() t.streams = append(t.streams, mname) t.mu.Unlock() return nil } // This will check if a template owns this stream. // jsAccount lock should be held func (jsa *jsAccount) checkTemplateOwnership(tname, sname string) bool { if jsa.templates == nil { return false } t, ok := jsa.templates[tname] if !ok { return false } // We found template, make sure we are in streams. for _, streamName := range t.streams { if sname == streamName { return true } } return false } type Number interface { int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64 } // friendlyBytes returns a string with the given bytes int64 // represented as a size, such as 1KB, 10MB, etc... func friendlyBytes[T Number](bytes T) string { fbytes := float64(bytes) base := 1024 pre := []string{"K", "M", "G", "T", "P", "E"} if fbytes < float64(base) { return fmt.Sprintf("%v B", fbytes) } exp := int(math.Log(fbytes) / math.Log(float64(base))) index := exp - 1 return fmt.Sprintf("%.2f %sB", fbytes/math.Pow(float64(base), float64(exp)), pre[index]) } func isValidName(name string) bool { if name == _EMPTY_ { return false } return !strings.ContainsAny(name, " \t\r\n\f.*>") } // CanonicalName will replace all token separators '.' with '_'. // This can be used when naming streams or consumers with multi-token subjects. func canonicalName(name string) string { return strings.ReplaceAll(name, ".", "_") } // To throttle the out of resources errors. func (s *Server) resourcesExceededError() { var didAlert bool s.rerrMu.Lock() if now := time.Now(); now.Sub(s.rerrLast) > 10*time.Second { s.Errorf("JetStream resource limits exceeded for server") s.rerrLast = now didAlert = true } s.rerrMu.Unlock() // If we are meta leader we should relinguish that here. if didAlert { if js := s.getJetStream(); js != nil { js.mu.RLock() if cc := js.cluster; cc != nil && cc.isLeader() { cc.meta.StepDown() } js.mu.RUnlock() } } } // For validating options. func validateJetStreamOptions(o *Options) error { // in non operator mode, the account names need to be configured if len(o.JsAccDefaultDomain) > 0 { if len(o.TrustedOperators) == 0 { for a, domain := range o.JsAccDefaultDomain { found := false if isReservedAccount(a) { found = true } else { for _, acc := range o.Accounts { if a == acc.GetName() { if len(acc.jsLimits) > 0 && domain != _EMPTY_ { return fmt.Errorf("default_js_domain contains account name %q with enabled JetStream", a) } found = true break } } } if !found { return fmt.Errorf("in non operator mode, `default_js_domain` references non existing account %q", a) } } } else { for a := range o.JsAccDefaultDomain { if !nkeys.IsValidPublicAccountKey(a) { return fmt.Errorf("default_js_domain contains account name %q, which is not a valid public account nkey", a) } } } for a, d := range o.JsAccDefaultDomain { sacc := DEFAULT_SYSTEM_ACCOUNT if o.SystemAccount != _EMPTY_ { sacc = o.SystemAccount } if a == sacc { return fmt.Errorf("system account %q can not be in default_js_domain", a) } if d == _EMPTY_ { continue } if sub := fmt.Sprintf(jsDomainAPI, d); !IsValidSubject(sub) { return fmt.Errorf("default_js_domain contains account %q with invalid domain name %q", a, d) } } } if o.JetStreamDomain != _EMPTY_ { if subj := fmt.Sprintf(jsDomainAPI, o.JetStreamDomain); !IsValidSubject(subj) { return fmt.Errorf("invalid domain name: derived %q is not a valid subject", subj) } if !isValidName(o.JetStreamDomain) { return fmt.Errorf("invalid domain name: may not contain ., * or >") } } // If not clustered no checks needed past here. if !o.JetStream || o.Cluster.Port == 0 { return nil } if o.ServerName == _EMPTY_ { return fmt.Errorf("jetstream cluster requires `server_name` to be set") } if o.Cluster.Name == _EMPTY_ { return fmt.Errorf("jetstream cluster requires `cluster.name` to be set") } h := strings.ToLower(o.JetStreamExtHint) switch h { case jsWillExtend, jsNoExtend, _EMPTY_: o.JetStreamExtHint = h default: return fmt.Errorf("expected 'no_extend' for string value, got '%s'", h) } if o.JetStreamMaxCatchup < 0 { return fmt.Errorf("jetstream max catchup cannot be negative") } return nil } // We had a bug that set a default de dupe window on mirror, despite that being not a valid config func fixCfgMirrorWithDedupWindow(cfg *StreamConfig) { if cfg == nil || cfg.Mirror == nil { return } if cfg.Duplicates != 0 { cfg.Duplicates = 0 } } func (s *Server) handleWritePermissionError() { //TODO Check if we should add s.jetStreamOOSPending in condition if s.JetStreamEnabled() { s.Errorf("File system permission denied while writing, disabling JetStream") go s.DisableJetStream() //TODO Send respective advisory if needed, same as in handleOutOfSpace } } nats-server-2.10.27/server/jetstream_api.go000066400000000000000000004207311477524627100206460ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" "encoding/json" "errors" "fmt" "math/rand" "os" "path/filepath" "runtime" "slices" "strconv" "strings" "sync/atomic" "time" "unicode" "github.com/nats-io/nuid" ) // Request API subjects for JetStream. const ( // All API endpoints. jsAllAPI = "$JS.API.>" // For constructing JetStream domain prefixes. jsDomainAPI = "$JS.%s.API.>" JSApiPrefix = "$JS.API" // JSApiAccountInfo is for obtaining general information about JetStream for this account. // Will return JSON response. JSApiAccountInfo = "$JS.API.INFO" // JSApiTemplateCreate is the endpoint to create new stream templates. // Will return JSON response. JSApiTemplateCreate = "$JS.API.STREAM.TEMPLATE.CREATE.*" JSApiTemplateCreateT = "$JS.API.STREAM.TEMPLATE.CREATE.%s" // JSApiTemplates is the endpoint to list all stream template names for this account. // Will return JSON response. JSApiTemplates = "$JS.API.STREAM.TEMPLATE.NAMES" // JSApiTemplateInfo is for obtaining general information about a named stream template. // Will return JSON response. JSApiTemplateInfo = "$JS.API.STREAM.TEMPLATE.INFO.*" JSApiTemplateInfoT = "$JS.API.STREAM.TEMPLATE.INFO.%s" // JSApiTemplateDelete is the endpoint to delete stream templates. // Will return JSON response. JSApiTemplateDelete = "$JS.API.STREAM.TEMPLATE.DELETE.*" JSApiTemplateDeleteT = "$JS.API.STREAM.TEMPLATE.DELETE.%s" // JSApiStreamCreate is the endpoint to create new streams. // Will return JSON response. JSApiStreamCreate = "$JS.API.STREAM.CREATE.*" JSApiStreamCreateT = "$JS.API.STREAM.CREATE.%s" // JSApiStreamUpdate is the endpoint to update existing streams. // Will return JSON response. JSApiStreamUpdate = "$JS.API.STREAM.UPDATE.*" JSApiStreamUpdateT = "$JS.API.STREAM.UPDATE.%s" // JSApiStreams is the endpoint to list all stream names for this account. // Will return JSON response. JSApiStreams = "$JS.API.STREAM.NAMES" // JSApiStreamList is the endpoint that will return all detailed stream information JSApiStreamList = "$JS.API.STREAM.LIST" // JSApiStreamInfo is for obtaining general information about a named stream. // Will return JSON response. JSApiStreamInfo = "$JS.API.STREAM.INFO.*" JSApiStreamInfoT = "$JS.API.STREAM.INFO.%s" // JSApiStreamDelete is the endpoint to delete streams. // Will return JSON response. JSApiStreamDelete = "$JS.API.STREAM.DELETE.*" JSApiStreamDeleteT = "$JS.API.STREAM.DELETE.%s" // JSApiStreamPurge is the endpoint to purge streams. // Will return JSON response. JSApiStreamPurge = "$JS.API.STREAM.PURGE.*" JSApiStreamPurgeT = "$JS.API.STREAM.PURGE.%s" // JSApiStreamSnapshot is the endpoint to snapshot streams. // Will return a stream of chunks with a nil chunk as EOF to // the deliver subject. Caller should respond to each chunk // with a nil body response for ack flow. JSApiStreamSnapshot = "$JS.API.STREAM.SNAPSHOT.*" JSApiStreamSnapshotT = "$JS.API.STREAM.SNAPSHOT.%s" // JSApiStreamRestore is the endpoint to restore a stream from a snapshot. // Caller should respond to each chunk with a nil body response. JSApiStreamRestore = "$JS.API.STREAM.RESTORE.*" JSApiStreamRestoreT = "$JS.API.STREAM.RESTORE.%s" // JSApiMsgDelete is the endpoint to delete messages from a stream. // Will return JSON response. JSApiMsgDelete = "$JS.API.STREAM.MSG.DELETE.*" JSApiMsgDeleteT = "$JS.API.STREAM.MSG.DELETE.%s" // JSApiMsgGet is the template for direct requests for a message by its stream sequence number. // Will return JSON response. JSApiMsgGet = "$JS.API.STREAM.MSG.GET.*" JSApiMsgGetT = "$JS.API.STREAM.MSG.GET.%s" // JSDirectMsgGet is the template for non-api layer direct requests for a message by its stream sequence number or last by subject. // Will return the message similar to how a consumer receives the message, no JSON processing. // If the message can not be found we will use a status header of 404. If the stream does not exist the client will get a no-responders or timeout. JSDirectMsgGet = "$JS.API.DIRECT.GET.*" JSDirectMsgGetT = "$JS.API.DIRECT.GET.%s" // This is a direct version of get last by subject, which will be the dominant pattern for KV access once 2.9 is released. // The stream and the key will be part of the subject to allow for no-marshal payloads and subject based security permissions. JSDirectGetLastBySubject = "$JS.API.DIRECT.GET.*.>" JSDirectGetLastBySubjectT = "$JS.API.DIRECT.GET.%s.%s" // jsDirectGetPre jsDirectGetPre = "$JS.API.DIRECT.GET" // JSApiConsumerCreate is the endpoint to create consumers for streams. // This was also the legacy endpoint for ephemeral consumers. // It now can take consumer name and optional filter subject, which when part of the subject controls access. // Will return JSON response. JSApiConsumerCreate = "$JS.API.CONSUMER.CREATE.*" JSApiConsumerCreateT = "$JS.API.CONSUMER.CREATE.%s" JSApiConsumerCreateEx = "$JS.API.CONSUMER.CREATE.*.>" JSApiConsumerCreateExT = "$JS.API.CONSUMER.CREATE.%s.%s.%s" // JSApiDurableCreate is the endpoint to create durable consumers for streams. // You need to include the stream and consumer name in the subject. JSApiDurableCreate = "$JS.API.CONSUMER.DURABLE.CREATE.*.*" JSApiDurableCreateT = "$JS.API.CONSUMER.DURABLE.CREATE.%s.%s" // JSApiConsumers is the endpoint to list all consumer names for the stream. // Will return JSON response. JSApiConsumers = "$JS.API.CONSUMER.NAMES.*" JSApiConsumersT = "$JS.API.CONSUMER.NAMES.%s" // JSApiConsumerList is the endpoint that will return all detailed consumer information JSApiConsumerList = "$JS.API.CONSUMER.LIST.*" JSApiConsumerListT = "$JS.API.CONSUMER.LIST.%s" // JSApiConsumerInfo is for obtaining general information about a consumer. // Will return JSON response. JSApiConsumerInfo = "$JS.API.CONSUMER.INFO.*.*" JSApiConsumerInfoT = "$JS.API.CONSUMER.INFO.%s.%s" // JSApiConsumerDelete is the endpoint to delete consumers. // Will return JSON response. JSApiConsumerDelete = "$JS.API.CONSUMER.DELETE.*.*" JSApiConsumerDeleteT = "$JS.API.CONSUMER.DELETE.%s.%s" // JSApiRequestNextT is the prefix for the request next message(s) for a consumer in worker/pull mode. JSApiRequestNextT = "$JS.API.CONSUMER.MSG.NEXT.%s.%s" // jsRequestNextPre jsRequestNextPre = "$JS.API.CONSUMER.MSG.NEXT." // For snapshots and restores. The ack will have additional tokens. jsSnapshotAckT = "$JS.SNAPSHOT.ACK.%s.%s" jsRestoreDeliverT = "$JS.SNAPSHOT.RESTORE.%s.%s" // JSApiStreamRemovePeer is the endpoint to remove a peer from a clustered stream and its consumers. // Will return JSON response. JSApiStreamRemovePeer = "$JS.API.STREAM.PEER.REMOVE.*" JSApiStreamRemovePeerT = "$JS.API.STREAM.PEER.REMOVE.%s" // JSApiStreamLeaderStepDown is the endpoint to have stream leader stepdown. // Will return JSON response. JSApiStreamLeaderStepDown = "$JS.API.STREAM.LEADER.STEPDOWN.*" JSApiStreamLeaderStepDownT = "$JS.API.STREAM.LEADER.STEPDOWN.%s" // JSApiConsumerLeaderStepDown is the endpoint to have consumer leader stepdown. // Will return JSON response. JSApiConsumerLeaderStepDown = "$JS.API.CONSUMER.LEADER.STEPDOWN.*.*" JSApiConsumerLeaderStepDownT = "$JS.API.CONSUMER.LEADER.STEPDOWN.%s.%s" // JSApiLeaderStepDown is the endpoint to have our metaleader stepdown. // Only works from system account. // Will return JSON response. JSApiLeaderStepDown = "$JS.API.META.LEADER.STEPDOWN" // JSApiRemoveServer is the endpoint to remove a peer server from the cluster. // Only works from system account. // Will return JSON response. JSApiRemoveServer = "$JS.API.SERVER.REMOVE" // JSApiAccountPurge is the endpoint to purge the js content of an account // Only works from system account. // Will return JSON response. JSApiAccountPurge = "$JS.API.ACCOUNT.PURGE.*" JSApiAccountPurgeT = "$JS.API.ACCOUNT.PURGE.%s" // JSApiServerStreamMove is the endpoint to move streams off a server // Only works from system account. // Will return JSON response. JSApiServerStreamMove = "$JS.API.ACCOUNT.STREAM.MOVE.*.*" JSApiServerStreamMoveT = "$JS.API.ACCOUNT.STREAM.MOVE.%s.%s" // JSApiServerStreamCancelMove is the endpoint to cancel a stream move // Only works from system account. // Will return JSON response. JSApiServerStreamCancelMove = "$JS.API.ACCOUNT.STREAM.CANCEL_MOVE.*.*" JSApiServerStreamCancelMoveT = "$JS.API.ACCOUNT.STREAM.CANCEL_MOVE.%s.%s" // The prefix for system level account API. jsAPIAccountPre = "$JS.API.ACCOUNT." // jsAckT is the template for the ack message stream coming back from a consumer // when they ACK/NAK, etc a message. jsAckT = "$JS.ACK.%s.%s" jsAckPre = "$JS.ACK." jsAckPreLen = len(jsAckPre) // jsFlowControl is for flow control subjects. jsFlowControlPre = "$JS.FC." // jsFlowControl is for FC responses. jsFlowControl = "$JS.FC.%s.%s.*" // JSAdvisoryPrefix is a prefix for all JetStream advisories. JSAdvisoryPrefix = "$JS.EVENT.ADVISORY" // JSMetricPrefix is a prefix for all JetStream metrics. JSMetricPrefix = "$JS.EVENT.METRIC" // JSMetricConsumerAckPre is a metric containing ack latency. JSMetricConsumerAckPre = "$JS.EVENT.METRIC.CONSUMER.ACK" // JSAdvisoryConsumerMaxDeliveryExceedPre is a notification published when a message exceeds its delivery threshold. JSAdvisoryConsumerMaxDeliveryExceedPre = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES" // JSAdvisoryConsumerMsgNakPre is a notification published when a message has been naked JSAdvisoryConsumerMsgNakPre = "$JS.EVENT.ADVISORY.CONSUMER.MSG_NAKED" // JSAdvisoryConsumerMsgTerminatedPre is a notification published when a message has been terminated. JSAdvisoryConsumerMsgTerminatedPre = "$JS.EVENT.ADVISORY.CONSUMER.MSG_TERMINATED" // JSAdvisoryStreamCreatedPre notification that a stream was created. JSAdvisoryStreamCreatedPre = "$JS.EVENT.ADVISORY.STREAM.CREATED" // JSAdvisoryStreamDeletedPre notification that a stream was deleted. JSAdvisoryStreamDeletedPre = "$JS.EVENT.ADVISORY.STREAM.DELETED" // JSAdvisoryStreamUpdatedPre notification that a stream was updated. JSAdvisoryStreamUpdatedPre = "$JS.EVENT.ADVISORY.STREAM.UPDATED" // JSAdvisoryConsumerCreatedPre notification that a template created. JSAdvisoryConsumerCreatedPre = "$JS.EVENT.ADVISORY.CONSUMER.CREATED" // JSAdvisoryConsumerDeletedPre notification that a template deleted. JSAdvisoryConsumerDeletedPre = "$JS.EVENT.ADVISORY.CONSUMER.DELETED" // JSAdvisoryStreamSnapshotCreatePre notification that a snapshot was created. JSAdvisoryStreamSnapshotCreatePre = "$JS.EVENT.ADVISORY.STREAM.SNAPSHOT_CREATE" // JSAdvisoryStreamSnapshotCompletePre notification that a snapshot was completed. JSAdvisoryStreamSnapshotCompletePre = "$JS.EVENT.ADVISORY.STREAM.SNAPSHOT_COMPLETE" // JSAdvisoryStreamRestoreCreatePre notification that a restore was start. JSAdvisoryStreamRestoreCreatePre = "$JS.EVENT.ADVISORY.STREAM.RESTORE_CREATE" // JSAdvisoryStreamRestoreCompletePre notification that a restore was completed. JSAdvisoryStreamRestoreCompletePre = "$JS.EVENT.ADVISORY.STREAM.RESTORE_COMPLETE" // JSAdvisoryDomainLeaderElectedPre notification that a jetstream domain has elected a leader. JSAdvisoryDomainLeaderElected = "$JS.EVENT.ADVISORY.DOMAIN.LEADER_ELECTED" // JSAdvisoryStreamLeaderElectedPre notification that a replicated stream has elected a leader. JSAdvisoryStreamLeaderElectedPre = "$JS.EVENT.ADVISORY.STREAM.LEADER_ELECTED" // JSAdvisoryStreamQuorumLostPre notification that a stream and its consumers are stalled. JSAdvisoryStreamQuorumLostPre = "$JS.EVENT.ADVISORY.STREAM.QUORUM_LOST" // JSAdvisoryConsumerLeaderElectedPre notification that a replicated consumer has elected a leader. JSAdvisoryConsumerLeaderElectedPre = "$JS.EVENT.ADVISORY.CONSUMER.LEADER_ELECTED" // JSAdvisoryConsumerQuorumLostPre notification that a consumer is stalled. JSAdvisoryConsumerQuorumLostPre = "$JS.EVENT.ADVISORY.CONSUMER.QUORUM_LOST" // JSAdvisoryServerOutOfStorage notification that a server has no more storage. JSAdvisoryServerOutOfStorage = "$JS.EVENT.ADVISORY.SERVER.OUT_OF_STORAGE" // JSAdvisoryServerRemoved notification that a server has been removed from the system. JSAdvisoryServerRemoved = "$JS.EVENT.ADVISORY.SERVER.REMOVED" // JSAdvisoryAPILimitReached notification that a server has reached the JS API hard limit. JSAdvisoryAPILimitReached = "$JS.EVENT.ADVISORY.API.LIMIT_REACHED" // JSAuditAdvisory is a notification about JetStream API access. // FIXME - Add in details about who.. JSAuditAdvisory = "$JS.EVENT.ADVISORY.API" ) var denyAllClientJs = []string{jsAllAPI, "$KV.>", "$OBJ.>"} var denyAllJs = []string{jscAllSubj, raftAllSubj, jsAllAPI, "$KV.>", "$OBJ.>"} func generateJSMappingTable(domain string) map[string]string { mappings := map[string]string{} // This set of mappings is very very very ugly. // It is a consequence of what we defined the domain prefix to be "$JS.domain.API" and it's mapping to "$JS.API" // For optics $KV and $OBJ where made to be independent subject spaces. // As materialized views of JS, they did not simply extend that subject space to say "$JS.API.KV" "$JS.API.OBJ" // This is very unfortunate!!! // Furthermore, it seemed bad to require different domain prefixes for JS/KV/OBJ. // Especially since the actual API for say KV, does use stream create from JS. // To avoid overlaps KV and OBJ views append the prefix to their API. // (Replacing $KV with the prefix allows users to create collisions with say the bucket name) // This mapping therefore needs to have extra token so that the mapping can properly discern between $JS, $KV, $OBJ for srcMappingSuffix, to := range map[string]string{ "INFO": JSApiAccountInfo, "STREAM.>": "$JS.API.STREAM.>", "CONSUMER.>": "$JS.API.CONSUMER.>", "DIRECT.>": "$JS.API.DIRECT.>", "META.>": "$JS.API.META.>", "SERVER.>": "$JS.API.SERVER.>", "ACCOUNT.>": "$JS.API.ACCOUNT.>", "$KV.>": "$KV.>", "$OBJ.>": "$OBJ.>", } { mappings[fmt.Sprintf("$JS.%s.API.%s", domain, srcMappingSuffix)] = to } return mappings } // JSMaxDescription is the maximum description length for streams and consumers. const JSMaxDescriptionLen = 4 * 1024 // JSMaxMetadataLen is the maximum length for streams and consumers metadata map. // It's calculated by summing length of all keys and values. const JSMaxMetadataLen = 128 * 1024 // JSMaxNameLen is the maximum name lengths for streams, consumers and templates. // Picked 255 as it seems to be a widely used file name limit const JSMaxNameLen = 255 // JSDefaultRequestQueueLimit is the default number of entries that we will // put on the global request queue before we react. const JSDefaultRequestQueueLimit = 10_000 // Responses for API calls. // ApiResponse is a standard response from the JetStream JSON API type ApiResponse struct { Type string `json:"type"` Error *ApiError `json:"error,omitempty"` } const JSApiSystemResponseType = "io.nats.jetstream.api.v1.system_response" // When passing back to the clients generalize store failures. var ( errStreamStoreFailed = errors.New("error creating store for stream") errConsumerStoreFailed = errors.New("error creating store for consumer") ) // ToError checks if the response has a error and if it does converts it to an error avoiding // the pitfalls described by https://yourbasic.org/golang/gotcha-why-nil-error-not-equal-nil/ func (r *ApiResponse) ToError() error { if r.Error == nil { return nil } return r.Error } const JSApiOverloadedType = "io.nats.jetstream.api.v1.system_overloaded" // ApiPaged includes variables used to create paged responses from the JSON API type ApiPaged struct { Total int `json:"total"` Offset int `json:"offset"` Limit int `json:"limit"` } // ApiPagedRequest includes parameters allowing specific pages to be requests from APIs responding with ApiPaged type ApiPagedRequest struct { Offset int `json:"offset"` } // JSApiAccountInfoResponse reports back information on jetstream for this account. type JSApiAccountInfoResponse struct { ApiResponse *JetStreamAccountStats } const JSApiAccountInfoResponseType = "io.nats.jetstream.api.v1.account_info_response" // JSApiStreamCreateResponse stream creation. type JSApiStreamCreateResponse struct { ApiResponse *StreamInfo DidCreate bool `json:"did_create,omitempty"` } const JSApiStreamCreateResponseType = "io.nats.jetstream.api.v1.stream_create_response" // JSApiStreamDeleteResponse stream removal. type JSApiStreamDeleteResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiStreamDeleteResponseType = "io.nats.jetstream.api.v1.stream_delete_response" // JSMaxSubjectDetails The limit of the number of subject details we will send in a stream info response. const JSMaxSubjectDetails = 100_000 type JSApiStreamInfoRequest struct { ApiPagedRequest DeletedDetails bool `json:"deleted_details,omitempty"` SubjectsFilter string `json:"subjects_filter,omitempty"` } type JSApiStreamInfoResponse struct { ApiResponse ApiPaged *StreamInfo } const JSApiStreamInfoResponseType = "io.nats.jetstream.api.v1.stream_info_response" // JSApiNamesLimit is the maximum entries we will return for streams or consumers lists. // TODO(dlc) - with header or request support could request chunked response. const JSApiNamesLimit = 1024 const JSApiListLimit = 256 type JSApiStreamNamesRequest struct { ApiPagedRequest // These are filters that can be applied to the list. Subject string `json:"subject,omitempty"` } // JSApiStreamNamesResponse list of streams. // A nil request is valid and means all streams. type JSApiStreamNamesResponse struct { ApiResponse ApiPaged Streams []string `json:"streams"` } const JSApiStreamNamesResponseType = "io.nats.jetstream.api.v1.stream_names_response" type JSApiStreamListRequest struct { ApiPagedRequest // These are filters that can be applied to the list. Subject string `json:"subject,omitempty"` } // JSApiStreamListResponse list of detailed stream information. // A nil request is valid and means all streams. type JSApiStreamListResponse struct { ApiResponse ApiPaged Streams []*StreamInfo `json:"streams"` Missing []string `json:"missing,omitempty"` } const JSApiStreamListResponseType = "io.nats.jetstream.api.v1.stream_list_response" // JSApiStreamPurgeRequest is optional request information to the purge API. // Subject will filter the purge request to only messages that match the subject, which can have wildcards. // Sequence will purge up to but not including this sequence and can be combined with subject filtering. // Keep will specify how many messages to keep. This can also be combined with subject filtering. // Note that Sequence and Keep are mutually exclusive, so both can not be set at the same time. type JSApiStreamPurgeRequest struct { // Purge up to but not including sequence. Sequence uint64 `json:"seq,omitempty"` // Subject to match against messages for the purge command. Subject string `json:"filter,omitempty"` // Number of messages to keep. Keep uint64 `json:"keep,omitempty"` } type JSApiStreamPurgeResponse struct { ApiResponse Success bool `json:"success,omitempty"` Purged uint64 `json:"purged"` } const JSApiStreamPurgeResponseType = "io.nats.jetstream.api.v1.stream_purge_response" // JSApiStreamUpdateResponse for updating a stream. type JSApiStreamUpdateResponse struct { ApiResponse *StreamInfo } const JSApiStreamUpdateResponseType = "io.nats.jetstream.api.v1.stream_update_response" // JSApiMsgDeleteRequest delete message request. type JSApiMsgDeleteRequest struct { Seq uint64 `json:"seq"` NoErase bool `json:"no_erase,omitempty"` } type JSApiMsgDeleteResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiMsgDeleteResponseType = "io.nats.jetstream.api.v1.stream_msg_delete_response" type JSApiStreamSnapshotRequest struct { // Subject to deliver the chunks to for the snapshot. DeliverSubject string `json:"deliver_subject"` // Do not include consumers in the snapshot. NoConsumers bool `json:"no_consumers,omitempty"` // Optional chunk size preference. // Best to just let server select. ChunkSize int `json:"chunk_size,omitempty"` // Check all message's checksums prior to snapshot. CheckMsgs bool `json:"jsck,omitempty"` } // JSApiStreamSnapshotResponse is the direct response to the snapshot request. type JSApiStreamSnapshotResponse struct { ApiResponse // Configuration of the given stream. Config *StreamConfig `json:"config,omitempty"` // Current State for the given stream. State *StreamState `json:"state,omitempty"` } const JSApiStreamSnapshotResponseType = "io.nats.jetstream.api.v1.stream_snapshot_response" // JSApiStreamRestoreRequest is the required restore request. type JSApiStreamRestoreRequest struct { // Configuration of the given stream. Config StreamConfig `json:"config"` // Current State for the given stream. State StreamState `json:"state"` } // JSApiStreamRestoreResponse is the direct response to the restore request. type JSApiStreamRestoreResponse struct { ApiResponse // Subject to deliver the chunks to for the snapshot restore. DeliverSubject string `json:"deliver_subject"` } const JSApiStreamRestoreResponseType = "io.nats.jetstream.api.v1.stream_restore_response" // JSApiStreamRemovePeerRequest is the required remove peer request. type JSApiStreamRemovePeerRequest struct { // Server name of the peer to be removed. Peer string `json:"peer"` } // JSApiStreamRemovePeerResponse is the response to a remove peer request. type JSApiStreamRemovePeerResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiStreamRemovePeerResponseType = "io.nats.jetstream.api.v1.stream_remove_peer_response" // JSApiStreamLeaderStepDownResponse is the response to a leader stepdown request. type JSApiStreamLeaderStepDownResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiStreamLeaderStepDownResponseType = "io.nats.jetstream.api.v1.stream_leader_stepdown_response" // JSApiConsumerLeaderStepDownResponse is the response to a consumer leader stepdown request. type JSApiConsumerLeaderStepDownResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiConsumerLeaderStepDownResponseType = "io.nats.jetstream.api.v1.consumer_leader_stepdown_response" // JSApiLeaderStepdownRequest allows placement control over the meta leader placement. type JSApiLeaderStepdownRequest struct { Placement *Placement `json:"placement,omitempty"` } // JSApiLeaderStepDownResponse is the response to a meta leader stepdown request. type JSApiLeaderStepDownResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiLeaderStepDownResponseType = "io.nats.jetstream.api.v1.meta_leader_stepdown_response" // JSApiMetaServerRemoveRequest will remove a peer from the meta group. type JSApiMetaServerRemoveRequest struct { // Server name of the peer to be removed. Server string `json:"peer"` // Peer ID of the peer to be removed. If specified this is used // instead of the server name. Peer string `json:"peer_id,omitempty"` } // JSApiMetaServerRemoveResponse is the response to a peer removal request in the meta group. type JSApiMetaServerRemoveResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiMetaServerRemoveResponseType = "io.nats.jetstream.api.v1.meta_server_remove_response" // JSApiMetaServerStreamMoveRequest will move a stream on a server to another // response to this will come as JSApiStreamUpdateResponse/JSApiStreamUpdateResponseType type JSApiMetaServerStreamMoveRequest struct { // Server name of the peer to be evacuated. Server string `json:"server,omitempty"` // Cluster the server is in Cluster string `json:"cluster,omitempty"` // Domain the sever is in Domain string `json:"domain,omitempty"` // Ephemeral placement tags for the move Tags []string `json:"tags,omitempty"` } const JSApiAccountPurgeResponseType = "io.nats.jetstream.api.v1.account_purge_response" // JSApiAccountPurgeResponse is the response to a purge request in the meta group. type JSApiAccountPurgeResponse struct { ApiResponse Initiated bool `json:"initiated,omitempty"` } // JSApiMsgGetRequest get a message request. type JSApiMsgGetRequest struct { Seq uint64 `json:"seq,omitempty"` LastFor string `json:"last_by_subj,omitempty"` NextFor string `json:"next_by_subj,omitempty"` } type JSApiMsgGetResponse struct { ApiResponse Message *StoredMsg `json:"message,omitempty"` } const JSApiMsgGetResponseType = "io.nats.jetstream.api.v1.stream_msg_get_response" // JSWaitQueueDefaultMax is the default max number of outstanding requests for pull consumers. const JSWaitQueueDefaultMax = 512 type JSApiConsumerCreateResponse struct { ApiResponse *ConsumerInfo } const JSApiConsumerCreateResponseType = "io.nats.jetstream.api.v1.consumer_create_response" type JSApiConsumerDeleteResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiConsumerDeleteResponseType = "io.nats.jetstream.api.v1.consumer_delete_response" type JSApiConsumerInfoResponse struct { ApiResponse *ConsumerInfo } const JSApiConsumerInfoResponseType = "io.nats.jetstream.api.v1.consumer_info_response" type JSApiConsumersRequest struct { ApiPagedRequest } type JSApiConsumerNamesResponse struct { ApiResponse ApiPaged Consumers []string `json:"consumers"` } const JSApiConsumerNamesResponseType = "io.nats.jetstream.api.v1.consumer_names_response" type JSApiConsumerListResponse struct { ApiResponse ApiPaged Consumers []*ConsumerInfo `json:"consumers"` Missing []string `json:"missing,omitempty"` } const JSApiConsumerListResponseType = "io.nats.jetstream.api.v1.consumer_list_response" // JSApiConsumerGetNextRequest is for getting next messages for pull based consumers. type JSApiConsumerGetNextRequest struct { Expires time.Duration `json:"expires,omitempty"` Batch int `json:"batch,omitempty"` MaxBytes int `json:"max_bytes,omitempty"` NoWait bool `json:"no_wait,omitempty"` Heartbeat time.Duration `json:"idle_heartbeat,omitempty"` } // JSApiStreamTemplateCreateResponse for creating templates. type JSApiStreamTemplateCreateResponse struct { ApiResponse *StreamTemplateInfo } const JSApiStreamTemplateCreateResponseType = "io.nats.jetstream.api.v1.stream_template_create_response" type JSApiStreamTemplateDeleteResponse struct { ApiResponse Success bool `json:"success,omitempty"` } const JSApiStreamTemplateDeleteResponseType = "io.nats.jetstream.api.v1.stream_template_delete_response" // JSApiStreamTemplateInfoResponse for information about stream templates. type JSApiStreamTemplateInfoResponse struct { ApiResponse *StreamTemplateInfo } const JSApiStreamTemplateInfoResponseType = "io.nats.jetstream.api.v1.stream_template_info_response" type JSApiStreamTemplatesRequest struct { ApiPagedRequest } // JSApiStreamTemplateNamesResponse list of templates type JSApiStreamTemplateNamesResponse struct { ApiResponse ApiPaged Templates []string `json:"streams"` } const JSApiStreamTemplateNamesResponseType = "io.nats.jetstream.api.v1.stream_template_names_response" // Structure that holds state for a JetStream API request that is processed // in a separate long-lived go routine. This is to avoid possibly blocking // ROUTE and GATEWAY connections. type jsAPIRoutedReq struct { jsub *subscription sub *subscription acc *Account subject string reply string msg []byte pa pubArg } func (js *jetStream) apiDispatch(sub *subscription, c *client, acc *Account, subject, reply string, rmsg []byte) { // Ignore system level directives meta stepdown and peer remove requests here. if subject == JSApiLeaderStepDown || subject == JSApiRemoveServer || strings.HasPrefix(subject, jsAPIAccountPre) { return } // No lock needed, those are immutable. s, rr := js.srv, js.apiSubs.Match(subject) hdr, msg := c.msgParts(rmsg) if len(sliceHeader(ClientInfoHdr, hdr)) == 0 { // Check if this is the system account. We will let these through for the account info only. sacc := s.SystemAccount() if sacc != acc { return } if subject != JSApiAccountInfo { // Only respond from the initial server entry to the NATS system. if c.kind == CLIENT || c.kind == LEAF { var resp = ApiResponse{ Type: JSApiSystemResponseType, Error: NewJSNotEnabledForAccountError(), } s.sendAPIErrResponse(nil, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } } // Short circuit for no interest. if len(rr.psubs)+len(rr.qsubs) == 0 { if (c.kind == CLIENT || c.kind == LEAF) && acc != s.SystemAccount() { ci, acc, _, _, _ := s.getRequestInfo(c, rmsg) var resp = ApiResponse{ Type: JSApiSystemResponseType, Error: NewJSBadRequestError(), } s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // We should only have psubs and only 1 per result. if len(rr.psubs) != 1 { s.Warnf("Malformed JetStream API Request: [%s] %q", subject, rmsg) if c.kind == CLIENT || c.kind == LEAF { ci, acc, _, _, _ := s.getRequestInfo(c, rmsg) var resp = ApiResponse{ Type: JSApiSystemResponseType, Error: NewJSBadRequestError(), } s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } jsub := rr.psubs[0] // If this is directly from a client connection ok to do in place. if c.kind != ROUTER && c.kind != GATEWAY && c.kind != LEAF { start := time.Now() jsub.icb(sub, c, acc, subject, reply, rmsg) if dur := time.Since(start); dur >= readLoopReportThreshold { s.Warnf("Internal subscription on %q took too long: %v", subject, dur) } return } // If we are here we have received this request over a non-client connection. // We need to make sure not to block. We will send the request to a long-lived // pool of go routines. // Increment inflight. Do this before queueing. atomic.AddInt64(&js.apiInflight, 1) // Copy the state. Note the JSAPI only uses the hdr index to piece apart the // header from the msg body. No other references are needed. // Check pending and warn if getting backed up. pending := s.jsAPIRoutedReqs.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa}) limit := atomic.LoadInt64(&js.queueLimit) if pending >= int(limit) { s.rateLimitFormatWarnf("JetStream API queue limit reached, dropping %d requests", pending) drained := int64(s.jsAPIRoutedReqs.drain()) atomic.AddInt64(&js.apiInflight, -drained) s.publishAdvisory(nil, JSAdvisoryAPILimitReached, JSAPILimitReachedAdvisory{ TypedEvent: TypedEvent{ Type: JSAPILimitReachedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Server: s.Name(), Domain: js.config.Domain, Dropped: drained, }) } } func (s *Server) processJSAPIRoutedRequests() { defer s.grWG.Done() s.mu.RLock() queue := s.jsAPIRoutedReqs client := &client{srv: s, kind: JETSTREAM} s.mu.RUnlock() js := s.getJetStream() for { select { case <-queue.ch: // Only pop one item at a time here, otherwise if the system is recovering // from queue buildup, then one worker will pull off all the tasks and the // others will be starved of work. for r, ok := queue.popOne(); ok && r != nil; r, ok = queue.popOne() { client.pa = r.pa start := time.Now() r.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg) if dur := time.Since(start); dur >= readLoopReportThreshold { s.Warnf("Internal subscription on %q took too long: %v", r.subject, dur) } atomic.AddInt64(&js.apiInflight, -1) } case <-s.quitCh: return } } } func (s *Server) setJetStreamExportSubs() error { js := s.getJetStream() if js == nil { return NewJSNotEnabledError() } // Start the go routine that will process API requests received by the // subscription below when they are coming from routes, etc.. const maxProcs = 16 mp := runtime.GOMAXPROCS(0) // Cap at 16 max for now on larger core setups. if mp > maxProcs { mp = maxProcs } s.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, "Routed JS API Requests") for i := 0; i < mp; i++ { s.startGoRoutine(s.processJSAPIRoutedRequests) } // This is the catch all now for all JetStream API calls. if _, err := s.sysSubscribe(jsAllAPI, js.apiDispatch); err != nil { return err } if err := s.SystemAccount().AddServiceExport(jsAllAPI, nil); err != nil { s.Warnf("Error setting up jetstream service exports: %v", err) return err } // API handles themselves. pairs := []struct { subject string handler msgHandler }{ {JSApiAccountInfo, s.jsAccountInfoRequest}, {JSApiTemplateCreate, s.jsTemplateCreateRequest}, {JSApiTemplates, s.jsTemplateNamesRequest}, {JSApiTemplateInfo, s.jsTemplateInfoRequest}, {JSApiTemplateDelete, s.jsTemplateDeleteRequest}, {JSApiStreamCreate, s.jsStreamCreateRequest}, {JSApiStreamUpdate, s.jsStreamUpdateRequest}, {JSApiStreams, s.jsStreamNamesRequest}, {JSApiStreamList, s.jsStreamListRequest}, {JSApiStreamInfo, s.jsStreamInfoRequest}, {JSApiStreamDelete, s.jsStreamDeleteRequest}, {JSApiStreamPurge, s.jsStreamPurgeRequest}, {JSApiStreamSnapshot, s.jsStreamSnapshotRequest}, {JSApiStreamRestore, s.jsStreamRestoreRequest}, {JSApiStreamRemovePeer, s.jsStreamRemovePeerRequest}, {JSApiStreamLeaderStepDown, s.jsStreamLeaderStepDownRequest}, {JSApiConsumerLeaderStepDown, s.jsConsumerLeaderStepDownRequest}, {JSApiMsgDelete, s.jsMsgDeleteRequest}, {JSApiMsgGet, s.jsMsgGetRequest}, {JSApiConsumerCreateEx, s.jsConsumerCreateRequest}, {JSApiConsumerCreate, s.jsConsumerCreateRequest}, {JSApiDurableCreate, s.jsConsumerCreateRequest}, {JSApiConsumers, s.jsConsumerNamesRequest}, {JSApiConsumerList, s.jsConsumerListRequest}, {JSApiConsumerInfo, s.jsConsumerInfoRequest}, {JSApiConsumerDelete, s.jsConsumerDeleteRequest}, } js.mu.Lock() defer js.mu.Unlock() for _, p := range pairs { sub := &subscription{subject: []byte(p.subject), icb: p.handler} if err := js.apiSubs.Insert(sub); err != nil { return err } } return nil } func (s *Server) sendAPIResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string) { acc.trackAPI() if reply != _EMPTY_ { s.sendInternalAccountMsg(nil, reply, response) } s.sendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response) } func (s *Server) sendAPIErrResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string) { acc.trackAPIErr() if reply != _EMPTY_ { s.sendInternalAccountMsg(nil, reply, response) } s.sendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response) } const errRespDelay = 500 * time.Millisecond func (s *Server) sendDelayedAPIErrResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string, rg *raftGroup) { js := s.getJetStream() if js == nil { return } var quitCh <-chan struct{} js.mu.RLock() if rg != nil && rg.node != nil { quitCh = rg.node.QuitC() } js.mu.RUnlock() s.startGoRoutine(func() { defer s.grWG.Done() select { case <-quitCh: case <-s.quitCh: case <-time.After(errRespDelay): acc.trackAPIErr() if reply != _EMPTY_ { s.sendInternalAccountMsg(nil, reply, response) } s.sendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response) } }) } func (s *Server) getRequestInfo(c *client, raw []byte) (pci *ClientInfo, acc *Account, hdr, msg []byte, err error) { hdr, msg = c.msgParts(raw) var ci ClientInfo if len(hdr) > 0 { if err := json.Unmarshal(sliceHeader(ClientInfoHdr, hdr), &ci); err != nil { return nil, nil, nil, nil, err } } if ci.Service != _EMPTY_ { acc, _ = s.LookupAccount(ci.Service) } else if ci.Account != _EMPTY_ { acc, _ = s.LookupAccount(ci.Account) } else { // Direct $SYS access. acc = c.acc if acc == nil { acc = s.SystemAccount() } } if acc == nil { return nil, nil, nil, nil, ErrMissingAccount } return &ci, acc, hdr, msg, nil } func (a *Account) trackAPI() { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa != nil { jsa.usageMu.Lock() jsa.usageApi++ jsa.apiTotal++ jsa.sendClusterUsageUpdate() atomic.AddInt64(&jsa.js.apiTotal, 1) jsa.usageMu.Unlock() } } func (a *Account) trackAPIErr() { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa != nil { jsa.usageMu.Lock() jsa.usageApi++ jsa.apiTotal++ jsa.usageErr++ jsa.apiErrors++ jsa.sendClusterUsageUpdate() atomic.AddInt64(&jsa.js.apiTotal, 1) atomic.AddInt64(&jsa.js.apiErrors, 1) jsa.usageMu.Unlock() } } const badAPIRequestT = "Malformed JetStream API Request: %q" // Helper function to check on JetStream being enabled but also on status of leafnodes // If the local account is not enabled but does have leafnode connectivity we will not // want to error immediately and let the other side decide. func (a *Account) checkJetStream() (enabled, shouldError bool) { a.mu.RLock() defer a.mu.RUnlock() return a.js != nil, a.nleafs+a.nrleafs == 0 } // Request for current usage and limits for this account. func (s *Server) jsAccountInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiAccountInfoResponse{ApiResponse: ApiResponse{Type: JSApiAccountInfoResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if !doErr { return } resp.Error = NewJSNotEnabledForAccountError() } else { stats := acc.JetStreamUsage() resp.JetStreamAccountStats = &stats } b, err := json.Marshal(resp) if err != nil { return } s.sendAPIResponse(ci, acc, subject, reply, string(msg), string(b)) } // Helpers for token extraction. func templateNameFromSubject(subject string) string { return tokenAt(subject, 6) } func streamNameFromSubject(subject string) string { return tokenAt(subject, 5) } func consumerNameFromSubject(subject string) string { return tokenAt(subject, 6) } // Request to create a new template. func (s *Server) jsTemplateCreateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamTemplateCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateCreateResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Not supported for now. if s.JetStreamIsClustered() { resp.Error = NewJSClusterUnSupportFeatureError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var cfg StreamTemplateConfig if err := json.Unmarshal(msg, &cfg); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } templateName := templateNameFromSubject(subject) if templateName != cfg.Name { resp.Error = NewJSTemplateNameNotMatchSubjectError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } t, err := acc.addStreamTemplate(&cfg) if err != nil { resp.Error = NewJSStreamTemplateCreateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } t.mu.Lock() tcfg := t.StreamTemplateConfig.deepCopy() streams := t.streams if streams == nil { streams = []string{} } t.mu.Unlock() resp.StreamTemplateInfo = &StreamTemplateInfo{Config: tcfg, Streams: streams} s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for the list of all template names. func (s *Server) jsTemplateNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamTemplateNamesResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateNamesResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Not supported for now. if s.JetStreamIsClustered() { resp.Error = NewJSClusterUnSupportFeatureError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var offset int if !isEmptyRequest(msg) { var req JSApiStreamTemplatesRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } offset = req.Offset } ts := acc.templates() slices.SortFunc(ts, func(i, j *streamTemplate) int { return cmp.Compare(i.StreamTemplateConfig.Name, j.StreamTemplateConfig.Name) }) tcnt := len(ts) if offset > tcnt { offset = tcnt } for _, t := range ts[offset:] { t.mu.Lock() name := t.Name t.mu.Unlock() resp.Templates = append(resp.Templates, name) if len(resp.Templates) >= JSApiNamesLimit { break } } resp.Total = tcnt resp.Limit = JSApiNamesLimit resp.Offset = offset if resp.Templates == nil { resp.Templates = []string{} } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for information about a stream template. func (s *Server) jsTemplateInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamTemplateInfoResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateInfoResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if !isEmptyRequest(msg) { resp.Error = NewJSNotEmptyRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } name := templateNameFromSubject(subject) t, err := acc.lookupStreamTemplate(name) if err != nil { resp.Error = NewJSStreamTemplateNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } t.mu.Lock() cfg := t.StreamTemplateConfig.deepCopy() streams := t.streams if streams == nil { streams = []string{} } t.mu.Unlock() resp.StreamTemplateInfo = &StreamTemplateInfo{Config: cfg, Streams: streams} s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to delete a stream template. func (s *Server) jsTemplateDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamTemplateDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateDeleteResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if !isEmptyRequest(msg) { resp.Error = NewJSNotEmptyRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } name := templateNameFromSubject(subject) err = acc.deleteStreamTemplate(name) if err != nil { resp.Error = NewJSStreamTemplateDeleteError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } func (s *Server) jsonResponse(v any) string { b, err := json.Marshal(v) if err != nil { s.Warnf("Problem marshaling JSON for JetStream API:", err) return "" } return string(b) } // Read lock must be held func (jsa *jsAccount) tieredReservation(tier string, cfg *StreamConfig) int64 { reservation := int64(0) if tier == _EMPTY_ { for _, sa := range jsa.streams { if sa.cfg.MaxBytes > 0 { if sa.cfg.Storage == cfg.Storage && sa.cfg.Name != cfg.Name { reservation += (int64(sa.cfg.Replicas) * sa.cfg.MaxBytes) } } } } else { for _, sa := range jsa.streams { if sa.cfg.Replicas == cfg.Replicas { if sa.cfg.MaxBytes > 0 { if isSameTier(&sa.cfg, cfg) && sa.cfg.Name != cfg.Name { reservation += (int64(sa.cfg.Replicas) * sa.cfg.MaxBytes) } } } } } return reservation } // Request to create a stream. func (s *Server) jsStreamCreateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var cfg StreamConfig if err := json.Unmarshal(msg, &cfg); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } streamName := streamNameFromSubject(subject) if streamName != cfg.Name { resp.Error = NewJSStreamMismatchError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check for path like separators in the name. if strings.ContainsAny(streamName, `\/`) { resp.Error = NewJSStreamNameContainsPathSeparatorsError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Can't create a stream with a sealed state. if cfg.Sealed { resp.Error = NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration for create can not be sealed")) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are told to do mirror direct but are not mirroring, error. if cfg.MirrorDirect && cfg.Mirror == nil { resp.Error = NewJSStreamInvalidConfigError(fmt.Errorf("stream has no mirror but does have mirror direct")) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Hand off to cluster for processing. if s.JetStreamIsClustered() { s.jsClusteredStreamRequest(ci, acc, subject, reply, rmsg, &cfg) return } if err := acc.jsNonClusteredStreamLimitsCheck(&cfg); err != nil { resp.Error = err s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } mset, err := acc.addStream(&cfg) if err != nil { if IsNatsErr(err, JSStreamStoreFailedF) { s.Warnf("Stream create failed for '%s > %s': %v", acc, streamName, err) err = errStreamStoreFailed } resp.Error = NewJSStreamCreateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), TimeStamp: time.Now().UTC(), Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo(), } resp.DidCreate = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to update a stream. func (s *Server) jsStreamUpdateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var ncfg StreamConfig if err := json.Unmarshal(msg, &ncfg); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } cfg, apiErr := s.checkStreamCfg(&ncfg, acc) if apiErr != nil { resp.Error = apiErr s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } streamName := streamNameFromSubject(subject) if streamName != cfg.Name { resp.Error = NewJSStreamMismatchError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Handle clustered version here. if s.JetStreamIsClustered() { // Always do in separate Go routine. go s.jsClusteredStreamUpdateRequest(ci, acc, subject, reply, copyBytes(rmsg), &cfg, nil) return } mset, err := acc.lookupStream(streamName) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if err := mset.update(&cfg); err != nil { resp.Error = NewJSStreamUpdateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Domain: s.getOpts().JetStreamDomain, Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo(), TimeStamp: time.Now().UTC(), } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for the list of all stream names. func (s *Server) jsStreamNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamNamesResponse{ApiResponse: ApiResponse{Type: JSApiStreamNamesResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var offset int var filter string if !isEmptyRequest(msg) { var req JSApiStreamNamesRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } offset = req.Offset if req.Subject != _EMPTY_ { filter = req.Subject } } // TODO(dlc) - Maybe hold these results for large results that we expect to be paged. // TODO(dlc) - If this list is long maybe do this in a Go routine? var numStreams int if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { // TODO(dlc) - Debug or Warn? return } js.mu.RLock() for stream, sa := range cc.streams[acc.Name] { if IsNatsErr(sa.err, JSClusterNotAssignedErr) { continue } if filter != _EMPTY_ { // These could not have subjects auto-filled in since they are raw and unprocessed. if len(sa.Config.Subjects) == 0 { if SubjectsCollide(filter, sa.Config.Name) { resp.Streams = append(resp.Streams, stream) } } else { for _, subj := range sa.Config.Subjects { if SubjectsCollide(filter, subj) { resp.Streams = append(resp.Streams, stream) break } } } } else { resp.Streams = append(resp.Streams, stream) } } js.mu.RUnlock() if len(resp.Streams) > 1 { slices.Sort(resp.Streams) } numStreams = len(resp.Streams) if offset > numStreams { offset = numStreams } if offset > 0 { resp.Streams = resp.Streams[offset:] } if len(resp.Streams) > JSApiNamesLimit { resp.Streams = resp.Streams[:JSApiNamesLimit] } } else { msets := acc.filteredStreams(filter) // Since we page results order matters. if len(msets) > 1 { slices.SortFunc(msets, func(i, j *stream) int { return cmp.Compare(i.cfg.Name, j.cfg.Name) }) } numStreams = len(msets) if offset > numStreams { offset = numStreams } for _, mset := range msets[offset:] { resp.Streams = append(resp.Streams, mset.cfg.Name) if len(resp.Streams) >= JSApiNamesLimit { break } } } resp.Total = numStreams resp.Limit = JSApiNamesLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for the list of all detailed stream info. // TODO(dlc) - combine with above long term func (s *Server) jsStreamListRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamListResponse{ ApiResponse: ApiResponse{Type: JSApiStreamListResponseType}, Streams: []*StreamInfo{}, } // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var offset int var filter string if !isEmptyRequest(msg) { var req JSApiStreamListRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } offset = req.Offset if req.Subject != _EMPTY_ { filter = req.Subject } } // Clustered mode will invoke a scatter and gather. if s.JetStreamIsClustered() { // Need to copy these off before sending.. don't move this inside startGoRoutine!!! msg = copyBytes(msg) s.startGoRoutine(func() { s.jsClusteredStreamListRequest(acc, ci, filter, offset, subject, reply, msg) }) return } // TODO(dlc) - Maybe hold these results for large results that we expect to be paged. // TODO(dlc) - If this list is long maybe do this in a Go routine? var msets []*stream if filter == _EMPTY_ { msets = acc.streams() } else { msets = acc.filteredStreams(filter) } slices.SortFunc(msets, func(i, j *stream) int { return cmp.Compare(i.cfg.Name, j.cfg.Name) }) scnt := len(msets) if offset > scnt { offset = scnt } for _, mset := range msets[offset:] { config := mset.config() resp.Streams = append(resp.Streams, &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: config, Domain: s.getOpts().JetStreamDomain, Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo(), TimeStamp: time.Now().UTC(), }) if len(resp.Streams) >= JSApiListLimit { break } } resp.Total = scnt resp.Limit = JSApiListLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for information about a stream. func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, a *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } streamName := streamNameFromSubject(subject) var resp = JSApiStreamInfoResponse{ApiResponse: ApiResponse{Type: JSApiStreamInfoResponseType}} // If someone creates a duplicate stream that is identical we will get this request forwarded to us. // Make sure the response type is for a create call. if rt := getHeader(JSResponseType, hdr); len(rt) > 0 && string(rt) == jsCreateResponse { resp.ApiResponse.Type = JSApiStreamCreateResponseType } var clusterWideConsCount int js, cc := s.getJetStreamCluster() if js == nil { return } // If we are in clustered mode we need to be the stream leader to proceed. if cc != nil { // Check to make sure the stream is assigned. js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, streamName) var offline bool if sa != nil { clusterWideConsCount = len(sa.consumers) offline = s.allPeersOffline(sa.Group) } js.mu.RUnlock() if isLeader && sa == nil { // We can't find the stream, so mimic what would be the errors below. if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // No stream present. resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() // Delaying an error response gives the leader a chance to respond before us s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil) } return } else if isLeader && offline { resp.Error = NewJSStreamOfflineError() s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil) return } // Check to see if we are a member of the group and if the group has no leader. isLeaderless := js.isGroupLeaderless(sa.Group) // We have the stream assigned and a leader, so only the stream leader should answer. if !acc.JetStreamIsStreamLeader(streamName) && !isLeaderless { if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() // Delaying an error response gives the leader a chance to respond before us s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), sa.Group) return } // We may be in process of electing a leader, but if this is a scale up from 1 we will still be the state leader // while the new members work through the election and catchup process. // Double check for that instead of exiting here and being silent. e.g. nats stream update test --replicas=3 js.mu.RLock() rg := sa.Group var ourID string if cc.meta != nil { ourID = cc.meta.ID() } // We have seen cases where rg is nil at this point, // so check explicitly and bail if that is the case. bail := rg == nil || !rg.isMember(ourID) if !bail { // We know we are a member here, if this group is new and we are preferred allow us to answer. // Also, we have seen cases where rg.node is nil at this point, // so check explicitly and bail if that is the case. bail = rg.Preferred != ourID || (rg.node != nil && time.Since(rg.node.Created()) > lostQuorumIntervalDefault) } js.mu.RUnlock() if bail { return } } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var details bool var subjects string var offset int if !isEmptyRequest(msg) { var req JSApiStreamInfoRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } details, subjects = req.DeletedDetails, req.SubjectsFilter offset = req.Offset } mset, err := acc.lookupStream(streamName) // Error is not to be expected at this point, but could happen if same stream trying to be created. if err != nil { if cc != nil { // This could be inflight, pause for a short bit and try again. // This will not be inline, so ok. time.Sleep(10 * time.Millisecond) mset, err = acc.lookupStream(streamName) } // Check again. if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } config := mset.config() resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.stateWithDetail(details), Config: config, Domain: s.getOpts().JetStreamDomain, Cluster: js.clusterInfo(mset.raftGroup()), Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo(), Alternates: js.streamAlternates(ci, config.Name), TimeStamp: time.Now().UTC(), } if clusterWideConsCount > 0 { resp.StreamInfo.State.Consumers = clusterWideConsCount } // Check if they have asked for subject details. if subjects != _EMPTY_ { st := mset.store.SubjectsTotals(subjects) if lst := len(st); lst > 0 { // Common for both cases. resp.Offset = offset resp.Limit = JSMaxSubjectDetails resp.Total = lst if offset == 0 && lst <= JSMaxSubjectDetails { resp.StreamInfo.State.Subjects = st } else { // Here we have to filter list due to offset or maximum constraints. subjs := make([]string, 0, len(st)) for subj := range st { subjs = append(subjs, subj) } // Sort it slices.Sort(subjs) if offset > len(subjs) { offset = len(subjs) } end := offset + JSMaxSubjectDetails if end > len(subjs) { end = len(subjs) } actualSize := end - offset var sd map[string]uint64 if actualSize > 0 { sd = make(map[string]uint64, actualSize) for _, ss := range subjs[offset:end] { sd[ss] = st[ss] } } resp.StreamInfo.State.Subjects = sd } } } // Check for out of band catchups. if mset.hasCatchupPeers() { mset.checkClusterInfo(resp.StreamInfo.Cluster) } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to have a stream leader stepdown. func (s *Server) jsStreamLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } // Have extra token for this one. name := tokenAt(subject, 6) var resp = JSApiStreamLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiStreamLeaderStepDownResponseType}} // If we are not in clustered mode this is a failed request. if !s.JetStreamIsClustered() { resp.Error = NewJSClusterRequiredError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are here we are clustered. See if we are the stream leader in order to proceed. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, name) js.mu.RUnlock() if isLeader && sa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { return } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if !isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check to see if we are a member of the group and if the group has no leader. if js.isGroupLeaderless(sa.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // We have the stream assigned and a leader, so only the stream leader should answer. if !acc.JetStreamIsStreamLeader(name) { return } mset, err := acc.lookupStream(name) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if mset == nil { resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) return } // Call actual stepdown. Do this in a Go routine. go func() { if node := mset.raftNode(); node != nil { mset.setLeader(false) // TODO (mh) eventually make sure all go routines exited and all channels are cleared time.Sleep(250 * time.Millisecond) node.StepDown() } resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) }() } // Request to have a consumer leader stepdown. func (s *Server) jsConsumerLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiConsumerLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiConsumerLeaderStepDownResponseType}} // If we are not in clustered mode this is a failed request. if !s.JetStreamIsClustered() { resp.Error = NewJSClusterRequiredError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are here we are clustered. See if we are the stream leader in order to proceed. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Have extra token for this one. stream := tokenAt(subject, 6) consumer := tokenAt(subject, 7) js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream) js.mu.RUnlock() if isLeader && sa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { return } var ca *consumerAssignment if sa.consumers != nil { ca = sa.consumers[consumer] } if ca == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check to see if we are a member of the group and if the group has no leader. if js.isGroupLeaderless(ca.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if !acc.JetStreamIsConsumerLeader(stream, consumer) { return } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if !isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } o := mset.lookupConsumer(consumer) if o == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } n := o.raftNode() if n == nil { resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) return } // Call actual stepdown. Do this in a Go routine. go func() { o.setLeader(false) // TODO (mh) eventually make sure all go routines exited and all channels are cleared time.Sleep(250 * time.Millisecond) n.StepDown() resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) }() } // Request to remove a peer from a clustered stream. func (s *Server) jsStreamRemovePeerRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } // Have extra token for this one. name := tokenAt(subject, 6) var resp = JSApiStreamRemovePeerResponse{ApiResponse: ApiResponse{Type: JSApiStreamRemovePeerResponseType}} // If we are not in clustered mode this is a failed request. if !s.JetStreamIsClustered() { resp.Error = NewJSClusterRequiredError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are here we are clustered. See if we are the stream leader in order to proceed. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, name) js.mu.RUnlock() // Make sure we are meta leader. if !isLeader { return } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var req JSApiStreamRemovePeerRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if req.Peer == _EMPTY_ { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if sa == nil { // No stream present. resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check to see if we are a member of the group and if the group has no leader. // Peers here is a server name, convert to node name. nodeName := getHash(req.Peer) js.mu.RLock() rg := sa.Group isMember := rg.isMember(nodeName) js.mu.RUnlock() // Make sure we are a member. if !isMember { resp.Error = NewJSClusterPeerNotMemberError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are here we have a valid peer member set for removal. if !js.removePeerFromStream(sa, nodeName) { resp.Error = NewJSPeerRemapError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to have the metaleader remove a peer from the system. func (s *Server) jsLeaderServerRemoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } if acc != s.SystemAccount() { return } js, cc := s.getJetStreamCluster() if js == nil || cc == nil || cc.meta == nil { return } // Extra checks here but only leader is listening. js.mu.RLock() isLeader := cc.isLeader() js.mu.RUnlock() if !isLeader { return } var resp = JSApiMetaServerRemoveResponse{ApiResponse: ApiResponse{Type: JSApiMetaServerRemoveResponseType}} if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var req JSApiMetaServerRemoveRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var found string js.mu.RLock() for _, p := range cc.meta.Peers() { // If Peer is specified, it takes precedence if req.Peer != _EMPTY_ { if p.ID == req.Peer { found = req.Peer break } continue } si, ok := s.nodeToInfo.Load(p.ID) if ok && si.(nodeInfo).name == req.Server { found = p.ID break } } js.mu.RUnlock() if found == _EMPTY_ { resp.Error = NewJSClusterServerNotMemberError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // So we have a valid peer. js.mu.Lock() cc.meta.ProposeRemovePeer(found) js.mu.Unlock() resp.Success = true s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } func (s *Server) peerSetToNames(ps []string) []string { names := make([]string, len(ps)) for i := 0; i < len(ps); i++ { if si, ok := s.nodeToInfo.Load(ps[i]); !ok { names[i] = ps[i] } else { names[i] = si.(nodeInfo).name } } return names } // looks up the peer id for a given server name. Cluster and domain name are optional filter criteria func (s *Server) nameToPeer(js *jetStream, serverName, clusterName, domainName string) string { js.mu.RLock() defer js.mu.RUnlock() if cc := js.cluster; cc != nil { for _, p := range cc.meta.Peers() { si, ok := s.nodeToInfo.Load(p.ID) if ok && si.(nodeInfo).name == serverName { if clusterName == _EMPTY_ || clusterName == si.(nodeInfo).cluster { if domainName == _EMPTY_ || domainName == si.(nodeInfo).domain { return p.ID } } } } } return _EMPTY_ } // Request to have the metaleader move a stream on a peer to another func (s *Server) jsLeaderServerStreamMoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } js, cc := s.getJetStreamCluster() if js == nil || cc == nil || cc.meta == nil { return } // Extra checks here but only leader is listening. js.mu.RLock() isLeader := cc.isLeader() js.mu.RUnlock() if !isLeader { return } accName := tokenAt(subject, 6) streamName := tokenAt(subject, 7) if acc.GetName() != accName && acc != s.SystemAccount() { return } var resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}} var req JSApiMetaServerStreamMoveRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } srcPeer := _EMPTY_ if req.Server != _EMPTY_ { srcPeer = s.nameToPeer(js, req.Server, req.Cluster, req.Domain) } targetAcc, ok := s.accounts.Load(accName) if !ok { resp.Error = NewJSNoAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var streamFound bool cfg := StreamConfig{} currPeers := []string{} currCluster := _EMPTY_ js.mu.Lock() streams, ok := cc.streams[accName] if ok { sa, ok := streams[streamName] if ok { cfg = *sa.Config streamFound = true currPeers = sa.Group.Peers currCluster = sa.Group.Cluster } } js.mu.Unlock() if !streamFound { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // if server was picked, make sure src peer exists and move it to first position. // removal will drop peers from the left if req.Server != _EMPTY_ { if srcPeer == _EMPTY_ { resp.Error = NewJSClusterServerNotMemberError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var peerFound bool for i := 0; i < len(currPeers); i++ { if currPeers[i] == srcPeer { copy(currPeers[1:], currPeers[:i]) currPeers[0] = srcPeer peerFound = true break } } if !peerFound { resp.Error = NewJSClusterPeerNotMemberError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } // make sure client is scoped to requested account ciNew := *(ci) ciNew.Account = accName // backup placement such that peers can be looked up with modified tag list var origPlacement *Placement if cfg.Placement != nil { tmp := *cfg.Placement origPlacement = &tmp } if len(req.Tags) > 0 { if cfg.Placement == nil { cfg.Placement = &Placement{} } cfg.Placement.Tags = append(cfg.Placement.Tags, req.Tags...) } peers, e := cc.selectPeerGroup(cfg.Replicas+1, currCluster, &cfg, currPeers, 1, nil) if len(peers) <= cfg.Replicas { // since expanding in the same cluster did not yield a result, try in different cluster peers = nil clusters := map[string]struct{}{} s.nodeToInfo.Range(func(_, ni any) bool { if currCluster != ni.(nodeInfo).cluster { clusters[ni.(nodeInfo).cluster] = struct{}{} } return true }) errs := &selectPeerError{} errs.accumulate(e) for cluster := range clusters { newPeers, e := cc.selectPeerGroup(cfg.Replicas, cluster, &cfg, nil, 0, nil) if len(newPeers) >= cfg.Replicas { peers = append([]string{}, currPeers...) peers = append(peers, newPeers[:cfg.Replicas]...) break } errs.accumulate(e) } if peers == nil { resp.Error = NewJSClusterNoPeersError(errs) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } cfg.Placement = origPlacement s.Noticef("Requested move for stream '%s > %s' R=%d from %+v to %+v", accName, streamName, cfg.Replicas, s.peerSetToNames(currPeers), s.peerSetToNames(peers)) // We will always have peers and therefore never do a callout, therefore it is safe to call inline s.jsClusteredStreamUpdateRequest(&ciNew, targetAcc.(*Account), subject, reply, rmsg, &cfg, peers) } // Request to have the metaleader move a stream on a peer to another func (s *Server) jsLeaderServerStreamCancelMoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } js, cc := s.getJetStreamCluster() if js == nil || cc == nil || cc.meta == nil { return } // Extra checks here but only leader is listening. js.mu.RLock() isLeader := cc.isLeader() js.mu.RUnlock() if !isLeader { return } var resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}} accName := tokenAt(subject, 6) streamName := tokenAt(subject, 7) if acc.GetName() != accName && acc != s.SystemAccount() { return } targetAcc, ok := s.accounts.Load(accName) if !ok { resp.Error = NewJSNoAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } streamFound := false cfg := StreamConfig{} currPeers := []string{} js.mu.Lock() streams, ok := cc.streams[accName] if ok { sa, ok := streams[streamName] if ok { cfg = *sa.Config streamFound = true currPeers = sa.Group.Peers } } js.mu.Unlock() if !streamFound { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if len(currPeers) <= cfg.Replicas { resp.Error = NewJSStreamMoveNotInProgressError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // make sure client is scoped to requested account ciNew := *(ci) ciNew.Account = accName peers := currPeers[:cfg.Replicas] // Remove placement in case tags don't match // This can happen if the move was initiated by modifying the tags. // This is an account operation. // This can NOT happen when the move was initiated by the system account. // There move honors the original tag list. if cfg.Placement != nil && len(cfg.Placement.Tags) != 0 { FOR_TAGCHECK: for _, peer := range peers { si, ok := s.nodeToInfo.Load(peer) if !ok { // can't verify tags, do the safe thing and error resp.Error = NewJSStreamGeneralError( fmt.Errorf("peer %s not present for tag validation", peer)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } nodeTags := si.(nodeInfo).tags for _, tag := range cfg.Placement.Tags { if !nodeTags.Contains(tag) { // clear placement as tags don't match cfg.Placement = nil break FOR_TAGCHECK } } } } s.Noticef("Requested cancel of move: R=%d '%s > %s' to peer set %+v and restore previous peer set %+v", cfg.Replicas, accName, streamName, s.peerSetToNames(currPeers), s.peerSetToNames(peers)) // We will always have peers and therefore never do a callout, therefore it is safe to call inline s.jsClusteredStreamUpdateRequest(&ciNew, targetAcc.(*Account), subject, reply, rmsg, &cfg, peers) } // Request to have an account purged func (s *Server) jsLeaderAccountPurgeRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } if acc != s.SystemAccount() { return } js := s.getJetStream() if js == nil { return } accName := tokenAt(subject, 5) var resp = JSApiAccountPurgeResponse{ApiResponse: ApiResponse{Type: JSApiAccountPurgeResponseType}} if !s.JetStreamIsClustered() { var streams []*stream var ac *Account if ac, err = s.lookupAccount(accName); err == nil && ac != nil { streams = ac.streams() } s.Noticef("Purge request for account %s (streams: %d, hasAccount: %t)", accName, len(streams), ac != nil) for _, mset := range streams { err := mset.delete() if err != nil { resp.Error = NewJSStreamDeleteError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } if err := os.RemoveAll(filepath.Join(js.config.StoreDir, accName)); err != nil { resp.Error = NewJSStreamGeneralError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Initiated = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } _, cc := s.getJetStreamCluster() if cc == nil || cc.meta == nil || !cc.isLeader() { return } if js.isMetaRecovering() { // While in recovery mode, the data structures are not fully initialized resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } js.mu.RLock() ns, nc := 0, 0 streams, hasAccount := cc.streams[accName] for _, osa := range streams { for _, oca := range osa.consumers { oca.deleted = true ca := &consumerAssignment{Group: oca.Group, Stream: oca.Stream, Name: oca.Name, Config: oca.Config, Subject: subject, Client: oca.Client} cc.meta.Propose(encodeDeleteConsumerAssignment(ca)) nc++ } sa := &streamAssignment{Group: osa.Group, Config: osa.Config, Subject: subject, Client: osa.Client} cc.meta.Propose(encodeDeleteStreamAssignment(sa)) ns++ } js.mu.RUnlock() s.Noticef("Purge request for account %s (streams: %d, consumer: %d, hasAccount: %t)", accName, ns, nc, hasAccount) resp.Initiated = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } // Request to have the meta leader stepdown. // These will only be received by the meta leader, so less checking needed. func (s *Server) jsLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } // This should only be coming from the System Account. if acc != s.SystemAccount() { s.RateLimitWarnf("JetStream API stepdown request from non-system account: %q user: %q", ci.serviceAccount(), ci.User) return } js, cc := s.getJetStreamCluster() if js == nil || cc == nil || cc.meta == nil { return } // Extra checks here but only leader is listening. js.mu.RLock() isLeader := cc.isLeader() js.mu.RUnlock() if !isLeader { return } var preferredLeader string var resp = JSApiLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiLeaderStepDownResponseType}} if !isEmptyRequest(msg) { var req JSApiLeaderStepdownRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if req.Placement != nil { if len(req.Placement.Tags) > 0 { // Tags currently not supported. resp.Error = NewJSClusterTagsError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } cn := req.Placement.Cluster var peers []string ourID := cc.meta.ID() for _, p := range cc.meta.Peers() { if si, ok := s.nodeToInfo.Load(p.ID); ok && si != nil { if ni := si.(nodeInfo); ni.offline || ni.cluster != cn || p.ID == ourID { continue } peers = append(peers, p.ID) } } if len(peers) == 0 { resp.Error = NewJSClusterNoPeersError(fmt.Errorf("no replacement peer connected")) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Randomize and select. if len(peers) > 1 { rand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] }) } preferredLeader = peers[0] } } // Call actual stepdown. err = cc.meta.StepDown(preferredLeader) if err != nil { resp.Error = NewJSRaftGeneralError(err, Unless(err)) } else { resp.Success = true } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } func isEmptyRequest(req []byte) bool { if len(req) == 0 { return true } if bytes.Equal(req, []byte("{}")) { return true } // If we are here we didn't get our simple match, but still could be valid. var v any if err := json.Unmarshal(req, &v); err != nil { return false } vm, ok := v.(map[string]any) if !ok { return false } return len(vm) == 0 } // Request to delete a stream. func (s *Server) jsStreamDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if !isEmptyRequest(msg) { resp.Error = NewJSNotEmptyRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } stream := streamNameFromSubject(subject) // Clustered. if s.JetStreamIsClustered() { s.jsClusteredStreamDeleteRequest(ci, acc, stream, subject, reply, msg) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if err := mset.delete(); err != nil { resp.Error = NewJSStreamDeleteError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to delete a message. // This expects a stream sequence number as the msg body. func (s *Server) jsMsgDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } stream := tokenAt(subject, 6) var resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}} // If we are in clustered mode we need to be the stream leader to proceed. if s.JetStreamIsClustered() { // Check to make sure the stream is assigned. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream) js.mu.RUnlock() if isLeader && sa == nil { // We can't find the stream, so mimic what would be the errors below. if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // No stream present. resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { return } // Check to see if we are a member of the group and if the group has no leader. if js.isGroupLeaderless(sa.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // We have the stream assigned and a leader, so only the stream leader should answer. if !acc.JetStreamIsStreamLeader(stream) { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var req JSApiMsgDeleteRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if mset.cfg.Sealed { resp.Error = NewJSStreamSealedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if mset.cfg.DenyDelete { resp.Error = NewJSStreamMsgDeleteFailedError(errors.New("message delete not permitted")) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if s.JetStreamIsClustered() { s.jsClusteredMsgDeleteRequest(ci, acc, mset, stream, subject, reply, &req, rmsg) return } var removed bool if req.NoErase { removed, err = mset.removeMsg(req.Seq) } else { removed, err = mset.eraseMsg(req.Seq) } if err != nil { resp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err)) } else if !removed { resp.Error = NewJSSequenceNotFoundError(req.Seq) } else { resp.Success = true } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to get a raw stream message. func (s *Server) jsMsgGetRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } stream := tokenAt(subject, 6) var resp = JSApiMsgGetResponse{ApiResponse: ApiResponse{Type: JSApiMsgGetResponseType}} // If we are in clustered mode we need to be the stream leader to proceed. if s.JetStreamIsClustered() { // Check to make sure the stream is assigned. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream) js.mu.RUnlock() if isLeader && sa == nil { // We can't find the stream, so mimic what would be the errors below. if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // No stream present. resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { return } // Check to see if we are a member of the group and if the group has no leader. if js.isGroupLeaderless(sa.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // We have the stream assigned and a leader, so only the stream leader should answer. if !acc.JetStreamIsStreamLeader(stream) { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var req JSApiMsgGetRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check that we do not have both options set. if req.Seq > 0 && req.LastFor != _EMPTY_ || req.Seq == 0 && req.LastFor == _EMPTY_ && req.NextFor == _EMPTY_ { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check that both last and next not both set. if req.LastFor != _EMPTY_ && req.NextFor != _EMPTY_ { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var svp StoreMsg var sm *StoreMsg if req.Seq > 0 && req.NextFor == _EMPTY_ { sm, err = mset.store.LoadMsg(req.Seq, &svp) } else if req.NextFor != _EMPTY_ { sm, _, err = mset.store.LoadNextMsg(req.NextFor, subjectHasWildcard(req.NextFor), req.Seq, &svp) } else { sm, err = mset.store.LoadLastMsg(req.LastFor, &svp) } if err != nil { resp.Error = NewJSNoMessageFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Message = &StoredMsg{ Subject: sm.subj, Sequence: sm.seq, Header: sm.hdr, Data: sm.msg, Time: time.Unix(0, sm.ts).UTC(), } // Don't send response through API layer for this call. s.sendInternalAccountMsg(nil, reply, s.jsonResponse(resp)) } // Request to purge a stream. func (s *Server) jsStreamPurgeRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } stream := streamNameFromSubject(subject) var resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}} // If we are in clustered mode we need to be the stream leader to proceed. if s.JetStreamIsClustered() { // Check to make sure the stream is assigned. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.RLock() isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream) js.mu.RUnlock() if isLeader && sa == nil { // We can't find the stream, so mimic what would be the errors below. if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // No stream present. resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if sa == nil { if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } // Check to see if we are a member of the group and if the group has no leader. if js.isGroupLeaderless(sa.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // We have the stream assigned and a leader, so only the stream leader should answer. if !acc.JetStreamIsStreamLeader(stream) { if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var purgeRequest *JSApiStreamPurgeRequest if !isEmptyRequest(msg) { var req JSApiStreamPurgeRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if req.Sequence > 0 && req.Keep > 0 { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } purgeRequest = &req } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if mset.cfg.Sealed { resp.Error = NewJSStreamSealedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if mset.cfg.DenyPurge { resp.Error = NewJSStreamPurgeFailedError(errors.New("stream purge not permitted")) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if s.JetStreamIsClustered() { s.jsClusteredStreamPurgeRequest(ci, acc, mset, stream, subject, reply, rmsg, purgeRequest) return } purged, err := mset.purge(purgeRequest) if err != nil { resp.Error = NewJSStreamGeneralError(err, Unless(err)) } else { resp.Purged = purged resp.Success = true } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } func (acc *Account) jsNonClusteredStreamLimitsCheck(cfg *StreamConfig) *ApiError { var replicas int if cfg != nil { replicas = cfg.Replicas } selectedLimits, tier, jsa, apiErr := acc.selectLimits(replicas) if apiErr != nil { return apiErr } jsa.mu.RLock() defer jsa.mu.RUnlock() if selectedLimits.MaxStreams > 0 && jsa.countStreams(tier, cfg) >= selectedLimits.MaxStreams { return NewJSMaximumStreamsLimitError() } reserved := jsa.tieredReservation(tier, cfg) if err := jsa.js.checkAllLimits(selectedLimits, cfg, reserved, 0); err != nil { return NewJSStreamLimitsError(err, Unless(err)) } return nil } // Request to restore a stream. func (s *Server) jsStreamRestoreRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamIsLeader() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var req JSApiStreamRestoreRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } stream := streamNameFromSubject(subject) if stream != req.Config.Name && req.Config.Name == _EMPTY_ { req.Config.Name = stream } // check stream config at the start of the restore process, not at the end cfg, apiErr := s.checkStreamCfg(&req.Config, acc) if apiErr != nil { resp.Error = apiErr s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if s.JetStreamIsClustered() { s.jsClusteredStreamRestoreRequest(ci, acc, &req, subject, reply, rmsg) return } if err := acc.jsNonClusteredStreamLimitsCheck(&cfg); err != nil { resp.Error = err s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if _, err := acc.lookupStream(stream); err == nil { resp.Error = NewJSStreamNameExistRestoreFailedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } s.processStreamRestore(ci, acc, &req.Config, subject, reply, string(msg)) } func (s *Server) processStreamRestore(ci *ClientInfo, acc *Account, cfg *StreamConfig, subject, reply, msg string) <-chan error { js := s.getJetStream() var resp = JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}} snapDir := filepath.Join(js.config.StoreDir, snapStagingDir) if _, err := os.Stat(snapDir); os.IsNotExist(err) { if err := os.MkdirAll(snapDir, defaultDirPerms); err != nil { resp.Error = &ApiError{Code: 503, Description: "JetStream unable to create temp storage for restore"} s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return nil } } tfile, err := os.CreateTemp(snapDir, "js-restore-") if err != nil { resp.Error = NewJSTempStorageFailedError() s.sendAPIErrResponse(ci, acc, subject, reply, msg, s.jsonResponse(&resp)) return nil } streamName := cfg.Name s.Noticef("Starting restore for stream '%s > %s'", acc.Name, streamName) start := time.Now().UTC() domain := s.getOpts().JetStreamDomain s.publishAdvisory(acc, JSAdvisoryStreamRestoreCreatePre+"."+streamName, &JSRestoreCreateAdvisory{ TypedEvent: TypedEvent{ Type: JSRestoreCreateAdvisoryType, ID: nuid.Next(), Time: start, }, Stream: streamName, Client: ci.forAdvisory(), Domain: domain, }) // Create our internal subscription to accept the snapshot. restoreSubj := fmt.Sprintf(jsRestoreDeliverT, streamName, nuid.Next()) type result struct { err error reply string } // For signaling to upper layers. resultCh := make(chan result, 1) activeQ := newIPQueue[int](s, fmt.Sprintf("[ACC:%s] stream '%s' restore", acc.Name, streamName)) // of int var total int // FIXME(dlc) - Probably take out of network path eventually due to disk I/O? processChunk := func(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { // We require reply subjects to communicate back failures, flow etc. If they do not have one log and cancel. if reply == _EMPTY_ { sub.client.processUnsub(sub.sid) resultCh <- result{ fmt.Errorf("restore for stream '%s > %s' requires reply subject for each chunk", acc.Name, streamName), reply, } return } // Account client messages have \r\n on end. This is an error. if len(msg) < LEN_CR_LF { sub.client.processUnsub(sub.sid) resultCh <- result{ fmt.Errorf("restore for stream '%s > %s' received short chunk", acc.Name, streamName), reply, } return } // Adjust. msg = msg[:len(msg)-LEN_CR_LF] // This means we are complete with our transfer from the client. if len(msg) == 0 { s.Debugf("Finished staging restore for stream '%s > %s'", acc.Name, streamName) resultCh <- result{err, reply} return } // We track total and check on server limits. // TODO(dlc) - We could check apriori and cancel initial request if we know it won't fit. total += len(msg) if js.wouldExceedLimits(FileStorage, total) { s.resourcesExceededError() resultCh <- result{NewJSInsufficientResourcesError(), reply} return } // Append chunk to temp file. Mark as issue if we encounter an error. if n, err := tfile.Write(msg); n != len(msg) || err != nil { resultCh <- result{err, reply} if reply != _EMPTY_ { s.sendInternalAccountMsg(acc, reply, "-ERR 'storage failure during restore'") } return } activeQ.push(len(msg)) s.sendInternalAccountMsg(acc, reply, nil) } sub, err := acc.subscribeInternal(restoreSubj, processChunk) if err != nil { tfile.Close() os.Remove(tfile.Name()) resp.Error = NewJSRestoreSubscribeFailedError(err, restoreSubj) s.sendAPIErrResponse(ci, acc, subject, reply, msg, s.jsonResponse(&resp)) return nil } // Mark the subject so the end user knows where to send the snapshot chunks. resp.DeliverSubject = restoreSubj s.sendAPIResponse(ci, acc, subject, reply, msg, s.jsonResponse(resp)) doneCh := make(chan error, 1) // Monitor the progress from another Go routine. s.startGoRoutine(func() { defer s.grWG.Done() defer func() { tfile.Close() os.Remove(tfile.Name()) sub.client.processUnsub(sub.sid) activeQ.unregister() }() const activityInterval = 5 * time.Second notActive := time.NewTimer(activityInterval) defer notActive.Stop() total := 0 for { select { case result := <-resultCh: err := result.err var mset *stream // If we staged properly go ahead and do restore now. if err == nil { s.Debugf("Finalizing restore for stream '%s > %s'", acc.Name, streamName) tfile.Seek(0, 0) mset, err = acc.RestoreStream(cfg, tfile) } else { errStr := err.Error() tmp := []rune(errStr) tmp[0] = unicode.ToUpper(tmp[0]) s.Warnf(errStr) } end := time.Now().UTC() // TODO(rip) - Should this have the error code in it?? s.publishAdvisory(acc, JSAdvisoryStreamRestoreCompletePre+"."+streamName, &JSRestoreCompleteAdvisory{ TypedEvent: TypedEvent{ Type: JSRestoreCompleteAdvisoryType, ID: nuid.Next(), Time: end, }, Stream: streamName, Start: start, End: end, Bytes: int64(total), Client: ci.forAdvisory(), Domain: domain, }) var resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}} if err != nil { resp.Error = NewJSStreamRestoreError(err, Unless(err)) s.Warnf("Restore failed for %s for stream '%s > %s' in %v", friendlyBytes(int64(total)), acc.Name, streamName, end.Sub(start)) } else { resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), TimeStamp: time.Now().UTC(), } s.Noticef("Completed restore of %s for stream '%s > %s' in %v", friendlyBytes(int64(total)), acc.Name, streamName, end.Sub(start).Round(time.Millisecond)) } // On the last EOF, send back the stream info or error status. s.sendInternalAccountMsg(acc, result.reply, s.jsonResponse(&resp)) // Signal to the upper layers. doneCh <- err return case <-activeQ.ch: if n, ok := activeQ.popOne(); ok { total += n notActive.Reset(activityInterval) } case <-notActive.C: err := fmt.Errorf("restore for stream '%s > %s' is stalled", acc, streamName) doneCh <- err return } } }) return doneCh } // Process a snapshot request. func (s *Server) jsStreamSnapshotRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } smsg := string(msg) stream := streamNameFromSubject(subject) // If we are in clustered mode we need to be the stream leader to proceed. if s.JetStreamIsClustered() && !acc.JetStreamIsStreamLeader(stream) { return } var resp = JSApiStreamSnapshotResponse{ApiResponse: ApiResponse{Type: JSApiStreamSnapshotResponseType}} if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } if isEmptyRequest(msg) { resp.Error = NewJSBadRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) } return } var req JSApiStreamSnapshotRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } if !IsValidSubject(req.DeliverSubject) { resp.Error = NewJSSnapshotDeliverSubjectInvalidError() s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } // We will do the snapshot in a go routine as well since check msgs may // stall this go routine. go func() { if req.CheckMsgs { s.Noticef("Starting health check and snapshot for stream '%s > %s'", mset.jsa.account.Name, mset.name()) } else { s.Noticef("Starting snapshot for stream '%s > %s'", mset.jsa.account.Name, mset.name()) } start := time.Now().UTC() sr, err := mset.snapshot(0, req.CheckMsgs, !req.NoConsumers) if err != nil { s.Warnf("Snapshot of stream '%s > %s' failed: %v", mset.jsa.account.Name, mset.name(), err) resp.Error = NewJSStreamSnapshotError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp)) return } config := mset.config() resp.State = &sr.State resp.Config = &config s.sendAPIResponse(ci, acc, subject, reply, smsg, s.jsonResponse(resp)) s.publishAdvisory(acc, JSAdvisoryStreamSnapshotCreatePre+"."+mset.name(), &JSSnapshotCreateAdvisory{ TypedEvent: TypedEvent{ Type: JSSnapshotCreatedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: mset.name(), State: sr.State, Client: ci.forAdvisory(), Domain: s.getOpts().JetStreamDomain, }) // Now do the real streaming. s.streamSnapshot(acc, mset, sr, &req) end := time.Now().UTC() s.publishAdvisory(acc, JSAdvisoryStreamSnapshotCompletePre+"."+mset.name(), &JSSnapshotCompleteAdvisory{ TypedEvent: TypedEvent{ Type: JSSnapshotCompleteAdvisoryType, ID: nuid.Next(), Time: end, }, Stream: mset.name(), Start: start, End: end, Client: ci.forAdvisory(), Domain: s.getOpts().JetStreamDomain, }) s.Noticef("Completed snapshot of %s for stream '%s > %s' in %v", friendlyBytes(int64(sr.State.Bytes)), mset.jsa.account.Name, mset.name(), end.Sub(start)) }() } // Default chunk size for now. const defaultSnapshotChunkSize = 128 * 1024 const defaultSnapshotWindowSize = 8 * 1024 * 1024 // 8MB // streamSnapshot will stream out our snapshot to the reply subject. func (s *Server) streamSnapshot(acc *Account, mset *stream, sr *SnapshotResult, req *JSApiStreamSnapshotRequest) { chunkSize := req.ChunkSize if chunkSize == 0 { chunkSize = defaultSnapshotChunkSize } // Setup for the chunk stream. reply := req.DeliverSubject r := sr.Reader defer r.Close() // Check interest for the snapshot deliver subject. inch := make(chan bool, 1) acc.sl.RegisterNotification(req.DeliverSubject, inch) defer acc.sl.ClearNotification(req.DeliverSubject, inch) hasInterest := <-inch if !hasInterest { // Allow 2 seconds or so for interest to show up. select { case <-inch: case <-time.After(2 * time.Second): } } // Create our ack flow handler. // This is very simple for now. ackSize := defaultSnapshotWindowSize / chunkSize if ackSize < 8 { ackSize = 8 } else if ackSize > 8*1024 { ackSize = 8 * 1024 } acks := make(chan struct{}, ackSize) acks <- struct{}{} // Track bytes outstanding. var out int32 // We will place sequence number and size of chunk sent in the reply. ackSubj := fmt.Sprintf(jsSnapshotAckT, mset.name(), nuid.Next()) ackSub, _ := mset.subscribeInternal(ackSubj+".>", func(_ *subscription, _ *client, _ *Account, subject, _ string, _ []byte) { cs, _ := strconv.Atoi(tokenAt(subject, 6)) // This is very crude and simple, but ok for now. // This only matters when sending multiple chunks. if atomic.AddInt32(&out, int32(-cs)) < defaultSnapshotWindowSize { select { case acks <- struct{}{}: default: } } }) defer mset.unsubscribe(ackSub) // TODO(dlc) - Add in NATS-Chunked-Sequence header var hdr []byte for index := 1; ; index++ { chunk := make([]byte, chunkSize) n, err := r.Read(chunk) chunk = chunk[:n] if err != nil { if n > 0 { mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, chunk, nil, 0)) } break } // Wait on acks for flow control if past our window size. // Wait up to 10ms for now if no acks received. if atomic.LoadInt32(&out) > defaultSnapshotWindowSize { select { case <-acks: // ok to proceed. case <-inch: // Lost interest hdr = []byte("NATS/1.0 408 No Interest\r\n\r\n") goto done case <-time.After(2 * time.Second): hdr = []byte("NATS/1.0 408 No Flow Response\r\n\r\n") goto done } } ackReply := fmt.Sprintf("%s.%d.%d", ackSubj, len(chunk), index) if hdr == nil { hdr = []byte("NATS/1.0 204\r\n\r\n") } mset.outq.send(newJSPubMsg(reply, _EMPTY_, ackReply, nil, chunk, nil, 0)) atomic.AddInt32(&out, int32(len(chunk))) } done: // Send last EOF // TODO(dlc) - place hash in header mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) } // For determining consumer request type. type ccReqType uint8 const ( ccNew = iota ccLegacyEphemeral ccLegacyDurable ) // Request to create a consumer where stream and optional consumer name are part of the subject, and optional // filtered subjects can be at the tail end. // Assumes stream and consumer names are single tokens. func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}} var req CreateConsumerRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } var js *jetStream isClustered := s.JetStreamIsClustered() // Determine if we should proceed here when we are in clustered mode. if isClustered { if req.Config.Direct { // Check to see if we have this stream and are the stream leader. if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) { return } } else { var cc *jetStreamCluster js, cc = s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } } var streamName, consumerName, filteredSubject string var rt ccReqType if n := numTokens(subject); n < 5 { s.Warnf(badAPIRequestT, msg) return } else if n == 5 { // Legacy ephemeral. rt = ccLegacyEphemeral streamName = streamNameFromSubject(subject) } else { // New style and durable legacy. if tokenAt(subject, 4) == "DURABLE" { rt = ccLegacyDurable if n != 7 { resp.Error = NewJSConsumerDurableNameNotInSubjectError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } streamName = tokenAt(subject, 6) consumerName = tokenAt(subject, 7) } else { streamName = streamNameFromSubject(subject) consumerName = consumerNameFromSubject(subject) // New has optional filtered subject as part of main subject.. if n > 6 { tokens := strings.Split(subject, tsep) filteredSubject = strings.Join(tokens[6:], tsep) } } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if streamName != req.Stream { resp.Error = NewJSStreamMismatchError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if consumerName != _EMPTY_ { // Check for path like separators in the name. if strings.ContainsAny(consumerName, `\/`) { resp.Error = NewJSConsumerNameContainsPathSeparatorsError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } // Should we expect a durable name if rt == ccLegacyDurable { if numTokens(subject) < 7 { resp.Error = NewJSConsumerDurableNameNotInSubjectError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Now check on requirements for durable request. if req.Config.Durable == _EMPTY_ { resp.Error = NewJSConsumerDurableNameNotSetError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if consumerName != req.Config.Durable { resp.Error = NewJSConsumerDurableNameNotMatchSubjectError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } // If new style and durable set make sure they match. if rt == ccNew { if req.Config.Durable != _EMPTY_ { if consumerName != req.Config.Durable { resp.Error = NewJSConsumerDurableNameNotMatchSubjectError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } } // New style ephemeral so we need to honor the name. req.Config.Name = consumerName } // Check for legacy ephemeral mis-configuration. if rt == ccLegacyEphemeral && req.Config.Durable != _EMPTY_ { resp.Error = NewJSConsumerEphemeralWithDurableNameError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // in case of multiple filters provided, error if new API is used. if filteredSubject != _EMPTY_ && len(req.Config.FilterSubjects) != 0 { resp.Error = NewJSConsumerMultipleFiltersNotAllowedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Check for a filter subject. if filteredSubject != _EMPTY_ && req.Config.FilterSubject != filteredSubject { resp.Error = NewJSConsumerCreateFilterSubjectMismatchError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if isClustered && !req.Config.Direct { // If we are inline with client, we still may need to do a callout for consumer info // during this call, so place in Go routine to not block client. // Router and Gateway API calls already in separate context. if c.kind != ROUTER && c.kind != GATEWAY { go s.jsClusteredConsumerRequest(ci, acc, subject, reply, rmsg, req.Stream, &req.Config, req.Action) } else { s.jsClusteredConsumerRequest(ci, acc, subject, reply, rmsg, req.Stream, &req.Config, req.Action) } return } // If we are here we are single server mode. if req.Config.Replicas > 1 { resp.Error = NewJSStreamReplicasNotSupportedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } stream, err := acc.lookupStream(req.Stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } o, err := stream.addConsumerWithAction(&req.Config, req.Action) if err != nil { if IsNatsErr(err, JSConsumerStoreFailedErrF) { cname := req.Config.Durable // Will be empty if ephemeral. s.Warnf("Consumer create failed for '%s > %s > %s': %v", acc, req.Stream, cname, err) err = errConsumerStoreFailed } resp.Error = NewJSConsumerCreateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.ConsumerInfo = o.initialInfo() s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for the list of all consumer names. func (s *Server) jsConsumerNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiConsumerNamesResponse{ ApiResponse: ApiResponse{Type: JSApiConsumerNamesResponseType}, Consumers: []string{}, } // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var offset int if !isEmptyRequest(msg) { var req JSApiConsumersRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } offset = req.Offset } streamName := streamNameFromSubject(subject) var numConsumers int if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { // TODO(dlc) - Debug or Warn? return } js.mu.RLock() sas := cc.streams[acc.Name] if sas == nil { js.mu.RUnlock() resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } sa := sas[streamName] if sa == nil || sa.err != nil { js.mu.RUnlock() resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } for consumer := range sa.consumers { resp.Consumers = append(resp.Consumers, consumer) } if len(resp.Consumers) > 1 { slices.Sort(resp.Consumers) } numConsumers = len(resp.Consumers) if offset > numConsumers { offset = numConsumers } resp.Consumers = resp.Consumers[offset:] if len(resp.Consumers) > JSApiNamesLimit { resp.Consumers = resp.Consumers[:JSApiNamesLimit] } js.mu.RUnlock() } else { mset, err := acc.lookupStream(streamName) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } obs := mset.getPublicConsumers() slices.SortFunc(obs, func(i, j *consumer) int { return cmp.Compare(i.name, j.name) }) numConsumers = len(obs) if offset > numConsumers { offset = numConsumers } for _, o := range obs[offset:] { resp.Consumers = append(resp.Consumers, o.String()) if len(resp.Consumers) >= JSApiNamesLimit { break } } } resp.Total = numConsumers resp.Limit = JSApiNamesLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for the list of all detailed consumer information. func (s *Server) jsConsumerListRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiConsumerListResponse{ ApiResponse: ApiResponse{Type: JSApiConsumerListResponseType}, Consumers: []*ConsumerInfo{}, } // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } var offset int if !isEmptyRequest(msg) { var req JSApiConsumersRequest if err := json.Unmarshal(msg, &req); err != nil { resp.Error = NewJSInvalidJSONError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } offset = req.Offset } streamName := streamNameFromSubject(subject) // Clustered mode will invoke a scatter and gather. if s.JetStreamIsClustered() { // Need to copy these off before sending.. don't move this inside startGoRoutine!!! msg = copyBytes(msg) s.startGoRoutine(func() { s.jsClusteredConsumerListRequest(acc, ci, offset, streamName, subject, reply, msg) }) return } mset, err := acc.lookupStream(streamName) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } obs := mset.getPublicConsumers() slices.SortFunc(obs, func(i, j *consumer) int { return cmp.Compare(i.name, j.name) }) ocnt := len(obs) if offset > ocnt { offset = ocnt } for _, o := range obs[offset:] { if cinfo := o.info(); cinfo != nil { resp.Consumers = append(resp.Consumers, cinfo) } if len(resp.Consumers) >= JSApiListLimit { break } } resp.Total = ocnt resp.Limit = JSApiListLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request for information about an consumer. func (s *Server) jsConsumerInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } streamName := streamNameFromSubject(subject) consumerName := consumerNameFromSubject(subject) var resp = JSApiConsumerInfoResponse{ApiResponse: ApiResponse{Type: JSApiConsumerInfoResponseType}} if !isEmptyRequest(msg) { resp.Error = NewJSNotEmptyRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are in clustered mode we need to be the consumer leader to proceed. if s.JetStreamIsClustered() { // Check to make sure the consumer is assigned. js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.RLock() meta := cc.meta js.mu.RUnlock() // Since these could wait on the Raft group lock, don't do so under the JS lock. ourID := meta.ID() groupLeaderless := meta.Leaderless() groupCreated := meta.Created() js.mu.RLock() isLeader, sa, ca := cc.isLeader(), js.streamAssignment(acc.Name, streamName), js.consumerAssignment(acc.Name, streamName, consumerName) var rg *raftGroup var offline, isMember bool if ca != nil { if rg = ca.Group; rg != nil { offline = s.allPeersOffline(rg) isMember = rg.isMember(ourID) } } // Capture consumer leader here. isConsumerLeader := cc.isConsumerLeader(acc.Name, streamName, consumerName) // Also capture if we think there is no meta leader. var isLeaderLess bool if !isLeader { isLeaderLess = groupLeaderless && time.Since(groupCreated) > lostQuorumIntervalDefault } js.mu.RUnlock() if isLeader && ca == nil { // We can't find the consumer, so mimic what would be the errors below. if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if sa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // If we are here the consumer is not present. resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } else if ca == nil { if isLeaderLess { resp.Error = NewJSClusterNotAvailError() // Delaying an error response gives the leader a chance to respond before us s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil) } return } else if isLeader && offline { resp.Error = NewJSConsumerOfflineError() s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil) return } // Check to see if we are a member of the group and if the group has no leader. if isMember && js.isGroupLeaderless(ca.Group) { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // We have the consumer assigned and a leader, so only the consumer leader should answer. if !isConsumerLeader { if isLeaderLess { resp.Error = NewJSClusterNotAvailError() // Delaying an error response gives the leader a chance to respond before us s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), ca.Group) return } var node RaftNode var leaderNotPartOfGroup bool // We have a consumer assignment. if isMember { js.mu.RLock() if rg.node != nil { node = rg.node if gl := node.GroupLeader(); gl != _EMPTY_ && !rg.isMember(gl) { leaderNotPartOfGroup = true } } js.mu.RUnlock() } // Check if we should ignore all together. if node == nil { // We have been assigned but have not created a node yet. If we are a member return // our config and defaults for state and no cluster info. if isMember { // Since we access consumerAssignment, need js lock. js.mu.RLock() resp.ConsumerInfo = &ConsumerInfo{ Stream: ca.Stream, Name: ca.Name, Created: ca.Created, Config: ca.Config, TimeStamp: time.Now().UTC(), } b := s.jsonResponse(resp) js.mu.RUnlock() s.sendAPIResponse(ci, acc, subject, reply, string(msg), b) } return } // If we are a member and we have a group leader or we had a previous leader consider bailing out. if !node.Leaderless() || node.HadPreviousLeader() { if leaderNotPartOfGroup { resp.Error = NewJSConsumerOfflineError() s.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil) } return } // If we are here we are a member and this is just a new consumer that does not have a leader yet. // Will fall through and return what we have. All consumers can respond but this should be very rare // but makes more sense to clients when they try to create, get a consumer exists, and then do consumer info. } } if !acc.JetStreamEnabled() { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } mset, err := acc.lookupStream(streamName) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } obs := mset.lookupConsumer(consumerName) if obs == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if resp.ConsumerInfo = obs.info(); resp.ConsumerInfo == nil { // This consumer returned nil which means it's closed. Respond with not found. resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // Request to delete an Consumer. func (s *Server) jsConsumerDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if c == nil || !s.JetStreamEnabled() { return } ci, acc, _, msg, err := s.getRequestInfo(c, rmsg) if err != nil { s.Warnf(badAPIRequestT, msg) return } var resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}} // Determine if we should proceed here when we are in clustered mode. if s.JetStreamIsClustered() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } if js.isLeaderless() { resp.Error = NewJSClusterNotAvailError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } // Make sure we are meta leader. if !s.JetStreamIsLeader() { return } } if hasJS, doErr := acc.checkJetStream(); !hasJS { if doErr { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) } return } if !isEmptyRequest(msg) { resp.Error = NewJSNotEmptyRequestError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } stream := streamNameFromSubject(subject) consumer := consumerNameFromSubject(subject) if s.JetStreamIsClustered() { s.jsClusteredConsumerDeleteRequest(ci, acc, stream, consumer, subject, reply, rmsg) return } mset, err := acc.lookupStream(stream) if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } obs := mset.lookupConsumer(consumer) if obs == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } if err := obs.delete(); err != nil { resp.Error = NewJSStreamGeneralError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp)) return } resp.Success = true s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp)) } // sendJetStreamAPIAuditAdvisor will send the audit event for a given event. func (s *Server) sendJetStreamAPIAuditAdvisory(ci *ClientInfo, acc *Account, subject, request, response string) { s.publishAdvisory(acc, JSAuditAdvisory, JSAPIAudit{ TypedEvent: TypedEvent{ Type: JSAPIAuditType, ID: nuid.Next(), Time: time.Now().UTC(), }, Server: s.Name(), Client: ci.forAdvisory(), Subject: subject, Request: request, Response: response, Domain: s.getOpts().JetStreamDomain, }) } nats-server-2.10.27/server/jetstream_benchmark_test.go000066400000000000000000001351011477524627100230600ustar00rootroot00000000000000// Copyright 2023-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 // +build !skip_js_tests,!skip_js_cluster_tests,!skip_js_cluster_tests_2 package server import ( "fmt" "math/rand" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats-server/v2/internal/fastrand" "github.com/nats-io/nats.go" ) func BenchmarkJetStreamConsume(b *testing.B) { const ( verbose = false streamName = "S" subject = "s" seed = 12345 publishTimeout = 30 * time.Second PublishBatchSize = 10000 ) runSyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string) (int, int, int) { const nextMsgTimeout = 3 * time.Second subOpts := []nats.SubOpt{ nats.BindStream(streamName), } sub, err := js.SubscribeSync(_EMPTY_, subOpts...) if err != nil { b.Fatalf("Failed to subscribe: %v", err) } defer sub.Unsubscribe() bitset := NewBitset(uint64(b.N)) uniqueConsumed, duplicates, errors := 0, 0, 0 b.ResetTimer() for uniqueConsumed < b.N { msg, err := sub.NextMsg(nextMsgTimeout) if err != nil { b.Fatalf("No more messages (received: %d/%d)", uniqueConsumed, b.N) } metadata, mdErr := msg.Metadata() if mdErr != nil { errors++ continue } ackErr := msg.Ack() if ackErr != nil { errors++ continue } seq := metadata.Sequence.Stream index := seq - 1 if bitset.get(index) { duplicates++ continue } uniqueConsumed++ bitset.set(index, true) if verbose && uniqueConsumed%1000 == 0 { b.Logf("Consumed: %d/%d", bitset.count(), b.N) } } b.StopTimer() return uniqueConsumed, duplicates, errors } runAsyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, ordered, durable bool) (int, int, int) { const timeout = 3 * time.Minute bitset := NewBitset(uint64(b.N)) doneCh := make(chan bool, 1) uniqueConsumed, duplicates, errors := 0, 0, 0 handleMsg := func(msg *nats.Msg) { metadata, mdErr := msg.Metadata() if mdErr != nil { // fmt.Printf("Metadata error: %v\n", mdErr) errors++ return } // Ordered defaults to AckNone policy, don't try to ACK if !ordered { ackErr := msg.Ack() if ackErr != nil { // fmt.Printf("Ack error: %v\n", ackErr) errors++ return } } seq := metadata.Sequence.Stream index := seq - 1 if bitset.get(index) { duplicates++ return } uniqueConsumed++ bitset.set(index, true) if uniqueConsumed == b.N { msg.Sub.Unsubscribe() doneCh <- true } if verbose && uniqueConsumed%1000 == 0 { b.Logf("Consumed %d/%d", uniqueConsumed, b.N) } } subOpts := []nats.SubOpt{ nats.BindStream(streamName), } if ordered { subOpts = append(subOpts, nats.OrderedConsumer()) } if durable { subOpts = append(subOpts, nats.Durable("c")) } sub, err := js.Subscribe(_EMPTY_, handleMsg, subOpts...) if err != nil { b.Fatalf("Failed to subscribe: %v", err) } defer sub.Unsubscribe() b.ResetTimer() select { case <-doneCh: b.StopTimer() case <-time.After(timeout): b.Fatalf("Timeout, %d/%d received, %d errors", uniqueConsumed, b.N, errors) } return uniqueConsumed, duplicates, errors } runPullConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, durable bool) (int, int, int) { const fetchMaxWait = nats.MaxWait(3 * time.Second) const fetchMaxMessages = 1000 bitset := NewBitset(uint64(b.N)) uniqueConsumed, duplicates, errors := 0, 0, 0 subOpts := []nats.SubOpt{ nats.BindStream(streamName), } consumerName := _EMPTY_ // Default ephemeral if durable { consumerName = "c" // Durable } sub, err := js.PullSubscribe("", consumerName, subOpts...) if err != nil { b.Fatalf("Failed to subscribe: %v", err) } defer sub.Unsubscribe() b.ResetTimer() fetchLoop: for { msgs, err := sub.Fetch(fetchMaxMessages, fetchMaxWait) if err != nil { b.Fatalf("Failed to fetch: %v", err) } processMsgsLoop: for _, msg := range msgs { metadata, mdErr := msg.Metadata() if mdErr != nil { errors++ continue processMsgsLoop } ackErr := msg.Ack() if ackErr != nil { errors++ continue processMsgsLoop } seq := metadata.Sequence.Stream index := seq - 1 if bitset.get(index) { duplicates++ continue processMsgsLoop } uniqueConsumed++ bitset.set(index, true) if uniqueConsumed == b.N { msg.Sub.Unsubscribe() break fetchLoop } if verbose && uniqueConsumed%1000 == 0 { b.Logf("Consumed %d/%d", uniqueConsumed, b.N) } } } b.StopTimer() return uniqueConsumed, duplicates, errors } type ConsumerType string const ( PushSync ConsumerType = "PUSH[Sync,Ephemeral]" PushAsync ConsumerType = "PUSH[Async,Ephemeral]" PushAsyncOrdered ConsumerType = "PUSH[Async,Ordered]" PushAsyncDurable ConsumerType = "PUSH[Async,Durable]" PullDurable ConsumerType = "PULL[Durable]" PullEphemeral ConsumerType = "PULL[Ephemeral]" ) benchmarksCases := []struct { clusterSize int replicas int messageSize int minMessages int }{ {1, 1, 10, 100_000}, // Single node, 10B messages, ~1MiB minimum {1, 1, 1024, 1_000}, // Single node, 1KB messages, ~1MiB minimum {3, 3, 10, 100_000}, // Cluster, R3, 10B messages, ~1MiB minimum {3, 3, 1024, 1_000}, // Cluster, R3, 1KB messages, ~1MiB minimum } //Each of the cases above is run with each of the consumer types consumerTypes := []ConsumerType{ PushSync, PushAsync, PushAsyncOrdered, PushAsyncDurable, PullDurable, PullEphemeral, } for _, bc := range benchmarksCases { name := fmt.Sprintf( "N=%d,R=%d,MsgSz=%db", bc.clusterSize, bc.replicas, bc.messageSize, ) b.Run( name, func(b *testing.B) { for _, ct := range consumerTypes { name := fmt.Sprintf( "%v", ct, ) b.Run( name, func(b *testing.B) { // Skip short runs, benchmark gets re-executed with a larger N if b.N < bc.minMessages { b.ResetTimer() return } if verbose { b.Logf("Running %s with %d messages", name, b.N) } if verbose { b.Logf("Setting up %d nodes", bc.clusterSize) } cl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) defer shutdown() defer nc.Close() if verbose { b.Logf("Creating stream with R=%d", bc.replicas) } streamConfig := &nats.StreamConfig{ Name: streamName, Subjects: []string{subject}, Replicas: bc.replicas, } if _, err := js.AddStream(streamConfig); err != nil { b.Fatalf("Error creating stream: %v", err) } // If replicated resource, connect to stream leader for lower variability if bc.replicas > 1 { connectURL := cl.streamLeader("$G", streamName).ClientURL() nc.Close() _, js = jsClientConnectURL(b, connectURL) } message := make([]byte, bc.messageSize) rand.New(rand.NewSource(int64(seed))).Read(message) // Publish b.N messages to the stream (in batches) for i := 1; i <= b.N; i++ { fastRandomMutation(message, 10) _, err := js.PublishAsync(subject, message) if err != nil { b.Fatalf("Failed to publish: %s", err) } // Limit outstanding published messages to PublishBatchSize if i%PublishBatchSize == 0 || i == b.N { select { case <-js.PublishAsyncComplete(): if verbose { b.Logf("Published %d/%d messages", i, b.N) } case <-time.After(publishTimeout): b.Fatalf("Publish timed out") } } } // Set size of each operation, for throughput calculation b.SetBytes(int64(bc.messageSize)) // Discard time spent during setup // Consumer may reset again further in b.ResetTimer() var consumed, duplicates, errors int const ( ordered = true unordered = false durable = true ephemeral = false ) switch ct { case PushSync: consumed, duplicates, errors = runSyncPushConsumer(b, js, streamName) case PushAsync: consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, ephemeral) case PushAsyncOrdered: consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, ordered, ephemeral) case PushAsyncDurable: consumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, durable) case PullDurable: consumed, duplicates, errors = runPullConsumer(b, js, streamName, durable) case PullEphemeral: consumed, duplicates, errors = runPullConsumer(b, js, streamName, ephemeral) default: b.Fatalf("Unknown consumer type: %v", ct) } // Benchmark ends here, (consumer may have stopped earlier) b.StopTimer() if consumed != b.N { b.Fatalf("Something doesn't add up: %d != %d", consumed, b.N) } b.ReportMetric(float64(duplicates)*100/float64(b.N), "%dupe") b.ReportMetric(float64(errors)*100/float64(b.N), "%error") }, ) } }, ) } } func BenchmarkJetStreamConsumeWithFilters(b *testing.B) { const ( verbose = false streamName = "S" subjectPrefix = "s" seed = 123456 messageSize = 32 consumerReplicas = 1 domainNameLength = 36 // Length of domain portion of subject, must be an even number publishBatchSize = 1000 publishTimeout = 10 * time.Second ) clusterSizeCases := []struct { clusterSize int // Single node or cluster replicas int // Stream replicas storage nats.StorageType // Stream storage }{ {1, 1, nats.MemoryStorage}, {3, 3, nats.MemoryStorage}, {1, 1, nats.FileStorage}, {3, 3, nats.FileStorage}, } benchmarksCases := []struct { domains int // Number of distinct domains subjectsPerDomain int // Number of distinct subjects within each domain filters int // Number of filters (..>) per consumer concurrentConsumers int // Number of consumer running }{ {100, 10, 5, 12}, {1000, 10, 25, 12}, {10_000, 10, 50, 12}, } for _, cs := range clusterSizeCases { name := fmt.Sprintf( "N=%d,R=%d,storage=%s", cs.clusterSize, cs.replicas, cs.storage.String(), ) b.Run( name, func(b *testing.B) { for _, bc := range benchmarksCases { name := fmt.Sprintf( "D=%d,DS=%d,F=%d,C=%d", bc.domains, bc.subjectsPerDomain, bc.filters, bc.concurrentConsumers, ) b.Run( name, func(b *testing.B) { cl, s, shutdown, nc, js := startJSClusterAndConnect(b, cs.clusterSize) defer shutdown() defer nc.Close() if verbose { b.Logf("Creating stream with R=%d", cs.replicas) } streamConfig := &nats.StreamConfig{ Name: streamName, Subjects: []string{subjectPrefix + ".>"}, Storage: cs.storage, Retention: nats.LimitsPolicy, MaxAge: time.Hour, Duplicates: 10 * time.Second, Discard: nats.DiscardOld, NoAck: false, MaxMsgs: -1, MaxBytes: -1, MaxConsumers: -1, Replicas: 1, MaxMsgsPerSubject: 1, } if _, err := js.AddStream(streamConfig); err != nil { b.Fatalf("Error creating stream: %v", err) } // If replicated resource, connect to stream leader for lower variability connectURL := s.ClientURL() if cs.replicas > 1 { connectURL = cl.streamLeader("$G", streamName).ClientURL() nc.Close() _, js = jsClientConnectURL(b, connectURL) } rng := rand.New(rand.NewSource(int64(seed))) message := make([]byte, messageSize) domain := make([]byte, domainNameLength/2) domains := make([]string, 0, bc.domains*bc.subjectsPerDomain) // Publish one message per subject for each domain published := 0 totalMessages := bc.domains * bc.subjectsPerDomain for d := 1; d <= bc.domains; d++ { rng.Read(domain) for s := 1; s <= bc.subjectsPerDomain; s++ { rng.Read(message) domainString := fmt.Sprintf("%X", domain) domains = append(domains, domainString) subject := fmt.Sprintf("%s.%s.%d", subjectPrefix, domainString, s) _, err := js.PublishAsync(subject, message) if err != nil { b.Fatalf("failed to publish: %s", err) } published += 1 // Wait for all pending to be published before trying to publish the next batch if published%publishBatchSize == 0 || published == totalMessages { select { case <-js.PublishAsyncComplete(): if verbose { b.Logf("Published %d/%d messages", published, totalMessages) } case <-time.After(publishTimeout): b.Fatalf("Publish timed out") } } } } // Number of messages that each new consumer expects to consume messagesPerIteration := bc.filters * bc.subjectsPerDomain // Each call to 'subscribe_consume_unsubscribe' is one benchmark operation. // i.e. subscribe_consume_unsubscribe will be called a total of b.N times (split among C threads) // Each operation consists of: // - Create filter // - Create consumer / Subscribe // - Consume expected number of messages // - Unsubscribe subscribeConsumeUnsubscribe := func(js nats.JetStreamContext, rng *rand.Rand) { // Select F unique domains to create F non-overlapping filters filterDomains := make(map[string]bool, bc.filters) filters := make([]string, 0, bc.filters) for len(filterDomains) < bc.filters { domain := domains[rng.Intn(len(domains))] if _, found := filterDomains[domain]; found { // Collision with existing filter, try again continue } filterDomains[domain] = true filters = append(filters, fmt.Sprintf("%s.%s.>", subjectPrefix, domain)) } if verbose { b.Logf("Subscribe with filters: %+v", filters) } // Consumer callback received := 0 consumeWg := sync.WaitGroup{} consumeWg.Add(1) cb := func(msg *nats.Msg) { received += 1 if received == messagesPerIteration { consumeWg.Done() if verbose { b.Logf("Received %d/%d messages", received, messagesPerIteration) } } } // Create consumer subOpts := []nats.SubOpt{ nats.BindStream(streamName), nats.OrderedConsumer(), nats.ConsumerReplicas(consumerReplicas), nats.ConsumerFilterSubjects(filters...), nats.ConsumerMemoryStorage(), } var sub *nats.Subscription sub, err := js.Subscribe("", cb, subOpts...) if err != nil { b.Fatalf("Failed to subscribe: %s", err) } defer func(sub *nats.Subscription) { err := sub.Unsubscribe() if err != nil { b.Logf("Failed to unsubscribe: %s", err) } }(sub) consumeWg.Wait() } // Wait for all consumer threads and main to be ready wgReady := sync.WaitGroup{} wgReady.Add(bc.concurrentConsumers + 1) // Wait until all consumer threads have completed wgCompleted := sync.WaitGroup{} wgCompleted.Add(bc.concurrentConsumers) // Operations left for consumer threads opsCount := atomic.Int32{} opsCount.Store(int32(b.N)) // Start a pool of C goroutines, each one with a dedicated connection. for i := 1; i <= bc.concurrentConsumers; i++ { go func(consumerId int) { // Connect nc, js := jsClientConnectURL(b, connectURL) defer nc.Close() // Signal completion of work defer wgCompleted.Done() rng := rand.New(rand.NewSource(int64(seed + consumerId))) // Ready, wait for everyone else wgReady.Done() wgReady.Wait() completed := 0 for opsCount.Add(-1) >= 0 { subscribeConsumeUnsubscribe(js, rng) completed += 1 } if verbose { b.Logf("Consumer thread %d completed %d of %d operations", consumerId, completed, b.N) } }(i) } // Wait for all consumers to be ready wgReady.Done() wgReady.Wait() // Start measuring time b.ResetTimer() // Wait for consumers to have chewed through b.N operations wgCompleted.Wait() b.StopTimer() // Throughput is not very important in this benchmark since each operation includes // subscribe, unsubscribe and retrieves just a few bytes //b.SetBytes(int64(messageSize * messagesPerIteration)) }, ) } }, ) } } func BenchmarkJetStreamPublish(b *testing.B) { const ( verbose = false seed = 12345 streamName = "S" ) runSyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string) (int, int) { published, errors := 0, 0 message := make([]byte, messageSize) rand.New(rand.NewSource(int64(seed))).Read(message) b.ResetTimer() for i := 1; i <= b.N; i++ { fastRandomMutation(message, 10) subject := subjects[fastrand.Uint32n(uint32(len(subjects)))] _, pubErr := js.Publish(subject, message) if pubErr != nil { errors++ } else { published++ } if verbose && i%1000 == 0 { b.Logf("Published %d/%d, %d errors", i, b.N, errors) } } b.StopTimer() return published, errors } runAsyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string, asyncWindow int) (int, int) { const publishCompleteMaxWait = 30 * time.Second rng := rand.New(rand.NewSource(int64(seed))) message := make([]byte, messageSize) rng.Read(message) published, errors := 0, 0 b.ResetTimer() for published < b.N { // Normally publish a full batch (of size `asyncWindow`) publishBatchSize := asyncWindow // Unless fewer are left to complete the benchmark if b.N-published < asyncWindow { publishBatchSize = b.N - published } pending := make([]nats.PubAckFuture, 0, publishBatchSize) for i := 0; i < publishBatchSize; i++ { fastRandomMutation(message, 10) subject := subjects[rng.Intn(len(subjects))] pubAckFuture, err := js.PublishAsync(subject, message) if err != nil { errors++ continue } pending = append(pending, pubAckFuture) } // All in this batch published, wait for completed select { case <-js.PublishAsyncComplete(): case <-time.After(publishCompleteMaxWait): b.Fatalf("Publish timed out") } // Verify one by one if they were published successfully for _, pubAckFuture := range pending { select { case <-pubAckFuture.Ok(): published++ case <-pubAckFuture.Err(): errors++ default: b.Fatalf("PubAck is still pending after publish completed") } } if verbose { b.Logf("Published %d/%d", published, b.N) } } b.StopTimer() return published, errors } type PublishType string const ( Sync PublishType = "Sync" Async PublishType = "Async" ) benchmarksCases := []struct { clusterSize int replicas int messageSize int numSubjects int minMessages int }{ {1, 1, 10, 1, 100_000}, // Single node, 10B messages, ~1MB minimum {1, 1, 1024, 1, 1_000}, // Single node, 1KB messages, ~1MB minimum {3, 3, 10, 1, 100_000}, // 3-nodes cluster, R=3, 10B messages, ~1MB minimum {3, 3, 1024, 1, 1_000}, // 3-nodes cluster, R=3, 10B messages, ~1MB minimum } // All the cases above are run with each of the publisher cases below publisherCases := []struct { pType PublishType asyncWindow int }{ {Sync, -1}, {Async, 1000}, {Async, 4000}, {Async, 8000}, } for _, bc := range benchmarksCases { name := fmt.Sprintf( "N=%d,R=%d,MsgSz=%db,Subjs=%d", bc.clusterSize, bc.replicas, bc.messageSize, bc.numSubjects, ) b.Run( name, func(b *testing.B) { for _, pc := range publisherCases { name := fmt.Sprintf("%v", pc.pType) if pc.pType == Async && pc.asyncWindow > 0 { name = fmt.Sprintf("%s[W:%d]", name, pc.asyncWindow) } b.Run( name, func(b *testing.B) { subjects := make([]string, bc.numSubjects) for i := 0; i < bc.numSubjects; i++ { subjects[i] = fmt.Sprintf("s-%d", i+1) } if verbose { b.Logf("Running %s with %d ops", name, b.N) } if verbose { b.Logf("Setting up %d nodes", bc.clusterSize) } cl, _, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize) defer shutdown() defer nc.Close() jsOpts := []nats.JSOpt{ nats.MaxWait(10 * time.Second), } if pc.asyncWindow > 0 && pc.pType == Async { jsOpts = append(jsOpts, nats.PublishAsyncMaxPending(pc.asyncWindow)) } js, err := nc.JetStream(jsOpts...) if err != nil { b.Fatalf("Unexpected error getting JetStream context: %v", err) } if verbose { b.Logf("Creating stream with R=%d and %d input subjects", bc.replicas, bc.numSubjects) } streamConfig := &nats.StreamConfig{ Name: streamName, Subjects: subjects, Replicas: bc.replicas, } if _, err := js.AddStream(streamConfig); err != nil { b.Fatalf("Error creating stream: %v", err) } // If replicated resource, connect to stream leader for lower variability if bc.replicas > 1 { connectURL := cl.streamLeader("$G", streamName).ClientURL() nc.Close() nc, err = nats.Connect(connectURL) if err != nil { b.Fatalf("Failed to create client connection to stream leader: %v", err) } defer nc.Close() js, err = nc.JetStream(jsOpts...) if err != nil { b.Fatalf("Unexpected error getting JetStream context for stream leader: %v", err) } } if verbose { b.Logf("Running %v publisher with message size: %dB", pc.pType, bc.messageSize) } b.SetBytes(int64(bc.messageSize)) // Benchmark starts here b.ResetTimer() var published, errors int switch pc.pType { case Sync: published, errors = runSyncPublisher(b, js, bc.messageSize, subjects) case Async: published, errors = runAsyncPublisher(b, js, bc.messageSize, subjects, pc.asyncWindow) } // Benchmark ends here b.StopTimer() if published+errors != b.N { b.Fatalf("Something doesn't add up: %d + %d != %d", published, errors, b.N) } b.ReportMetric(float64(errors)*100/float64(b.N), "%error") }, ) } }, ) } } func BenchmarkJetStreamInterestStreamWithLimit(b *testing.B) { const ( verbose = true seed = 12345 publishBatchSize = 100 messageSize = 256 numSubjects = 2500 subjectPrefix = "S" numPublishers = 4 randomData = true warmupMessages = 1 ) if verbose { b.Logf( "BatchSize: %d, MsgSize: %d, Subjects: %d, Publishers: %d, Random Message: %v", publishBatchSize, messageSize, numSubjects, numPublishers, randomData, ) } // Benchmark parameters: sub-benchmarks are executed for every combination of the following 3 groups // Unless a more restrictive filter is specified, e.g.: // BenchmarkJetStreamInterestStreamWithLimit/.*R=3.*/Storage=Memory/unlimited // Parameter: Number of nodes and number of stream replicas clusterAndReplicasCases := []struct { clusterSize int replicas int }{ {1, 1}, // Single node, R=1 {3, 3}, // 3-nodes cluster, R=3 } // Parameter: Stream storage type storageTypeCases := []nats.StorageType{ nats.MemoryStorage, nats.FileStorage, } // Parameter: Stream limit configuration limitConfigCases := map[string]func(*nats.StreamConfig){ "unlimited": func(config *nats.StreamConfig) { }, "MaxMsg=1000": func(config *nats.StreamConfig) { config.MaxMsgs = 100 }, "MaxMsg=10": func(config *nats.StreamConfig) { config.MaxMsgs = 10 }, "MaxPerSubject=10": func(config *nats.StreamConfig) { config.MaxMsgsPerSubject = 10 }, "MaxAge=1s": func(config *nats.StreamConfig) { config.MaxAge = 1 * time.Second }, "MaxBytes=1MB": func(config *nats.StreamConfig) { config.MaxBytes = 1024 * 1024 }, } // Context shared by publishers routines type PublishersContext = struct { readyWg sync.WaitGroup completedWg sync.WaitGroup messagesLeft int lock sync.Mutex errors int } // Helper: Publish synchronously as Goroutine publish := func(publisherId int, ctx *PublishersContext, js nats.JetStreamContext) { defer ctx.completedWg.Done() errors := 0 messageBuf := make([]byte, messageSize) rand.New(rand.NewSource(int64(seed + publisherId))).Read(messageBuf) // Warm up: publish a few messages for i := 0; i < warmupMessages; i++ { subject := fmt.Sprintf("%s.%d", subjectPrefix, fastrand.Uint32n(numSubjects)) if randomData { fastRandomMutation(messageBuf, 10) } _, err := js.Publish(subject, messageBuf) if err != nil { b.Logf("Warning: failed to publish warmup message: %s", err) } } // Signal this publisher is ready ctx.readyWg.Done() for { // Obtain a batch of messages to publish batchSize := 0 { ctx.lock.Lock() if ctx.messagesLeft >= publishBatchSize { batchSize = publishBatchSize } else if ctx.messagesLeft < publishBatchSize { batchSize = ctx.messagesLeft } ctx.messagesLeft -= batchSize ctx.lock.Unlock() } // Nothing left to publish, terminate if batchSize == 0 { ctx.lock.Lock() ctx.errors += errors ctx.lock.Unlock() return } // Publish a batch of messages for i := 0; i < batchSize; i++ { subject := fmt.Sprintf("%s.%d", subjectPrefix, fastrand.Uint32n(numSubjects)) if randomData { fastRandomMutation(messageBuf, 10) } _, err := js.Publish(subject, messageBuf) if err != nil { errors += 1 } } } } // Benchmark matrix: (cluster and replicas) * (storage type) * (stream limit) for _, benchmarkCase := range clusterAndReplicasCases { b.Run( fmt.Sprintf( "N=%d,R=%d", benchmarkCase.clusterSize, benchmarkCase.replicas, ), func(b *testing.B) { for _, storageType := range storageTypeCases { b.Run( fmt.Sprintf("Storage=%v", storageType), func(b *testing.B) { for limitDescription, limitConfigFunc := range limitConfigCases { b.Run( limitDescription, func(b *testing.B) { // Print benchmark parameters if verbose { b.Logf( "Stream: %+v, Storage: [%v] Limit: [%s], Ops: %d", benchmarkCase, storageType, limitDescription, b.N, ) } // Setup server or cluster cl, ls, shutdown, nc, js := startJSClusterAndConnect(b, benchmarkCase.clusterSize) defer shutdown() defer nc.Close() // Common stream configuration streamConfig := &nats.StreamConfig{ Name: "S", Subjects: []string{fmt.Sprintf("%s.>", subjectPrefix)}, Replicas: benchmarkCase.replicas, Storage: storageType, Discard: DiscardOld, Retention: DiscardOld, } // Configure stream limit limitConfigFunc(streamConfig) // Create stream if _, err := js.AddStream(streamConfig); err != nil { b.Fatalf("Error creating stream: %v", err) } // Set up publishers shared context var pubCtx PublishersContext pubCtx.readyWg.Add(numPublishers) pubCtx.completedWg.Add(numPublishers) // Hold this lock until all publishers are ready pubCtx.lock.Lock() pubCtx.messagesLeft = b.N connectURL := ls.ClientURL() // If replicated resource, connect to stream leader for lower variability if benchmarkCase.replicas > 1 { connectURL = cl.streamLeader("$G", "S").ClientURL() } // Spawn publishers routines, each with its own connection and JS context for i := 0; i < numPublishers; i++ { nc, err := nats.Connect(connectURL) if err != nil { b.Fatal(err) } defer nc.Close() js, err := nc.JetStream() if err != nil { b.Fatal(err) } go publish(i, &pubCtx, js) } // Wait for all publishers to be ready pubCtx.readyWg.Wait() // Set size of each operation, for throughput calculation b.SetBytes(messageSize) // Benchmark starts here b.ResetTimer() // Unblock the publishers pubCtx.lock.Unlock() // Wait for all publishers to complete pubCtx.completedWg.Wait() // Benchmark ends here b.StopTimer() // Sanity check, publishers may have died before completing if pubCtx.messagesLeft != 0 { b.Fatalf("Some messages left: %d", pubCtx.messagesLeft) } b.ReportMetric(float64(pubCtx.errors)*100/float64(b.N), "%error") }, ) } }, ) } }, ) } } func BenchmarkJetStreamKV(b *testing.B) { const ( verbose = false kvName = "BUCKET" keyPrefix = "K_" seed = 12345 ) runKVGet := func(b *testing.B, kv nats.KeyValue, keys []string) int { rng := rand.New(rand.NewSource(int64(seed))) errors := 0 b.ResetTimer() for i := 1; i <= b.N; i++ { key := keys[rng.Intn(len(keys))] _, err := kv.Get(key) if err != nil { errors++ continue } if verbose && i%1000 == 0 { b.Logf("Completed %d/%d Get ops", i, b.N) } } b.StopTimer() return errors } runKVPut := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int { value := make([]byte, valueSize) rand.New(rand.NewSource(int64(seed))).Read(value) errors := 0 b.ResetTimer() for i := 1; i <= b.N; i++ { key := keys[fastrand.Uint32n(uint32(len(keys)))] fastRandomMutation(value, 10) _, err := kv.Put(key, value) if err != nil { errors++ continue } if verbose && i%1000 == 0 { b.Logf("Completed %d/%d Put ops", i, b.N) } } b.StopTimer() return errors } runKVUpdate := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int { value := make([]byte, valueSize) rand.New(rand.NewSource(int64(seed))).Read(value) errors := 0 b.ResetTimer() for i := 1; i <= b.N; i++ { key := keys[fastrand.Uint32n(uint32(len(keys)))] kve, getErr := kv.Get(key) if getErr != nil { errors++ continue } fastRandomMutation(value, 10) _, updateErr := kv.Update(key, value, kve.Revision()) if updateErr != nil { errors++ continue } if verbose && i%1000 == 0 { b.Logf("Completed %d/%d Update ops", i, b.N) } } b.StopTimer() return errors } type WorkloadType string const ( Get WorkloadType = "GET" Put WorkloadType = "PUT" Update WorkloadType = "CAS" ) benchmarksCases := []struct { clusterSize int replicas int numKeys int valueSize int }{ {1, 1, 100, 100}, // 1 node with 100 keys, 100B values {1, 1, 1000, 100}, // 1 node with 1000 keys, 100B values {3, 3, 100, 100}, // 3 nodes with 100 keys, 100B values {3, 3, 1000, 100}, // 3 nodes with 1000 keys, 100B values {3, 3, 1000, 1024}, // 3 nodes with 1000 keys, 1KB values } workloadCases := []WorkloadType{ Get, Put, Update, } for _, bc := range benchmarksCases { bName := fmt.Sprintf( "N=%d,R=%d,B=1,K=%d,ValSz=%db", bc.clusterSize, bc.replicas, bc.numKeys, bc.valueSize, ) b.Run( bName, func(b *testing.B) { for _, wc := range workloadCases { wName := fmt.Sprintf("%v", wc) b.Run( wName, func(b *testing.B) { if verbose { b.Logf("Running %s workload %s with %d messages", wName, bName, b.N) } if verbose { b.Logf("Setting up %d nodes", bc.clusterSize) } // Pre-generate all keys keys := make([]string, 0, bc.numKeys) for i := 1; i <= bc.numKeys; i++ { key := fmt.Sprintf("%s%d", keyPrefix, i) keys = append(keys, key) } // Setup server or cluster cl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) defer shutdown() defer nc.Close() // Create bucket if verbose { b.Logf("Creating KV %s with R=%d", kvName, bc.replicas) } kvConfig := &nats.KeyValueConfig{ Bucket: kvName, Replicas: bc.replicas, } kv, err := js.CreateKeyValue(kvConfig) if err != nil { b.Fatalf("Error creating KV: %v", err) } // Initialize all keys rng := rand.New(rand.NewSource(int64(seed))) value := make([]byte, bc.valueSize) for _, key := range keys { rng.Read(value) _, err := kv.Create(key, value) if err != nil { b.Fatalf("Failed to initialize %s/%s: %v", kvName, key, err) } } // If replicated resource, connect to stream leader for lower variability if bc.replicas > 1 { nc.Close() connectURL := cl.streamLeader("$G", fmt.Sprintf("KV_%s", kvName)).ClientURL() nc, js = jsClientConnectURL(b, connectURL) defer nc.Close() } kv, err = js.KeyValue(kv.Bucket()) if err != nil { b.Fatalf("Error binding to KV: %v", err) } // Set size of each operation, for throughput calculation b.SetBytes(int64(bc.valueSize)) // Discard time spent during setup // May reset again further in b.ResetTimer() var errors int switch wc { case Get: errors = runKVGet(b, kv, keys) case Put: errors = runKVPut(b, kv, keys, bc.valueSize) case Update: errors = runKVUpdate(b, kv, keys, bc.valueSize) default: b.Fatalf("Unknown workload type: %v", wc) } // Benchmark ends here, (may have stopped earlier) b.StopTimer() b.ReportMetric(float64(errors)*100/float64(b.N), "%error") }, ) } }, ) } } func BenchmarkJetStreamObjStore(b *testing.B) { const ( verbose = false objStoreName = "B" keyPrefix = "K_" seed = 12345 initKeys = true // read/write ratios ReadOnly = 1.0 WriteOnly = 0.0 ) // rwRatio to string rwRatioToString := func(rwRatio float64) string { switch rwRatio { case ReadOnly: return "readOnly" case WriteOnly: return "writeOnly" default: return fmt.Sprintf("%0.1f", rwRatio) } } // benchmark for object store by performing read/write operations with data of random size RunObjStoreBenchmark := func(b *testing.B, objStore nats.ObjectStore, minObjSz int, maxObjSz int, numKeys int, rwRatio float64) (int, int, int) { var ( errors int reads int writes int ) dataBuf := make([]byte, maxObjSz) rng := rand.New(rand.NewSource(int64(seed))) rng.Read(dataBuf) // Each operation is processing a random amount of bytes within a size range which // will be either read from or written to an object store bucket. However, here we are // approximating the size of the processed data with a simple average of the range. b.SetBytes(int64((minObjSz + maxObjSz) / 2)) for i := 1; i <= b.N; i++ { key := fmt.Sprintf("%s_%d", keyPrefix, rng.Intn(numKeys)) var err error rwOp := rng.Float64() switch { case rwOp <= rwRatio: // Read Op _, err = objStore.GetBytes(key) reads++ case rwOp > rwRatio: // Write Op // dataSz is a random value between min-max object size and cannot be less than 1 byte dataSz := rng.Intn(maxObjSz-minObjSz+1) + minObjSz data := dataBuf[:dataSz] fastRandomMutation(data, 10) _, err = objStore.PutBytes(key, data) writes++ } if err != nil { errors++ } if verbose && i%1000 == 0 { b.Logf("Completed: %d reads, %d writes, %d errors. %d/%d total operations have been completed.", reads, writes, errors, i, b.N) } } return errors, reads, writes } // benchmark cases table benchmarkCases := []struct { storage nats.StorageType numKeys int minObjSz int maxObjSz int }{ {nats.MemoryStorage, 100, 1024, 102400}, // mem storage, 100 objects sized (1KB-100KB) {nats.MemoryStorage, 100, 102400, 1048576}, // mem storage, 100 objects sized (100KB-1MB) {nats.MemoryStorage, 1000, 10240, 102400}, // mem storage, 1k objects of various size (10KB - 100KB) {nats.FileStorage, 100, 1024, 102400}, // file storage, 100 objects sized (1KB-100KB) {nats.FileStorage, 1000, 10240, 1048576}, // file storage, 1k objects of various size (10KB - 1MB) {nats.FileStorage, 100, 102400, 1048576}, // file storage, 100 objects sized (100KB-1MB) {nats.FileStorage, 100, 1048576, 10485760}, // file storage, 100 objects sized (1MB-10MB) {nats.FileStorage, 10, 10485760, 104857600}, // file storage, 10 objects sized (10MB-100MB) } var ( clusterSizeCases = []int{1, 3} rwRatioCases = []float64{ReadOnly, WriteOnly, 0.8} ) // Test with either single node or 3 node cluster for _, clusterSize := range clusterSizeCases { replicas := clusterSize cName := fmt.Sprintf("N=%d,R=%d", clusterSize, replicas) b.Run( cName, func(b *testing.B) { for _, rwRatio := range rwRatioCases { rName := fmt.Sprintf("workload=%s", rwRatioToString(rwRatio)) b.Run( rName, func(b *testing.B) { // Test all tabled benchmark cases for _, bc := range benchmarkCases { bName := fmt.Sprintf("K=%d,storage=%s,minObjSz=%db,maxObjSz=%db", bc.numKeys, bc.storage, bc.minObjSz, bc.maxObjSz) b.Run( bName, func(b *testing.B) { // Test setup rng := rand.New(rand.NewSource(int64(seed))) if verbose { b.Logf("Setting up %d nodes", replicas) } // Setup server or cluster cl, _, shutdown, nc, js := startJSClusterAndConnect(b, clusterSize) defer shutdown() defer nc.Close() // Initialize object store if verbose { b.Logf("Creating ObjectStore %s with R=%d", objStoreName, replicas) } objStoreConfig := &nats.ObjectStoreConfig{ Bucket: objStoreName, Replicas: replicas, Storage: bc.storage, } objStore, err := js.CreateObjectStore(objStoreConfig) if err != nil { b.Fatalf("Error creating ObjectStore: %v", err) } // If replicated resource, connect to stream leader for lower variability if clusterSize > 1 { nc.Close() connectURL := cl.streamLeader("$G", fmt.Sprintf("OBJ_%s", objStoreName)).ClientURL() nc, js := jsClientConnectURL(b, connectURL) defer nc.Close() objStore, err = js.ObjectStore(objStoreName) if err != nil { b.Fatalf("Error binding to ObjectStore: %v", err) } } // Initialize keys if initKeys { for n := 0; n < bc.numKeys; n++ { key := fmt.Sprintf("%s_%d", keyPrefix, n) dataSz := rng.Intn(bc.maxObjSz-bc.minObjSz+1) + bc.minObjSz value := make([]byte, dataSz) rng.Read(value) _, err := objStore.PutBytes(key, value) if err != nil { b.Fatalf("Failed to initialize %s/%s: %v", objStoreName, key, err) } } } b.ResetTimer() // Run benchmark errors, reads, writes := RunObjStoreBenchmark(b, objStore, bc.minObjSz, bc.maxObjSz, bc.numKeys, rwRatio) // Report metrics b.ReportMetric(float64(errors)*100/float64(b.N), "%error") b.ReportMetric(float64(reads), "reads") b.ReportMetric(float64(writes), "writes") }, ) } }, ) } }, ) } } func BenchmarkJetStreamPublishConcurrent(b *testing.B) { const ( subject = "test-subject" streamName = "test-stream" ) type BenchPublisher struct { // nats connection for this publisher conn *nats.Conn // jetstream context js nats.JetStreamContext // message buffer messageData []byte // number of publish calls publishCalls int // number of publish errors publishErrors int } messageSizeCases := []int64{ 10, // 10B 1024, // 1KiB 102400, // 100KiB } numPubsCases := []int{ 12, } replicasCases := []struct { clusterSize int replicas int }{ {1, 1}, {3, 3}, } workload := func(b *testing.B, numPubs int, messageSize int64, clientUrl string) { // create N publishers publishers := make([]BenchPublisher, numPubs) for i := range publishers { // create publisher connection and jetstream context ncPub, err := nats.Connect(clientUrl) if err != nil { b.Fatal(err) } defer ncPub.Close() jsPub, err := ncPub.JetStream() if err != nil { b.Fatal(err) } // initialize publisher publishers[i] = BenchPublisher{ conn: ncPub, js: jsPub, messageData: make([]byte, messageSize), publishCalls: 0, publishErrors: 0, } rand.New(rand.NewSource(int64(i))).Read(publishers[i].messageData) } // waits for all publishers sub-routines and for main thread to be ready var workloadReadyWg sync.WaitGroup workloadReadyWg.Add(1 + numPubs) // wait group blocks main thread until publish workload is completed, it is decremented after stream receives b.N messages from all publishers var benchCompleteWg sync.WaitGroup benchCompleteWg.Add(1) // wait group to ensure all publishers have been torn down var finishedPublishersWg sync.WaitGroup finishedPublishersWg.Add(numPubs) // start go routines for all publishers, wait till all publishers are initialized before starting publish workload for i := range publishers { go func(pubId int) { // signal that this publisher has been torn down defer finishedPublishersWg.Done() // publisher sub-routine is ready workloadReadyWg.Done() // start workload when main thread and all other publishers are ready workloadReadyWg.Wait() // publish until stream receives b.N messages for { // random bytes as payload fastRandomMutation(publishers[pubId].messageData, 10) // attempt to publish message pubAck, err := publishers[pubId].js.Publish(subject, publishers[pubId].messageData) publishers[pubId].publishCalls += 1 if err != nil { publishers[pubId].publishErrors += 1 continue } // all messages have been published to stream if pubAck.Sequence == uint64(b.N) { benchCompleteWg.Done() } // a publisher has already published b.N messages, stop publishing if pubAck.Sequence >= uint64(b.N) { return } } }(i) } // set bytes per operation b.SetBytes(messageSize) // main thread is ready workloadReadyWg.Done() // start the clock b.ResetTimer() // wait till termination cond reached benchCompleteWg.Wait() // stop the clock b.StopTimer() // wait for all publishers to shutdown finishedPublishersWg.Wait() // sum up publish calls and errors publishCalls := 0 publishErrors := 0 for _, pub := range publishers { publishCalls += pub.publishCalls publishErrors += pub.publishErrors } // report error rate errorRate := 100 * float64(publishErrors) / float64(publishCalls) b.ReportMetric(errorRate, "%error") } // benchmark case matrix for _, replicasCase := range replicasCases { b.Run( fmt.Sprintf("N=%d,R=%d", replicasCase.clusterSize, replicasCase.replicas), func(b *testing.B) { for _, messageSize := range messageSizeCases { b.Run( fmt.Sprintf("msgSz=%db", messageSize), func(b *testing.B) { for _, numPubs := range numPubsCases { b.Run( fmt.Sprintf("pubs=%d", numPubs), func(b *testing.B) { // start jetstream cluster cl, ls, shutdown, nc, js := startJSClusterAndConnect(b, replicasCase.clusterSize) defer shutdown() defer nc.Close() clientUrl := ls.ClientURL() // create stream _, err := js.AddStream(&nats.StreamConfig{ Name: streamName, Subjects: []string{subject}, Replicas: replicasCase.replicas, }) if err != nil { b.Fatal(err) } defer js.DeleteStream(streamName) // If replicated resource, connect to stream leader for lower variability if replicasCase.replicas > 1 { nc.Close() clientUrl = cl.streamLeader("$G", streamName).ClientURL() nc, _ = jsClientConnectURL(b, clientUrl) defer nc.Close() } // run workload workload(b, numPubs, messageSize, clientUrl) }, ) } }) } }) } } // Helper function to stand up a JS-enabled single server or cluster func startJSClusterAndConnect(b *testing.B, clusterSize int) (c *cluster, s *Server, shutdown func(), nc *nats.Conn, js nats.JetStreamContext) { b.Helper() var err error if clusterSize == 1 { s = RunBasicJetStreamServer(b) shutdown = func() { s.Shutdown() } s.optsMu.Lock() s.opts.SyncInterval = 5 * time.Minute s.optsMu.Unlock() } else { c = createJetStreamClusterExplicit(b, "BENCH_PUB", clusterSize) c.waitOnClusterReadyWithNumPeers(clusterSize) c.waitOnLeader() s = c.leader() shutdown = func() { c.shutdown() } for _, s := range c.servers { s.optsMu.Lock() s.opts.SyncInterval = 5 * time.Minute s.optsMu.Unlock() } } nc, err = nats.Connect(s.ClientURL()) if err != nil { b.Fatalf("failed to connect: %s", err) } js, err = nc.JetStream() if err != nil { b.Fatalf("failed to init jetstream: %s", err) } return c, s, shutdown, nc, js } nats-server-2.10.27/server/jetstream_chaos_test.go000066400000000000000000001034141477524627100222250ustar00rootroot00000000000000// Copyright 2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build js_chaos_tests // +build js_chaos_tests package server import ( "bytes" "encoding/json" "fmt" "math/rand" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" ) // Support functions for "chaos" testing (random injected failures) type ChaosMonkeyController interface { // Launch the monkey as background routine and return start() // Stop a monkey that was previously started stop() // Run the monkey synchronously, until it is manually stopped via stopCh run() } type ClusterChaosMonkey interface { // Set defaults and validates the monkey parameters validate(t *testing.T, c *cluster) // Run the monkey synchronously, until it is manually stopped via stopCh run(t *testing.T, c *cluster, stopCh <-chan bool) } // Chaos Monkey Controller that acts on a cluster type clusterChaosMonkeyController struct { t *testing.T cluster *cluster wg sync.WaitGroup stopCh chan bool ccm ClusterChaosMonkey } func createClusterChaosMonkeyController(t *testing.T, c *cluster, ccm ClusterChaosMonkey) ChaosMonkeyController { ccm.validate(t, c) return &clusterChaosMonkeyController{ t: t, cluster: c, stopCh: make(chan bool, 3), ccm: ccm, } } func (m *clusterChaosMonkeyController) start() { m.t.Logf("🵠Starting monkey") m.wg.Add(1) go func() { defer m.wg.Done() m.run() }() } func (m *clusterChaosMonkeyController) stop() { m.t.Logf("🵠Stopping monkey") m.stopCh <- true m.wg.Wait() m.t.Logf("🵠Monkey stopped") } func (m *clusterChaosMonkeyController) run() { m.ccm.run(m.t, m.cluster, m.stopCh) } // Cluster Chaos Monkey that selects a random subset of the nodes in a cluster (according to min/max provided), // shuts them down for a given duration (according to min/max provided), then brings them back up. // Then sleeps for a given time, and does it again until stopped. type clusterBouncerChaosMonkey struct { minDowntime time.Duration maxDowntime time.Duration minDownServers int maxDownServers int pause time.Duration } func (m *clusterBouncerChaosMonkey) validate(t *testing.T, c *cluster) { if m.minDowntime > m.maxDowntime { t.Fatalf("Min downtime %v cannot be larger than max downtime %v", m.minDowntime, m.maxDowntime) } if m.minDownServers > m.maxDownServers { t.Fatalf("Min down servers %v cannot be larger than max down servers %v", m.minDownServers, m.maxDownServers) } } func (m *clusterBouncerChaosMonkey) run(t *testing.T, c *cluster, stopCh <-chan bool) { for { // Pause between actions select { case <-stopCh: return case <-time.After(m.pause): } // Pick a random subset of servers numServersDown := rand.Intn(1+m.maxDownServers-m.minDownServers) + m.minDownServers servers := c.selectRandomServers(numServersDown) serverNames := []string{} for _, s := range servers { serverNames = append(serverNames, s.info.Name) } // Pick a random outage interval minOutageNanos := m.minDowntime.Nanoseconds() maxOutageNanos := m.maxDowntime.Nanoseconds() outageDurationNanos := rand.Int63n(1+maxOutageNanos-minOutageNanos) + minOutageNanos outageDuration := time.Duration(outageDurationNanos) // Take down selected servers t.Logf("🵠Taking down %d/%d servers for %v (%v)", numServersDown, len(c.servers), outageDuration, serverNames) c.stopSubset(servers) // Wait for the "outage" duration select { case <-stopCh: return case <-time.After(outageDuration): } // Restart servers and wait for cluster to be healthy t.Logf("🵠Restoring cluster") c.restartAllSamePorts() c.waitOnClusterHealthz() c.waitOnClusterReady() c.waitOnAllCurrent() c.waitOnLeader() } } // Additional cluster methods for chaos testing func (c *cluster) waitOnClusterHealthz() { c.t.Helper() for _, cs := range c.servers { c.waitOnServerHealthz(cs) } } func (c *cluster) stopSubset(toStop []*Server) { c.t.Helper() for _, s := range toStop { s.Shutdown() } } func (c *cluster) selectRandomServers(numServers int) []*Server { c.t.Helper() if numServers > len(c.servers) { panic(fmt.Sprintf("Can't select %d servers in a cluster of %d", numServers, len(c.servers))) } var selectedServers []*Server selectedServers = append(selectedServers, c.servers...) rand.Shuffle(len(selectedServers), func(x, y int) { selectedServers[x], selectedServers[y] = selectedServers[y], selectedServers[x] }) return selectedServers[0:numServers] } // Other helpers func jsClientConnectCluster(t testing.TB, c *cluster) (*nats.Conn, nats.JetStreamContext) { serverConnectURLs := make([]string, len(c.servers)) for i, server := range c.servers { serverConnectURLs[i] = server.ClientURL() } connectURL := strings.Join(serverConnectURLs, ",") nc, err := nats.Connect(connectURL) if err != nil { t.Fatalf("Failed to connect: %s", err) } js, err := nc.JetStream() if err != nil { t.Fatalf("Failed to init JetStream context: %s", err) } return nc, js } func toIndentedJsonString(v any) string { s, err := json.MarshalIndent(v, "", " ") if err != nil { panic(err) } return string(s) } // Bounces the entire set of nodes, then brings them back up. // Fail if some nodes don't come back online. func TestJetStreamChaosClusterBounce(t *testing.T) { const duration = 60 * time.Second const clusterSize = 3 c := createJetStreamClusterExplicit(t, "R3", clusterSize) defer c.shutdown() chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, maxDownServers: clusterSize, pause: 3 * time.Second, }, ) chaos.start() defer chaos.stop() <-time.After(duration) } // Bounces a subset of the nodes, then brings them back up. // Fails if some nodes don't come back online. func TestJetStreamChaosClusterBounceSubset(t *testing.T) { const duration = 60 * time.Second const clusterSize = 3 c := createJetStreamClusterExplicit(t, "R3", clusterSize) defer c.shutdown() chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: 1, maxDownServers: clusterSize, pause: 3 * time.Second, }, ) chaos.start() defer chaos.stop() <-time.After(duration) } const ( chaosConsumerTestsClusterName = "CONSUMERS_CHAOS_TEST" chaosConsumerTestsStreamName = "CONSUMER_CHAOS_TEST_STREAM" chaosConsumerTestsSubject = "foo" chaosConsumerTestsDebug = false ) // Creates stream and fills it with the given number of messages. // Each message is the string representation of the stream sequence number, // e.g. the first message (seqno: 1) contains data "1". // This allows consumers to verify the content of each message without tracking additional state func createStreamForConsumerChaosTest(t *testing.T, c *cluster, replicas, numMessages int) { t.Helper() const publishBatchSize = 1_000 pubNc, pubJs := jsClientConnectCluster(t, c) defer pubNc.Close() _, err := pubJs.AddStream(&nats.StreamConfig{ Name: chaosConsumerTestsStreamName, Subjects: []string{chaosConsumerTestsSubject}, Replicas: replicas, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } ackFutures := make([]nats.PubAckFuture, 0, publishBatchSize) for i := 1; i <= numMessages; i++ { message := []byte(fmt.Sprintf("%d", i)) pubAckFuture, err := pubJs.PublishAsync(chaosConsumerTestsSubject, message, nats.ExpectLastSequence(uint64(i-1))) if err != nil { t.Fatalf("Publish error: %s", err) } ackFutures = append(ackFutures, pubAckFuture) if (i > 0 && i%publishBatchSize == 0) || i == numMessages { select { case <-pubJs.PublishAsyncComplete(): for _, pubAckFuture := range ackFutures { select { case <-pubAckFuture.Ok(): // Noop case pubAckErr := <-pubAckFuture.Err(): t.Fatalf("Error publishing: %s", pubAckErr) case <-time.After(30 * time.Second): t.Fatalf("Timeout verifying pubAck for message: %s", pubAckFuture.Msg().Data) } } ackFutures = make([]nats.PubAckFuture, 0, publishBatchSize) t.Logf("Published %d/%d messages", i, numMessages) case <-time.After(30 * time.Second): t.Fatalf("Publish timed out") } } } } // Verify ordered delivery despite cluster-wide outages func TestJetStreamChaosConsumerOrdered(t *testing.T) { const numMessages = 5_000 const numBatch = 500 const maxRetries = 100 const retryDelay = 500 * time.Millisecond const fetchTimeout = 250 * time.Millisecond const clusterSize = 3 const replicas = 3 c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize) defer c.shutdown() createStreamForConsumerChaosTest(t, c, replicas, numMessages) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, // Whole cluster outage maxDownServers: clusterSize, pause: 1 * time.Second, }, ) subNc, subJs := jsClientConnectCluster(t, c) defer subNc.Close() sub, err := subJs.SubscribeSync( chaosConsumerTestsSubject, nats.OrderedConsumer(), ) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() if chaosConsumerTestsDebug { t.Logf("Initial subscription: %s", toIndentedJsonString(sub)) } chaos.start() defer chaos.stop() for i := 1; i <= numMessages; i++ { var msg *nats.Msg var nextMsgErr error var expectedMsgData = []byte(fmt.Sprintf("%d", i)) nextMsgRetryLoop: for r := 0; r <= maxRetries; r++ { msg, nextMsgErr = sub.NextMsg(fetchTimeout) if nextMsgErr == nil { break nextMsgRetryLoop } else if r == maxRetries { t.Fatalf("Exceeded max retries for NextMsg") } else if nextMsgErr == nats.ErrBadSubscription { t.Fatalf("Subscription is invalid: %s", toIndentedJsonString(sub)) } else { time.Sleep(retryDelay) } } metadata, err := msg.Metadata() if err != nil { t.Fatalf("Failed to get message metadata: %v", err) } if metadata.Sequence.Stream != uint64(i) { t.Fatalf("Expecting stream sequence %d, got %d instead", i, metadata.Sequence.Stream) } if !bytes.Equal(msg.Data, expectedMsgData) { t.Fatalf("Expecting message %s, got %s instead", expectedMsgData, msg.Data) } // Simulate application processing (and gives the monkey some time to brew chaos) time.Sleep(10 * time.Millisecond) if i%numBatch == 0 { t.Logf("Consumed %d/%d", i, numMessages) } } } // Verify ordered delivery despite cluster-wide outages func TestJetStreamChaosConsumerAsync(t *testing.T) { const numMessages = 5_000 const numBatch = 500 const timeout = 30 * time.Second // No (new) messages for 30s => terminate const maxRetries = 25 const retryDelay = 500 * time.Millisecond const clusterSize = 3 const replicas = 3 c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize) defer c.shutdown() createStreamForConsumerChaosTest(t, c, replicas, numMessages) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, maxDownServers: clusterSize, pause: 2 * time.Second, }, ) subNc, subJs := jsClientConnectCluster(t, c) defer subNc.Close() timeoutTimer := time.NewTimer(timeout) deliveryCount := uint64(0) received := NewBitset(numMessages) handleMsg := func(msg *nats.Msg) { deliveryCount += 1 metadata, err := msg.Metadata() if err != nil { t.Fatalf("Failed to get message metadata: %v", err) } seq := metadata.Sequence.Stream var expectedMsgData = []byte(fmt.Sprintf("%d", seq)) if !bytes.Equal(msg.Data, expectedMsgData) { t.Fatalf("Expecting message content '%s', got '%s' instead", expectedMsgData, msg.Data) } isDupe := received.get(seq - 1) if isDupe { if chaosConsumerTestsDebug { t.Logf("Duplicate message delivery, seq: %d", seq) } return } // Mark this sequence as received received.set(seq-1, true) if received.count() < numMessages { // Reset timeout timeoutTimer.Reset(timeout) } else { // All received, speed up the shutdown timeoutTimer.Reset(1 * time.Second) } if received.count()%numBatch == 0 { t.Logf("Consumed %d/%d", received.count(), numMessages) } // Simulate application processing (and gives the monkey some time to brew chaos) time.Sleep(10 * time.Millisecond) ackRetryLoop: for i := 0; i <= maxRetries; i++ { ackErr := msg.Ack() if ackErr == nil { break ackRetryLoop } else if i == maxRetries { t.Fatalf("Failed to ACK message %d (retried %d times)", seq, maxRetries) } else { time.Sleep(retryDelay) } } } subOpts := []nats.SubOpt{} sub, err := subJs.Subscribe(chaosConsumerTestsSubject, handleMsg, subOpts...) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() chaos.start() defer chaos.stop() // Wait for long enough silence. // Either a stall, or all messages received <-timeoutTimer.C // Shut down consumer sub.Unsubscribe() uniqueDeliveredCount := received.count() t.Logf( "Delivered %d/%d messages %d duplicate deliveries", uniqueDeliveredCount, numMessages, deliveryCount-uniqueDeliveredCount, ) if uniqueDeliveredCount != numMessages { t.Fatalf("No new message delivered in the last %s, %d/%d messages never delivered", timeout, numMessages-uniqueDeliveredCount, numMessages) } } // Verify durable consumer retains state despite cluster-wide outages // The consumer connection is also periodically closed, and the consumer 'resumes' on a different one func TestJetStreamChaosConsumerDurable(t *testing.T) { const numMessages = 5_000 const numBatch = 500 const timeout = 30 * time.Second // No (new) messages for 60s => terminate const clusterSize = 3 const replicas = 3 const maxRetries = 25 const retryDelay = 500 * time.Millisecond const durableConsumerName = "durable" c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize) defer c.shutdown() createStreamForConsumerChaosTest(t, c, replicas, numMessages) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: 1, maxDownServers: clusterSize, pause: 3 * time.Second, }, ) var nc *nats.Conn var sub *nats.Subscription var subLock sync.Mutex var handleMsgFun func(msg *nats.Msg) var natsURL string { var sb strings.Builder for _, s := range c.servers { sb.WriteString(s.ClientURL()) sb.WriteString(",") } natsURL = sb.String() } resetDurableConsumer := func() { subLock.Lock() defer subLock.Unlock() if nc != nil { nc.Close() } var newNc *nats.Conn connectRetryLoop: for r := 0; r <= maxRetries; r++ { var connErr error newNc, connErr = nats.Connect(natsURL) if connErr == nil { break connectRetryLoop } else if r == maxRetries { t.Fatalf("Failed to connect, exceeded max retries, last error: %s", connErr) } else { time.Sleep(retryDelay) } } var newJs nats.JetStreamContext jsRetryLoop: for r := 0; r <= maxRetries; r++ { var jsErr error newJs, jsErr = newNc.JetStream(nats.MaxWait(10 * time.Second)) if jsErr == nil { break jsRetryLoop } else if r == maxRetries { t.Fatalf("Failed to get JS, exceeded max retries, last error: %s", jsErr) } else { time.Sleep(retryDelay) } } subOpts := []nats.SubOpt{ nats.Durable(durableConsumerName), } var newSub *nats.Subscription subscribeRetryLoop: for i := 0; i <= maxRetries; i++ { var subErr error newSub, subErr = newJs.Subscribe(chaosConsumerTestsSubject, handleMsgFun, subOpts...) if subErr == nil { ci, err := newJs.ConsumerInfo(chaosConsumerTestsStreamName, durableConsumerName) if err == nil { if chaosConsumerTestsDebug { t.Logf("Consumer info:\n %s", toIndentedJsonString(ci)) } } else { t.Logf("Failed to retrieve consumer info: %s", err) } break subscribeRetryLoop } else if i == maxRetries { t.Fatalf("Exceeded max retries creating subscription: %v", subErr) } else { time.Sleep(retryDelay) } } nc, sub = newNc, newSub } timeoutTimer := time.NewTimer(timeout) deliveryCount := uint64(0) received := NewBitset(numMessages) handleMsgFun = func(msg *nats.Msg) { subLock.Lock() if msg.Sub != sub { // Message from a previous instance of durable consumer, drop defer subLock.Unlock() return } subLock.Unlock() deliveryCount += 1 metadata, err := msg.Metadata() if err != nil { t.Fatalf("Failed to get message metadata: %v", err) } seq := metadata.Sequence.Stream var expectedMsgData = []byte(fmt.Sprintf("%d", seq)) if !bytes.Equal(msg.Data, expectedMsgData) { t.Fatalf("Expecting message content '%s', got '%s' instead", expectedMsgData, msg.Data) } isDupe := received.get(seq - 1) if isDupe { if chaosConsumerTestsDebug { t.Logf("Duplicate message delivery, seq: %d", seq) } return } // Mark this sequence as received received.set(seq-1, true) if received.count() < numMessages { // Reset timeout timeoutTimer.Reset(timeout) } else { // All received, speed up the shutdown timeoutTimer.Reset(1 * time.Second) } // Simulate application processing (and gives the monkey some time to brew chaos) time.Sleep(10 * time.Millisecond) ackRetryLoop: for i := 0; i <= maxRetries; i++ { ackErr := msg.Ack() if ackErr == nil { break ackRetryLoop } else if i == maxRetries { t.Fatalf("Failed to ACK message %d (retried %d times)", seq, maxRetries) } else { time.Sleep(retryDelay) } } if received.count()%numBatch == 0 { t.Logf("Consumed %d/%d, duplicate deliveries: %d", received.count(), numMessages, deliveryCount-received.count()) // Close connection and resume consuming on a different one resetDurableConsumer() } } resetDurableConsumer() chaos.start() defer chaos.stop() // Wait for long enough silence. // Either a stall, or all messages received <-timeoutTimer.C // Shut down consumer if sub != nil { sub.Unsubscribe() } uniqueDeliveredCount := received.count() t.Logf( "Delivered %d/%d messages %d duplicate deliveries", uniqueDeliveredCount, numMessages, deliveryCount-uniqueDeliveredCount, ) if uniqueDeliveredCount != numMessages { t.Fatalf("No new message delivered in the last %s, %d/%d messages never delivered", timeout, numMessages-uniqueDeliveredCount, numMessages) } } func TestJetStreamChaosConsumerPull(t *testing.T) { const numMessages = 5_000 const numBatch = 500 const maxRetries = 100 const retryDelay = 500 * time.Millisecond const fetchTimeout = 250 * time.Millisecond const fetchBatchSize = 100 const clusterSize = 3 const replicas = 3 const durableConsumerName = "durable" c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize) defer c.shutdown() createStreamForConsumerChaosTest(t, c, replicas, numMessages) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, // Whole cluster outage maxDownServers: clusterSize, pause: 1 * time.Second, }, ) subNc, subJs := jsClientConnectCluster(t, c) defer subNc.Close() sub, err := subJs.PullSubscribe( chaosConsumerTestsSubject, durableConsumerName, ) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() if chaosConsumerTestsDebug { t.Logf("Initial subscription: %s", toIndentedJsonString(sub)) } chaos.start() defer chaos.stop() fetchMaxWait := nats.MaxWait(fetchTimeout) received := NewBitset(numMessages) deliveredCount := uint64(0) for received.count() < numMessages { var msgs []*nats.Msg var fetchErr error fetchRetryLoop: for r := 0; r <= maxRetries; r++ { msgs, fetchErr = sub.Fetch(fetchBatchSize, fetchMaxWait) if fetchErr == nil { break fetchRetryLoop } else if r == maxRetries { t.Fatalf("Exceeded max retries for Fetch, last error: %s", fetchErr) } else if fetchErr == nats.ErrBadSubscription { t.Fatalf("Subscription is invalid: %s", toIndentedJsonString(sub)) } else { // t.Logf("Fetch error: %v", fetchErr) time.Sleep(retryDelay) } } for _, msg := range msgs { deliveredCount += 1 metadata, err := msg.Metadata() if err != nil { t.Fatalf("Failed to get message metadata: %v", err) } streamSeq := metadata.Sequence.Stream expectedMsgData := []byte(fmt.Sprintf("%d", streamSeq)) if !bytes.Equal(msg.Data, expectedMsgData) { t.Fatalf("Expecting message %s, got %s instead", expectedMsgData, msg.Data) } isDupe := received.get(streamSeq - 1) received.set(streamSeq-1, true) // Simulate application processing (and gives the monkey some time to brew chaos) time.Sleep(10 * time.Millisecond) ackRetryLoop: for r := 0; r <= maxRetries; r++ { ackErr := msg.Ack() if ackErr == nil { break ackRetryLoop } else if r == maxRetries { t.Fatalf("Failed to ACK message %d, last error: %s", streamSeq, ackErr) } else { time.Sleep(retryDelay) } } if !isDupe && received.count()%numBatch == 0 { t.Logf("Consumed %d/%d (duplicates: %d)", received.count(), numMessages, deliveredCount-received.count()) } } } } const ( chaosKvTestsClusterName = "KV_CHAOS_TEST" chaosKvTestsBucketName = "KV_CHAOS_TEST_BUCKET" chaosKvTestsSubject = "foo" chaosKvTestsDebug = false ) // Creates KV store (a.k.a. bucket). func createBucketForKvChaosTest(t *testing.T, c *cluster, replicas int) { t.Helper() pubNc, pubJs := jsClientConnectCluster(t, c) defer pubNc.Close() config := nats.KeyValueConfig{ Bucket: chaosKvTestsBucketName, Replicas: replicas, Description: "Test bucket", } kvs, err := pubJs.CreateKeyValue(&config) if err != nil { t.Fatalf("Error creating bucket: %v", err) } status, err := kvs.Status() if err != nil { t.Fatalf("Error retrieving bucket status: %v", err) } t.Logf("Bucket created: %s", status.Bucket()) } // Single client performs a set of PUT on a single key. // If PUT is successful, perform a GET on the same key. // If GET is successful, ensure key revision and value match the most recent successful write. func TestJetStreamChaosKvPutGet(t *testing.T) { const numOps = 100_000 const clusterSize = 3 const replicas = 3 const key = "key" const staleReadsOk = true // Set to false to check for violations of 'read committed' consistency c := createJetStreamClusterExplicit(t, chaosKvTestsClusterName, clusterSize) defer c.shutdown() createBucketForKvChaosTest(t, c, replicas) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, // Whole cluster outage maxDownServers: clusterSize, pause: 1 * time.Second, }, ) nc, js := jsClientConnectCluster(t, c) defer nc.Close() // Create KV bucket kv, err := js.KeyValue(chaosKvTestsBucketName) if err != nil { t.Fatalf("Failed to get KV store: %v", err) } // Initialize the only key firstRevision, err := kv.Create(key, []byte("INITIAL VALUE")) if err != nil { t.Fatalf("Failed to create key: %v", err) } else if firstRevision != 1 { t.Fatalf("Unexpected revision: %d", firstRevision) } // Start chaos chaos.start() defer chaos.stop() staleReadsCount := uint64(0) successCount := uint64(0) previousRevision := firstRevision putGetLoop: for i := 1; i <= numOps; i++ { if i%1000 == 0 { t.Logf("Completed %d/%d PUT+GET operations", i, numOps) } // PUT a value putValue := fmt.Sprintf("value-%d", i) putRevision, err := kv.Put(key, []byte(putValue)) if err != nil { t.Logf("PUT error: %v", err) continue putGetLoop } // Check revision is monotonically increasing if putRevision <= previousRevision { t.Fatalf("PUT produced revision %d which is not greater than the previous successful PUT revision: %d", putRevision, previousRevision) } previousRevision = putRevision // If PUT was successful, GET the same kve, err := kv.Get(key) if err == nats.ErrKeyNotFound { t.Fatalf("GET key not found, but key does exists (last PUT revision: %d)", putRevision) } else if err != nil { t.Logf("GET error: %v", err) continue putGetLoop } getValue := string(kve.Value()) getRevision := kve.Revision() if putRevision > getRevision { // Stale read, violates 'read committed' consistency criteria if !staleReadsOk { t.Fatalf("PUT value %s (rev: %d) then read value %s (rev: %d)", putValue, putRevision, getValue, getRevision) } else { staleReadsCount += 1 } } else if putRevision < getRevision { // Returned revision is higher than any ever written, this should never happen t.Fatalf("GET returned revision %d, but most recent expected revision is %d", getRevision, putRevision) } else if putValue != getValue { // Returned revision matches latest, but values do not, this should never happen t.Fatalf("GET returned revision %d with value %s, but value %s was just committed for that same revision", getRevision, getValue, putValue) } else { // Get returned the latest revision/value successCount += 1 if chaosKvTestsDebug { t.Logf("PUT+GET %s=%s (rev: %d)", key, putValue, putRevision) } } } t.Logf("Completed %d PUT+GET cycles of which %d successful, %d GETs returned a stale value", numOps, successCount, staleReadsCount) } // A variant TestJetStreamChaosKvPutGet where PUT is retried until successful, and GET is retried until it returns the latest known key revision. // This validates than a confirmed PUT value is never lost, and becomes eventually visible. func TestJetStreamChaosKvPutGetWithRetries(t *testing.T) { const numOps = 10_000 const maxRetries = 20 const retryDelay = 100 * time.Millisecond const clusterSize = 3 const replicas = 3 const key = "key" c := createJetStreamClusterExplicit(t, chaosKvTestsClusterName, clusterSize) defer c.shutdown() createBucketForKvChaosTest(t, c, replicas) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, // Whole cluster outage maxDownServers: clusterSize, pause: 1 * time.Second, }, ) nc, js := jsClientConnectCluster(t, c) defer nc.Close() kv, err := js.KeyValue(chaosKvTestsBucketName) if err != nil { t.Fatalf("Failed to get KV store: %v", err) } // Initialize key value firstRevision, err := kv.Create(key, []byte("INITIAL VALUE")) if err != nil { t.Fatalf("Failed to create key: %v", err) } else if firstRevision != 1 { t.Fatalf("Unexpected revision: %d", firstRevision) } // Start chaos chaos.start() defer chaos.stop() staleReadCount := 0 previousRevision := firstRevision putGetLoop: for i := 1; i <= numOps; i++ { if i%1000 == 0 { t.Logf("Completed %d/%d PUT+GET operations", i, numOps) } putValue := fmt.Sprintf("value-%d", i) putRevision := uint64(0) // Put new value for key, retry until successful or out of retries putRetryLoop: for r := 0; r <= maxRetries; r++ { var putErr error putRevision, putErr = kv.Put(key, []byte(putValue)) if putErr == nil { break putRetryLoop } else if r == maxRetries { t.Fatalf("Failed to PUT (retried %d times): %v", maxRetries, putErr) } else { if chaosKvTestsDebug { t.Logf("PUT error: %v", putErr) } time.Sleep(retryDelay) } } // Ensure key version is monotonically increasing if putRevision <= previousRevision { t.Fatalf("Latest PUT created revision %d which is not greater than the previous revision: %d", putRevision, previousRevision) } previousRevision = putRevision // Read value for key, retry until successful, and validate corresponding version and value getRetryLoop: for r := 0; r <= maxRetries; r++ { var getErr error kve, getErr := kv.Get(key) if getErr != nil && r == maxRetries { t.Fatalf("Failed to GET (retried %d times): %v", maxRetries, getErr) } else if getErr != nil { if chaosKvTestsDebug { t.Logf("GET error: %v", getErr) } time.Sleep(retryDelay) continue getRetryLoop } // GET successful, check revision and value getValue := string(kve.Value()) getRevision := kve.Revision() if putRevision == getRevision { if putValue != getValue { t.Fatalf("Unexpected value %s for revision %d, expected: %s", getValue, getRevision, putValue) } if chaosKvTestsDebug { t.Logf("PUT+GET %s=%s (rev: %d) (retry: %d)", key, putValue, putRevision, r) } continue putGetLoop } else if getRevision > putRevision { t.Fatalf("GET returned version that should not exist yet: %d, last created: %d", getRevision, putRevision) } else { // get revision < put revision staleReadCount += 1 if chaosKvTestsDebug { t.Logf("GET got stale value: %v (rev: %d, latest: %d)", getValue, getRevision, putRevision) } time.Sleep(retryDelay) continue getRetryLoop } } } t.Logf("Client completed %d PUT+GET cycles, %d GET returned a stale value", numOps, staleReadCount) } // Multiple clients updating a finite set of keys with CAS semantics. // TODO check that revision is never lower than last one seen // TODO check that KeyNotFound is never returned, as keys are initialized beforehand func TestJetStreamChaosKvCAS(t *testing.T) { const numOps = 10_000 const maxRetries = 50 const retryDelay = 300 * time.Millisecond const clusterSize = 3 const replicas = 3 const numKeys = 15 const numClients = 5 c := createJetStreamClusterExplicit(t, chaosKvTestsClusterName, clusterSize) defer c.shutdown() createBucketForKvChaosTest(t, c, replicas) chaos := createClusterChaosMonkeyController( t, c, &clusterBouncerChaosMonkey{ minDowntime: 0 * time.Second, maxDowntime: 2 * time.Second, minDownServers: clusterSize, // Whole cluster outage maxDownServers: clusterSize, pause: 1 * time.Second, }, ) nc, js := jsClientConnectCluster(t, c) defer nc.Close() // Create bucket kv, err := js.KeyValue(chaosKvTestsBucketName) if err != nil { t.Fatalf("Failed to get KV store: %v", err) } // Create set of keys and initialize them with dummy value keys := make([]string, numKeys) for k := 0; k < numKeys; k++ { key := fmt.Sprintf("key-%d", k) keys[k] = key _, err := kv.Create(key, []byte("Initial value")) if err != nil { t.Fatalf("Failed to create key: %v", err) } } wgStart := sync.WaitGroup{} wgComplete := sync.WaitGroup{} // Client routine client := func(clientId int, kv nats.KeyValue) { defer wgComplete.Done() rng := rand.New(rand.NewSource(int64(clientId))) successfulUpdates := 0 casRejectUpdates := 0 otherUpdateErrors := 0 // Map to track last known revision for each of the keys knownRevisions := map[string]uint64{} for _, key := range keys { knownRevisions[key] = 0 } // Wait for all clients to reach this point before proceeding wgStart.Done() wgStart.Wait() for i := 1; i <= numOps; i++ { if i%1000 == 0 { t.Logf("Client %d completed %d/%d updates", clientId, i, numOps) } // Pick random key from the set key := keys[rng.Intn(numKeys)] // Prepare unique value to be written value := fmt.Sprintf("client: %d operation %d", clientId, i) // Try to update a key with CAS newRevision, updateErr := kv.Update(key, []byte(value), knownRevisions[key]) if updateErr == nil { // Update successful knownRevisions[key] = newRevision successfulUpdates += 1 if chaosKvTestsDebug { t.Logf("Client %d updated key %s, new revision: %d", clientId, key, newRevision) } } else if updateErr != nil && strings.Contains(fmt.Sprint(updateErr), "wrong last sequence") { // CAS rejected update, learn current revision for this key casRejectUpdates += 1 for r := 0; r <= maxRetries; r++ { kve, getErr := kv.Get(key) if getErr == nil { currentRevision := kve.Revision() if currentRevision < knownRevisions[key] { // Revision number moved backward, this should never happen t.Fatalf("Current revision for key %s is %d, which is lower than the last known revision %d", key, currentRevision, knownRevisions[key]) } knownRevisions[key] = currentRevision if chaosKvTestsDebug { t.Logf("Client %d learn key %s revision: %d", clientId, key, currentRevision) } break } else if r == maxRetries { t.Fatalf("Failed to GET (retried %d times): %v", maxRetries, getErr) } else { time.Sleep(retryDelay) } } } else { // Other update error otherUpdateErrors += 1 if chaosKvTestsDebug { t.Logf("Client %d update error for key %s: %v", clientId, key, updateErr) } time.Sleep(retryDelay) } } t.Logf("Client %d done, %d kv updates, %d CAS rejected, %d other errors", clientId, successfulUpdates, casRejectUpdates, otherUpdateErrors) } // Launch all clients for i := 1; i <= numClients; i++ { cNc, cJs := jsClientConnectCluster(t, c) defer cNc.Close() cKv, err := cJs.KeyValue(chaosKvTestsBucketName) if err != nil { t.Fatalf("Failed to get KV store: %v", err) } wgStart.Add(1) wgComplete.Add(1) go client(i, cKv) } // Wait for clients to be connected and ready wgStart.Wait() // Start failures chaos.start() defer chaos.stop() // Wait for all clients to be done wgComplete.Wait() } nats-server-2.10.27/server/jetstream_cluster.go000066400000000000000000010133401477524627100215510ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" crand "crypto/rand" "encoding/binary" "encoding/json" "errors" "fmt" "math" "math/rand" "os" "path/filepath" "reflect" "slices" "strconv" "strings" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/minio/highwayhash" "github.com/nats-io/nuid" ) // jetStreamCluster holds information about the meta group and stream assignments. type jetStreamCluster struct { // The metacontroller raftNode. meta RaftNode // For stream and consumer assignments. All servers will have this be the same. // ACCOUNT -> STREAM -> Stream Assignment -> Consumers streams map[string]map[string]*streamAssignment // These are inflight proposals and used to apply limits when there are // concurrent requests that would otherwise be accepted. // We also record the group for the stream. This is needed since if we have // concurrent requests for same account and stream we need to let it process to get // a response but they need to be same group, peers etc. and sync subjects. inflight map[string]map[string]*inflightInfo // Signals meta-leader should check the stream assignments. streamsCheck bool // Server. s *Server // Internal client. c *client // Processing assignment results. streamResults *subscription consumerResults *subscription // System level request to have the leader stepdown. stepdown *subscription // System level requests to remove a peer. peerRemove *subscription // System level request to move a stream peerStreamMove *subscription // System level request to cancel a stream move peerStreamCancelMove *subscription // To pop out the monitorCluster before the raft layer. qch chan struct{} } // Used to track inflight stream add requests to properly re-use same group and sync subject. type inflightInfo struct { rg *raftGroup sync string } // Used to guide placement of streams and meta controllers in clustered JetStream. type Placement struct { Cluster string `json:"cluster,omitempty"` Tags []string `json:"tags,omitempty"` } // Define types of the entry. type entryOp uint8 // ONLY ADD TO THE END, DO NOT INSERT IN BETWEEN WILL BREAK SERVER INTEROP. const ( // Meta ops. assignStreamOp entryOp = iota assignConsumerOp removeStreamOp removeConsumerOp // Stream ops. streamMsgOp purgeStreamOp deleteMsgOp // Consumer ops. updateDeliveredOp updateAcksOp // Compressed consumer assignments. assignCompressedConsumerOp // Filtered Consumer skip. updateSkipOp // Update Stream. updateStreamOp // For updating information on pending pull requests. addPendingRequest removePendingRequest // For sending compressed streams, either through RAFT or catchup. compressedStreamMsgOp // For sending deleted gaps on catchups for replicas. deleteRangeOp ) // raftGroups are controlled by the metagroup controller. // The raftGroups will house streams and consumers. type raftGroup struct { Name string `json:"name"` Peers []string `json:"peers"` Storage StorageType `json:"store"` Cluster string `json:"cluster,omitempty"` Preferred string `json:"preferred,omitempty"` // Internal node RaftNode } // streamAssignment is what the meta controller uses to assign streams to peers. type streamAssignment struct { Client *ClientInfo `json:"client,omitempty"` Created time.Time `json:"created"` Config *StreamConfig `json:"stream"` Group *raftGroup `json:"group"` Sync string `json:"sync"` Subject string `json:"subject,omitempty"` Reply string `json:"reply,omitempty"` Restore *StreamState `json:"restore_state,omitempty"` // Internal consumers map[string]*consumerAssignment responded bool recovering bool reassigning bool // i.e. due to placement issues, lack of resources, etc. resetting bool // i.e. there was an error, and we're stopping and starting the stream err error } // consumerAssignment is what the meta controller uses to assign consumers to streams. type consumerAssignment struct { Client *ClientInfo `json:"client,omitempty"` Created time.Time `json:"created"` Name string `json:"name"` Stream string `json:"stream"` Config *ConsumerConfig `json:"consumer"` Group *raftGroup `json:"group"` Subject string `json:"subject,omitempty"` Reply string `json:"reply,omitempty"` State *ConsumerState `json:"state,omitempty"` // Internal responded bool recovering bool pending bool deleted bool err error } // streamPurge is what the stream leader will replicate when purging a stream. type streamPurge struct { Client *ClientInfo `json:"client,omitempty"` Stream string `json:"stream"` LastSeq uint64 `json:"last_seq"` Subject string `json:"subject"` Reply string `json:"reply"` Request *JSApiStreamPurgeRequest `json:"request,omitempty"` } // streamMsgDelete is what the stream leader will replicate when deleting a message. type streamMsgDelete struct { Client *ClientInfo `json:"client,omitempty"` Stream string `json:"stream"` Seq uint64 `json:"seq"` NoErase bool `json:"no_erase,omitempty"` Subject string `json:"subject"` Reply string `json:"reply"` } const ( defaultStoreDirName = "_js_" defaultMetaGroupName = "_meta_" defaultMetaFSBlkSize = 1024 * 1024 jsExcludePlacement = "!jetstream" ) // Returns information useful in mixed mode. func (s *Server) trackedJetStreamServers() (js, total int) { s.mu.RLock() defer s.mu.RUnlock() if !s.isRunning() || !s.eventsEnabled() { return -1, -1 } s.nodeToInfo.Range(func(k, v any) bool { si := v.(nodeInfo) if si.js { js++ } total++ return true }) return js, total } func (s *Server) getJetStreamCluster() (*jetStream, *jetStreamCluster) { if s.isShuttingDown() { return nil, nil } js := s.getJetStream() if js == nil { return nil, nil } // Only set once, do not need a lock. return js, js.cluster } func (s *Server) JetStreamIsClustered() bool { return s.jsClustered.Load() } func (s *Server) JetStreamIsLeader() bool { return s.isMetaLeader.Load() } func (s *Server) JetStreamIsCurrent() bool { js := s.getJetStream() if js == nil { return false } // Grab what we need and release js lock. js.mu.RLock() var meta RaftNode cc := js.cluster if cc != nil { meta = cc.meta } js.mu.RUnlock() if cc == nil { // Non-clustered mode return true } return meta.Current() } func (s *Server) JetStreamSnapshotMeta() error { js := s.getJetStream() if js == nil { return NewJSNotEnabledError() } js.mu.RLock() cc := js.cluster isLeader := cc.isLeader() meta := cc.meta js.mu.RUnlock() if !isLeader { return errNotLeader } snap, err := js.metaSnapshot() if err != nil { return err } return meta.InstallSnapshot(snap) } func (s *Server) JetStreamStepdownStream(account, stream string) error { js, cc := s.getJetStreamCluster() if js == nil { return NewJSNotEnabledError() } if cc == nil { return NewJSClusterNotActiveError() } // Grab account acc, err := s.LookupAccount(account) if err != nil { return err } // Grab stream mset, err := acc.lookupStream(stream) if err != nil { return err } if node := mset.raftNode(); node != nil && node.Leader() { node.StepDown() } return nil } func (s *Server) JetStreamStepdownConsumer(account, stream, consumer string) error { js, cc := s.getJetStreamCluster() if js == nil { return NewJSNotEnabledError() } if cc == nil { return NewJSClusterNotActiveError() } // Grab account acc, err := s.LookupAccount(account) if err != nil { return err } // Grab stream mset, err := acc.lookupStream(stream) if err != nil { return err } o := mset.lookupConsumer(consumer) if o == nil { return NewJSConsumerNotFoundError() } if node := o.raftNode(); node != nil && node.Leader() { node.StepDown() } return nil } func (s *Server) JetStreamSnapshotStream(account, stream string) error { js, cc := s.getJetStreamCluster() if js == nil { return NewJSNotEnabledForAccountError() } if cc == nil { return NewJSClusterNotActiveError() } // Grab account acc, err := s.LookupAccount(account) if err != nil { return err } // Grab stream mset, err := acc.lookupStream(stream) if err != nil { return err } // Hold lock when installing snapshot. mset.mu.Lock() if mset.node == nil { mset.mu.Unlock() return nil } err = mset.node.InstallSnapshot(mset.stateSnapshotLocked()) mset.mu.Unlock() return err } func (s *Server) JetStreamClusterPeers() []string { js := s.getJetStream() if js == nil { return nil } js.mu.RLock() defer js.mu.RUnlock() cc := js.cluster if !cc.isLeader() || cc.meta == nil { return nil } peers := cc.meta.Peers() var nodes []string for _, p := range peers { si, ok := s.nodeToInfo.Load(p.ID) if !ok || si == nil { continue } ni := si.(nodeInfo) // Ignore if offline, no JS, or no current stats have been received. if ni.offline || !ni.js || ni.stats == nil { continue } nodes = append(nodes, si.(nodeInfo).name) } return nodes } // Read lock should be held. func (cc *jetStreamCluster) isLeader() bool { if cc == nil { // Non-clustered mode return true } return cc.meta != nil && cc.meta.Leader() } // isStreamCurrent will determine if the stream is up to date. // For R1 it will make sure the stream is present on this server. // Read lock should be held. func (cc *jetStreamCluster) isStreamCurrent(account, stream string) bool { if cc == nil { // Non-clustered mode return true } as := cc.streams[account] if as == nil { return false } sa := as[stream] if sa == nil { return false } rg := sa.Group if rg == nil { return false } if rg.node == nil || rg.node.Current() { // Check if we are processing a snapshot and are catching up. acc, err := cc.s.LookupAccount(account) if err != nil { return false } mset, err := acc.lookupStream(stream) if err != nil { return false } if mset.isCatchingUp() { return false } // Success. return true } return false } // isStreamHealthy will determine if the stream is up to date or very close. // For R1 it will make sure the stream is present on this server. func (js *jetStream) isStreamHealthy(acc *Account, sa *streamAssignment) error { js.mu.RLock() s, cc := js.srv, js.cluster if cc == nil { // Non-clustered mode js.mu.RUnlock() return nil } if sa == nil || sa.Group == nil { js.mu.RUnlock() return errors.New("stream assignment or group missing") } streamName := sa.Config.Name node := sa.Group.node js.mu.RUnlock() // First lookup stream and make sure its there. mset, err := acc.lookupStream(streamName) if err != nil { return errors.New("stream not found") } switch { case mset.cfg.Replicas <= 1: return nil // No further checks for R=1 streams case node == nil: return errors.New("group node missing") case node != mset.raftNode(): s.Warnf("Detected stream cluster node skew '%s > %s'", acc.GetName(), streamName) node.Delete() mset.resetClusteredState(nil) return errors.New("cluster node skew detected") case !mset.isMonitorRunning(): return errors.New("monitor goroutine not running") case !node.Healthy(): return errors.New("group node unhealthy") case mset.isCatchingUp(): return errors.New("stream catching up") default: return nil } } // isConsumerHealthy will determine if the consumer is up to date. // For R1 it will make sure the consunmer is present on this server. func (js *jetStream) isConsumerHealthy(mset *stream, consumer string, ca *consumerAssignment) error { if mset == nil { return errors.New("stream missing") } js.mu.RLock() s, cc := js.srv, js.cluster if cc == nil { // Non-clustered mode js.mu.RUnlock() return nil } if ca == nil || ca.Group == nil { js.mu.RUnlock() return errors.New("consumer assignment or group missing") } node := ca.Group.node js.mu.RUnlock() // Check if not running at all. o := mset.lookupConsumer(consumer) if o == nil { return errors.New("consumer not found") } rc, _ := o.replica() switch { case rc <= 1: return nil // No further checks for R=1 consumers case node == nil: return errors.New("group node missing") case node != o.raftNode(): mset.mu.RLock() accName, streamName := mset.acc.GetName(), mset.cfg.Name mset.mu.RUnlock() s.Warnf("Detected consumer cluster node skew '%s > %s > %s'", accName, streamName, consumer) node.Delete() o.deleteWithoutAdvisory() // When we try to restart we nil out the node and reprocess the consumer assignment. js.mu.Lock() ca.Group.node = nil js.mu.Unlock() js.processConsumerAssignment(ca) return errors.New("cluster node skew detected") case !o.isMonitorRunning(): return errors.New("monitor goroutine not running") case !node.Healthy(): return errors.New("group node unhealthy") default: return nil } } // subjectsOverlap checks all existing stream assignments for the account cross-cluster for subject overlap // Use only for clustered JetStream // Read lock should be held. func (jsc *jetStreamCluster) subjectsOverlap(acc string, subjects []string, osa *streamAssignment) bool { asa := jsc.streams[acc] for _, sa := range asa { // can't overlap yourself, assume osa pre-checked for deep equal if passed if osa != nil && sa == osa { continue } for _, subj := range sa.Config.Subjects { for _, tsubj := range subjects { if SubjectsCollide(tsubj, subj) { return true } } } } return false } func (a *Account) getJetStreamFromAccount() (*Server, *jetStream, *jsAccount) { a.mu.RLock() jsa := a.js a.mu.RUnlock() if jsa == nil { return nil, nil, nil } jsa.mu.RLock() js := jsa.js jsa.mu.RUnlock() if js == nil { return nil, nil, nil } // Lock not needed, set on creation. s := js.srv return s, js, jsa } func (s *Server) JetStreamIsStreamLeader(account, stream string) bool { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return cc.isStreamLeader(account, stream) } func (a *Account) JetStreamIsStreamLeader(stream string) bool { s, js, jsa := a.getJetStreamFromAccount() if s == nil || js == nil || jsa == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return js.cluster.isStreamLeader(a.Name, stream) } func (s *Server) JetStreamIsStreamCurrent(account, stream string) bool { js, cc := s.getJetStreamCluster() if js == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return cc.isStreamCurrent(account, stream) } func (a *Account) JetStreamIsConsumerLeader(stream, consumer string) bool { s, js, jsa := a.getJetStreamFromAccount() if s == nil || js == nil || jsa == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return js.cluster.isConsumerLeader(a.Name, stream, consumer) } func (s *Server) JetStreamIsConsumerLeader(account, stream, consumer string) bool { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return cc.isConsumerLeader(account, stream, consumer) } func (s *Server) enableJetStreamClustering() error { if !s.isRunning() { return nil } js := s.getJetStream() if js == nil { return NewJSNotEnabledForAccountError() } // Already set. if js.cluster != nil { return nil } s.Noticef("Starting JetStream cluster") // We need to determine if we have a stable cluster name and expected number of servers. s.Debugf("JetStream cluster checking for stable cluster name and peers") hasLeafNodeSystemShare := s.canExtendOtherDomain() if s.isClusterNameDynamic() && !hasLeafNodeSystemShare { return errors.New("JetStream cluster requires cluster name") } if s.configuredRoutes() == 0 && !hasLeafNodeSystemShare { return errors.New("JetStream cluster requires configured routes or solicited leafnode for the system account") } return js.setupMetaGroup() } // isClustered returns if we are clustered. // Lock should not be held. func (js *jetStream) isClustered() bool { // This is only ever set, no need for lock here. return js.cluster != nil } // isClusteredNoLock returns if we are clustered, but unlike isClustered() does // not use the jetstream's lock, instead, uses an atomic operation. // There are situations where some code wants to know if we are clustered but // can't use js.isClustered() without causing a lock inversion. func (js *jetStream) isClusteredNoLock() bool { return atomic.LoadInt32(&js.clustered) == 1 } func (js *jetStream) setupMetaGroup() error { s := js.srv s.Noticef("Creating JetStream metadata controller") // Setup our WAL for the metagroup. sysAcc := s.SystemAccount() if sysAcc == nil { return ErrNoSysAccount } storeDir := filepath.Join(js.config.StoreDir, sysAcc.Name, defaultStoreDirName, defaultMetaGroupName) js.srv.optsMu.RLock() syncAlways := js.srv.opts.SyncAlways syncInterval := js.srv.opts.SyncInterval js.srv.optsMu.RUnlock() fs, err := newFileStoreWithCreated( FileStoreConfig{StoreDir: storeDir, BlockSize: defaultMetaFSBlkSize, AsyncFlush: false, SyncAlways: syncAlways, SyncInterval: syncInterval, srv: s}, StreamConfig{Name: defaultMetaGroupName, Storage: FileStorage}, time.Now().UTC(), s.jsKeyGen(s.getOpts().JetStreamKey, defaultMetaGroupName), s.jsKeyGen(s.getOpts().JetStreamOldKey, defaultMetaGroupName), ) if err != nil { s.Errorf("Error creating filestore: %v", err) return err } cfg := &RaftConfig{Name: defaultMetaGroupName, Store: storeDir, Log: fs} // If we are soliciting leafnode connections and we are sharing a system account and do not disable it with a hint, // we want to move to observer mode so that we extend the solicited cluster or supercluster but do not form our own. cfg.Observer = s.canExtendOtherDomain() && s.getOpts().JetStreamExtHint != jsNoExtend var bootstrap bool if ps, err := readPeerState(storeDir); err != nil { s.Noticef("JetStream cluster bootstrapping") bootstrap = true peers := s.ActivePeers() s.Debugf("JetStream cluster initial peers: %+v", peers) if err := s.bootstrapRaftNode(cfg, peers, false); err != nil { return err } if cfg.Observer { s.Noticef("Turning JetStream metadata controller Observer Mode on") } } else { s.Noticef("JetStream cluster recovering state") // correlate the value of observer with observations from a previous run. if cfg.Observer { switch ps.domainExt { case extExtended: s.Noticef("Keeping JetStream metadata controller Observer Mode on - due to previous contact") case extNotExtended: s.Noticef("Turning JetStream metadata controller Observer Mode off - due to previous contact") cfg.Observer = false case extUndetermined: s.Noticef("Turning JetStream metadata controller Observer Mode on - no previous contact") s.Noticef("In cases where JetStream will not be extended") s.Noticef("and waiting for leader election until first contact is not acceptable,") s.Noticef(`manually disable Observer Mode by setting the JetStream Option "extension_hint: %s"`, jsNoExtend) } } else { // To track possible configuration changes, responsible for an altered value of cfg.Observer, // set extension state to undetermined. ps.domainExt = extUndetermined if err := writePeerState(storeDir, ps); err != nil { return err } } } // Start up our meta node. n, err := s.startRaftNode(sysAcc.GetName(), cfg, pprofLabels{ "type": "metaleader", "account": sysAcc.Name, }) if err != nil { s.Warnf("Could not start metadata controller: %v", err) return err } // If we are bootstrapped with no state, start campaign early. if bootstrap { n.Campaign() } c := s.createInternalJetStreamClient() js.mu.Lock() defer js.mu.Unlock() js.cluster = &jetStreamCluster{ meta: n, streams: make(map[string]map[string]*streamAssignment), s: s, c: c, qch: make(chan struct{}), } atomic.StoreInt32(&js.clustered, 1) c.registerWithAccount(sysAcc) // Set to true before we start. js.metaRecovering = true js.srv.startGoRoutine( js.monitorCluster, pprofLabels{ "type": "metaleader", "account": sysAcc.Name, }, ) return nil } func (js *jetStream) getMetaGroup() RaftNode { js.mu.RLock() defer js.mu.RUnlock() if js.cluster == nil { return nil } return js.cluster.meta } func (js *jetStream) server() *Server { // Lock not needed, only set once on creation. return js.srv } // Will respond if we do not think we have a metacontroller leader. func (js *jetStream) isLeaderless() bool { js.mu.RLock() cc := js.cluster if cc == nil || cc.meta == nil { js.mu.RUnlock() return false } meta := cc.meta js.mu.RUnlock() // If we don't have a leader. // Make sure we have been running for enough time. if meta.Leaderless() && time.Since(meta.Created()) > lostQuorumIntervalDefault { return true } return false } // Will respond iff we are a member and we know we have no leader. func (js *jetStream) isGroupLeaderless(rg *raftGroup) bool { if rg == nil || js == nil { return false } js.mu.RLock() cc := js.cluster started := js.started // If we are not a member we can not say.. if cc.meta == nil { js.mu.RUnlock() return false } if !rg.isMember(cc.meta.ID()) { js.mu.RUnlock() return false } // Single peer groups always have a leader if we are here. if rg.node == nil { js.mu.RUnlock() return false } node := rg.node js.mu.RUnlock() // If we don't have a leader. if node.Leaderless() { // Threshold for jetstream startup. const startupThreshold = 10 * time.Second if node.HadPreviousLeader() { // Make sure we have been running long enough to intelligently determine this. if time.Since(started) > startupThreshold { return true } } // Make sure we have been running for enough time. if time.Since(node.Created()) > lostQuorumIntervalDefault { return true } } return false } func (s *Server) JetStreamIsStreamAssigned(account, stream string) bool { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return false } acc, _ := s.LookupAccount(account) if acc == nil { return false } js.mu.RLock() assigned := cc.isStreamAssigned(acc, stream) js.mu.RUnlock() return assigned } // streamAssigned informs us if this server has this stream assigned. func (jsa *jsAccount) streamAssigned(stream string) bool { jsa.mu.RLock() js, acc := jsa.js, jsa.account jsa.mu.RUnlock() if js == nil { return false } js.mu.RLock() assigned := js.cluster.isStreamAssigned(acc, stream) js.mu.RUnlock() return assigned } // Read lock should be held. func (cc *jetStreamCluster) isStreamAssigned(a *Account, stream string) bool { // Non-clustered mode always return true. if cc == nil { return true } if cc.meta == nil { return false } as := cc.streams[a.Name] if as == nil { return false } sa := as[stream] if sa == nil { return false } rg := sa.Group if rg == nil { return false } // Check if we are the leader of this raftGroup assigned to the stream. ourID := cc.meta.ID() for _, peer := range rg.Peers { if peer == ourID { return true } } return false } // Read lock should be held. func (cc *jetStreamCluster) isStreamLeader(account, stream string) bool { // Non-clustered mode always return true. if cc == nil { return true } if cc.meta == nil { return false } var sa *streamAssignment if as := cc.streams[account]; as != nil { sa = as[stream] } if sa == nil { return false } rg := sa.Group if rg == nil { return false } // Check if we are the leader of this raftGroup assigned to the stream. ourID := cc.meta.ID() for _, peer := range rg.Peers { if peer == ourID { if len(rg.Peers) == 1 || (rg.node != nil && rg.node.Leader()) { return true } } } return false } // Read lock should be held. func (cc *jetStreamCluster) isConsumerLeader(account, stream, consumer string) bool { // Non-clustered mode always return true. if cc == nil { return true } if cc.meta == nil { return false } var sa *streamAssignment if as := cc.streams[account]; as != nil { sa = as[stream] } if sa == nil { return false } // Check if we are the leader of this raftGroup assigned to this consumer. ca := sa.consumers[consumer] if ca == nil { return false } rg := ca.Group ourID := cc.meta.ID() for _, peer := range rg.Peers { if peer == ourID { if len(rg.Peers) == 1 || (rg.node != nil && rg.node.Leader()) { return true } } } return false } // Remove the stream `streamName` for the account `accName` from the inflight // proposals map. This is done on success (processStreamAssignment) or on // failure (processStreamAssignmentResults). // (Write) Lock held on entry. func (cc *jetStreamCluster) removeInflightProposal(accName, streamName string) { streams, ok := cc.inflight[accName] if !ok { return } delete(streams, streamName) if len(streams) == 0 { delete(cc.inflight, accName) } } // Return the cluster quit chan. func (js *jetStream) clusterQuitC() chan struct{} { js.mu.RLock() defer js.mu.RUnlock() if js.cluster != nil { return js.cluster.qch } return nil } // Mark that the meta layer is recovering. func (js *jetStream) setMetaRecovering() { js.mu.Lock() defer js.mu.Unlock() if js.cluster != nil { // metaRecovering js.metaRecovering = true } } // Mark that the meta layer is no longer recovering. func (js *jetStream) clearMetaRecovering() { js.mu.Lock() defer js.mu.Unlock() js.metaRecovering = false } // Return whether the meta layer is recovering. func (js *jetStream) isMetaRecovering() bool { js.mu.RLock() defer js.mu.RUnlock() return js.metaRecovering } // During recovery track any stream and consumer delete and update operations. type recoveryUpdates struct { removeStreams map[string]*streamAssignment removeConsumers map[string]map[string]*consumerAssignment addStreams map[string]*streamAssignment updateStreams map[string]*streamAssignment updateConsumers map[string]map[string]*consumerAssignment } // Called after recovery of the cluster on startup to check for any orphans. // Streams and consumers are recovered from disk, and the meta layer's mappings // should clean them up, but under crash scenarios there could be orphans. func (js *jetStream) checkForOrphans() { // Can not hold jetstream lock while trying to delete streams or consumers. js.mu.Lock() s, cc := js.srv, js.cluster s.Debugf("JetStream cluster checking for orphans") // We only want to cleanup any orphans if we know we are current with the meta-leader. meta := cc.meta if meta == nil || meta.Leaderless() { js.mu.Unlock() s.Debugf("JetStream cluster skipping check for orphans, no meta-leader") return } if !meta.Healthy() { js.mu.Unlock() s.Debugf("JetStream cluster skipping check for orphans, not current with the meta-leader") return } var streams []*stream var consumers []*consumer for accName, jsa := range js.accounts { asa := cc.streams[accName] jsa.mu.RLock() for stream, mset := range jsa.streams { if sa := asa[stream]; sa == nil { streams = append(streams, mset) } else { // This one is good, check consumers now. for _, o := range mset.getConsumers() { if sa.consumers[o.String()] == nil { consumers = append(consumers, o) } } } } jsa.mu.RUnlock() } js.mu.Unlock() for _, mset := range streams { mset.mu.RLock() accName, stream := mset.acc.Name, mset.cfg.Name mset.mu.RUnlock() s.Warnf("Detected orphaned stream '%s > %s', will cleanup", accName, stream) if err := mset.delete(); err != nil { s.Warnf("Deleting stream encountered an error: %v", err) } } for _, o := range consumers { o.mu.RLock() accName, mset, consumer := o.acc.Name, o.mset, o.name o.mu.RUnlock() stream := "N/A" if mset != nil { mset.mu.RLock() stream = mset.cfg.Name mset.mu.RUnlock() } if o.isDurable() { s.Warnf("Detected orphaned durable consumer '%s > %s > %s', will cleanup", accName, stream, consumer) } else { s.Debugf("Detected orphaned consumer '%s > %s > %s', will cleanup", accName, stream, consumer) } if err := o.delete(); err != nil { s.Warnf("Deleting consumer encountered an error: %v", err) } } } // Check and delete any orphans we may come across. func (s *Server) checkForNRGOrphans() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil || js.isMetaRecovering() { // No cluster means no NRGs. Also return if still recovering. return } // Track which assets R>1 should be on this server. nrgMap := make(map[string]struct{}) trackGroup := func(rg *raftGroup) { // If R>1 track this as a legit NRG. if rg.node != nil { nrgMap[rg.Name] = struct{}{} } } // Register our meta. js.mu.RLock() meta := cc.meta if meta == nil { js.mu.RUnlock() // Bail with no meta node. return } ourID := meta.ID() nrgMap[meta.Group()] = struct{}{} // Collect all valid groups from our assignments. for _, asa := range cc.streams { for _, sa := range asa { if sa.Group.isMember(ourID) && sa.Restore == nil { trackGroup(sa.Group) for _, ca := range sa.consumers { if ca.Group.isMember(ourID) { trackGroup(ca.Group) } } } } } js.mu.RUnlock() // Check NRGs that are running. var needDelete []RaftNode s.rnMu.RLock() for name, n := range s.raftNodes { if _, ok := nrgMap[name]; !ok { needDelete = append(needDelete, n) } } s.rnMu.RUnlock() for _, n := range needDelete { s.Warnf("Detected orphaned NRG %q, will cleanup", n.Group()) n.Delete() } } func (js *jetStream) monitorCluster() { s, n := js.server(), js.getMetaGroup() qch, rqch, lch, aq := js.clusterQuitC(), n.QuitC(), n.LeadChangeC(), n.ApplyQ() defer s.grWG.Done() s.Debugf("Starting metadata monitor") defer s.Debugf("Exiting metadata monitor") // Make sure to stop the raft group on exit to prevent accidental memory bloat. defer n.Stop() defer s.isMetaLeader.Store(false) const compactInterval = time.Minute t := time.NewTicker(compactInterval) defer t.Stop() // Used to check cold boot cluster when possibly in mixed mode. const leaderCheckInterval = time.Second lt := time.NewTicker(leaderCheckInterval) defer lt.Stop() // Check the general health once an hour. const healthCheckInterval = 1 * time.Hour ht := time.NewTicker(healthCheckInterval) defer ht.Stop() // Utility to check health. checkHealth := func() { if hs := s.healthz(nil); hs.Error != _EMPTY_ { s.Warnf("%v", hs.Error) } // Also check for orphaned NRGs. s.checkForNRGOrphans() } var ( isLeader bool lastSnapTime time.Time compactSizeMin = uint64(8 * 1024 * 1024) // 8MB minSnapDelta = 30 * time.Second ) // Highwayhash key for generating hashes. key := make([]byte, 32) crand.Read(key) // Set to true to start. js.setMetaRecovering() // Snapshotting function. doSnapshot := func() { // Suppress during recovery. if js.isMetaRecovering() { return } // For the meta layer we want to snapshot when asked if we need one or have any entries that we can compact. if ne, _ := n.Size(); ne > 0 || n.NeedSnapshot() { snap, err := js.metaSnapshot() if err != nil { s.Warnf("Error generating JetStream cluster snapshot: %v", err) } else if err = n.InstallSnapshot(snap); err == nil { lastSnapTime = time.Now() } else if err != errNoSnapAvailable && err != errNodeClosed { s.Warnf("Error snapshotting JetStream cluster state: %v", err) } } } ru := &recoveryUpdates{ removeStreams: make(map[string]*streamAssignment), removeConsumers: make(map[string]map[string]*consumerAssignment), addStreams: make(map[string]*streamAssignment), updateStreams: make(map[string]*streamAssignment), updateConsumers: make(map[string]map[string]*consumerAssignment), } // Make sure to cancel any pending checkForOrphans calls if the // monitor goroutine exits. var oc *time.Timer defer stopAndClearTimer(&oc) for { select { case <-s.quitCh: // Server shutting down, but we might receive this before qch, so try to snapshot. doSnapshot() return case <-rqch: return case <-qch: // Clean signal from shutdown routine so do best effort attempt to snapshot meta layer. doSnapshot() // Return the signal back since shutdown will be waiting. close(qch) return case <-aq.ch: ces := aq.pop() for _, ce := range ces { if ce == nil { // Process any removes that are still valid after recovery. for _, cas := range ru.removeConsumers { for _, ca := range cas { js.processConsumerRemoval(ca) } } for _, sa := range ru.removeStreams { js.processStreamRemoval(sa) } // Process stream additions. for _, sa := range ru.addStreams { js.processStreamAssignment(sa) } // Process pending updates. for _, sa := range ru.updateStreams { js.processUpdateStreamAssignment(sa) } // Now consumers. for _, cas := range ru.updateConsumers { for _, ca := range cas { js.processConsumerAssignment(ca) } } // Signals we have replayed all of our metadata. js.clearMetaRecovering() // Clear. ru = nil s.Debugf("Recovered JetStream cluster metadata") oc = time.AfterFunc(30*time.Second, js.checkForOrphans) // Do a health check here as well. go checkHealth() continue } if didSnap, didStreamRemoval, _, err := js.applyMetaEntries(ce.Entries, ru); err == nil { var nb uint64 // Some entries can fail without an error when shutting down, don't move applied forward. if !js.isShuttingDown() { _, nb = n.Applied(ce.Index) } if js.hasPeerEntries(ce.Entries) || didStreamRemoval || (didSnap && !isLeader) { doSnapshot() } else if nb > compactSizeMin && time.Since(lastSnapTime) > minSnapDelta { doSnapshot() } ce.ReturnToPool() } else { s.Warnf("Error applying JetStream cluster entries: %v", err) } } aq.recycle(&ces) case isLeader = <-lch: // Process the change. js.processLeaderChange(isLeader) if isLeader { s.sendInternalMsgLocked(serverStatsPingReqSubj, _EMPTY_, nil, nil) // Install a snapshot as we become leader. js.checkClusterSize() doSnapshot() } case <-t.C: doSnapshot() // Periodically check the cluster size. if n.Leader() { js.checkClusterSize() } case <-ht.C: // Do this in a separate go routine. go checkHealth() case <-lt.C: s.Debugf("Checking JetStream cluster state") // If we have a current leader or had one in the past we can cancel this here since the metaleader // will be in charge of all peer state changes. // For cold boot only. if !n.Leaderless() || n.HadPreviousLeader() { lt.Stop() continue } // If we are here we do not have a leader and we did not have a previous one, so cold start. // Check to see if we can adjust our cluster size down iff we are in mixed mode and we have // seen a total that is what our original estimate was. cs := n.ClusterSize() if js, total := s.trackedJetStreamServers(); js < total && total >= cs && js != cs { s.Noticef("Adjusting JetStream expected peer set size to %d from original %d", js, cs) n.AdjustBootClusterSize(js) } } } } // This is called on first leader transition to double check the peers and cluster set size. func (js *jetStream) checkClusterSize() { s, n := js.server(), js.getMetaGroup() if n == nil { return } // We will check that we have a correct cluster set size by checking for any non-js servers // which can happen in mixed mode. ps := n.(*raft).currentPeerState() if len(ps.knownPeers) >= ps.clusterSize { return } // Grab our active peers. peers := s.ActivePeers() // If we have not registered all of our peers yet we can't do // any adjustments based on a mixed mode. We will periodically check back. if len(peers) < ps.clusterSize { return } s.Debugf("Checking JetStream cluster size") // If we are here our known set as the leader is not the same as the cluster size. // Check to see if we have a mixed mode setup. var totalJS int for _, p := range peers { if si, ok := s.nodeToInfo.Load(p); ok && si != nil { if si.(nodeInfo).js { totalJS++ } } } // If we have less then our cluster size adjust that here. Can not do individual peer removals since // they will not be in the tracked peers. if totalJS < ps.clusterSize { s.Debugf("Adjusting JetStream cluster size from %d to %d", ps.clusterSize, totalJS) if err := n.AdjustClusterSize(totalJS); err != nil { s.Warnf("Error adjusting JetStream cluster size: %v", err) } } } // Represents our stable meta state that we can write out. type writeableStreamAssignment struct { Client *ClientInfo `json:"client,omitempty"` Created time.Time `json:"created"` Config *StreamConfig `json:"stream"` Group *raftGroup `json:"group"` Sync string `json:"sync"` Consumers []*consumerAssignment } func (js *jetStream) clusterStreamConfig(accName, streamName string) (StreamConfig, bool) { js.mu.RLock() defer js.mu.RUnlock() if sa, ok := js.cluster.streams[accName][streamName]; ok { return *sa.Config, true } return StreamConfig{}, false } func (js *jetStream) metaSnapshot() ([]byte, error) { start := time.Now() js.mu.RLock() s := js.srv cc := js.cluster nsa := 0 nca := 0 for _, asa := range cc.streams { nsa += len(asa) } streams := make([]writeableStreamAssignment, 0, nsa) for _, asa := range cc.streams { for _, sa := range asa { wsa := writeableStreamAssignment{ Client: sa.Client.forAssignmentSnap(), Created: sa.Created, Config: sa.Config, Group: sa.Group, Sync: sa.Sync, Consumers: make([]*consumerAssignment, 0, len(sa.consumers)), } for _, ca := range sa.consumers { // Skip if the consumer is pending, we can't include it in our snapshot. // If the proposal fails after we marked it pending, it would result in a ghost consumer. if ca.pending { continue } cca := *ca cca.Stream = wsa.Config.Name // Needed for safe roll-backs. cca.Client = cca.Client.forAssignmentSnap() cca.Subject, cca.Reply = _EMPTY_, _EMPTY_ wsa.Consumers = append(wsa.Consumers, &cca) nca++ } streams = append(streams, wsa) } } if len(streams) == 0 { js.mu.RUnlock() return nil, nil } // Track how long it took to marshal the JSON mstart := time.Now() b, err := json.Marshal(streams) mend := time.Since(mstart) js.mu.RUnlock() // Must not be possible for a JSON marshaling error to result // in an empty snapshot. if err != nil { return nil, err } // Track how long it took to compress the JSON cstart := time.Now() snap := s2.Encode(nil, b) cend := time.Since(cstart) if took := time.Since(start); took > time.Second { s.rateLimitFormatWarnf("Metalayer snapshot took %.3fs (streams: %d, consumers: %d, marshal: %.3fs, s2: %.3fs, uncompressed: %d, compressed: %d)", took.Seconds(), nsa, nca, mend.Seconds(), cend.Seconds(), len(b), len(snap)) } return snap, nil } func (js *jetStream) applyMetaSnapshot(buf []byte, ru *recoveryUpdates, isRecovering bool) error { var wsas []writeableStreamAssignment if len(buf) > 0 { jse, err := s2.Decode(nil, buf) if err != nil { return err } if err = json.Unmarshal(jse, &wsas); err != nil { return err } } // Build our new version here outside of js. streams := make(map[string]map[string]*streamAssignment) for _, wsa := range wsas { fixCfgMirrorWithDedupWindow(wsa.Config) as := streams[wsa.Client.serviceAccount()] if as == nil { as = make(map[string]*streamAssignment) streams[wsa.Client.serviceAccount()] = as } sa := &streamAssignment{Client: wsa.Client, Created: wsa.Created, Config: wsa.Config, Group: wsa.Group, Sync: wsa.Sync} if len(wsa.Consumers) > 0 { sa.consumers = make(map[string]*consumerAssignment) for _, ca := range wsa.Consumers { if ca.Stream == _EMPTY_ { ca.Stream = sa.Config.Name // Rehydrate from the stream name. } sa.consumers[ca.Name] = ca } } as[wsa.Config.Name] = sa } js.mu.Lock() cc := js.cluster var saAdd, saDel, saChk []*streamAssignment // Walk through the old list to generate the delete list. for account, asa := range cc.streams { nasa := streams[account] for sn, sa := range asa { if nsa := nasa[sn]; nsa == nil { saDel = append(saDel, sa) } else { saChk = append(saChk, nsa) } } } // Walk through the new list to generate the add list. for account, nasa := range streams { asa := cc.streams[account] for sn, sa := range nasa { if asa[sn] == nil { saAdd = append(saAdd, sa) } } } // Now walk the ones to check and process consumers. var caAdd, caDel []*consumerAssignment for _, sa := range saChk { // Make sure to add in all the new ones from sa. for _, ca := range sa.consumers { caAdd = append(caAdd, ca) } if osa := js.streamAssignment(sa.Client.serviceAccount(), sa.Config.Name); osa != nil { for _, ca := range osa.consumers { // Consumer was either removed, or recreated with a different raft group. if nca := sa.consumers[ca.Name]; nca == nil { caDel = append(caDel, ca) } else if nca.Group != nil && ca.Group != nil && nca.Group.Name != ca.Group.Name { caDel = append(caDel, ca) } } } } js.mu.Unlock() // Do removals first. for _, sa := range saDel { js.setStreamAssignmentRecovering(sa) if isRecovering { key := sa.recoveryKey() ru.removeStreams[key] = sa delete(ru.addStreams, key) delete(ru.updateStreams, key) delete(ru.updateConsumers, key) delete(ru.removeConsumers, key) } else { js.processStreamRemoval(sa) } } // Now do add for the streams. Also add in all consumers. for _, sa := range saAdd { js.setStreamAssignmentRecovering(sa) js.processStreamAssignment(sa) // We can simply process the consumers. for _, ca := range sa.consumers { js.setConsumerAssignmentRecovering(ca) js.processConsumerAssignment(ca) } } // Perform updates on those in saChk. These were existing so make // sure to process any changes. for _, sa := range saChk { js.setStreamAssignmentRecovering(sa) if isRecovering { key := sa.recoveryKey() ru.updateStreams[key] = sa delete(ru.addStreams, key) delete(ru.removeStreams, key) } else { js.processUpdateStreamAssignment(sa) } } // Now do the deltas for existing stream's consumers. for _, ca := range caDel { js.setConsumerAssignmentRecovering(ca) if isRecovering { key := ca.recoveryKey() skey := ca.streamRecoveryKey() if _, ok := ru.removeConsumers[skey]; !ok { ru.removeConsumers[skey] = map[string]*consumerAssignment{} } ru.removeConsumers[skey][key] = ca if consumers, ok := ru.updateConsumers[skey]; ok { delete(consumers, key) } } else { js.processConsumerRemoval(ca) } } for _, ca := range caAdd { js.setConsumerAssignmentRecovering(ca) if isRecovering { key := ca.recoveryKey() skey := ca.streamRecoveryKey() if consumers, ok := ru.removeConsumers[skey]; ok { delete(consumers, key) } if _, ok := ru.updateConsumers[skey]; !ok { ru.updateConsumers[skey] = map[string]*consumerAssignment{} } ru.updateConsumers[skey][key] = ca } else { js.processConsumerAssignment(ca) } } return nil } // Called on recovery to make sure we do not process like original. func (js *jetStream) setStreamAssignmentRecovering(sa *streamAssignment) { js.mu.Lock() defer js.mu.Unlock() sa.responded = true sa.recovering = true sa.Restore = nil if sa.Group != nil { sa.Group.Preferred = _EMPTY_ } } // Called on recovery to make sure we do not process like original. func (js *jetStream) setConsumerAssignmentRecovering(ca *consumerAssignment) { js.mu.Lock() defer js.mu.Unlock() ca.responded = true ca.recovering = true if ca.Group != nil { ca.Group.Preferred = _EMPTY_ } } // Just copies over and changes out the group so it can be encoded. // Lock should be held. func (sa *streamAssignment) copyGroup() *streamAssignment { csa, cg := *sa, *sa.Group csa.Group = &cg csa.Group.Peers = copyStrings(sa.Group.Peers) return &csa } // Just copies over and changes out the group so it can be encoded. // Lock should be held. func (ca *consumerAssignment) copyGroup() *consumerAssignment { cca, cg := *ca, *ca.Group cca.Group = &cg cca.Group.Peers = copyStrings(ca.Group.Peers) return &cca } // Lock should be held. func (sa *streamAssignment) missingPeers() bool { return len(sa.Group.Peers) < sa.Config.Replicas } // Called when we detect a new peer. Only the leader will process checking // for any streams, and consequently any consumers. func (js *jetStream) processAddPeer(peer string) { js.mu.Lock() defer js.mu.Unlock() s, cc := js.srv, js.cluster if cc == nil || cc.meta == nil { return } isLeader := cc.isLeader() // Now check if we are meta-leader. We will check for any re-assignments. if !isLeader { return } sir, ok := s.nodeToInfo.Load(peer) if !ok || sir == nil { return } si := sir.(nodeInfo) for _, asa := range cc.streams { for _, sa := range asa { if sa.missingPeers() { // Make sure the right cluster etc. if si.cluster != sa.Client.Cluster { continue } // If we are here we can add in this peer. csa := sa.copyGroup() csa.Group.Peers = append(csa.Group.Peers, peer) // Send our proposal for this csa. Also use same group definition for all the consumers as well. cc.meta.Propose(encodeAddStreamAssignment(csa)) for _, ca := range sa.consumers { // Ephemerals are R=1, so only auto-remap durables, or R>1. if ca.Config.Durable != _EMPTY_ || len(ca.Group.Peers) > 1 { cca := ca.copyGroup() cca.Group.Peers = csa.Group.Peers cc.meta.Propose(encodeAddConsumerAssignment(cca)) } } } } } } func (js *jetStream) processRemovePeer(peer string) { // We may be already disabled. if js == nil || js.disabled.Load() { return } js.mu.Lock() s, cc := js.srv, js.cluster if cc == nil || cc.meta == nil { js.mu.Unlock() return } isLeader := cc.isLeader() // All nodes will check if this is them. isUs := cc.meta.ID() == peer js.mu.Unlock() if isUs { s.Errorf("JetStream being DISABLED, our server was removed from the cluster") adv := &JSServerRemovedAdvisory{ TypedEvent: TypedEvent{ Type: JSServerRemovedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Server: s.Name(), ServerID: s.ID(), Cluster: s.cachedClusterName(), Domain: s.getOpts().JetStreamDomain, } s.publishAdvisory(nil, JSAdvisoryServerRemoved, adv) go s.DisableJetStream() } // Now check if we are meta-leader. We will attempt re-assignment. if !isLeader { return } js.mu.Lock() defer js.mu.Unlock() for _, asa := range cc.streams { for _, sa := range asa { if rg := sa.Group; rg.isMember(peer) { js.removePeerFromStreamLocked(sa, peer) } } } } // Assumes all checks have already been done. func (js *jetStream) removePeerFromStream(sa *streamAssignment, peer string) bool { js.mu.Lock() defer js.mu.Unlock() return js.removePeerFromStreamLocked(sa, peer) } // Lock should be held. func (js *jetStream) removePeerFromStreamLocked(sa *streamAssignment, peer string) bool { if rg := sa.Group; !rg.isMember(peer) { return false } s, cc, csa := js.srv, js.cluster, sa.copyGroup() if cc == nil || cc.meta == nil { return false } replaced := cc.remapStreamAssignment(csa, peer) if !replaced { s.Warnf("JetStream cluster could not replace peer for stream '%s > %s'", sa.Client.serviceAccount(), sa.Config.Name) } // Send our proposal for this csa. Also use same group definition for all the consumers as well. cc.meta.Propose(encodeAddStreamAssignment(csa)) rg := csa.Group for _, ca := range sa.consumers { // Ephemerals are R=1, so only auto-remap durables, or R>1. if ca.Config.Durable != _EMPTY_ { cca := ca.copyGroup() cca.Group.Peers, cca.Group.Preferred = rg.Peers, _EMPTY_ cc.meta.Propose(encodeAddConsumerAssignment(cca)) } else if ca.Group.isMember(peer) { // These are ephemerals. Check to see if we deleted this peer. cc.meta.Propose(encodeDeleteConsumerAssignment(ca)) } } return replaced } // Check if we have peer related entries. func (js *jetStream) hasPeerEntries(entries []*Entry) bool { for _, e := range entries { if e.Type == EntryRemovePeer || e.Type == EntryAddPeer { return true } } return false } const ksep = ":" func (sa *streamAssignment) recoveryKey() string { if sa == nil { return _EMPTY_ } return sa.Client.serviceAccount() + ksep + sa.Config.Name } func (ca *consumerAssignment) streamRecoveryKey() string { if ca == nil { return _EMPTY_ } return ca.Client.serviceAccount() + ksep + ca.Stream } func (ca *consumerAssignment) recoveryKey() string { if ca == nil { return _EMPTY_ } return ca.Client.serviceAccount() + ksep + ca.Stream + ksep + ca.Name } func (js *jetStream) applyMetaEntries(entries []*Entry, ru *recoveryUpdates) (bool, bool, bool, error) { var didSnap, didRemoveStream, didRemoveConsumer bool isRecovering := js.isMetaRecovering() for _, e := range entries { if e.Type == EntrySnapshot { js.applyMetaSnapshot(e.Data, ru, isRecovering) didSnap = true } else if e.Type == EntryRemovePeer { if !isRecovering { js.processRemovePeer(string(e.Data)) } } else if e.Type == EntryAddPeer { if !isRecovering { js.processAddPeer(string(e.Data)) } } else { buf := e.Data switch entryOp(buf[0]) { case assignStreamOp: sa, err := decodeStreamAssignment(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode stream assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setStreamAssignmentRecovering(sa) key := sa.recoveryKey() ru.addStreams[key] = sa delete(ru.removeStreams, key) } else if js.processStreamAssignment(sa) { didRemoveStream = true } case removeStreamOp: sa, err := decodeStreamAssignment(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode stream assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setStreamAssignmentRecovering(sa) key := sa.recoveryKey() ru.removeStreams[key] = sa delete(ru.addStreams, key) delete(ru.updateStreams, key) delete(ru.updateConsumers, key) delete(ru.removeConsumers, key) } else { js.processStreamRemoval(sa) didRemoveStream = true } case assignConsumerOp: ca, err := decodeConsumerAssignment(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode consumer assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setConsumerAssignmentRecovering(ca) key := ca.recoveryKey() skey := ca.streamRecoveryKey() if consumers, ok := ru.removeConsumers[skey]; ok { delete(consumers, key) } if _, ok := ru.updateConsumers[skey]; !ok { ru.updateConsumers[skey] = map[string]*consumerAssignment{} } ru.updateConsumers[skey][key] = ca } else { js.processConsumerAssignment(ca) } case assignCompressedConsumerOp: ca, err := decodeConsumerAssignmentCompressed(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode compressed consumer assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setConsumerAssignmentRecovering(ca) key := ca.recoveryKey() skey := ca.streamRecoveryKey() if consumers, ok := ru.removeConsumers[skey]; ok { delete(consumers, key) } if _, ok := ru.updateConsumers[skey]; !ok { ru.updateConsumers[skey] = map[string]*consumerAssignment{} } ru.updateConsumers[skey][key] = ca } else { js.processConsumerAssignment(ca) } case removeConsumerOp: ca, err := decodeConsumerAssignment(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode consumer assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setConsumerAssignmentRecovering(ca) key := ca.recoveryKey() skey := ca.streamRecoveryKey() if _, ok := ru.removeConsumers[skey]; !ok { ru.removeConsumers[skey] = map[string]*consumerAssignment{} } ru.removeConsumers[skey][key] = ca if consumers, ok := ru.updateConsumers[skey]; ok { delete(consumers, key) } } else { js.processConsumerRemoval(ca) didRemoveConsumer = true } case updateStreamOp: sa, err := decodeStreamAssignment(buf[1:]) if err != nil { js.srv.Errorf("JetStream cluster failed to decode stream assignment: %q", buf[1:]) return didSnap, didRemoveStream, didRemoveConsumer, err } if isRecovering { js.setStreamAssignmentRecovering(sa) key := sa.recoveryKey() ru.updateStreams[key] = sa delete(ru.addStreams, key) delete(ru.removeStreams, key) } else { js.processUpdateStreamAssignment(sa) // Since an update can be lowering replica count, we want upper layer to treat // similar to a removal and snapshot to collapse old entries. didRemoveStream = true } default: panic(fmt.Sprintf("JetStream Cluster Unknown meta entry op type: %v", entryOp(buf[0]))) } } } return didSnap, didRemoveStream, didRemoveConsumer, nil } func (rg *raftGroup) isMember(id string) bool { if rg == nil { return false } for _, peer := range rg.Peers { if peer == id { return true } } return false } func (rg *raftGroup) setPreferred() { if rg == nil || len(rg.Peers) == 0 { return } if len(rg.Peers) == 1 { rg.Preferred = rg.Peers[0] } else { // For now just randomly select a peer for the preferred. pi := rand.Int31n(int32(len(rg.Peers))) rg.Preferred = rg.Peers[pi] } } // createRaftGroup is called to spin up this raft group if needed. func (js *jetStream) createRaftGroup(accName string, rg *raftGroup, storage StorageType, labels pprofLabels) error { js.mu.Lock() s, cc := js.srv, js.cluster if cc == nil || cc.meta == nil { js.mu.Unlock() return NewJSClusterNotActiveError() } // If this is a single peer raft group or we are not a member return. if len(rg.Peers) <= 1 || !rg.isMember(cc.meta.ID()) { js.mu.Unlock() // Nothing to do here. return nil } // Check if we already have this assigned. retry: if node := s.lookupRaftNode(rg.Name); node != nil { if node.State() == Closed { // We're waiting for this node to finish shutting down before we replace it. js.mu.Unlock() node.WaitForStop() js.mu.Lock() goto retry } s.Debugf("JetStream cluster already has raft group %q assigned", rg.Name) // Check and see if the group has the same peers. If not then we // will update the known peers, which will send a peerstate if leader. groupPeerIDs := append([]string{}, rg.Peers...) var samePeers bool if nodePeers := node.Peers(); len(rg.Peers) == len(nodePeers) { nodePeerIDs := make([]string, 0, len(nodePeers)) for _, n := range nodePeers { nodePeerIDs = append(nodePeerIDs, n.ID) } slices.Sort(groupPeerIDs) slices.Sort(nodePeerIDs) samePeers = slices.Equal(groupPeerIDs, nodePeerIDs) } if !samePeers { node.UpdateKnownPeers(groupPeerIDs) } rg.node = node js.mu.Unlock() return nil } s.Debugf("JetStream cluster creating raft group:%+v", rg) js.mu.Unlock() sysAcc := s.SystemAccount() if sysAcc == nil { s.Debugf("JetStream cluster detected shutdown processing raft group: %+v", rg) return errors.New("shutting down") } // Check here to see if we have a max HA Assets limit set. if maxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets; maxHaAssets > 0 { if s.numRaftNodes() > maxHaAssets { s.Warnf("Maximum HA Assets limit reached: %d", maxHaAssets) // Since the meta leader assigned this, send a statsz update to them to get them up to date. go s.sendStatszUpdate() return errors.New("system limit reached") } } storeDir := filepath.Join(js.config.StoreDir, sysAcc.Name, defaultStoreDirName, rg.Name) var store StreamStore if storage == FileStorage { // If the server is set to sync always, do the same for the Raft log. js.srv.optsMu.RLock() syncAlways := js.srv.opts.SyncAlways syncInterval := js.srv.opts.SyncInterval js.srv.optsMu.RUnlock() fs, err := newFileStoreWithCreated( FileStoreConfig{StoreDir: storeDir, BlockSize: defaultMediumBlockSize, AsyncFlush: false, SyncAlways: syncAlways, SyncInterval: syncInterval, srv: s}, StreamConfig{Name: rg.Name, Storage: FileStorage, Metadata: labels}, time.Now().UTC(), s.jsKeyGen(s.getOpts().JetStreamKey, rg.Name), s.jsKeyGen(s.getOpts().JetStreamOldKey, rg.Name), ) if err != nil { s.Errorf("Error creating filestore WAL: %v", err) return err } store = fs } else { ms, err := newMemStore(&StreamConfig{Name: rg.Name, Storage: MemoryStorage}) if err != nil { s.Errorf("Error creating memstore WAL: %v", err) return err } store = ms } cfg := &RaftConfig{Name: rg.Name, Store: storeDir, Log: store, Track: true} if _, err := readPeerState(storeDir); err != nil { s.bootstrapRaftNode(cfg, rg.Peers, true) } n, err := s.startRaftNode(accName, cfg, labels) if err != nil || n == nil { s.Debugf("Error creating raft group: %v", err) return err } // Need locking here for the assignment to avoid data-race reports js.mu.Lock() rg.node = n // See if we are preferred and should start campaign immediately. if n.ID() == rg.Preferred && n.Term() == 0 { n.Campaign() } js.mu.Unlock() return nil } func (mset *stream) raftGroup() *raftGroup { if mset == nil { return nil } mset.mu.RLock() defer mset.mu.RUnlock() if mset.sa == nil { return nil } return mset.sa.Group } func (mset *stream) raftNode() RaftNode { if mset == nil { return nil } mset.mu.RLock() defer mset.mu.RUnlock() return mset.node } func (mset *stream) removeNode() { mset.mu.Lock() defer mset.mu.Unlock() if n := mset.node; n != nil { n.Delete() mset.node = nil } } func (mset *stream) clearRaftNode() { if mset == nil { return } mset.mu.Lock() defer mset.mu.Unlock() mset.node = nil } // Helper function to generate peer info. // lists and sets for old and new. func genPeerInfo(peers []string, split int) (newPeers, oldPeers []string, newPeerSet, oldPeerSet map[string]bool) { newPeers = peers[split:] oldPeers = peers[:split] newPeerSet = make(map[string]bool, len(newPeers)) oldPeerSet = make(map[string]bool, len(oldPeers)) for i, peer := range peers { if i < split { oldPeerSet[peer] = true } else { newPeerSet[peer] = true } } return } // This will wait for a period of time until all consumers are registered and have // their consumer assignments assigned. // Should only be called from monitorStream. func (mset *stream) waitOnConsumerAssignments() { mset.mu.RLock() s, js, acc, sa, name, replicas := mset.srv, mset.js, mset.acc, mset.sa, mset.cfg.Name, mset.cfg.Replicas mset.mu.RUnlock() if s == nil || js == nil || acc == nil || sa == nil { return } js.mu.RLock() numExpectedConsumers := len(sa.consumers) js.mu.RUnlock() // Max to wait. const maxWaitTime = 10 * time.Second const sleepTime = 500 * time.Millisecond // Wait up to 10s timeout := time.Now().Add(maxWaitTime) for time.Now().Before(timeout) { var numReady int for _, o := range mset.getConsumers() { // Make sure we are registered with our consumer assignment. if ca := o.consumerAssignment(); ca != nil { if replicas > 1 && !o.isMonitorRunning() { break } numReady++ } else { break } } // Check if we are good. if numReady >= numExpectedConsumers { break } s.Debugf("Waiting for consumers for interest based stream '%s > %s'", acc.Name, name) select { case <-s.quitCh: return case <-mset.monitorQuitC(): return case <-time.After(sleepTime): } } if actual := mset.numConsumers(); actual < numExpectedConsumers { s.Warnf("All consumers not online for '%s > %s': expected %d but only have %d", acc.Name, name, numExpectedConsumers, actual) } } // Monitor our stream node for this stream. func (js *jetStream) monitorStream(mset *stream, sa *streamAssignment, sendSnapshot bool) { s, cc := js.server(), js.cluster defer s.grWG.Done() if mset != nil { defer mset.monitorWg.Done() } js.mu.RLock() n := sa.Group.node meta := cc.meta js.mu.RUnlock() if n == nil || meta == nil { s.Warnf("No RAFT group for '%s > %s'", sa.Client.serviceAccount(), sa.Config.Name) return } // Make sure only one is running. if mset != nil { if mset.checkInMonitor() { return } defer mset.clearMonitorRunning() } // Make sure to stop the raft group on exit to prevent accidental memory bloat. // This should be below the checkInMonitor call though to avoid stopping it out // from underneath the one that is running since it will be the same raft node. defer func() { // We might be closing during shutdown, don't pre-emptively stop here since we'll still want to install snapshots. if mset != nil && !mset.closed.Load() { n.Stop() } }() qch, mqch, lch, aq, uch, ourPeerId := n.QuitC(), mset.monitorQuitC(), n.LeadChangeC(), n.ApplyQ(), mset.updateC(), meta.ID() s.Debugf("Starting stream monitor for '%s > %s' [%s]", sa.Client.serviceAccount(), sa.Config.Name, n.Group()) defer s.Debugf("Exiting stream monitor for '%s > %s' [%s]", sa.Client.serviceAccount(), sa.Config.Name, n.Group()) // Make sure we do not leave the apply channel to fill up and block the raft layer. defer func() { if n.State() == Closed { return } if n.Leader() { n.StepDown() } // Drain the commit queue... aq.drain() }() const ( compactInterval = 2 * time.Minute compactSizeMin = 8 * 1024 * 1024 compactNumMin = 65536 ) // Spread these out for large numbers on server restart. rci := time.Duration(rand.Int63n(int64(time.Minute))) t := time.NewTicker(compactInterval + rci) defer t.Stop() js.mu.RLock() isLeader := cc.isStreamLeader(sa.Client.serviceAccount(), sa.Config.Name) isRestore := sa.Restore != nil js.mu.RUnlock() acc, err := s.LookupAccount(sa.Client.serviceAccount()) if err != nil { s.Warnf("Could not retrieve account for stream '%s > %s'", sa.Client.serviceAccount(), sa.Config.Name) return } accName := acc.GetName() // Used to represent how we can detect a changed state quickly and without representing // a complete and detailed state which could be costly in terms of memory, cpu and GC. // This only entails how many messages, and the first and last sequence of the stream. // This is all that is needed to detect a change, and we can get this from FilteredState() // with an empty filter. var lastState SimpleState // Don't allow the upper layer to install snapshots until we have // fully recovered from disk. isRecovering := true doSnapshot := func() { if mset == nil || isRecovering || isRestore { return } // Before we actually calculate the detailed state and encode it, let's check the // simple state to detect any changes. curState := mset.store.FilteredState(0, _EMPTY_) // If the state hasn't changed but the log has gone way over // the compaction size then we will want to compact anyway. // This shouldn't happen for streams like it can for pull // consumers on idle streams but better to be safe than sorry! ne, nb := n.Size() if curState == lastState && ne < compactNumMin && nb < compactSizeMin { return } if err := n.InstallSnapshot(mset.stateSnapshot()); err == nil { lastState = curState } else if err != errNoSnapAvailable && err != errNodeClosed && err != errCatchupsRunning { s.RateLimitWarnf("Failed to install snapshot for '%s > %s' [%s]: %v", mset.acc.Name, mset.name(), n.Group(), err) } } // We will establish a restoreDoneCh no matter what. Will never be triggered unless // we replace with the restore chan. restoreDoneCh := make(<-chan error) // For migration tracking. var mmt *time.Ticker var mmtc <-chan time.Time startMigrationMonitoring := func() { if mmt == nil { mmt = time.NewTicker(500 * time.Millisecond) mmtc = mmt.C } } stopMigrationMonitoring := func() { if mmt != nil { mmt.Stop() mmt, mmtc = nil, nil } } defer stopMigrationMonitoring() // This is to optionally track when we are ready as a non-leader for direct access participation. // Either direct or if we are a direct mirror, or both. var dat *time.Ticker var datc <-chan time.Time startDirectAccessMonitoring := func() { if dat == nil { dat = time.NewTicker(2 * time.Second) datc = dat.C } } stopDirectMonitoring := func() { if dat != nil { dat.Stop() dat, datc = nil, nil } } defer stopDirectMonitoring() // For checking interest state if applicable. var cist *time.Ticker var cistc <-chan time.Time // 2 minutes plus up to 30s jitter. checkInterestInterval := 2*time.Minute + time.Duration(rand.Intn(30))*time.Second if mset != nil && mset.isInterestRetention() { // Wait on our consumers to be assigned and running before proceeding. // This can become important when a server has lots of assets // since we process streams first then consumers as an asset class. mset.waitOnConsumerAssignments() // Setup our periodic check here. We will check once we have restored right away. cist = time.NewTicker(checkInterestInterval) cistc = cist.C } // This is triggered during a scale up from R1 to clustered mode. We need the new followers to catchup, // similar to how we trigger the catchup mechanism post a backup/restore. // We can arrive here NOT being the leader, so we send the snapshot only if we are, and in this case // reset the notion that we need to send the snapshot. If we are not, then the first time the server // will switch to leader (in the loop below), we will send the snapshot. if sendSnapshot && isLeader && mset != nil && n != nil && !isRecovering { n.SendSnapshot(mset.stateSnapshot()) sendSnapshot = false } for { select { case <-s.quitCh: // Server shutting down, but we might receive this before qch, so try to snapshot. doSnapshot() return case <-mqch: return case <-qch: // Clean signal from shutdown routine so do best effort attempt to snapshot. doSnapshot() return case <-aq.ch: var ne, nb uint64 // If we bump clfs we will want to write out snapshot if within our time window. pclfs := mset.getCLFS() ces := aq.pop() for _, ce := range ces { // No special processing needed for when we are caught up on restart. if ce == nil { isRecovering = false // If we are interest based make sure to check consumers if interest retention policy. // This is to make sure we process any outstanding acks from all consumers. if mset != nil && mset.isInterestRetention() { fire := time.Duration(rand.Intn(5)+5) * time.Second time.AfterFunc(fire, mset.checkInterestState) } // If we became leader during this time and we need to send a snapshot to our // followers, i.e. as a result of a scale-up from R1, do it now. if sendSnapshot && isLeader && mset != nil && n != nil { n.SendSnapshot(mset.stateSnapshot()) sendSnapshot = false } continue } // Apply our entries. if err := js.applyStreamEntries(mset, ce, isRecovering); err == nil { // Update our applied. ne, nb = n.Applied(ce.Index) ce.ReturnToPool() } else { // Our stream was closed out from underneath of us, simply return here. if err == errStreamClosed || err == errCatchupStreamStopped || err == ErrServerNotRunning { aq.recycle(&ces) return } s.Warnf("Error applying entries to '%s > %s': %v", accName, sa.Config.Name, err) if isClusterResetErr(err) { if mset.isMirror() && mset.IsLeader() { mset.retryMirrorConsumer() continue } // We will attempt to reset our cluster state. if mset.resetClusteredState(err) { aq.recycle(&ces) return } } else if isOutOfSpaceErr(err) { // If applicable this will tear all of this down, but don't assume so and return. s.handleOutOfSpace(mset) } } } aq.recycle(&ces) // Check about snapshotting // If we have at least min entries to compact, go ahead and try to snapshot/compact. if ne >= compactNumMin || nb > compactSizeMin || mset.getCLFS() > pclfs { doSnapshot() } case isLeader = <-lch: if isLeader { if mset != nil && n != nil && sendSnapshot && !isRecovering { // If we *are* recovering at the time then this will get done when the apply queue // handles the nil guard to show the catchup ended. n.SendSnapshot(mset.stateSnapshot()) sendSnapshot = false } if isRestore { acc, _ := s.LookupAccount(sa.Client.serviceAccount()) restoreDoneCh = s.processStreamRestore(sa.Client, acc, sa.Config, _EMPTY_, sa.Reply, _EMPTY_) continue } else if n != nil && n.NeedSnapshot() { doSnapshot() } // Always cancel if this was running. stopDirectMonitoring() } else if !n.Leaderless() { js.setStreamAssignmentRecovering(sa) } // Process our leader change. js.processStreamLeaderChange(mset, isLeader) // We may receive a leader change after the stream assignment which would cancel us // monitoring for this closely. So re-assess our state here as well. // Or the old leader is no longer part of the set and transferred leadership // for this leader to resume with removal migrating := mset.isMigrating() // Check for migrations here. We set the state on the stream assignment update below. if isLeader && migrating { startMigrationMonitoring() } // Here we are checking if we are not the leader but we have been asked to allow // direct access. We now allow non-leaders to participate in the queue group. if !isLeader && mset != nil { mset.mu.RLock() ad, md := mset.cfg.AllowDirect, mset.cfg.MirrorDirect mset.mu.RUnlock() if ad || md { startDirectAccessMonitoring() } } case <-cistc: cist.Reset(checkInterestInterval) // We may be adjusting some things with consumers so do this in its own go routine. go mset.checkInterestState() case <-datc: if mset == nil || isRecovering { continue } // If we are leader we can stop, we know this is setup now. if isLeader { stopDirectMonitoring() continue } mset.mu.Lock() ad, md, current := mset.cfg.AllowDirect, mset.cfg.MirrorDirect, mset.isCurrent() if !current { const syncThreshold = 90.0 // We are not current, but current means exactly caught up. Under heavy publish // loads we may never reach this, so check if we are within 90% caught up. _, c, a := mset.node.Progress() if c == 0 { mset.mu.Unlock() continue } if p := float64(a) / float64(c) * 100.0; p < syncThreshold { mset.mu.Unlock() continue } else { s.Debugf("Stream '%s > %s' enabling direct gets at %.0f%% synchronized", sa.Client.serviceAccount(), sa.Config.Name, p) } } // We are current, cancel monitoring and create the direct subs as needed. if ad { mset.subscribeToDirect() } if md { mset.subscribeToMirrorDirect() } mset.mu.Unlock() // Stop direct monitoring. stopDirectMonitoring() case <-t.C: doSnapshot() case <-uch: // keep stream assignment current sa = mset.streamAssignment() // We get this when we have a new stream assignment caused by an update. // We want to know if we are migrating. if migrating := mset.isMigrating(); migrating { if isLeader && mmtc == nil { startMigrationMonitoring() } } else { stopMigrationMonitoring() } case <-mmtc: if !isLeader { // We are no longer leader, so not our job. stopMigrationMonitoring() continue } // Check to see where we are.. rg := mset.raftGroup() // Track the new peers and check the ones that are current. mset.mu.RLock() replicas := mset.cfg.Replicas mset.mu.RUnlock() if len(rg.Peers) <= replicas { // Migration no longer happening, so not our job anymore stopMigrationMonitoring() continue } // Make sure we have correct cluster information on the other peers. ci := js.clusterInfo(rg) mset.checkClusterInfo(ci) newPeers, oldPeers, newPeerSet, oldPeerSet := genPeerInfo(rg.Peers, len(rg.Peers)-replicas) // If we are part of the new peerset and we have been passed the baton. // We will handle scale down. if newPeerSet[ourPeerId] { // First need to check on any consumers and make sure they have moved properly before scaling down ourselves. js.mu.RLock() var needToWait bool for name, c := range sa.consumers { for _, peer := range c.Group.Peers { // If we have peers still in the old set block. if oldPeerSet[peer] { s.Debugf("Scale down of '%s > %s' blocked by consumer '%s'", accName, sa.Config.Name, name) needToWait = true break } } if needToWait { break } } js.mu.RUnlock() if needToWait { continue } // We are good to go, can scale down here. for _, p := range oldPeers { n.ProposeRemovePeer(p) } csa := sa.copyGroup() csa.Group.Peers = newPeers csa.Group.Preferred = ourPeerId csa.Group.Cluster = s.cachedClusterName() cc.meta.ForwardProposal(encodeUpdateStreamAssignment(csa)) s.Noticef("Scaling down '%s > %s' to %+v", accName, sa.Config.Name, s.peerSetToNames(newPeers)) } else { // We are the old leader here, from the original peer set. // We are simply waiting on the new peerset to be caught up so we can transfer leadership. var newLeaderPeer, newLeader string neededCurrent, current := replicas/2+1, 0 for _, r := range ci.Replicas { if r.Current && newPeerSet[r.Peer] { current++ if newLeader == _EMPTY_ { newLeaderPeer, newLeader = r.Peer, r.Name } } } // Check if we have a quorom. if current >= neededCurrent { s.Noticef("Transfer of stream leader for '%s > %s' to '%s'", accName, sa.Config.Name, newLeader) n.ProposeKnownPeers(newPeers) n.StepDown(newLeaderPeer) } } case err := <-restoreDoneCh: // We have completed a restore from snapshot on this server. The stream assignment has // already been assigned but the replicas will need to catch up out of band. Consumers // will need to be assigned by forwarding the proposal and stamping the initial state. s.Debugf("Stream restore for '%s > %s' completed", sa.Client.serviceAccount(), sa.Config.Name) if err != nil { s.Debugf("Stream restore failed: %v", err) } isRestore = false sa.Restore = nil // If we were successful lookup up our stream now. if err == nil { if mset, err = acc.lookupStream(sa.Config.Name); mset != nil { mset.monitorWg.Add(1) defer mset.monitorWg.Done() mset.setStreamAssignment(sa) // Make sure to update our updateC which would have been nil. uch = mset.updateC() // Also update our mqch mqch = mset.monitorQuitC() // Setup a periodic check here if we are interest based as well. if mset.isInterestRetention() { cist = time.NewTicker(checkInterestInterval) cistc = cist.C } } } if err != nil { if mset != nil { mset.delete() } js.mu.Lock() sa.err = err if n != nil { n.Delete() } result := &streamAssignmentResult{ Account: sa.Client.serviceAccount(), Stream: sa.Config.Name, Restore: &JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}}, } result.Restore.Error = NewJSStreamAssignmentError(err, Unless(err)) js.mu.Unlock() // Send response to the metadata leader. They will forward to the user as needed. s.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result) return } if !isLeader { panic("Finished restore but not leader") } // Trigger the stream followers to catchup. if n = mset.raftNode(); n != nil { n.SendSnapshot(mset.stateSnapshot()) } js.processStreamLeaderChange(mset, isLeader) // Check to see if we have restored consumers here. // These are not currently assigned so we will need to do so here. if consumers := mset.getPublicConsumers(); len(consumers) > 0 { for _, o := range consumers { name, cfg := o.String(), o.config() rg := cc.createGroupForConsumer(&cfg, sa) // Pick a preferred leader. rg.setPreferred() // Place our initial state here as well for assignment distribution. state, _ := o.store.State() ca := &consumerAssignment{ Group: rg, Stream: sa.Config.Name, Name: name, Config: &cfg, Client: sa.Client, Created: o.createdTime(), State: state, } // We make these compressed in case state is complex. addEntry := encodeAddConsumerAssignmentCompressed(ca) cc.meta.ForwardProposal(addEntry) // Check to make sure we see the assignment. go func() { ticker := time.NewTicker(time.Second) defer ticker.Stop() for range ticker.C { js.mu.RLock() ca, meta := js.consumerAssignment(ca.Client.serviceAccount(), sa.Config.Name, name), cc.meta js.mu.RUnlock() if ca == nil { s.Warnf("Consumer assignment has not been assigned, retrying") if meta != nil { meta.ForwardProposal(addEntry) } else { return } } else { return } } }() } } } } } // Determine if we are migrating func (mset *stream) isMigrating() bool { if mset == nil { return false } mset.mu.RLock() js, sa := mset.js, mset.sa mset.mu.RUnlock() js.mu.RLock() defer js.mu.RUnlock() // During migration we will always be R>1, even when we start R1. // So if we do not have a group or node we no we are not migrating. if sa == nil || sa.Group == nil || sa.Group.node == nil { return false } // The sign of migration is if our group peer count != configured replica count. if sa.Config.Replicas == len(sa.Group.Peers) { return false } return true } // resetClusteredState is called when a clustered stream had an error (e.g sequence mismatch, bad snapshot) and needs to be reset. func (mset *stream) resetClusteredState(err error) bool { mset.mu.RLock() s, js, jsa, sa, acc, node := mset.srv, mset.js, mset.jsa, mset.sa, mset.acc, mset.node stype, isLeader, tierName, replicas := mset.cfg.Storage, mset.isLeader(), mset.tier, mset.cfg.Replicas mset.mu.RUnlock() // Stepdown regardless if we are the leader here. if isLeader && node != nil { node.StepDown() } // If we detect we are shutting down just return. if js != nil && js.isShuttingDown() { s.Debugf("Will not reset stream, JetStream shutting down") return false } // Server if js.limitsExceeded(stype) { s.Warnf("Will not reset stream, server resources exceeded") return false } // Account if exceeded, _ := jsa.limitsExceeded(stype, tierName, replicas); exceeded { s.Warnf("stream '%s > %s' errored, account resources exceeded", acc, mset.name()) return false } if node != nil { if errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries { // Don't delete all state, could've just been temporarily unable to reach the leader. node.Stop() } else { // We delete our raft state. Will recreate. node.Delete() } } // Preserve our current state and messages unless we have a first sequence mismatch. shouldDelete := err == errFirstSequenceMismatch // Need to do the rest in a separate Go routine. go func() { mset.monitorWg.Wait() mset.resetAndWaitOnConsumers() // Stop our stream. mset.stop(shouldDelete, false) if sa != nil { js.mu.Lock() if js.shuttingDown { js.mu.Unlock() return } s.Warnf("Resetting stream cluster state for '%s > %s'", sa.Client.serviceAccount(), sa.Config.Name) // Mark stream assignment as resetting, so we don't double-account reserved resources. // But only if we're not also releasing the resources as part of the delete. sa.resetting = !shouldDelete // Now wipe groups from assignments. sa.Group.node = nil var consumers []*consumerAssignment if cc := js.cluster; cc != nil && cc.meta != nil { ourID := cc.meta.ID() for _, ca := range sa.consumers { if rg := ca.Group; rg != nil && rg.isMember(ourID) { rg.node = nil // Erase group raft/node state. consumers = append(consumers, ca) } } } js.mu.Unlock() // This will reset the stream and consumers. // Reset stream. js.processClusterCreateStream(acc, sa) // Reset consumers. for _, ca := range consumers { js.processClusterCreateConsumer(ca, nil, false) } } }() return true } func isControlHdr(hdr []byte) bool { return bytes.HasPrefix(hdr, []byte("NATS/1.0 100 ")) } // Apply our stream entries. func (js *jetStream) applyStreamEntries(mset *stream, ce *CommittedEntry, isRecovering bool) error { for _, e := range ce.Entries { if e.Type == EntryNormal { buf, op := e.Data, entryOp(e.Data[0]) switch op { case streamMsgOp, compressedStreamMsgOp: if mset == nil { continue } s := js.srv mbuf := buf[1:] if op == compressedStreamMsgOp { var err error mbuf, err = s2.Decode(nil, mbuf) if err != nil { panic(err.Error()) } } subject, reply, hdr, msg, lseq, ts, err := decodeStreamMsg(mbuf) if err != nil { if node := mset.raftNode(); node != nil { s.Errorf("JetStream cluster could not decode stream msg for '%s > %s' [%s]", mset.account(), mset.name(), node.Group()) } panic(err.Error()) } // Check for flowcontrol here. if len(msg) == 0 && len(hdr) > 0 && reply != _EMPTY_ && isControlHdr(hdr) { if !isRecovering { mset.sendFlowControlReply(reply) } continue } // Grab last sequence and CLFS. last, clfs := mset.lastSeqAndCLFS() // We can skip if we know this is less than what we already have. if lseq-clfs < last { s.Debugf("Apply stream entries for '%s > %s' skipping message with sequence %d with last of %d", mset.account(), mset.name(), lseq+1-clfs, last) mset.mu.Lock() // Check for any preAcks in case we are interest based. mset.clearAllPreAcks(lseq + 1 - clfs) mset.mu.Unlock() continue } // Skip by hand here since first msg special case. // Reason is sequence is unsigned and for lseq being 0 // the lseq under stream would have to be -1. if lseq == 0 && last != 0 { continue } // Messages to be skipped have no subject or timestamp or msg or hdr. if subject == _EMPTY_ && ts == 0 && len(msg) == 0 && len(hdr) == 0 { // Skip and update our lseq. last := mset.store.SkipMsg() mset.mu.Lock() mset.setLastSeq(last) mset.clearAllPreAcks(last) mset.mu.Unlock() continue } // Process the actual message here. err = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts) // If we have inflight make sure to clear after processing. // TODO(dlc) - technically check on inflight != nil could cause datarace. // But do not want to acquire lock since tracking this will be rare. if mset.inflight != nil { mset.clMu.Lock() delete(mset.inflight, lseq) mset.clMu.Unlock() } if err != nil { if err == errLastSeqMismatch { var state StreamState mset.store.FastState(&state) // If we have no msgs and the other side is delivering us a sequence past where we // should be reset. This is possible if the other side has a stale snapshot and no longer // has those messages. So compact and retry to reset. if state.Msgs == 0 { mset.store.Compact(lseq + 1) // Retry err = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts) } // FIXME(dlc) - We could just run a catchup with a request defining the span between what we expected // and what we got. } // Only return in place if we are going to reset our stream or we are out of space, or we are closed. if isClusterResetErr(err) || isOutOfSpaceErr(err) || err == errStreamClosed { return err } s.Debugf("Apply stream entries for '%s > %s' got error processing message: %v", mset.account(), mset.name(), err) } case deleteMsgOp: md, err := decodeMsgDelete(buf[1:]) if err != nil { if node := mset.raftNode(); node != nil { s := js.srv s.Errorf("JetStream cluster could not decode delete msg for '%s > %s' [%s]", mset.account(), mset.name(), node.Group()) } panic(err.Error()) } s, cc := js.server(), js.cluster var removed bool if md.NoErase { removed, err = mset.removeMsg(md.Seq) } else { removed, err = mset.eraseMsg(md.Seq) } // Cluster reset error. if err == ErrStoreEOF { return err } if err != nil && !isRecovering { s.Debugf("JetStream cluster failed to delete stream msg %d from '%s > %s': %v", md.Seq, md.Client.serviceAccount(), md.Stream, err) } js.mu.RLock() isLeader := cc.isStreamLeader(md.Client.serviceAccount(), md.Stream) js.mu.RUnlock() if isLeader && !isRecovering { var resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}} if err != nil { resp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err)) s.sendAPIErrResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp)) } else if !removed { resp.Error = NewJSSequenceNotFoundError(md.Seq) s.sendAPIErrResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp)) } else { resp.Success = true s.sendAPIResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp)) } } case purgeStreamOp: sp, err := decodeStreamPurge(buf[1:]) if err != nil { if node := mset.raftNode(); node != nil { s := js.srv s.Errorf("JetStream cluster could not decode purge msg for '%s > %s' [%s]", mset.account(), mset.name(), node.Group()) } panic(err.Error()) } // If no explicit request, fill in with leader stamped last sequence to protect ourselves on replay during server start. if sp.Request == nil || sp.Request.Sequence == 0 { purgeSeq := sp.LastSeq + 1 if sp.Request == nil { sp.Request = &JSApiStreamPurgeRequest{Sequence: purgeSeq} } else if sp.Request.Keep == 0 { sp.Request.Sequence = purgeSeq } else if isRecovering { continue } } s := js.server() purged, err := mset.purge(sp.Request) if err != nil { s.Warnf("JetStream cluster failed to purge stream %q for account %q: %v", sp.Stream, sp.Client.serviceAccount(), err) } js.mu.RLock() isLeader := js.cluster.isStreamLeader(sp.Client.serviceAccount(), sp.Stream) js.mu.RUnlock() if isLeader && !isRecovering { var resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}} if err != nil { resp.Error = NewJSStreamGeneralError(err, Unless(err)) s.sendAPIErrResponse(sp.Client, mset.account(), sp.Subject, sp.Reply, _EMPTY_, s.jsonResponse(resp)) } else { resp.Purged = purged resp.Success = true s.sendAPIResponse(sp.Client, mset.account(), sp.Subject, sp.Reply, _EMPTY_, s.jsonResponse(resp)) } } default: panic(fmt.Sprintf("JetStream Cluster Unknown group entry op type: %v", op)) } } else if e.Type == EntrySnapshot { if mset == nil { continue } // Everything operates on new replicated state. Will convert legacy snapshots to this for processing. var ss *StreamReplicatedState onBadState := func(err error) { // If we are the leader or recovering, meaning we own the snapshot, // we should stepdown and clear our raft state since our snapshot is bad. if isRecovering || mset.IsLeader() { mset.mu.RLock() s, accName, streamName := mset.srv, mset.acc.GetName(), mset.cfg.Name mset.mu.RUnlock() s.Warnf("Detected bad stream state, resetting '%s > %s'", accName, streamName) mset.resetClusteredState(err) } } // Check if we are the new binary encoding. if IsEncodedStreamState(e.Data) { var err error ss, err = DecodeStreamState(e.Data) if err != nil { onBadState(err) return err } } else { var snap streamSnapshot if err := json.Unmarshal(e.Data, &snap); err != nil { onBadState(err) return err } // Convert over to StreamReplicatedState ss = &StreamReplicatedState{ Msgs: snap.Msgs, Bytes: snap.Bytes, FirstSeq: snap.FirstSeq, LastSeq: snap.LastSeq, Failed: snap.Failed, } if len(snap.Deleted) > 0 { ss.Deleted = append(ss.Deleted, DeleteSlice(snap.Deleted)) } } if isRecovering || !mset.IsLeader() { if err := mset.processSnapshot(ss); err != nil { return err } } } else if e.Type == EntryRemovePeer { js.mu.RLock() var ourID string if js.cluster != nil && js.cluster.meta != nil { ourID = js.cluster.meta.ID() } js.mu.RUnlock() // We only need to do processing if this is us. if peer := string(e.Data); peer == ourID && mset != nil { // Double check here with the registered stream assignment. shouldRemove := true if sa := mset.streamAssignment(); sa != nil && sa.Group != nil { js.mu.RLock() shouldRemove = !sa.Group.isMember(ourID) js.mu.RUnlock() } if shouldRemove { mset.stop(true, false) } } } } return nil } // Returns the PeerInfo for all replicas of a raft node. This is different than node.Peers() // and is used for external facing advisories. func (s *Server) replicas(node RaftNode) []*PeerInfo { now := time.Now() var replicas []*PeerInfo for _, rp := range node.Peers() { if sir, ok := s.nodeToInfo.Load(rp.ID); ok && sir != nil { si := sir.(nodeInfo) pi := &PeerInfo{Peer: rp.ID, Name: si.name, Current: rp.Current, Active: now.Sub(rp.Last), Offline: si.offline, Lag: rp.Lag} replicas = append(replicas, pi) } } return replicas } // Process a leader change for the clustered stream. func (js *jetStream) processStreamLeaderChange(mset *stream, isLeader bool) { if mset == nil { return } sa := mset.streamAssignment() if sa == nil { return } // Clear inflight if we have it. mset.clMu.Lock() mset.inflight = nil mset.clMu.Unlock() js.mu.Lock() s, account, err := js.srv, sa.Client.serviceAccount(), sa.err client, subject, reply := sa.Client, sa.Subject, sa.Reply hasResponded := sa.responded sa.responded = true peers := copyStrings(sa.Group.Peers) js.mu.Unlock() streamName := mset.name() if isLeader { s.Noticef("JetStream cluster new stream leader for '%s > %s'", account, streamName) s.sendStreamLeaderElectAdvisory(mset) mset.checkAllowMsgCompress(peers) } else { // We are stepping down. // Make sure if we are doing so because we have lost quorum that we send the appropriate advisories. if node := mset.raftNode(); node != nil && !node.Quorum() && time.Since(node.Created()) > 5*time.Second { s.sendStreamLostQuorumAdvisory(mset) } // Clear clseq. If we become leader again, it will be fixed up // automatically on the next processClusteredInboundMsg call. mset.clMu.Lock() if mset.clseq > 0 { mset.clseq = 0 } mset.clMu.Unlock() } // Tell stream to switch leader status. mset.setLeader(isLeader) if !isLeader || hasResponded { return } acc, _ := s.LookupAccount(account) if acc == nil { return } // Send our response. var resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}} if err != nil { resp.Error = NewJSStreamCreateError(err, Unless(err)) s.sendAPIErrResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) } else { resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Cluster: js.clusterInfo(mset.raftGroup()), Sources: mset.sourcesInfo(), Mirror: mset.mirrorInfo(), TimeStamp: time.Now().UTC(), } resp.DidCreate = true s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) if node := mset.raftNode(); node != nil { mset.sendCreateAdvisory() } } } // Fixed value ok for now. const lostQuorumAdvInterval = 10 * time.Second // Determines if we should send lost quorum advisory. We throttle these after first one. func (mset *stream) shouldSendLostQuorum() bool { mset.mu.Lock() defer mset.mu.Unlock() if time.Since(mset.lqsent) >= lostQuorumAdvInterval { mset.lqsent = time.Now() return true } return false } func (s *Server) sendStreamLostQuorumAdvisory(mset *stream) { if mset == nil { return } node, stream, acc := mset.raftNode(), mset.name(), mset.account() if node == nil { return } if !mset.shouldSendLostQuorum() { return } s.Warnf("JetStream cluster stream '%s > %s' has NO quorum, stalled", acc.GetName(), stream) subj := JSAdvisoryStreamQuorumLostPre + "." + stream adv := &JSStreamQuorumLostAdvisory{ TypedEvent: TypedEvent{ Type: JSStreamQuorumLostAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: stream, Replicas: s.replicas(node), Domain: s.getOpts().JetStreamDomain, } // Send to the user's account if not the system account. if acc != s.SystemAccount() { s.publishAdvisory(acc, subj, adv) } // Now do system level one. Place account info in adv, and nil account means system. adv.Account = acc.GetName() s.publishAdvisory(nil, subj, adv) } func (s *Server) sendStreamLeaderElectAdvisory(mset *stream) { if mset == nil { return } node, stream, acc := mset.raftNode(), mset.name(), mset.account() if node == nil { return } subj := JSAdvisoryStreamLeaderElectedPre + "." + stream adv := &JSStreamLeaderElectedAdvisory{ TypedEvent: TypedEvent{ Type: JSStreamLeaderElectedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: stream, Leader: s.serverNameForNode(node.GroupLeader()), Replicas: s.replicas(node), Domain: s.getOpts().JetStreamDomain, } // Send to the user's account if not the system account. if acc != s.SystemAccount() { s.publishAdvisory(acc, subj, adv) } // Now do system level one. Place account info in adv, and nil account means system. adv.Account = acc.GetName() s.publishAdvisory(nil, subj, adv) } // Will lookup a stream assignment. // Lock should be held. func (js *jetStream) streamAssignment(account, stream string) (sa *streamAssignment) { cc := js.cluster if cc == nil { return nil } if as := cc.streams[account]; as != nil { sa = as[stream] } return sa } // processStreamAssignment is called when followers have replicated an assignment. func (js *jetStream) processStreamAssignment(sa *streamAssignment) bool { js.mu.Lock() s, cc := js.srv, js.cluster accName, stream := sa.Client.serviceAccount(), sa.Config.Name noMeta := cc == nil || cc.meta == nil var ourID string if !noMeta { ourID = cc.meta.ID() } var isMember bool if sa.Group != nil && ourID != _EMPTY_ { isMember = sa.Group.isMember(ourID) } // Remove this stream from the inflight proposals cc.removeInflightProposal(accName, sa.Config.Name) if s == nil || noMeta { js.mu.Unlock() return false } accStreams := cc.streams[accName] if accStreams == nil { accStreams = make(map[string]*streamAssignment) } else if osa := accStreams[stream]; osa != nil && osa != sa { // Copy over private existing state from former SA. if sa.Group != nil { sa.Group.node = osa.Group.node } sa.consumers = osa.consumers sa.responded = osa.responded sa.err = osa.err } // Update our state. accStreams[stream] = sa cc.streams[accName] = accStreams hasResponded := sa.responded js.mu.Unlock() acc, err := s.LookupAccount(accName) if err != nil { ll := fmt.Sprintf("Account [%s] lookup for stream create failed: %v", accName, err) if isMember { if !hasResponded { // If we can not lookup the account and we are a member, send this result back to the metacontroller leader. result := &streamAssignmentResult{ Account: accName, Stream: stream, Response: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}, } result.Response.Error = NewJSNoAccountError() s.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result) } s.Warnf(ll) } else { s.Debugf(ll) } return false } var didRemove bool // Check if this is for us.. if isMember { js.processClusterCreateStream(acc, sa) } else if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil { // We have one here even though we are not a member. This can happen on re-assignment. s.removeStream(mset, sa) } // If this stream assignment does not have a sync subject (bug) set that the meta-leader should check when elected. if sa.Sync == _EMPTY_ { js.mu.Lock() cc.streamsCheck = true js.mu.Unlock() return false } return didRemove } // processUpdateStreamAssignment is called when followers have replicated an updated assignment. func (js *jetStream) processUpdateStreamAssignment(sa *streamAssignment) { js.mu.RLock() s, cc := js.srv, js.cluster js.mu.RUnlock() if s == nil || cc == nil { // TODO(dlc) - debug at least return } accName := sa.Client.serviceAccount() stream := sa.Config.Name js.mu.Lock() if cc.meta == nil { js.mu.Unlock() return } ourID := cc.meta.ID() var isMember bool if sa.Group != nil { isMember = sa.Group.isMember(ourID) } accStreams := cc.streams[accName] if accStreams == nil { js.mu.Unlock() return } osa := accStreams[stream] if osa == nil { js.mu.Unlock() return } // Copy over private existing state from former SA. if sa.Group != nil { sa.Group.node = osa.Group.node } sa.consumers = osa.consumers sa.err = osa.err // If we detect we are scaling down to 1, non-clustered, and we had a previous node, clear it here. if sa.Config.Replicas == 1 && sa.Group.node != nil { sa.Group.node = nil } // Update our state. accStreams[stream] = sa cc.streams[accName] = accStreams // Make sure we respond if we are a member. if isMember { sa.responded = false } else { // Make sure to clean up any old node in case this stream moves back here. if sa.Group != nil { sa.Group.node = nil } } js.mu.Unlock() acc, err := s.LookupAccount(accName) if err != nil { s.Warnf("Update Stream Account %s, error on lookup: %v", accName, err) return } // Check if this is for us.. if isMember { js.processClusterUpdateStream(acc, osa, sa) } else if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil { // We have one here even though we are not a member. This can happen on re-assignment. s.removeStream(mset, sa) } } // Common function to remove ourselves from this server. // This can happen on re-assignment, move, etc func (s *Server) removeStream(mset *stream, nsa *streamAssignment) { if mset == nil { return } // Make sure to use the new stream assignment, not our own. s.Debugf("JetStream removing stream '%s > %s' from this server", nsa.Client.serviceAccount(), nsa.Config.Name) if node := mset.raftNode(); node != nil { if node.Leader() { node.StepDown(nsa.Group.Preferred) } // shutdown monitor by shutting down raft. node.Delete() } var isShuttingDown bool // Make sure this node is no longer attached to our stream assignment. if js, _ := s.getJetStreamCluster(); js != nil { js.mu.Lock() nsa.Group.node = nil isShuttingDown = js.shuttingDown js.mu.Unlock() } if !isShuttingDown { // wait for monitor to be shutdown. mset.monitorWg.Wait() } mset.stop(true, false) } // processClusterUpdateStream is called when we have a stream assignment that // has been updated for an existing assignment and we are a member. func (js *jetStream) processClusterUpdateStream(acc *Account, osa, sa *streamAssignment) { if sa == nil { return } js.mu.Lock() s, rg := js.srv, sa.Group client, subject, reply := sa.Client, sa.Subject, sa.Reply alreadyRunning, numReplicas := osa.Group.node != nil, len(rg.Peers) needsNode := rg.node == nil storage, cfg := sa.Config.Storage, sa.Config hasResponded := sa.responded sa.responded = true recovering := sa.recovering js.mu.Unlock() mset, err := acc.lookupStream(cfg.Name) if err == nil && mset != nil { // Make sure we have not had a new group assigned to us. if osa.Group.Name != sa.Group.Name { s.Warnf("JetStream cluster detected stream remapping for '%s > %s' from %q to %q", acc, cfg.Name, osa.Group.Name, sa.Group.Name) mset.removeNode() alreadyRunning, needsNode = false, true // Make sure to clear from original. js.mu.Lock() osa.Group.node = nil js.mu.Unlock() } if !alreadyRunning && numReplicas > 1 { if needsNode { // Since we are scaling up we want to make sure our sync subject // is registered before we start our raft node. mset.mu.Lock() mset.startClusterSubs() mset.mu.Unlock() js.createRaftGroup(acc.GetName(), rg, storage, pprofLabels{ "type": "stream", "account": mset.accName(), "stream": mset.name(), }) } mset.monitorWg.Add(1) // Start monitoring.. s.startGoRoutine( func() { js.monitorStream(mset, sa, needsNode) }, pprofLabels{ "type": "stream", "account": mset.accName(), "stream": mset.name(), }, ) } else if numReplicas == 1 && alreadyRunning { // We downgraded to R1. Make sure we cleanup the raft node and the stream monitor. mset.removeNode() // In case we need to shutdown the cluster specific subs, etc. mset.mu.Lock() // Stop responding to sync requests. mset.stopClusterSubs() // Clear catchup state mset.clearAllCatchupPeers() mset.mu.Unlock() // Remove from meta layer. js.mu.Lock() rg.node = nil js.mu.Unlock() } // Set the new stream assignment. mset.setStreamAssignment(sa) // Call update. if err = mset.updateWithAdvisory(cfg, !recovering); err != nil { s.Warnf("JetStream cluster error updating stream %q for account %q: %v", cfg.Name, acc.Name, err) } } // If not found we must be expanding into this node since if we are here we know we are a member. if err == ErrJetStreamStreamNotFound { js.processStreamAssignment(sa) return } if err != nil { js.mu.Lock() sa.err = err result := &streamAssignmentResult{ Account: sa.Client.serviceAccount(), Stream: sa.Config.Name, Response: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}, Update: true, } result.Response.Error = NewJSStreamGeneralError(err, Unless(err)) js.mu.Unlock() // Send response to the metadata leader. They will forward to the user as needed. s.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result) return } isLeader := mset.IsLeader() // Check for missing syncSubject bug. if isLeader && osa != nil && osa.Sync == _EMPTY_ { if node := mset.raftNode(); node != nil { node.StepDown() } return } // If we were a single node being promoted assume leadership role for purpose of responding. if !hasResponded && !isLeader && !alreadyRunning { isLeader = true } // Check if we should bail. if !isLeader || hasResponded || recovering { return } // Send our response. var resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}} resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Cluster: js.clusterInfo(mset.raftGroup()), Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo(), TimeStamp: time.Now().UTC(), } s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) } // processClusterCreateStream is called when we have a stream assignment that // has been committed and this server is a member of the peer group. func (js *jetStream) processClusterCreateStream(acc *Account, sa *streamAssignment) { if sa == nil { return } js.mu.RLock() s, rg := js.srv, sa.Group alreadyRunning := rg.node != nil storage := sa.Config.Storage restore := sa.Restore js.mu.RUnlock() // Process the raft group and make sure it's running if needed. err := js.createRaftGroup(acc.GetName(), rg, storage, pprofLabels{ "type": "stream", "account": acc.Name, "stream": sa.Config.Name, }) // If we are restoring, create the stream if we are R>1 and not the preferred who handles the // receipt of the snapshot itself. shouldCreate := true if restore != nil { if len(rg.Peers) == 1 || rg.node != nil && rg.node.ID() == rg.Preferred { shouldCreate = false } else { js.mu.Lock() sa.Restore = nil js.mu.Unlock() } } // Our stream. var mset *stream // Process here if not restoring or not the leader. if shouldCreate && err == nil { // Go ahead and create or update the stream. mset, err = acc.lookupStream(sa.Config.Name) if err == nil && mset != nil { osa := mset.streamAssignment() // If we already have a stream assignment and they are the same exact config, short circuit here. if osa != nil { if reflect.DeepEqual(osa.Config, sa.Config) { if sa.Group.Name == osa.Group.Name && reflect.DeepEqual(sa.Group.Peers, osa.Group.Peers) { // Since this already exists we know it succeeded, just respond to this caller. js.mu.RLock() client, subject, reply, recovering := sa.Client, sa.Subject, sa.Reply, sa.recovering js.mu.RUnlock() if !recovering { var resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}} resp.StreamInfo = &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Cluster: js.clusterInfo(mset.raftGroup()), Sources: mset.sourcesInfo(), Mirror: mset.mirrorInfo(), TimeStamp: time.Now().UTC(), } s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) } return } else { // We had a bug where we could have multiple assignments for the same // stream but with different group assignments, including multiple raft // groups. So check for that here. We can only bet on the last one being // consistent in the long run, so let it continue if we see this condition. s.Warnf("JetStream cluster detected duplicate assignment for stream %q for account %q", sa.Config.Name, acc.Name) if osa.Group.node != nil && osa.Group.node != sa.Group.node { osa.Group.node.Delete() osa.Group.node = nil } } } } mset.setStreamAssignment(sa) // Check if our config has really been updated. cfg := mset.config() if !reflect.DeepEqual(&cfg, sa.Config) { if err = mset.updateWithAdvisory(sa.Config, false); err != nil { s.Warnf("JetStream cluster error updating stream %q for account %q: %v", sa.Config.Name, acc.Name, err) if osa != nil { // Process the raft group and make sure it's running if needed. js.createRaftGroup(acc.GetName(), osa.Group, storage, pprofLabels{ "type": "stream", "account": mset.accName(), "stream": mset.name(), }) mset.setStreamAssignment(osa) } if rg.node != nil { rg.node.Delete() rg.node = nil } } } } else if err == NewJSStreamNotFoundError() { // Add in the stream here. mset, err = acc.addStreamWithAssignment(sa.Config, nil, sa) } if mset != nil { mset.setCreatedTime(sa.Created) } } // This is an error condition. if err != nil { // If we're shutting down we could get a variety of errors, for example: // 'JetStream not enabled for account' when looking up the stream. // Normally we can continue and delete state, but need to be careful when shutting down. if js.isShuttingDown() { s.Debugf("Could not create stream, JetStream shutting down") return } if IsNatsErr(err, JSStreamStoreFailedF) { s.Warnf("Stream create failed for '%s > %s': %v", sa.Client.serviceAccount(), sa.Config.Name, err) err = errStreamStoreFailed } js.mu.Lock() sa.err = err hasResponded := sa.responded // If out of space do nothing for now. if isOutOfSpaceErr(err) { hasResponded = true } if rg.node != nil { rg.node.Delete() } var result *streamAssignmentResult if !hasResponded { result = &streamAssignmentResult{ Account: sa.Client.serviceAccount(), Stream: sa.Config.Name, Response: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}, } result.Response.Error = NewJSStreamCreateError(err, Unless(err)) } js.mu.Unlock() // Send response to the metadata leader. They will forward to the user as needed. if result != nil { s.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result) } return } // Re-capture node. js.mu.RLock() node := rg.node js.mu.RUnlock() // Start our monitoring routine. if node != nil { if !alreadyRunning { if mset != nil { mset.monitorWg.Add(1) } s.startGoRoutine( func() { js.monitorStream(mset, sa, false) }, pprofLabels{ "type": "stream", "account": mset.accName(), "stream": mset.name(), }, ) } } else { // Single replica stream, process manually here. // If we are restoring, process that first. if sa.Restore != nil { // We are restoring a stream here. restoreDoneCh := s.processStreamRestore(sa.Client, acc, sa.Config, _EMPTY_, sa.Reply, _EMPTY_) s.startGoRoutine(func() { defer s.grWG.Done() select { case err := <-restoreDoneCh: if err == nil { mset, err = acc.lookupStream(sa.Config.Name) if mset != nil { mset.setStreamAssignment(sa) mset.setCreatedTime(sa.Created) } } if err != nil { if mset != nil { mset.delete() } js.mu.Lock() sa.err = err result := &streamAssignmentResult{ Account: sa.Client.serviceAccount(), Stream: sa.Config.Name, Restore: &JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}}, } result.Restore.Error = NewJSStreamRestoreError(err, Unless(err)) js.mu.Unlock() // Send response to the metadata leader. They will forward to the user as needed. b, _ := json.Marshal(result) // Avoids auto-processing and doing fancy json with newlines. s.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, b) return } js.processStreamLeaderChange(mset, true) // Check to see if we have restored consumers here. // These are not currently assigned so we will need to do so here. if consumers := mset.getPublicConsumers(); len(consumers) > 0 { js.mu.RLock() cc := js.cluster js.mu.RUnlock() for _, o := range consumers { name, cfg := o.String(), o.config() rg := cc.createGroupForConsumer(&cfg, sa) // Place our initial state here as well for assignment distribution. ca := &consumerAssignment{ Group: rg, Stream: sa.Config.Name, Name: name, Config: &cfg, Client: sa.Client, Created: o.createdTime(), } addEntry := encodeAddConsumerAssignment(ca) cc.meta.ForwardProposal(addEntry) // Check to make sure we see the assignment. go func() { ticker := time.NewTicker(time.Second) defer ticker.Stop() for range ticker.C { js.mu.RLock() ca, meta := js.consumerAssignment(ca.Client.serviceAccount(), sa.Config.Name, name), cc.meta js.mu.RUnlock() if ca == nil { s.Warnf("Consumer assignment has not been assigned, retrying") if meta != nil { meta.ForwardProposal(addEntry) } else { return } } else { return } } }() } } case <-s.quitCh: return } }) } else { js.processStreamLeaderChange(mset, true) } } } // processStreamRemoval is called when followers have replicated an assignment. func (js *jetStream) processStreamRemoval(sa *streamAssignment) { js.mu.Lock() s, cc := js.srv, js.cluster if s == nil || cc == nil || cc.meta == nil { // TODO(dlc) - debug at least js.mu.Unlock() return } stream := sa.Config.Name isMember := sa.Group.isMember(cc.meta.ID()) wasLeader := cc.isStreamLeader(sa.Client.serviceAccount(), stream) // Check if we already have this assigned. accStreams := cc.streams[sa.Client.serviceAccount()] needDelete := accStreams != nil && accStreams[stream] != nil if needDelete { delete(accStreams, stream) if len(accStreams) == 0 { delete(cc.streams, sa.Client.serviceAccount()) } } js.mu.Unlock() if needDelete { js.processClusterDeleteStream(sa, isMember, wasLeader) } } func (js *jetStream) processClusterDeleteStream(sa *streamAssignment, isMember, wasLeader bool) { if sa == nil { return } js.mu.RLock() s := js.srv node := sa.Group.node hadLeader := node == nil || !node.Leaderless() offline := s.allPeersOffline(sa.Group) var isMetaLeader bool if cc := js.cluster; cc != nil { isMetaLeader = cc.isLeader() } recovering := sa.recovering js.mu.RUnlock() stopped := false var resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}} var err error var acc *Account // Go ahead and delete the stream if we have it and the account here. if acc, _ = s.LookupAccount(sa.Client.serviceAccount()); acc != nil { if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil { // shut down monitor by shutting down raft if n := mset.raftNode(); n != nil { n.Delete() } // wait for monitor to be shut down mset.monitorWg.Wait() err = mset.stop(true, wasLeader) stopped = true } else if isMember { s.Warnf("JetStream failed to lookup running stream while removing stream '%s > %s' from this server", sa.Client.serviceAccount(), sa.Config.Name) } } else if isMember { s.Warnf("JetStream failed to lookup account while removing stream '%s > %s' from this server", sa.Client.serviceAccount(), sa.Config.Name) } // Always delete the node if present. if node != nil { node.Delete() } // This is a stop gap cleanup in case // 1) the account does not exist (and mset couldn't be stopped) and/or // 2) node was nil (and couldn't be deleted) if !stopped || node == nil { if sacc := s.SystemAccount(); sacc != nil { saccName := sacc.GetName() os.RemoveAll(filepath.Join(js.config.StoreDir, saccName, defaultStoreDirName, sa.Group.Name)) // cleanup dependent consumer groups if !stopped { for _, ca := range sa.consumers { // Make sure we cleanup any possible running nodes for the consumers. if isMember && ca.Group != nil && ca.Group.node != nil { ca.Group.node.Delete() } os.RemoveAll(filepath.Join(js.config.StoreDir, saccName, defaultStoreDirName, ca.Group.Name)) } } } } accDir := filepath.Join(js.config.StoreDir, sa.Client.serviceAccount()) streamDir := filepath.Join(accDir, streamsDir) os.RemoveAll(filepath.Join(streamDir, sa.Config.Name)) // no op if not empty os.Remove(streamDir) os.Remove(accDir) // Normally we want only the leader to respond here, but if we had no leader then all members will respond to make // sure we get feedback to the user. if !isMember || (hadLeader && !wasLeader) { // If all the peers are offline and we are the meta leader we will also respond, so suppress returning here. if !(offline && isMetaLeader) { return } } // Do not respond if the account does not exist any longer if acc == nil || recovering { return } if err != nil { resp.Error = NewJSStreamGeneralError(err, Unless(err)) s.sendAPIErrResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, s.jsonResponse(resp)) } else { resp.Success = true s.sendAPIResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, s.jsonResponse(resp)) } } // processConsumerAssignment is called when followers have replicated an assignment for a consumer. func (js *jetStream) processConsumerAssignment(ca *consumerAssignment) { js.mu.RLock() s, cc := js.srv, js.cluster accName, stream, consumerName := ca.Client.serviceAccount(), ca.Stream, ca.Name noMeta := cc == nil || cc.meta == nil shuttingDown := js.shuttingDown var ourID string if !noMeta { ourID = cc.meta.ID() } var isMember bool if ca.Group != nil && ourID != _EMPTY_ { isMember = ca.Group.isMember(ourID) } js.mu.RUnlock() if s == nil || noMeta || shuttingDown { return } js.mu.Lock() sa := js.streamAssignment(accName, stream) if sa == nil { js.mu.Unlock() s.Debugf("Consumer create failed, could not locate stream '%s > %s'", accName, stream) return } // Might need this below. numReplicas := sa.Config.Replicas // Track if this existed already. var wasExisting bool // Check if we have an existing consumer assignment. if sa.consumers == nil { sa.consumers = make(map[string]*consumerAssignment) } else if oca := sa.consumers[ca.Name]; oca != nil { wasExisting = true // Copy over private existing state from former CA. if ca.Group != nil { ca.Group.node = oca.Group.node } ca.responded = oca.responded ca.err = oca.err } // Capture the optional state. We will pass it along if we are a member to apply. // This is only applicable when restoring a stream with consumers. state := ca.State ca.State = nil // Place into our internal map under the stream assignment. // Ok to replace an existing one, we check on process call below. sa.consumers[ca.Name] = ca ca.pending = false js.mu.Unlock() acc, err := s.LookupAccount(accName) if err != nil { ll := fmt.Sprintf("Account [%s] lookup for consumer create failed: %v", accName, err) if isMember { if !js.isMetaRecovering() { // If we can not lookup the account and we are a member, send this result back to the metacontroller leader. result := &consumerAssignmentResult{ Account: accName, Stream: stream, Consumer: consumerName, Response: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}, } result.Response.Error = NewJSNoAccountError() s.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result) } s.Warnf(ll) } else { s.Debugf(ll) } return } // Check if this is for us.. if isMember { js.processClusterCreateConsumer(ca, state, wasExisting) } else { // We need to be removed here, we are no longer assigned. // Grab consumer if we have it. var o *consumer if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil { o = mset.lookupConsumer(ca.Name) } // Check if we have a raft node running, meaning we are no longer part of the group but were. js.mu.Lock() if node := ca.Group.node; node != nil { // We have one here even though we are not a member. This can happen on re-assignment. s.Debugf("JetStream removing consumer '%s > %s > %s' from this server", sa.Client.serviceAccount(), sa.Config.Name, ca.Name) if node.Leader() { s.Debugf("JetStream consumer '%s > %s > %s' is being removed and was the leader, will perform stepdown", sa.Client.serviceAccount(), sa.Config.Name, ca.Name) peers, cn := node.Peers(), s.cachedClusterName() migrating := numReplicas != len(peers) // Select a new peer to transfer to. If we are a migrating make sure its from the new cluster. var npeer string for _, r := range peers { if !r.Current { continue } if !migrating { npeer = r.ID break } else if sir, ok := s.nodeToInfo.Load(r.ID); ok && sir != nil { si := sir.(nodeInfo) if si.cluster != cn { npeer = r.ID break } } } // Clear the raftnode from our consumer so that a subsequent o.delete will not also issue a stepdown. if o != nil { o.clearRaftNode() } // Manually handle the stepdown and deletion of the node. node.UpdateKnownPeers(ca.Group.Peers) node.StepDown(npeer) node.Delete() } else { node.UpdateKnownPeers(ca.Group.Peers) } } // Always clear the old node. ca.Group.node = nil ca.err = nil js.mu.Unlock() if o != nil { o.deleteWithoutAdvisory() } } } func (js *jetStream) processConsumerRemoval(ca *consumerAssignment) { js.mu.Lock() s, cc := js.srv, js.cluster if s == nil || cc == nil || cc.meta == nil { // TODO(dlc) - debug at least js.mu.Unlock() return } wasLeader := cc.isConsumerLeader(ca.Client.serviceAccount(), ca.Stream, ca.Name) // Delete from our state. var needDelete bool if accStreams := cc.streams[ca.Client.serviceAccount()]; accStreams != nil { if sa := accStreams[ca.Stream]; sa != nil && sa.consumers != nil && sa.consumers[ca.Name] != nil { oca := sa.consumers[ca.Name] // Make sure this removal is for what we have, otherwise ignore. if ca.Group != nil && oca.Group != nil && ca.Group.Name == oca.Group.Name { needDelete = true oca.deleted = true delete(sa.consumers, ca.Name) } } } js.mu.Unlock() if needDelete { js.processClusterDeleteConsumer(ca, wasLeader) } } type consumerAssignmentResult struct { Account string `json:"account"` Stream string `json:"stream"` Consumer string `json:"consumer"` Response *JSApiConsumerCreateResponse `json:"response,omitempty"` } // processClusterCreateConsumer is when we are a member of the group and need to create the consumer. func (js *jetStream) processClusterCreateConsumer(ca *consumerAssignment, state *ConsumerState, wasExisting bool) { if ca == nil { return } js.mu.RLock() s := js.srv rg := ca.Group alreadyRunning := rg != nil && rg.node != nil accName, stream, consumer := ca.Client.serviceAccount(), ca.Stream, ca.Name js.mu.RUnlock() acc, err := s.LookupAccount(accName) if err != nil { s.Warnf("JetStream cluster failed to lookup axccount %q: %v", accName, err) return } // Go ahead and create or update the consumer. mset, err := acc.lookupStream(stream) if err != nil { if !js.isMetaRecovering() { js.mu.Lock() s.Warnf("Consumer create failed, could not locate stream '%s > %s > %s'", ca.Client.serviceAccount(), ca.Stream, ca.Name) ca.err = NewJSStreamNotFoundError() result := &consumerAssignmentResult{ Account: ca.Client.serviceAccount(), Stream: ca.Stream, Consumer: ca.Name, Response: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}, } result.Response.Error = NewJSStreamNotFoundError() s.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result) js.mu.Unlock() } return } // Check if we already have this consumer running. o := mset.lookupConsumer(consumer) if !alreadyRunning { // Process the raft group and make sure its running if needed. storage := mset.config().Storage if ca.Config.MemoryStorage { storage = MemoryStorage } // No-op if R1. js.createRaftGroup(accName, rg, storage, pprofLabels{ "type": "consumer", "account": mset.accName(), "stream": ca.Stream, "consumer": ca.Name, }) } else { // If we are clustered update the known peers. js.mu.RLock() node := rg.node js.mu.RUnlock() if node != nil { node.UpdateKnownPeers(ca.Group.Peers) } } // Check if we already have this consumer running. var didCreate, isConfigUpdate, needsLocalResponse bool if o == nil { // Add in the consumer if needed. if o, err = mset.addConsumerWithAssignment(ca.Config, ca.Name, ca, js.isMetaRecovering(), ActionCreateOrUpdate); err == nil { didCreate = true } } else { // This consumer exists. // Only update if config is really different. cfg := o.config() if isConfigUpdate = !reflect.DeepEqual(&cfg, ca.Config); isConfigUpdate { // Call into update, ignore consumer exists error here since this means an old deliver subject is bound // which can happen on restart etc. if err := o.updateConfig(ca.Config); err != nil && err != NewJSConsumerNameExistError() { // This is essentially an update that has failed. Respond back to metaleader if we are not recovering. js.mu.RLock() if !js.metaRecovering { result := &consumerAssignmentResult{ Account: accName, Stream: stream, Consumer: consumer, Response: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}, } result.Response.Error = NewJSConsumerNameExistError() s.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result) } s.Warnf("Consumer create failed during update for '%s > %s > %s': %v", ca.Client.serviceAccount(), ca.Stream, ca.Name, err) js.mu.RUnlock() return } } var sendState bool js.mu.RLock() n := rg.node // Check if we already had a consumer assignment and its still pending. cca, oca := ca, o.consumerAssignment() if oca != nil { if !oca.responded { // We can't override info for replying here otherwise leader once elected can not respond. // So copy over original client and the reply from the old ca. cac := *ca cac.Client = oca.Client cac.Reply = oca.Reply cca = &cac needsLocalResponse = true } // If we look like we are scaling up, let's send our current state to the group. sendState = len(ca.Group.Peers) > len(oca.Group.Peers) && o.IsLeader() && n != nil // Signal that this is an update if ca.Reply != _EMPTY_ { isConfigUpdate = true } } js.mu.RUnlock() if sendState { if snap, err := o.store.EncodedState(); err == nil { n.SendSnapshot(snap) } } // Set CA for our consumer. o.setConsumerAssignment(cca) s.Debugf("JetStream cluster, consumer '%s > %s > %s' was already running", ca.Client.serviceAccount(), ca.Stream, ca.Name) } // If we have an initial state set apply that now. if state != nil && o != nil { o.mu.Lock() err = o.setStoreState(state) o.mu.Unlock() } if err != nil { // If we're shutting down we could get a variety of errors. // Normally we can continue and delete state, but need to be careful when shutting down. if js.isShuttingDown() { s.Debugf("Could not create consumer, JetStream shutting down") return } if IsNatsErr(err, JSConsumerStoreFailedErrF) { s.Warnf("Consumer create failed for '%s > %s > %s': %v", ca.Client.serviceAccount(), ca.Stream, ca.Name, err) err = errConsumerStoreFailed } js.mu.Lock() ca.err = err hasResponded := ca.responded // If out of space do nothing for now. if isOutOfSpaceErr(err) { hasResponded = true } if rg.node != nil { rg.node.Delete() // Clear the node here. rg.node = nil } // If we did seem to create a consumer make sure to stop it. if o != nil { o.stop() } var result *consumerAssignmentResult if !hasResponded && !js.metaRecovering { result = &consumerAssignmentResult{ Account: ca.Client.serviceAccount(), Stream: ca.Stream, Consumer: ca.Name, Response: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}, } result.Response.Error = NewJSConsumerCreateError(err, Unless(err)) } else if err == errNoInterest { // This is a stranded ephemeral, let's clean this one up. subject := fmt.Sprintf(JSApiConsumerDeleteT, ca.Stream, ca.Name) mset.outq.send(newJSPubMsg(subject, _EMPTY_, _EMPTY_, nil, nil, nil, 0)) } js.mu.Unlock() if result != nil { // Send response to the metadata leader. They will forward to the user as needed. b, _ := json.Marshal(result) // Avoids auto-processing and doing fancy json with newlines. s.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, b) } } else { js.mu.RLock() node := rg.node js.mu.RUnlock() if didCreate { o.setCreatedTime(ca.Created) } else { // Check for scale down to 1.. if node != nil && len(rg.Peers) == 1 { o.clearNode() o.setLeader(true) // Need to clear from rg too. js.mu.Lock() rg.node = nil client, subject, reply := ca.Client, ca.Subject, ca.Reply js.mu.Unlock() var resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}} resp.ConsumerInfo = o.info() s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) return } } if node == nil { // Single replica consumer, process manually here. js.mu.Lock() // Force response in case we think this is an update. if !js.metaRecovering && isConfigUpdate { ca.responded = false } js.mu.Unlock() js.processConsumerLeaderChange(o, true) } else { // Clustered consumer. // Start our monitoring routine if needed. if !alreadyRunning && o.shouldStartMonitor() { s.startGoRoutine( func() { js.monitorConsumer(o, ca) }, pprofLabels{ "type": "consumer", "account": mset.accName(), "stream": mset.name(), "consumer": ca.Name, }, ) } // For existing consumer, only send response if not recovering. if wasExisting && !js.isMetaRecovering() { if o.IsLeader() || (!didCreate && needsLocalResponse) { // Process if existing as an update. Double check that this is not recovered. js.mu.RLock() client, subject, reply, recovering := ca.Client, ca.Subject, ca.Reply, ca.recovering js.mu.RUnlock() if !recovering { var resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}} resp.ConsumerInfo = o.info() s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) } } } } } } func (js *jetStream) processClusterDeleteConsumer(ca *consumerAssignment, wasLeader bool) { if ca == nil { return } js.mu.RLock() s := js.srv node := ca.Group.node offline := s.allPeersOffline(ca.Group) var isMetaLeader bool if cc := js.cluster; cc != nil { isMetaLeader = cc.isLeader() } recovering := ca.recovering js.mu.RUnlock() var resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}} var err error var acc *Account // Go ahead and delete the consumer if we have it and the account. if acc, _ = s.LookupAccount(ca.Client.serviceAccount()); acc != nil { if mset, _ := acc.lookupStream(ca.Stream); mset != nil { if o := mset.lookupConsumer(ca.Name); o != nil { err = o.stopWithFlags(true, false, true, wasLeader) } } } else if ca.Group != nil { // We have a missing account, see if we can cleanup. if sacc := s.SystemAccount(); sacc != nil { os.RemoveAll(filepath.Join(js.config.StoreDir, sacc.GetName(), defaultStoreDirName, ca.Group.Name)) } } // Always delete the node if present. if node != nil { node.Delete() } if !wasLeader || ca.Reply == _EMPTY_ { if !(offline && isMetaLeader) { return } } // Do not respond if the account does not exist any longer or this is during recovery. if acc == nil || recovering { return } if err != nil { resp.Error = NewJSStreamNotFoundError(Unless(err)) s.sendAPIErrResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(resp)) } else { resp.Success = true s.sendAPIResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(resp)) } } // Returns the consumer assignment, or nil if not present. // Lock should be held. func (js *jetStream) consumerAssignment(account, stream, consumer string) *consumerAssignment { if sa := js.streamAssignment(account, stream); sa != nil { return sa.consumers[consumer] } return nil } // consumerAssigned informs us if this server has this consumer assigned. func (jsa *jsAccount) consumerAssigned(stream, consumer string) bool { jsa.mu.RLock() js, acc := jsa.js, jsa.account jsa.mu.RUnlock() if js == nil { return false } js.mu.RLock() defer js.mu.RUnlock() return js.cluster.isConsumerAssigned(acc, stream, consumer) } // Read lock should be held. func (cc *jetStreamCluster) isConsumerAssigned(a *Account, stream, consumer string) bool { // Non-clustered mode always return true. if cc == nil { return true } if cc.meta == nil { return false } var sa *streamAssignment accStreams := cc.streams[a.Name] if accStreams != nil { sa = accStreams[stream] } if sa == nil { // TODO(dlc) - This should not happen. return false } ca := sa.consumers[consumer] if ca == nil { return false } rg := ca.Group // Check if we are the leader of this raftGroup assigned to the stream. ourID := cc.meta.ID() for _, peer := range rg.Peers { if peer == ourID { return true } } return false } // Returns our stream and underlying raft node. func (o *consumer) streamAndNode() (*stream, RaftNode) { if o == nil { return nil, nil } o.mu.RLock() defer o.mu.RUnlock() return o.mset, o.node } // Return the replica count for this consumer. If the consumer has been // stopped, this will return an error. func (o *consumer) replica() (int, error) { o.mu.RLock() oCfg := o.cfg mset := o.mset o.mu.RUnlock() if mset == nil { return 0, errBadConsumer } sCfg := mset.config() return oCfg.replicas(&sCfg), nil } func (o *consumer) raftGroup() *raftGroup { if o == nil { return nil } o.mu.RLock() defer o.mu.RUnlock() if o.ca == nil { return nil } return o.ca.Group } func (o *consumer) clearRaftNode() { if o == nil { return } o.mu.Lock() defer o.mu.Unlock() o.node = nil } func (o *consumer) raftNode() RaftNode { if o == nil { return nil } o.mu.RLock() defer o.mu.RUnlock() return o.node } func (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) { s, n, cc := js.server(), o.raftNode(), js.cluster defer s.grWG.Done() defer o.clearMonitorRunning() if n == nil { s.Warnf("No RAFT group for '%s > %s > %s'", o.acc.Name, ca.Stream, ca.Name) return } // Make sure to stop the raft group on exit to prevent accidental memory bloat. // This should be below the checkInMonitor call though to avoid stopping it out // from underneath the one that is running since it will be the same raft node. defer n.Stop() qch, lch, aq, uch, ourPeerId := n.QuitC(), n.LeadChangeC(), n.ApplyQ(), o.updateC(), cc.meta.ID() s.Debugf("Starting consumer monitor for '%s > %s > %s' [%s]", o.acc.Name, ca.Stream, ca.Name, n.Group()) defer s.Debugf("Exiting consumer monitor for '%s > %s > %s' [%s]", o.acc.Name, ca.Stream, ca.Name, n.Group()) const ( compactInterval = 2 * time.Minute compactSizeMin = 64 * 1024 // What is stored here is always small for consumers. compactNumMin = 1024 minSnapDelta = 10 * time.Second ) // Spread these out for large numbers on server restart. rci := time.Duration(rand.Int63n(int64(time.Minute))) t := time.NewTicker(compactInterval + rci) defer t.Stop() // Highwayhash key for generating hashes. key := make([]byte, 32) crand.Read(key) // Hash of the last snapshot (fixed size in memory). var lastSnap []byte var lastSnapTime time.Time // Don't allow the upper layer to install snapshots until we have // fully recovered from disk. recovering := true doSnapshot := func(force bool) { // Bail if trying too fast and not in a forced situation. if recovering || (!force && time.Since(lastSnapTime) < minSnapDelta) { return } // Check several things to see if we need a snapshot. ne, nb := n.Size() if !n.NeedSnapshot() { // Check if we should compact etc. based on size of log. if !force && ne < compactNumMin && nb < compactSizeMin { return } } if snap, err := o.store.EncodedState(); err == nil { hash := highwayhash.Sum(snap, key) // If the state hasn't changed but the log has gone way over // the compaction size then we will want to compact anyway. // This can happen for example when a pull consumer fetches a // lot on an idle stream, log entries get distributed but the // state never changes, therefore the log never gets compacted. if !bytes.Equal(hash[:], lastSnap) || ne >= compactNumMin || nb >= compactSizeMin { if err := n.InstallSnapshot(snap); err == nil { lastSnap, lastSnapTime = hash[:], time.Now() } else if err != errNoSnapAvailable && err != errNodeClosed && err != errCatchupsRunning { s.RateLimitWarnf("Failed to install snapshot for '%s > %s > %s' [%s]: %v", o.acc.Name, ca.Stream, ca.Name, n.Group(), err) } } } } // For migration tracking. var mmt *time.Ticker var mmtc <-chan time.Time startMigrationMonitoring := func() { if mmt == nil { mmt = time.NewTicker(500 * time.Millisecond) mmtc = mmt.C } } stopMigrationMonitoring := func() { if mmt != nil { mmt.Stop() mmt, mmtc = nil, nil } } defer stopMigrationMonitoring() // Track if we are leader. var isLeader bool for { select { case <-s.quitCh: // Server shutting down, but we might receive this before qch, so try to snapshot. doSnapshot(false) return case <-qch: // Clean signal from shutdown routine so do best effort attempt to snapshot. doSnapshot(false) return case <-aq.ch: ces := aq.pop() for _, ce := range ces { // No special processing needed for when we are caught up on restart. if ce == nil { recovering = false if n.NeedSnapshot() { doSnapshot(true) } } else if err := js.applyConsumerEntries(o, ce, isLeader); err == nil { var ne, nb uint64 // We can't guarantee writes are flushed while we're shutting down. Just rely on replay during recovery. if !js.isShuttingDown() { ne, nb = n.Applied(ce.Index) } ce.ReturnToPool() // If we have at least min entries to compact, go ahead and snapshot/compact. if nb > 0 && ne >= compactNumMin || nb > compactSizeMin { doSnapshot(false) } } else if err != errConsumerClosed { s.Warnf("Error applying consumer entries to '%s > %s'", ca.Client.serviceAccount(), ca.Name) } } aq.recycle(&ces) case isLeader = <-lch: if recovering && !isLeader { js.setConsumerAssignmentRecovering(ca) } // Process the change. if err := js.processConsumerLeaderChange(o, isLeader); err == nil { // Check our state if we are under an interest based stream. if mset := o.getStream(); mset != nil { var ss StreamState mset.store.FastState(&ss) o.checkStateForInterestStream(&ss) } } // We may receive a leader change after the consumer assignment which would cancel us // monitoring for this closely. So re-assess our state here as well. // Or the old leader is no longer part of the set and transferred leadership // for this leader to resume with removal rg := o.raftGroup() // Check for migrations (peer count and replica count differ) here. // We set the state on the stream assignment update below. replicas, err := o.replica() if err != nil { continue } if isLeader && len(rg.Peers) != replicas { startMigrationMonitoring() } else { stopMigrationMonitoring() } case <-uch: // keep consumer assignment current ca = o.consumerAssignment() // We get this when we have a new consumer assignment caused by an update. // We want to know if we are migrating. rg := o.raftGroup() // If we are migrating, monitor for the new peers to be caught up. replicas, err := o.replica() if err != nil { continue } if isLeader && len(rg.Peers) != replicas { startMigrationMonitoring() } else { stopMigrationMonitoring() } case <-mmtc: if !isLeader { // We are no longer leader, so not our job. stopMigrationMonitoring() continue } rg := o.raftGroup() ci := js.clusterInfo(rg) replicas, err := o.replica() if err != nil { continue } if len(rg.Peers) <= replicas { // Migration no longer happening, so not our job anymore stopMigrationMonitoring() continue } newPeers, oldPeers, newPeerSet, _ := genPeerInfo(rg.Peers, len(rg.Peers)-replicas) // If we are part of the new peerset and we have been passed the baton. // We will handle scale down. if newPeerSet[ourPeerId] { for _, p := range oldPeers { n.ProposeRemovePeer(p) } cca := ca.copyGroup() cca.Group.Peers = newPeers cca.Group.Cluster = s.cachedClusterName() cc.meta.ForwardProposal(encodeAddConsumerAssignment(cca)) s.Noticef("Scaling down '%s > %s > %s' to %+v", ca.Client.serviceAccount(), ca.Stream, ca.Name, s.peerSetToNames(newPeers)) } else { var newLeaderPeer, newLeader, newCluster string neededCurrent, current := replicas/2+1, 0 for _, r := range ci.Replicas { if r.Current && newPeerSet[r.Peer] { current++ if newCluster == _EMPTY_ { newLeaderPeer, newLeader, newCluster = r.Peer, r.Name, r.cluster } } } // Check if we have a quorom if current >= neededCurrent { s.Noticef("Transfer of consumer leader for '%s > %s > %s' to '%s'", ca.Client.serviceAccount(), ca.Stream, ca.Name, newLeader) n.StepDown(newLeaderPeer) } } case <-t.C: doSnapshot(false) } } } func (js *jetStream) applyConsumerEntries(o *consumer, ce *CommittedEntry, isLeader bool) error { for _, e := range ce.Entries { if e.Type == EntrySnapshot { if !isLeader { // No-op needed? state, err := decodeConsumerState(e.Data) if err != nil { if mset, node := o.streamAndNode(); mset != nil && node != nil { s := js.srv s.Errorf("JetStream cluster could not decode consumer snapshot for '%s > %s > %s' [%s]", mset.account(), mset.name(), o, node.Group()) } panic(err.Error()) } if err = o.store.Update(state); err != nil { o.mu.RLock() s, acc, mset, name := o.srv, o.acc, o.mset, o.name o.mu.RUnlock() if s != nil && mset != nil { s.Warnf("Consumer '%s > %s > %s' error on store update from snapshot entry: %v", acc, mset.name(), name, err) } } // Check our interest state if applicable. if mset := o.getStream(); mset != nil { var ss StreamState mset.store.FastState(&ss) // We used to register preacks here if our ack floor was higher than the last sequence. // Now when streams catch up they properly call checkInterestState() and periodically run this as well. // If our states drift this could have allocated lots of pre-acks. o.checkStateForInterestStream(&ss) } } } else if e.Type == EntryRemovePeer { js.mu.RLock() var ourID string if js.cluster != nil && js.cluster.meta != nil { ourID = js.cluster.meta.ID() } js.mu.RUnlock() if peer := string(e.Data); peer == ourID { shouldRemove := true if mset := o.getStream(); mset != nil { if sa := mset.streamAssignment(); sa != nil && sa.Group != nil { js.mu.RLock() shouldRemove = !sa.Group.isMember(ourID) js.mu.RUnlock() } } if shouldRemove { o.stopWithFlags(true, false, false, false) } } } else if e.Type == EntryAddPeer { // Ignore for now. } else { buf := e.Data switch entryOp(buf[0]) { case updateDeliveredOp: dseq, sseq, dc, ts, err := decodeDeliveredUpdate(buf[1:]) if err != nil { if mset, node := o.streamAndNode(); mset != nil && node != nil { s := js.srv s.Errorf("JetStream cluster could not decode consumer delivered update for '%s > %s > %s' [%s]", mset.account(), mset.name(), o, node.Group()) } panic(err.Error()) } // Make sure to update delivered under the lock. o.mu.Lock() err = o.store.UpdateDelivered(dseq, sseq, dc, ts) o.ldt = time.Now() o.mu.Unlock() if err != nil { panic(err.Error()) } case updateAcksOp: dseq, sseq, err := decodeAckUpdate(buf[1:]) if err != nil { if mset, node := o.streamAndNode(); mset != nil && node != nil { s := js.srv s.Errorf("JetStream cluster could not decode consumer ack update for '%s > %s > %s' [%s]", mset.account(), mset.name(), o, node.Group()) } panic(err.Error()) } if err := o.processReplicatedAck(dseq, sseq); err == errConsumerClosed { return err } case updateSkipOp: o.mu.Lock() if !o.isLeader() { var le = binary.LittleEndian if sseq := le.Uint64(buf[1:]); sseq > o.sseq { o.sseq = sseq } } o.mu.Unlock() case addPendingRequest: o.mu.Lock() if !o.isLeader() { if o.prm == nil { o.prm = make(map[string]struct{}) } o.prm[string(buf[1:])] = struct{}{} } o.mu.Unlock() case removePendingRequest: o.mu.Lock() if !o.isLeader() { if o.prm != nil { delete(o.prm, string(buf[1:])) } } o.mu.Unlock() default: panic(fmt.Sprintf("JetStream Cluster Unknown group entry op type: %v", entryOp(buf[0]))) } } } return nil } var errConsumerClosed = errors.New("consumer closed") func (o *consumer) processReplicatedAck(dseq, sseq uint64) error { o.mu.Lock() // Update activity. o.lat = time.Now() // Do actual ack update to store. // Always do this to have it recorded. o.store.UpdateAcks(dseq, sseq) mset := o.mset if o.closed || mset == nil { o.mu.Unlock() return errConsumerClosed } if mset.closed.Load() { o.mu.Unlock() return errStreamClosed } // Check if we have a reply that was requested. if reply := o.replies[sseq]; reply != _EMPTY_ { o.outq.sendMsg(reply, nil) delete(o.replies, sseq) } if o.retention == LimitsPolicy { o.mu.Unlock() return nil } var sagap uint64 if o.cfg.AckPolicy == AckAll { if o.isLeader() { sagap = sseq - o.asflr } else { // We are a follower so only have the store state, so read that in. state, err := o.store.State() if err != nil { o.mu.Unlock() return err } sagap = sseq - state.AckFloor.Stream } } o.mu.Unlock() if sagap > 1 { // FIXME(dlc) - This is very inefficient, will need to fix. for seq := sseq; seq > sseq-sagap; seq-- { mset.ackMsg(o, seq) } } else { mset.ackMsg(o, sseq) } return nil } var errBadAckUpdate = errors.New("jetstream cluster bad replicated ack update") var errBadDeliveredUpdate = errors.New("jetstream cluster bad replicated delivered update") func decodeAckUpdate(buf []byte) (dseq, sseq uint64, err error) { var bi, n int if dseq, n = binary.Uvarint(buf); n < 0 { return 0, 0, errBadAckUpdate } bi += n if sseq, n = binary.Uvarint(buf[bi:]); n < 0 { return 0, 0, errBadAckUpdate } return dseq, sseq, nil } func decodeDeliveredUpdate(buf []byte) (dseq, sseq, dc uint64, ts int64, err error) { var bi, n int if dseq, n = binary.Uvarint(buf); n < 0 { return 0, 0, 0, 0, errBadDeliveredUpdate } bi += n if sseq, n = binary.Uvarint(buf[bi:]); n < 0 { return 0, 0, 0, 0, errBadDeliveredUpdate } bi += n if dc, n = binary.Uvarint(buf[bi:]); n < 0 { return 0, 0, 0, 0, errBadDeliveredUpdate } bi += n if ts, n = binary.Varint(buf[bi:]); n < 0 { return 0, 0, 0, 0, errBadDeliveredUpdate } return dseq, sseq, dc, ts, nil } func (js *jetStream) processConsumerLeaderChange(o *consumer, isLeader bool) error { stepDownIfLeader := func() error { if node := o.raftNode(); node != nil && isLeader { node.StepDown() } return errors.New("failed to update consumer leader status") } if o == nil || o.isClosed() { return stepDownIfLeader() } ca := o.consumerAssignment() if ca == nil { return stepDownIfLeader() } js.mu.Lock() s, account, err := js.srv, ca.Client.serviceAccount(), ca.err client, subject, reply, streamName, consumerName := ca.Client, ca.Subject, ca.Reply, ca.Stream, ca.Name hasResponded := ca.responded ca.responded = true js.mu.Unlock() acc, _ := s.LookupAccount(account) if acc == nil { return stepDownIfLeader() } if isLeader { s.Noticef("JetStream cluster new consumer leader for '%s > %s > %s'", ca.Client.serviceAccount(), streamName, consumerName) s.sendConsumerLeaderElectAdvisory(o) } else { // We are stepping down. // Make sure if we are doing so because we have lost quorum that we send the appropriate advisories. if node := o.raftNode(); node != nil && !node.Quorum() && time.Since(node.Created()) > 5*time.Second { s.sendConsumerLostQuorumAdvisory(o) } } // Tell consumer to switch leader status. o.setLeader(isLeader) if !isLeader || hasResponded { if isLeader { o.clearInitialInfo() } return nil } var resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}} if err != nil { resp.Error = NewJSConsumerCreateError(err, Unless(err)) s.sendAPIErrResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) } else { resp.ConsumerInfo = o.initialInfo() s.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp)) o.sendCreateAdvisory() } return nil } // Determines if we should send lost quorum advisory. We throttle these after first one. func (o *consumer) shouldSendLostQuorum() bool { o.mu.Lock() defer o.mu.Unlock() if time.Since(o.lqsent) >= lostQuorumAdvInterval { o.lqsent = time.Now() return true } return false } func (s *Server) sendConsumerLostQuorumAdvisory(o *consumer) { if o == nil { return } node, stream, consumer, acc := o.raftNode(), o.streamName(), o.String(), o.account() if node == nil { return } if !o.shouldSendLostQuorum() { return } s.Warnf("JetStream cluster consumer '%s > %s > %s' has NO quorum, stalled.", acc.GetName(), stream, consumer) subj := JSAdvisoryConsumerQuorumLostPre + "." + stream + "." + consumer adv := &JSConsumerQuorumLostAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerQuorumLostAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: stream, Consumer: consumer, Replicas: s.replicas(node), Domain: s.getOpts().JetStreamDomain, } // Send to the user's account if not the system account. if acc != s.SystemAccount() { s.publishAdvisory(acc, subj, adv) } // Now do system level one. Place account info in adv, and nil account means system. adv.Account = acc.GetName() s.publishAdvisory(nil, subj, adv) } func (s *Server) sendConsumerLeaderElectAdvisory(o *consumer) { if o == nil { return } node, stream, consumer, acc := o.raftNode(), o.streamName(), o.String(), o.account() if node == nil { return } subj := JSAdvisoryConsumerLeaderElectedPre + "." + stream + "." + consumer adv := &JSConsumerLeaderElectedAdvisory{ TypedEvent: TypedEvent{ Type: JSConsumerLeaderElectedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: stream, Consumer: consumer, Leader: s.serverNameForNode(node.GroupLeader()), Replicas: s.replicas(node), Domain: s.getOpts().JetStreamDomain, } // Send to the user's account if not the system account. if acc != s.SystemAccount() { s.publishAdvisory(acc, subj, adv) } // Now do system level one. Place account info in adv, and nil account means system. adv.Account = acc.GetName() s.publishAdvisory(nil, subj, adv) } type streamAssignmentResult struct { Account string `json:"account"` Stream string `json:"stream"` Response *JSApiStreamCreateResponse `json:"create_response,omitempty"` Restore *JSApiStreamRestoreResponse `json:"restore_response,omitempty"` Update bool `json:"is_update,omitempty"` } // Determine if this is an insufficient resources' error type. func isInsufficientResourcesErr(resp *JSApiStreamCreateResponse) bool { return resp != nil && resp.Error != nil && IsNatsErr(resp.Error, JSInsufficientResourcesErr, JSMemoryResourcesExceededErr, JSStorageResourcesExceededErr) } // Process error results of stream and consumer assignments. // Success will be handled by stream leader. func (js *jetStream) processStreamAssignmentResults(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { var result streamAssignmentResult if err := json.Unmarshal(msg, &result); err != nil { // TODO(dlc) - log return } acc, _ := js.srv.LookupAccount(result.Account) if acc == nil { // TODO(dlc) - log return } js.mu.Lock() defer js.mu.Unlock() s, cc := js.srv, js.cluster if cc == nil || cc.meta == nil { return } // This should have been done already in processStreamAssignment, but in // case we have a code path that gets here with no processStreamAssignment, // then we will do the proper thing. Otherwise will be a no-op. cc.removeInflightProposal(result.Account, result.Stream) if sa := js.streamAssignment(result.Account, result.Stream); sa != nil && !sa.reassigning { canDelete := !result.Update && time.Since(sa.Created) < 5*time.Second // See if we should retry in case this cluster is full but there are others. if cfg, ci := sa.Config, sa.Client; cfg != nil && ci != nil && isInsufficientResourcesErr(result.Response) && canDelete { // If cluster is defined we can not retry. if cfg.Placement == nil || cfg.Placement.Cluster == _EMPTY_ { // If we have additional clusters to try we can retry. // We have already verified that ci != nil. if len(ci.Alternates) > 0 { if rg, err := js.createGroupForStream(ci, cfg); err != nil { s.Warnf("Retrying cluster placement for stream '%s > %s' failed due to placement error: %+v", result.Account, result.Stream, err) } else { if org := sa.Group; org != nil && len(org.Peers) > 0 { s.Warnf("Retrying cluster placement for stream '%s > %s' due to insufficient resources in cluster %q", result.Account, result.Stream, s.clusterNameForNode(org.Peers[0])) } else { s.Warnf("Retrying cluster placement for stream '%s > %s' due to insufficient resources", result.Account, result.Stream) } // Pick a new preferred leader. rg.setPreferred() // Get rid of previous attempt. cc.meta.Propose(encodeDeleteStreamAssignment(sa)) // Propose new. sa.Group, sa.err = rg, nil cc.meta.Propose(encodeAddStreamAssignment(sa)) // When the new stream assignment is processed, sa.reassigning will be // automatically set back to false. Until then, don't process any more // assignment results. sa.reassigning = true return } } } } // Respond to the user here. var resp string if result.Response != nil { resp = s.jsonResponse(result.Response) } else if result.Restore != nil { resp = s.jsonResponse(result.Restore) } if !sa.responded || result.Update { sa.responded = true js.srv.sendAPIErrResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, resp) } // Remove this assignment if possible. if canDelete { sa.err = NewJSClusterNotAssignedError() cc.meta.Propose(encodeDeleteStreamAssignment(sa)) } } } func (js *jetStream) processConsumerAssignmentResults(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { var result consumerAssignmentResult if err := json.Unmarshal(msg, &result); err != nil { // TODO(dlc) - log return } acc, _ := js.srv.LookupAccount(result.Account) if acc == nil { // TODO(dlc) - log return } js.mu.Lock() defer js.mu.Unlock() s, cc := js.srv, js.cluster if cc == nil || cc.meta == nil { return } if sa := js.streamAssignment(result.Account, result.Stream); sa != nil && sa.consumers != nil { if ca := sa.consumers[result.Consumer]; ca != nil && !ca.responded { js.srv.sendAPIErrResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(result.Response)) ca.responded = true // Check if this failed. // TODO(dlc) - Could have mixed results, should track per peer. // Make sure this is recent response, do not delete existing consumers. if result.Response.Error != nil && result.Response.Error != NewJSConsumerNameExistError() && time.Since(ca.Created) < 2*time.Second { // So while we are deleting we will not respond to list/names requests. ca.err = NewJSClusterNotAssignedError() cc.meta.Propose(encodeDeleteConsumerAssignment(ca)) s.Warnf("Proposing to delete consumer `%s > %s > %s' due to assignment response error: %v", result.Account, result.Stream, result.Consumer, result.Response.Error) } } } } const ( streamAssignmentSubj = "$SYS.JSC.STREAM.ASSIGNMENT.RESULT" consumerAssignmentSubj = "$SYS.JSC.CONSUMER.ASSIGNMENT.RESULT" ) // Lock should be held. func (js *jetStream) startUpdatesSub() { cc, s, c := js.cluster, js.srv, js.cluster.c if cc.streamResults == nil { cc.streamResults, _ = s.systemSubscribe(streamAssignmentSubj, _EMPTY_, false, c, js.processStreamAssignmentResults) } if cc.consumerResults == nil { cc.consumerResults, _ = s.systemSubscribe(consumerAssignmentSubj, _EMPTY_, false, c, js.processConsumerAssignmentResults) } if cc.stepdown == nil { cc.stepdown, _ = s.systemSubscribe(JSApiLeaderStepDown, _EMPTY_, false, c, s.jsLeaderStepDownRequest) } if cc.peerRemove == nil { cc.peerRemove, _ = s.systemSubscribe(JSApiRemoveServer, _EMPTY_, false, c, s.jsLeaderServerRemoveRequest) } if cc.peerStreamMove == nil { cc.peerStreamMove, _ = s.systemSubscribe(JSApiServerStreamMove, _EMPTY_, false, c, s.jsLeaderServerStreamMoveRequest) } if cc.peerStreamCancelMove == nil { cc.peerStreamCancelMove, _ = s.systemSubscribe(JSApiServerStreamCancelMove, _EMPTY_, false, c, s.jsLeaderServerStreamCancelMoveRequest) } if js.accountPurge == nil { js.accountPurge, _ = s.systemSubscribe(JSApiAccountPurge, _EMPTY_, false, c, s.jsLeaderAccountPurgeRequest) } } // Lock should be held. func (js *jetStream) stopUpdatesSub() { cc := js.cluster if cc.streamResults != nil { cc.s.sysUnsubscribe(cc.streamResults) cc.streamResults = nil } if cc.consumerResults != nil { cc.s.sysUnsubscribe(cc.consumerResults) cc.consumerResults = nil } if cc.stepdown != nil { cc.s.sysUnsubscribe(cc.stepdown) cc.stepdown = nil } if cc.peerRemove != nil { cc.s.sysUnsubscribe(cc.peerRemove) cc.peerRemove = nil } if cc.peerStreamMove != nil { cc.s.sysUnsubscribe(cc.peerStreamMove) cc.peerStreamMove = nil } if cc.peerStreamCancelMove != nil { cc.s.sysUnsubscribe(cc.peerStreamCancelMove) cc.peerStreamCancelMove = nil } if js.accountPurge != nil { cc.s.sysUnsubscribe(js.accountPurge) js.accountPurge = nil } } func (s *Server) sendDomainLeaderElectAdvisory() { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.RLock() node := cc.meta js.mu.RUnlock() adv := &JSDomainLeaderElectedAdvisory{ TypedEvent: TypedEvent{ Type: JSDomainLeaderElectedAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Leader: node.GroupLeader(), Replicas: s.replicas(node), Cluster: s.cachedClusterName(), Domain: s.getOpts().JetStreamDomain, } s.publishAdvisory(nil, JSAdvisoryDomainLeaderElected, adv) } func (js *jetStream) processLeaderChange(isLeader bool) { if js == nil { return } s := js.srv if s == nil { return } // Update our server atomic. s.isMetaLeader.Store(isLeader) if isLeader { s.Noticef("Self is new JetStream cluster metadata leader") s.sendDomainLeaderElectAdvisory() } else { var node string if meta := js.getMetaGroup(); meta != nil { node = meta.GroupLeader() } if node == _EMPTY_ { s.Noticef("JetStream cluster no metadata leader") } else if srv := js.srv.serverNameForNode(node); srv == _EMPTY_ { s.Noticef("JetStream cluster new remote metadata leader") } else if clst := js.srv.clusterNameForNode(node); clst == _EMPTY_ { s.Noticef("JetStream cluster new metadata leader: %s", srv) } else { s.Noticef("JetStream cluster new metadata leader: %s/%s", srv, clst) } } js.mu.Lock() defer js.mu.Unlock() if isLeader { js.startUpdatesSub() } else { js.stopUpdatesSub() // TODO(dlc) - stepdown. } // If we have been signaled to check the streams, this is for a bug that left stream // assignments with no sync subject after an update and no way to sync/catchup outside of the RAFT layer. if isLeader && js.cluster.streamsCheck { cc := js.cluster for acc, asa := range cc.streams { for _, sa := range asa { if sa.Sync == _EMPTY_ { s.Warnf("Stream assignment corrupt for stream '%s > %s'", acc, sa.Config.Name) nsa := &streamAssignment{Group: sa.Group, Config: sa.Config, Subject: sa.Subject, Reply: sa.Reply, Client: sa.Client} nsa.Sync = syncSubjForStream() cc.meta.Propose(encodeUpdateStreamAssignment(nsa)) } } } // Clear check. cc.streamsCheck = false } } // Lock should be held. func (cc *jetStreamCluster) remapStreamAssignment(sa *streamAssignment, removePeer string) bool { // Invoke placement algo passing RG peers that stay (existing) and the peer that is being removed (ignore) var retain, ignore []string for _, v := range sa.Group.Peers { if v == removePeer { ignore = append(ignore, v) } else { retain = append(retain, v) } } newPeers, placementError := cc.selectPeerGroup(len(sa.Group.Peers), sa.Group.Cluster, sa.Config, retain, 0, ignore) if placementError == nil { sa.Group.Peers = newPeers // Don't influence preferred leader. sa.Group.Preferred = _EMPTY_ return true } // If R1 just return to avoid bricking the stream. if sa.Group.node == nil || len(sa.Group.Peers) == 1 { return false } // If we are here let's remove the peer at least, as long as we are R>1 for i, peer := range sa.Group.Peers { if peer == removePeer { sa.Group.Peers[i] = sa.Group.Peers[len(sa.Group.Peers)-1] sa.Group.Peers = sa.Group.Peers[:len(sa.Group.Peers)-1] break } } return false } type selectPeerError struct { excludeTag bool offline bool noStorage bool uniqueTag bool misc bool noJsClust bool noMatchTags map[string]struct{} } func (e *selectPeerError) Error() string { b := strings.Builder{} writeBoolErrReason := func(hasErr bool, errMsg string) { if !hasErr { return } b.WriteString(", ") b.WriteString(errMsg) } b.WriteString("no suitable peers for placement") writeBoolErrReason(e.offline, "peer offline") writeBoolErrReason(e.excludeTag, "exclude tag set") writeBoolErrReason(e.noStorage, "insufficient storage") writeBoolErrReason(e.uniqueTag, "server tag not unique") writeBoolErrReason(e.misc, "miscellaneous issue") writeBoolErrReason(e.noJsClust, "jetstream not enabled in cluster") if len(e.noMatchTags) != 0 { b.WriteString(", tags not matched [") var firstTagWritten bool for tag := range e.noMatchTags { if firstTagWritten { b.WriteString(", ") } firstTagWritten = true b.WriteRune('\'') b.WriteString(tag) b.WriteRune('\'') } b.WriteString("]") } return b.String() } func (e *selectPeerError) addMissingTag(t string) { if e.noMatchTags == nil { e.noMatchTags = map[string]struct{}{} } e.noMatchTags[t] = struct{}{} } func (e *selectPeerError) accumulate(eAdd *selectPeerError) { if eAdd == nil { return } acc := func(val *bool, valAdd bool) { if valAdd { *val = valAdd } } acc(&e.offline, eAdd.offline) acc(&e.excludeTag, eAdd.excludeTag) acc(&e.noStorage, eAdd.noStorage) acc(&e.uniqueTag, eAdd.uniqueTag) acc(&e.misc, eAdd.misc) acc(&e.noJsClust, eAdd.noJsClust) for tag := range eAdd.noMatchTags { e.addMissingTag(tag) } } // selectPeerGroup will select a group of peers to start a raft group. // when peers exist already the unique tag prefix check for the replaceFirstExisting will be skipped // js lock should be held. func (cc *jetStreamCluster) selectPeerGroup(r int, cluster string, cfg *StreamConfig, existing []string, replaceFirstExisting int, ignore []string) ([]string, *selectPeerError) { if cluster == _EMPTY_ || cfg == nil { return nil, &selectPeerError{misc: true} } var maxBytes uint64 if cfg.MaxBytes > 0 { maxBytes = uint64(cfg.MaxBytes) } // Check for tags. var tags []string if cfg.Placement != nil && len(cfg.Placement.Tags) > 0 { tags = cfg.Placement.Tags } // Used for weighted sorting based on availability. type wn struct { id string avail uint64 ha int ns int } var nodes []wn // peers is a randomized list s, peers := cc.s, cc.meta.Peers() uniqueTagPrefix := s.getOpts().JetStreamUniqueTag if uniqueTagPrefix != _EMPTY_ { for _, tag := range tags { if strings.HasPrefix(tag, uniqueTagPrefix) { // disable uniqueness check if explicitly listed in tags uniqueTagPrefix = _EMPTY_ break } } } var uniqueTags = make(map[string]*nodeInfo) checkUniqueTag := func(ni *nodeInfo) (bool, *nodeInfo) { for _, t := range ni.tags { if strings.HasPrefix(t, uniqueTagPrefix) { if n, ok := uniqueTags[t]; !ok { uniqueTags[t] = ni return true, ni } else { return false, n } } } // default requires the unique prefix to be present return false, nil } // Map existing. var ep map[string]struct{} if le := len(existing); le > 0 { if le >= r { return existing[:r], nil } ep = make(map[string]struct{}) for i, p := range existing { ep[p] = struct{}{} if uniqueTagPrefix == _EMPTY_ { continue } si, ok := s.nodeToInfo.Load(p) if !ok || si == nil || i < replaceFirstExisting { continue } ni := si.(nodeInfo) // collect unique tags, but do not require them as this node is already part of the peerset checkUniqueTag(&ni) } } // Map ignore var ip map[string]struct{} if li := len(ignore); li > 0 { ip = make(map[string]struct{}) for _, p := range ignore { ip[p] = struct{}{} } } // Grab the number of streams and HA assets currently assigned to each peer. // HAAssets under usage is async, so calculate here in realtime based on assignments. peerStreams := make(map[string]int, len(peers)) peerHA := make(map[string]int, len(peers)) for _, asa := range cc.streams { for _, sa := range asa { isHA := len(sa.Group.Peers) > 1 for _, peer := range sa.Group.Peers { peerStreams[peer]++ if isHA { peerHA[peer]++ } } } } maxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets // An error is a result of multiple individual placement decisions. // Which is why we keep taps on how often which one happened. err := selectPeerError{} // Shuffle them up. rand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] }) for _, p := range peers { si, ok := s.nodeToInfo.Load(p.ID) if !ok || si == nil { err.misc = true continue } ni := si.(nodeInfo) // Only select from the designated named cluster. if ni.cluster != cluster { s.Debugf("Peer selection: discard %s@%s reason: not target cluster %s", ni.name, ni.cluster, cluster) continue } // If we know its offline or we do not have config or err don't consider. if ni.offline || ni.cfg == nil || ni.stats == nil { s.Debugf("Peer selection: discard %s@%s reason: offline", ni.name, ni.cluster) err.offline = true continue } // If ignore skip if _, ok := ip[p.ID]; ok { continue } // If existing also skip, we will add back in to front of the list when done. if _, ok := ep[p.ID]; ok { continue } if ni.tags.Contains(jsExcludePlacement) { s.Debugf("Peer selection: discard %s@%s tags: %v reason: %s present", ni.name, ni.cluster, ni.tags, jsExcludePlacement) err.excludeTag = true continue } if len(tags) > 0 { matched := true for _, t := range tags { if !ni.tags.Contains(t) { matched = false s.Debugf("Peer selection: discard %s@%s tags: %v reason: mandatory tag %s not present", ni.name, ni.cluster, ni.tags, t) err.addMissingTag(t) break } } if !matched { continue } } var available uint64 if ni.stats != nil { switch cfg.Storage { case MemoryStorage: used := ni.stats.ReservedMemory if ni.stats.Memory > used { used = ni.stats.Memory } if ni.cfg.MaxMemory > int64(used) { available = uint64(ni.cfg.MaxMemory) - used } case FileStorage: used := ni.stats.ReservedStore if ni.stats.Store > used { used = ni.stats.Store } if ni.cfg.MaxStore > int64(used) { available = uint64(ni.cfg.MaxStore) - used } } } // Otherwise check if we have enough room if maxBytes set. if maxBytes > 0 && maxBytes > available { s.Warnf("Peer selection: discard %s@%s (Max Bytes: %d) exceeds available %s storage of %d bytes", ni.name, ni.cluster, maxBytes, cfg.Storage.String(), available) err.noStorage = true continue } // HAAssets contain _meta_ which we want to ignore, hence > and not >=. if maxHaAssets > 0 && ni.stats != nil && ni.stats.HAAssets > maxHaAssets { s.Warnf("Peer selection: discard %s@%s (HA Asset Count: %d) exceeds max ha asset limit of %d for stream placement", ni.name, ni.cluster, ni.stats.HAAssets, maxHaAssets) err.misc = true continue } if uniqueTagPrefix != _EMPTY_ { if unique, owner := checkUniqueTag(&ni); !unique { if owner != nil { s.Debugf("Peer selection: discard %s@%s tags:%v reason: unique prefix %s owned by %s@%s", ni.name, ni.cluster, ni.tags, owner.name, owner.cluster) } else { s.Debugf("Peer selection: discard %s@%s tags:%v reason: unique prefix %s not present", ni.name, ni.cluster, ni.tags) } err.uniqueTag = true continue } } // Add to our list of potential nodes. nodes = append(nodes, wn{p.ID, available, peerHA[p.ID], peerStreams[p.ID]}) } // If we could not select enough peers, fail. if len(nodes) < (r - len(existing)) { s.Debugf("Peer selection: required %d nodes but found %d (cluster: %s replica: %d existing: %v/%d peers: %d result-peers: %d err: %+v)", (r - len(existing)), len(nodes), cluster, r, existing, replaceFirstExisting, len(peers), len(nodes), err) if len(peers) == 0 { err.noJsClust = true } return nil, &err } // Sort based on available from most to least, breaking ties by number of total streams assigned to the peer. slices.SortFunc(nodes, func(i, j wn) int { if i.avail == j.avail { return cmp.Compare(i.ns, j.ns) } return -cmp.Compare(i.avail, j.avail) // reverse }) // If we are placing a replicated stream, let's sort based on HAAssets, as that is more important to balance. if cfg.Replicas > 1 { slices.SortStableFunc(nodes, func(i, j wn) int { return cmp.Compare(i.ha, j.ha) }) } var results []string if len(existing) > 0 { results = append(results, existing...) r -= len(existing) } for _, r := range nodes[:r] { results = append(results, r.id) } return results, nil } func groupNameForStream(peers []string, storage StorageType) string { return groupName("S", peers, storage) } func groupNameForConsumer(peers []string, storage StorageType) string { return groupName("C", peers, storage) } func groupName(prefix string, peers []string, storage StorageType) string { gns := getHash(nuid.Next()) return fmt.Sprintf("%s-R%d%s-%s", prefix, len(peers), storage.String()[:1], gns) } // returns stream count for this tier as well as applicable reservation size (not including cfg) // jetStream read lock should be held func tieredStreamAndReservationCount(asa map[string]*streamAssignment, tier string, cfg *StreamConfig) (int, int64) { var numStreams int var reservation int64 for _, sa := range asa { // Don't count the stream toward the limit if it already exists. if (tier == _EMPTY_ || isSameTier(sa.Config, cfg)) && sa.Config.Name != cfg.Name { numStreams++ if sa.Config.MaxBytes > 0 && sa.Config.Storage == cfg.Storage { // If tier is empty, all storage is flat and we should adjust for replicas. // Otherwise if tiered, storage replication already taken into consideration. if tier == _EMPTY_ && cfg.Replicas > 1 { reservation += sa.Config.MaxBytes * int64(cfg.Replicas) } else { reservation += sa.Config.MaxBytes } } } } return numStreams, reservation } // createGroupForStream will create a group for assignment for the stream. // Lock should be held. func (js *jetStream) createGroupForStream(ci *ClientInfo, cfg *StreamConfig) (*raftGroup, *selectPeerError) { replicas := cfg.Replicas if replicas == 0 { replicas = 1 } // Default connected cluster from the request origin. cc, cluster := js.cluster, ci.Cluster // If specified, override the default. clusterDefined := cfg.Placement != nil && cfg.Placement.Cluster != _EMPTY_ if clusterDefined { cluster = cfg.Placement.Cluster } clusters := []string{cluster} if !clusterDefined { clusters = append(clusters, ci.Alternates...) } // Need to create a group here. errs := &selectPeerError{} for _, cn := range clusters { peers, err := cc.selectPeerGroup(replicas, cn, cfg, nil, 0, nil) if len(peers) < replicas { errs.accumulate(err) continue } return &raftGroup{Name: groupNameForStream(peers, cfg.Storage), Storage: cfg.Storage, Peers: peers, Cluster: cn}, nil } return nil, errs } func (acc *Account) selectLimits(replicas int) (*JetStreamAccountLimits, string, *jsAccount, *ApiError) { // Grab our jetstream account info. acc.mu.RLock() jsa := acc.js acc.mu.RUnlock() if jsa == nil { return nil, _EMPTY_, nil, NewJSNotEnabledForAccountError() } jsa.usageMu.RLock() selectedLimits, tierName, ok := jsa.selectLimits(replicas) jsa.usageMu.RUnlock() if !ok { return nil, _EMPTY_, nil, NewJSNoLimitsError() } return &selectedLimits, tierName, jsa, nil } // Read lock needs to be held func (js *jetStream) jsClusteredStreamLimitsCheck(acc *Account, cfg *StreamConfig) *ApiError { var replicas int if cfg != nil { replicas = cfg.Replicas } selectedLimits, tier, _, apiErr := acc.selectLimits(replicas) if apiErr != nil { return apiErr } asa := js.cluster.streams[acc.Name] numStreams, reservations := tieredStreamAndReservationCount(asa, tier, cfg) // Check for inflight proposals... if cc := js.cluster; cc != nil && cc.inflight != nil { streams := cc.inflight[acc.Name] numStreams += len(streams) // If inflight contains the same stream, don't count toward exceeding maximum. if cfg != nil { if _, ok := streams[cfg.Name]; ok { numStreams-- } } } if selectedLimits.MaxStreams > 0 && numStreams >= selectedLimits.MaxStreams { return NewJSMaximumStreamsLimitError() } // Check for account limits here before proposing. if err := js.checkAccountLimits(selectedLimits, cfg, reservations); err != nil { return NewJSStreamLimitsError(err, Unless(err)) } return nil } func (s *Server) jsClusteredStreamRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, config *StreamConfig) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } var resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}} ccfg, apiErr := s.checkStreamCfg(config, acc) if apiErr != nil { resp.Error = apiErr s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } cfg := &ccfg // Now process the request and proposal. js.mu.Lock() defer js.mu.Unlock() var self *streamAssignment var rg *raftGroup var syncSubject string // Capture if we have existing assignment first. if osa := js.streamAssignment(acc.Name, cfg.Name); osa != nil { if !reflect.DeepEqual(osa.Config, cfg) { resp.Error = NewJSStreamNameExistError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // This is an equal assignment. self, rg, syncSubject = osa, osa.Group, osa.Sync } if cfg.Sealed { resp.Error = NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration for create can not be sealed")) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Check for subject collisions here. if cc.subjectsOverlap(acc.Name, cfg.Subjects, self) { resp.Error = NewJSStreamSubjectOverlapError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } apiErr = js.jsClusteredStreamLimitsCheck(acc, cfg) // Check for stream limits here before proposing. These need to be tracked from meta layer, not jsa. if apiErr != nil { resp.Error = apiErr s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Make sure inflight is setup properly. if cc.inflight == nil { cc.inflight = make(map[string]map[string]*inflightInfo) } streams, ok := cc.inflight[acc.Name] if !ok { streams = make(map[string]*inflightInfo) cc.inflight[acc.Name] = streams } // Raft group selection and placement. if rg == nil { // Check inflight before proposing in case we have an existing inflight proposal. if existing, ok := streams[cfg.Name]; ok { // We have existing for same stream. Re-use same group and syncSubject. rg, syncSubject = existing.rg, existing.sync } } // Create a new one here if needed. if rg == nil { nrg, err := js.createGroupForStream(ci, cfg) if err != nil { resp.Error = NewJSClusterNoPeersError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } rg = nrg // Pick a preferred leader. rg.setPreferred() } if syncSubject == _EMPTY_ { syncSubject = syncSubjForStream() } // Sync subject for post snapshot sync. sa := &streamAssignment{Group: rg, Sync: syncSubject, Config: cfg, Subject: subject, Reply: reply, Client: ci, Created: time.Now().UTC()} if err := cc.meta.Propose(encodeAddStreamAssignment(sa)); err == nil { // On success, add this as an inflight proposal so we can apply limits // on concurrent create requests while this stream assignment has // possibly not been processed yet. if streams, ok := cc.inflight[acc.Name]; ok && self == nil { streams[cfg.Name] = &inflightInfo{rg, syncSubject} } } } var ( errReqTimeout = errors.New("timeout while waiting for response") errReqSrvExit = errors.New("server shutdown while waiting for response") ) // blocking utility call to perform requests on the system account // returns (synchronized) v or error func sysRequest[T any](s *Server, subjFormat string, args ...any) (*T, error) { isubj := fmt.Sprintf(subjFormat, args...) s.mu.Lock() if s.sys == nil { s.mu.Unlock() return nil, ErrNoSysAccount } inbox := s.newRespInbox() results := make(chan *T, 1) s.sys.replies[inbox] = func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) { var v T if err := json.Unmarshal(msg, &v); err != nil { s.Warnf("Error unmarshalling response for request '%s':%v", isubj, err) return } select { case results <- &v: default: s.Warnf("Failed placing request response on internal channel") } } s.mu.Unlock() s.sendInternalMsgLocked(isubj, inbox, nil, nil) defer func() { s.mu.Lock() defer s.mu.Unlock() if s.sys != nil && s.sys.replies != nil { delete(s.sys.replies, inbox) } }() ttl := time.NewTimer(2 * time.Second) defer ttl.Stop() select { case <-s.quitCh: return nil, errReqSrvExit case <-ttl.C: return nil, errReqTimeout case data := <-results: return data, nil } } func (s *Server) jsClusteredStreamUpdateRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, cfg *StreamConfig, peerSet []string) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } // Now process the request and proposal. js.mu.Lock() defer js.mu.Unlock() meta := cc.meta if meta == nil { return } var resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}} osa := js.streamAssignment(acc.Name, cfg.Name) if osa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } var newCfg *StreamConfig if jsa := js.accounts[acc.Name]; jsa != nil { js.mu.Unlock() ncfg, err := jsa.configUpdateCheck(osa.Config, cfg, s) js.mu.Lock() if err != nil { resp.Error = NewJSStreamUpdateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } else { newCfg = ncfg } } else { resp.Error = NewJSNotEnabledForAccountError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Check for mirror changes which are not allowed. if !reflect.DeepEqual(newCfg.Mirror, osa.Config.Mirror) { resp.Error = NewJSStreamMirrorNotUpdatableError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Check for subject collisions here. if cc.subjectsOverlap(acc.Name, cfg.Subjects, osa) { resp.Error = NewJSStreamSubjectOverlapError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Make copy so to not change original. rg := osa.copyGroup().Group // Check for a move request. var isMoveRequest, isMoveCancel bool if lPeerSet := len(peerSet); lPeerSet > 0 { isMoveRequest = true // check if this is a cancellation if lPeerSet == osa.Config.Replicas && lPeerSet <= len(rg.Peers) { isMoveCancel = true // can only be a cancellation if the peer sets overlap as expected for i := 0; i < lPeerSet; i++ { if peerSet[i] != rg.Peers[i] { isMoveCancel = false break } } } } else { isMoveRequest = newCfg.Placement != nil && !reflect.DeepEqual(osa.Config.Placement, newCfg.Placement) } // Check for replica changes. isReplicaChange := newCfg.Replicas != osa.Config.Replicas // We stage consumer updates and do them after the stream update. var consumers []*consumerAssignment // Check if this is a move request, but no cancellation, and we are already moving this stream. if isMoveRequest && !isMoveCancel && osa.Config.Replicas != len(rg.Peers) { // obtain stats to include in error message msg := _EMPTY_ if s.allPeersOffline(rg) { msg = fmt.Sprintf("all %d peers offline", len(rg.Peers)) } else { // Need to release js lock. js.mu.Unlock() if si, err := sysRequest[StreamInfo](s, clusterStreamInfoT, ci.serviceAccount(), cfg.Name); err != nil { msg = fmt.Sprintf("error retrieving info: %s", err.Error()) } else if si != nil { currentCount := 0 if si.Cluster.Leader != _EMPTY_ { currentCount++ } combinedLag := uint64(0) for _, r := range si.Cluster.Replicas { if r.Current { currentCount++ } combinedLag += r.Lag } msg = fmt.Sprintf("total peers: %d, current peers: %d, combined lag: %d", len(rg.Peers), currentCount, combinedLag) } // Re-acquire here. js.mu.Lock() } resp.Error = NewJSStreamMoveInProgressError(msg) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Can not move and scale at same time. if isMoveRequest && isReplicaChange { resp.Error = NewJSStreamMoveAndScaleError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } if isReplicaChange { isScaleUp := newCfg.Replicas > len(rg.Peers) // We are adding new peers here. if isScaleUp { // Check that we have the allocation available. if err := js.jsClusteredStreamLimitsCheck(acc, newCfg); err != nil { resp.Error = err s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Check if we do not have a cluster assigned, and if we do not make sure we // try to pick one. This could happen with older streams that were assigned by // previous servers. if rg.Cluster == _EMPTY_ { // Prefer placement directrives if we have them. if newCfg.Placement != nil && newCfg.Placement.Cluster != _EMPTY_ { rg.Cluster = newCfg.Placement.Cluster } else { // Fall back to the cluster assignment from the client. rg.Cluster = ci.Cluster } } peers, err := cc.selectPeerGroup(newCfg.Replicas, rg.Cluster, newCfg, rg.Peers, 0, nil) if err != nil { resp.Error = NewJSClusterNoPeersError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Single nodes are not recorded by the NRG layer so we can rename. if len(peers) == 1 { rg.Name = groupNameForStream(peers, rg.Storage) } else if len(rg.Peers) == 1 { // This is scale up from being a singelton, set preferred to that singelton. rg.Preferred = rg.Peers[0] } rg.Peers = peers } else { // We are deleting nodes here. We want to do our best to preserve the current leader. // We have support now from above that guarantees we are in our own Go routine, so can // ask for stream info from the stream leader to make sure we keep the leader in the new list. var curLeader string if !s.allPeersOffline(rg) { // Need to release js lock. js.mu.Unlock() if si, err := sysRequest[StreamInfo](s, clusterStreamInfoT, ci.serviceAccount(), cfg.Name); err != nil { s.Warnf("Did not receive stream info results for '%s > %s' due to: %s", acc, cfg.Name, err) } else if si != nil { if cl := si.Cluster; cl != nil && cl.Leader != _EMPTY_ { curLeader = getHash(cl.Leader) } } // Re-acquire here. js.mu.Lock() } // If we identified a leader make sure its part of the new group. selected := make([]string, 0, newCfg.Replicas) if curLeader != _EMPTY_ { selected = append(selected, curLeader) } for _, peer := range rg.Peers { if len(selected) == newCfg.Replicas { break } if peer == curLeader { continue } if si, ok := s.nodeToInfo.Load(peer); ok && si != nil { if si.(nodeInfo).offline { continue } selected = append(selected, peer) } } rg.Peers = selected } // Need to remap any consumers. for _, ca := range osa.consumers { // Legacy ephemerals are R=1 but present as R=0, so only auto-remap named consumers, or if we are downsizing the consumer peers. // If stream is interest or workqueue policy always remaps since they require peer parity with stream. numPeers := len(ca.Group.Peers) isAutoScale := ca.Config.Replicas == 0 && (ca.Config.Durable != _EMPTY_ || ca.Config.Name != _EMPTY_) if isAutoScale || numPeers > len(rg.Peers) || cfg.Retention != LimitsPolicy { cca := ca.copyGroup() // Adjust preferred as needed. if numPeers == 1 && isScaleUp { cca.Group.Preferred = ca.Group.Peers[0] } else { cca.Group.Preferred = _EMPTY_ } // Assign new peers. cca.Group.Peers = rg.Peers // If the replicas was not 0 make sure it matches here. if cca.Config.Replicas != 0 { cca.Config.Replicas = len(rg.Peers) } // We can not propose here before the stream itself so we collect them. consumers = append(consumers, cca) } else if !isScaleUp { // We decided to leave this consumer's peer group alone but we are also scaling down. // We need to make sure we do not have any peers that are no longer part of the stream. // Note we handle down scaling of a consumer above if its number of peers were > new stream peers. var needReplace []string for _, rp := range ca.Group.Peers { // Check if we have an orphaned peer now for this consumer. if !rg.isMember(rp) { needReplace = append(needReplace, rp) } } if len(needReplace) > 0 { newPeers := copyStrings(rg.Peers) rand.Shuffle(len(newPeers), func(i, j int) { newPeers[i], newPeers[j] = newPeers[j], newPeers[i] }) // If we had a small size then the peer set, restrict to the same number. if lp := len(ca.Group.Peers); lp < len(newPeers) { newPeers = newPeers[:lp] } cca := ca.copyGroup() // Assign new peers. cca.Group.Peers = newPeers // If the replicas was not 0 make sure it matches here. if cca.Config.Replicas != 0 { cca.Config.Replicas = len(newPeers) } // Check if all peers are invalid. This can happen with R1 under replicated streams that are being scaled down. if len(needReplace) == len(ca.Group.Peers) { // We have to transfer state to new peers. // we will grab our state and attach to the new assignment. // TODO(dlc) - In practice we would want to make sure the consumer is paused. // Need to release js lock. js.mu.Unlock() if ci, err := sysRequest[ConsumerInfo](s, clusterConsumerInfoT, acc, osa.Config.Name, ca.Name); err != nil { s.Warnf("Did not receive consumer info results for '%s > %s > %s' due to: %s", acc, osa.Config.Name, ca.Name, err) } else if ci != nil { cca.State = &ConsumerState{ Delivered: SequencePair{ Consumer: ci.Delivered.Consumer, Stream: ci.Delivered.Stream, }, AckFloor: SequencePair{ Consumer: ci.AckFloor.Consumer, Stream: ci.AckFloor.Stream, }, } } // Re-acquire here. js.mu.Lock() } // We can not propose here before the stream itself so we collect them. consumers = append(consumers, cca) } } } } else if isMoveRequest { if len(peerSet) == 0 { nrg, err := js.createGroupForStream(ci, newCfg) if err != nil { resp.Error = NewJSClusterNoPeersError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // filter peers present in both sets for _, peer := range rg.Peers { found := false for _, newPeer := range nrg.Peers { if peer == newPeer { found = true break } } if !found { peerSet = append(peerSet, peer) } } peerSet = append(peerSet, nrg.Peers...) } if len(rg.Peers) == 1 { rg.Preferred = peerSet[0] } rg.Peers = peerSet for _, ca := range osa.consumers { cca := ca.copyGroup() r := cca.Config.replicas(osa.Config) // shuffle part of cluster peer set we will be keeping randPeerSet := copyStrings(peerSet[len(peerSet)-newCfg.Replicas:]) rand.Shuffle(newCfg.Replicas, func(i, j int) { randPeerSet[i], randPeerSet[j] = randPeerSet[j], randPeerSet[i] }) // move overlapping peers at the end of randPeerSet and keep a tally of non overlapping peers dropPeerSet := make([]string, 0, len(cca.Group.Peers)) for _, p := range cca.Group.Peers { found := false for i, rp := range randPeerSet { if p == rp { randPeerSet[i] = randPeerSet[newCfg.Replicas-1] randPeerSet[newCfg.Replicas-1] = p found = true break } } if !found { dropPeerSet = append(dropPeerSet, p) } } cPeerSet := randPeerSet[newCfg.Replicas-r:] // In case of a set or cancel simply assign if len(peerSet) == newCfg.Replicas { cca.Group.Peers = cPeerSet } else { cca.Group.Peers = append(dropPeerSet, cPeerSet...) } // make sure it overlaps with peers and remove if not if cca.Group.Preferred != _EMPTY_ { found := false for _, p := range cca.Group.Peers { if p == cca.Group.Preferred { found = true break } } if !found { cca.Group.Preferred = _EMPTY_ } } // We can not propose here before the stream itself so we collect them. consumers = append(consumers, cca) } } else { // All other updates make sure no preferred is set. rg.Preferred = _EMPTY_ } sa := &streamAssignment{Group: rg, Sync: osa.Sync, Created: osa.Created, Config: newCfg, Subject: subject, Reply: reply, Client: ci} meta.Propose(encodeUpdateStreamAssignment(sa)) // Process any staged consumers. for _, ca := range consumers { meta.Propose(encodeAddConsumerAssignment(ca)) } } func (s *Server) jsClusteredStreamDeleteRequest(ci *ClientInfo, acc *Account, stream, subject, reply string, rmsg []byte) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.Lock() defer js.mu.Unlock() if cc.meta == nil { return } osa := js.streamAssignment(acc.Name, stream) if osa == nil { var resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}} resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } sa := &streamAssignment{Group: osa.Group, Config: osa.Config, Subject: subject, Reply: reply, Client: ci} cc.meta.Propose(encodeDeleteStreamAssignment(sa)) } // Process a clustered purge request. func (s *Server) jsClusteredStreamPurgeRequest( ci *ClientInfo, acc *Account, mset *stream, stream, subject, reply string, rmsg []byte, preq *JSApiStreamPurgeRequest, ) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.Lock() sa := js.streamAssignment(acc.Name, stream) if sa == nil { resp := JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}} resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) js.mu.Unlock() return } if n := sa.Group.node; n != nil { sp := &streamPurge{Stream: stream, LastSeq: mset.state().LastSeq, Subject: subject, Reply: reply, Client: ci, Request: preq} n.Propose(encodeStreamPurge(sp)) js.mu.Unlock() return } js.mu.Unlock() if mset == nil { return } var resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}} purged, err := mset.purge(preq) if err != nil { resp.Error = NewJSStreamGeneralError(err, Unless(err)) } else { resp.Purged = purged resp.Success = true } s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) } func (s *Server) jsClusteredStreamRestoreRequest( ci *ClientInfo, acc *Account, req *JSApiStreamRestoreRequest, subject, reply string, rmsg []byte) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.Lock() defer js.mu.Unlock() if cc.meta == nil { return } cfg := &req.Config resp := JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}} if err := js.jsClusteredStreamLimitsCheck(acc, cfg); err != nil { resp.Error = err s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } if sa := js.streamAssignment(ci.serviceAccount(), cfg.Name); sa != nil { resp.Error = NewJSStreamNameExistRestoreFailedError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Raft group selection and placement. rg, err := js.createGroupForStream(ci, cfg) if err != nil { resp.Error = NewJSClusterNoPeersError(err) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Pick a preferred leader. rg.setPreferred() sa := &streamAssignment{Group: rg, Sync: syncSubjForStream(), Config: cfg, Subject: subject, Reply: reply, Client: ci, Created: time.Now().UTC()} // Now add in our restore state and pre-select a peer to handle the actual receipt of the snapshot. sa.Restore = &req.State cc.meta.Propose(encodeAddStreamAssignment(sa)) } // Determine if all peers for this group are offline. func (s *Server) allPeersOffline(rg *raftGroup) bool { if rg == nil { return false } // Check to see if this stream has any servers online to respond. for _, peer := range rg.Peers { if si, ok := s.nodeToInfo.Load(peer); ok && si != nil { if !si.(nodeInfo).offline { return false } } } return true } // This will do a scatter and gather operation for all streams for this account. This is only called from metadata leader. // This will be running in a separate Go routine. func (s *Server) jsClusteredStreamListRequest(acc *Account, ci *ClientInfo, filter string, offset int, subject, reply string, rmsg []byte) { defer s.grWG.Done() js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.RLock() var streams []*streamAssignment for _, sa := range cc.streams[acc.Name] { if IsNatsErr(sa.err, JSClusterNotAssignedErr) { continue } if filter != _EMPTY_ { // These could not have subjects auto-filled in since they are raw and unprocessed. if len(sa.Config.Subjects) == 0 { if SubjectsCollide(filter, sa.Config.Name) { streams = append(streams, sa) } } else { for _, subj := range sa.Config.Subjects { if SubjectsCollide(filter, subj) { streams = append(streams, sa) break } } } } else { streams = append(streams, sa) } } // Needs to be sorted for offsets etc. if len(streams) > 1 { slices.SortFunc(streams, func(i, j *streamAssignment) int { return cmp.Compare(i.Config.Name, j.Config.Name) }) } scnt := len(streams) if offset > scnt { offset = scnt } if offset > 0 { streams = streams[offset:] } if len(streams) > JSApiListLimit { streams = streams[:JSApiListLimit] } var resp = JSApiStreamListResponse{ ApiResponse: ApiResponse{Type: JSApiStreamListResponseType}, Streams: make([]*StreamInfo, 0, len(streams)), } js.mu.RUnlock() if len(streams) == 0 { resp.Limit = JSApiListLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) return } // Create an inbox for our responses and send out our requests. s.mu.Lock() inbox := s.newRespInbox() rc := make(chan *StreamInfo, len(streams)) // Store our handler. s.sys.replies[inbox] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { var si StreamInfo if err := json.Unmarshal(msg, &si); err != nil { s.Warnf("Error unmarshalling clustered stream info response:%v", err) return } select { case rc <- &si: default: s.Warnf("Failed placing remote stream info result on internal channel") } } s.mu.Unlock() // Cleanup after. defer func() { s.mu.Lock() if s.sys != nil && s.sys.replies != nil { delete(s.sys.replies, inbox) } s.mu.Unlock() }() var missingNames []string sent := map[string]int{} // Send out our requests here. js.mu.RLock() for _, sa := range streams { if s.allPeersOffline(sa.Group) { // Place offline onto our results by hand here. si := &StreamInfo{ Config: *sa.Config, Created: sa.Created, Cluster: js.offlineClusterInfo(sa.Group), TimeStamp: time.Now().UTC(), } resp.Streams = append(resp.Streams, si) missingNames = append(missingNames, sa.Config.Name) } else { isubj := fmt.Sprintf(clusterStreamInfoT, sa.Client.serviceAccount(), sa.Config.Name) s.sendInternalMsgLocked(isubj, inbox, nil, nil) sent[sa.Config.Name] = len(sa.consumers) } } // Don't hold lock. js.mu.RUnlock() const timeout = 4 * time.Second notActive := time.NewTimer(timeout) defer notActive.Stop() LOOP: for len(sent) > 0 { select { case <-s.quitCh: return case <-notActive.C: s.Warnf("Did not receive all stream info results for %q", acc) for sName := range sent { missingNames = append(missingNames, sName) } break LOOP case si := <-rc: consCount := sent[si.Config.Name] if consCount > 0 { si.State.Consumers = consCount } delete(sent, si.Config.Name) resp.Streams = append(resp.Streams, si) // Check to see if we are done. if len(resp.Streams) == len(streams) { break LOOP } } } // Needs to be sorted as well. if len(resp.Streams) > 1 { slices.SortFunc(resp.Streams, func(i, j *StreamInfo) int { return cmp.Compare(i.Config.Name, j.Config.Name) }) } resp.Total = scnt resp.Limit = JSApiListLimit resp.Offset = offset resp.Missing = missingNames s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) } // This will do a scatter and gather operation for all consumers for this stream and account. // This will be running in a separate Go routine. func (s *Server) jsClusteredConsumerListRequest(acc *Account, ci *ClientInfo, offset int, stream, subject, reply string, rmsg []byte) { defer s.grWG.Done() js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.RLock() var consumers []*consumerAssignment if sas := cc.streams[acc.Name]; sas != nil { if sa := sas[stream]; sa != nil { // Copy over since we need to sort etc. for _, ca := range sa.consumers { consumers = append(consumers, ca) } } } // Needs to be sorted. if len(consumers) > 1 { slices.SortFunc(consumers, func(i, j *consumerAssignment) int { return cmp.Compare(i.Config.Name, j.Config.Name) }) } ocnt := len(consumers) if offset > ocnt { offset = ocnt } if offset > 0 { consumers = consumers[offset:] } if len(consumers) > JSApiListLimit { consumers = consumers[:JSApiListLimit] } // Send out our requests here. var resp = JSApiConsumerListResponse{ ApiResponse: ApiResponse{Type: JSApiConsumerListResponseType}, Consumers: []*ConsumerInfo{}, } js.mu.RUnlock() if len(consumers) == 0 { resp.Limit = JSApiListLimit resp.Offset = offset s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) return } // Create an inbox for our responses and send out requests. s.mu.Lock() inbox := s.newRespInbox() rc := make(chan *ConsumerInfo, len(consumers)) // Store our handler. s.sys.replies[inbox] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) { var ci ConsumerInfo if err := json.Unmarshal(msg, &ci); err != nil { s.Warnf("Error unmarshaling clustered consumer info response:%v", err) return } select { case rc <- &ci: default: s.Warnf("Failed placing consumer info result on internal chan") } } s.mu.Unlock() // Cleanup after. defer func() { s.mu.Lock() if s.sys != nil && s.sys.replies != nil { delete(s.sys.replies, inbox) } s.mu.Unlock() }() var missingNames []string sent := map[string]struct{}{} // Send out our requests here. js.mu.RLock() for _, ca := range consumers { if s.allPeersOffline(ca.Group) { // Place offline onto our results by hand here. ci := &ConsumerInfo{ Config: ca.Config, Created: ca.Created, Cluster: js.offlineClusterInfo(ca.Group), TimeStamp: time.Now().UTC(), } resp.Consumers = append(resp.Consumers, ci) missingNames = append(missingNames, ca.Name) } else { isubj := fmt.Sprintf(clusterConsumerInfoT, ca.Client.serviceAccount(), stream, ca.Name) s.sendInternalMsgLocked(isubj, inbox, nil, nil) sent[ca.Name] = struct{}{} } } // Don't hold lock. js.mu.RUnlock() const timeout = 4 * time.Second notActive := time.NewTimer(timeout) defer notActive.Stop() LOOP: for len(sent) > 0 { select { case <-s.quitCh: return case <-notActive.C: s.Warnf("Did not receive all consumer info results for '%s > %s'", acc, stream) for cName := range sent { missingNames = append(missingNames, cName) } break LOOP case ci := <-rc: delete(sent, ci.Name) resp.Consumers = append(resp.Consumers, ci) // Check to see if we are done. if len(resp.Consumers) == len(consumers) { break LOOP } } } // Needs to be sorted as well. if len(resp.Consumers) > 1 { slices.SortFunc(resp.Consumers, func(i, j *ConsumerInfo) int { return cmp.Compare(i.Name, j.Name) }) } resp.Total = ocnt resp.Limit = JSApiListLimit resp.Offset = offset resp.Missing = missingNames s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) } func encodeStreamPurge(sp *streamPurge) []byte { var bb bytes.Buffer bb.WriteByte(byte(purgeStreamOp)) json.NewEncoder(&bb).Encode(sp) return bb.Bytes() } func decodeStreamPurge(buf []byte) (*streamPurge, error) { var sp streamPurge err := json.Unmarshal(buf, &sp) return &sp, err } func (s *Server) jsClusteredConsumerDeleteRequest(ci *ClientInfo, acc *Account, stream, consumer, subject, reply string, rmsg []byte) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.Lock() defer js.mu.Unlock() if cc.meta == nil { return } var resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}} sa := js.streamAssignment(acc.Name, stream) if sa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } if sa.consumers == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } oca := sa.consumers[consumer] if oca == nil { resp.Error = NewJSConsumerNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } oca.deleted = true ca := &consumerAssignment{Group: oca.Group, Stream: stream, Name: consumer, Config: oca.Config, Subject: subject, Reply: reply, Client: ci} cc.meta.Propose(encodeDeleteConsumerAssignment(ca)) } func encodeMsgDelete(md *streamMsgDelete) []byte { var bb bytes.Buffer bb.WriteByte(byte(deleteMsgOp)) json.NewEncoder(&bb).Encode(md) return bb.Bytes() } func decodeMsgDelete(buf []byte) (*streamMsgDelete, error) { var md streamMsgDelete err := json.Unmarshal(buf, &md) return &md, err } func (s *Server) jsClusteredMsgDeleteRequest(ci *ClientInfo, acc *Account, mset *stream, stream, subject, reply string, req *JSApiMsgDeleteRequest, rmsg []byte) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } js.mu.Lock() sa := js.streamAssignment(acc.Name, stream) if sa == nil { s.Debugf("Message delete failed, could not locate stream '%s > %s'", acc.Name, stream) js.mu.Unlock() return } // Check for single replica items. if n := sa.Group.node; n != nil { md := streamMsgDelete{Seq: req.Seq, NoErase: req.NoErase, Stream: stream, Subject: subject, Reply: reply, Client: ci} n.Propose(encodeMsgDelete(&md)) js.mu.Unlock() return } js.mu.Unlock() if mset == nil { return } var err error var removed bool if req.NoErase { removed, err = mset.removeMsg(req.Seq) } else { removed, err = mset.eraseMsg(req.Seq) } var resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}} if err != nil { resp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err)) } else if !removed { resp.Error = NewJSSequenceNotFoundError(req.Seq) } else { resp.Success = true } s.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp)) } func encodeAddStreamAssignment(sa *streamAssignment) []byte { csa := *sa csa.Client = csa.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(assignStreamOp)) json.NewEncoder(&bb).Encode(csa) return bb.Bytes() } func encodeUpdateStreamAssignment(sa *streamAssignment) []byte { csa := *sa csa.Client = csa.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(updateStreamOp)) json.NewEncoder(&bb).Encode(csa) return bb.Bytes() } func encodeDeleteStreamAssignment(sa *streamAssignment) []byte { csa := *sa csa.Client = csa.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(removeStreamOp)) json.NewEncoder(&bb).Encode(csa) return bb.Bytes() } func decodeStreamAssignment(buf []byte) (*streamAssignment, error) { var sa streamAssignment err := json.Unmarshal(buf, &sa) if err != nil { return nil, err } fixCfgMirrorWithDedupWindow(sa.Config) return &sa, err } func encodeDeleteRange(dr *DeleteRange) []byte { var bb bytes.Buffer bb.WriteByte(byte(deleteRangeOp)) json.NewEncoder(&bb).Encode(dr) return bb.Bytes() } func decodeDeleteRange(buf []byte) (*DeleteRange, error) { var dr DeleteRange err := json.Unmarshal(buf, &dr) if err != nil { return nil, err } return &dr, err } // createGroupForConsumer will create a new group from same peer set as the stream. func (cc *jetStreamCluster) createGroupForConsumer(cfg *ConsumerConfig, sa *streamAssignment) *raftGroup { if len(sa.Group.Peers) == 0 || cfg.Replicas > len(sa.Group.Peers) { return nil } peers := copyStrings(sa.Group.Peers) var _ss [5]string active := _ss[:0] // Calculate all active peers. for _, peer := range peers { if sir, ok := cc.s.nodeToInfo.Load(peer); ok && sir != nil { if !sir.(nodeInfo).offline { active = append(active, peer) } } } if quorum := cfg.Replicas/2 + 1; quorum > len(active) { // Not enough active to satisfy the request. return nil } // If we want less then our parent stream, select from active. if cfg.Replicas > 0 && cfg.Replicas < len(peers) { // Pedantic in case stream is say R5 and consumer is R3 and 3 or more offline, etc. if len(active) < cfg.Replicas { return nil } // First shuffle the active peers and then select to account for replica = 1. rand.Shuffle(len(active), func(i, j int) { active[i], active[j] = active[j], active[i] }) peers = active[:cfg.Replicas] } storage := sa.Config.Storage if cfg.MemoryStorage { storage = MemoryStorage } return &raftGroup{Name: groupNameForConsumer(peers, storage), Storage: storage, Peers: peers} } // jsClusteredConsumerRequest is first point of entry to create a consumer in clustered mode. func (s *Server) jsClusteredConsumerRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, stream string, cfg *ConsumerConfig, action ConsumerAction) { js, cc := s.getJetStreamCluster() if js == nil || cc == nil { return } var resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}} streamCfg, ok := js.clusterStreamConfig(acc.Name, stream) if !ok { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } selectedLimits, _, _, apiErr := acc.selectLimits(cfg.replicas(&streamCfg)) if apiErr != nil { resp.Error = apiErr s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } srvLim := &s.getOpts().JetStreamLimits // Make sure we have sane defaults setConsumerConfigDefaults(cfg, &streamCfg, srvLim, selectedLimits) if err := checkConsumerCfg(cfg, srvLim, &streamCfg, acc, selectedLimits, false); err != nil { resp.Error = err s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } js.mu.Lock() defer js.mu.Unlock() if cc.meta == nil { return } // Lookup the stream assignment. sa := js.streamAssignment(acc.Name, stream) if sa == nil { resp.Error = NewJSStreamNotFoundError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Was a consumer name provided? var oname string if isDurableConsumer(cfg) || cfg.Name != _EMPTY_ { if cfg.Name != _EMPTY_ { oname = cfg.Name } else { oname = cfg.Durable } } // Check for max consumers here to short circuit if possible. // Start with limit on a stream, but if one is defined at the level of the account // and is lower, use that limit. if action == ActionCreate || action == ActionCreateOrUpdate { maxc := sa.Config.MaxConsumers if maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) { maxc = selectedLimits.MaxConsumers } if maxc > 0 { // Don't count DIRECTS. total := 0 for cn, ca := range sa.consumers { // If the consumer name is specified and we think it already exists, then // we're likely updating an existing consumer, so don't count it. Otherwise // we will incorrectly return NewJSMaximumConsumersLimitError for an update. if oname != _EMPTY_ && cn == oname && sa.consumers[oname] != nil { continue } if ca.Config != nil && !ca.Config.Direct { total++ } } if total >= maxc { resp.Error = NewJSMaximumConsumersLimitError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } } } // Also short circuit if DeliverLastPerSubject is set with no FilterSubject. if cfg.DeliverPolicy == DeliverLastPerSubject { if cfg.FilterSubject == _EMPTY_ && len(cfg.FilterSubjects) == 0 { resp.Error = NewJSConsumerInvalidPolicyError(fmt.Errorf("consumer delivery policy is deliver last per subject, but FilterSubject is not set")) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } } // Setup proper default for ack wait if we are in explicit ack mode. if cfg.AckWait == 0 && (cfg.AckPolicy == AckExplicit || cfg.AckPolicy == AckAll) { cfg.AckWait = JsAckWaitDefault } // Setup default of -1, meaning no limit for MaxDeliver. if cfg.MaxDeliver == 0 { cfg.MaxDeliver = -1 } // Set proper default for max ack pending if we are ack explicit and none has been set. if cfg.AckPolicy == AckExplicit && cfg.MaxAckPending == 0 { cfg.MaxAckPending = JsDefaultMaxAckPending } var ca *consumerAssignment // See if we have an existing one already under same durable name or // if name was set by the user. if oname != _EMPTY_ { if ca = sa.consumers[oname]; ca != nil && !ca.deleted { if action == ActionCreate && !reflect.DeepEqual(cfg, ca.Config) { resp.Error = NewJSConsumerAlreadyExistsError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Do quick sanity check on new cfg to prevent here if possible. if err := acc.checkNewConsumerConfig(ca.Config, cfg); err != nil { resp.Error = NewJSConsumerCreateError(err, Unless(err)) s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } } } // If this is new consumer. if ca == nil { if action == ActionUpdate { resp.Error = NewJSConsumerDoesNotExistError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } rg := cc.createGroupForConsumer(cfg, sa) if rg == nil { resp.Error = NewJSInsufficientResourcesError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Pick a preferred leader. rg.setPreferred() // Inherit cluster from stream. rg.Cluster = sa.Group.Cluster // We need to set the ephemeral here before replicating. if !isDurableConsumer(cfg) { // We chose to have ephemerals be R=1 unless stream is interest or workqueue. // Consumer can override. if sa.Config.Retention == LimitsPolicy && cfg.Replicas <= 1 { rg.Peers = []string{rg.Preferred} rg.Name = groupNameForConsumer(rg.Peers, rg.Storage) } if cfg.Name != _EMPTY_ { oname = cfg.Name } else { // Make sure name is unique. for { oname = createConsumerName() if sa.consumers != nil { if sa.consumers[oname] != nil { continue } } break } } } if len(rg.Peers) > 1 { if maxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets; maxHaAssets != 0 { for _, peer := range rg.Peers { if ni, ok := s.nodeToInfo.Load(peer); ok { ni := ni.(nodeInfo) if stats := ni.stats; stats != nil && stats.HAAssets > maxHaAssets { resp.Error = NewJSInsufficientResourcesError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) s.Warnf("%s@%s (HA Asset Count: %d) exceeds max ha asset limit of %d"+ " for (durable) consumer %s placement on stream %s", ni.name, ni.cluster, ni.stats.HAAssets, maxHaAssets, oname, stream) return } } } } } // Check if we are work queue policy. // We will do pre-checks here to avoid thrashing meta layer. if sa.Config.Retention == WorkQueuePolicy && !cfg.Direct { if cfg.AckPolicy != AckExplicit { resp.Error = NewJSConsumerWQRequiresExplicitAckError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } subjects := gatherSubjectFilters(cfg.FilterSubject, cfg.FilterSubjects) if len(subjects) == 0 && len(sa.consumers) > 0 { resp.Error = NewJSConsumerWQMultipleUnfilteredError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } // Check here to make sure we have not collided with another. if len(sa.consumers) > 0 { for _, oca := range sa.consumers { if oca.Name == oname { continue } for _, psubj := range gatherSubjectFilters(oca.Config.FilterSubject, oca.Config.FilterSubjects) { for _, subj := range subjects { if SubjectsCollide(subj, psubj) { resp.Error = NewJSConsumerWQConsumerNotUniqueError() s.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp)) return } } } } } } ca = &consumerAssignment{ Group: rg, Stream: stream, Name: oname, Config: cfg, Subject: subject, Reply: reply, Client: ci, Created: time.Now().UTC(), } } else { nca := ca.copyGroup() rBefore := nca.Config.replicas(sa.Config) rAfter := cfg.replicas(sa.Config) var curLeader string if rBefore != rAfter { // We are modifying nodes here. We want to do our best to preserve the current leader. // We have support now from above that guarantees we are in our own Go routine, so can // ask for stream info from the stream leader to make sure we keep the leader in the new list. if !s.allPeersOffline(ca.Group) { // Need to release js lock. js.mu.Unlock() if ci, err := sysRequest[ConsumerInfo](s, clusterConsumerInfoT, ci.serviceAccount(), sa.Config.Name, cfg.Durable); err != nil { s.Warnf("Did not receive consumer info results for '%s > %s > %s' due to: %s", acc, sa.Config.Name, cfg.Durable, err) } else if ci != nil { if cl := ci.Cluster; cl != nil { curLeader = getHash(cl.Leader) } } // Re-acquire here. js.mu.Lock() } } if rBefore < rAfter { newPeerSet := nca.Group.Peers // scale up by adding new members from the stream peer set that are not yet in the consumer peer set streamPeerSet := copyStrings(sa.Group.Peers) rand.Shuffle(rAfter, func(i, j int) { streamPeerSet[i], streamPeerSet[j] = streamPeerSet[j], streamPeerSet[i] }) for _, p := range streamPeerSet { found := false for _, sp := range newPeerSet { if sp == p { found = true break } } if !found { newPeerSet = append(newPeerSet, p) if len(newPeerSet) == rAfter { break } } } nca.Group.Peers = newPeerSet nca.Group.Preferred = curLeader } else if rBefore > rAfter { newPeerSet := nca.Group.Peers // mark leader preferred and move it to end nca.Group.Preferred = curLeader if nca.Group.Preferred != _EMPTY_ { for i, p := range newPeerSet { if nca.Group.Preferred == p { newPeerSet[i] = newPeerSet[len(newPeerSet)-1] newPeerSet[len(newPeerSet)-1] = p } } } // scale down by removing peers from the end newPeerSet = newPeerSet[len(newPeerSet)-rAfter:] nca.Group.Peers = newPeerSet } // Update config and client info on copy of existing. nca.Config = cfg nca.Client = ci nca.Subject = subject nca.Reply = reply ca = nca } // Do formal proposal. if err := cc.meta.Propose(encodeAddConsumerAssignment(ca)); err == nil { // Mark this as pending. if sa.consumers == nil { sa.consumers = make(map[string]*consumerAssignment) } ca.pending = true sa.consumers[ca.Name] = ca } } func encodeAddConsumerAssignment(ca *consumerAssignment) []byte { cca := *ca cca.Client = cca.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(assignConsumerOp)) json.NewEncoder(&bb).Encode(cca) return bb.Bytes() } func encodeDeleteConsumerAssignment(ca *consumerAssignment) []byte { cca := *ca cca.Client = cca.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(removeConsumerOp)) json.NewEncoder(&bb).Encode(cca) return bb.Bytes() } func decodeConsumerAssignment(buf []byte) (*consumerAssignment, error) { var ca consumerAssignment err := json.Unmarshal(buf, &ca) return &ca, err } func encodeAddConsumerAssignmentCompressed(ca *consumerAssignment) []byte { cca := *ca cca.Client = cca.Client.forProposal() var bb bytes.Buffer bb.WriteByte(byte(assignCompressedConsumerOp)) s2e := s2.NewWriter(&bb) json.NewEncoder(s2e).Encode(cca) s2e.Close() return bb.Bytes() } func decodeConsumerAssignmentCompressed(buf []byte) (*consumerAssignment, error) { var ca consumerAssignment bb := bytes.NewBuffer(buf) s2d := s2.NewReader(bb) return &ca, json.NewDecoder(s2d).Decode(&ca) } var errBadStreamMsg = errors.New("jetstream cluster bad replicated stream msg") func decodeStreamMsg(buf []byte) (subject, reply string, hdr, msg []byte, lseq uint64, ts int64, err error) { var le = binary.LittleEndian if len(buf) < 26 { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } lseq = le.Uint64(buf) buf = buf[8:] ts = int64(le.Uint64(buf)) buf = buf[8:] sl := int(le.Uint16(buf)) buf = buf[2:] if len(buf) < sl { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } subject = string(buf[:sl]) buf = buf[sl:] if len(buf) < 2 { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } rl := int(le.Uint16(buf)) buf = buf[2:] if len(buf) < rl { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } reply = string(buf[:rl]) buf = buf[rl:] if len(buf) < 2 { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } hl := int(le.Uint16(buf)) buf = buf[2:] if len(buf) < hl { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } if hdr = buf[:hl]; len(hdr) == 0 { hdr = nil } buf = buf[hl:] if len(buf) < 4 { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } ml := int(le.Uint32(buf)) buf = buf[4:] if len(buf) < ml { return _EMPTY_, _EMPTY_, nil, nil, 0, 0, errBadStreamMsg } if msg = buf[:ml]; len(msg) == 0 { msg = nil } return subject, reply, hdr, msg, lseq, ts, nil } // Helper to return if compression allowed. func (mset *stream) compressAllowed() bool { mset.clMu.Lock() defer mset.clMu.Unlock() return mset.compressOK } func encodeStreamMsg(subject, reply string, hdr, msg []byte, lseq uint64, ts int64) []byte { return encodeStreamMsgAllowCompress(subject, reply, hdr, msg, lseq, ts, false) } // Threshold for compression. // TODO(dlc) - Eventually make configurable. const compressThreshold = 8192 // 8k // If allowed and contents over the threshold we will compress. func encodeStreamMsgAllowCompress(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, compressOK bool) []byte { // Clip the subject, reply, header and msgs down. Operate on // uint64 lengths to avoid overflowing. slen := min(uint64(len(subject)), math.MaxUint16) rlen := min(uint64(len(reply)), math.MaxUint16) hlen := min(uint64(len(hdr)), math.MaxUint16) mlen := min(uint64(len(msg)), math.MaxUint32) total := slen + rlen + hlen + mlen shouldCompress := compressOK && total > compressThreshold elen := int(1 + 8 + 8 + total) elen += (2 + 2 + 2 + 4) // Encoded lengths, 4bytes buf := make([]byte, 1, elen) buf[0] = byte(streamMsgOp) var le = binary.LittleEndian buf = le.AppendUint64(buf, lseq) buf = le.AppendUint64(buf, uint64(ts)) buf = le.AppendUint16(buf, uint16(slen)) buf = append(buf, subject[:slen]...) buf = le.AppendUint16(buf, uint16(rlen)) buf = append(buf, reply[:rlen]...) buf = le.AppendUint16(buf, uint16(hlen)) buf = append(buf, hdr[:hlen]...) buf = le.AppendUint32(buf, uint32(mlen)) buf = append(buf, msg[:mlen]...) // Check if we should compress. if shouldCompress { nbuf := make([]byte, s2.MaxEncodedLen(elen)) nbuf[0] = byte(compressedStreamMsgOp) ebuf := s2.Encode(nbuf[1:], buf[1:]) // Only pay the cost of decode on the other side if we compressed. // S2 will allow us to try without major penalty for non-compressable data. if len(ebuf) < len(buf) { buf = nbuf[:len(ebuf)+1] } } return buf } // Determine if all peers in our set support the binary snapshot. func (mset *stream) supportsBinarySnapshot() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.supportsBinarySnapshotLocked() } // Determine if all peers in our set support the binary snapshot. // Lock should be held. func (mset *stream) supportsBinarySnapshotLocked() bool { s, n := mset.srv, mset.node if s == nil || n == nil { return false } // Grab our peers and walk them to make sure we can all support binary stream snapshots. id, peers := n.ID(), n.Peers() for _, p := range peers { if p.ID == id { // We know we support ourselves. continue } // Since release 2.10.16 only deny if we know the other node does not support. if sir, ok := s.nodeToInfo.Load(p.ID); ok && sir != nil && !sir.(nodeInfo).binarySnapshots { return false } } return true } // StreamSnapshot is used for snapshotting and out of band catch up in clustered mode. // Legacy, replace with binary stream snapshots. type streamSnapshot struct { Msgs uint64 `json:"messages"` Bytes uint64 `json:"bytes"` FirstSeq uint64 `json:"first_seq"` LastSeq uint64 `json:"last_seq"` Failed uint64 `json:"clfs"` Deleted []uint64 `json:"deleted,omitempty"` } // Grab a snapshot of a stream for clustered mode. func (mset *stream) stateSnapshot() []byte { mset.mu.RLock() defer mset.mu.RUnlock() return mset.stateSnapshotLocked() } // Grab a snapshot of a stream for clustered mode. // Lock should be held. func (mset *stream) stateSnapshotLocked() []byte { // Decide if we can support the new style of stream snapshots. if mset.supportsBinarySnapshotLocked() { snap, err := mset.store.EncodedStreamState(mset.getCLFS()) if err != nil { return nil } return snap } // Older v1 version with deleted as a sorted []uint64. state := mset.store.State() snap := &streamSnapshot{ Msgs: state.Msgs, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq, Failed: mset.getCLFS(), Deleted: state.Deleted, } b, _ := json.Marshal(snap) return b } // Will check if we can do message compression in RAFT and catchup logic. func (mset *stream) checkAllowMsgCompress(peers []string) { allowed := true for _, id := range peers { sir, ok := mset.srv.nodeToInfo.Load(id) if !ok || sir == nil { allowed = false break } // Check for capability. if si := sir.(nodeInfo); si.cfg == nil || !si.cfg.CompressOK { allowed = false break } } mset.mu.Lock() mset.compressOK = allowed mset.mu.Unlock() } // To warn when we are getting too far behind from what has been proposed vs what has been committed. const streamLagWarnThreshold = 10_000 // processClusteredMsg will propose the inbound message to the underlying raft group. func (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg []byte) error { // For possible error response. var response []byte mset.mu.RLock() canRespond := !mset.cfg.NoAck && len(reply) > 0 name, stype, store := mset.cfg.Name, mset.cfg.Storage, mset.store s, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node maxMsgSize, lseq := int(mset.cfg.MaxMsgSize), mset.lseq interestPolicy, discard, maxMsgs, maxBytes := mset.cfg.Retention != LimitsPolicy, mset.cfg.Discard, mset.cfg.MaxMsgs, mset.cfg.MaxBytes isLeader, isSealed, compressOK := mset.isLeader(), mset.cfg.Sealed, mset.compressOK mset.mu.RUnlock() // This should not happen but possible now that we allow scale up, and scale down where this could trigger. if node == nil { return mset.processJetStreamMsg(subject, reply, hdr, msg, 0, 0) } // Check that we are the leader. This can be false if we have scaled up from an R1 that had inbound queued messages. if !isLeader { return NewJSClusterNotLeaderError() } // Bail here if sealed. if isSealed { var resp = JSPubAckResponse{PubAck: &PubAck{Stream: mset.name()}, Error: NewJSStreamSealedError()} b, _ := json.Marshal(resp) mset.outq.sendMsg(reply, b) return NewJSStreamSealedError() } // Check here pre-emptively if we have exceeded this server limits. if js.limitsExceeded(stype) { s.resourcesExceededError() if canRespond { b, _ := json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: name}, Error: NewJSInsufficientResourcesError()}) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, b, nil, 0)) } // Stepdown regardless. if node := mset.raftNode(); node != nil { node.StepDown() } return NewJSInsufficientResourcesError() } // Check here pre-emptively if we have exceeded our account limits. if exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded { if err == nil { err = NewJSAccountResourcesExceededError() } s.RateLimitWarnf("JetStream account limits exceeded for '%s': %s", jsa.acc().GetName(), err.Error()) if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.Error = err response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } return err } // Check msgSize if we have a limit set there. Again this works if it goes through but better to be pre-emptive. if maxMsgSize >= 0 && (len(hdr)+len(msg)) > maxMsgSize { err := fmt.Errorf("JetStream message size exceeds limits for '%s > %s'", jsa.acc().Name, mset.cfg.Name) s.RateLimitWarnf("%s", err.Error()) if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.Error = NewJSStreamMessageExceedsMaximumError() response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } return err } // Some header checks can be checked pre proposal. Most can not. var msgId string if len(hdr) > 0 { // Since we encode header len as u16 make sure we do not exceed. // Again this works if it goes through but better to be pre-emptive. if len(hdr) > math.MaxUint16 { err := fmt.Errorf("JetStream header size exceeds limits for '%s > %s'", jsa.acc().Name, mset.cfg.Name) s.RateLimitWarnf("%s", err.Error()) if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.Error = NewJSStreamHeaderExceedsMaximumError() response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } return err } // Expected last sequence per subject. // We can check for last sequence per subject but only if the expected seq <= lseq. if seq, exists := getExpectedLastSeqPerSubject(hdr); exists && store != nil && seq <= lseq { var smv StoreMsg var fseq uint64 sm, err := store.LoadLastMsg(subject, &smv) if sm != nil { fseq = sm.seq } if err == ErrStoreMsgNotFound && seq == 0 { fseq, err = 0, nil } if err != nil || fseq != seq { if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamWrongLastSequenceError(fseq) b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return fmt.Errorf("last sequence by subject mismatch: %d vs %d", seq, fseq) } } // Expected stream name can also be pre-checked. if sname := getExpectedStream(hdr); sname != _EMPTY_ && sname != name { if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamNotMatchError() b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return errStreamMismatch } // Check for MsgIds here at the cluster level to avoid excessive CLFS accounting. // Will help during restarts. if msgId = getMsgId(hdr); msgId != _EMPTY_ { mset.mu.Lock() if dde := mset.checkMsgId(msgId); dde != nil { var buf [256]byte pubAck := append(buf[:0], mset.pubAck...) seq := dde.seq mset.mu.Unlock() if canRespond { response := append(pubAck, strconv.FormatUint(seq, 10)...) response = append(response, ",\"duplicate\": true}"...) outq.sendMsg(reply, response) } return errMsgIdDuplicate } // FIXME(dlc) - locking conflict with accessing mset.clseq // For now we stage with zero, and will update in processStreamMsg. mset.storeMsgIdLocked(&ddentry{msgId, 0, time.Now().UnixNano()}) mset.mu.Unlock() } } // Proceed with proposing this message. // We only use mset.clseq for clustering and in case we run ahead of actual commits. // Check if we need to set initial value here mset.clMu.Lock() if mset.clseq == 0 || mset.clseq < lseq+mset.clfs { // Re-capture lseq = mset.lastSeq() mset.clseq = lseq + mset.clfs } // Check if we have an interest policy and discard new with max msgs or bytes. // We need to deny here otherwise it could succeed on some peers and not others // depending on consumer ack state. So we deny here, if we allow that means we know // it would succeed on every peer. if interestPolicy && discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) { // Track inflight. if mset.inflight == nil { mset.inflight = make(map[uint64]uint64) } if stype == FileStorage { mset.inflight[mset.clseq] = fileStoreMsgSize(subject, hdr, msg) } else { mset.inflight[mset.clseq] = memStoreMsgSize(subject, hdr, msg) } var state StreamState mset.store.FastState(&state) var err error if maxMsgs > 0 && state.Msgs+uint64(len(mset.inflight)) > uint64(maxMsgs) { err = ErrMaxMsgs } else if maxBytes > 0 { // TODO(dlc) - Could track this rollup independently. var bytesPending uint64 for _, nb := range mset.inflight { bytesPending += nb } if state.Bytes+bytesPending > uint64(maxBytes) { err = ErrMaxBytes } } if err != nil { delete(mset.inflight, mset.clseq) mset.clMu.Unlock() if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}} resp.Error = NewJSStreamStoreFailedError(err, Unless(err)) response, _ = json.Marshal(resp) outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } return err } } esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), compressOK) // Do proposal. err := node.Propose(esm) if err == nil { mset.clseq++ // If we are using the system account for NRG, add in the extra sent msgs and bytes to our account // so that the end user / account owner has visibility. if node.IsSystemAccount() && mset.acc != nil && r > 1 { atomic.AddInt64(&mset.acc.outMsgs, int64(r-1)) atomic.AddInt64(&mset.acc.outBytes, int64(len(esm)*(r-1))) } } // Check to see if we are being overrun. // TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured. if mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold { lerr := fmt.Errorf("JetStream stream '%s > %s' has high message lag", jsa.acc().Name, name) s.RateLimitWarnf("%s", lerr.Error()) } mset.clMu.Unlock() if err != nil { if canRespond { var resp = &JSPubAckResponse{PubAck: &PubAck{Stream: mset.cfg.Name}} resp.Error = &ApiError{Code: 503, Description: err.Error()} response, _ = json.Marshal(resp) // If we errored out respond here. outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } if isOutOfSpaceErr(err) { s.handleOutOfSpace(mset) } } return err } // For requesting messages post raft snapshot to catch up streams post server restart. // Any deleted msgs etc will be handled inline on catchup. type streamSyncRequest struct { Peer string `json:"peer,omitempty"` FirstSeq uint64 `json:"first_seq"` LastSeq uint64 `json:"last_seq"` DeleteRangesOk bool `json:"delete_ranges"` } // Given a stream state that represents a snapshot, calculate the sync request based on our current state. // Stream lock must be held. func (mset *stream) calculateSyncRequest(state *StreamState, snap *StreamReplicatedState) *streamSyncRequest { // Shouldn't happen, but consequences are pretty bad if we have the lock held and // our caller tries to take the lock again on panic defer, as in processSnapshot. if state == nil || snap == nil || mset.node == nil { return nil } // Quick check if we are already caught up. if state.LastSeq >= snap.LastSeq { return nil } return &streamSyncRequest{FirstSeq: state.LastSeq + 1, LastSeq: snap.LastSeq, Peer: mset.node.ID(), DeleteRangesOk: true} } // processSnapshotDeletes will update our current store based on the snapshot // but only processing deletes and new FirstSeq / purges. func (mset *stream) processSnapshotDeletes(snap *StreamReplicatedState) { mset.mu.Lock() var state StreamState mset.store.FastState(&state) // Always adjust if FirstSeq has moved beyond our state. var didReset bool if snap.FirstSeq > state.FirstSeq { mset.store.Compact(snap.FirstSeq) mset.store.FastState(&state) mset.lseq = state.LastSeq mset.clearAllPreAcksBelowFloor(state.FirstSeq) didReset = true } s := mset.srv mset.mu.Unlock() if didReset { s.Warnf("Catchup for stream '%s > %s' resetting first sequence: %d on catchup request", mset.account(), mset.name(), snap.FirstSeq) } if len(snap.Deleted) > 0 { mset.store.SyncDeleted(snap.Deleted) } } func (mset *stream) setCatchupPeer(peer string, lag uint64) { if peer == _EMPTY_ { return } mset.mu.Lock() if mset.catchups == nil { mset.catchups = make(map[string]uint64) } mset.catchups[peer] = lag mset.mu.Unlock() } // Will decrement by one. func (mset *stream) updateCatchupPeer(peer string) { if peer == _EMPTY_ { return } mset.mu.Lock() if lag := mset.catchups[peer]; lag > 0 { mset.catchups[peer] = lag - 1 } mset.mu.Unlock() } func (mset *stream) decrementCatchupPeer(peer string, num uint64) { if peer == _EMPTY_ { return } mset.mu.Lock() if lag := mset.catchups[peer]; lag > 0 { if lag >= num { lag -= num } else { lag = 0 } mset.catchups[peer] = lag } mset.mu.Unlock() } func (mset *stream) clearCatchupPeer(peer string) { mset.mu.Lock() if mset.catchups != nil { delete(mset.catchups, peer) } mset.mu.Unlock() } // Lock should be held. func (mset *stream) clearAllCatchupPeers() { if mset.catchups != nil { mset.catchups = nil } } func (mset *stream) lagForCatchupPeer(peer string) uint64 { mset.mu.RLock() defer mset.mu.RUnlock() if mset.catchups == nil { return 0 } return mset.catchups[peer] } func (mset *stream) hasCatchupPeers() bool { mset.mu.RLock() defer mset.mu.RUnlock() return len(mset.catchups) > 0 } func (mset *stream) setCatchingUp() { mset.catchup.Store(true) } func (mset *stream) clearCatchingUp() { mset.catchup.Store(false) } func (mset *stream) isCatchingUp() bool { return mset.catchup.Load() } // Determine if a non-leader is current. // Lock should be held. func (mset *stream) isCurrent() bool { if mset.node == nil { return true } return mset.node.Current() && !mset.catchup.Load() } // Maximum requests for the whole server that can be in flight at the same time. const maxConcurrentSyncRequests = 32 var ( errCatchupCorruptSnapshot = errors.New("corrupt stream snapshot detected") errCatchupStalled = errors.New("catchup stalled") errCatchupStreamStopped = errors.New("stream has been stopped") // when a catchup is terminated due to the stream going away. errCatchupBadMsg = errors.New("bad catchup msg") errCatchupWrongSeqForSkip = errors.New("wrong sequence for skipped msg") errCatchupAbortedNoLeader = errors.New("catchup aborted, no leader") errCatchupTooManyRetries = errors.New("catchup failed, too many retries") ) // Process a stream snapshot. func (mset *stream) processSnapshot(snap *StreamReplicatedState) (e error) { // Update any deletes, etc. mset.processSnapshotDeletes(snap) mset.setCLFS(snap.Failed) mset.mu.Lock() var state StreamState mset.store.FastState(&state) sreq := mset.calculateSyncRequest(&state, snap) s, js, subject, n, st := mset.srv, mset.js, mset.sa.Sync, mset.node, mset.cfg.Storage qname := fmt.Sprintf("[ACC:%s] stream '%s' snapshot", mset.acc.Name, mset.cfg.Name) mset.mu.Unlock() // Bug that would cause this to be empty on stream update. if subject == _EMPTY_ { return errCatchupCorruptSnapshot } // Just return if up to date or already exceeded limits. if sreq == nil || js.limitsExceeded(st) { return nil } // Pause the apply channel for our raft group while we catch up. if err := n.PauseApply(); err != nil { return err } defer func() { // Don't bother resuming if server or stream is gone. if e != errCatchupStreamStopped && e != ErrServerNotRunning { n.ResumeApply() } }() // Set our catchup state. mset.setCatchingUp() defer mset.clearCatchingUp() var sub *subscription var err error const ( startInterval = 5 * time.Second activityInterval = 30 * time.Second ) notActive := time.NewTimer(startInterval) defer notActive.Stop() defer func() { if sub != nil { s.sysUnsubscribe(sub) } // Make sure any consumers are updated for the pending amounts. mset.mu.Lock() for _, o := range mset.consumers { o.mu.Lock() if o.isLeader() { o.streamNumPending() } o.mu.Unlock() } mset.mu.Unlock() // If we are interest based make sure to check our ack floor state. // We will delay a bit to allow consumer states to also catchup. if mset.isInterestRetention() { fire := time.Duration(rand.Intn(10)+5) * time.Second time.AfterFunc(fire, mset.checkInterestState) } }() var releaseSem bool releaseSyncOutSem := func() { if !releaseSem { return } // Need to use select for the server shutdown case. select { case s.syncOutSem <- struct{}{}: default: } releaseSem = false } // On exit, we will release our semaphore if we acquired it. defer releaseSyncOutSem() // Do not let this go on forever. const maxRetries = 3 var numRetries int RETRY: // On retry, we need to release the semaphore we got. Call will be no-op // if releaseSem boolean has not been set to true on successfully getting // the semaphore. releaseSyncOutSem() if n.Leaderless() { // Prevent us from spinning if we've installed a snapshot from a leader but there's no leader online. // We wait a bit to check if a leader has come online in the meantime, if so we can continue. var canContinue bool if numRetries == 0 { time.Sleep(startInterval) canContinue = !n.Leaderless() } if !canContinue { return fmt.Errorf("%w for stream '%s > %s'", errCatchupAbortedNoLeader, mset.account(), mset.name()) } } // If we have a sub clear that here. if sub != nil { s.sysUnsubscribe(sub) sub = nil } if !s.isRunning() { return ErrServerNotRunning } numRetries++ if numRetries > maxRetries { // Force a hard reset here. return errCatchupTooManyRetries } // Block here if we have too many requests in flight. <-s.syncOutSem releaseSem = true // We may have been blocked for a bit, so the reset needs to ensure that we // consume the already fired timer. if !notActive.Stop() { select { case <-notActive.C: default: } } notActive.Reset(startInterval) // Grab sync request again on failures. if sreq == nil { mset.mu.RLock() var state StreamState mset.store.FastState(&state) sreq = mset.calculateSyncRequest(&state, snap) mset.mu.RUnlock() if sreq == nil { return nil } } // Used to transfer message from the wire to another Go routine internally. type im struct { msg []byte reply string } // This is used to notify the leader that it should stop the runCatchup // because we are either bailing out or going to retry due to an error. notifyLeaderStopCatchup := func(mrec *im, err error) { if mrec.reply == _EMPTY_ { return } s.sendInternalMsgLocked(mrec.reply, _EMPTY_, nil, err.Error()) } msgsQ := newIPQueue[*im](s, qname) defer msgsQ.unregister() // Send our catchup request here. reply := syncReplySubject() sub, err = s.sysSubscribe(reply, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { // Make copy since we are using a buffer from the inbound client/route. msgsQ.push(&im{copyBytes(msg), reply}) }) if err != nil { s.Errorf("Could not subscribe to stream catchup: %v", err) goto RETRY } // Send our sync request. b, _ := json.Marshal(sreq) s.sendInternalMsgLocked(subject, reply, nil, b) // Remember when we sent this out to avoid loop spins on errors below. reqSendTime := time.Now() // Clear our sync request. sreq = nil // Run our own select loop here. for qch, lch := n.QuitC(), n.LeadChangeC(); ; { select { case <-msgsQ.ch: notActive.Reset(activityInterval) mrecs := msgsQ.pop() for _, mrec := range mrecs { msg := mrec.msg // Check for eof signaling. if len(msg) == 0 { msgsQ.recycle(&mrecs) return nil } if _, err := mset.processCatchupMsg(msg); err == nil { if mrec.reply != _EMPTY_ { s.sendInternalMsgLocked(mrec.reply, _EMPTY_, nil, nil) } } else if isOutOfSpaceErr(err) { notifyLeaderStopCatchup(mrec, err) return err } else if err == NewJSInsufficientResourcesError() { notifyLeaderStopCatchup(mrec, err) if mset.js.limitsExceeded(mset.cfg.Storage) { s.resourcesExceededError() } else { s.Warnf("Catchup for stream '%s > %s' errored, account resources exceeded: %v", mset.account(), mset.name(), err) } msgsQ.recycle(&mrecs) return err } else { notifyLeaderStopCatchup(mrec, err) s.Warnf("Catchup for stream '%s > %s' errored, will retry: %v", mset.account(), mset.name(), err) msgsQ.recycle(&mrecs) // Make sure we do not spin and make things worse. const minRetryWait = 2 * time.Second elapsed := time.Since(reqSendTime) if elapsed < minRetryWait { select { case <-s.quitCh: return ErrServerNotRunning case <-qch: return errCatchupStreamStopped case <-time.After(minRetryWait - elapsed): } } goto RETRY } } notActive.Reset(activityInterval) msgsQ.recycle(&mrecs) case <-notActive.C: if mrecs := msgsQ.pop(); len(mrecs) > 0 { mrec := mrecs[0] notifyLeaderStopCatchup(mrec, errCatchupStalled) msgsQ.recycle(&mrecs) } s.Warnf("Catchup for stream '%s > %s' stalled", mset.account(), mset.name()) goto RETRY case <-s.quitCh: return ErrServerNotRunning case <-qch: return errCatchupStreamStopped case isLeader := <-lch: if isLeader { n.StepDown() goto RETRY } } } } // processCatchupMsg will be called to process out of band catchup msgs from a sync request. func (mset *stream) processCatchupMsg(msg []byte) (uint64, error) { if len(msg) == 0 { return 0, errCatchupBadMsg } op := entryOp(msg[0]) if op != streamMsgOp && op != compressedStreamMsgOp && op != deleteRangeOp { return 0, errCatchupBadMsg } mbuf := msg[1:] if op == deleteRangeOp { dr, err := decodeDeleteRange(mbuf) if err != nil { return 0, errCatchupBadMsg } // Handle the delete range. // Make sure the sequences match up properly. mset.mu.Lock() if len(mset.preAcks) > 0 { for seq := dr.First; seq < dr.First+dr.Num; seq++ { mset.clearAllPreAcks(seq) } } if err = mset.store.SkipMsgs(dr.First, dr.Num); err != nil { mset.mu.Unlock() return 0, errCatchupWrongSeqForSkip } mset.lseq = dr.First + dr.Num - 1 lseq := mset.lseq mset.mu.Unlock() return lseq, nil } if op == compressedStreamMsgOp { var err error mbuf, err = s2.Decode(nil, mbuf) if err != nil { panic(err.Error()) } } subj, _, hdr, msg, seq, ts, err := decodeStreamMsg(mbuf) if err != nil { return 0, errCatchupBadMsg } mset.mu.Lock() st := mset.cfg.Storage ddloaded := mset.ddloaded tierName := mset.tier replicas := mset.cfg.Replicas if mset.hasAllPreAcks(seq, subj) { mset.clearAllPreAcks(seq) // Mark this to be skipped subj, ts = _EMPTY_, 0 } mset.mu.Unlock() if mset.js.limitsExceeded(st) { return 0, NewJSInsufficientResourcesError() } else if exceeded, apiErr := mset.jsa.limitsExceeded(st, tierName, replicas); apiErr != nil { return 0, apiErr } else if exceeded { return 0, NewJSInsufficientResourcesError() } // Put into our store // Messages to be skipped have no subject or timestamp. // TODO(dlc) - formalize with skipMsgOp if subj == _EMPTY_ && ts == 0 { if lseq := mset.store.SkipMsg(); lseq != seq { return 0, errCatchupWrongSeqForSkip } } else if err := mset.store.StoreRawMsg(subj, hdr, msg, seq, ts); err != nil { return 0, err } mset.mu.Lock() defer mset.mu.Unlock() // Update our lseq. mset.setLastSeq(seq) // Check for MsgId and if we have one here make sure to update our internal map. if len(hdr) > 0 { if msgId := getMsgId(hdr); msgId != _EMPTY_ { if !ddloaded { mset.rebuildDedupe() } mset.storeMsgIdLocked(&ddentry{msgId, seq, ts}) } } return seq, nil } func (mset *stream) handleClusterSyncRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { var sreq streamSyncRequest if err := json.Unmarshal(msg, &sreq); err != nil { // Log error. return } mset.srv.startGoRoutine(func() { mset.runCatchup(reply, &sreq) }) } // Lock should be held. func (js *jetStream) offlineClusterInfo(rg *raftGroup) *ClusterInfo { s := js.srv ci := &ClusterInfo{Name: s.ClusterName(), RaftGroup: rg.Name} for _, peer := range rg.Peers { if sir, ok := s.nodeToInfo.Load(peer); ok && sir != nil { si := sir.(nodeInfo) pi := &PeerInfo{Peer: peer, Name: si.name, Current: false, Offline: true} ci.Replicas = append(ci.Replicas, pi) } } return ci } // clusterInfo will report on the status of the raft group. func (js *jetStream) clusterInfo(rg *raftGroup) *ClusterInfo { if js == nil { return nil } js.mu.RLock() defer js.mu.RUnlock() s := js.srv if rg == nil || rg.node == nil { return &ClusterInfo{ Name: s.cachedClusterName(), Leader: s.Name(), } } n := rg.node ci := &ClusterInfo{ Name: s.cachedClusterName(), Leader: s.serverNameForNode(n.GroupLeader()), RaftGroup: rg.Name, } now := time.Now() id, peers := n.ID(), n.Peers() // If we are leaderless, do not suppress putting us in the peer list. if ci.Leader == _EMPTY_ { id = _EMPTY_ } for _, rp := range peers { if rp.ID != id && rg.isMember(rp.ID) { var lastSeen time.Duration if now.After(rp.Last) && rp.Last.Unix() != 0 { lastSeen = now.Sub(rp.Last) } current := rp.Current if current && lastSeen > lostQuorumInterval { current = false } // Create a peer info with common settings if the peer has not been seen // yet (which can happen after the whole cluster is stopped and only some // of the nodes are restarted). pi := &PeerInfo{ Current: current, Offline: true, Active: lastSeen, Lag: rp.Lag, Peer: rp.ID, } // If node is found, complete/update the settings. if sir, ok := s.nodeToInfo.Load(rp.ID); ok && sir != nil { si := sir.(nodeInfo) pi.Name, pi.Offline, pi.cluster = si.name, si.offline, si.cluster } else { // If not, then add a name that indicates that the server name // is unknown at this time, and clear the lag since it is misleading // (the node may not have that much lag). // Note: We return now the Peer ID in PeerInfo, so the "(peerID: %s)" // would technically not be required, but keeping it for now. pi.Name, pi.Lag = fmt.Sprintf("Server name unknown at this time (peerID: %s)", rp.ID), 0 } ci.Replicas = append(ci.Replicas, pi) } } // Order the result based on the name so that we get something consistent // when doing repeated stream info in the CLI, etc... slices.SortFunc(ci.Replicas, func(i, j *PeerInfo) int { return cmp.Compare(i.Name, j.Name) }) return ci } func (mset *stream) checkClusterInfo(ci *ClusterInfo) { for _, r := range ci.Replicas { peer := getHash(r.Name) if lag := mset.lagForCatchupPeer(peer); lag > 0 { r.Current = false r.Lag = lag } } } // Return a list of alternates, ranked by preference order to the request, of stream mirrors. // This allows clients to select or get more information about read replicas that could be a // better option to connect to versus the original source. func (js *jetStream) streamAlternates(ci *ClientInfo, stream string) []StreamAlternate { if js == nil { return nil } js.mu.RLock() defer js.mu.RUnlock() s, cc := js.srv, js.cluster // Track our domain. domain := s.getOpts().JetStreamDomain // No clustering just return nil. if cc == nil { return nil } acc, _ := s.LookupAccount(ci.serviceAccount()) if acc == nil { return nil } // Collect our ordering first for clusters. weights := make(map[string]int) all := []string{ci.Cluster} all = append(all, ci.Alternates...) for i := 0; i < len(all); i++ { weights[all[i]] = len(all) - i } var alts []StreamAlternate for _, sa := range cc.streams[acc.Name] { // Add in ourselves and any mirrors. if sa.Config.Name == stream || (sa.Config.Mirror != nil && sa.Config.Mirror.Name == stream) { alts = append(alts, StreamAlternate{Name: sa.Config.Name, Domain: domain, Cluster: sa.Group.Cluster}) } } // If just us don't fill in. if len(alts) == 1 { return nil } // Sort based on our weights that originate from the request itself. // reverse sort slices.SortFunc(alts, func(i, j StreamAlternate) int { return -cmp.Compare(weights[i.Cluster], weights[j.Cluster]) }) return alts } // Internal request for stream info, this is coming on the wire so do not block here. func (mset *stream) handleClusterStreamInfoRequest(_ *subscription, c *client, _ *Account, subject, reply string, _ []byte) { go mset.processClusterStreamInfoRequest(reply) } func (mset *stream) processClusterStreamInfoRequest(reply string) { mset.mu.RLock() sysc, js, sa, config := mset.sysc, mset.srv.js.Load(), mset.sa, mset.cfg isLeader := mset.isLeader() mset.mu.RUnlock() // By design all members will receive this. Normally we only want the leader answering. // But if we have stalled and lost quorom all can respond. if sa != nil && !js.isGroupLeaderless(sa.Group) && !isLeader { return } // If we are not the leader let someone else possibly respond first. if !isLeader { time.Sleep(500 * time.Millisecond) } si := &StreamInfo{ Created: mset.createdTime(), State: mset.state(), Config: config, Cluster: js.clusterInfo(mset.raftGroup()), Sources: mset.sourcesInfo(), Mirror: mset.mirrorInfo(), TimeStamp: time.Now().UTC(), } // Check for out of band catchups. if mset.hasCatchupPeers() { mset.checkClusterInfo(si.Cluster) } sysc.sendInternalMsg(reply, _EMPTY_, nil, si) } // 64MB for now, for the total server. This is max we will blast out if asked to // do so to another server for purposes of catchups. // This number should be ok on 1Gbit interface. const defaultMaxTotalCatchupOutBytes = int64(64 * 1024 * 1024) // Current total outstanding catchup bytes. func (s *Server) gcbTotal() int64 { s.gcbMu.RLock() defer s.gcbMu.RUnlock() return s.gcbOut } // Returns true if Current total outstanding catchup bytes is below // the maximum configured. func (s *Server) gcbBelowMax() bool { s.gcbMu.RLock() defer s.gcbMu.RUnlock() return s.gcbOut <= s.gcbOutMax } // Adds `sz` to the server's total outstanding catchup bytes and to `localsz` // under the gcbMu lock. The `localsz` points to the local outstanding catchup // bytes of the runCatchup go routine of a given stream. func (s *Server) gcbAdd(localsz *int64, sz int64) { s.gcbMu.Lock() atomic.AddInt64(localsz, sz) s.gcbOut += sz if s.gcbOut >= s.gcbOutMax && s.gcbKick == nil { s.gcbKick = make(chan struct{}) } s.gcbMu.Unlock() } // Removes `sz` from the server's total outstanding catchup bytes and from // `localsz`, but only if `localsz` is non 0, which would signal that gcSubLast // has already been invoked. See that function for details. // Must be invoked under the gcbMu lock. func (s *Server) gcbSubLocked(localsz *int64, sz int64) { if atomic.LoadInt64(localsz) == 0 { return } atomic.AddInt64(localsz, -sz) s.gcbOut -= sz if s.gcbKick != nil && s.gcbOut < s.gcbOutMax { close(s.gcbKick) s.gcbKick = nil } } // Locked version of gcbSubLocked() func (s *Server) gcbSub(localsz *int64, sz int64) { s.gcbMu.Lock() s.gcbSubLocked(localsz, sz) s.gcbMu.Unlock() } // Similar to gcbSub() but reset `localsz` to 0 at the end under the gcbMu lock. // This will signal further calls to gcbSub() for this `localsz` pointer that // nothing should be done because runCatchup() has exited and any remaining // outstanding bytes value has already been decremented. func (s *Server) gcbSubLast(localsz *int64) { s.gcbMu.Lock() s.gcbSubLocked(localsz, *localsz) *localsz = 0 s.gcbMu.Unlock() } // Returns our kick chan, or nil if it does not exist. func (s *Server) cbKickChan() <-chan struct{} { s.gcbMu.RLock() defer s.gcbMu.RUnlock() return s.gcbKick } func (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) { s := mset.srv defer s.grWG.Done() const maxOutBytes = int64(64 * 1024 * 1024) // 64MB for now, these are all internal, from server to server const maxOutMsgs = int32(256 * 1024) // 256k in case we have lots of small messages or skip msgs. outb := int64(0) outm := int32(0) // On abnormal exit make sure to update global total. defer s.gcbSubLast(&outb) // Flow control processing. ackReplySize := func(subj string) int64 { if li := strings.LastIndexByte(subj, btsep); li > 0 && li < len(subj) { return parseAckReplyNum(subj[li+1:]) } return 0 } nextBatchC := make(chan struct{}, 4) nextBatchC <- struct{}{} remoteQuitCh := make(chan struct{}) const activityInterval = 30 * time.Second notActive := time.NewTimer(activityInterval) defer notActive.Stop() // Setup ackReply for flow control. ackReply := syncAckSubject() ackSub, _ := s.sysSubscribe(ackReply, func(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { if len(msg) > 0 { s.Warnf("Catchup for stream '%s > %s' was aborted on the remote due to: %q", mset.account(), mset.name(), msg) s.sysUnsubscribe(sub) close(remoteQuitCh) return } sz := ackReplySize(subject) s.gcbSub(&outb, sz) atomic.AddInt32(&outm, -1) mset.updateCatchupPeer(sreq.Peer) // Kick ourselves and anyone else who might have stalled on global state. select { case nextBatchC <- struct{}{}: default: } // Reset our activity notActive.Reset(activityInterval) }) defer s.sysUnsubscribe(ackSub) ackReplyT := strings.ReplaceAll(ackReply, ".*", ".%d") // Grab our state. var state StreamState // mset.store never changes after being set, don't need lock. mset.store.FastState(&state) // Setup sequences to walk through. seq, last := sreq.FirstSeq, sreq.LastSeq mset.setCatchupPeer(sreq.Peer, last-seq) // Check if we can compress during this. compressOk := mset.compressAllowed() var spb int const minWait = 5 * time.Second sendNextBatchAndContinue := func(qch chan struct{}) bool { // Check if we know we will not enter the loop because we are done. if seq > last { s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) // EOF s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) return false } // If we already sent a batch, we will try to make sure we can at least send a minimum // batch before sending the next batch. if spb > 0 { // Wait til we can send at least 4k const minBatchWait = int32(4 * 1024) mw := time.NewTimer(minWait) for done := maxOutMsgs-atomic.LoadInt32(&outm) > minBatchWait; !done; { select { case <-nextBatchC: done = maxOutMsgs-atomic.LoadInt32(&outm) > minBatchWait if !done { // Wait for a small bit. time.Sleep(100 * time.Millisecond) } else { // GC friendly. mw.Stop() } case <-mw.C: done = true case <-s.quitCh: return false case <-qch: return false case <-remoteQuitCh: return false } } spb = 0 } // Send an encoded msg. sendEM := func(em []byte) { // Place size in reply subject for flow control. l := int64(len(em)) reply := fmt.Sprintf(ackReplyT, l) s.gcbAdd(&outb, l) atomic.AddInt32(&outm, 1) s.sendInternalMsgLocked(sendSubject, reply, nil, em) spb++ } // If we support gap markers. var dr DeleteRange drOk := sreq.DeleteRangesOk // Will send our delete range. // Should already be checked for being valid. sendDR := func() { if dr.Num == 1 { // Send like a normal skip msg. sendEM(encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, dr.First, 0)) } else { // We have a run, send a gap record. We send these without reply or tracking. s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, encodeDeleteRange(&dr)) // Clear out the pending for catchup. mset.decrementCatchupPeer(sreq.Peer, dr.Num) } // Reset always. dr.First, dr.Num = 0, 0 } // See if we should use LoadNextMsg instead of walking sequence by sequence if we have an order magnitude more interior deletes. // Only makes sense with delete range capabilities. useLoadNext := drOk && (uint64(state.NumDeleted) > 10*state.Msgs) var smv StoreMsg for ; seq <= last && atomic.LoadInt64(&outb) <= maxOutBytes && atomic.LoadInt32(&outm) <= maxOutMsgs && s.gcbBelowMax(); seq++ { var sm *StoreMsg var err error // If we should use load next do so here. if useLoadNext { var nseq uint64 sm, nseq, err = mset.store.LoadNextMsg(fwcs, true, seq, &smv) if err == nil && nseq > seq { // If we jumped over the requested last sequence, clamp it down. // Otherwise, we would send too much to the follower. if nseq > last { nseq = last sm = nil } dr.First, dr.Num = seq, nseq-seq // Jump ahead seq = nseq } else if err == ErrStoreEOF { dr.First, dr.Num = seq, last-seq // Clear EOF here for normal processing. err = nil // Jump ahead seq = last } } else { sm, err = mset.store.LoadMsg(seq, &smv) } // if this is not a deleted msg, bail out. if err != nil && err != ErrStoreMsgNotFound && err != errDeletedMsg { if err == ErrStoreEOF { var state StreamState mset.store.FastState(&state) if seq > state.LastSeq { // The snapshot has a larger last sequence then we have. This could be due to a truncation // when trying to recover after corruption, still not 100% sure. Could be off by 1 too somehow, // but tested a ton of those with no success. s.Warnf("Catchup for stream '%s > %s' completed, but requested sequence %d was larger then current state: %+v", mset.account(), mset.name(), seq, state) // Try our best to redo our invalidated snapshot as well. if n := mset.raftNode(); n != nil { if snap := mset.stateSnapshot(); snap != nil { n.InstallSnapshot(snap) } } // If we allow gap markers check if we have one pending. if drOk && dr.First > 0 { sendDR() } // Signal EOF s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) return false } } s.Warnf("Error loading message for catchup '%s > %s': %v", mset.account(), mset.name(), err) return false } if sm != nil { // If we allow gap markers check if we have one pending. if drOk && dr.First > 0 { sendDR() } // Send the normal message now. sendEM(encodeStreamMsgAllowCompress(sm.subj, _EMPTY_, sm.hdr, sm.msg, sm.seq, sm.ts, compressOk)) } else { if drOk { if dr.First == 0 { dr.First, dr.Num = seq, 1 } else { dr.Num++ } } else { // Skip record for deleted msg. sendEM(encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, seq, 0)) } } // Check if we are done. if seq == last { // Need to see if we have a pending delete range. if drOk && dr.First > 0 { sendDR() } s.Noticef("Catchup for stream '%s > %s' complete", mset.account(), mset.name()) // EOF s.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil) return false } select { case <-remoteQuitCh: return false default: } } if drOk && dr.First > 0 { sendDR() } return true } // Check is this stream got closed. mset.mu.RLock() qch := mset.qch mset.mu.RUnlock() if qch == nil { return } // Run as long as we are still active and need catchup. // FIXME(dlc) - Purge event? Stream delete? for { // Get this each time, will be non-nil if globally blocked and we will close to wake everyone up. cbKick := s.cbKickChan() select { case <-s.quitCh: return case <-qch: return case <-remoteQuitCh: mset.clearCatchupPeer(sreq.Peer) return case <-notActive.C: s.Warnf("Catchup for stream '%s > %s' stalled", mset.account(), mset.name()) mset.clearCatchupPeer(sreq.Peer) return case <-nextBatchC: if !sendNextBatchAndContinue(qch) { mset.clearCatchupPeer(sreq.Peer) return } case <-cbKick: if !sendNextBatchAndContinue(qch) { mset.clearCatchupPeer(sreq.Peer) return } case <-time.After(500 * time.Millisecond): if !sendNextBatchAndContinue(qch) { mset.clearCatchupPeer(sreq.Peer) return } } } } const jscAllSubj = "$JSC.>" func syncSubjForStream() string { return syncSubject("$JSC.SYNC") } func syncReplySubject() string { return syncSubject("$JSC.R") } func infoReplySubject() string { return syncSubject("$JSC.R") } func syncAckSubject() string { return syncSubject("$JSC.ACK") + ".*" } func syncSubject(pre string) string { var sb strings.Builder sb.WriteString(pre) sb.WriteByte(btsep) var b [replySuffixLen]byte rn := rand.Int63() for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } sb.Write(b[:]) return sb.String() } const ( clusterStreamInfoT = "$JSC.SI.%s.%s" clusterConsumerInfoT = "$JSC.CI.%s.%s.%s" jsaUpdatesSubT = "$JSC.ARU.%s.*" jsaUpdatesPubT = "$JSC.ARU.%s.%s" ) nats-server-2.10.27/server/jetstream_cluster_1_test.go000066400000000000000000006377341477524627100230520ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests // +build !skip_js_tests,!skip_js_cluster_tests package server import ( "bytes" "context" crand "crypto/rand" "encoding/binary" "encoding/json" "errors" "fmt" "math/rand" "os" "path/filepath" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) func TestJetStreamClusterConfig(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 16GB, max_file_store: 10TB, store_dir: '%s'} cluster { listen: 127.0.0.1:-1 } `)) check := func(errStr string) { t.Helper() opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), errStr) { t.Fatalf("Expected an error of `%s`, got `%v`", errStr, err) } } check("requires `server_name`") conf = createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: "TEST" jetstream: {max_mem_store: 16GB, max_file_store: 10TB, store_dir: '%s'} cluster { listen: 127.0.0.1:-1 } `)) check("requires `cluster.name`") } func TestJetStreamClusterLeader(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() // Kill our current leader and force an election. c.leader().Shutdown() c.waitOnLeader() // Now killing our current leader should leave us leaderless. c.leader().Shutdown() c.expectNoLeader() } func TestJetStreamClusterExpand(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 2) defer c.shutdown() c.addInNewServer() c.waitOnPeerCount(3) } func TestJetStreamClusterAccountInfo(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc := clientConnectToServer(t, c.randomServer()) defer nc.Close() reply := nats.NewInbox() sub, _ := nc.SubscribeSync(reply) if err := nc.PublishRequest(JSApiAccountInfo, reply, nil); err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, 1) resp, _ := sub.NextMsg(0) var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.JetStreamAccountStats == nil || info.Error != nil { t.Fatalf("Did not receive correct response: %+v", info.Error) } // Make sure we only got 1 response. // Technically this will always work since its a singelton service export. if nmsgs, _, _ := sub.Pending(); nmsgs > 0 { t.Fatalf("Expected only a single response, got %d more", nmsgs) } } func TestJetStreamClusterStreamLimitWithAccountDefaults(t *testing.T) { // 2MB memory, 8MB disk c := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, "R3L", 3) defer c.shutdown() // Client based API s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 2, MaxBytes: 4 * 1024 * 1024, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Replicas: 2, MaxBytes: 15 * 1024 * 1024, }) require_Contains(t, err.Error(), "no suitable peers for placement", "insufficient storage") } func TestJetStreamClusterInfoRaftGroup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R1S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Storage: nats.FileStorage, Replicas: 3, }) require_NoError(t, err) nfoResp, err := nc.Request("$JS.API.STREAM.INFO.TEST", nil, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(nfoResp.Data, &si) require_NoError(t, err) if si.Cluster == nil { t.Fatalf("Expected cluster info, got none") } stream, err := acc.lookupStream("TEST") require_NoError(t, err) if si.Cluster.RaftGroup != stream.raftGroup().Name { t.Fatalf("Expected raft group %q to equal %q", si.Cluster.RaftGroup, stream.raftGroup().Name) } var sscfg StreamConfig rCfgData, err := os.ReadFile(filepath.Join(s.opts.StoreDir, "jetstream", "$SYS", "_js_", stream.raftGroup().Name, "meta.inf")) require_NoError(t, err) err = json.Unmarshal(rCfgData, &sscfg) require_NoError(t, err) if !reflect.DeepEqual(sscfg.Metadata, map[string]string{"account": "$G", "stream": "TEST", "type": "stream"}) { t.Fatalf("Invalid raft stream metadata: %v", sscfg.Metadata) } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "DURABLE", Replicas: 3}) require_NoError(t, err) consumer := stream.lookupConsumer("DURABLE") var ci ConsumerInfo nfoResp, err = nc.Request("$JS.API.CONSUMER.INFO.TEST.DURABLE", nil, time.Second) require_NoError(t, err) var cscfg ConsumerConfig rCfgData, err = os.ReadFile(filepath.Join(s.opts.StoreDir, "jetstream", "$SYS", "_js_", consumer.raftGroup().Name, "meta.inf")) require_NoError(t, err) err = json.Unmarshal(rCfgData, &cscfg) require_NoError(t, err) if !reflect.DeepEqual(cscfg.Metadata, map[string]string{"account": "$G", "consumer": "DURABLE", "stream": "TEST", "type": "consumer"}) { t.Fatalf("Invalid raft stream metadata: %v", cscfg.Metadata) } err = json.Unmarshal(nfoResp.Data, &ci) require_NoError(t, err) if ci.Cluster.RaftGroup != consumer.raftGroup().Name { t.Fatalf("Expected raft group %q to equal %q", ci.Cluster.RaftGroup, consumer.raftGroup().Name) } } func TestJetStreamClusterSingleReplicaStreams(t *testing.T) { c := createJetStreamClusterExplicit(t, "R1S", 3) defer c.shutdown() // Client based API s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil { t.Fatalf("Expected si to have cluster info") } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Now create a consumer. This should be pinned to same server that our stream was allocated to. // First do a normal sub. sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) // Now create a consumer as well. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci == nil || ci.Name != "dlc" || ci.Stream != "TEST" { t.Fatalf("ConsumerInfo is not correct %+v", ci) } // Now make sure that if we kill and restart the server that this stream and consumer return. sl := c.streamLeader("$G", "TEST") sl.Shutdown() c.restartServer(sl) c.waitOnStreamLeader("$G", "TEST") si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Now durable consumer. c.waitOnConsumerLeader("$G", "TEST", "dlc") if _, err = js.ConsumerInfo("TEST", "dlc"); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterMultiReplicaStreams(t *testing.T) { c := createJetStreamClusterExplicit(t, "RNS", 5) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Now create a consumer. This should be affinitize to the same set of servers as the stream. // First do a normal sub. sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) // Now create a consumer as well. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci == nil || ci.Name != "dlc" || ci.Stream != "TEST" || ci.NumPending != uint64(toSend) { t.Fatalf("ConsumerInfo is not correct %+v", ci) } } func TestJetStreamClusterMultiReplicaStreamsDefaultFileMem(t *testing.T) { const testConfig = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` c := createJetStreamClusterWithTemplate(t, testConfig, "RNS", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, MaxBytes: 1024, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Now create a consumer. This should be affinitize to the same set of servers as the stream. // First do a normal sub. sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) // Now create a consumer as well. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci == nil || ci.Name != "dlc" || ci.Stream != "TEST" || ci.NumPending != uint64(toSend) { t.Fatalf("ConsumerInfo is not correct %+v", ci) } } func TestJetStreamClusterMemoryStore(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3M", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, Storage: nats.MemoryStorage, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 100 messages. msg, toSend := []byte("Hello MemoryStore"), 100 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } if si.Cluster == nil || len(si.Cluster.Replicas) != 2 { t.Fatalf("Cluster info is incorrect: %+v", si.Cluster) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Do a normal sub. sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) } func TestJetStreamClusterDelete(t *testing.T) { c := createJetStreamClusterExplicit(t, "RNS", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "C22", Subjects: []string{"foo", "bar", "baz"}, Replicas: 2, Storage: nats.FileStorage, MaxMsgs: 100, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Error adding stream: %v", err) } // Now create a consumer. if _, err := js.AddConsumer("C22", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, }); err != nil { t.Fatalf("Error adding consumer: %v", err) } // Now delete the consumer. if err := js.DeleteConsumer("C22", "dlc"); err != nil { t.Fatalf("Error deleting consumer: %v", err) } // Now delete the stream. if err := js.DeleteStream("C22"); err != nil { t.Fatalf("Error deleting stream: %v", err) } // This will get the current information about usage and limits for this account. checkFor(t, time.Second, 15*time.Millisecond, func() error { info, err := js.AccountInfo() if err != nil { return err } if info.Streams != 0 { return fmt.Errorf("Expected no remaining streams, got %d", info.Streams) } return nil }) } func TestJetStreamClusterStreamPurge(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg, toSend := []byte("Hello JS Clustering"), 100 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Now purge the stream. if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 || si.State.FirstSeq != uint64(toSend+1) { t.Fatalf("Expected no msgs, got: %+v", si.State) } } func TestJetStreamClusterStreamUpdateSubjects(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we can update subjects. cfg.Subjects = []string{"bar", "baz"} si, err := js.UpdateStream(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil { t.Fatalf("Expected a stream info, got none") } if !reflect.DeepEqual(si.Config.Subjects, cfg.Subjects) { t.Fatalf("Expected subjects to be updated: got %+v", si.Config.Subjects) } // Make sure it registered js2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err = js2.Publish("foo", nil); err == nil { t.Fatalf("Expected this to fail") } if _, err = js2.Publish("baz", nil); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } func TestJetStreamClusterBadStreamUpdate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } msg, toSend := []byte("Keep Me"), 50 for i := 0; i < toSend; i++ { if _, err := js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Make sure a bad update will not remove our stream. cfg.Subjects = []string{"foo..bar"} if _, err := js.UpdateStream(cfg); err == nil || err == nats.ErrTimeout { t.Fatalf("Expected error but got none or timeout") } // Make sure we did not delete our original stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if !reflect.DeepEqual(si.Config.Subjects, []string{"foo", "bar"}) { t.Fatalf("Expected subjects to be original ones, got %+v", si.Config.Subjects) } } func TestJetStreamClusterConsumerRedeliveredInfo(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{Name: "TEST"} if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.Publish("TEST", []byte("CI")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } sub, _ := nc.SubscribeSync("R") sub.AutoUnsubscribe(2) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: "R", AckPolicy: nats.AckExplicitPolicy, AckWait: 100 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, 2) sub.Unsubscribe() ci, err = js.ConsumerInfo("TEST", ci.Name) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci.NumRedelivered != 1 { t.Fatalf("Expected 1 redelivered, got %d", ci.NumRedelivered) } } func TestJetStreamClusterConsumerState(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 5) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Make sure we are not connected to any of the stream servers so that we do not do client reconnect // when we take out the consumer leader. if s.JetStreamIsStreamAssigned("$G", "TEST") { nc.Close() for _, ns := range c.servers { if !ns.JetStreamIsStreamAssigned("$G", "TEST") { s = ns nc, js = jsClientConnect(t, s) defer nc.Close() break } } } sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Pull 5 messages and ack. for _, m := range fetchMsgs(t, sub, 5, 5*time.Second) { m.AckSync() } // Let state propagate for exact comparison below. time.Sleep(200 * time.Millisecond) ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } if ci.AckFloor.Consumer != 5 { t.Fatalf("Expected ack floor of %d, got %d", 5, ci.AckFloor.Consumer) } c.consumerLeader("$G", "TEST", "dlc").Shutdown() c.waitOnConsumerLeader("$G", "TEST", "dlc") nci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } // nil out timestamp for better comparison nci.Delivered.Last, ci.Delivered.Last = nil, nil if nci.Delivered != ci.Delivered { t.Fatalf("Consumer delivered did not match after leader switch, wanted %+v, got %+v", ci.Delivered, nci.Delivered) } nci.AckFloor.Last, ci.AckFloor.Last = nil, nil if nci.AckFloor != ci.AckFloor { t.Fatalf("Consumer ackfloor did not match after leader switch, wanted %+v, got %+v", ci.AckFloor, nci.AckFloor) } // Now make sure we can receive new messages. // Pull last 5. for _, m := range fetchMsgs(t, sub, 5, 5*time.Second) { m.AckSync() } nci, _ = sub.ConsumerInfo() if nci.Delivered.Consumer != 10 || nci.Delivered.Stream != 10 { t.Fatalf("Received bad delivered: %+v", nci.Delivered) } if nci.AckFloor.Consumer != 10 || nci.AckFloor.Stream != 10 { t.Fatalf("Received bad ackfloor: %+v", nci.AckFloor) } if nci.NumAckPending != 0 { t.Fatalf("Received bad ackpending: %+v", nci.NumAckPending) } } func TestJetStreamClusterFullConsumerState(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } fetchMsgs(t, sub, 1, 5*time.Second) // Now purge the stream. if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } } func TestJetStreamClusterMetaSnapshotsAndCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Shut one down. rs := c.randomServer() rs.Shutdown() c.waitOnLeader() s := c.leader() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() numStreams := 4 // Create 4 streams // FIXME(dlc) - R2 make sure we place properly. for i := 0; i < numStreams; i++ { sn := fmt.Sprintf("T-%d", i+1) _, err := js.AddStream(&nats.StreamConfig{Name: sn}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } c.leader().JetStreamSnapshotMeta() rs = c.restartServer(rs) c.checkClusterFormed() c.waitOnServerCurrent(rs) rs.Shutdown() c.waitOnLeader() for i := 0; i < numStreams; i++ { sn := fmt.Sprintf("T-%d", i+1) err := js.DeleteStream(sn) if err != nil { t.Fatalf("Unexpected error: %v", err) } } rs = c.restartServer(rs) c.checkClusterFormed() c.waitOnServerCurrent(rs) } func TestJetStreamClusterMetaSnapshotsMultiChange(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 2) defer c.shutdown() s := c.leader() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() // Add in 2 streams with 1 consumer each. if _, err := js.AddStream(&nats.StreamConfig{Name: "S1"}); err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnStreamLeader(globalAccountName, "S1") _, err := js.AddConsumer("S1", &nats.ConsumerConfig{Durable: "S1C1", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnConsumerLeader(globalAccountName, "S1", "S1C1") if _, err = js.AddStream(&nats.StreamConfig{Name: "S2"}); err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnStreamLeader(globalAccountName, "S2") _, err = js.AddConsumer("S2", &nats.ConsumerConfig{Durable: "S2C1", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnConsumerLeader(globalAccountName, "S2", "S2C1") // Add in a new server to the group. This way we know we can delete the original streams and consumers. rs := c.addInNewServer() c.waitOnServerCurrent(rs) rsn := rs.Name() // Shut it down. rs.Shutdown() // Wait for the peer to be removed. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { for _, p := range s.JetStreamClusterPeers() { if p == rsn { return fmt.Errorf("Old server still in peer set") } } return nil }) // We want to make changes here that test each delta scenario for the meta snapshots. // Add new stream and consumer. if _, err = js.AddStream(&nats.StreamConfig{Name: "S3"}); err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnStreamLeader(globalAccountName, "S3") _, err = js.AddConsumer("S3", &nats.ConsumerConfig{Durable: "S3C1", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnConsumerLeader(globalAccountName, "S3", "S3C1") // Delete stream S2 resp, _ := nc.Request(fmt.Sprintf(JSApiStreamDeleteT, "S2"), nil, time.Second) var dResp JSApiStreamDeleteResponse if err := json.Unmarshal(resp.Data, &dResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !dResp.Success || dResp.Error != nil { t.Fatalf("Got a bad response %+v", dResp.Error) } // Delete the consumer on S1 but add another. resp, _ = nc.Request(fmt.Sprintf(JSApiConsumerDeleteT, "S1", "S1C1"), nil, time.Second) var cdResp JSApiConsumerDeleteResponse if err = json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !cdResp.Success || cdResp.Error != nil { t.Fatalf("Got a bad response %+v", cdResp) } // Add new consumer on S1 _, err = js.AddConsumer("S1", &nats.ConsumerConfig{Durable: "S1C2", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnConsumerLeader(globalAccountName, "S1", "S1C2") cl := c.leader() cl.JetStreamSnapshotMeta() c.waitOnServerCurrent(cl) rs = c.restartServer(rs) c.checkClusterFormed() c.waitOnServerCurrent(rs) } func TestJetStreamClusterStreamSynchedTimeStamps(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Storage: nats.MemoryStorage, Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err = js.Publish("foo", []byte("TSS")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Grab the message and timestamp from our current leader sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } meta, _ := m.Metadata() sub.Unsubscribe() sl := c.streamLeader("$G", "foo") sl.Shutdown() c.waitOnLeader() c.waitOnStreamLeader("$G", "foo") nc, js = jsClientConnect(t, c.leader()) defer nc.Close() sm, err := js.GetMsg("foo", 1) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !sm.Time.Equal(meta.Timestamp) { t.Fatalf("Expected same timestamps, got %v vs %v", sm.Time, meta.Timestamp) } } // Test to mimic what R.I. was seeing. func TestJetStreamClusterRestoreSingleConsumer(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err = js.Publish("foo", []byte("TSS")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } if m, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else { m.AckSync() } c.stopAll() c.restartAll() c.waitOnLeader() c.waitOnStreamLeader("$G", "foo") s = c.randomServer() nc, js = jsClientConnect(t, s) defer nc.Close() var names []string for name := range js.StreamNames() { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(names)) } // Now do detailed version. var infos []*nats.StreamInfo for info := range js.StreamsInfo() { infos = append(infos, info) } if len(infos) != 1 { t.Fatalf("Expected 1 stream but got %d", len(infos)) } si, err := js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "foo" { t.Fatalf("StreamInfo is not correct %+v", si) } // Now check for consumer. names = names[:0] for name := range js.ConsumerNames("foo") { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected 1 consumer but got %d", len(names)) } } func TestJetStreamClusterMaxBytesForStream(t *testing.T) { // Has max_file_store of 2GB c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() info, err := js.AccountInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we still are dynamic. if info.Limits.MaxStore != -1 || info.Limits.MaxMemory != -1 { t.Fatalf("Expected dynamic limits for the account, got %+v\n", info.Limits) } // Stream config. cfg := &nats.StreamConfig{ Name: "TEST", Replicas: 2, MaxBytes: 2 * 1024 * 1024 * 1024, // 2GB } _, err = js.AddStream(cfg) require_NoError(t, err) // Make sure going over the single server limit though is enforced (for now). cfg.Name = "TEST2" cfg.MaxBytes *= 2 _, err = js.AddStream(cfg) require_Contains(t, err.Error(), "no suitable peers for placement") } func TestJetStreamClusterStreamPublishWithActiveConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err = js.Publish("foo", []byte("TSS")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } // FIXME(dlc) - Need to track this down. c.waitOnConsumerLeader("$G", "foo", "dlc") if m, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else { m.AckSync() } // Send 10 messages. for i := 1; i <= 10; i++ { payload := []byte(fmt.Sprintf("MSG-%d", i)) if _, err = js.Publish("foo", payload); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkSubsPending(t, sub, 10) // Sanity check for duplicate deliveries.. if nmsgs, _, _ := sub.Pending(); nmsgs > 10 { t.Fatalf("Expected only %d responses, got %d more", 10, nmsgs) } for i := 1; i <= 10; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } payload := []byte(fmt.Sprintf("MSG-%d", i)) if !bytes.Equal(m.Data, payload) { t.Fatalf("Did not get expected msg, expected %q, got %q", payload, m.Data) } } ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } c.consumerLeader("$G", "foo", "dlc").Shutdown() c.waitOnConsumerLeader("$G", "foo", "dlc") ci2, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } ci.Cluster = nil ci2.Cluster = nil // nil out timestamp for better comparison ci.Delivered.Last, ci2.Delivered.Last = nil, nil ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil if !reflect.DeepEqual(ci, ci2) { t.Fatalf("Consumer info did not match: %+v vs %+v", ci, ci2) } // In case the server above was also stream leader. c.waitOnStreamLeader("$G", "foo") // Now send more.. // Send 10 more messages. for i := 11; i <= 20; i++ { payload := []byte(fmt.Sprintf("MSG-%d", i)) if _, err = js.Publish("foo", payload); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkSubsPending(t, sub, 10) // Sanity check for duplicate deliveries.. if nmsgs, _, _ := sub.Pending(); nmsgs > 10 { t.Fatalf("Expected only %d responses, got %d more", 10, nmsgs) } for i := 11; i <= 20; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } payload := []byte(fmt.Sprintf("MSG-%d", i)) if !bytes.Equal(m.Data, payload) { t.Fatalf("Did not get expected msg, expected %q, got %q", payload, m.Data) } } } func TestJetStreamClusterStreamOverlapSubjects(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST2", Subjects: []string{"foo"}}); err == nil || err == nats.ErrTimeout { t.Fatalf("Expected error but got none or timeout: %v", err) } // Now grab list of streams and make sure the second is not there. var names []string for name := range js.StreamNames() { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(names)) } // Now do a detailed version. var infos []*nats.StreamInfo for info := range js.StreamsInfo() { infos = append(infos, info) } if len(infos) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(infos)) } } func TestJetStreamClusterStreamInfoList(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() createStream := func(name string) { t.Helper() if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { t.Fatalf("Unexpected error: %v", err) } } createStream("foo") createStream("bar") createStream("baz") sendBatch := func(subject string, n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } sendBatch("foo", 10) sendBatch("bar", 22) sendBatch("baz", 33) // Now get the stream list info. var infos []*nats.StreamInfo checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { infos = infos[:0] for info := range js.StreamsInfo() { infos = append(infos, info) } if len(infos) != 3 { return fmt.Errorf("StreamInfo expected 3 results, got %d", len(infos)) } return nil }) for _, si := range infos { switch si.Config.Name { case "foo": if si.State.Msgs != 10 { t.Fatalf("Expected %d msgs but got %d", 10, si.State.Msgs) } case "bar": if si.State.Msgs != 22 { t.Fatalf("Expected %d msgs but got %d", 22, si.State.Msgs) } case "baz": if si.State.Msgs != 33 { t.Fatalf("Expected %d msgs but got %d", 33, si.State.Msgs) } } } } func TestJetStreamClusterConsumerInfoList(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Place messages so we can generate consumer state. for i := 0; i < 10; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } createConsumer := func(name string) *nats.Subscription { t.Helper() sub, err := js.PullSubscribe("TEST", name) if err != nil { t.Fatalf("Unexpected error: %v", err) } return sub } subFoo := createConsumer("foo") subBar := createConsumer("bar") subBaz := createConsumer("baz") // Place consumers in various states. for _, ss := range []struct { sub *nats.Subscription fetch int ack int }{ {subFoo, 4, 2}, {subBar, 2, 0}, {subBaz, 8, 6}, } { msgs := fetchMsgs(t, ss.sub, ss.fetch, 5*time.Second) for i := 0; i < ss.ack; i++ { msgs[i].AckSync() } } // Now get the consumer list info. var infos []*nats.ConsumerInfo for info := range js.ConsumersInfo("TEST") { infos = append(infos, info) } if len(infos) != 3 { t.Fatalf("ConsumerInfo expected 3 results, got %d", len(infos)) } for _, ci := range infos { switch ci.Name { case "foo": if ci.Delivered.Consumer != 4 { t.Fatalf("Expected %d delivered but got %d", 4, ci.Delivered.Consumer) } if ci.AckFloor.Consumer != 2 { t.Fatalf("Expected %d for ack floor but got %d", 2, ci.AckFloor.Consumer) } case "bar": if ci.Delivered.Consumer != 2 { t.Fatalf("Expected %d delivered but got %d", 2, ci.Delivered.Consumer) } if ci.AckFloor.Consumer != 0 { t.Fatalf("Expected %d for ack floor but got %d", 0, ci.AckFloor.Consumer) } case "baz": if ci.Delivered.Consumer != 8 { t.Fatalf("Expected %d delivered but got %d", 8, ci.Delivered.Consumer) } if ci.AckFloor.Consumer != 6 { t.Fatalf("Expected %d for ack floor but got %d", 6, ci.AckFloor.Consumer) } } } } func TestJetStreamClusterStreamUpdate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sc := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, MaxMsgs: 10, Discard: DiscardNew, } if _, err := js.AddStream(sc); err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 1; i <= int(sc.MaxMsgs); i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", i)) if _, err := js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Expect error here. if _, err := js.Publish("foo", []byte("fail")); err == nil { t.Fatalf("Expected publish to fail") } // Now update MaxMsgs, select non-leader s = c.randomNonStreamLeader("$G", "TEST") nc, js = jsClientConnect(t, s) defer nc.Close() sc.MaxMsgs = 20 si, err := js.UpdateStream(sc) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Config.MaxMsgs != 20 { t.Fatalf("Expected to have config updated with max msgs of %d, got %d", 20, si.Config.MaxMsgs) } // Do one that will fail. Wait and make sure we only are getting one response. sc.Name = "TEST22" rsub, _ := nc.SubscribeSync(nats.NewInbox()) defer rsub.Unsubscribe() nc.Flush() req, _ := json.Marshal(sc) if err := nc.PublishRequest(fmt.Sprintf(JSApiStreamUpdateT, "TEST"), rsub.Subject, req); err != nil { t.Fatalf("Unexpected error: %v", err) } // Wait incase more than one reply sent. time.Sleep(250 * time.Millisecond) if nmsgs, _, _ := rsub.Pending(); err != nil || nmsgs != 1 { t.Fatalf("Expected only one response, got %d", nmsgs) } m, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } var scResp JSApiStreamCreateResponse if err := json.Unmarshal(m.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.StreamInfo != nil || scResp.Error == nil { t.Fatalf("Did not receive correct response: %+v", scResp) } } func TestJetStreamClusterStreamExtendedUpdates(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } updateStream := func() *nats.StreamInfo { si, err := js.UpdateStream(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } return si } // Subjects can be updated cfg.Subjects = []string{"bar", "baz"} if si := updateStream(); !reflect.DeepEqual(si.Config.Subjects, cfg.Subjects) { t.Fatalf("Did not get expected stream info: %+v", si) } // Mirror changes are not supported for now cfg.Subjects = nil cfg.Mirror = &nats.StreamSource{Name: "ORDERS"} _, err := js.UpdateStream(cfg) require_Error(t, err, NewJSStreamMirrorNotUpdatableError()) } func TestJetStreamClusterDoubleAdd(t *testing.T) { c := createJetStreamClusterExplicit(t, "R32", 2) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Streams should allow double add. if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Check Consumers. cfg := &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy} if _, err := js.AddConsumer("TEST", cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // Check double add ok. if _, err := js.AddConsumer("TEST", cfg); err != nil { t.Fatalf("Expected no error but got: %v", err) } } func TestJetStreamClusterDefaultMaxAckPending(t *testing.T) { c := createJetStreamClusterExplicit(t, "R32", 2) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Do Consumers too. cfg := &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy} ci, err := js.AddConsumer("TEST", cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Check that we have a default set now for the max ack pending. if ci.Config.MaxAckPending != JsDefaultMaxAckPending { t.Fatalf("Expected a default for max ack pending of %d, got %d", JsDefaultMaxAckPending, ci.Config.MaxAckPending) } } func TestJetStreamClusterStreamNormalCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 for i := 1; i <= toSend; i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", i)) if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sl := c.streamLeader("$G", "TEST") sl.Shutdown() c.waitOnStreamLeader("$G", "TEST") // Send 10 more while one replica offline. for i := toSend; i <= toSend*2; i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", i)) if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Delete the first from the second batch. dreq := JSApiMsgDeleteRequest{Seq: uint64(toSend)} dreqj, err := json.Marshal(dreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, _ := nc.Request(fmt.Sprintf(JSApiMsgDeleteT, "TEST"), dreqj, time.Second) var delMsgResp JSApiMsgDeleteResponse if err = json.Unmarshal(resp.Data, &delMsgResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !delMsgResp.Success || delMsgResp.Error != nil { t.Fatalf("Got a bad response %+v", delMsgResp.Error) } sl = c.restartServer(sl) c.checkClusterFormed() c.waitOnServerCurrent(sl) c.waitOnStreamCurrent(sl, "$G", "TEST") } func TestJetStreamClusterStreamSnapshotCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } pseq := uint64(1) sendBatch := func(n int) { t.Helper() // Send a batch. for i := 0; i < n; i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", pseq)) if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } pseq++ } } sendBatch(2) sl := c.streamLeader("$G", "TEST") sl.Shutdown() c.waitOnStreamLeader("$G", "TEST") sendBatch(100) deleteMsg := func(seq uint64) { if err := js.DeleteMsg("TEST", seq); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Delete the first from the second batch. deleteMsg(pseq / 2) // Delete the next one too. deleteMsg(pseq/2 + 1) nsl := c.streamLeader("$G", "TEST") nsl.JetStreamSnapshotStream("$G", "TEST") // Do some activity post snapshot as well. // Delete next to last. deleteMsg(pseq - 2) // Send another batch. sendBatch(100) mset, err := nsl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) ostate := mset.stateWithDetail(true) sl = c.restartServer(sl) c.checkClusterFormed() c.waitOnServerCurrent(sl) c.waitOnStreamCurrent(sl, "$G", "TEST") mset, err = sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if nstate := mset.stateWithDetail(true); !reflect.DeepEqual(ostate, nstate) { return fmt.Errorf("States do not match after recovery: %+v vs %+v", ostate, nstate) } return nil }) } func TestJetStreamClusterDeleteMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // R=1 make sure delete works. _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 for i := 1; i <= toSend; i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", i)) if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } deleteMsg := func(seq uint64) { if err := js.DeleteMsg("TEST", seq); err != nil { t.Fatalf("Unexpected error: %v", err) } } deleteMsg(1) // Also make sure purge of R=1 works too. if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } } func TestJetStreamClusterDeleteMsgAndRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // R=1 make sure delete works. _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 for i := 1; i <= toSend; i++ { msg := []byte(fmt.Sprintf("HELLO JSC-%d", i)) if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } deleteMsg := func(seq uint64) { if err := js.DeleteMsg("TEST", seq); err != nil { t.Fatalf("Unexpected error: %v", err) } } deleteMsg(1) c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") } func TestJetStreamClusterStreamSnapshotCatchupWithPurge(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() s := c.randomServer() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sl := c.streamLeader("$G", "TEST") sl.Shutdown() c.waitOnStreamLeader("$G", "TEST") toSend := 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } nsl := c.streamLeader("$G", "TEST") if err := nsl.JetStreamSnapshotStream("$G", "TEST"); err != nil { t.Fatalf("Error snapshotting stream: %v", err) } time.Sleep(250 * time.Millisecond) sl = c.restartServer(sl) c.checkClusterFormed() // Now purge the stream while we are recovering. if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } c.waitOnServerCurrent(sl) c.waitOnStreamCurrent(sl, "$G", "TEST") nsl.Shutdown() c.waitOnStreamLeader("$G", "TEST") if _, err := js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterExtendedStreamInfo(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 50 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } leader := c.streamLeader("$G", "TEST").Name() si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil { t.Fatalf("Expected cluster info") } if si.Cluster.Name != c.name { t.Fatalf("Expected cluster name of %q, got %q", c.name, si.Cluster.Name) } if si.Cluster.Leader != leader { t.Fatalf("Expected leader of %q, got %q", leader, si.Cluster.Leader) } if len(si.Cluster.Replicas) != 2 { t.Fatalf("Expected %d replicas, got %d", 2, len(si.Cluster.Replicas)) } // Make sure that returned array is ordered for i := 0; i < 50; i++ { si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, len(si.Cluster.Replicas) == 2) s1 := si.Cluster.Replicas[0].Name s2 := si.Cluster.Replicas[1].Name if s1 > s2 { t.Fatalf("Expected replicas to be ordered, got %s then %s", s1, s2) } } // Faster timeout since we loop below checking for condition. js2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } // We may need to wait a bit for peers to catch up. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { for _, peer := range si.Cluster.Replicas { if !peer.Current { if si, err = js2.StreamInfo("TEST"); err != nil { t.Fatalf("Could not retrieve stream info") } return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) // Shutdown the leader. oldLeader := c.streamLeader("$G", "TEST") oldLeader.Shutdown() c.waitOnStreamLeader("$G", "TEST") // Re-request. leader = c.streamLeader("$G", "TEST").Name() si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil { t.Fatalf("Expected cluster info") } if si.Cluster.Leader != leader { t.Fatalf("Expected leader of %q, got %q", leader, si.Cluster.Leader) } if len(si.Cluster.Replicas) != 2 { t.Fatalf("Expected %d replicas, got %d", 2, len(si.Cluster.Replicas)) } for _, peer := range si.Cluster.Replicas { if peer.Name == oldLeader.Name() { if peer.Current { t.Fatalf("Expected old leader to be reported as not current: %+v", peer) } } else if !peer.Current { t.Fatalf("Expected replica to be current: %+v", peer) } } // Now send a few more messages then restart the oldLeader. for i := 0; i < 10; i++ { if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } oldLeader = c.restartServer(oldLeader) c.checkClusterFormed() c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamCurrent(oldLeader, "$G", "TEST") // Re-request. leader = c.streamLeader("$G", "TEST").Name() si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil { t.Fatalf("Expected cluster info") } if si.Cluster.Leader != leader { t.Fatalf("Expected leader of %q, got %q", leader, si.Cluster.Leader) } if len(si.Cluster.Replicas) != 2 { t.Fatalf("Expected %d replicas, got %d", 2, len(si.Cluster.Replicas)) } // We may need to wait a bit for peers to catch up. checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { for _, peer := range si.Cluster.Replicas { if !peer.Current { if si, err = js2.StreamInfo("TEST"); err != nil { t.Fatalf("Could not retrieve stream info") } return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Now do consumer. sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() fetchMsgs(t, sub, 10, 5*time.Second) leader = c.consumerLeader("$G", "TEST", "dlc").Name() ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } if ci.Cluster.Leader != leader { t.Fatalf("Expected leader of %q, got %q", leader, ci.Cluster.Leader) } if len(ci.Cluster.Replicas) != 2 { t.Fatalf("Expected %d replicas, got %d", 2, len(ci.Cluster.Replicas)) } checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { for _, peer := range si.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) } func TestJetStreamClusterExtendedStreamInfoSingleReplica(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 50 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } leader := c.streamLeader("$G", "TEST").Name() si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil { t.Fatalf("Expected cluster info") } if si.Cluster.Name != c.name { t.Fatalf("Expected cluster name of %q, got %q", c.name, si.Cluster.Name) } if si.Cluster.Leader != leader { t.Fatalf("Expected leader of %q, got %q", leader, si.Cluster.Leader) } if len(si.Cluster.Replicas) != 0 { t.Fatalf("Expected no replicas but got %d", len(si.Cluster.Replicas)) } // Make sure we can grab consumer lists from any var infos []*nats.ConsumerInfo for info := range js.ConsumersInfo("TEST") { infos = append(infos, info) } if len(infos) != 0 { t.Fatalf("ConsumerInfo expected no paged results, got %d", len(infos)) } // Now add in a consumer. cfg := &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy} if _, err := js.AddConsumer("TEST", cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } infos = infos[:0] for info := range js.ConsumersInfo("TEST") { infos = append(infos, info) } if len(infos) != 1 { t.Fatalf("ConsumerInfo expected 1 result, got %d", len(infos)) } // Now do direct names list as well. var names []string for name := range js.ConsumerNames("TEST") { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 consumer but got %d", len(names)) } } func TestJetStreamClusterInterestRetention(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Retention: nats.InterestPolicy, Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } sl := c.streamLeader("$G", "foo") cl := c.consumerLeader("$G", "foo", "dlc") if sl == cl { _, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "foo"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnStreamLeader("$G", "foo") } if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error getting msg: %v", err) } m.AckSync() waitForZero := func() { checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected 0 msgs, got state: %+v", si.State) } return nil }) } waitForZero() // Add in 50 messages. for i := 0; i < 50; i++ { if _, err = js.Publish("foo", []byte("more")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkSubsPending(t, sub, 50) // Now delete the consumer and make sure the stream goes to zero. if err := js.DeleteConsumer("foo", "dlc"); err != nil { t.Fatalf("Unexpected error: %v", err) } waitForZero() } // https://github.com/nats-io/nats-server/issues/2243 func TestJetStreamClusterWorkQueueRetention(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "FOO", Subjects: []string{"foo.*"}, Replicas: 2, Retention: nats.WorkQueuePolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.PullSubscribe("foo.test", "test") if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err = js.Publish("foo.test", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } si, err := js.StreamInfo("FOO") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 1 { t.Fatalf("Expected 1 msg, got state: %+v", si.State) } // Fetch from our pull consumer and ack. for _, m := range fetchMsgs(t, sub, 1, 5*time.Second) { m.AckSync() } // Make sure the messages are removed. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("FOO") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected 0 msgs, got state: %+v", si.State) } return nil }) } func TestJetStreamClusterMirrorAndSourceWorkQueues(t *testing.T) { c := createJetStreamClusterExplicit(t, "WQ", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "WQ22", Subjects: []string{"foo"}, Replicas: 2, Retention: nats.WorkQueuePolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 2, Mirror: &nats.StreamSource{Name: "WQ22"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Replicas: 2, Sources: []*nats.StreamSource{{Name: "WQ22"}}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Allow direct sync consumers to connect. time.Sleep(1500 * time.Millisecond) if _, err = js.Publish("foo", []byte("ok")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { if si, _ := js.StreamInfo("WQ22"); si.State.Msgs != 0 { return fmt.Errorf("Expected no msgs for %q, got %d", "WQ22", si.State.Msgs) } if si, _ := js.StreamInfo("M"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for %q, got %d", "M", si.State.Msgs) } if si, _ := js.StreamInfo("S"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for %q, got %d", "S", si.State.Msgs) } return nil }) } func TestJetStreamClusterMirrorAndSourceInterestPolicyStream(t *testing.T) { c := createJetStreamClusterExplicit(t, "WQ", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "IP22", Subjects: []string{"foo"}, Replicas: 3, Retention: nats.InterestPolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 2, Mirror: &nats.StreamSource{Name: "IP22"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Replicas: 2, Sources: []*nats.StreamSource{{Name: "IP22"}}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Allow sync consumers to connect. time.Sleep(1500 * time.Millisecond) if _, err = js.Publish("foo", []byte("ok")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { // This one will be 0 since no other interest exists. if si, _ := js.StreamInfo("IP22"); si.State.Msgs != 0 { return fmt.Errorf("Expected no msgs for %q, got %d", "IP22", si.State.Msgs) } if si, _ := js.StreamInfo("M"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for %q, got %d", "M", si.State.Msgs) } if si, _ := js.StreamInfo("S"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for %q, got %d", "S", si.State.Msgs) } return nil }) // Now create other interest on IP22. sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() // Allow consumer state to propagate. time.Sleep(500 * time.Millisecond) if _, err = js.Publish("foo", []byte("ok")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { // This one will be 0 since no other interest exists. if si, _ := js.StreamInfo("IP22"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for %q, got %d", "IP22", si.State.Msgs) } if si, _ := js.StreamInfo("M"); si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs for %q, got %d", "M", si.State.Msgs) } if si, _ := js.StreamInfo("S"); si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs for %q, got %d", "S", si.State.Msgs) } return nil }) } func TestJetStreamClusterInterestRetentionWithFilteredConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"*"}, Retention: nats.InterestPolicy, Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } fsub, err := js.SubscribeSync("foo", nats.Durable("d1")) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer fsub.Unsubscribe() bsub, err := js.SubscribeSync("bar", nats.Durable("d2")) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer bsub.Unsubscribe() msg := []byte("FILTERED") sendMsg := func(subj string) { t.Helper() if _, err = js.Publish(subj, msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } getAndAck := func(sub *nats.Subscription) { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error getting msg: %v", err) } m.AckSync() } jsq, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkState := func(expected uint64) { t.Helper() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { t.Helper() si, err := jsq.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != expected { return fmt.Errorf("Expected %d msgs, got %d", expected, si.State.Msgs) } return nil }) } sendMsg("foo") checkState(1) getAndAck(fsub) checkState(0) sendMsg("bar") sendMsg("foo") checkState(2) getAndAck(bsub) checkState(1) getAndAck(fsub) checkState(0) // Now send a bunch of messages and then delete the consumer. for i := 0; i < 10; i++ { sendMsg("foo") sendMsg("bar") } checkState(20) if err := js.DeleteConsumer("TEST", "d1"); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := js.DeleteConsumer("TEST", "d2"); err != nil { t.Fatalf("Unexpected error: %v", err) } checkState(0) // Now make sure pull based consumers work same. if _, err := js.PullSubscribe("foo", "dlc"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now send a bunch of messages and then delete the consumer. for i := 0; i < 10; i++ { sendMsg("foo") sendMsg("bar") } checkState(10) if err := js.DeleteConsumer("TEST", "dlc"); err != nil { t.Fatalf("Unexpected error: %v", err) } checkState(0) } func TestJetStreamClusterEphemeralConsumerNoImmediateInterest(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // We want to relax the strict interest requirement. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{DeliverSubject: "r"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } cl := c.consumerLeader("$G", "TEST", ci.Name) mset, err := cl.GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Expected to find a stream for %q", "TEST") } o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } o.setInActiveDeleteThreshold(500 * time.Millisecond) // Make sure the consumer goes away though eventually. // Should be 5 seconds wait. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if _, err := js.ConsumerInfo("TEST", ci.Name); err != nil { return nil } return fmt.Errorf("Consumer still present") }) } func TestJetStreamClusterEphemeralConsumerCleanup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 2}) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.Subscribe("foo", func(m *nats.Msg) {}) if err != nil { t.Fatalf("Unexpected error: %v", err) } ci, _ := sub.ConsumerInfo() if ci == nil { t.Fatalf("Unexpected error: no consumer info") } // We will look up by hand this consumer to set inactive threshold lower for this test. cl := c.consumerLeader("$G", "foo", ci.Name) if cl == nil { t.Fatalf("Could not find consumer leader") } mset, err := cl.GlobalAccount().lookupStream("foo") if err != nil { t.Fatalf("Expected to find a stream for %q", "foo") } o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } o.setInActiveDeleteThreshold(10 * time.Millisecond) msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } getConsumers := func() []string { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() var names []string for name := range js.ConsumerNames("foo", nats.Context(ctx)) { names = append(names, name) } return names } checkConsumer := func(expected int) { consumers := getConsumers() if len(consumers) != expected { t.Fatalf("Expected %d consumers but got %d", expected, len(consumers)) } } checkConsumer(1) // Now Unsubscribe, since this is ephemeral this will make this go away. sub.Unsubscribe() checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if consumers := getConsumers(); len(consumers) == 0 { return nil } else { return fmt.Errorf("Still %d consumers remaining", len(consumers)) } }) } func TestJetStreamClusterEphemeralConsumersNotReplicated(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } ci, _ := sub.ConsumerInfo() if ci == nil { t.Fatalf("Unexpected error: no consumer info") } if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkSubsPending(t, sub, 1) sub.NextMsg(0) if ci.Cluster == nil || len(ci.Cluster.Replicas) != 0 { t.Fatalf("Expected ephemeral to be R=1, got %+v", ci.Cluster) } scl := c.serverByName(ci.Cluster.Leader) if scl == nil { t.Fatalf("Could not select server where ephemeral consumer is running") } // Test migrations. If we are also metadata leader will not work so skip. if scl == c.leader() { return } scl.Shutdown() c.waitOnStreamLeader("$G", "foo") if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } if _, err := sub.NextMsg(500 * time.Millisecond); err != nil { t.Logf("Expected to see another message, but behavior is optimistic so can fail") } } func TestJetStreamClusterUserSnapshotAndRestore(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 200 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Create consumer with no state. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "rip", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Create another consumer as well and give it a non-simplistic state. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, AckWait: 10 * time.Second}) if err != nil { t.Fatalf("Unexpected error: %v", err) } jsub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Ack first 50. for _, m := range fetchMsgs(t, jsub, 50, 5*time.Second) { m.AckSync() } // Now ack every third message for next 50. for i, m := range fetchMsgs(t, jsub, 50, 5*time.Second) { if i%3 == 0 { m.AckSync() } } // Snapshot consumer info. ci, err := jsub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } sreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 512, } req, _ := json.Marshal(sreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } // Grab state for comparison. state := *resp.State config := *resp.Config var snapshot []byte done := make(chan bool) sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- true return } // Could be writing to a file here too. snapshot = append(snapshot, m.Data...) // Flow ack m.Respond(nil) }) defer sub.Unsubscribe() // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } var rresp JSApiStreamRestoreResponse rreq := &JSApiStreamRestoreRequest{ Config: config, State: state, } req, _ = json.Marshal(rreq) // Make sure a restore to an existing stream fails. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, "TEST"), req, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } json.Unmarshal(rmsg.Data, &rresp) if !IsNatsErr(rresp.Error, JSStreamNameExistRestoreFailedErr) { t.Fatalf("Did not get correct error response: %+v", rresp.Error) } if _, err := js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now make sure a restore will work. // Delete our stream first. if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.StreamInfo("TEST"); err == nil || !strings.Contains(err.Error(), "not found") { t.Fatalf("Expected not found error: %v", err) } // This should work properly. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, "TEST"), req, 5*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } if rresp.DeliverSubject == _EMPTY_ { t.Fatalf("No deliver subject set on response: %+v", rresp) } // Send our snapshot back in to restore the stream. // Can be any size message. var chunk [1024]byte for r := bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) } rmsg, err = nc.Request(rresp.DeliverSubject, nil, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" || si.State.Msgs != uint64(toSend) { t.Fatalf("StreamInfo is not correct %+v", si) } // Make sure the replicas become current eventually. They will be doing catchup. checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, _ := js.StreamInfo("TEST") if si == nil || si.Cluster == nil { t.Fatalf("Did not get stream info") } for _, pi := range si.Cluster.Replicas { if !pi.Current { return fmt.Errorf("Peer not current: %+v", pi) } } return nil }) // Wait on the system to elect a leader for the restored consumer. c.waitOnConsumerLeader("$G", "TEST", "dlc") // Now check for the consumer being recreated. nci, err := js.ConsumerInfo("TEST", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } // nil out timestamp for better comparison nci.Delivered.Last, ci.Delivered.Last = nil, nil if nci.Delivered != ci.Delivered { t.Fatalf("Delivered states do not match %+v vs %+v", nci.Delivered, ci.Delivered) } nci.AckFloor.Last, ci.AckFloor.Last = nil, nil if nci.AckFloor != ci.AckFloor { t.Fatalf("Ack floors did not match %+v vs %+v", nci.AckFloor, ci.AckFloor) } // Make sure consumer works. // It should pick up with the next delivery spot, so check for that as first message. // We should have all the messages for first delivery delivered. wantSeq := 101 for _, m := range fetchMsgs(t, jsub, 100, 5*time.Second) { meta, err := m.Metadata() if err != nil { t.Fatalf("Unexpected error: %v", err) } if meta.Sequence.Stream != uint64(wantSeq) { t.Fatalf("Expected stream sequence of %d, but got %d", wantSeq, meta.Sequence.Stream) } m.AckSync() wantSeq++ } // Check that redelivered come in now.. redelivered := 50/3 + 1 fetchMsgs(t, jsub, redelivered, 15*time.Second) // Now make sure the other server was properly caughtup. // Need to call this by hand for now. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var sdResp JSApiStreamLeaderStepDownResponse if err := json.Unmarshal(rmsg.Data, &sdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if sdResp.Error != nil { t.Fatalf("Unexpected error: %+v", sdResp.Error) } c.waitOnStreamLeader("$G", "TEST") si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %+v", err) } if si.State.Msgs != uint64(toSend) { t.Fatalf("Unexpected stream info: %+v", si) } // Check idle consumer c.waitOnConsumerLeader("$G", "TEST", "rip") // Now check for the consumer being recreated. if _, err := js.ConsumerInfo("TEST", "rip"); err != nil { t.Fatalf("Unexpected error: %+v", err) } } func TestJetStreamClusterUserSnapshotAndRestoreConfigChanges(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // FIXME(dlc) - Do case with R=1 cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 for i := 0; i < toSend; i++ { if _, err := js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } getSnapshot := func() ([]byte, *StreamState) { t.Helper() sreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 1024, } req, _ := json.Marshal(sreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } var snapshot []byte done := make(chan bool) sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- true return } // Could be writing to a file here too. snapshot = append(snapshot, m.Data...) // Flow ack m.Respond(nil) }) defer sub.Unsubscribe() // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } return snapshot, resp.State } restore := func(cfg *StreamConfig, state *StreamState, snap []byte) *nats.StreamInfo { rreq := &JSApiStreamRestoreRequest{ Config: *cfg, State: *state, } req, err := json.Marshal(rreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, cfg.Name), req, 5*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var rresp JSApiStreamRestoreResponse json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } if rresp.DeliverSubject == _EMPTY_ { t.Fatalf("No deliver subject set on response: %+v", rresp) } // Send our snapshot back in to restore the stream. // Can be any size message. var chunk [1024]byte for r := bytes.NewReader(snap); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) } rmsg, err = nc.Request(rresp.DeliverSubject, nil, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } si, err := js.StreamInfo(cfg.Name) if err != nil { t.Fatalf("Unexpected error: %v", err) } return si } snap, state := getSnapshot() if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now change subjects. ncfg := &StreamConfig{ Name: "TEST", Subjects: []string{"bar", "baz"}, Storage: FileStorage, Replicas: 2, } if si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) { t.Fatalf("Did not get expected stream info: %+v", si) } if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Storage ncfg.Storage = MemoryStorage if si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) { t.Fatalf("Did not get expected stream info: %+v", si) } if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now replicas ncfg.Replicas = 3 if si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) { t.Fatalf("Did not get expected stream info: %+v", si) } } func TestJetStreamClusterAccountInfoAndLimits(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() // Adjust our limits. c.updateLimits("$G", map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 1024, MaxStore: 8000, MaxStreams: 3, MaxConsumers: 2, }, }) // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "baz", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Create with same config is idempotent, and must not exceed max streams as it already exists. if _, err := js.AddStream(&nats.StreamConfig{Name: "baz", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch := func(subject string, n int) { t.Helper() for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } sendBatch("foo", 25) sendBatch("bar", 75) sendBatch("baz", 10) accountStats := func() *nats.AccountInfo { t.Helper() info, err := js.AccountInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } return info } // If subject is not 3 letters or payload not 2 this needs to change. const msgSize = uint64(22 + 3 + 6 + 8) stats := accountStats() if stats.Streams != 3 { t.Fatalf("Should have been tracking 3 streams, found %d", stats.Streams) } expectedSize := 25*msgSize + 75*msgSize*2 + 10*msgSize*3 // This may lag. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { if stats.Store != expectedSize { err := fmt.Errorf("Expected store size to be %d, got %+v\n", expectedSize, stats) stats = accountStats() return err } return nil }) // Check limit enforcement. if _, err := js.AddStream(&nats.StreamConfig{Name: "fail", Replicas: 3}); err == nil { t.Fatalf("Expected an error but got none") } // We should be at 7995 at the moment with a limit of 8000, so any message will go over. if _, err := js.Publish("baz", []byte("JSC-NOT-OK")); err == nil { t.Fatalf("Expected publish error but got none") } // Check consumers _, err := js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Create (with explicit create API) for the same consumer must be idempotent, and not trigger limit. obsReq := CreateConsumerRequest{ Stream: "foo", Config: ConsumerConfig{Durable: "bar"}, Action: ActionCreate, } req, err := json.Marshal(obsReq) require_NoError(t, err) msg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "foo", "bar"), req, time.Second) require_NoError(t, err) var resp JSApiConsumerInfoResponse require_NoError(t, json.Unmarshal(msg.Data, &resp)) if resp.Error != nil { t.Fatalf("Unexpected error: %v", resp.Error) } msg, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, "foo", "bar"), req, time.Second) require_NoError(t, err) var resp2 JSApiConsumerInfoResponse require_NoError(t, json.Unmarshal(msg.Data, &resp2)) if resp2.Error != nil { t.Fatalf("Unexpected error: %v", resp2.Error) } // This should fail. _, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dlc22", AckPolicy: nats.AckExplicitPolicy}) if err == nil { t.Fatalf("Expected error but got none") } } func TestJetStreamClusterMaxStreamsReached(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomNonLeader()) defer nc.Close() // Adjust our limits. c.updateLimits("$G", map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 1024, MaxStore: 1024, MaxStreams: 1, }, }) // Many stream creations in parallel for the same stream should not result in // maximum number of streams reached error. All should have a successful response. var wg sync.WaitGroup for i := 0; i < 15; i++ { wg.Add(1) go func() { defer wg.Done() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) }() } wg.Wait() require_NoError(t, js.DeleteStream("TEST")) // Adjust our limits. c.updateLimits("$G", map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 1024, MaxStore: 1024, MaxStreams: 2, }, }) // Setup streams beforehand. for d := 0; d < 2; d++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST-%d", d), Subjects: []string{fmt.Sprintf("foo.%d", d)}, Replicas: 1, }) require_NoError(t, err) } // Many stream creations in parallel for streams that already exist should not result in // maximum number of streams reached error. All should have a successful response. for i := 0; i < 15; i++ { wg.Add(1) d := i % 2 go func() { defer wg.Done() _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST-%d", d), Subjects: []string{fmt.Sprintf("foo.%d", d)}, Replicas: 1, }) require_NoError(t, err) }() } wg.Wait() } func TestJetStreamClusterStreamLimits(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Check that large R will fail. if _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 5}); err == nil { t.Fatalf("Expected error but got none") } maxMsgs := 5 _, err := js.AddStream(&nats.StreamConfig{ Name: "foo", Replicas: 3, Retention: nats.LimitsPolicy, Discard: DiscardNew, MaxMsgSize: 11, MaxMsgs: int64(maxMsgs), MaxAge: 250 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Large message should fail. if _, err := js.Publish("foo", []byte("0123456789ZZZ")); err == nil { t.Fatalf("Expected publish to fail") } for i := 0; i < maxMsgs; i++ { if _, err := js.Publish("foo", []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // These should fail. if _, err := js.Publish("foo", []byte("JSC-OK")); err == nil { t.Fatalf("Expected publish to fail") } // Make sure when space frees up we can send more. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected 0 msgs, got state: %+v", si.State) } return nil }) if _, err := js.Publish("foo", []byte("ROUND2")); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterStreamInterestOnlyPolicy(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "foo", Replicas: 3, Retention: nats.InterestPolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 // With no interest these should be no-ops. for i := 0; i < toSend; i++ { if _, err := js.Publish("foo", []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } si, err := js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { t.Fatalf("Expected no messages with no interest, got %d", si.State.Msgs) } // Now create a consumer. sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 0; i < toSend; i++ { if _, err := js.Publish("foo", []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkSubsPending(t, sub, toSend) si, err = js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages with interest, got %d", toSend, si.State.Msgs) } if si.State.FirstSeq != uint64(toSend+1) { t.Fatalf("Expected first sequence of %d, got %d", toSend+1, si.State.FirstSeq) } // Now delete the consumer. sub.Unsubscribe() // That should make it go away. if _, err := js.ConsumerInfo("foo", "dlc"); err == nil { t.Fatalf("Expected not found error, got none") } // Wait for the messages to be purged. checkFor(t, 5*time.Second, 20*time.Millisecond, func() error { si, err := js.StreamInfo("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs == 0 { return nil } return fmt.Errorf("Wanted 0 messages, got %d", si.State.Msgs) }) } // These are disabled for now. func TestJetStreamClusterStreamTemplates(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, _ := jsClientConnect(t, s) defer nc.Close() // List API var tListResp JSApiStreamTemplateNamesResponse resp, err := nc.Request(JSApiTemplates, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err := json.Unmarshal(resp.Data, &tListResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if tListResp.Error == nil { t.Fatalf("Expected an unsupported error, got none") } if !strings.Contains(tListResp.Error.Description, "not currently supported in clustered mode") { t.Fatalf("Did not get correct error response: %+v", tListResp.Error) } // Create // Now do templates. mcfg := &StreamConfig{ Subjects: []string{"kv.*"}, Storage: MemoryStorage, } template := &StreamTemplateConfig{ Name: "kv", Config: mcfg, MaxStreams: 4, } req, err := json.Marshal(template) if err != nil { t.Fatalf("Unexpected error: %v", err) } var stResp JSApiStreamTemplateCreateResponse resp, err = nc.Request(fmt.Sprintf(JSApiTemplateCreateT, template.Name), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err = json.Unmarshal(resp.Data, &stResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if stResp.Error == nil { t.Fatalf("Expected an unsupported error, got none") } if !strings.Contains(stResp.Error.Description, "not currently supported in clustered mode") { t.Fatalf("Did not get correct error response: %+v", stResp.Error) } } func TestJetStreamClusterExtendedAccountInfo(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sendBatch := func(subject string, n int) { t.Helper() for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } // Add in some streams with msgs and consumers. if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-1", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-1"); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-1", 25) if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-2", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-2"); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-2", 50) if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-3", Replicas: 3, Storage: nats.MemoryStorage}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-3"); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-3", 100) // Go client will lag so use direct for now. getAccountInfo := func() *nats.AccountInfo { t.Helper() info, err := js.AccountInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } return info } // Wait to accumulate. time.Sleep(500 * time.Millisecond) ai := getAccountInfo() if ai.Streams != 3 || ai.Consumers != 3 { t.Fatalf("AccountInfo not correct: %+v", ai) } if ai.API.Total < 7 { t.Fatalf("Expected at least 7 total API calls, got %d", ai.API.Total) } // Now do a failure to make sure we track API errors. js.StreamInfo("NO-STREAM") js.ConsumerInfo("TEST-1", "NO-CONSUMER") js.ConsumerInfo("TEST-2", "NO-CONSUMER") js.ConsumerInfo("TEST-3", "NO-CONSUMER") ai = getAccountInfo() if ai.API.Errors != 4 { t.Fatalf("Expected 4 API calls to be errors, got %d", ai.API.Errors) } } func TestJetStreamClusterPeerRemovalAPI(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() // Client based API ml := c.leader() nc, err := nats.Connect(ml.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() // Expect error if unknown peer req := &JSApiMetaServerRemoveRequest{Server: "S-9"} jsreq, err := json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMetaServerRemoveResponse if err := json.Unmarshal(rmsg.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error == nil { t.Fatalf("Expected an error, got none") } sub, err := nc.SubscribeSync(JSAdvisoryServerRemoved) if err != nil { t.Fatalf("Unexpected error: %v", err) } rs := c.randomNonLeader() req = &JSApiMetaServerRemoveRequest{Server: rs.Name()} jsreq, err = json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err = nc.Request(JSApiRemoveServer, jsreq, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp.Error = nil if err := json.Unmarshal(rmsg.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } c.waitOnLeader() ml = c.leader() checkSubsPending(t, sub, 1) madv, _ := sub.NextMsg(0) var adv JSServerRemovedAdvisory if err := json.Unmarshal(madv.Data, &adv); err != nil { t.Fatalf("Unexpected error: %v", err) } if adv.Server != rs.Name() { t.Fatalf("Expected advisory about %s being removed, got %+v", rs.Name(), adv) } checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { for _, s := range ml.JetStreamClusterPeers() { if s == rs.Name() { return fmt.Errorf("Still in the peer list") } } return nil }) } func TestJetStreamClusterPeerRemovalAndStreamReassignment(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() // Client based API s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Admin based API ml := c.leader() nc, err = nats.Connect(ml.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() // Select the non-leader server for the stream to remove. if len(si.Cluster.Replicas) < 2 { t.Fatalf("Not enough replicas found: %+v", si.Cluster) } toRemove, cl := si.Cluster.Replicas[0].Name, c.leader() if toRemove == cl.Name() { toRemove = si.Cluster.Replicas[1].Name } req := &JSApiMetaServerRemoveRequest{Server: toRemove} jsreq, err := json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMetaServerRemoveResponse if err := json.Unmarshal(rmsg.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } // In case that server was also meta-leader. c.waitOnLeader() checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { for _, s := range ml.JetStreamClusterPeers() { if s == toRemove { return fmt.Errorf("Server still in the peer list") } } return nil }) // Now wait until the stream is now current. checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } // We should not see the old server at all. for _, p := range si.Cluster.Replicas { if p.Name == toRemove { t.Fatalf("Peer not removed yet: %+v", toRemove) } if !p.Current { return fmt.Errorf("Expected replica to be current: %+v", p) } } if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(si.Cluster.Replicas)) } return nil }) } func TestJetStreamClusterPeerRemovalAndStreamReassignmentWithoutSpace(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Admin based API ml := c.leader() nc, err = nats.Connect(ml.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() // Select the non-leader server for the stream to remove. if len(si.Cluster.Replicas) < 2 { t.Fatalf("Not enough replicas found: %+v", si.Cluster) } toRemove, cl := si.Cluster.Replicas[0].Name, c.leader() if toRemove == cl.Name() { toRemove = si.Cluster.Replicas[1].Name } req := &JSApiMetaServerRemoveRequest{Server: toRemove} jsreq, err := json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err := nc.Request(JSApiRemoveServer, jsreq, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMetaServerRemoveResponse if err := json.Unmarshal(rmsg.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { for _, s := range ml.JetStreamClusterPeers() { if s == toRemove { return fmt.Errorf("Server still in the peer list") } } return nil }) // Make sure only 2 peers at this point. c.waitOnPeerCount(2) // Now wait until the stream is now current. streamCurrent := func(nr int) { checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } // We should not see the old server at all. for _, p := range si.Cluster.Replicas { if p.Name == toRemove { return fmt.Errorf("Peer not removed yet: %+v", toRemove) } if !p.Current { return fmt.Errorf("Expected replica to be current: %+v", p) } } if len(si.Cluster.Replicas) != nr { return fmt.Errorf("Expected %d replicas, got %d", nr, len(si.Cluster.Replicas)) } return nil }) } // Make sure the peer was removed from the stream and that we did not fill the new spot. streamCurrent(1) // Now add in a new server and make sure it gets added to our stream. c.addInNewServer() c.waitOnPeerCount(3) streamCurrent(2) } func TestJetStreamClusterPeerExclusionTag(t *testing.T) { c := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, "C", 3, func(serverName, clusterName, storeDir, conf string) string { switch serverName { case "S-1": return fmt.Sprintf("%s\nserver_tags: [server:%s, intersect, %s]", conf, serverName, jsExcludePlacement) case "S-2": return fmt.Sprintf("%s\nserver_tags: [server:%s, intersect]", conf, serverName) default: return fmt.Sprintf("%s\nserver_tags: [server:%s]", conf, serverName) } }) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for i, c := range []nats.StreamConfig{ {Replicas: 1, Placement: &nats.Placement{Tags: []string{"server:S-1"}}}, {Replicas: 2, Placement: &nats.Placement{Tags: []string{"intersect"}}}, {Replicas: 3}, // not enough server without !jetstream } { c.Name = fmt.Sprintf("TEST%d", i) c.Subjects = []string{c.Name} _, err := js.AddStream(&c) require_Error(t, err) require_Contains(t, err.Error(), "no suitable peers for placement", "exclude tag set") } // Test update failure cfg := &nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}, Replicas: 2} _, err := js.AddStream(cfg) require_NoError(t, err) cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_Error(t, err) require_Contains(t, err.Error(), "no suitable peers for placement", "exclude tag set") // Test tag reload removing !jetstream tag, and allowing placement again srv := c.serverByName("S-1") v, err := srv.Varz(nil) require_NoError(t, err) require_True(t, v.Tags.Contains(jsExcludePlacement)) content, err := os.ReadFile(srv.configFile) require_NoError(t, err) newContent := strings.ReplaceAll(string(content), fmt.Sprintf(", %s]", jsExcludePlacement), "]") changeCurrentConfigContentWithNewContent(t, srv.configFile, []byte(newContent)) ncSys := natsConnect(t, c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) defer ncSys.Close() sub, err := ncSys.SubscribeSync(fmt.Sprintf("$SYS.SERVER.%s.STATSZ", srv.ID())) require_NoError(t, err) require_NoError(t, srv.Reload()) v, err = srv.Varz(nil) require_NoError(t, err) require_True(t, !v.Tags.Contains(jsExcludePlacement)) // it is possible that sub already received a stasz message prior to reload, retry once cmp := false for i := 0; i < 2 && !cmp; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) cmp = strings.Contains(string(m.Data), `"tags":["server:s-1","intersect"]`) } require_True(t, cmp) cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) } func TestJetStreamClusterAccountPurge(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, accpub := createKey(t) accClaim := jwt.NewAccountClaims(accpub) accClaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 * 5 accClaim.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 * 5 accJwt := encodeClaim(t, accClaim, accpub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, func(serverName, clustername, storeDir, conf string) string { return conf + fmt.Sprintf(` operator: %s system_account: %s resolver: { type: full dir: '%s/jwt' timeout: "10ms" }`, ojwt, syspub, storeDir) }) defer c.shutdown() c.waitOnLeader() updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt, 3) c.waitOnAccount(accpub) createTestData := func(t *testing.T) { nc, js := jsClientConnect(t, c.randomNonLeader(), nats.UserCredentials(accCreds)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(accpub, "TEST1") ci, err := js.AddConsumer("TEST1", &nats.ConsumerConfig{Durable: "DUR1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) require_True(t, ci.Config.Replicas == 0) ci, err = js.AddConsumer("TEST1", &nats.ConsumerConfig{Durable: "DUR2", AckPolicy: nats.AckExplicitPolicy, Replicas: 1}) require_NoError(t, err) require_True(t, ci.Config.Replicas == 1) toSend := uint64(1_000) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"bar"}, Replicas: 1, }) require_NoError(t, err) ci, err = js.AddConsumer("TEST2", &nats.ConsumerConfig{Durable: "DUR1", AckPolicy: nats.AckExplicitPolicy, Replicas: 0}) require_NoError(t, err) require_True(t, ci.Config.Replicas == 0) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("bar", nil) require_NoError(t, err) } } inspectDirs := func(t *testing.T, sysTotal, accTotal int) error { t.Helper() sysDirs := 0 accDirs := 0 for _, s := range c.servers { files, err := os.ReadDir(filepath.Join(s.getOpts().StoreDir, "jetstream", syspub, "_js_")) require_NoError(t, err) sysDirs += len(files) - 1 // sub 1 for _meta_ files, err = os.ReadDir(filepath.Join(s.getOpts().StoreDir, "jetstream", accpub, "streams")) if err == nil || err.(*os.PathError).Error() == "no such file or directory" { accDirs += len(files) } } if sysDirs != sysTotal || accDirs != accTotal { return fmt.Errorf("expected directory count does not match %d == %d, %d == %d", sysDirs, sysTotal, accDirs, accTotal) } return nil } checkForDirs := func(t *testing.T, sysTotal, accTotal int) { t.Helper() checkFor(t, 20*time.Second, 250*time.Millisecond, func() error { return inspectDirs(t, sysTotal, accTotal) }) } purge := func(t *testing.T) { t.Helper() ncsys, err := nats.Connect(c.randomServer().ClientURL(), nats.UserCredentials(sysCreds)) require_NoError(t, err) defer ncsys.Close() request := func() error { var resp JSApiAccountPurgeResponse m, err := ncsys.Request(fmt.Sprintf(JSApiAccountPurgeT, accpub), nil, time.Second) if err != nil { return err } if err := json.Unmarshal(m.Data, &resp); err != nil { return err } if !resp.Initiated { return fmt.Errorf("not started") } return nil } checkFor(t, 30*time.Second, 250*time.Millisecond, request) } t.Run("startup-cleanup", func(t *testing.T) { _, newCleanupAcc1 := createKey(t) _, newCleanupAcc2 := createKey(t) for _, s := range c.servers { os.MkdirAll(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc1, streamsDir), defaultDirPerms) os.MkdirAll(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc2), defaultDirPerms) } createTestData(t) checkForDirs(t, 6, 4) c.stopAll() c.restartAll() for _, s := range c.servers { accDir := filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc1) _, e := os.Stat(filepath.Join(accDir, streamsDir)) require_Error(t, e) require_True(t, os.IsNotExist(e)) _, e = os.Stat(accDir) require_Error(t, e) require_True(t, os.IsNotExist(e)) _, e = os.Stat(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc2)) require_Error(t, e) require_True(t, os.IsNotExist(e)) } checkForDirs(t, 6, 4) // Make sure we have a leader for all assets before moving to the next test c.waitOnStreamLeader(accpub, "TEST1") c.waitOnConsumerLeader(accpub, "TEST1", "DUR1") c.waitOnConsumerLeader(accpub, "TEST1", "DUR2") c.waitOnStreamLeader(accpub, "TEST2") c.waitOnConsumerLeader(accpub, "TEST2", "DUR1") }) t.Run("purge-with-restart", func(t *testing.T) { createTestData(t) checkForDirs(t, 6, 4) purge(t) checkForDirs(t, 0, 0) c.stopAll() c.restartAll() checkForDirs(t, 0, 0) }) t.Run("purge-with-reuse", func(t *testing.T) { createTestData(t) checkForDirs(t, 6, 4) purge(t) checkForDirs(t, 0, 0) createTestData(t) checkForDirs(t, 6, 4) purge(t) checkForDirs(t, 0, 0) }) t.Run("purge-deleted-account", func(t *testing.T) { createTestData(t) checkForDirs(t, 6, 4) c.stopAll() for _, s := range c.servers { require_NoError(t, os.Remove(s.getOpts().StoreDir+"/jwt/"+accpub+".jwt")) } c.restartAll() checkForDirs(t, 6, 4) purge(t) checkForDirs(t, 0, 0) c.stopAll() c.restartAll() checkForDirs(t, 0, 0) }) } func TestJetStreamClusterScaleConsumer(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterTempl, "C", 3) defer c.shutdown() srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) durCfg := &nats.ConsumerConfig{Durable: "DUR", AckPolicy: nats.AckExplicitPolicy} ci, err := js.AddConsumer("TEST", durCfg) require_NoError(t, err) require_True(t, ci.Config.Replicas == 0) toSend := uint64(1_000) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } s, err := js.PullSubscribe("foo", "DUR") require_NoError(t, err) consumeOne := func(expSeq uint64) error { if ci, err := js.ConsumerInfo("TEST", "DUR"); err != nil { return err } else if ci.Delivered.Stream != expSeq { return fmt.Errorf("pre: not expected delivered stream %d, got %d", expSeq, ci.Delivered.Stream) } else if ci.Delivered.Consumer != expSeq { return fmt.Errorf("pre: not expected delivered consumer %d, got %d", expSeq, ci.Delivered.Consumer) } else if ci.AckFloor.Stream != expSeq { return fmt.Errorf("pre: not expected ack stream %d, got %d", expSeq, ci.AckFloor.Stream) } else if ci.AckFloor.Consumer != expSeq { return fmt.Errorf("pre: not expected ack consumer %d, got %d", expSeq, ci.AckFloor.Consumer) } if m, err := s.Fetch(1); err != nil { return err } else if err := m[0].AckSync(); err != nil { return err } expSeq = expSeq + 1 if ci, err := js.ConsumerInfo("TEST", "DUR"); err != nil { return err } else if ci.Delivered.Stream != expSeq { return fmt.Errorf("post: not expected delivered stream %d, got %d", expSeq, ci.Delivered.Stream) } else if ci.Delivered.Consumer != expSeq { return fmt.Errorf("post: not expected delivered consumer %d, got %d", expSeq, ci.Delivered.Consumer) } else if ci.AckFloor.Stream != expSeq { return fmt.Errorf("post: not expected ack stream %d, got %d", expSeq, ci.AckFloor.Stream) } else if ci.AckFloor.Consumer != expSeq { return fmt.Errorf("post: not expected ack consumer %d, got %d", expSeq, ci.AckFloor.Consumer) } return nil } require_NoError(t, consumeOne(0)) // scale down, up, down and up to default == 3 again for i, r := range []int{1, 3, 1, 0} { durCfg.Replicas = r if r == 0 { r = si.Config.Replicas } js.UpdateConsumer("TEST", durCfg) checkFor(t, time.Second*30, time.Millisecond*250, func() error { if ci, err = js.ConsumerInfo("TEST", "DUR"); err != nil { return err } else if ci.Cluster == nil { return errors.New("no cluster info") } else if ci.Cluster.Leader == _EMPTY_ { return errors.New("no leader") } else if len(ci.Cluster.Replicas) != r-1 { return fmt.Errorf("not enough replica, got %d wanted %d", len(ci.Cluster.Replicas), r-1) } else { for _, r := range ci.Cluster.Replicas { if !r.Current || r.Offline || r.Lag != 0 { return fmt.Errorf("replica %s not current %t offline %t lag %d", r.Name, r.Current, r.Offline, r.Lag) } } } return nil }) require_NoError(t, consumeOne(uint64(i+1))) } } func TestJetStreamClusterConsumerScaleUp(t *testing.T) { c := createJetStreamCluster(t, jsClusterTempl, "HUB", _EMPTY_, 3, 22020, true) defer c.shutdown() // Client based API srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() scfg := nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, } _, err := js.AddStream(&scfg) require_NoError(t, err) defer js.DeleteStream("TEST") dcfg := nats.ConsumerConfig{ Durable: "DUR", AckPolicy: nats.AckExplicitPolicy, Replicas: 0} _, err = js.AddConsumer("TEST", &dcfg) require_NoError(t, err) for i := 0; i < 100; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } scfg.Replicas = 2 _, err = js.UpdateStream(&scfg) require_NoError(t, err) // The scale up issue shows itself as permanent loss of consumer leadership // So give it some time for the change to propagate to new consumer peers and the quorum to disrupt // 2 seconds is a value arrived by experimentally, no sleep or a sleep of 1sec always had the test pass a lot. time.Sleep(2 * time.Second) c.waitOnStreamLeader("$G", "TEST") // There is also a timing component to the issue triggering. c.waitOnConsumerLeader("$G", "TEST", "DUR") } func TestJetStreamClusterPeerOffline(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() ml := c.leader() rs := c.randomNonLeader() checkPeer := func(ml, rs *Server, shouldBeOffline bool) { t.Helper() checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { var found bool for _, s := range ml.JetStreamClusterPeers() { if s == rs.Name() { found = true break } } if !shouldBeOffline && !found { return fmt.Errorf("Server %q not in the peers list", rs.Name()) } else if shouldBeOffline && found { return fmt.Errorf("Server %q should not be in the peers list", rs.Name()) } var ok bool ml.nodeToInfo.Range(func(k, v any) bool { if si := v.(nodeInfo); si.name == rs.Name() { if shouldBeOffline && si.offline || !shouldBeOffline && !si.offline { ok = true return false } } return true }) if !ok { if shouldBeOffline { return fmt.Errorf("Server %q should be marked as online", rs.Name()) } return fmt.Errorf("Server %q is still marked as online", rs.Name()) } return nil }) } // Shutdown the server and make sure that it is now showing as offline. rs.Shutdown() checkPeer(ml, rs, true) // Now restart that server and check that is no longer offline. oldrs := rs rs, _ = RunServerWithConfig(rs.getOpts().ConfigFile) defer rs.Shutdown() // Replaced old with new server for i := 0; i < len(c.servers); i++ { if c.servers[i] == oldrs { c.servers[i] = rs } } // Wait for cluster to be formed checkClusterFormed(t, c.servers...) // Make sure that we have a leader (there can always be a re-election) c.waitOnLeader() ml = c.leader() // Now check that rs is not offline checkPeer(ml, rs, false) } func TestJetStreamClusterNoQuorumStepdown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Setup subscription for leader elected. lesub, err := nc.SubscribeSync(JSAdvisoryStreamLeaderElectedPre + ".*") if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "NO-Q", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we received our leader elected advisory. leadv, _ := lesub.NextMsg(0) if leadv == nil { t.Fatalf("Expected to receive a leader elected advisory") } var le JSStreamLeaderElectedAdvisory if err := json.Unmarshal(leadv.Data, &le); err != nil { t.Fatalf("Unexpected error: %v", err) } if ln := c.streamLeader("$G", "NO-Q").Name(); le.Leader != ln { t.Fatalf("Expected to have leader %q in elect advisory, got %q", ln, le.Leader) } payload := []byte("Hello JSC") for i := 0; i < 10; i++ { if _, err := js.Publish("NO-Q", payload); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Setup subscription for leader elected. clesub, err := nc.SubscribeSync(JSAdvisoryConsumerLeaderElectedPre + ".*.*") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make durable to have R match Stream. sub, err := js.SubscribeSync("NO-Q", nats.Durable("rr")) if err != nil { t.Fatalf("Unexpected error: %v", err) } ci, err := sub.ConsumerInfo() if err != nil || ci == nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we received our consumer leader elected advisory. leadv, _ = clesub.NextMsg(0) if leadv == nil { t.Fatalf("Expected to receive a consumer leader elected advisory") } // Shutdown the non-leader. c.randomNonStreamLeader("$G", "NO-Q").Shutdown() // This should eventually have us stepdown as leader since we would have lost quorum with R=2. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { if sl := c.streamLeader("$G", "NO-Q"); sl == nil { return nil } return fmt.Errorf("Still have leader for stream") }) notAvailableErr := func(err error) bool { return err != nil && (strings.Contains(err.Error(), "unavailable") || err == context.DeadlineExceeded) } checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if cl := c.consumerLeader("$G", "NO-Q", ci.Name); cl == nil { return nil } return fmt.Errorf("Still have leader for consumer") }) if _, err = js.ConsumerInfo("NO-Q", ci.Name); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if _, err := sub.ConsumerInfo(); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } // Now let's take out the other non meta-leader // We should get same error for general API calls. c.randomNonLeader().Shutdown() c.expectNoLeader() // Now make sure the general JS API responds with system unavailable. if _, err = js.AccountInfo(); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "NO-Q33", Replicas: 2}); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if _, err := js.UpdateStream(&nats.StreamConfig{Name: "NO-Q33", Replicas: 2}); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if err := js.DeleteStream("NO-Q"); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if err := js.PurgeStream("NO-Q"); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if err := js.DeleteMsg("NO-Q", 1); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } // Consumer if _, err := js.AddConsumer("NO-Q", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if err := js.DeleteConsumer("NO-Q", "dlc"); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } if _, err := js.ConsumerInfo("NO-Q", "dlc"); !notAvailableErr(err) { t.Fatalf("Expected an 'unavailable' error, got %v", err) } // Listers for info := range js.StreamsInfo() { t.Fatalf("Unexpected stream info, got %v", info) } for info := range js.ConsumersInfo("NO-Q") { t.Fatalf("Unexpected consumer info, got %v", info) } } func TestJetStreamClusterCreateResponseAdvisoriesHaveSubject(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.API") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST", nats.Durable("DLC")); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, 6) for m, err := sub.NextMsg(0); err == nil; m, err = sub.NextMsg(0) { var audit JSAPIAudit if err := json.Unmarshal(m.Data, &audit); err != nil { t.Fatalf("Unexpected error: %v", err) } if audit.Subject == _EMPTY_ { t.Fatalf("Expected subject, got nothing") } } } func TestJetStreamClusterRestartAndRemoveAdvisories(t *testing.T) { // FIXME(dlc) - Flaky on Travis, skip for now. skip(t) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.API") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() csub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.*.CREATED.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer csub.Unsubscribe() nc.Flush() sendBatch := func(subject string, n int) { t.Helper() for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("JSC-OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } // Add in some streams with msgs and consumers. if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-1", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-1", nats.Durable("DC")); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-1", 25) if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-2", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-2", nats.Durable("DC")); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-2", 50) if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST-3", Replicas: 3, Storage: nats.MemoryStorage}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("TEST-3", nats.Durable("DC")); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("TEST-3", 100) drainSub := func(sub *nats.Subscription) { for _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) { } } // Wait for the advisories for all streams and consumers. checkSubsPending(t, sub, 12) // 3 streams, 3*2 consumers, 3 stream names lookups for creating consumers. drainSub(sub) // Created audit events. checkSubsPending(t, csub, 6) drainSub(csub) usub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.*.UPDATED.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer usub.Unsubscribe() nc.Flush() checkSubsPending(t, csub, 0) checkSubsPending(t, sub, 0) checkSubsPending(t, usub, 0) // Now restart the other two servers we are not connected to. for _, cs := range c.servers { if cs != s { cs.Shutdown() c.restartServer(cs) } } c.waitOnAllCurrent() checkSubsPending(t, csub, 0) checkSubsPending(t, sub, 0) checkSubsPending(t, usub, 0) dsub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.*.DELETED.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer dsub.Unsubscribe() nc.Flush() c.waitOnConsumerLeader("$G", "TEST-1", "DC") c.waitOnLeader() // Now check delete advisories as well. if err := js.DeleteConsumer("TEST-1", "DC"); err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, csub, 0) checkSubsPending(t, dsub, 1) checkSubsPending(t, sub, 1) checkSubsPending(t, usub, 0) drainSub(dsub) if err := js.DeleteStream("TEST-3"); err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, dsub, 2) // Stream and the consumer underneath. checkSubsPending(t, sub, 2) } func TestJetStreamClusterNoDuplicateOnNodeRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "ND", 2) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sl := c.streamLeader("$G", "TEST") if s == sl { nc.Close() nc, js = jsClientConnect(t, s) defer nc.Close() } sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } js.Publish("foo", []byte("msg1")) if m, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else { m.AckSync() } sl.Shutdown() c.restartServer(sl) c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "dlc") // Send second msg js.Publish("foo", []byte("msg2")) msg, err := sub.NextMsg(5 * time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } if string(msg.Data) != "msg2" { t.Fatalf("Unexpected message: %s", msg.Data) } msg.AckSync() // Make sure we don't get a duplicate. msg, err = sub.NextMsg(250 * time.Millisecond) if err == nil { t.Fatalf("Should have gotten an error, got %s", msg.Data) } } func TestJetStreamClusterNoDupePeerSelection(t *testing.T) { c := createJetStreamClusterExplicit(t, "NDP", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Create 10 streams. Make sure none of them have a replica // that is the same as the leader. for i := 1; i <= 10; i++ { si, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST-%d", i), Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil || si.Cluster.Leader == "" || len(si.Cluster.Replicas) != 2 { t.Fatalf("Unexpected cluster state for stream info: %+v\n", si.Cluster) } // Make sure that the replicas are not same as the leader. for _, pi := range si.Cluster.Replicas { if pi.Name == si.Cluster.Leader { t.Fatalf("Found replica that is same as leader, meaning 2 nodes placed on same server") } } // Now do a consumer and check same thing. sub, err := js.SubscribeSync(si.Config.Name) if err != nil { t.Fatalf("Unexpected error: %v", err) } ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } for _, pi := range ci.Cluster.Replicas { if pi.Name == ci.Cluster.Leader { t.Fatalf("Found replica that is same as leader, meaning 2 nodes placed on same server") } } } } func TestJetStreamClusterStreamRemovePeer(t *testing.T) { c := createJetStreamClusterExplicit(t, "RNS", 5) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("TEST", nats.Durable("cat")) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) // Do ephemeral too. esub, err := js.SubscribeSync("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, esub, toSend) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "pull", AckPolicy: nats.AckNonePolicy}) require_NoError(t, err) pullSub, err := js.PullSubscribe("TEST", "pull") require_NoError(t, err) // First fetch the messages that are already there. msgs, err := pullSub.Fetch(toSend, nats.MaxWait(500*time.Millisecond)) require_NoError(t, err) require_Equal(t, toSend, len(msgs)) // Now prepare a check to see if we get unwated `Consumer Deleted` error on peer remove. pullResults := make(chan error, 1) go func() { _, err := pullSub.Fetch(1, nats.MaxWait(30*time.Second)) // Let's check if we get unwted `Consumer Deleted` error on peer remove. // Everything else is fine (Leader Changed, Timeout, etc.) if err != nats.ErrConsumerDeleted { close(pullResults) } else { pullResults <- err } }() ci, err := esub.ConsumerInfo() if err != nil { t.Fatalf("Could not fetch consumer info: %v", err) } // Capture ephemeral's server and name. es, en := ci.Cluster.Leader, ci.Name // Grab stream info. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } peers := []string{si.Cluster.Leader} for _, p := range si.Cluster.Replicas { peers = append(peers, p.Name) } // Pick a truly random server to remove. rand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] }) toRemove := peers[0] // First test bad peer. req := &JSApiStreamRemovePeerRequest{Peer: "NOT VALID"} jsreq, err := json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Need to call this by hand for now. resp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "TEST"), jsreq, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var rpResp JSApiStreamRemovePeerResponse if err := json.Unmarshal(resp.Data, &rpResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if rpResp.Error == nil || !strings.Contains(rpResp.Error.Description, "peer not a member") { t.Fatalf("Expected error for bad peer, got %+v", rpResp.Error) } rpResp.Error = nil req = &JSApiStreamRemovePeerRequest{Peer: toRemove} jsreq, err = json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err = nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "TEST"), jsreq, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err := json.Unmarshal(resp.Data, &rpResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if rpResp.Error != nil { t.Fatalf("Unexpected error: %+v", rpResp.Error) } c.waitOnStreamLeader("$G", "TEST") checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(si.Cluster.Replicas)) } for _, peer := range si.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } if si.Cluster.Leader == toRemove { return fmt.Errorf("Peer not removed yet: %+v", toRemove) } for _, p := range si.Cluster.Replicas { if p.Name == toRemove { return fmt.Errorf("Peer not removed yet: %+v", toRemove) } } return nil }) c.waitOnConsumerLeader("$G", "TEST", "cat") c.waitOnConsumerLeader("$G", "TEST", "pull") // Now check consumer info as well. checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "cat", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("Could not fetch consumer info: %v", err) } if len(ci.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(ci.Cluster.Replicas)) } for _, peer := range ci.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } if ci.Cluster.Leader == toRemove { return fmt.Errorf("Peer not removed yet: %+v", toRemove) } for _, p := range ci.Cluster.Replicas { if p.Name == toRemove { return fmt.Errorf("Peer not removed yet: %+v", toRemove) } } return nil }) // Check if we got the `Consumer Deleted` error on the pull consumer. select { case err := <-pullResults: if err != nil { t.Fatalf("Expected timeout error or nil, got %v", err) } default: } // Now check ephemeral consumer info. // Make sure we did not stamp same new group into the ephemeral where R=1. ci, err = esub.ConsumerInfo() // If the leader was same as what we just removed, this should fail. if es == toRemove { if err != nats.ErrConsumerNotFound { t.Fatalf("Expected a not found error, got %v", err) } // Also make sure this was removed all together. // We may proactively move things in the future. for cn := range js.ConsumerNames("TEST") { if cn == en { t.Fatalf("Expected ephemeral consumer to be deleted since we removed its only peer") } } } else { if err != nil { t.Fatalf("Could not fetch consumer info: %v", err) } if len(ci.Cluster.Replicas) != 0 { t.Fatalf("Expected no replicas for ephemeral, got %d", len(ci.Cluster.Replicas)) } } } func TestJetStreamClusterStreamLeaderStepDown(t *testing.T) { c := createJetStreamClusterExplicit(t, "RNS", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("TEST", nats.Durable("cat")) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() oldLeader := c.streamLeader("$G", "TEST").Name() // Need to call this by hand for now. resp, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var sdResp JSApiStreamLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &sdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if sdResp.Error != nil { t.Fatalf("Unexpected error: %+v", sdResp.Error) } // Grab shorter timeout jetstream context. js, err = nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } if si.Cluster.Leader == oldLeader { return fmt.Errorf("Still have old leader") } if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(si.Cluster.Replicas)) } for _, peer := range si.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) // Now do consumer. oldLeader = c.consumerLeader("$G", "TEST", "cat").Name() // Need to call this by hand for now. resp, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "cat"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cdResp JSApiConsumerLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if cdResp.Error != nil { t.Fatalf("Unexpected error: %+v", cdResp.Error) } checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "cat") if err != nil { return fmt.Errorf("Could not fetch consumer info: %v", err) } if ci.Cluster.Leader == oldLeader { return fmt.Errorf("Still have old leader") } if len(ci.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(ci.Cluster.Replicas)) } for _, peer := range ci.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) } func TestJetStreamClusterRemoveServer(t *testing.T) { skip(t) c := createJetStreamClusterExplicit(t, "RNS", 5) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } cname := ci.Name sl := c.streamLeader("$G", "TEST") c.removeJetStream(sl) c.waitOnLeader() c.waitOnStreamLeader("$G", "TEST") // Faster timeout since we loop below checking for condition. js, err = nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Check the stream info is eventually correct. checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(si.Cluster.Replicas)) } for _, peer := range si.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) // Now do consumer. c.waitOnConsumerLeader("$G", "TEST", cname) checkFor(t, 20*time.Second, 50*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", cname) if err != nil { return fmt.Errorf("Could not fetch consumer info: %v", err) } if len(ci.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(ci.Cluster.Replicas)) } for _, peer := range ci.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } } return nil }) } func TestJetStreamClusterPurgeReplayAfterRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "P3F", 3) defer c.shutdown() // Client based API s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch := func(n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } sendBatch(10) if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } sendBatch(10) c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") s = c.randomServer() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 10 { t.Fatalf("Expected 10 msgs after restart, got %d", si.State.Msgs) } } func TestJetStreamClusterStreamGetMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } mreq := &JSApiMsgGetRequest{Seq: 1} req, err := json.Marshal(mreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } rmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, "TEST"), req, time.Second) if err != nil { t.Fatalf("Could not retrieve stream message: %v", err) } if err != nil { t.Fatalf("Could not retrieve stream message: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(rmsg.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Message == nil || resp.Error != nil { t.Fatalf("Did not receive correct response: %+v", resp.Error) } } func TestJetStreamClusterStreamDirectGetMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, _ := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: MemoryStorage, Replicas: 3, MaxMsgsPer: 1, AllowDirect: true, } addStream(t, nc, cfg) sendStreamMsg(t, nc, "foo", "bar") getSubj := fmt.Sprintf(JSDirectMsgGetT, "TEST") getMsg := func(req *JSApiMsgGetRequest) *nats.Msg { var b []byte var err error if req != nil { b, err = json.Marshal(req) require_NoError(t, err) } m, err := nc.Request(getSubj, b, time.Second) require_NoError(t, err) return m } m := getMsg(&JSApiMsgGetRequest{LastFor: "foo"}) require_True(t, string(m.Data) == "bar") require_True(t, m.Header.Get(JSStream) == "TEST") require_True(t, m.Header.Get(JSSequence) == "1") require_True(t, m.Header.Get(JSSubject) == "foo") require_True(t, m.Subject != "foo") require_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_) } func TestJetStreamClusterStreamPerf(t *testing.T) { // Comment out to run, holding place for now. skip(t) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } numConnections := 4 var conns []nats.JetStream for i := 0; i < numConnections; i++ { s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() conns = append(conns, js) } toSend := 100000 numProducers := 8 payload := []byte("Hello JSC") startCh := make(chan bool) var wg sync.WaitGroup for n := 0; n < numProducers; n++ { wg.Add(1) go func() { defer wg.Done() js := conns[rand.Intn(numConnections)] <-startCh for i := 0; i < int(toSend)/numProducers; i++ { if _, err = js.Publish("foo", payload); err != nil { t.Errorf("Unexpected publish error: %v", err) } } }() } // Wait for Go routines. time.Sleep(250 * time.Millisecond) start := time.Now() close(startCh) wg.Wait() tt := time.Since(start) fmt.Printf("Took %v to send %d msgs with %d producers and R=3!\n", tt, toSend, numProducers) fmt.Printf("%.0f msgs/sec\n\n", float64(toSend)/tt.Seconds()) } func TestJetStreamClusterConsumerPerf(t *testing.T) { // Comment out to run, holding place for now. skip(t) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 500000 msg := make([]byte, 64) crand.Read(msg) for i := 0; i < toSend; i++ { nc.Publish("TEST", msg) } nc.Flush() checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return fmt.Errorf("Unexpected error: %v", err) } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected to have %d messages, got %d", toSend, si.State.Msgs) } return nil }) received := int32(0) deliverTo := "r" done := make(chan bool) total := int32(toSend) var start time.Time nc.Subscribe(deliverTo, func(m *nats.Msg) { if r := atomic.AddInt32(&received, 1); r >= total { done <- true } else if r == 1 { start = time.Now() } }) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{DeliverSubject: deliverTo, Durable: "gf"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } select { case <-done: case <-time.After(10 * time.Second): t.Fatalf("Timed out?") } tt := time.Since(start) fmt.Printf("Took %v to receive %d msgs\n", tt, toSend) fmt.Printf("%.0f msgs/sec\n\n", float64(toSend)/tt.Seconds()) } // This test creates a queue consumer for the delivery subject, // and make sure it connects to the server that is not the leader // of the stream. A bug was not stripping the $JS.ACK reply subject // correctly, which means that ack sent on the reply subject was // dropped by the route. func TestJetStreamClusterQueueSubConsumer(t *testing.T) { c := createJetStreamClusterExplicit(t, "R2S", 2) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, Replicas: 1, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } inbox := nats.NewInbox() obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "ivan", DeliverSubject: inbox, DeliverGroup: "queue", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, }, } req, err := json.Marshal(obsReq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "ivan"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error != nil { t.Fatalf("Unexpected error, got %+v", ccResp.Error) } ci, err := js.ConsumerInfo("TEST", "ivan") if err != nil { t.Fatalf("Error getting consumer info: %v", err) } // Now create a client that does NOT connect to the stream leader. // Start with url from first server in the cluster. u := c.servers[0].ClientURL() // If leader is "S-1", then use S-2 to connect to, which is at servers[1]. if ci.Cluster.Leader == "S-1" { u = c.servers[1].ClientURL() } qsubnc, err := nats.Connect(u) if err != nil { t.Fatalf("Error connecting: %v", err) } defer qsubnc.Close() ch := make(chan struct{}, 2) if _, err := qsubnc.QueueSubscribe(inbox, "queue", func(m *nats.Msg) { m.Respond(nil) ch <- struct{}{} }); err != nil { t.Fatalf("Error creating sub: %v", err) } // Use the other connection to publish a message if _, err := js.Publish("foo.bar", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } // Wait that we receive the message first. select { case <-ch: case <-time.After(time.Second): t.Fatal("Did not receive message") } // Message should be ack'ed and not redelivered. select { case <-ch: t.Fatal("Message redelivered!!!") case <-time.After(250 * time.Millisecond): // OK } } func TestJetStreamClusterLeaderStepdown(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() c.waitOnLeader() cl := c.leader() // Now ask the system account to have the leader stepdown. s := c.randomNonLeader() nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() resp, err := nc.Request(JSApiLeaderStepDown, nil, time.Second) if err != nil { t.Fatalf("Error on stepdown request: %v", err) } var sdr JSApiLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &sdr); err != nil { t.Fatalf("Unexpected error: %v", err) } if sdr.Error != nil || !sdr.Success { t.Fatalf("Unexpected error for leader stepdown: %+v", sdr.Error) } c.waitOnLeader() if cl == c.leader() { t.Fatalf("Expected a new metaleader, got same") } } func TestJetStreamClusterSourcesFilteringAndUpdating(t *testing.T) { owt := srcConsumerWaitTime srcConsumerWaitTime = 2 * time.Second defer func() { srcConsumerWaitTime = owt }() c := createJetStreamClusterExplicit(t, "MSR", 5) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sendBatch := func(subject string, n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkSync := func(msgsTest, msgsM uint64) { t.Helper() checkFor(t, 30*time.Second, time.Second, func() error { if tsi, err := js.StreamInfo("TEST"); err != nil { return err } else if msi, err := js.StreamInfo("M"); err != nil { return err } else if tsi.State.Msgs != msgsTest { return fmt.Errorf("received %d msgs from TEST, expected %d", tsi.State.Msgs, msgsTest) } else if msi.State.Msgs != msgsM { return fmt.Errorf("received %d msgs from M, expected %d", msi.State.Msgs, msgsM) } return nil }) } // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 2, }) require_NoError(t, err) defer js.DeleteStream("TEST") // Create M stream with a single source on "foo" _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Sources: []*nats.StreamSource{{Name: "TEST", FilterSubject: "foo"}}, Replicas: 2, }) require_NoError(t, err) defer js.DeleteStream("M") // check a message on "bar" doesn't get sourced sendBatch("bar", 100) checkSync(100, 0) // check a message on "foo" does get sourced sendBatch("foo", 100) checkSync(200, 100) // change remove the source on "foo" and add a new source on "bar" _, err = js.UpdateStream(&nats.StreamConfig{ Name: "M", Sources: []*nats.StreamSource{{Name: "TEST", FilterSubject: "bar"}}, Replicas: 2, }) require_NoError(t, err) // As it is a new source (never been sourced before) it starts sourcing at the start of TEST // and therefore sources the message on "bar" that is in TEST checkSync(200, 200) // new messages on "foo" are being filtered as it's not being currently sourced sendBatch("foo", 100) checkSync(300, 200) // new messages on "bar" are being sourced sendBatch("bar", 100) checkSync(400, 300) // re-add the source for "foo" keep the source on "bar" _, err = js.UpdateStream(&nats.StreamConfig{ Name: "M", Sources: []*nats.StreamSource{{Name: "TEST", FilterSubject: "bar"}, {Name: "TEST", FilterSubject: "foo"}}, Replicas: 2, }) require_NoError(t, err) // check the 'backfill' of messages on "foo" that were published while the source was inactive checkSync(400, 400) // causes startingSequenceForSources() to be called nc.Close() c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamLeader("$G", "M") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // check that it restarted the sources' consumers at the right place checkSync(400, 400) // check both sources are still active sendBatch("bar", 100) checkSync(500, 500) sendBatch("foo", 100) checkSync(600, 600) // Check that purging the stream and does not cause the re-sourcing of the messages js.PurgeStream("M") checkSync(600, 0) // Even after a leader change or restart nc.Close() c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamLeader("$G", "M") // Allow direct sync consumers to connect. // This test could pass if we do not hook up and try to deliver the messages when we should not. time.Sleep(1500 * time.Millisecond) nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkSync(600, 0) } func TestJetStreamClusterSourcesUpdateOriginError(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSR", 5) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sendBatch := func(subject string, n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkSync := func(msgsTest, msgsM uint64) { t.Helper() checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { if tsi, err := js.StreamInfo("TEST"); err != nil { return err } else if msi, err := js.StreamInfo("M"); err != nil { return err } else if tsi.State.Msgs != msgsTest { return fmt.Errorf("received %d msgs from TEST, expected %d", tsi.State.Msgs, msgsTest) } else if msi.State.Msgs != msgsM { return fmt.Errorf("received %d msgs from M, expected %d", msi.State.Msgs, msgsM) } return nil }) } // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Sources: []*nats.StreamSource{{Name: "TEST", FilterSubject: "foo"}}, Replicas: 2, }) require_NoError(t, err) // Send 100 msgs. sendBatch("foo", 100) checkSync(100, 100) // update makes source invalid _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"bar"}, Replicas: 2, }) require_NoError(t, err) // TODO check for downstream error propagation _, err = js.Publish("foo", nil) require_Error(t, err) sendBatch("bar", 100) // The source stream remains at 100 msgs as it still uses foo as it's filter checkSync(200, 100) nc.Close() c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamLeader("$G", "M") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkSync(200, 100) _, err = js.Publish("foo", nil) require_Error(t, err) require_Equal(t, err.Error(), "nats: no response from stream") sendBatch("bar", 100) // The source stream remains at 100 msgs as it still uses foo as it's filter checkSync(300, 100) } func TestJetStreamClusterMirrorAndSourcesClusterRestart(t *testing.T) { owt := srcConsumerWaitTime srcConsumerWaitTime = 2 * time.Second defer func() { srcConsumerWaitTime = owt }() test := func(t *testing.T, mirror bool, filter bool) { c := createJetStreamClusterExplicit(t, "MSR", 5) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz.*"}, Replicas: 2, }) require_NoError(t, err) filterSubj := _EMPTY_ if filter { filterSubj = "foo" } // Create Mirror/Source now. if mirror { _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST", FilterSubject: filterSubj}, Replicas: 2, }) } else { _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Sources: []*nats.StreamSource{{Name: "TEST", FilterSubject: filterSubj}}, Replicas: 2, }) } require_NoError(t, err) expectedMsgCount := uint64(0) sendBatch := func(subject string, n int) { t.Helper() if subject == "foo" || !filter { expectedMsgCount += uint64(n) } // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkSync := func(msgsTest, msgsM uint64) { t.Helper() checkFor(t, 40*time.Second, time.Second, func() error { if tsi, err := js.StreamInfo("TEST"); err != nil { return err } else if msi, err := js.StreamInfo("M"); err != nil { return err } else if tsi.State.Msgs != msgsTest { return fmt.Errorf("received %d msgs from TEST, expected %d", tsi.State.Msgs, msgsTest) } else if msi.State.Msgs != msgsM { return fmt.Errorf("received %d msgs from M, expected %d", msi.State.Msgs, msgsM) } return nil }) } sendBatch("foo", 100) checkSync(100, expectedMsgCount) sendBatch("bar", 100) checkSync(200, expectedMsgCount) nc.Close() c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamLeader("$G", "M") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkSync(200, expectedMsgCount) sendBatch("foo", 100) checkSync(300, expectedMsgCount) sendBatch("bar", 100) checkSync(400, expectedMsgCount) } t.Run("mirror-filter", func(t *testing.T) { test(t, true, true) }) t.Run("mirror-nofilter", func(t *testing.T) { test(t, true, false) }) t.Run("source-filter", func(t *testing.T) { test(t, false, true) }) t.Run("source-nofilter", func(t *testing.T) { test(t, false, false) }) } func TestJetStreamClusterMirrorAndSourcesFilteredConsumers(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, "MS5", 5) defer c.shutdown() // Client for API requests. s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz.*"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Create Mirror now. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } dsubj := nats.NewInbox() nc.SubscribeSync(dsubj) nc.Flush() createConsumer := func(sn, fs string) { t.Helper() _, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } expectFail := func(sn, fs string) { t.Helper() _, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs}) if err == nil { t.Fatalf("Expected error but got none") } } createConsumer("M", "foo") createConsumer("M", "bar") createConsumer("M", "baz.foo") expectFail("M", ".") expectFail("M", ">.foo") // Make sure wider scoped subjects work as well. createConsumer("M", "*") createConsumer("M", ">") // Now do some sources. if _, err := js.AddStream(&nats.StreamConfig{Name: "O1", Subjects: []string{"foo.*"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "O2", Subjects: []string{"bar.*"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Create downstream now. _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{Name: "O1"}, {Name: "O2"}}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } createConsumer("S", "foo.1") createConsumer("S", "bar.1") // Now cross account stuff. nc2, js2 := jsClientConnect(t, s, nats.UserInfo("rip", "pass")) defer nc2.Close() if _, err := js2.AddStream(&nats.StreamConfig{Name: "ORIGIN", Subjects: []string{"foo.*"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } cfg := StreamConfig{ Name: "SCA", Storage: FileStorage, Sources: []*StreamSource{{ Name: "ORIGIN", External: &ExternalStream{ ApiPrefix: "RI.JS.API", DeliverPrefix: "RI.DELIVER.SYNC.SOURCES", }, }}, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var scResp JSApiStreamCreateResponse if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.StreamInfo == nil || scResp.Error != nil { t.Fatalf("Did not receive correct response: %+v", scResp.Error) } // Externals skip the checks for now. createConsumer("SCA", "foo.1") createConsumer("SCA", "bar.1") createConsumer("SCA", "baz") } func TestJetStreamClusterCrossAccountMirrorsAndSources(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, "C1", 3) defer c.shutdown() // Create source stream under RI account. s := c.randomServer() nc, js := jsClientConnect(t, s, nats.UserInfo("rip", "pass")) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // use large number to tease out FC issues toSend := 3000 for i := 0; i < toSend; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } nc2, js2 := jsClientConnect(t, s) defer nc2.Close() // Have to do this direct until we get Go client support. // Need to match jsClusterMirrorSourceImportsTempl imports. _, err := js2.AddStream(&nats.StreamConfig{ Name: "MY_MIRROR_TEST", Mirror: &nats.StreamSource{ Name: "TEST", External: &nats.ExternalStream{ APIPrefix: "RI.JS.API", DeliverPrefix: "RI.DELIVER.SYNC.MIRRORS", }, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { si, err := js2.StreamInfo("MY_MIRROR_TEST") if err != nil { t.Fatalf("Could not retrieve stream info: %s", err) } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d msgs, got state: %+v", toSend, si.State) } return nil }) // Now do sources as well. _, err = js2.AddStream(&nats.StreamConfig{ Name: "MY_SOURCE_TEST", Sources: []*nats.StreamSource{ { Name: "TEST", External: &nats.ExternalStream{ APIPrefix: "RI.JS.API", DeliverPrefix: "RI.DELIVER.SYNC.SOURCES", }, }, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("MY_SOURCE_TEST") if err != nil { t.Fatalf("Could not retrieve stream info") } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d msgs, got state: %+v", toSend, si.State) } return nil }) } func TestJetStreamClusterFailMirrorsAndSources(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, "C1", 3) defer c.shutdown() // Create source stream under RI account. s := c.randomServer() nc, js := jsClientConnect(t, s, nats.UserInfo("rip", "pass")) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2, Subjects: []string{"test.>"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } nc2, _ := jsClientConnect(t, s, nats.UserInfo("rip", "pass")) defer nc2.Close() testPrefix := func(testName string, id ErrorIdentifier, cfg StreamConfig) { t.Run(testName, func(t *testing.T) { req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc2.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var scResp JSApiStreamCreateResponse if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.Error == nil { t.Fatalf("Did expect an error but got none") } else if !IsNatsErr(scResp.Error, id) { t.Fatalf("Expected different error: %s", scResp.Error.Description) } }) } testPrefix("mirror-bad-apiprefix", JSStreamExternalApiOverlapErrF, StreamConfig{ Name: "MY_MIRROR_TEST", Storage: FileStorage, Mirror: &StreamSource{ Name: "TEST", External: &ExternalStream{ ApiPrefix: "$JS.API", DeliverPrefix: "here", }, }, }) testPrefix("source-bad-apiprefix", JSStreamExternalApiOverlapErrF, StreamConfig{ Name: "MY_SOURCE_TEST", Storage: FileStorage, Sources: []*StreamSource{{ Name: "TEST", External: &ExternalStream{ ApiPrefix: "$JS.API", DeliverPrefix: "here", }, }, }, }) } func TestJetStreamClusterConsumerDeliveredSyncReporting(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.WorkQueuePolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo.bar", "mc") require_NoError(t, err) // Make us match first, but not next 10. _, err = js.Publish("foo.bar", nil) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish("foo.baz", nil) require_NoError(t, err) } msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) // Now we want to make sure that jsz reporting will show the same // state, including delivered, which will have skipped to the end. // The skip can happen on several factors, but for here we just send // another pull request which we will let fail. _, err = sub.Fetch(1, nats.MaxWait(200*time.Millisecond)) require_Error(t, err) opts := &JSzOptions{Accounts: true, Streams: true, Consumer: true} for _, s := range c.servers { jsz, err := s.Jsz(opts) require_NoError(t, err) ci := jsz.AccountDetails[0].Streams[0].Consumer[0] require_Equal(t, ci.Delivered.Consumer, 1) require_Equal(t, ci.Delivered.Stream, 1) } } // This is to test follower ack fill logic when pending not empty. // There was a bug that would update p.Sequence in the stores (mem & file) // that would cause the logic to fail. Redeliveries were required to trigger. func TestJetStreamClusterConsumerAckSyncReporting(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.WorkQueuePolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo.bar", "mc", nats.AckWait(250*time.Millisecond)) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish("foo.bar", nil) require_NoError(t, err) } msgs, err := sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) // Let redeliveries kick in. time.Sleep(time.Second) msgs, err = sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) // Randomize rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) dontAck := uint64(7) var skipped, last *nats.Msg for _, m := range msgs { meta, err := m.Metadata() require_NoError(t, err) if meta.Sequence.Stream == dontAck { skipped = m continue } if meta.Sequence.Stream == 10 { last = m continue } err = m.AckSync() require_NoError(t, err) } require_NotNil(t, skipped) require_NotNil(t, last) checkAckFloor := func(consumer, stream uint64) { opts := &JSzOptions{Accounts: true, Streams: true, Consumer: true} checkFor(t, 3*time.Second, 200*time.Millisecond, func() error { for _, s := range c.servers { jsz, err := s.Jsz(opts) if err != nil { return err } ci := jsz.AccountDetails[0].Streams[0].Consumer[0] if ci.AckFloor.Consumer != consumer { return fmt.Errorf("AckFloor.Consumer is not %d: %v", consumer, ci.AckFloor.Consumer) } if ci.AckFloor.Stream != stream { return fmt.Errorf("AckFloor.Stream is not %d: %v", stream, ci.AckFloor.Stream) } } return nil }) } // Now we want to make sure that jsz reporting will show the same // state for ack floor. checkAckFloor(dontAck-1, dontAck-1) // Now ack the skipped message err = skipped.AckSync() require_NoError(t, err) checkAckFloor(9, 9) // Now ack the last message err = last.AckSync() require_NoError(t, err) checkAckFloor(20, 10) } func TestJetStreamClusterConsumerDeleteInterestPolicyMultipleConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.InterestPolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // Make trhe first sequence high. We already protect against it but for extra sanity. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) require_NoError(t, err) // Create 2 consumers. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C2", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo.bar", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 100) sub, err := js.PullSubscribe("foo.bar", "C1") require_NoError(t, err) msgs, err := sub.Fetch(50) require_NoError(t, err) require_Equal(t, len(msgs), 50) for _, m := range msgs { m.AckSync() } // Now delete second one and make sure accounting correct. err = js.DeleteConsumer("TEST", "C2") require_NoError(t, err) time.Sleep(100 * time.Millisecond) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 50) } func TestJetStreamClusterConsumerAckNoneInterestPolicyShouldNotRetainAfterDelivery(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.InterestPolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // Make trhe first sequence high. We already protect against it but for extra sanity. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C1", AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo.bar", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 100) sub, err := js.PullSubscribe("foo.bar", "C1") require_NoError(t, err) msgs, err := sub.Fetch(100) require_NoError(t, err) require_Equal(t, len(msgs), 100) for _, m := range msgs { m.AckSync() } si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 0) } func TestJetStreamClusterConsumerDeleteAckNoneInterestPolicyWithOthers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.InterestPolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // Make trhe first sequence high. We already protect against it but for extra sanity. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) require_NoError(t, err) // Create 2 consumers. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C2", AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo.bar", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 100) sub, err := js.PullSubscribe("foo.bar", "C1") require_NoError(t, err) msgs, err := sub.Fetch(100) require_NoError(t, err) require_Equal(t, len(msgs), 100) for _, m := range msgs { m.AckSync() } // AckNone will hold. si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 100) // Now delete second one and make sure accounting correct. err = js.DeleteConsumer("TEST", "C2") require_NoError(t, err) time.Sleep(100 * time.Millisecond) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 0) } func TestJetStreamClusterConsumerDeleteInterestPolicyPerf(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.InterestPolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // Make trhe first sequence high. We already protect against it but for extra sanity. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) require_NoError(t, err) // Create 3 consumers. 1 Ack explicit, 1 AckAll and 1 AckNone _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C2", AckPolicy: nats.AckAllPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C3", AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) for i := 0; i < 500_000; i++ { js.PublishAsync("foo.bar", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // For C1 grab 100 and ack evens. sub, err := js.PullSubscribe("foo.bar", "C1") require_NoError(t, err) msgs, err := sub.Fetch(100) require_NoError(t, err) require_Equal(t, len(msgs), 100) for _, m := range msgs { meta, _ := m.Metadata() if meta.Sequence.Stream%2 == 0 { m.AckSync() } } // For C2 grab 500 and ack 100. sub, err = js.PullSubscribe("foo.bar", "C2") require_NoError(t, err) msgs, err = sub.Fetch(500, nats.MaxWait(10*time.Second)) require_NoError(t, err) require_Equal(t, len(msgs), 500) msgs[99].AckSync() // Simulate stream viewer, get first 10 from C3 sub, err = js.PullSubscribe("foo.bar", "C3") require_NoError(t, err) msgs, err = sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) time.Sleep(500 * time.Millisecond) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 499_995) // Before fix this was in the seconds. All the while the stream is locked. // This should be short now. start := time.Now() err = js.DeleteConsumer("TEST", "C3") require_NoError(t, err) if elapsed := time.Since(start); elapsed > 50*time.Millisecond { t.Fatalf("Deleting AckNone consumer took too long: %v", elapsed) } time.Sleep(500 * time.Millisecond) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 499_950) // Now do AckAll start = time.Now() err = js.DeleteConsumer("TEST", "C2") require_NoError(t, err) if elapsed := time.Since(start); elapsed > 50*time.Millisecond { t.Fatalf("Deleting AckAll consumer took too long: %v", elapsed) } time.Sleep(500 * time.Millisecond) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 499_950) // Now do AckExplicit start = time.Now() err = js.DeleteConsumer("TEST", "C1") require_NoError(t, err) if elapsed := time.Since(start); elapsed > 50*time.Millisecond { t.Fatalf("Deleting AckExplicit consumer took too long: %v", elapsed) } time.Sleep(500 * time.Millisecond) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 0) } // Make sure to not allow non-system accounts to move meta leader. func TestJetStreamClusterMetaStepdownFromNonSysAccount(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() // Client based API nc, _ := jsClientConnect(t, s) defer nc.Close() ml := c.leader() _, err := nc.Request(JSApiLeaderStepDown, nil, time.Second) require_Error(t, err, nats.ErrTimeout) // Make sure we did not move. c.waitOnLeader() require_Equal(t, ml, c.leader()) // System user can move it. snc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() resp, err := snc.Request(JSApiLeaderStepDown, nil, time.Second) require_NoError(t, err) var sdr JSApiLeaderStepDownResponse require_NoError(t, json.Unmarshal(resp.Data, &sdr)) require_True(t, sdr.Success) require_Equal(t, sdr.Error, nil) // Make sure we did move this time. c.waitOnLeader() require_NotEqual(t, ml, c.leader()) } func TestJetStreamClusterMaxDeliveriesOnInterestStreams(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub1, err := js.PullSubscribe("foo.*", "c1", nats.AckWait(10*time.Millisecond), nats.MaxDeliver(1)) require_NoError(t, err) sub2, err := js.PullSubscribe("foo.*", "c2", nats.AckWait(10*time.Millisecond), nats.MaxDeliver(1)) require_NoError(t, err) js.Publish("foo.bar", []byte("HELLO")) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 1) msgs, err := sub1.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) msgs, err = sub2.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) // Wait for redelivery to both consumers which will do nothing. time.Sleep(250 * time.Millisecond) // Now check that stream and consumer infos are correct. si, err = js.StreamInfo("TEST") require_NoError(t, err) // Messages that are skipped due to max deliveries should NOT remove messages. require_Equal(t, si.State.Msgs, 1) require_Equal(t, si.State.Consumers, 2) for _, cname := range []string{"c1", "c2"} { ci, err := js.ConsumerInfo("TEST", cname) require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 1) require_Equal(t, ci.Delivered.Stream, 1) require_Equal(t, ci.AckFloor.Consumer, 1) require_Equal(t, ci.AckFloor.Stream, 1) require_Equal(t, ci.NumAckPending, 0) require_Equal(t, ci.NumRedelivered, 1) require_Equal(t, ci.NumPending, 0) } } func TestJetStreamClusterMetaRecoveryUpdatesDeletesConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() js := c.leader().getJetStream() create := []*Entry{ {EntryNormal, encodeAddStreamAssignment(&streamAssignment{ Config: &StreamConfig{Name: "TEST", Storage: FileStorage}, })}, {EntryNormal, encodeAddConsumerAssignment(&consumerAssignment{ Stream: "TEST", Config: &ConsumerConfig{Name: "consumer"}, })}, } delete := []*Entry{ {EntryNormal, encodeDeleteStreamAssignment(&streamAssignment{ Config: &StreamConfig{Name: "TEST", Storage: FileStorage}, })}, } // Need to be recovering so that we accumulate recoveryUpdates. js.setMetaRecovering() ru := &recoveryUpdates{ removeStreams: make(map[string]*streamAssignment), removeConsumers: make(map[string]map[string]*consumerAssignment), addStreams: make(map[string]*streamAssignment), updateStreams: make(map[string]*streamAssignment), updateConsumers: make(map[string]map[string]*consumerAssignment), } // Push recovery entries that create the stream & consumer. _, _, _, err := js.applyMetaEntries(create, ru) require_NoError(t, err) require_Len(t, len(ru.updateConsumers), 1) // Now push another recovery entry that deletes the stream. The // entry that creates the consumer should now be gone. _, _, _, err = js.applyMetaEntries(delete, ru) require_NoError(t, err) require_Len(t, len(ru.removeStreams), 1) require_Len(t, len(ru.updateConsumers), 0) } func TestJetStreamClusterMetaRecoveryRecreateFileStreamAsMemory(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() js := c.leader().getJetStream() createFileStream := []*Entry{ {EntryNormal, encodeAddStreamAssignment(&streamAssignment{ Config: &StreamConfig{Name: "TEST", Storage: FileStorage}, })}, } deleteFileStream := []*Entry{ {EntryNormal, encodeDeleteStreamAssignment(&streamAssignment{ Config: &StreamConfig{Name: "TEST", Storage: FileStorage}, })}, } createMemoryStream := []*Entry{ {EntryNormal, encodeAddStreamAssignment(&streamAssignment{ Config: &StreamConfig{Name: "TEST", Storage: FileStorage}, })}, } createConsumer := []*Entry{ {EntryNormal, encodeAddConsumerAssignment(&consumerAssignment{ Stream: "TEST", Config: &ConsumerConfig{Name: "consumer"}, })}, } // Need to be recovering so that we accumulate recoveryUpdates. js.setMetaRecovering() ru := &recoveryUpdates{ removeStreams: make(map[string]*streamAssignment), removeConsumers: make(map[string]map[string]*consumerAssignment), addStreams: make(map[string]*streamAssignment), updateStreams: make(map[string]*streamAssignment), updateConsumers: make(map[string]map[string]*consumerAssignment), } // We created a file-based stream first, but deleted it shortly after. _, _, _, err := js.applyMetaEntries(createFileStream, ru) require_NoError(t, err) require_Len(t, len(ru.addStreams), 1) require_Len(t, len(ru.removeStreams), 0) // Now push another recovery entry that deletes the stream. // The file-based stream should not have been created. _, _, _, err = js.applyMetaEntries(deleteFileStream, ru) require_NoError(t, err) require_Len(t, len(ru.addStreams), 0) require_Len(t, len(ru.removeStreams), 1) // Now stage a memory-based stream to be created. _, _, _, err = js.applyMetaEntries(createMemoryStream, ru) require_NoError(t, err) require_Len(t, len(ru.addStreams), 1) require_Len(t, len(ru.removeStreams), 0) require_Len(t, len(ru.updateConsumers), 0) // Also create a consumer on that memory-based stream. _, _, _, err = js.applyMetaEntries(createConsumer, ru) require_NoError(t, err) require_Len(t, len(ru.addStreams), 1) require_Len(t, len(ru.removeStreams), 0) require_Len(t, len(ru.updateConsumers), 1) } func TestJetStreamClusterMetaRecoveryConsumerCreateAndRemove(t *testing.T) { tests := []struct { title string encodeAddConsumerAssignment func(ca *consumerAssignment) []byte }{ {title: "simple", encodeAddConsumerAssignment: encodeAddConsumerAssignment}, {title: "compressed", encodeAddConsumerAssignment: encodeAddConsumerAssignmentCompressed}, } for _, test := range tests { t.Run(test.title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() js := c.leader().getJetStream() ca := &consumerAssignment{Stream: "TEST", Name: "consumer"} createConsumer := []*Entry{{EntryNormal, test.encodeAddConsumerAssignment(ca)}} deleteConsumer := []*Entry{{EntryNormal, encodeDeleteConsumerAssignment(ca)}} // Need to be recovering so that we accumulate recoveryUpdates. js.setMetaRecovering() ru := &recoveryUpdates{ removeStreams: make(map[string]*streamAssignment), removeConsumers: make(map[string]map[string]*consumerAssignment), addStreams: make(map[string]*streamAssignment), updateStreams: make(map[string]*streamAssignment), updateConsumers: make(map[string]map[string]*consumerAssignment), } // Creating the consumer should append to update consumers list. _, _, _, err := js.applyMetaEntries(createConsumer, ru) require_NoError(t, err) require_Len(t, len(ru.updateConsumers[":TEST"]), 1) require_Len(t, len(ru.removeConsumers), 0) // Deleting the consumer should append to remove consumers list and remove from update list. _, _, _, err = js.applyMetaEntries(deleteConsumer, ru) require_NoError(t, err) require_Len(t, len(ru.removeConsumers[":TEST"]), 1) require_Len(t, len(ru.updateConsumers[":TEST"]), 0) // When re-creating the consumer, add to update list and remove from remove list. _, _, _, err = js.applyMetaEntries(createConsumer, ru) require_NoError(t, err) require_Len(t, len(ru.updateConsumers[":TEST"]), 1) require_Len(t, len(ru.removeConsumers[":TEST"]), 0) }) } } // Make sure if we received acks that are out of bounds, meaning past our // last sequence or before our first that they are ignored and errored if applicable. func TestJetStreamClusterConsumerAckOutOfBounds(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish("foo.bar", []byte("OK")) require_NoError(t, err) } sub, err := js.PullSubscribe("foo.*", "C") require_NoError(t, err) msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) msgs[0].AckSync() // Now ack way past the last sequence. _, err = nc.Request("$JS.ACK.TEST.C.1.10000000000.0.0.0", nil, 250*time.Millisecond) require_Error(t, err, nats.ErrTimeout) // Make sure that now changes happened to our state. ci, err := js.ConsumerInfo("TEST", "C") require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 1) require_Equal(t, ci.Delivered.Stream, 1) require_Equal(t, ci.AckFloor.Consumer, 1) require_Equal(t, ci.AckFloor.Stream, 1) s := c.consumerLeader("$G", "TEST", "C") s.Shutdown() s.WaitForShutdown() c.restartServer(s) c.waitOnConsumerLeader(globalAccountName, "TEST", "C") // Confirm new leader has same state for delivered and ack floor. ci, err = js.ConsumerInfo("TEST", "C") require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 1) require_Equal(t, ci.Delivered.Stream, 1) require_Equal(t, ci.AckFloor.Consumer, 1) require_Equal(t, ci.AckFloor.Stream, 1) } func TestJetStreamClusterCatchupLoadNextMsgTooManyDeletes(t *testing.T) { tests := []struct { title string catchupRequest *streamSyncRequest setup func(js nats.JetStreamContext) assert func(sub *nats.Subscription) }{ { title: "within-delete-gap", setup: func(js nats.JetStreamContext) {}, }, { title: "EOF", setup: func(js nats.JetStreamContext) { err := js.DeleteMsg("TEST", 100) require_NoError(t, err) }, }, } for _, test := range tests { t.Run(test.title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) require_NoError(t, err) // Starts and ends with subject "foo", we'll purge so there's a large gap of deletes in the middle. // This should force runCatchup to use LoadNextMsg instead of LoadMsg. for i := 0; i < 100; i++ { subject := "bar" if i == 0 || i == 99 { subject = "foo" } _, err = js.Publish(subject, nil) require_NoError(t, err) } err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: "bar"}) require_NoError(t, err) // Optionally run some extra setup. test.setup(js) // Reconnect to stream leader. l := c.streamLeader(globalAccountName, "TEST") nc.Close() nc, _ = jsClientConnect(t, l, nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() // Setup wiretap and grab stream. sendSubject := "test-wiretap" sub, err := nc.SubscribeSync(sendSubject) require_NoError(t, err) err = nc.Flush() // Must flush, otherwise our subscription could be too late. require_NoError(t, err) acc, err := l.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) // Run custom catchup request and the test's asserts. sreq := &streamSyncRequest{Peer: "peer", FirstSeq: 5, LastSeq: 5, DeleteRangesOk: true} require_True(t, mset.srv.startGoRoutine(func() { mset.runCatchup(sendSubject, sreq) })) // Our first message should be a skip msg. msg, err := sub.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, entryOp(msg.Data[0]), streamMsgOp) subj, _, _, _, seq, ts, err := decodeStreamMsg(msg.Data[1:]) require_NoError(t, err) require_Equal(t, seq, 5) require_Equal(t, subj, _EMPTY_) require_Equal(t, ts, 0) // And end with EOF. msg, err = sub.NextMsg(time.Second) require_NoError(t, err) require_Len(t, len(msg.Data), 0) }) } } func TestJetStreamClusterConsumerInfoAfterCreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nl := c.randomNonLeader() nc, js := jsClientConnect(t, nl) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // We pause applies for the server we're connected to. // This is fine for the RAFT log and allowing the consumer to be created, // but we will not be able to apply the consumer assignment for some time. mjs := nl.getJetStream() require_NotNil(t, js) mg := mjs.getMetaGroup() require_NotNil(t, mg) err = mg.(*raft).PauseApply() require_NoError(t, err) // Add consumer. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) require_NoError(t, err) // Consumer info should not fail, this server should not short-circuit because // it was not able to apply the consumer assignment. _, err = js.ConsumerInfo("TEST", "CONSUMER") require_NoError(t, err) // Resume applies. mg.(*raft).ResumeApply() // Check consumer info still works. _, err = js.ConsumerInfo("TEST", "CONSUMER") require_NoError(t, err) } func TestJetStreamClusterStreamUpscalePeersAfterDownscale(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() checkPeerSet := func() { t.Helper() checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) if err != nil { return err } mset, err := acc.lookupStream("TEST") if err != nil { return err } peers := mset.raftNode().Peers() if len(peers) != 5 { return fmt.Errorf("expected 5 peers, got %d", len(peers)) } } return nil }) } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 5, }) require_NoError(t, err) checkPeerSet() _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 5, }) require_NoError(t, err) checkPeerSet() } func TestJetStreamClusterClearAllPreAcksOnRemoveMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "CONSUMER", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for i := 0; i < 3; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } // Wait for all servers to converge on the same state. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { return checkState(t, c, globalAccountName, "TEST") }) // Register pre-acks on all servers. // Normally this can't happen as the stream leader will have the message that's acked available, just for testing. for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) // Register pre-acks for the 3 messages. mset.registerPreAckLock(o, 1) mset.registerPreAckLock(o, 2) mset.registerPreAckLock(o, 3) } // Check there's an expected amount of pre-acks, and there are no pre-acks for the given sequence. checkPreAcks := func(seq uint64, expected int) { t.Helper() checkFor(t, 5*time.Second, time.Second, func() error { for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) if err != nil { return err } mset, err := acc.lookupStream("TEST") if err != nil { return err } mset.mu.RLock() numPreAcks := len(mset.preAcks) numSeqPreAcks := len(mset.preAcks[seq]) mset.mu.RUnlock() if numPreAcks != expected { return fmt.Errorf("expected %d pre-acks, got %d", expected, numPreAcks) } if seq > 0 && numSeqPreAcks != 0 { return fmt.Errorf("expected 0 pre-acks for seq %d, got %d", seq, numSeqPreAcks) } } return nil }) } // Check all pre-acks were registered. checkPreAcks(0, 3) // Deleting the message should clear the pre-ack. err = js.DeleteMsg("TEST", 1) require_NoError(t, err) checkPreAcks(1, 2) // Erasing the message should clear the pre-ack. err = js.SecureDeleteMsg("TEST", 2) require_NoError(t, err) checkPreAcks(2, 1) // Purging should clear all pre-acks below the purged floor. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 4}) require_NoError(t, err) checkPreAcks(3, 0) } func TestJetStreamClusterStreamHealthCheckMustNotRecreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() waitForStreamAssignments := func() { t.Helper() checkFor(t, 5*time.Second, time.Second, func() error { for _, s := range c.servers { if s.getJetStream().streamAssignment(globalAccountName, "TEST") == nil { return fmt.Errorf("stream assignment not found on %s", s.Name()) } } return nil }) } waitForNoStreamAssignments := func() { t.Helper() checkFor(t, 5*time.Second, time.Second, func() error { for _, s := range c.servers { if s.getJetStream().streamAssignment(globalAccountName, "TEST") != nil { return fmt.Errorf("stream assignment still available on %s", s.Name()) } } return nil }) } getStreamAssignment := func(rs *Server) (*jetStream, *Account, *streamAssignment, *stream) { acc, err := rs.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NotNil(t, err) sjs := rs.getJetStream() sjs.mu.RLock() defer sjs.mu.RUnlock() sas := sjs.cluster.streams[globalAccountName] require_True(t, sas != nil) sa := sas["TEST"] require_True(t, sa != nil) sa.Created = time.Time{} return sjs, acc, sa, mset } checkNodeIsClosed := func(sa *streamAssignment) { t.Helper() require_True(t, sa.Group != nil) rg := sa.Group require_True(t, rg.node != nil) n := rg.node require_Equal(t, n.State(), Closed) } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) waitForStreamAssignments() // We manually stop the RAFT node and ensure it doesn't get restarted. rs := c.randomNonStreamLeader(globalAccountName, "TEST") sjs, acc, sa, mset := getStreamAssignment(rs) require_True(t, sa.Group != nil) rg := sa.Group require_True(t, rg.node != nil) n := rg.node n.Stop() n.WaitForStop() // We wait for the monitor to exit, so we can set the flag back manually. checkFor(t, 5*time.Second, time.Second, func() error { mset.mu.RLock() defer mset.mu.RUnlock() if mset.inMonitor { return errors.New("waiting for monitor to stop") } return nil }) mset.mu.Lock() mset.inMonitor = true mset.mu.Unlock() // The RAFT node should be closed. Checking health must not change that. // Simulates a race condition where we're shutting down, but we're still in the stream monitor. checkNodeIsClosed(sa) sjs.isStreamHealthy(acc, sa) checkNodeIsClosed(sa) err = js.DeleteStream("TEST") require_NoError(t, err) waitForNoStreamAssignments() // Underlying layer would be aware the health check made a copy. // So we sneakily set these values back, which simulates a race condition where // the health check is called while the deletion is in progress. This could happen // depending on how the locks are used. sjs.mu.Lock() sjs.cluster.streams = make(map[string]map[string]*streamAssignment) sjs.cluster.streams[globalAccountName] = make(map[string]*streamAssignment) sjs.cluster.streams[globalAccountName]["TEST"] = sa sa.Group.node = n sjs.mu.Unlock() // The underlying stream has been deleted. Checking health must not recreate the stream. checkNodeIsClosed(sa) sjs.isStreamHealthy(acc, sa) checkNodeIsClosed(sa) } func TestJetStreamClusterConsumerHealthCheckMustNotRecreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() waitForConsumerAssignments := func() { t.Helper() checkFor(t, 5*time.Second, time.Second, func() error { for _, s := range c.servers { if s.getJetStream().consumerAssignment(globalAccountName, "TEST", "CONSUMER") == nil { return fmt.Errorf("stream assignment not found on %s", s.Name()) } } return nil }) } waitForNoConsumerAssignments := func() { t.Helper() checkFor(t, 5*time.Second, time.Second, func() error { for _, s := range c.servers { if s.getJetStream().consumerAssignment(globalAccountName, "TEST", "CONSUMER") != nil { return fmt.Errorf("stream assignment still available on %s", s.Name()) } } return nil }) } getConsumerAssignment := func(rs *Server) (*jetStream, *streamAssignment, *consumerAssignment, *stream) { acc, err := rs.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NotNil(t, err) sjs := rs.getJetStream() sjs.mu.RLock() defer sjs.mu.RUnlock() sas := sjs.cluster.streams[globalAccountName] require_True(t, sas != nil) sa := sas["TEST"] require_True(t, sa != nil) ca := sa.consumers["CONSUMER"] require_True(t, ca != nil) ca.Created = time.Time{} return sjs, sa, ca, mset } checkNodeIsClosed := func(ca *consumerAssignment) { t.Helper() require_True(t, ca.Group != nil) rg := ca.Group require_True(t, rg.node != nil) n := rg.node require_Equal(t, n.State(), Closed) } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, Retention: nats.InterestPolicy, // Replicated consumers by default }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) require_NoError(t, err) waitForConsumerAssignments() // We manually stop the RAFT node and ensure it doesn't get restarted. rs := c.randomNonConsumerLeader(globalAccountName, "TEST", "CONSUMER") sjs, sa, ca, mset := getConsumerAssignment(rs) require_True(t, ca.Group != nil) rg := ca.Group require_True(t, rg.node != nil) n := rg.node n.Stop() n.WaitForStop() // The RAFT node should be closed. Checking health must not change that. // Simulates a race condition where we're shutting down. checkNodeIsClosed(ca) sjs.isConsumerHealthy(mset, "CONSUMER", ca) checkNodeIsClosed(ca) // We create a new RAFT group, the health check should detect this skew and restart. err = sjs.createRaftGroup(globalAccountName, ca.Group, MemoryStorage, pprofLabels{}) require_NoError(t, err) sjs.mu.Lock() // We set creating to now, since previously it would delete all data but NOT restart if created within <10s. ca.Created = time.Now() // Setting ca.pending, since a side effect of js.processConsumerAssignment is that it resets it. ca.pending = true sjs.mu.Unlock() sjs.isConsumerHealthy(mset, "CONSUMER", ca) require_False(t, ca.pending) err = js.DeleteConsumer("TEST", "CONSUMER") require_NoError(t, err) waitForNoConsumerAssignments() // Underlying layer would be aware the health check made a copy. // So we sneakily set these values back, which simulates a race condition where // the health check is called while the deletion is in progress. This could happen // depending on how the locks are used. sjs.mu.Lock() sjs.cluster.streams = make(map[string]map[string]*streamAssignment) sjs.cluster.streams[globalAccountName] = make(map[string]*streamAssignment) sjs.cluster.streams[globalAccountName]["TEST"] = sa ca.Created = time.Time{} ca.Group.node = n ca.deleted = false sjs.mu.Unlock() // The underlying consumer has been deleted. Checking health must not recreate the consumer. checkNodeIsClosed(ca) sjs.isConsumerHealthy(mset, "CONSUMER", ca) checkNodeIsClosed(ca) } func TestJetStreamClusterPeerRemoveStreamConsumerDesync(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Must have at least one message in the stream. _, err = js.Publish("foo", nil) require_NoError(t, err) sl := c.streamLeader(globalAccountName, "TEST") acc, err := sl.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) rs := c.randomNonStreamLeader(globalAccountName, "TEST") peer := rs.Name() rn := mset.raftNode() mset.mu.Lock() esm := encodeStreamMsgAllowCompress("foo", _EMPTY_, nil, nil, mset.clseq, time.Now().UnixNano(), false) mset.clseq++ mset.mu.Unlock() // Propose both remove peer and a normal entry within the same append entry. err = rn.ProposeMulti([]*Entry{ newEntry(EntryRemovePeer, []byte(peer)), newEntry(EntryNormal, esm), }) require_NoError(t, err) // If the previous normal entry was skipped, we'd get a seq mismatch error here. _, err = js.Publish("foo", nil) require_NoError(t, err) // Now check the same but for a consumer. _, err = js.PullSubscribe("foo", "CONSUMER") require_NoError(t, err) cl := c.consumerLeader(globalAccountName, "TEST", "CONSUMER") acc, err = cl.lookupAccount(globalAccountName) require_NoError(t, err) mset, err = acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) updateDeliveredBuffer := func() []byte { var b [4*binary.MaxVarintLen64 + 1]byte b[0] = byte(updateDeliveredOp) n := 1 n += binary.PutUvarint(b[n:], 100) n += binary.PutUvarint(b[n:], 100) n += binary.PutUvarint(b[n:], 1) n += binary.PutVarint(b[n:], time.Now().UnixNano()) return b[:n] } rs = c.randomNonConsumerLeader(globalAccountName, "TEST", "CONSUMER") peer = rs.Name() rn = o.raftNode() // Propose both remove peer and a normal entry within the same append entry. err = rn.ProposeMulti([]*Entry{ newEntry(EntryRemovePeer, []byte(peer)), newEntry(EntryNormal, updateDeliveredBuffer()), }) require_NoError(t, err) // Check the normal entry was applied. checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { o.mu.Lock() defer o.mu.Unlock() cs, err := o.store.State() if err != nil { return err } if cs.Delivered.Consumer != 100 || cs.Delivered.Stream != 100 { return fmt.Errorf("expected sequence 100, got: %v", cs.Delivered) } return nil }) } func TestJetStreamClusterStuckConsumerAfterLeaderChangeWithUnknownDeliveries(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Publish some messages into the stream. for i := 0; i < 3; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } // Ensure all servers are up-to-date. checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { return checkState(t, c, globalAccountName, "TEST") }) sub, err := js.PullSubscribe("foo", "CONSUMER") require_NoError(t, err) defer sub.Unsubscribe() // We only fetch 1 message here, since the condition is hard to trigger otherwise. // But, we're simulating fetching 3 messages and the consumer leader changing while // deliveries are happening. This will result in the new consumer leader not knowing // that the last two messages were also delivered (since we don't wait for quorum before delivering). msgs, err := sub.Fetch(1) require_NoError(t, err) require_Len(t, len(msgs), 1) // The client could send an acknowledgement, while the new consumer leader doesn't know about it // ever being delivered. It must NOT adjust any state and ignore the request to remain consistent. _, err = nc.Request("$JS.ACK.TEST.CONSUMER.1.3.3.0.0", nil, time.Second) require_Error(t, err, nats.ErrTimeout) // Acknowledging a message that is known to be delivered is accepted still. _, err = nc.Request("$JS.ACK.TEST.CONSUMER.1.1.1.0.0", nil, time.Second) require_NoError(t, err) // Check for consistent consumer info. ci, err := js.ConsumerInfo("TEST", "CONSUMER") require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 1) require_Equal(t, ci.Delivered.Stream, 1) require_Equal(t, ci.AckFloor.Consumer, 1) require_Equal(t, ci.AckFloor.Stream, 1) // Fetching for new messages MUST return the two messages the new consumer leader didn't // know were delivered before. If we wouldn't deliver these we'd have a stuck consumer. msgs, err = sub.Fetch(2) require_NoError(t, err) require_Len(t, len(msgs), 2) for _, msg := range msgs { require_NoError(t, msg.AckSync()) } // Check for consistent consumer info. ci, err = js.ConsumerInfo("TEST", "CONSUMER") require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 3) require_Equal(t, ci.Delivered.Stream, 3) require_Equal(t, ci.AckFloor.Consumer, 3) require_Equal(t, ci.AckFloor.Stream, 3) } // This is for when we are still using $SYS for NRG replication but we want to make sure // we track this in something visible to the end user. func TestJetStreamClusterAccountStatsForReplicatedStreams(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 5, }) require_NoError(t, err) // Let's connect to stream leader to make sent messages predictable, otherwise we get those to come up based on routing to stream leader. s := c.streamLeader(globalAccountName, "TEST") require_True(t, s != nil) nc, js = jsClientConnect(t, s) defer nc.Close() // NRG traffic can be compressed, so make this unique so we can check stats correctly. msg := make([]byte, 1024*1024) crand.Read(msg) // Publish some messages into the stream. for i := 0; i < 10; i++ { _, err = js.Publish("foo", msg) require_NoError(t, err) } time.Sleep(250 * time.Millisecond) // Now grab the account stats for us and make sure we account for the replicated messages. // Opts to grab our account. opts := &AccountStatzOptions{ Accounts: []string{globalAccountName}, } as, err := s.AccountStatz(opts) require_NoError(t, err) require_Equal(t, len(as.Accounts), 1) accStats := as.Accounts[0] // We need to account for possibility that the stream create was also on this server, hence the >= vs strict ==. require_True(t, accStats.Received.Msgs >= 10) require_True(t, accStats.Received.Bytes >= 1024*1024) // For sent, we will have 10 pub acks, and then should have 40 extra messages that are sent and accounted for // during the nrg propsal to the R5 peers. require_True(t, accStats.Sent.Msgs >= 50) require_True(t, accStats.Sent.Bytes >= accStats.Received.Bytes*4) } func TestJetStreamClusterRecreateConsumerFromMetaSnapshot(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Initial setup. _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) require_NoError(t, err) // Wait for all servers to be fully up-to-date. checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { if err = checkState(t, c, globalAccountName, "TEST"); err != nil { return err } for _, s := range c.servers { if acc, err := s.lookupAccount(globalAccountName); err != nil { return err } else if mset, err := acc.lookupStream("TEST"); err != nil { return err } else if o := mset.lookupConsumer("CONSUMER"); o == nil { return errors.New("consumer doesn't exist") } } return nil }) // Shutdown a random server. rs := c.randomServer() rs.Shutdown() rs.WaitForShutdown() // Recreate connection, since we could have shutdown the server we were connected to. nc.Close() c.waitOnLeader() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Recreate consumer. require_NoError(t, js.DeleteConsumer("TEST", "CONSUMER")) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) require_NoError(t, err) // Wait for all servers (except for the one that's down) to have recreated the consumer. var consumerRg string checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { consumerRg = _EMPTY_ for _, s := range c.servers { if s == rs { continue } if acc, err := s.lookupAccount(globalAccountName); err != nil { return err } else if mset, err := acc.lookupStream("TEST"); err != nil { return err } else if o := mset.lookupConsumer("CONSUMER"); o == nil { return errors.New("consumer doesn't exist") } else if ccrg := o.raftNode().Group(); consumerRg == _EMPTY_ { consumerRg = ccrg } else if consumerRg != ccrg { return errors.New("consumer raft groups don't match") } } return nil }) // Install snapshots on all remaining servers to "hide" the intermediate consumer recreate requests. for _, s := range c.servers { if s != rs { sjs := s.getJetStream() require_NotNil(t, sjs) snap, err := sjs.metaSnapshot() require_NoError(t, err) sjs.mu.RLock() meta := sjs.cluster.meta sjs.mu.RUnlock() require_NoError(t, meta.InstallSnapshot(snap)) } } // Restart the server, it should receive a meta snapshot and recognize the consumer recreation. rs = c.restartServer(rs) checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { consumerRg = _EMPTY_ for _, s := range c.servers { if acc, err := s.lookupAccount(globalAccountName); err != nil { return err } else if mset, err := acc.lookupStream("TEST"); err != nil { return err } else if o := mset.lookupConsumer("CONSUMER"); o == nil { return errors.New("consumer doesn't exist") } else if rn := o.raftNode(); rn == nil { return errors.New("consumer raft node doesn't exist") } else if ccrg := rn.Group(); ccrg == _EMPTY_ { return errors.New("consumer raft group doesn't exist") } else if consumerRg == _EMPTY_ { consumerRg = ccrg } else if consumerRg != ccrg { return errors.New("consumer raft groups don't match") } } return nil }) } // // DO NOT ADD NEW TESTS IN THIS FILE (unless to balance test times) // Add at the end of jetstream_cluster__test.go, with being the highest value. // nats-server-2.10.27/server/jetstream_cluster_2_test.go000066400000000000000000006232641477524627100230440ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests_2 // +build !skip_js_tests,!skip_js_cluster_tests_2 package server import ( "bytes" "context" crand "crypto/rand" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "math/rand" "os" "path/filepath" "reflect" "runtime" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats.go" ) func TestJetStreamClusterJSAPIImport(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterImportsTempl, "C1", 3) defer c.shutdown() // Client based API - This will connect to the non-js account which imports JS. // Connect below does an AccountInfo call. s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Note if this was ephemeral we would need to setup export/import for that subject. sub, err := js.SubscribeSync("TEST", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we can look up both. if _, err := js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := sub.ConsumerInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } // Names list.. var names []string for name := range js.StreamNames() { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(names)) } // Now send to stream. if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } sub, err = js.PullSubscribe("TEST", "tr") if err != nil { t.Fatalf("Unexpected error: %v", err) } msgs := fetchMsgs(t, sub, 1, 5*time.Second) m := msgs[0] if m.Subject != "TEST" { t.Fatalf("Expected subject of %q, got %q", "TEST", m.Subject) } if m.Header != nil { t.Fatalf("Expected no header on the message, got: %v", m.Header) } meta, err := m.Metadata() if err != nil { t.Fatalf("Unexpected error: %v", err) } if meta.Sequence.Consumer != 1 || meta.Sequence.Stream != 1 || meta.NumDelivered != 1 || meta.NumPending != 0 { t.Fatalf("Bad meta: %+v", meta) } js.Publish("TEST", []byte("Second")) js.Publish("TEST", []byte("Third")) checkFor(t, time.Second, 15*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "tr") if err != nil { return fmt.Errorf("Error getting consumer info: %v", err) } if ci.NumPending != 2 { return fmt.Errorf("NumPending still not 1: %v", ci.NumPending) } return nil }) // Ack across accounts. m, err = nc.Request("$JS.API.CONSUMER.MSG.NEXT.TEST.tr", []byte("+NXT"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } meta, err = m.Metadata() if err != nil { t.Fatalf("Unexpected error: %v", err) } if meta.Sequence.Consumer != 2 || meta.Sequence.Stream != 2 || meta.NumDelivered != 1 || meta.NumPending != 1 { t.Fatalf("Bad meta: %+v", meta) } // AckNext _, err = nc.Request(m.Reply, []byte("+NXT"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterMultiRestartBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10000 messages. msg, toSend := make([]byte, 4*1024), 10000 crand.Read(msg) for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return fmt.Errorf("Unexpected error: %v", err) } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected to have %d messages, got %d", toSend, si.State.Msgs) } return nil }) // For this bug, we will stop and remove the complete state from one server. s := c.randomServer() opts := s.getOpts() s.Shutdown() removeDir(t, opts.StoreDir) // Then restart it. c.restartAll() c.waitOnAllCurrent() c.waitOnStreamLeader("$G", "TEST") s = c.serverByName(s.Name()) c.waitOnStreamCurrent(s, "$G", "TEST") // Now restart them all.. c.stopAll() c.restartAll() c.waitOnLeader() c.waitOnStreamLeader("$G", "TEST") // Create new client. nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Make sure the replicas are current. js2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 20*time.Second, 250*time.Millisecond, func() error { si, _ := js2.StreamInfo("TEST") if si == nil || si.Cluster == nil { return fmt.Errorf("No stream info or cluster") } for _, pi := range si.Cluster.Replicas { if !pi.Current { return fmt.Errorf("Peer not current: %+v", pi) } } return nil }) } func TestJetStreamClusterServerLimits(t *testing.T) { // 2MB memory, 8MB disk c := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, "R3L", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() msg, toSend := make([]byte, 4*1024), 5000 crand.Read(msg) // Memory first. max_mem := uint64(2*1024*1024) + uint64(len(msg)) _, err := js.AddStream(&nats.StreamConfig{ Name: "TM", Replicas: 3, Storage: nats.MemoryStorage, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Grab stream leader. sl := c.streamLeader("ONE", "TM") require_True(t, sl != nil) for i := 0; i < toSend; i++ { if _, err = js.Publish("TM", msg); err != nil { break } } if err == nil || !strings.HasPrefix(err.Error(), "nats: insufficient resources") { t.Fatalf("Expected a ErrJetStreamResourcesExceeded error, got %v", err) } // Since we have run all servers out of resources, no leader will be elected to respond to stream info. // So check manually. acc, err := sl.LookupAccount("ONE") require_NoError(t, err) mset, err := acc.lookupStream("TM") require_NoError(t, err) if state := mset.state(); state.Bytes > max_mem { t.Fatalf("Expected bytes of %v to not be greater then %v", friendlyBytes(int64(state.Bytes)), friendlyBytes(int64(max_mem)), ) } c.waitOnLeader() // Now disk. max_disk := uint64(8*1024*1024) + uint64(len(msg)) _, err = js.AddStream(&nats.StreamConfig{ Name: "TF", Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Grab stream leader. sl = c.streamLeader("ONE", "TF") require_True(t, sl != nil) for i := 0; i < toSend; i++ { if _, err = js.Publish("TF", msg); err != nil { break } } if err == nil || !strings.HasPrefix(err.Error(), "nats: insufficient resources") { t.Fatalf("Expected a ErrJetStreamResourcesExceeded error, got %v", err) } // Since we have run all servers out of resources, no leader will be elected to respond to stream info. // So check manually. acc, err = sl.LookupAccount("ONE") require_NoError(t, err) mset, err = acc.lookupStream("TF") require_NoError(t, err) if state := mset.state(); state.Bytes > max_disk { t.Fatalf("Expected bytes of %v to not be greater then %v", friendlyBytes(int64(state.Bytes)), friendlyBytes(int64(max_disk)), ) } } func TestJetStreamClusterAccountLoadFailure(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, "R3L", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.leader()) defer nc.Close() // Remove the "ONE" account from non-leader s := c.randomNonLeader() s.mu.Lock() s.accounts.Delete("ONE") s.mu.Unlock() _, err := js.AddStream(&nats.StreamConfig{Name: "F", Replicas: 3}) if err == nil || !strings.Contains(err.Error(), "account not found") { t.Fatalf("Expected an 'account not found' error but got %v", err) } } func TestJetStreamClusterAckPendingWithExpired(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, MaxAge: 500 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 100 messages. msg, toSend := make([]byte, 256), 100 crand.Read(msg) for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci.NumAckPending != toSend { t.Fatalf("Expected %d to be pending, got %d", toSend, ci.NumAckPending) } // Wait for messages to expire. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected 0 msgs, got state: %+v", si.State) } return nil }) // Once expired these messages can not be redelivered so should not be considered ack pending at this point. // Now ack.. ci, err = sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci.NumAckPending != 0 { t.Fatalf("Expected nothing to be ack pending, got %d", ci.NumAckPending) } } func TestJetStreamClusterAckPendingWithMaxRedelivered(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 100 messages. msg, toSend := []byte("HELLO"), 100 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("foo", nats.MaxDeliver(2), nats.Durable("dlc"), nats.AckWait(10*time.Millisecond), nats.MaxAckPending(50), ) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend*2) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := sub.ConsumerInfo() if err != nil { return err } if ci.NumAckPending != 0 { return fmt.Errorf("Expected nothing to be ack pending, got %d", ci.NumAckPending) } return nil }) } func TestJetStreamClusterMixedMode(t *testing.T) { for _, test := range []struct { name string tmpl string }{ {"multi-account", jsClusterLimitsTempl}, {"global-account", jsMixedModeGlobalAccountTempl}, } { t.Run(test.name, func(t *testing.T) { c := createMixedModeCluster(t, test.tmpl, "MM5", _EMPTY_, 3, 2, true) defer c.shutdown() // Client based API - Non-JS server. nc, js := jsClientConnect(t, c.serverByName("S-5")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } ml := c.leader() if ml == nil { t.Fatalf("No metaleader") } // Make sure we are tracking only the JS peers. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { peers := ml.JetStreamClusterPeers() if len(peers) == 3 { return nil } return fmt.Errorf("Not correct number of peers, expected %d, got %d", 3, len(peers)) }) // Grab the underlying raft structure and make sure the system adjusts its cluster set size. meta := ml.getJetStream().getMetaGroup().(*raft) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ps := meta.currentPeerState() if len(ps.knownPeers) != 3 { return fmt.Errorf("Expected known peers to be 3, but got %+v", ps.knownPeers) } if ps.clusterSize < 3 { return fmt.Errorf("Expected cluster size to be 3, but got %+v", ps) } return nil }) }) } } func TestJetStreamClusterLeafnodeSpokes(t *testing.T) { c := createJetStreamCluster(t, jsClusterTempl, "HUB", _EMPTY_, 3, 22020, false) defer c.shutdown() lnc1 := c.createLeafNodesWithStartPortAndDomain("R1", 3, 22110, _EMPTY_) defer lnc1.shutdown() lnc2 := c.createLeafNodesWithStartPortAndDomain("R2", 3, 22120, _EMPTY_) defer lnc2.shutdown() lnc3 := c.createLeafNodesWithStartPortAndDomain("R3", 3, 22130, _EMPTY_) defer lnc3.shutdown() // Wait on all peers. c.waitOnPeerCount(12) // Make sure shrinking works. lnc3.shutdown() c.waitOnPeerCount(9) lnc3 = c.createLeafNodesWithStartPortAndDomain("LNC3", 3, 22130, _EMPTY_) defer lnc3.shutdown() c.waitOnPeerCount(12) } func TestJetStreamClusterLeafNodeDenyNoDupe(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: CORE, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 18033, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) ln := c.createLeafNodeWithTemplate("LN-SPOKE", tmpl) defer ln.Shutdown() checkLeafNodeConnectedCount(t, ln, 2) // Now disconnect our leafnode connections by restarting the server we are connected to.. for _, s := range c.servers { if s.ClusterName() != c.name { continue } if nln := s.NumLeafNodes(); nln > 0 { s.Shutdown() c.restartServer(s) } } // Make sure we are back connected. checkLeafNodeConnectedCount(t, ln, 2) // Now grab leaf varz and make sure we have no dupe deny clauses. vz, err := ln.Varz(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Grab the correct remote. for _, remote := range vz.LeafNode.Remotes { if remote.LocalAccount == ln.SystemAccount().Name { if remote.Deny != nil && len(remote.Deny.Exports) > 3 { // denyAll := []string{jscAllSubj, raftAllSubj, jsAllAPI} t.Fatalf("Dupe entries found: %+v", remote.Deny) } break } } } // Multiple JS domains. func TestJetStreamClusterSingleLeafNodeWithoutSharedSystemAccount(t *testing.T) { c := createJetStreamCluster(t, strings.Replace(jsClusterAccountsTempl, "store_dir", "domain: CORE, store_dir", 1), "HUB", _EMPTY_, 3, 14333, true) defer c.shutdown() ln := c.createSingleLeafNodeNoSystemAccount() defer ln.Shutdown() // The setup here has a single leafnode server with two accounts. One has JS, the other does not. // We want to to test the following. // 1. For the account without JS, we simply will pass through to the HUB. Meaning since our local account // does not have it, we simply inherit the hub's by default. // 2. For the JS enabled account, we are isolated and use our local one only. // Check behavior of the account without JS. // Normally this should fail since our local account is not enabled. However, since we are bridging // via the leafnode we expect this to work here. nc, js := jsClientConnectEx(t, ln, []nats.JSOpt{nats.Domain("CORE")}, nats.UserInfo("n", "p")) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil || si.Cluster.Name != "HUB" { t.Fatalf("Expected stream to be placed in %q", "HUB") } // Do some other API calls. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "C1", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } seen := 0 for name := range js.StreamNames() { seen++ if name != "TEST" { t.Fatalf("Expected only %q but got %q", "TEST", name) } } if seen != 1 { t.Fatalf("Expected only 1 stream, got %d", seen) } if _, err := js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } if err := js.DeleteConsumer("TEST", "C1"); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.UpdateStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"bar"}, Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now check the enabled account. // Check the enabled account only talks to its local JS domain by default. nc, js = jsClientConnect(t, ln, nats.UserInfo("y", "p")) defer nc.Close() sub, err := nc.SubscribeSync(JSAdvisoryStreamCreatedPre + ".>") if err != nil { t.Fatalf("Unexpected error: %v", err) } si, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster != nil { t.Fatalf("Expected no cluster designation for stream since created on single LN server") } // Wait for a bit and make sure we only get one of these. // The HUB domain should be cut off by default. time.Sleep(250 * time.Millisecond) checkSubsPending(t, sub, 1) // Drain. for _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) { } // Now try to talk to the HUB JS domain through a new context that uses a different mapped subject. // This is similar to how we let users cross JS domains between accounts as well. js, err = nc.JetStream(nats.APIPrefix("$JS.HUB.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } // This should fail here with jetstream not enabled. if _, err := js.AccountInfo(); err != nats.ErrJetStreamNotEnabled { t.Fatalf("Unexpected error: %v", err) } // Now add in a mapping to the connected account in the HUB. // This aligns with the APIPrefix context above and works across leafnodes. // TODO(dlc) - Should we have a mapping section for leafnode solicit? c.addSubjectMapping("ONE", "$JS.HUB.API.>", "$JS.API.>") // Now it should work. if _, err := js.AccountInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we can add a stream, etc. si, err = js.AddStream(&nats.StreamConfig{ Name: "TEST22", Subjects: []string{"bar"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil || si.Cluster.Name != "HUB" { t.Fatalf("Expected stream to be placed in %q", "HUB") } jsLocal, err := nc.JetStream() if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } // Create a mirror on the local leafnode for stream TEST22. _, err = jsLocal.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{ Name: "TEST22", External: &nats.ExternalStream{APIPrefix: "$JS.HUB.API"}, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Publish a message to the HUB's TEST22 stream. if _, err := js.Publish("bar", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Make sure the message arrives in our mirror. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := jsLocal.StreamInfo("M") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) // Now do the reverse and create a sourced stream in the HUB from our local stream on leafnode. // Inside the HUB we need to be able to find our local leafnode JetStream assets, so we need // a mapping in the LN server to allow this to work. Normally this will just be in server config. acc, err := ln.LookupAccount("JSY") if err != nil { c.t.Fatalf("Unexpected error on %v: %v", ln, err) } if err := acc.AddMapping("$JS.LN.API.>", "$JS.API.>"); err != nil { c.t.Fatalf("Error adding mapping: %v", err) } // js is the HUB JetStream context here. _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{ Name: "M", External: &nats.ExternalStream{APIPrefix: "$JS.LN.API"}, }}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure the message arrives in our sourced stream. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) } // JetStream Domains func TestJetStreamClusterDomains(t *testing.T) { // This adds in domain config option to template. tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: CORE, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 12232, true) defer c.shutdown() // This leafnode is a single server with no domain but sharing the system account. // This extends the CORE domain through this leafnode. ln := c.createLeafNodeWithTemplate("LN-SYS", strings.ReplaceAll(jsClusterTemplWithSingleLeafNode, "store_dir:", "extension_hint: will_extend, domain: CORE, store_dir:")) defer ln.Shutdown() checkLeafNodeConnectedCount(t, ln, 2) // This shows we have extended this system. c.waitOnPeerCount(4) if ml := c.leader(); ml == ln { t.Fatalf("Detected a meta-leader in the leafnode: %s", ml) } // Now create another LN but with a domain defined. tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) spoke := c.createLeafNodeWithTemplate("LN-SPOKE", tmpl) defer spoke.Shutdown() checkLeafNodeConnectedCount(t, spoke, 2) // Should be the same, should not extend the CORE domain. c.waitOnPeerCount(4) // The domain signals to the system that we are our own JetStream domain and should not extend CORE. // We want to check to make sure we have all the deny properly setup. spoke.mu.Lock() // var hasDE, hasDI bool for _, ln := range spoke.leafs { ln.mu.Lock() remote := ln.leaf.remote ln.mu.Unlock() remote.RLock() if remote.RemoteLeafOpts.LocalAccount == "$SYS" { for _, s := range denyAllJs { if r := ln.perms.pub.deny.Match(s); len(r.psubs) != 1 { t.Fatalf("Expected to have deny permission for %s", s) } if r := ln.perms.sub.deny.Match(s); len(r.psubs) != 1 { t.Fatalf("Expected to have deny permission for %s", s) } } } remote.RUnlock() } spoke.mu.Unlock() // Now do some operations. // Check the enabled account only talks to its local JS domain by default. nc, js := jsClientConnect(t, spoke) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster != nil { t.Fatalf("Expected no cluster designation for stream since created on single LN server") } // Now try to talk to the CORE JS domain through a new context that uses a different mapped subject. jsCore, err := nc.JetStream(nats.APIPrefix("$JS.CORE.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } if _, err := jsCore.AccountInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we can add a stream, etc. si, err = jsCore.AddStream(&nats.StreamConfig{ Name: "TEST22", Subjects: []string{"bar"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster == nil || si.Cluster.Name != "CORE" { t.Fatalf("Expected stream to be placed in %q, got %q", "CORE", si.Cluster.Name) } jsLocal, err := nc.JetStream() if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } // Create a mirror on our local leafnode for stream TEST22. _, err = jsLocal.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{ Name: "TEST22", External: &nats.ExternalStream{APIPrefix: "$JS.CORE.API"}, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Publish a message to the CORE's TEST22 stream. if _, err := jsCore.Publish("bar", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Make sure the message arrives in our mirror. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := jsLocal.StreamInfo("M") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) // jsCore is the CORE JetStream domain. // Create a sourced stream in the CORE that is sourced from our mirror stream in our leafnode. _, err = jsCore.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{ Name: "M", External: &nats.ExternalStream{APIPrefix: "$JS.SPOKE.API"}, }}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure the message arrives in our sourced stream. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := jsCore.StreamInfo("S") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) // Now connect directly to the CORE cluster and make sure we can operate there. nc, jsLocal = jsClientConnect(t, c.randomServer()) defer nc.Close() // Create the js contexts again. jsSpoke, err := nc.JetStream(nats.APIPrefix("$JS.SPOKE.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } // Publish a message to the CORE's TEST22 stream. if _, err := jsLocal.Publish("bar", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Make sure the message arrives in our mirror. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := jsSpoke.StreamInfo("M") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs, got state: %+v", si.State) } return nil }) // Make sure the message arrives in our sourced stream. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := jsLocal.StreamInfo("S") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs, got state: %+v", si.State) } return nil }) // We are connected to the CORE domain/system. Create a JetStream context referencing ourselves. jsCore, err = nc.JetStream(nats.APIPrefix("$JS.CORE.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } si, err = jsCore.StreamInfo("S") if err != nil { t.Fatalf("Could not get stream info: %v", err) } if si.State.Msgs != 2 { t.Fatalf("Expected 2 msgs, got state: %+v", si.State) } } func TestJetStreamClusterDomainsWithNoJSHub(t *testing.T) { // Create our hub cluster with no JetStream defined. c := createMixedModeCluster(t, jsClusterAccountsTempl, "NOJS5", _EMPTY_, 0, 5, false) defer c.shutdown() ln := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStream() defer ln.Shutdown() lnd := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain("SPOKE", "nojs") defer lnd.Shutdown() // Client based API - Connected to the core cluster with no JS but account has JS. s := c.randomServer() // Make sure the JS interest from the LNs has made it to this server. checkSubInterest(t, s, "NOJS", "$JS.SPOKE.API.INFO", time.Second) nc, _ := jsClientConnect(t, s, nats.UserInfo("nojs", "p")) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do by hand to make sure we only get one response. sis := fmt.Sprintf(strings.ReplaceAll(JSApiStreamCreateT, JSApiPrefix, "$JS.SPOKE.API"), "TEST") rs := nats.NewInbox() sub, _ := nc.SubscribeSync(rs) nc.PublishRequest(sis, rs, req) // Wait for response. checkSubsPending(t, sub, 1) // Double check to make sure we only have 1. if nr, _, err := sub.Pending(); err != nil || nr != 1 { t.Fatalf("Expected 1 response, got %d and %v", nr, err) } resp, err := sub.NextMsg(time.Second) require_NoError(t, err) // This StreamInfo should *not* have a domain set. // Do by hand until this makes it to the Go client. var si StreamInfo if err = json.Unmarshal(resp.Data, &si); err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Domain != _EMPTY_ { t.Fatalf("Expected to have NO domain set but got %q", si.Domain) } // Now let's create a stream specifically on the SPOKE domain. js, err := nc.JetStream(nats.Domain("SPOKE")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST22", Subjects: []string{"bar"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now lookup by hand to check domain. resp, err = nc.Request("$JS.SPOKE.API.STREAM.INFO.TEST22", nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err = json.Unmarshal(resp.Data, &si); err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Domain != "SPOKE" { t.Fatalf("Expected to have domain set to %q but got %q", "SPOKE", si.Domain) } } // Issue #2205 func TestJetStreamClusterDomainsAndAPIResponses(t *testing.T) { // This adds in domain config option to template. tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: CORE, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 12232, true) defer c.shutdown() // Now create spoke LN cluster. tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "SPOKE", 5, 23913) defer lnc.shutdown() lnc.waitOnClusterReady() // Make the physical connection to the CORE. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now create JS domain context and try to do same in LN cluster. // The issue referenced above details a bug where we can not receive a positive response. nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() jsSpoke, err := nc.JetStream(nats.APIPrefix("$JS.SPOKE.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } si, err := jsSpoke.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != "SPOKE" { t.Fatalf("Expected %q as the cluster, got %q", "SPOKE", si.Cluster.Name) } } // Issue #2202 func TestJetStreamClusterDomainsAndSameNameSources(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: CORE, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 9323, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE-1, store_dir:", 1) spoke1 := c.createLeafNodeWithTemplate("LN-SPOKE-1", tmpl) defer spoke1.Shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE-2, store_dir:", 1) spoke2 := c.createLeafNodeWithTemplate("LN-SPOKE-2", tmpl) defer spoke2.Shutdown() checkLeafNodeConnectedCount(t, spoke1, 2) checkLeafNodeConnectedCount(t, spoke2, 2) subjFor := func(s *Server) string { switch s { case spoke1: return "foo" case spoke2: return "bar" } return "TEST" } // Create the same name stream in both spoke domains. for _, s := range []*Server{spoke1, spoke2} { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{subjFor(s)}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } nc.Close() } // Now connect to the hub and create a sourced stream from both leafnode streams. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{ { Name: "TEST", External: &nats.ExternalStream{APIPrefix: "$JS.SPOKE-1.API"}, }, { Name: "TEST", External: &nats.ExternalStream{APIPrefix: "$JS.SPOKE-2.API"}, }, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Publish a message to each spoke stream and we will check that our sourced stream gets both. for _, s := range []*Server{spoke1, spoke2} { nc, js := jsClientConnect(t, s) defer nc.Close() js.Publish(subjFor(s), []byte("DOUBLE TROUBLE")) si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 1 { t.Fatalf("Expected 1 msg, got %d", si.State.Msgs) } nc.Close() } // Now make sure we have 2 msgs in our sourced stream. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo("S") require_NoError(t, err) if si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs, got %d", si.State.Msgs) } return nil }) // Make sure we can see our external information. // This not in the Go client yet so manual for now. resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "S"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ssi StreamInfo if err = json.Unmarshal(resp.Data, &ssi); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(ssi.Sources) != 2 { t.Fatalf("Expected 2 source streams, got %d", len(ssi.Sources)) } if ssi.Sources[0].External == nil { t.Fatalf("Expected a non-nil external designation") } pre := ssi.Sources[0].External.ApiPrefix if pre != "$JS.SPOKE-1.API" && pre != "$JS.SPOKE-2.API" { t.Fatalf("Expected external api of %q, got %q", "$JS.SPOKE-[1|2].API", ssi.Sources[0].External.ApiPrefix) } // Also create a mirror. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{ Name: "TEST", External: &nats.ExternalStream{APIPrefix: "$JS.SPOKE-1.API"}, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "M"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err = json.Unmarshal(resp.Data, &ssi); err != nil { t.Fatalf("Unexpected error: %v", err) } if ssi.Mirror == nil || ssi.Mirror.External == nil { t.Fatalf("Expected a non-nil external designation for our mirror") } if ssi.Mirror.External.ApiPrefix != "$JS.SPOKE-1.API" { t.Fatalf("Expected external api of %q, got %q", "$JS.SPOKE-1.API", ssi.Sources[0].External.ApiPrefix) } } // When a leafnode enables JS on an account that is not enabled on the remote cluster account this should fail // Accessing a jet stream in a different availability domain requires the client provide a damain name, or // the server having set up appropriate defaults (default_js_domain. tested in leafnode_test.go) func TestJetStreamClusterSingleLeafNodeEnablingJetStream(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 11322, true) defer c.shutdown() ln := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStream() defer ln.Shutdown() // Check that we have JS in the $G account on the leafnode. nc, js := jsClientConnect(t, ln) defer nc.Close() if _, err := js.AccountInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } // Connect our client to the "nojs" account in the cluster but make sure JS works since its enabled via the leafnode. s := c.randomServer() nc, js = jsClientConnect(t, s, nats.UserInfo("nojs", "p")) defer nc.Close() _, err := js.AccountInfo() // error is context deadline exceeded as the local account has no js and can't reach the remote one require_True(t, err == context.DeadlineExceeded) } func TestJetStreamClusterLeafNodesWithoutJS(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 11233, true) defer c.shutdown() testJS := func(s *Server, domain string, doDomainAPI bool) { nc, js := jsClientConnect(t, s) defer nc.Close() if doDomainAPI { var err error apiPre := fmt.Sprintf("$JS.%s.API", domain) if js, err = nc.JetStream(nats.APIPrefix(apiPre)); err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } } ai, err := js.AccountInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } if ai.Domain != domain { t.Fatalf("Expected domain of %q, got %q", domain, ai.Domain) } } ln := c.createLeafNodeWithTemplate("LN-SYS-S-NOJS", jsClusterTemplWithSingleLeafNodeNoJS) defer ln.Shutdown() checkLeafNodeConnectedCount(t, ln, 2) // Check that we can access JS in the $G account on the cluster through the leafnode. testJS(ln, "HUB", true) ln.Shutdown() // Now create a leafnode cluster with No JS and make sure that works. lnc := c.createLeafNodesNoJS("LN-SYS-C-NOJS", 3) defer lnc.shutdown() testJS(lnc.randomServer(), "HUB", true) lnc.shutdown() // Do mixed mode but with a JS config block that specifies domain and just sets it to disabled. // This is the preferred method for mixed mode, always define JS server config block just disable // in those you do not want it running. // e.g. jetstream: {domain: "SPOKE", enabled: false} tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) lncm := c.createLeafNodesWithTemplateMixedMode(tmpl, "SPOKE", 3, 2, true) defer lncm.shutdown() // Now grab a non-JS server, last two are non-JS. sl := lncm.servers[0] testJS(sl, "SPOKE", false) // Test that mappings work as well and we can access the hub. testJS(sl, "HUB", true) } func TestJetStreamClusterLeafNodesWithSameDomainNames(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 11233, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: HUB, store_dir:", 1) lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "SPOKE", 3, 11311) defer lnc.shutdown() c.waitOnPeerCount(6) } func TestJetStreamClusterLeafDifferentAccounts(t *testing.T) { c := createJetStreamCluster(t, jsClusterAccountsTempl, "HUB", _EMPTY_, 2, 23133, false) defer c.shutdown() ln := c.createLeafNodesWithStartPortAndDomain("LN", 2, 22110, _EMPTY_) defer ln.shutdown() // Wait on all peers. c.waitOnPeerCount(4) nc, js := jsClientConnect(t, ln.randomServer()) defer nc.Close() // Make sure we can properly identify the right account when the leader received the request. // We need to map the client info header to the new account once received by the hub. if _, err := js.AccountInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterStreamInfoDeletedDetails(t *testing.T) { c := createJetStreamClusterExplicit(t, "R2", 2) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("HELLO"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now remove some messages. deleteMsg := func(seq uint64) { if err := js.DeleteMsg("TEST", seq); err != nil { t.Fatalf("Unexpected error: %v", err) } } deleteMsg(2) deleteMsg(4) deleteMsg(6) // Need to do these via direct server request for now. resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var si StreamInfo if err = json.Unmarshal(resp.Data, &si); err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.NumDeleted != 3 { t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted) } if len(si.State.Deleted) != 0 { t.Fatalf("Expected not deleted details, but got %+v", si.State.Deleted) } // Now request deleted details. req := JSApiStreamInfoRequest{DeletedDetails: true} b, _ := json.Marshal(req) resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), b, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err = json.Unmarshal(resp.Data, &si); err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.NumDeleted != 3 { t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted) } if len(si.State.Deleted) != 3 { t.Fatalf("Expected deleted details, but got %+v", si.State.Deleted) } } func TestJetStreamClusterMirrorAndSourceExpiration(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSE", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Origin if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } bi := 1 sendBatch := func(n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { msg := fmt.Sprintf("ID: %d", bi) bi++ if _, err := js.PublishAsync("TEST", []byte(msg)); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkStream := func(stream string, num uint64) { t.Helper() checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo(stream) if err != nil { return err } if si.State.Msgs != num { return fmt.Errorf("Expected %d msgs, got %d", num, si.State.Msgs) } return nil }) } checkSource := func(num uint64) { t.Helper(); checkStream("S", num) } checkMirror := func(num uint64) { t.Helper(); checkStream("M", num) } checkTest := func(num uint64) { t.Helper(); checkStream("TEST", num) } var err error _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, Replicas: 2, MaxAge: 500 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // We want this to not be same as TEST leader for this test. sl := c.streamLeader("$G", "TEST") for ss := sl; ss == sl; { _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{Name: "TEST"}}, Replicas: 2, MaxAge: 500 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ss = c.streamLeader("$G", "S"); ss == sl { // Delete and retry. js.DeleteStream("S") } } // Allow direct sync consumers to connect. time.Sleep(1500 * time.Millisecond) sendBatch(100) checkTest(100) checkMirror(100) checkSource(100) // Make sure they expire. checkMirror(0) checkSource(0) // Now stop the server housing the leader of the source stream. sl.Shutdown() c.restartServer(sl) checkClusterFormed(t, c.servers...) c.waitOnStreamLeader("$G", "S") c.waitOnStreamLeader("$G", "M") // Make sure can process correctly after we have expired all of the messages. sendBatch(100) // Need to check both in parallel. scheck, mcheck := uint64(0), uint64(0) checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { if scheck != 100 { if si, _ := js.StreamInfo("S"); si != nil { scheck = si.State.Msgs } } if mcheck != 100 { if si, _ := js.StreamInfo("M"); si != nil { mcheck = si.State.Msgs } } if scheck == 100 && mcheck == 100 { return nil } return fmt.Errorf("Both not at 100 yet, S=%d, M=%d", scheck, mcheck) }) checkTest(200) } func TestJetStreamClusterMirrorAndSourceSubLeaks(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSL", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() startSubs := c.stableTotalSubs() var ss []*nats.StreamSource // Create 10 origin streams for i := 0; i < 10; i++ { sn := fmt.Sprintf("ORDERS-%d", i+1) ss = append(ss, &nats.StreamSource{Name: sn}) if _, err := js.AddStream(&nats.StreamConfig{Name: sn}); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Create mux'd stream that sources all of the origin streams. _, err := js.AddStream(&nats.StreamConfig{ Name: "MUX", Replicas: 2, Sources: ss, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now create a mirror of the mux stream. _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR", Replicas: 2, Mirror: &nats.StreamSource{Name: "MUX"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Get stable subs count. afterSubs := c.stableTotalSubs() js.DeleteStream("MIRROR") js.DeleteStream("MUX") for _, si := range ss { js.DeleteStream(si.Name) } // Some subs take longer to settle out so we give ourselves a small buffer. // There will be 1 sub for client on each server (such as _INBOX.IvVJ2DOXUotn4RUSZZCFvp.*) // and 2 or 3 subs such as `_R_.xxxxx.>` on each server, so a total of 12 subs. if deleteSubs := c.stableTotalSubs(); deleteSubs > startSubs+12 { t.Fatalf("Expected subs to return to %d from a high of %d, but got %d", startSubs, afterSubs, deleteSubs) } } func TestJetStreamClusterCreateConcurrentDurableConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSL", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Create origin stream, must be R > 1 if _, err := js.AddStream(&nats.StreamConfig{Name: "ORDERS", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.QueueSubscribeSync("ORDERS", "wq", nats.Durable("shared")); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now try to create durables concurrently. start := make(chan struct{}) var wg sync.WaitGroup created := uint32(0) errs := make(chan error, 10) for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() <-start _, err := js.QueueSubscribeSync("ORDERS", "wq", nats.Durable("shared")) if err == nil { atomic.AddUint32(&created, 1) } else if !strings.Contains(err.Error(), "consumer name already") { errs <- err } }() } close(start) wg.Wait() if lc := atomic.LoadUint32(&created); lc != 10 { t.Fatalf("Expected all 10 to be created, got %d", lc) } if len(errs) > 0 { t.Fatalf("Failed to create some sub: %v", <-errs) } } // https://github.com/nats-io/nats-server/issues/2144 func TestJetStreamClusterUpdateStreamToExisting(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSL", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS1", Replicas: 3, Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "ORDERS2", Replicas: 3, Subjects: []string{"bar"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.UpdateStream(&nats.StreamConfig{ Name: "ORDERS2", Replicas: 3, Subjects: []string{"foo"}, }) if err == nil { t.Fatalf("Expected an error but got none") } } func TestJetStreamClusterCrossAccountInterop(t *testing.T) { template := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: HUB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { JS { jetstream: enabled users = [ { user: "rip", pass: "pass" } ] exports [ { service: "$JS.API.CONSUMER.INFO.>" } { service: "$JS.HUB.API.CONSUMER.>", response: stream } { stream: "M.SYNC.>" } # For the mirror ] } IA { jetstream: enabled users = [ { user: "dlc", pass: "pass" } ] imports [ { service: { account: JS, subject: "$JS.API.CONSUMER.INFO.TEST.DLC"}, to: "FROM.DLC" } { service: { account: JS, subject: "$JS.HUB.API.CONSUMER.>"}, to: "js.xacc.API.CONSUMER.>" } { stream: { account: JS, subject: "M.SYNC.>"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := createJetStreamClusterWithTemplate(t, template, "HUB", 3) defer c.shutdown() // Create the stream and the consumer under the JS/rip user. s := c.randomServer() nc, js := jsClientConnect(t, s, nats.UserInfo("rip", "pass")) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "DLC", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Also create a stream via the domain qualified API. js, err = nc.JetStream(nats.APIPrefix("$JS.HUB.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "ORDERS", Replicas: 2}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now we want to access the consumer info from IA/dlc. nc2, js2 := jsClientConnect(t, c.randomServer(), nats.UserInfo("dlc", "pass")) defer nc2.Close() if _, err := nc2.Request("FROM.DLC", nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure domain mappings etc work across accounts. // Setup a mirror. _, err = js2.AddStream(&nats.StreamConfig{ Name: "MIRROR", Mirror: &nats.StreamSource{ Name: "ORDERS", External: &nats.ExternalStream{ APIPrefix: "js.xacc.API", DeliverPrefix: "M.SYNC", }, }, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send 10 messages.. msg, toSend := []byte("Hello mapped domains"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("ORDERS", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("MIRROR") if err != nil { return fmt.Errorf("Unexpected error: %v", err) } if si.State.Msgs != 10 { return fmt.Errorf("Expected 10 msgs, got state: %+v", si.State) } return nil }) } // https://github.com/nats-io/nats-server/issues/2242 func TestJetStreamClusterMsgIdDuplicateBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSL", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sendMsgID := func(id string) (*nats.PubAck, error) { t.Helper() m := nats.NewMsg("foo") m.Header.Add(JSMsgId, id) m.Data = []byte("HELLO WORLD") return js.PublishMsg(m) } if _, err := sendMsgID("1"); err != nil { t.Fatalf("Unexpected error: %v", err) } // This should fail with duplicate detected. if pa, _ := sendMsgID("1"); pa == nil || !pa.Duplicate { t.Fatalf("Expected duplicate but got none: %+v", pa) } // This should be fine. if _, err := sendMsgID("2"); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterNilMsgWithHeaderThroughSourcedStream(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 12232, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) spoke := c.createLeafNodeWithTemplate("SPOKE", tmpl) defer spoke.Shutdown() checkLeafNodeConnectedCount(t, spoke, 2) // Client for API requests. nc, js := jsClientConnect(t, spoke) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } jsHub, err := nc.JetStream(nats.APIPrefix("$JS.HUB.API")) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } _, err = jsHub.AddStream(&nats.StreamConfig{ Name: "S", Replicas: 2, Sources: []*nats.StreamSource{{ Name: "TEST", External: &nats.ExternalStream{APIPrefix: "$JS.SPOKE.API"}, }}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now send a message to the origin stream with nil body and a header. m := nats.NewMsg("foo") m.Header.Add("X-Request-ID", "e9a639b4-cecb-4fbe-8376-1ef511ae1f8d") m.Data = []byte("HELLO WORLD") if _, err = jsHub.PublishMsg(m); err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := jsHub.SubscribeSync("foo", nats.BindStream("S")) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if string(msg.Data) != "HELLO WORLD" { t.Fatalf("Message corrupt? Expecting %q got %q", "HELLO WORLD", msg.Data) } } // Make sure varz reports the server usage not replicated usage etc. func TestJetStreamClusterVarzReporting(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // ~100k per message. msg := []byte(strings.Repeat("A", 99_960)) msz := fileStoreMsgSize("TEST", nil, msg) total := msz * 10 for i := 0; i < 10; i++ { if _, err := js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // To show the bug we need this to allow remote usage to replicate. time.Sleep(2 * usageTick) v, err := s.Varz(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } if v.JetStream.Stats.Store > total { t.Fatalf("Single server varz JetStream store usage should be <= %d, got %d", total, v.JetStream.Stats.Store) } info, err := js.AccountInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Store < total*3 { t.Fatalf("Expected account information to show usage ~%d, got %d", total*3, info.Store) } } func TestJetStreamClusterPurgeBySequence(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.*.*"}, Storage: st, Replicas: 2, MaxMsgsPer: 5, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) for i := 0; i < 20; i++ { if _, err = js.Publish("kv.myapp.username", []byte(fmt.Sprintf("value %d", i))); err != nil { t.Fatalf("request failed: %s", err) } } for i := 0; i < 20; i++ { if _, err = js.Publish("kv.myapp.password", []byte(fmt.Sprintf("value %d", i))); err != nil { t.Fatalf("request failed: %s", err) } } expectSequences := func(t *testing.T, subject string, seq ...int) { sub, err := js.SubscribeSync(subject) if err != nil { t.Fatalf("sub failed: %s", err) } defer sub.Unsubscribe() for _, i := range seq { msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("didn't get message: %s", err) } meta, err := msg.Metadata() if err != nil { t.Fatalf("didn't get metadata: %s", err) } if meta.Sequence.Stream != uint64(i) { t.Fatalf("expected sequence %d got %d", i, meta.Sequence.Stream) } } } expectSequences(t, "kv.myapp.username", 16, 17, 18, 19, 20) expectSequences(t, "kv.myapp.password", 36, 37, 38, 39, 40) // delete up to but not including 18 of username... jr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "kv.myapp.username", Sequence: 18}) _, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "KV"), jr, time.Second) if err != nil { t.Fatalf("request failed: %s", err) } // 18 should still be there expectSequences(t, "kv.myapp.username", 18, 19, 20) }) } } func TestJetStreamClusterMaxConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "MAXC", Storage: nats.MemoryStorage, Subjects: []string{"in.maxc.>"}, MaxConsumers: 1, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } si, err := js.StreamInfo("MAXC") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Config.MaxConsumers != 1 { t.Fatalf("Expected max of 1, got %d", si.Config.MaxConsumers) } // Make sure we get the right error. // This should succeed. if _, err := js.SubscribeSync("in.maxc.foo"); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.SubscribeSync("in.maxc.bar"); err == nil { t.Fatalf("Eexpected error but got none") } } func TestJetStreamClusterMaxConsumersMultipleConcurrentRequests(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "MAXCC", Storage: nats.MemoryStorage, Subjects: []string{"in.maxcc.>"}, MaxConsumers: 1, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } si, err := js.StreamInfo("MAXCC") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Config.MaxConsumers != 1 { t.Fatalf("Expected max of 1, got %d", si.Config.MaxConsumers) } startCh := make(chan bool) var wg sync.WaitGroup wg.Add(10) for n := 0; n < 10; n++ { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() go func(js nats.JetStreamContext) { defer wg.Done() <-startCh js.SubscribeSync("in.maxcc.foo") }(js) } // Wait for Go routines. time.Sleep(250 * time.Millisecond) close(startCh) wg.Wait() var names []string for n := range js.ConsumerNames("MAXCC") { names = append(names, n) } if nc := len(names); nc > 1 { t.Fatalf("Expected only 1 consumer, got %d", nc) } metaLeader := c.leader() mjs := metaLeader.getJetStream() mjs.mu.RLock() sa := mjs.streamAssignment(globalAccountName, "MAXCC") require_NotNil(t, sa) for _, ca := range sa.consumers { require_False(t, ca.pending) } mjs.mu.RUnlock() } func TestJetStreamClusterAccountMaxStreamsAndConsumersMultipleConcurrentRequests(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { A { jetstream { max_file: 9663676416 max_streams: 2 max_consumers: 1 } users = [ { user: "a", pass: "pwd" } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := createJetStreamClusterWithTemplate(t, tmpl, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo("a", "pwd")) defer nc.Close() cfg := &nats.StreamConfig{ Name: "MAXCC", Storage: nats.MemoryStorage, Subjects: []string{"in.maxcc.>"}, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } si, err := js.StreamInfo("MAXCC") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Config.MaxConsumers != -1 { t.Fatalf("Expected max of -1, got %d", si.Config.MaxConsumers) } startCh := make(chan bool) var wg sync.WaitGroup wg.Add(10) for n := 0; n < 10; n++ { nc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo("a", "pwd")) defer nc.Close() go func(js nats.JetStreamContext, idx int) { defer wg.Done() <-startCh // Test adding new streams js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("OTHER_%d", idx), Replicas: 3, }) // Test adding consumers to MAXCC stream js.SubscribeSync("in.maxcc.foo", nats.BindStream("MAXCC")) }(js, n) } // Wait for Go routines. time.Sleep(250 * time.Millisecond) close(startCh) wg.Wait() var names []string for n := range js.StreamNames() { names = append(names, n) } if nc := len(names); nc > 2 { t.Fatalf("Expected only 2 streams, got %d", nc) } names = names[:0] for n := range js.ConsumerNames("MAXCC") { names = append(names, n) } if nc := len(names); nc > 1 { t.Fatalf("Expected only 1 consumer, got %d", nc) } } func TestJetStreamClusterPanicDecodingConsumerState(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() rch := make(chan struct{}, 2) nc, js := jsClientConnect(t, c.randomServer(), nats.ReconnectWait(50*time.Millisecond), nats.MaxReconnects(-1), nats.ReconnectHandler(func(_ *nats.Conn) { rch <- struct{}{} }), ) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ORDERS.*"}, Storage: nats.FileStorage, Replicas: 3, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardNew, MaxMsgs: -1, MaxAge: time.Hour * 24 * 365, }); err != nil { t.Fatalf("Error creating stream: %v", err) } sub, err := js.PullSubscribe("ORDERS.created", "durable", nats.MaxAckPending(1000)) if err != nil { t.Fatalf("Error creating pull subscriber: %v", err) } sendMsg := func(subject string) { t.Helper() if _, err := js.Publish(subject, []byte("msg")); err != nil { t.Fatalf("Error on publish: %v", err) } } for i := 0; i < 100; i++ { sendMsg("ORDERS.something") sendMsg("ORDERS.created") } for total := 0; total != 100; { msgs, err := sub.Fetch(100-total, nats.MaxWait(2*time.Second)) if err != nil { t.Fatalf("Failed to fetch message: %v", err) } for _, m := range msgs { m.AckSync() total++ } } c.stopAll() c.restartAllSamePorts() c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "durable") select { case <-rch: case <-time.After(2 * time.Second): t.Fatal("Did not reconnect") } for i := 0; i < 100; i++ { sendMsg("ORDERS.something") sendMsg("ORDERS.created") } for total := 0; total != 100; { msgs, err := sub.Fetch(100-total, nats.MaxWait(2*time.Second)) if err != nil { t.Fatalf("Error on fetch: %v", err) } for _, m := range msgs { m.AckSync() total++ } } } // Had a report of leaked subs with pull subscribers. func TestJetStreamClusterPullConsumerLeakedSubs(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"Domains.*"}, Replicas: 1, Retention: nats.InterestPolicy, }); err != nil { t.Fatalf("Error creating stream: %v", err) } sub, err := js.PullSubscribe("Domains.Domain", "Domains-Api", nats.MaxAckPending(20_000)) if err != nil { t.Fatalf("Error creating pull subscriber: %v", err) } defer sub.Unsubscribe() // Load up a bunch of requests. numRequests := 20 for i := 0; i < numRequests; i++ { js.PublishAsync("Domains.Domain", []byte("QUESTION")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } numSubs := c.stableTotalSubs() // With batch of 1 we do not see any issues, so set to 10. // Currently Go client uses auto unsub based on the batch size. for i := 0; i < numRequests/10; i++ { msgs, err := sub.Fetch(10) if err != nil { t.Fatalf("Unexpected error: %v", err) } for _, m := range msgs { m.AckSync() } } // Make sure the stream is empty.. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { t.Fatalf("Stream should be empty, got %+v", si) } // Make sure we did not leak any subs. if numSubsAfter := c.stableTotalSubs(); numSubsAfter != numSubs { t.Fatalf("Subs leaked: %d before, %d after", numSubs, numSubsAfter) } } func TestJetStreamClusterPushConsumerQueueGroup(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }); err != nil { t.Fatalf("Error creating stream: %v", err) } js.Publish("foo", []byte("QG")) // Do consumer by hand for now. inbox := nats.NewInbox() obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "dlc", DeliverSubject: inbox, DeliverGroup: "22", AckPolicy: AckNone, }, } req, err := json.Marshal(obsReq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "dlc"), req, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error != nil { t.Fatalf("Unexpected error, got %+v", ccResp.Error) } sub, _ := nc.SubscribeSync(inbox) if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Expected a timeout, we should not get messages here") } qsub, _ := nc.QueueSubscribeSync(inbox, "22") checkSubsPending(t, qsub, 1) // Test deleting the plain sub has not affect. sub.Unsubscribe() js.Publish("foo", []byte("QG")) checkSubsPending(t, qsub, 2) qsub.Unsubscribe() qsub2, _ := nc.QueueSubscribeSync(inbox, "22") js.Publish("foo", []byte("QG")) checkSubsPending(t, qsub2, 1) // Catch all sub. sub, _ = nc.SubscribeSync(inbox) qsub2.Unsubscribe() // Should be no more interest. // Send another, make sure we do not see the message flow here. js.Publish("foo", []byte("QG")) if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Expected a timeout, we should not get messages here") } } func TestJetStreamClusterConsumerLastActiveReporting(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{Name: "foo", Replicas: 2} if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } sendMsg := func() { t.Helper() if _, err := js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sub, err := js.SubscribeSync("foo", nats.Durable("dlc")) if err != nil { t.Fatalf("Unexpected error: %v", err) } // TODO(dlc) - Do by hand for now until Go client has this. consumerInfo := func(name string) *ConsumerInfo { t.Helper() resp, err := nc.Request(fmt.Sprintf(JSApiConsumerInfoT, "foo", name), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cinfo JSApiConsumerInfoResponse if err := json.Unmarshal(resp.Data, &cinfo); err != nil { t.Fatalf("Unexpected error: %v", err) } if cinfo.ConsumerInfo == nil || cinfo.Error != nil { t.Fatalf("Got a bad response %+v", cinfo) } return cinfo.ConsumerInfo } if ci := consumerInfo("dlc"); ci.Delivered.Last != nil || ci.AckFloor.Last != nil { t.Fatalf("Expected last to be nil by default, got %+v", ci) } checkTimeDiff := func(t1, t2 *time.Time) { t.Helper() // Compare on a seconds level rt1, rt2 := t1.UTC().Round(time.Second), t2.UTC().Round(time.Second) if rt1 != rt2 { d := rt1.Sub(rt2) if d > time.Second || d < -time.Second { t.Fatalf("Times differ too much, expected %v got %v", rt1, rt2) } } } checkDelivered := func(name string) { t.Helper() now := time.Now() ci := consumerInfo(name) if ci.Delivered.Last == nil { t.Fatalf("Expected delivered last to not be nil after activity, got %+v", ci.Delivered) } checkTimeDiff(&now, ci.Delivered.Last) } checkLastAck := func(name string, m *nats.Msg) { t.Helper() now := time.Now() if err := m.AckSync(); err != nil { t.Fatalf("Unexpected error: %v", err) } ci := consumerInfo(name) if ci.AckFloor.Last == nil { t.Fatalf("Expected ack floor last to not be nil after ack, got %+v", ci.AckFloor) } // Compare on a seconds level checkTimeDiff(&now, ci.AckFloor.Last) } checkAck := func(name string) { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkLastAck(name, m) } // Push sendMsg() checkSubsPending(t, sub, 1) checkDelivered("dlc") checkAck("dlc") // Check pull. sub, err = js.PullSubscribe("foo", "rip") if err != nil { t.Fatalf("Unexpected error: %v", err) } sendMsg() // Should still be nil since pull. if ci := consumerInfo("rip"); ci.Delivered.Last != nil || ci.AckFloor.Last != nil { t.Fatalf("Expected last to be nil by default, got %+v", ci) } msgs, err := sub.Fetch(1) if err != nil || len(msgs) == 0 { t.Fatalf("Unexpected error: %v", err) } checkDelivered("rip") checkLastAck("rip", msgs[0]) // Now test to make sure this state is held correctly across a cluster. ci := consumerInfo("rip") nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "foo", "rip"), nil, time.Second) c.waitOnConsumerLeader("$G", "foo", "rip") nci := consumerInfo("rip") if nci.Delivered.Last == nil { t.Fatalf("Expected delivered last to not be nil, got %+v", nci.Delivered) } if nci.AckFloor.Last == nil { t.Fatalf("Expected ack floor last to not be nil, got %+v", nci.AckFloor) } checkTimeDiff(ci.Delivered.Last, nci.Delivered.Last) checkTimeDiff(ci.AckFloor.Last, nci.AckFloor.Last) } func TestJetStreamClusterRaceOnRAFTCreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() srv := c.servers[0] nc, err := nats.Connect(srv.ClientURL()) if err != nil { t.Fatal(err) } defer nc.Close() js, err := nc.JetStream() if err != nil { t.Fatal(err) } if _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }); err != nil { t.Fatalf("Error creating stream: %v", err) } c.waitOnStreamLeader(globalAccountName, "TEST") js, err = nc.JetStream(nats.MaxWait(2 * time.Second)) if err != nil { t.Fatal(err) } size := 10 wg := sync.WaitGroup{} wg.Add(size) for i := 0; i < size; i++ { go func() { defer wg.Done() // We don't care about possible failures here, we just want // parallel creation of a consumer. js.PullSubscribe("foo", "shared") }() } wg.Wait() } func TestJetStreamClusterDeadlockOnVarz(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() srv := c.servers[0] nc, err := nats.Connect(srv.ClientURL()) if err != nil { t.Fatal(err) } defer nc.Close() js, err := nc.JetStream() if err != nil { t.Fatal(err) } size := 10 wg := sync.WaitGroup{} wg.Add(size) ch := make(chan struct{}) for i := 0; i < size; i++ { go func(i int) { defer wg.Done() <-ch js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST%d", i), Subjects: []string{"foo"}, Replicas: 3, }) }(i) } close(ch) for i := 0; i < 10; i++ { srv.Varz(nil) time.Sleep(time.Millisecond) } wg.Wait() } // Issue #2397 func TestJetStreamClusterStreamCatchupNoState(t *testing.T) { c := createJetStreamClusterExplicit(t, "R2S", 2) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Hold onto servers. sl := c.streamLeader("$G", "TEST") if sl == nil { t.Fatalf("Did not get a server") } nsl := c.randomNonStreamLeader("$G", "TEST") if nsl == nil { t.Fatalf("Did not get a server") } // Grab low level stream and raft node. mset, err := nsl.GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } node := mset.raftNode() if node == nil { t.Fatalf("Could not get stream group name") } gname := node.Group() numRequests := 100 for i := 0; i < numRequests; i++ { // This will force a snapshot which will prune the normal log. // We will remove the snapshot to simulate the error condition. if i == 10 { if err := node.InstallSnapshot(mset.stateSnapshot()); err != nil { t.Fatalf("Error installing snapshot: %v", err) } } _, err := js.Publish("foo.created", []byte("REQ")) require_NoError(t, err) } config := nsl.JetStreamConfig() if config == nil { t.Fatalf("No config") } lconfig := sl.JetStreamConfig() if lconfig == nil { t.Fatalf("No config") } nc.Close() c.stopAll() // Remove all state by truncating for the non-leader. for _, fn := range []string{"1.blk", "1.idx", "1.fss"} { fname := filepath.Join(config.StoreDir, "$G", "streams", "TEST", "msgs", fn) fd, err := os.OpenFile(fname, os.O_RDWR, defaultFilePerms) if err != nil { continue } fd.Truncate(0) fd.Close() } // For both make sure we have no raft snapshots. snapDir := filepath.Join(lconfig.StoreDir, "$SYS", "_js_", gname, "snapshots") os.RemoveAll(snapDir) // Remove all our raft state, we do not want to hold onto our term and index which // results in a coin toss for who becomes the leader. raftDir := filepath.Join(config.StoreDir, "$SYS", "_js_", gname) os.RemoveAll(raftDir) // Now restart. c.restartAll() for _, cs := range c.servers { c.waitOnStreamCurrent(cs, "$G", "TEST") } nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() c.waitOnStreamLeader("$G", "TEST") _, err = js.Publish("foo.created", []byte("ZZZ")) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.LastSeq != 101 { t.Fatalf("bad state after restart: %+v", si.State) } } // Issue #2525 func TestJetStreamClusterLargeHeaders(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("add stream failed: %s", err) } // We use u16 to encode msg header len. Make sure we do the right thing when > 65k. data := make([]byte, 8*1024) crand.Read(data) val := hex.EncodeToString(data)[:8*1024] m := nats.NewMsg("foo") for i := 1; i <= 10; i++ { m.Header.Add(fmt.Sprintf("LargeHeader-%d", i), val) } m.Data = []byte("Hello Large Headers!") if _, err = js.PublishMsg(m); err == nil { t.Fatalf("Expected an error but got none") } } func TestJetStreamClusterFlowControlRequiresHeartbeats(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", DeliverSubject: nats.NewInbox(), FlowControl: true, }); err == nil || IsNatsErr(err, JSConsumerWithFlowControlNeedsHeartbeats) { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamClusterMixedModeColdStartPrune(t *testing.T) { // Purposely make this unbalanced. Without changes this will never form a quorum to elect the meta-leader. c := createMixedModeCluster(t, jsMixedModeGlobalAccountTempl, "MMCS5", _EMPTY_, 3, 4, false) defer c.shutdown() // Make sure we report cluster size. checkClusterSize := func(s *Server) { t.Helper() jsi, err := s.Jsz(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } if jsi.Meta == nil { t.Fatalf("Expected a cluster info") } if jsi.Meta.Size != 3 { t.Fatalf("Expected cluster size to be adjusted to %d, but got %d", 3, jsi.Meta.Size) } } checkClusterSize(c.leader()) checkClusterSize(c.randomNonLeader()) } func TestJetStreamClusterMirrorAndSourceCrossNonNeighboringDomain(t *testing.T) { storeDir1 := t.TempDir() conf1 := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain1, store_dir: '%s'} accounts { A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS no_auth_user: a1 leafnodes: { listen: 127.0.0.1:-1 } `, storeDir1))) s1, _ := RunServerWithConfig(conf1) defer s1.Shutdown() storeDir2 := t.TempDir() conf2 := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain2, store_dir: '%s'} accounts { A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS no_auth_user: a1 leafnodes:{ remotes:[{ url:nats://a1:a1@127.0.0.1:%d, account: A}, { url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } `, storeDir2, s1.opts.LeafNode.Port, s1.opts.LeafNode.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() storeDir3 := t.TempDir() conf3 := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain3, store_dir: '%s'} accounts { A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS no_auth_user: a1 leafnodes:{ remotes:[{ url:nats://a1:a1@127.0.0.1:%d, account: A}, { url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } `, storeDir3, s1.opts.LeafNode.Port, s1.opts.LeafNode.Port))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkLeafNodeConnectedCount(t, s1, 4) checkLeafNodeConnectedCount(t, s2, 2) checkLeafNodeConnectedCount(t, s3, 2) c2 := natsConnect(t, s2.ClientURL()) defer c2.Close() js2, err := c2.JetStream(nats.Domain("domain2")) require_NoError(t, err) ai2, err := js2.AccountInfo() require_NoError(t, err) require_Equal(t, ai2.Domain, "domain2") _, err = js2.AddStream(&nats.StreamConfig{ Name: "disk", Storage: nats.FileStorage, Subjects: []string{"disk"}, }) require_NoError(t, err) _, err = js2.Publish("disk", nil) require_NoError(t, err) si, err := js2.StreamInfo("disk") require_NoError(t, err) require_True(t, si.State.Msgs == 1) c3 := natsConnect(t, s3.ClientURL()) defer c3.Close() js3, err := c3.JetStream(nats.Domain("domain3")) require_NoError(t, err) ai3, err := js3.AccountInfo() require_NoError(t, err) require_Equal(t, ai3.Domain, "domain3") _, err = js3.AddStream(&nats.StreamConfig{ Name: "stream-mirror", Storage: nats.FileStorage, Mirror: &nats.StreamSource{ Name: "disk", External: &nats.ExternalStream{APIPrefix: "$JS.domain2.API"}, }, }) require_NoError(t, err) _, err = js3.AddStream(&nats.StreamConfig{ Name: "stream-source", Storage: nats.FileStorage, Sources: []*nats.StreamSource{{ Name: "disk", External: &nats.ExternalStream{APIPrefix: "$JS.domain2.API"}, }}, }) require_NoError(t, err) checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { if si, _ := js3.StreamInfo("stream-mirror"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for mirror, got %d", si.State.Msgs) } if si, _ := js3.StreamInfo("stream-source"); si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg for source, got %d", si.State.Msgs) } return nil }) } func TestJetStreamClusterSeal(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() // Need to be done by hand until makes its way to Go client. createStream := func(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *JSApiStreamCreateResponse { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) var scResp JSApiStreamCreateResponse err = json.Unmarshal(resp.Data, &scResp) require_NoError(t, err) return &scResp } updateStream := func(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *JSApiStreamUpdateResponse { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) require_NoError(t, err) var scResp JSApiStreamUpdateResponse err = json.Unmarshal(resp.Data, &scResp) require_NoError(t, err) return &scResp } testSeal := func(t *testing.T, s *Server, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() // Should not be able to create a stream that starts sealed. scr := createStream(t, nc, &StreamConfig{Name: "SEALED", Replicas: replicas, Storage: MemoryStorage, Sealed: true}) if scr.Error == nil { t.Fatalf("Expected an error but got none") } // Create our stream. scr = createStream(t, nc, &StreamConfig{Name: "SEALED", Replicas: replicas, MaxAge: time.Minute, Storage: MemoryStorage}) if scr.Error != nil { t.Fatalf("Unexpected error: %v", scr.Error) } for i := 0; i < 100; i++ { js.Publish("SEALED", []byte("OK")) } // Update to sealed. sur := updateStream(t, nc, &StreamConfig{Name: "SEALED", Replicas: replicas, MaxAge: time.Minute, Storage: MemoryStorage, Sealed: true}) if sur.Error != nil { t.Fatalf("Unexpected error: %v", sur.Error) } // Grab stream info and make sure its reflected as sealed. resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "SEALED"), nil, time.Second) require_NoError(t, err) var sir JSApiStreamInfoResponse err = json.Unmarshal(resp.Data, &sir) require_NoError(t, err) if sir.Error != nil { t.Fatalf("Unexpected error: %v", sir.Error) } si := sir.StreamInfo if !si.Config.Sealed { t.Fatalf("Expected the stream to be marked sealed, got %+v\n", si.Config) } // Make sure we also updated any max age and moved to discard new. if si.Config.MaxAge != 0 { t.Fatalf("Expected MaxAge to be cleared, got %v", si.Config.MaxAge) } if si.Config.Discard != DiscardNew { t.Fatalf("Expected DiscardPolicy to be set to new, got %v", si.Config.Discard) } // Also make sure we set denyDelete and denyPurge. if !si.Config.DenyDelete { t.Fatalf("Expected the stream to be marked as DenyDelete, got %+v\n", si.Config) } if !si.Config.DenyPurge { t.Fatalf("Expected the stream to be marked as DenyPurge, got %+v\n", si.Config) } if si.Config.AllowRollup { t.Fatalf("Expected the stream to be marked as not AllowRollup, got %+v\n", si.Config) } // Sealing is not reversible, so make sure we get an error trying to undo. sur = updateStream(t, nc, &StreamConfig{Name: "SEALED", Replicas: replicas, Storage: MemoryStorage, Sealed: false}) if sur.Error == nil { t.Fatalf("Expected an error but got none") } // Now test operations like publish a new msg, delete, purge etc all fail. if _, err := js.Publish("SEALED", []byte("OK")); err == nil { t.Fatalf("Expected a publish to fail") } if err := js.DeleteMsg("SEALED", 1); err == nil { t.Fatalf("Expected a delete to fail") } if err := js.PurgeStream("SEALED"); err == nil { t.Fatalf("Expected a purge to fail") } if err := js.DeleteStream("SEALED"); err != nil { t.Fatalf("Expected a delete to succeed, got %v", err) } } t.Run("Single", func(t *testing.T) { testSeal(t, s, 1) }) t.Run("Clustered", func(t *testing.T) { testSeal(t, c.randomServer(), 3) }) } // Issue #2568 func TestJetStreamClusteredStreamCreateIdempotent(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &StreamConfig{ Name: "AUDIT", Storage: MemoryStorage, Subjects: []string{"foo"}, Replicas: 3, DenyDelete: true, DenyPurge: true, } addStream(t, nc, cfg) addStream(t, nc, cfg) } func TestJetStreamClusterRollupsRequirePurge(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &StreamConfig{ Name: "SENSORS", Storage: FileStorage, Subjects: []string{"sensor.*.temp"}, MaxMsgsPer: 10, AllowRollup: true, DenyPurge: true, Replicas: 2, } j, err := json.Marshal(cfg) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), j, time.Second) require_NoError(t, err) var cr JSApiStreamCreateResponse err = json.Unmarshal(resp.Data, &cr) require_NoError(t, err) if cr.Error == nil || cr.Error.Description != "roll-ups require the purge permission" { t.Fatalf("unexpected error: %v", cr.Error) } } func TestJetStreamClusterRollups(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &StreamConfig{ Name: "SENSORS", Storage: FileStorage, Subjects: []string{"sensor.*.temp"}, MaxMsgsPer: 10, AllowRollup: true, Replicas: 2, } addStream(t, nc, cfg) var bt [16]byte var le = binary.LittleEndian // Generate 1000 random measurements for 10 sensors for i := 0; i < 1000; i++ { id, temp := strconv.Itoa(rand.Intn(9)+1), rand.Int31n(42)+60 // 60-102 degrees. le.PutUint16(bt[0:], uint16(temp)) js.PublishAsync(fmt.Sprintf("sensor.%v.temp", id), bt[:]) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Grab random sensor and do a rollup by averaging etc. sensor := fmt.Sprintf("sensor.%v.temp", strconv.Itoa(rand.Intn(9)+1)) sub, err := js.SubscribeSync(sensor) require_NoError(t, err) var total, samples int for m, err := sub.NextMsg(time.Second); err == nil; m, err = sub.NextMsg(time.Second) { total += int(le.Uint16(m.Data)) samples++ } sub.Unsubscribe() avg := uint16(total / samples) le.PutUint16(bt[0:], avg) rollup := nats.NewMsg(sensor) rollup.Data = bt[:] rollup.Header.Set(JSMsgRollup, JSMsgRollupSubject) _, err = js.PublishMsg(rollup) require_NoError(t, err) sub, err = js.SubscribeSync(sensor) require_NoError(t, err) // Make sure only 1 left. checkSubsPending(t, sub, 1) sub.Unsubscribe() // Now do all. rollup.Header.Set(JSMsgRollup, JSMsgRollupAll) _, err = js.PublishMsg(rollup) require_NoError(t, err) // Same thing as above should hold true. sub, err = js.SubscribeSync(sensor) require_NoError(t, err) // Make sure only 1 left. checkSubsPending(t, sub, 1) sub.Unsubscribe() // Also should only be 1 msgs in total stream left with JSMsgRollupAll si, err := js.StreamInfo("SENSORS") require_NoError(t, err) if si.State.Msgs != 1 { t.Fatalf("Expected only 1 msg left after rollup all, got %+v", si.State) } } func TestJetStreamClusterRollupSubjectAndWatchers(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &StreamConfig{ Name: "KVW", Storage: FileStorage, Subjects: []string{"kv.*"}, MaxMsgsPer: 10, AllowRollup: true, Replicas: 2, } addStream(t, nc, cfg) sub, err := js.SubscribeSync("kv.*") if err != nil { t.Fatalf("Unexpected error: %v", err) } send := func(key, value string) { t.Helper() _, err := js.Publish("kv."+key, []byte(value)) require_NoError(t, err) } rollup := func(key, value string) { t.Helper() m := nats.NewMsg("kv." + key) m.Data = []byte(value) m.Header.Set(JSMsgRollup, JSMsgRollupSubject) _, err := js.PublishMsg(m) require_NoError(t, err) } expectUpdate := func(key, value string, seq uint64) { t.Helper() m, err := sub.NextMsg(time.Second) require_NoError(t, err) if m.Subject != "kv."+key { t.Fatalf("Keys don't match: %q vs %q", m.Subject[3:], key) } if string(m.Data) != value { t.Fatalf("Values don't match: %q vs %q", m.Data, value) } meta, err := m.Metadata() require_NoError(t, err) if meta.Sequence.Consumer != seq { t.Fatalf("Sequences don't match: %v vs %v", meta.Sequence.Consumer, value) } } rollup("name", "derek") expectUpdate("name", "derek", 1) rollup("age", "22") expectUpdate("age", "22", 2) send("name", "derek") expectUpdate("name", "derek", 3) send("age", "22") expectUpdate("age", "22", 4) send("age", "33") expectUpdate("age", "33", 5) send("name", "ivan") expectUpdate("name", "ivan", 6) send("name", "rip") expectUpdate("name", "rip", 7) rollup("age", "50") expectUpdate("age", "50", 8) } func TestJetStreamClusterAppendOnly(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &StreamConfig{ Name: "AUDIT", Storage: MemoryStorage, Subjects: []string{"foo"}, Replicas: 3, DenyDelete: true, DenyPurge: true, } si := addStream(t, nc, cfg) if !si.Config.DenyDelete || !si.Config.DenyPurge { t.Fatalf("Expected DenyDelete and DenyPurge to be set, got %+v", si.Config) } for i := 0; i < 10; i++ { js.Publish("foo", []byte("ok")) } // Delete should not be allowed. if err := js.DeleteMsg("AUDIT", 1); err == nil { t.Fatalf("Expected an error for delete but got none") } if err := js.PurgeStream("AUDIT"); err == nil { t.Fatalf("Expected an error for purge but got none") } cfg.DenyDelete = false cfg.DenyPurge = false req, err := json.Marshal(cfg) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Error == nil { t.Fatalf("Expected an error") } } // Related to #2642 func TestJetStreamClusterStreamUpdateSyncBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } msg, toSend := []byte("OK"), 100 for i := 0; i < toSend; i++ { if _, err := js.PublishAsync("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } cfg.Subjects = []string{"foo", "bar", "baz"} if _, err := js.UpdateStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // Shutdown a server. The bug is that the update wiped the sync subject used to catchup a stream that has the RAFT layer snapshotted. nsl := c.randomNonStreamLeader("$G", "TEST") nsl.Shutdown() // make sure a leader exists c.waitOnStreamLeader("$G", "TEST") for i := 0; i < toSend*4; i++ { if _, err := js.PublishAsync("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Throw in deletes as well. for seq := uint64(200); seq < uint64(300); seq += 4 { if err := js.DeleteMsg("TEST", seq); err != nil { t.Fatalf("Unexpected error: %v", err) } } // We need to snapshot to force upper layer catchup vs RAFT layer. c.waitOnAllCurrent() mset, err := c.streamLeader("$G", "TEST").GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Expected to find a stream for %q", "TEST") } if err := mset.raftNode().InstallSnapshot(mset.stateSnapshot()); err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnAllCurrent() nsl = c.restartServer(nsl) c.waitOnStreamLeader("$G", "TEST") c.waitOnStreamCurrent(nsl, "$G", "TEST") mset, _ = nsl.GlobalAccount().lookupStream("TEST") cloneState := mset.state() mset, _ = c.streamLeader("$G", "TEST").GlobalAccount().lookupStream("TEST") leaderState := mset.state() if !reflect.DeepEqual(cloneState, leaderState) { t.Fatalf("States do not match: %+v vs %+v", cloneState, leaderState) } } // Issue #2666 func TestJetStreamClusterKVMultipleConcurrentCreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "TEST", History: 1, TTL: 150 * time.Millisecond, Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } startCh := make(chan bool) var wg sync.WaitGroup for n := 0; n < 5; n++ { wg.Add(1) go func() { defer wg.Done() <-startCh if r, err := kv.Create("name", []byte("dlc")); err == nil { if _, err = kv.Update("name", []byte("rip"), r); err != nil { t.Log("Unexpected Update error: ", err) } } }() } // Wait for Go routines to start. time.Sleep(100 * time.Millisecond) close(startCh) wg.Wait() // Just make sure its there and picks up the phone. if _, err := js.StreamInfo("KV_TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now make sure we do ok when servers are restarted and we need to deal with dangling clfs state. // First non-leader. rs := c.randomNonStreamLeader("$G", "KV_TEST") rs.Shutdown() rs = c.restartServer(rs) c.waitOnStreamCurrent(rs, "$G", "KV_TEST") if _, err := kv.Put("name", []byte("ik")); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now the actual leader. sl := c.streamLeader("$G", "KV_TEST") sl.Shutdown() sl = c.restartServer(sl) c.waitOnStreamLeader("$G", "KV_TEST") c.waitOnStreamCurrent(sl, "$G", "KV_TEST") if _, err := kv.Put("name", []byte("mh")); err != nil { t.Fatalf("Unexpected error: %v", err) } time.Sleep(time.Second) } func TestJetStreamClusterAccountInfoForSystemAccount(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() _, err := js.AccountInfo() require_Error(t, err, nats.ErrJetStreamNotEnabledForAccount) } func TestJetStreamClusterListFilter(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() testList := func(t *testing.T, srv *Server, r int) { nc, js := jsClientConnect(t, srv) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ONE", Subjects: []string{"one.>"}, Replicas: r, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TWO", Subjects: []string{"two.>"}, Replicas: r, }) require_NoError(t, err) resp, err := nc.Request(JSApiStreamList, []byte("{}"), time.Second) require_NoError(t, err) list := &JSApiStreamListResponse{} err = json.Unmarshal(resp.Data, list) require_NoError(t, err) if len(list.Streams) != 2 { t.Fatalf("Expected 2 responses got %d", len(list.Streams)) } resp, err = nc.Request(JSApiStreamList, []byte(`{"subject":"two.x"}`), time.Second) require_NoError(t, err) list = &JSApiStreamListResponse{} err = json.Unmarshal(resp.Data, list) require_NoError(t, err) if len(list.Streams) != 1 { t.Fatalf("Expected 1 response got %d", len(list.Streams)) } if list.Streams[0].Config.Name != "TWO" { t.Fatalf("Expected stream TWO in result got %#v", list.Streams[0]) } } t.Run("Single", func(t *testing.T) { testList(t, s, 1) }) t.Run("Clustered", func(t *testing.T) { testList(t, c.randomServer(), 3) }) } func TestJetStreamClusterConsumerUpdates(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "JSC", 5) defer c.shutdown() testConsumerUpdate := func(t *testing.T, s *Server, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() // Create a stream. _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: replicas, }) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo", []byte("OK")) } cfg := &nats.ConsumerConfig{ Durable: "dlc", Description: "Update TEST", FilterSubject: "foo", DeliverSubject: "d.foo", AckPolicy: nats.AckExplicitPolicy, AckWait: time.Minute, MaxDeliver: 5, MaxAckPending: 50, } _, err = js.AddConsumer("TEST", cfg) require_NoError(t, err) // Update delivery subject, which worked before, but upon review had issues unless replica count == clustered size. cfg.DeliverSubject = "d.bar" _, err = js.AddConsumer("TEST", cfg) require_NoError(t, err) // Bind deliver subject. sub, err := nc.SubscribeSync("d.bar") require_NoError(t, err) defer sub.Unsubscribe() ncfg := *cfg // Deliver Subject ncfg.DeliverSubject = "d.baz" // Description cfg.Description = "New Description" _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) // MaxAckPending checkSubsPending(t, sub, 50) cfg.MaxAckPending = 75 _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) checkSubsPending(t, sub, 75) // Drain sub, do not ack first ten though so we can test shortening AckWait. for i := 0; i < 100; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) if i >= 10 { m.AckSync() } } // AckWait checkSubsPending(t, sub, 0) cfg.AckWait = 200 * time.Millisecond _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) checkSubsPending(t, sub, 10) // Rate Limit cfg.RateLimit = 8 * 1024 _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) cfg.RateLimit = 0 _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) // These all should fail. ncfg = *cfg ncfg.DeliverPolicy = nats.DeliverLastPolicy _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg ncfg.OptStartSeq = 22 _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg now := time.Now() ncfg.OptStartTime = &now _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg ncfg.AckPolicy = nats.AckAllPolicy _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg ncfg.ReplayPolicy = nats.ReplayOriginalPolicy _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg ncfg.Heartbeat = time.Second _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) ncfg = *cfg ncfg.FlowControl = true _, err = js.UpdateConsumer("TEST", &ncfg) require_Error(t, err) } t.Run("Single", func(t *testing.T) { testConsumerUpdate(t, s, 1) }) t.Run("Clustered", func(t *testing.T) { testConsumerUpdate(t, c.randomServer(), 2) }) } func TestJetStreamClusterConsumerMaxDeliverUpdate(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}, Replicas: 3}) require_NoError(t, err) maxDeliver := 2 _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "ard", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", MaxDeliver: maxDeliver, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "ard") require_NoError(t, err) checkMaxDeliver := func() { t.Helper() for i := 0; i <= maxDeliver; i++ { msgs, err := sub.Fetch(2, nats.MaxWait(100*time.Millisecond)) if i < maxDeliver { require_NoError(t, err) require_Len(t, 1, len(msgs)) _ = msgs[0].Nak() } else { require_Error(t, err, nats.ErrTimeout) } } } _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) checkMaxDeliver() // update maxDeliver maxDeliver++ _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "ard", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", MaxDeliver: maxDeliver, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) checkMaxDeliver() } func TestJetStreamClusterAccountReservations(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, "C1", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() accMax := 3 errResources := errors.New("nats: insufficient storage resources available") test := func(t *testing.T, replica int) { mb := int64((1+accMax)-replica) * 1024 * 1024 * 1024 // GB, corrected for replication factor _, err := js.AddStream(&nats.StreamConfig{Name: "S1", Subjects: []string{"s1"}, MaxBytes: mb, Replicas: replica}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "S2", Subjects: []string{"s2"}, MaxBytes: 1024, Replicas: replica}) require_Error(t, err, errResources) _, err = js.UpdateStream(&nats.StreamConfig{Name: "S1", Subjects: []string{"s1"}, MaxBytes: mb / 2, Replicas: replica}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "S2", Subjects: []string{"s2"}, MaxBytes: mb / 2, Replicas: replica}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "S3", Subjects: []string{"s3"}, MaxBytes: 1024, Replicas: replica}) require_Error(t, err, errResources) _, err = js.UpdateStream(&nats.StreamConfig{Name: "S2", Subjects: []string{"s2"}, MaxBytes: mb/2 + 1, Replicas: replica}) require_Error(t, err, errResources) require_NoError(t, js.DeleteStream("S1")) require_NoError(t, js.DeleteStream("S2")) } test(t, 3) test(t, 1) } func TestJetStreamClusterConcurrentAccountLimits(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, "cluster", 3) defer c.shutdown() startCh := make(chan bool) var wg sync.WaitGroup var swg sync.WaitGroup failCount := int32(0) start := func(name string) { wg.Add(1) defer wg.Done() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() swg.Done() <-startCh _, err := js.AddStream(&nats.StreamConfig{ Name: name, Replicas: 3, MaxBytes: 1024 * 1024 * 1024, }) if err != nil { atomic.AddInt32(&failCount, 1) require_Equal(t, err.Error(), "nats: insufficient storage resources available") } } swg.Add(2) go start("foo") go start("bar") swg.Wait() // Now start both at same time. close(startCh) wg.Wait() require_True(t, failCount == 1) } func TestJetStreamClusterBalancedPlacement(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesTempl, "CB", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // We have 10GB (2GB X 5) available. // Use MaxBytes for ease of test (used works too) and place 5 1GB streams with R=2. for i := 1; i <= 5; i++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("S-%d", i), Replicas: 2, MaxBytes: 1 * 1024 * 1024 * 1024, }) require_NoError(t, err) } // Make sure the next one fails properly. _, err := js.AddStream(&nats.StreamConfig{ Name: "FAIL", Replicas: 2, MaxBytes: 1 * 1024 * 1024 * 1024, }) require_Contains(t, err.Error(), "no suitable peers for placement", "insufficient storage") } func TestJetStreamClusterConsumerPendingBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() nc2, js2 := jsClientConnect(t, c.randomServer()) defer nc2.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 3}) require_NoError(t, err) startCh, doneCh := make(chan bool), make(chan error) go func() { <-startCh _, err := js2.AddConsumer("foo", &nats.ConsumerConfig{ Durable: "dlc", FilterSubject: "foo", DeliverSubject: "x", }) doneCh <- err }() n := 10_000 for i := 0; i < n; i++ { nc.Publish("foo", []byte("ok")) if i == 222 { startCh <- true } } // Wait for them to all be there. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("foo") require_NoError(t, err) if si.State.Msgs != uint64(n) { return fmt.Errorf("Not received all messages") } return nil }) select { case err := <-doneCh: if err != nil { t.Fatalf("Error creating consumer: %v", err) } case <-time.After(5 * time.Second): t.Fatalf("Timed out?") } checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := js.ConsumerInfo("foo", "dlc") require_NoError(t, err) if ci.NumPending != uint64(n) { return fmt.Errorf("Expected NumPending to be %d, got %d", n, ci.NumPending) } return nil }) } func TestJetStreamClusterPullPerf(t *testing.T) { skip(t) c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() js.AddStream(&nats.StreamConfig{Name: "f22"}) defer js.DeleteStream("f22") n, msg := 1_000_000, []byte(strings.Repeat("A", 1000)) for i := 0; i < n; i++ { js.PublishAsync("f22", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("f22") require_NoError(t, err) fmt.Printf("msgs: %d, total_bytes: %v\n", si.State.Msgs, friendlyBytes(int64(si.State.Bytes))) // OrderedConsumer - fastest push based. start := time.Now() received, done := 0, make(chan bool) _, err = js.Subscribe("f22", func(m *nats.Msg) { received++ if received >= n { done <- true } }, nats.OrderedConsumer()) require_NoError(t, err) // Wait to receive all messages. select { case <-done: case <-time.After(30 * time.Second): t.Fatalf("Did not receive all of our messages") } tt := time.Since(start) fmt.Printf("Took %v to receive %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) // Now do pull based, this is custom for now. // Current nats.PullSubscribe maxes at about 1/2 the performance even with large batches. _, err = js.AddConsumer("f22", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckAllPolicy, MaxAckPending: 1000, }) require_NoError(t, err) r := 0 _, err = nc.Subscribe("xx", func(m *nats.Msg) { r++ if r >= n { done <- true } if r%750 == 0 { m.AckSync() } }) require_NoError(t, err) // Simulate an non-ending request. req := &JSApiConsumerGetNextRequest{Batch: n, Expires: 60 * time.Second} jreq, err := json.Marshal(req) require_NoError(t, err) start = time.Now() rsubj := fmt.Sprintf(JSApiRequestNextT, "f22", "dlc") err = nc.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) // Wait to receive all messages. select { case <-done: case <-time.After(60 * time.Second): t.Fatalf("Did not receive all of our messages") } tt = time.Since(start) fmt.Printf("Took %v to receive %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) } // Test that we get the right signaling when a consumer leader change occurs for any pending requests. func TestJetStreamClusterPullConsumerLeaderChange(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", }) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") sub, err := nc.SubscribeSync("reply") require_NoError(t, err) defer sub.Unsubscribe() drainSub := func() { t.Helper() for _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) { } checkSubsPending(t, sub, 0) } // Queue up a request that can live for a bit. req := &JSApiConsumerGetNextRequest{Expires: 2 * time.Second} jreq, err := json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(rsubj, "reply", jreq) require_NoError(t, err) // Make sure request is recorded and replicated. time.Sleep(100 * time.Millisecond) checkSubsPending(t, sub, 0) // Now have consumer leader change and make sure we get signaled that our request is not valid. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader("$G", "TEST", "dlc") checkSubsPending(t, sub, 1) m, err := sub.NextMsg(0) require_NoError(t, err) // Make sure this is an alert that tells us our request is no longer valid. if m.Header.Get("Status") != "409" { t.Fatalf("Expected a 409 status code, got %q", m.Header.Get("Status")) } checkSubsPending(t, sub, 0) // Add a few messages to the stream to fulfill a request. for i := 0; i < 10; i++ { _, err := js.Publish("foo", []byte("HELLO")) require_NoError(t, err) } req = &JSApiConsumerGetNextRequest{Batch: 10, Expires: 10 * time.Second} jreq, err = json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(rsubj, "reply", jreq) require_NoError(t, err) checkSubsPending(t, sub, 10) drainSub() // Now do a leader change again, make sure we do not get anything about that request. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader("$G", "TEST", "dlc") time.Sleep(100 * time.Millisecond) checkSubsPending(t, sub, 0) // Make sure we do not get anything if we expire, etc. req = &JSApiConsumerGetNextRequest{Batch: 10, Expires: 250 * time.Millisecond} jreq, err = json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(rsubj, "reply", jreq) require_NoError(t, err) // Let it expire. time.Sleep(350 * time.Millisecond) checkSubsPending(t, sub, 1) // Now do a leader change again, make sure we do not get anything about that request. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader("$G", "TEST", "dlc") checkSubsPending(t, sub, 1) } func TestJetStreamClusterEphemeralPullConsumerServerShutdown(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 2, Subjects: []string{"foo"}, }) require_NoError(t, err) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", }) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", ci.Name) sub, err := nc.SubscribeSync("reply") require_NoError(t, err) defer sub.Unsubscribe() // Queue up a request that can live for a bit. req := &JSApiConsumerGetNextRequest{Expires: 2 * time.Second} jreq, err := json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(rsubj, "reply", jreq) require_NoError(t, err) // Make sure request is recorded and replicated. time.Sleep(100 * time.Millisecond) checkSubsPending(t, sub, 0) // Now shutdown the server where this ephemeral lives. c.consumerLeader("$G", "TEST", ci.Name).Shutdown() checkSubsPending(t, sub, 1) m, err := sub.NextMsg(0) require_NoError(t, err) // Make sure this is an alert that tells us our request is no longer valid. if m.Header.Get("Status") != "409" { t.Fatalf("Expected a 409 status code, got %q", m.Header.Get("Status")) } } func TestJetStreamClusterNAKBackoffs(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 2, Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("NAK")) require_NoError(t, err) sub, err := js.SubscribeSync("foo", nats.Durable("dlc"), nats.AckWait(5*time.Second), nats.ManualAck()) require_NoError(t, err) defer sub.Unsubscribe() checkSubsPending(t, sub, 1) m, err := sub.NextMsg(0) require_NoError(t, err) // Default nak will redeliver almost immediately. // We can now add a parse duration string after whitespace to the NAK proto. start := time.Now() dnak := []byte(fmt.Sprintf("%s 200ms", AckNak)) m.Respond(dnak) checkSubsPending(t, sub, 1) elapsed := time.Since(start) if elapsed < 200*time.Millisecond { t.Fatalf("Took too short to redeliver, expected ~200ms but got %v", elapsed) } if elapsed > time.Second { t.Fatalf("Took too long to redeliver, expected ~200ms but got %v", elapsed) } // Now let's delay and make sure that is honored when a new consumer leader takes over. m, err = sub.NextMsg(0) require_NoError(t, err) dnak = []byte(fmt.Sprintf("%s 1s", AckNak)) start = time.Now() m.Respond(dnak) // Wait for NAK state to propagate. time.Sleep(100 * time.Millisecond) // Ask leader to stepdown. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader("$G", "TEST", "dlc") checkSubsPending(t, sub, 1) elapsed = time.Since(start) if elapsed < time.Second { t.Fatalf("Took too short to redeliver, expected ~1s but got %v", elapsed) } if elapsed > 3*time.Second { t.Fatalf("Took too long to redeliver, expected ~1s but got %v", elapsed) } // Test json version. delay, err := json.Marshal(&ConsumerNakOptions{Delay: 20 * time.Millisecond}) require_NoError(t, err) dnak = []byte(fmt.Sprintf("%s %s", AckNak, delay)) m, err = sub.NextMsg(0) require_NoError(t, err) start = time.Now() m.Respond(dnak) checkSubsPending(t, sub, 1) elapsed = time.Since(start) if elapsed < 20*time.Millisecond { t.Fatalf("Took too short to redeliver, expected ~20ms but got %v", elapsed) } if elapsed > 100*time.Millisecond { t.Fatalf("Took too long to redeliver, expected ~20ms but got %v", elapsed) } } func TestJetStreamClusterRedeliverBackoffs(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 2, Subjects: []string{"foo", "bar"}, }) require_NoError(t, err) // Produce some messages on bar so that when we create the consumer // on "foo", we don't have a 1:1 between consumer/stream sequence. for i := 0; i < 10; i++ { js.Publish("bar", []byte("msg")) } // Test when BackOff is configured and AckWait and MaxDeliver are as well. // Currently the BackOff will override AckWait, but we want MaxDeliver to be set to be at least len(BackOff)+1. ccReq := &CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "dlc", FilterSubject: "foo", DeliverSubject: "x", AckPolicy: AckExplicit, AckWait: 30 * time.Second, MaxDeliver: 2, BackOff: []time.Duration{25 * time.Millisecond, 100 * time.Millisecond, 250 * time.Millisecond}, }, } req, err := json.Marshal(ccReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "dlc"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error == nil || ccResp.Error.ErrCode != 10116 { t.Fatalf("Expected an error when MaxDeliver is <= len(BackOff), got %+v", ccResp.Error) } // Set MaxDeliver to 6. ccReq.Config.MaxDeliver = 6 req, err = json.Marshal(ccReq) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "dlc"), req, time.Second) require_NoError(t, err) ccResp.Error = nil err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected error: %+v", ccResp.Error) } if cfg := ccResp.ConsumerInfo.Config; cfg.AckWait != 25*time.Millisecond || cfg.MaxDeliver != 6 { t.Fatalf("Expected AckWait to be first BackOff (25ms) and MaxDeliver set to 6, got %+v", cfg) } var received []time.Time var mu sync.Mutex sub, err := nc.Subscribe("x", func(m *nats.Msg) { mu.Lock() received = append(received, time.Now()) mu.Unlock() }) require_NoError(t, err) // Send a message. start := time.Now() _, err = js.Publish("foo", []byte("m22")) require_NoError(t, err) checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { mu.Lock() nr := len(received) mu.Unlock() if nr >= 6 { return nil } return fmt.Errorf("Only seen %d of 6", nr) }) sub.Unsubscribe() expected := ccReq.Config.BackOff // We expect the MaxDeliver to go until 6, so fill in two additional ones. expected = append(expected, 250*time.Millisecond, 250*time.Millisecond) for i, tr := range received[1:] { d := tr.Sub(start) // Adjust start for next calcs. start = start.Add(d) if d < expected[i]-5*time.Millisecond || d > expected[i]*2+5*time.Millisecond { t.Fatalf("Timing is off for %d, expected ~%v, but got %v", i, expected[i], d) } } } func TestJetStreamClusterConsumerUpgrade(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() testUpdate := func(t *testing.T, s *Server) { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "X"}) require_NoError(t, err) _, err = js.Publish("X", []byte("OK")) require_NoError(t, err) // First create a consumer that is push based. _, err = js.AddConsumer("X", &nats.ConsumerConfig{Durable: "dlc", DeliverSubject: "Y"}) require_NoError(t, err) } t.Run("Single", func(t *testing.T) { testUpdate(t, s) }) t.Run("Clustered", func(t *testing.T) { testUpdate(t, c.randomServer()) }) } func TestJetStreamClusterAddConsumerWithInfo(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() testConsInfo := func(t *testing.T, s *Server) { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish("foo", []byte("msg")) require_NoError(t, err) } for i := 0; i < 100; i++ { inbox := nats.NewInbox() sub := natsSubSync(t, nc, inbox) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, DeliverPolicy: nats.DeliverAllPolicy, FilterSubject: "foo", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) if ci.NumPending != 10 { t.Fatalf("Iter=%v - expected 10 messages pending on create, got %v", i+1, ci.NumPending) } js.DeleteConsumer("TEST", ci.Name) sub.Unsubscribe() } } t.Run("Single", func(t *testing.T) { testConsInfo(t, s) }) t.Run("Clustered", func(t *testing.T) { testConsInfo(t, c.randomServer()) }) } func TestJetStreamClusterStreamReplicaUpdates(t *testing.T) { c := createJetStreamClusterExplicit(t, "R7S", 7) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Start out at R1 cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, } _, err := js.AddStream(cfg) require_NoError(t, err) numMsgs := 1000 for i := 0; i < numMsgs; i++ { js.PublishAsync("foo", []byte("HELLO WORLD")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } updateReplicas := func(r int) { t.Helper() si, err := js.StreamInfo("TEST") require_NoError(t, err) leader := si.Cluster.Leader cfg.Replicas = r _, err = js.UpdateStream(cfg) require_NoError(t, err) c.waitOnStreamLeader("$G", "TEST") checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err = js.StreamInfo("TEST") require_NoError(t, err) if len(si.Cluster.Replicas) != r-1 { return fmt.Errorf("Expected %d replicas, got %d", r-1, len(si.Cluster.Replicas)) } return nil }) // Make sure we kept same leader. if si.Cluster.Leader != leader { t.Fatalf("Leader changed, expected %q got %q", leader, si.Cluster.Leader) } // Make sure all are current. for _, r := range si.Cluster.Replicas { c.waitOnStreamCurrent(c.serverByName(r.Name), "$G", "TEST") } // Check msgs. if si.State.Msgs != uint64(numMsgs) { t.Fatalf("Expected %d msgs, got %d", numMsgs, si.State.Msgs) } // Make sure we have the right number of HA Assets running on the leader. s := c.serverByName(leader) jsi, err := s.Jsz(nil) require_NoError(t, err) nha := 1 // meta always present. if len(si.Cluster.Replicas) > 0 { nha++ } if nha != jsi.HAAssets { t.Fatalf("Expected %d HA asset(s), but got %d", nha, jsi.HAAssets) } } // Update from 1-3 updateReplicas(3) // Update from 3-5 updateReplicas(5) // Update from 5-3 updateReplicas(3) // Update from 3-1 updateReplicas(1) } func TestJetStreamClusterStreamAndConsumerScaleUpAndDown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Start out at R3 cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, } _, err := js.AddStream(cfg) require_NoError(t, err) sub, err := js.SubscribeSync("foo", nats.Durable("cat")) require_NoError(t, err) numMsgs := 10 for i := 0; i < numMsgs; i++ { _, err := js.Publish("foo", []byte("HELLO WORLD")) require_NoError(t, err) } checkSubsPending(t, sub, numMsgs) // Now ask leader to stepdown. rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) var sdResp JSApiStreamLeaderStepDownResponse err = json.Unmarshal(rmsg.Data, &sdResp) require_NoError(t, err) if sdResp.Error != nil || !sdResp.Success { t.Fatalf("Unexpected error: %+v", sdResp.Error) } c.waitOnStreamLeader("$G", "TEST") updateReplicas := func(r int) { t.Helper() cfg.Replicas = r _, err := js.UpdateStream(cfg) require_NoError(t, err) c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "cat") ci, err := js.ConsumerInfo("TEST", "cat") require_NoError(t, err) if ci.Cluster.Leader == _EMPTY_ { t.Fatalf("Expected a consumer leader but got none in consumer info") } if len(ci.Cluster.Replicas)+1 != r { t.Fatalf("Expected consumer info to have %d peers, got %d", r, len(ci.Cluster.Replicas)+1) } } // Capture leader, we want to make sure when we scale down this does not change. sl := c.streamLeader("$G", "TEST") // Scale down to 1. updateReplicas(1) if sl != c.streamLeader("$G", "TEST") { t.Fatalf("Expected same leader, but it changed") } // Make sure we can still send to the stream. for i := 0; i < numMsgs; i++ { _, err := js.Publish("foo", []byte("HELLO WORLD")) require_NoError(t, err) } si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != uint64(2*numMsgs) { t.Fatalf("Expected %d msgs, got %d", 3*numMsgs, si.State.Msgs) } checkSubsPending(t, sub, 2*numMsgs) // Now back up. updateReplicas(3) // Send more. for i := 0; i < numMsgs; i++ { _, err := js.Publish("foo", []byte("HELLO WORLD")) require_NoError(t, err) } si, err = js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != uint64(3*numMsgs) { t.Fatalf("Expected %d msgs, got %d", 3*numMsgs, si.State.Msgs) } checkSubsPending(t, sub, 3*numMsgs) // Make sure cluster replicas are current. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err = js.StreamInfo("TEST") require_NoError(t, err) for _, r := range si.Cluster.Replicas { if !r.Current { return fmt.Errorf("Expected replica to be current: %+v", r) } } return nil }) checkState := func(s *Server) { t.Helper() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) state := mset.state() if state.Msgs != uint64(3*numMsgs) || state.FirstSeq != 1 || state.LastSeq != 30 || state.Bytes != 1320 { return fmt.Errorf("Wrong state: %+v for server: %v", state, s) } return nil }) } // Now check each indidvidual stream on each server to make sure replication occurred. for _, s := range c.servers { checkState(s) } } func TestJetStreamClusterInterestRetentionWithFilteredConsumersExtra(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() subjectNameZero := "foo.bar" subjectNameOne := "foo.baz" // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.InterestPolicy, Replicas: 3}) require_NoError(t, err) checkState := func(expected uint64) { t.Helper() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != expected { return fmt.Errorf("Expected %d msgs, got %d", expected, si.State.Msgs) } return nil }) } subZero, err := js.PullSubscribe(subjectNameZero, "dlc-0") require_NoError(t, err) subOne, err := js.PullSubscribe(subjectNameOne, "dlc-1") require_NoError(t, err) msg := []byte("FILTERED") // Now send a bunch of messages for i := 0; i < 1000; i++ { _, err = js.PublishAsync(subjectNameZero, msg) require_NoError(t, err) _, err = js.PublishAsync(subjectNameOne, msg) require_NoError(t, err) } // should be 2000 in total checkState(2000) // fetch and acknowledge, count records to ensure no errors acknowledging getAndAckBatch := func(sub *nats.Subscription) { t.Helper() successCounter := 0 msgs, err := sub.Fetch(1000) require_NoError(t, err) for _, m := range msgs { err = m.AckSync() require_NoError(t, err) successCounter++ } if successCounter != 1000 { t.Fatalf("Unexpected number of acknowledges %d for subscription %v", successCounter, sub) } } // fetch records subscription zero getAndAckBatch(subZero) // fetch records for subscription one getAndAckBatch(subOne) // Make sure stream is zero. checkState(0) } func TestJetStreamClusterStreamConsumersCount(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sname := "TEST_STREAM_CONS_COUNT" _, err := js.AddStream(&nats.StreamConfig{Name: sname, Subjects: []string{"foo"}, Replicas: 3}) require_NoError(t, err) // Create some R1 consumers for i := 0; i < 10; i++ { inbox := nats.NewInbox() natsSubSync(t, nc, inbox) _, err = js.AddConsumer(sname, &nats.ConsumerConfig{DeliverSubject: inbox}) require_NoError(t, err) } // Now check that the consumer count in stream info/list is 10 checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { // Check stream info si, err := js.StreamInfo(sname) if err != nil { return fmt.Errorf("Error getting stream info: %v", err) } if n := si.State.Consumers; n != 10 { return fmt.Errorf("From StreamInfo, expecting 10 consumers, got %v", n) } // Now from stream list for si := range js.StreamsInfo() { if n := si.State.Consumers; n != 10 { return fmt.Errorf("From StreamsInfo, expecting 10 consumers, got %v", n) } } return nil }) } func TestJetStreamClusterFilteredAndIdleConsumerNRGGrowth(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sname := "TEST" _, err := js.AddStream(&nats.StreamConfig{Name: sname, Subjects: []string{"foo.*"}, Replicas: 3}) require_NoError(t, err) sub, err := js.SubscribeSync("foo.baz", nats.Durable("dlc")) require_NoError(t, err) for i := 0; i < 10_000; i++ { js.PublishAsync("foo.bar", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, 0) // Grab consumer's underlying info and make sure NRG log not running away do to no-op skips on filtered consumer. // Need a non-leader for the consumer, they are only ones getting skip ops to keep delivered updated. cl := c.consumerLeader("$G", "TEST", "dlc") var s *Server for _, s = range c.servers { if s != cl { break } } mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("dlc") if o == nil { t.Fatalf("Error looking up consumer %q", "dlc") } // compactNumMin from monitorConsumer is 8192 atm. const compactNumMin = 8192 if entries, _ := o.raftNode().Size(); entries > compactNumMin { t.Fatalf("Expected <= %d entries, got %d", compactNumMin, entries) } // Now make the consumer leader stepdown and make sure we have the proper snapshot. resp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) var cdResp JSApiConsumerLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if cdResp.Error != nil { t.Fatalf("Unexpected error: %+v", cdResp.Error) } c.waitOnConsumerLeader("$G", "TEST", "dlc") } func TestJetStreamClusterMirrorOrSourceNotActiveReporting(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}, Replicas: 3}) require_NoError(t, err) si, err := js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, }) require_NoError(t, err) // We would previous calculate a large number if we actually never heard from the peer yet. // We want to make sure if we have never heard from the other side report -1 as Active. // It is possible if testing infra is slow that this could be legit, but should be pretty small. if si.Mirror.Active != -1 && si.Mirror.Active > 10*time.Millisecond { t.Fatalf("Expected an Active of -1, but got %v", si.Mirror.Active) } } func TestJetStreamClusterStreamAdvisories(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() checkAdv := func(t *testing.T, sub *nats.Subscription, expectedPrefixes ...string) { t.Helper() seen := make([]bool, len(expectedPrefixes)) for i := 0; i < len(expectedPrefixes); i++ { msg := natsNexMsg(t, sub, time.Second) var gotOne bool for j, pfx := range expectedPrefixes { if !seen[j] && strings.HasPrefix(msg.Subject, pfx) { seen[j] = true gotOne = true break } } if !gotOne { t.Fatalf("Expected one of prefixes %q, got %q", expectedPrefixes, msg.Subject) } } } // Used to keep stream names pseudo unique. t.Name() has slashes in it which caused problems. var testN int checkAdvisories := func(t *testing.T, s *Server, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() testN++ streamName := "TEST_ADVISORIES_" + fmt.Sprintf("%d", testN) sub := natsSubSync(t, nc, "$JS.EVENT.ADVISORY.STREAM.*."+streamName) si, err := js.AddStream(&nats.StreamConfig{ Name: streamName, Storage: nats.FileStorage, Replicas: replicas, }) require_NoError(t, err) advisories := []string{JSAdvisoryStreamCreatedPre} if replicas > 1 { advisories = append(advisories, JSAdvisoryStreamLeaderElectedPre) } checkAdv(t, sub, advisories...) si.Config.MaxMsgs = 1000 _, err = js.UpdateStream(&si.Config) require_NoError(t, err) checkAdv(t, sub, JSAdvisoryStreamUpdatedPre) snapreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 512, } var snapshot []byte done := make(chan bool) nc.Subscribe(snapreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- true return } // Could be writing to a file here too. snapshot = append(snapshot, m.Data...) // Flow ack m.Respond(nil) }) req, _ := json.Marshal(snapreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, streamName), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } var snapresp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &snapresp) if snapresp.Error != nil { t.Fatalf("Did not get correct error response: %+v", snapresp.Error) } // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } checkAdv(t, sub, JSAdvisoryStreamSnapshotCreatePre) checkAdv(t, sub, JSAdvisoryStreamSnapshotCompletePre) err = js.DeleteStream(streamName) require_NoError(t, err) checkAdv(t, sub, JSAdvisoryStreamDeletedPre) state := *snapresp.State config := *snapresp.Config resreq := &JSApiStreamRestoreRequest{ Config: config, State: state, } req, _ = json.Marshal(resreq) rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, streamName), req, 5*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resresp JSApiStreamRestoreResponse json.Unmarshal(rmsg.Data, &resresp) if resresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", resresp.Error) } // Send our snapshot back in to restore the stream. // Can be any size message. var chunk [1024]byte for r := bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(resresp.DeliverSubject, chunk[:n], time.Second) } rmsg, err = nc.Request(resresp.DeliverSubject, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } resresp.Error = nil json.Unmarshal(rmsg.Data, &resresp) if resresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", resresp.Error) } checkAdv(t, sub, JSAdvisoryStreamRestoreCreatePre) // At this point, the stream_created advisory may be sent before // or after the restore_complete advisory because they are sent // using different "send queues". That is, the restore uses the // server's event queue while the stream_created is sent from // the stream's own send queue. advisories = append(advisories, JSAdvisoryStreamRestoreCompletePre) checkAdv(t, sub, advisories...) } t.Run("Single", func(t *testing.T) { checkAdvisories(t, s, 1) }) t.Run("Clustered_R1", func(t *testing.T) { checkAdvisories(t, c.randomServer(), 1) }) t.Run("Clustered_R3", func(t *testing.T) { checkAdvisories(t, c.randomServer(), 3) }) } // If the config files have duplicate routes this can have the metagroup estimate a size for the system // which prevents reaching quorum and electing a meta-leader. func TestJetStreamClusterDuplicateRoutesDisruptJetStreamMetaGroup(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: RR listen: 127.0.0.1:%d routes = [ nats-route://127.0.0.1:%d nats-route://127.0.0.1:%d nats-route://127.0.0.1:%d # These will be dupes nats-route://127.0.0.1:%d nats-route://127.0.0.1:%d nats-route://127.0.0.1:%d ] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := &cluster{servers: make([]*Server, 0, 3), opts: make([]*Options, 0, 3), name: "RR", t: t} rports := []int{22208, 22209, 22210} for i, p := range rports { sname, sd := fmt.Sprintf("S%d", i+1), t.TempDir() cf := fmt.Sprintf(tmpl, sname, sd, p, rports[0], rports[1], rports[2], rports[0], rports[1], rports[2]) s, o := RunServerWithConfig(createConfFile(t, []byte(cf))) c.servers, c.opts = append(c.servers, s), append(c.opts, o) } defer c.shutdown() checkClusterFormed(t, c.servers...) c.waitOnClusterReady() } func TestJetStreamClusterDuplicateMsgIdsOnCatchupAndLeaderTakeover(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, }) require_NoError(t, err) // Shutdown a non-leader. nc.Close() sr := c.randomNonStreamLeader("$G", "TEST") sr.Shutdown() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() m := nats.NewMsg("TEST") m.Data = []byte("OK") n := 10 for i := 0; i < n; i++ { m.Header.Set(JSMsgId, strconv.Itoa(i)) _, err := js.PublishMsg(m) require_NoError(t, err) } m.Header.Set(JSMsgId, "8") pa, err := js.PublishMsg(m) require_NoError(t, err) if !pa.Duplicate { t.Fatalf("Expected msg to be a duplicate") } // Now force a snapshot, want to test catchup above RAFT layer. sl := c.streamLeader("$G", "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) if node := mset.raftNode(); node == nil { t.Fatalf("Could not get stream group name") } else if err := node.InstallSnapshot(mset.stateSnapshot()); err != nil { t.Fatalf("Error installing snapshot: %v", err) } // Now restart sr = c.restartServer(sr) c.waitOnStreamCurrent(sr, "$G", "TEST") c.waitOnStreamLeader("$G", "TEST") // Now make them the leader. for sr != c.streamLeader("$G", "TEST") { nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) c.waitOnStreamLeader("$G", "TEST") } // Make sure this gets rejected. pa, err = js.PublishMsg(m) require_NoError(t, err) if !pa.Duplicate { t.Fatalf("Expected msg to be a duplicate") } } func TestJetStreamClusterConsumerLeaderChangeDeadlock(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Create a stream and durable with ack explicit _, err := js.AddStream(&nats.StreamConfig{Name: "test", Subjects: []string{"foo"}, Replicas: 3}) require_NoError(t, err) _, err = js.AddConsumer("test", &nats.ConsumerConfig{ Durable: "test", DeliverSubject: "bar", AckPolicy: nats.AckExplicitPolicy, AckWait: 250 * time.Millisecond, }) require_NoError(t, err) // Wait for a leader c.waitOnConsumerLeader("$G", "test", "test") cl := c.consumerLeader("$G", "test", "test") // Publish a message _, err = js.Publish("foo", []byte("msg")) require_NoError(t, err) // Create nats consumer on "bar" and don't ack it sub := natsSubSync(t, nc, "bar") natsNexMsg(t, sub, time.Second) // Wait for redeliveries, to make sure it is in the redelivery map natsNexMsg(t, sub, time.Second) natsNexMsg(t, sub, time.Second) mset, err := cl.GlobalAccount().lookupStream("test") require_NoError(t, err) require_True(t, mset != nil) // There are parts in the code (for instance when signaling to consumers // that there are new messages) where we get the mset lock and iterate // over the consumers and get consumer lock. We are going to do that // in a go routine while we send a consumer step down request from // another go routine. We will watch for possible deadlock and if // found report it. ch := make(chan struct{}) doneCh := make(chan struct{}) go func() { defer close(doneCh) for { mset.mu.Lock() for _, o := range mset.consumers { o.mu.Lock() time.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond) o.mu.Unlock() } mset.mu.Unlock() select { case <-ch: return default: } } }() // Now cause a leader changes for i := 0; i < 5; i++ { m, err := nc.Request("$JS.API.CONSUMER.LEADER.STEPDOWN.test.test", nil, 2*time.Second) // Ignore error here and check for deadlock below if err != nil { break } // if there is a message, check that it is success var resp JSApiConsumerLeaderStepDownResponse err = json.Unmarshal(m.Data, &resp) require_NoError(t, err) require_True(t, resp.Success) c.waitOnConsumerLeader("$G", "test", "test") } close(ch) select { case <-doneCh: // OK! case <-time.After(2 * time.Second): buf := make([]byte, 1000000) n := runtime.Stack(buf, true) t.Fatalf("Suspected deadlock, printing current stack. The test suite may timeout and will also dump the stack\n%s\n", buf[:n]) } } // We were compacting to keep the raft log manageable but not snapshotting, which meant that restarted // servers could complain about no snapshot and could not sync after that condition. // Changes also address https://github.com/nats-io/nats-server/issues/2936 func TestJetStreamClusterMemoryConsumerCompactVsSnapshot(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Create a stream and durable with ack explicit _, err := js.AddStream(&nats.StreamConfig{ Name: "test", Storage: nats.MemoryStorage, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("test", &nats.ConsumerConfig{ Durable: "d", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // Bring a non-leader down. s := c.randomNonConsumerLeader("$G", "test", "d") s.Shutdown() // In case that was also mete or stream leader. c.waitOnLeader() c.waitOnStreamLeader("$G", "test") // In case we were connected there. nc.Close() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Generate some state. for i := 0; i < 2000; i++ { _, err := js.PublishAsync("test", nil) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sub, err := js.PullSubscribe("test", "d") require_NoError(t, err) for i := 0; i < 2; i++ { for _, m := range fetchMsgs(t, sub, 1000, 5*time.Second) { m.AckSync() } } // Restart our downed server. s = c.restartServer(s) c.checkClusterFormed() c.waitOnServerCurrent(s) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := js.ConsumerInfo("test", "d") require_NoError(t, err) for _, r := range ci.Cluster.Replicas { if !r.Current || r.Lag != 0 { return fmt.Errorf("Replica not current: %+v", r) } } return nil }) } func TestJetStreamClusterMemoryConsumerInterestRetention(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "test", Storage: nats.MemoryStorage, Retention: nats.InterestPolicy, Replicas: 3, }) require_NoError(t, err) sub, err := js.SubscribeSync("test", nats.Durable("dlc")) require_NoError(t, err) for i := 0; i < 1000; i++ { _, err := js.PublishAsync("test", nil) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } toAck := 100 for i := 0; i < toAck; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) m.AckSync() } checkFor(t, time.Second, 15*time.Millisecond, func() error { si, err := js.StreamInfo("test") if err != nil { return err } if n := si.State.Msgs; n != 900 { return fmt.Errorf("Waiting for msgs count to be 900, got %v", n) } return nil }) si, err := js.StreamInfo("test") require_NoError(t, err) ci, err := sub.ConsumerInfo() require_NoError(t, err) // Make sure acks are not only replicated but processed in a way to remove messages from the replica streams. _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "test"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader("$G", "test") _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "test", "dlc"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader("$G", "test", "dlc") nsi, err := js.StreamInfo("test") require_NoError(t, err) if !reflect.DeepEqual(nsi.State, si.State) { t.Fatalf("Stream states do not match: %+v vs %+v", si.State, nsi.State) } nci, err := sub.ConsumerInfo() require_NoError(t, err) // Last may be skewed a very small amount. ci.AckFloor.Last, nci.AckFloor.Last = nil, nil if nci.AckFloor != ci.AckFloor { t.Fatalf("Consumer AckFloors are not the same: %+v vs %+v", ci.AckFloor, nci.AckFloor) } } func TestJetStreamClusterDeleteAndRestoreAndRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}) require_NoError(t, err) for i := 0; i < 10; i++ { _, err := js.Publish("TEST", []byte("OK")) require_NoError(t, err) } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) require_NoError(t, js.DeleteConsumer("TEST", "dlc")) require_NoError(t, js.DeleteStream("TEST")) _, err = js.AddStream(&nats.StreamConfig{Name: "TEST"}) require_NoError(t, err) for i := 0; i < 22; i++ { _, err := js.Publish("TEST", []byte("OK")) require_NoError(t, err) } sub, err := js.SubscribeSync("TEST", nats.Durable("dlc"), nats.Description("SECOND")) require_NoError(t, err) for i := 0; i < 5; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) m.AckSync() } // Now restart. sl := c.streamLeader("$G", "TEST") sl.Shutdown() sl = c.restartServer(sl) c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "dlc") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 22 { return fmt.Errorf("State is not correct after restart, expected 22 msgs, got %d", si.State.Msgs) } return nil }) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) if ci.AckFloor.Consumer != 5 { t.Fatalf("Bad ack floor: %+v", ci.AckFloor) } // Now delete and make sure consumer does not come back. // First add a delete something else. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST2"}) require_NoError(t, err) require_NoError(t, js.DeleteStream("TEST2")) // Now the consumer. require_NoError(t, js.DeleteConsumer("TEST", "dlc")) sl.Shutdown() c.restartServer(sl) c.waitOnStreamLeader("$G", "TEST") // In rare circumstances this could be recovered and then quickly deleted. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if _, err := js.ConsumerInfo("TEST", "dlc"); err == nil { return fmt.Errorf("Not cleaned up yet") } return nil }) } func TestJetStreamClusterMirrorSourceLoop(t *testing.T) { test := func(t *testing.T, s *Server, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() // Create a source/mirror loop _, err := js.AddStream(&nats.StreamConfig{ Name: "1", Subjects: []string{"foo", "bar"}, Replicas: replicas, Sources: []*nats.StreamSource{{Name: "DECOY"}, {Name: "2"}}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "DECOY", Subjects: []string{"baz"}, Replicas: replicas, Sources: []*nats.StreamSource{{Name: "NOTTHERE"}}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "2", Replicas: replicas, Sources: []*nats.StreamSource{{Name: "3"}}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "3", Replicas: replicas, Sources: []*nats.StreamSource{{Name: "1"}}, }) require_Error(t, err) require_Equal(t, err.Error(), "nats: detected cycle") } t.Run("Single", func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() test(t, s, 1) }) t.Run("Clustered", func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 5) defer c.shutdown() test(t, c.randomServer(), 2) }) } func TestJetStreamClusterMirrorDeDupWindow(t *testing.T) { owt := srcConsumerWaitTime srcConsumerWaitTime = 2 * time.Second defer func() { srcConsumerWaitTime = owt }() c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "S", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) require_True(t, si.Cluster != nil) require_True(t, si.Config.Replicas == 3) require_True(t, len(si.Cluster.Replicas) == 2) send := func(count int) { t.Helper() for i := 0; i < count; i++ { _, err := js.Publish("foo", []byte("msg")) require_NoError(t, err) } } // Send 100 messages send(100) // Now create a valid one. si, err = js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 3, Mirror: &nats.StreamSource{Name: "S"}, }) require_NoError(t, err) require_True(t, si.Cluster != nil) require_True(t, si.Config.Replicas == 3) require_True(t, len(si.Cluster.Replicas) == 2) check := func(expected int) { t.Helper() // Wait for all messages to be in mirror checkFor(t, 15*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo("M") if err != nil { return err } if n := si.State.Msgs; int(n) != expected { return fmt.Errorf("Expected %v msgs, got %v", expected, n) } return nil }) } check(100) // Restart cluster nc.Close() c.stopAll() c.restartAll() c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "S") c.waitOnStreamLeader(globalAccountName, "M") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() si, err = js.StreamInfo("M") require_NoError(t, err) require_True(t, si.Cluster != nil) require_True(t, si.Config.Replicas == 3) require_True(t, len(si.Cluster.Replicas) == 2) // Send 100 messages send(100) check(200) } func TestJetStreamClusterNewHealthz(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "R1", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "R3", Subjects: []string{"bar"}, Replicas: 3, }) require_NoError(t, err) // Create subscribers (durable and ephemeral for each) fsube, err := js.SubscribeSync("foo") require_NoError(t, err) fsubd, err := js.SubscribeSync("foo", nats.Durable("d")) require_NoError(t, err) _, err = js.SubscribeSync("bar") require_NoError(t, err) bsubd, err := js.SubscribeSync("bar", nats.Durable("d")) require_NoError(t, err) for i := 0; i < 20; i++ { _, err = js.Publish("foo", []byte("foo")) require_NoError(t, err) } checkSubsPending(t, fsube, 20) checkSubsPending(t, fsubd, 20) // Select the server where we know the R1 stream is running. sl := c.streamLeader("$G", "R1") sl.Shutdown() // Do same on R3 so that sl has to recover some things before healthz should be good. c.waitOnStreamLeader("$G", "R3") for i := 0; i < 10; i++ { _, err = js.Publish("bar", []byte("bar")) require_NoError(t, err) } // Ephemeral is skipped, might have been on the downed server. checkSubsPending(t, bsubd, 10) sl = c.restartServer(sl) c.waitOnServerHealthz(sl) } func TestJetStreamClusterConsumerOverrides(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Test replica override. // Make sure we can not go "wider" than the parent stream. ccReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "d", AckPolicy: AckExplicit, Replicas: 5, }, } req, err := json.Marshal(ccReq) require_NoError(t, err) ci, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "d"), req, time.Second) require_NoError(t, err) var resp JSApiConsumerCreateResponse err = json.Unmarshal(ci.Data, &resp) require_NoError(t, err) if resp.Error == nil || !IsNatsErr(resp.Error, JSConsumerReplicasExceedsStream) { t.Fatalf("Expected an error when replicas > parent stream, got %+v", resp.Error) } // Durables inherit the replica count from the stream, so make sure we can override that. ccReq.Config.Replicas = 1 req, err = json.Marshal(ccReq) require_NoError(t, err) ci, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "d"), req, time.Second) require_NoError(t, err) resp.Error = nil err = json.Unmarshal(ci.Data, &resp) require_NoError(t, err) require_True(t, resp.Error == nil) checkCount := func(durable string, expected int) { t.Helper() count := 0 for _, s := range c.servers { if mset, err := s.GlobalAccount().lookupStream("TEST"); err == nil { if o := mset.lookupConsumer(durable); o != nil { count++ } } } if count != expected { t.Fatalf("Expected %d consumers in cluster, got %d", expected, count) } } checkCount("d", 1) // Now override storage and force storage to memory based. ccReq.Config.MemoryStorage = true ccReq.Config.Durable = "m" ccReq.Config.Replicas = 3 req, err = json.Marshal(ccReq) require_NoError(t, err) ci, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "m"), req, time.Second) require_NoError(t, err) resp.Error = nil err = json.Unmarshal(ci.Data, &resp) require_NoError(t, err) require_True(t, resp.Error == nil) checkCount("m", 3) // Make sure memory setting is for both consumer raft log and consumer store. s := c.consumerLeader("$G", "TEST", "m") require_True(t, s != nil) mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("m") require_True(t, o != nil) st := o.store.Type() n := o.raftNode() require_True(t, n != nil) rn := n.(*raft) rn.RLock() wal := rn.wal rn.RUnlock() require_True(t, wal.Type() == MemoryStorage) require_True(t, st == MemoryStorage) // Now make sure we account properly for the consumers. // Add in normal here first. _, err = js.SubscribeSync("foo", nats.Durable("d22")) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Consumers == 3) err = js.DeleteConsumer("TEST", "d") require_NoError(t, err) // Also make sure the stream leader direct store reports same with mixed and matched. s = c.streamLeader("$G", "TEST") require_True(t, s != nil) mset, err = s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) state := mset.Store().State() require_True(t, state.Consumers == 2) // Fast state version as well. fstate := mset.stateWithDetail(false) require_True(t, fstate.Consumers == 2) // Make sure delete accounting works too. err = js.DeleteConsumer("TEST", "m") require_NoError(t, err) state = mset.Store().State() require_True(t, state.Consumers == 1) // Fast state version as well. fstate = mset.stateWithDetail(false) require_True(t, fstate.Consumers == 1) } func TestJetStreamClusterStreamRepublish(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "RP", Storage: MemoryStorage, Subjects: []string{"foo", "bar", "baz"}, Replicas: 3, RePublish: &RePublish{ Source: ">", Destination: "RP.>", }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("RP.>") require_NoError(t, err) msg, toSend := []byte("OK TO REPUBLISH?"), 100 for i := 0; i < toSend; i++ { _, err = js.PublishAsync("foo", msg) require_NoError(t, err) _, err = js.PublishAsync("bar", msg) require_NoError(t, err) _, err = js.PublishAsync("baz", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, toSend*3) lseq := map[string]int{ "foo": 0, "bar": 0, "baz": 0, } for i := 1; i <= toSend; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) // Grab info from Header require_True(t, m.Header.Get(JSStream) == "RP") // Make sure sequence is correct. seq, err := strconv.Atoi(m.Header.Get(JSSequence)) require_NoError(t, err) require_True(t, seq == i) // Make sure timestamp is correct ts, err := time.Parse(time.RFC3339Nano, m.Header.Get(JSTimeStamp)) require_NoError(t, err) origMsg, err := js.GetMsg("RP", uint64(seq)) require_NoError(t, err) require_True(t, ts == origMsg.Time) // Make sure last sequence matches last seq we received on this subject. last, err := strconv.Atoi(m.Header.Get(JSLastSequence)) require_NoError(t, err) require_True(t, last == lseq[m.Subject]) lseq[m.Subject] = seq } } func TestJetStreamClusterConsumerDeliverNewNotConsumingBeforeStepDownOrRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) inbox := nats.NewInbox() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, Durable: "dur", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, FilterSubject: "foo", }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", "msg") } checkCount := func(expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "dur") if err != nil { return err } if n := int(ci.NumPending); n != expected { return fmt.Errorf("Expected %v pending, got %v", expected, n) } return nil }) } checkCount(10) resp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dur"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cdResp JSApiConsumerLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if cdResp.Error != nil { t.Fatalf("Unexpected error: %+v", cdResp.Error) } c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") checkCount(10) // Check also servers restart nc.Close() c.stopAll() c.restartAll() c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkCount(10) // Make sure messages can be consumed sub := natsSubSync(t, nc, inbox) for i := 0; i < 10; i++ { msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("i=%v next msg error: %v", i, err) } msg.AckSync() } checkCount(0) } func TestJetStreamClusterConsumerDeliverNewMaxRedeliveriesAndServerRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) inbox := nats.NewInbox() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, Durable: "dur", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, MaxDeliver: 3, AckWait: 250 * time.Millisecond, FilterSubject: "foo.bar", }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") sendStreamMsg(t, nc, "foo.bar", "msg") sub := natsSubSync(t, nc, inbox) for i := 0; i < 3; i++ { natsNexMsg(t, sub, time.Second) } // Now check that there is no more redeliveries if msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout, got msg=%+v err=%v", msg, err) } // Check server restart nc.Close() c.stopAll() c.restartAll() c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() sub = natsSubSync(t, nc, inbox) // We should not have messages being redelivered. if msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout, got msg=%+v err=%v", msg, err) } } func TestJetStreamClusterNoRestartAdvisories(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"bar"}, Replicas: 1, }) require_NoError(t, err) // Create 10 consumers for i := 0; i < 10; i++ { dur := fmt.Sprintf("dlc-%d", i) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: dur, AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) } msg := bytes.Repeat([]byte("Z"), 1024) for i := 0; i < 1000; i++ { js.PublishAsync("foo", msg) js.PublishAsync("bar", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Add state to a consumer. sub, err := js.PullSubscribe("foo", "dlc-2") require_NoError(t, err) for _, m := range fetchMsgs(t, sub, 5, time.Second) { m.AckSync() } nc.Close() // Required to show the bug. c.leader().JetStreamSnapshotMeta() nc, _ = jsClientConnect(t, c.consumerLeader("$G", "TEST", "dlc-2")) defer nc.Close() sub, err = nc.SubscribeSync("$JS.EVENT.ADVISORY.API") require_NoError(t, err) // Shutdown and Restart. s := c.randomNonConsumerLeader("$G", "TEST", "dlc-2") s.Shutdown() s = c.restartServer(s) c.waitOnServerHealthz(s) checkSubsPending(t, sub, 0) nc, _ = jsClientConnect(t, c.randomNonStreamLeader("$G", "TEST")) defer nc.Close() sub, err = nc.SubscribeSync("$JS.EVENT.ADVISORY.STREAM.UPDATED.>") require_NoError(t, err) s = c.streamLeader("$G", "TEST2") s.Shutdown() s = c.restartServer(s) c.waitOnServerHealthz(s) checkSubsPending(t, sub, 0) } func TestJetStreamClusterR1StreamPlacementNoReservation(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sp := make(map[string]int) for i := 0; i < 100; i++ { sname := fmt.Sprintf("T-%d", i) _, err := js.AddStream(&nats.StreamConfig{ Name: sname, }) require_NoError(t, err) sp[c.streamLeader("$G", sname).Name()]++ } for serverName, num := range sp { if num > 60 { t.Fatalf("Streams not distributed, expected ~30-35 but got %d for server %q", num, serverName) } } } func TestJetStreamClusterConsumerAndStreamNamesWithPathSeparators(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "usr/bin"}) require_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName) _, err = js.AddStream(&nats.StreamConfig{Name: `Documents\readme.txt`}) require_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName) // Now consumers. _, err = js.AddStream(&nats.StreamConfig{Name: "T"}) require_NoError(t, err) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: "a/b", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: `a\b`, AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName) } func TestJetStreamClusterFilteredMirrors(t *testing.T) { c := createJetStreamClusterExplicit(t, "MSR", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz"}, }) require_NoError(t, err) msg := bytes.Repeat([]byte("Z"), 3) for i := 0; i < 100; i++ { js.PublishAsync("foo", msg) js.PublishAsync("bar", msg) js.PublishAsync("baz", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Create Mirror now. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST", FilterSubject: "foo"}, }) require_NoError(t, err) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("M") require_NoError(t, err) if si.State.Msgs != 100 { return fmt.Errorf("Expected 100 msgs, got state: %+v", si.State) } return nil }) sub, err := js.PullSubscribe("foo", "d", nats.BindStream("M")) require_NoError(t, err) // Make sure we only have "foo" and that sequence numbers preserved. sseq, dseq := uint64(1), uint64(1) for _, m := range fetchMsgs(t, sub, 100, 5*time.Second) { require_True(t, m.Subject == "foo") meta, err := m.Metadata() require_NoError(t, err) require_True(t, meta.Sequence.Consumer == dseq) dseq++ require_True(t, meta.Sequence.Stream == sseq) sseq += 3 } } // Test for making sure we error on same cluster name. func TestJetStreamClusterSameClusterLeafNodes(t *testing.T) { c := createJetStreamCluster(t, jsClusterAccountsTempl, "SAME", _EMPTY_, 3, 11233, true) defer c.shutdown() // Do by hand since by default we check for connections. tmpl := c.createLeafSolicit(jsClusterTemplWithLeafNode) lc := createJetStreamCluster(t, tmpl, "SAME", "S-", 2, 22111, false) defer lc.shutdown() time.Sleep(200 * time.Millisecond) // Make sure no leafnodes are connected. for _, s := range lc.servers { checkLeafNodeConnectedCount(t, s, 0) } } // https://github.com/nats-io/nats-server/issues/3178 func TestJetStreamClusterLeafNodeSPOFMigrateLeaders(t *testing.T) { tmpl := strings.Replace(jsClusterTempl, "store_dir:", "domain: REMOTE, store_dir:", 1) c := createJetStreamClusterWithTemplate(t, tmpl, "HUB", 2) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: CORE, store_dir:", 1) lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "LNC", 2, 22110) defer lnc.shutdown() lnc.waitOnClusterReady() // Place JS assets in LN, and we will do a pull consumer from the HUB. nc, js := jsClientConnect(t, lnc.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) require_NoError(t, err) require_True(t, si.Cluster.Name == "LNC") for i := 0; i < 100; i++ { js.PublishAsync("foo", []byte("HELLO")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Create the consumer. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "d", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() dsubj := "$JS.CORE.API.CONSUMER.MSG.NEXT.TEST.d" // Grab directly using domain based subject but from the HUB cluster. _, err = nc.Request(dsubj, nil, time.Second) require_NoError(t, err) // Now we will force the consumer leader's server to drop and stall leafnode connections. cl := lnc.consumerLeader("$G", "TEST", "d") cl.setJetStreamMigrateOnRemoteLeaf() cl.closeAndDisableLeafnodes() // Now make sure we can eventually get a message again. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { _, err = nc.Request(dsubj, nil, 500*time.Millisecond) return err }) nc, _ = jsClientConnect(t, lnc.randomServer()) defer nc.Close() // Now make sure the consumer, or any other asset, can not become a leader on this node while the leafnode // is disconnected. csd := fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "d") for i := 0; i < 10; i++ { nc.Request(csd, nil, time.Second) lnc.waitOnConsumerLeader(globalAccountName, "TEST", "d") if lnc.consumerLeader(globalAccountName, "TEST", "d") == cl { t.Fatalf("Consumer leader should not migrate to server without a leafnode connection") } } // Now make sure once leafnode is back we can have leaders on this server. cl.reEnableLeafnodes() checkLeafNodeConnectedCount(t, cl, 2) // Make sure we can migrate back to this server now that we are connected. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { nc.Request(csd, nil, time.Second) lnc.waitOnConsumerLeader(globalAccountName, "TEST", "d") if lnc.consumerLeader(globalAccountName, "TEST", "d") == cl { return nil } return fmt.Errorf("Not this server yet") }) } func TestJetStreamClusterStreamCatchupWithTruncateAndPriorSnapshot(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Shutdown a replica rs := c.randomNonStreamLeader("$G", "TEST") rs.Shutdown() if s == rs { nc.Close() s = c.randomServer() nc, js = jsClientConnect(t, s) defer nc.Close() } msg, toSend := []byte("OK"), 100 for i := 0; i < toSend; i++ { _, err := js.PublishAsync("foo", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(2 * time.Second): t.Fatalf("Did not receive completion signal") } sl := c.streamLeader("$G", "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) // Force snapshot require_NoError(t, mset.raftNode().InstallSnapshot(mset.stateSnapshot())) // Now truncate the store on purpose. err = mset.store.Truncate(50) require_NoError(t, err) // Restart Server. rs = c.restartServer(rs) // Make sure we can become current. // With bug we would fail here. c.waitOnStreamCurrent(rs, "$G", "TEST") } func TestJetStreamClusterNoOrphanedDueToNoConnection(t *testing.T) { orgEventsHBInterval := eventsHBInterval eventsHBInterval = 500 * time.Millisecond defer func() { eventsHBInterval = orgEventsHBInterval }() c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) checkSysServers := func() { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for _, s := range c.servers { s.mu.RLock() num := len(s.sys.servers) s.mu.RUnlock() if num != 2 { return fmt.Errorf("Expected server %q to have 2 servers, got %v", s, num) } } return nil }) } checkSysServers() nc.Close() s.mu.RLock() val := (s.sys.orphMax / eventsHBInterval) + 2 s.mu.RUnlock() time.Sleep(val * eventsHBInterval) checkSysServers() } func TestJetStreamClusterStreamResetOnExpirationDuringPeerDownAndRestartWithLeaderChange(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, MaxAge: time.Second, }) require_NoError(t, err) n := 100 for i := 0; i < n; i++ { js.PublishAsync("foo", []byte("NORESETPLS")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Shutdown a non-leader before expiration. nsl := c.randomNonStreamLeader("$G", "TEST") nsl.Shutdown() // Wait for all messages to expire. checkFor(t, 5*time.Second, time.Second, func() error { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() si, err := js.StreamInfo("TEST", nats.Context(ctx)) if err != nil { return err } if si.State.Msgs == 0 { return nil } return fmt.Errorf("Wanted 0 messages, got %d", si.State.Msgs) }) // Now restart the non-leader server, twice. First time clears raft, // second will not have any index state or raft to tell it what is first sequence. nsl = c.restartServer(nsl) c.checkClusterFormed() c.waitOnServerCurrent(nsl) // Now clear raft WAL. mset, err := nsl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) require_NoError(t, mset.raftNode().InstallSnapshot(mset.stateSnapshot())) nsl.Shutdown() nsl = c.restartServer(nsl) c.checkClusterFormed() c.waitOnServerCurrent(nsl) // We will now check this server directly. mset, err = nsl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) if state := mset.state(); state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) { t.Fatalf("Expected first sequence of %d, got %d", n+1, state.FirstSeq) } si, err := js.StreamInfo("TEST") require_NoError(t, err) if state := si.State; state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) { t.Fatalf("Expected first sequence of %d, got %d", n+1, state.FirstSeq) } // Now move the leader there and double check, but above test is sufficient. checkFor(t, 30*time.Second, 250*time.Millisecond, func() error { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) if err != nil { return err } c.waitOnStreamLeader("$G", "TEST") if c.streamLeader("$G", "TEST") == nsl { return nil } return fmt.Errorf("No correct leader yet") }) if state := mset.state(); state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) { t.Fatalf("Expected first sequence of %d, got %d", n+1, state.FirstSeq) } si, err = js.StreamInfo("TEST") require_NoError(t, err) if state := si.State; state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) { t.Fatalf("Expected first sequence of %d, got %d", n+1, state.FirstSeq) } } func TestJetStreamClusterPullConsumerMaxWaiting(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"test.*"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, MaxWaiting: 10, }) require_NoError(t, err) // Cannot be updated. _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, MaxWaiting: 1, }) if !strings.Contains(err.Error(), "can not be updated") { t.Fatalf(`expected "cannot be updated" error, got %s`, err) } } func TestJetStreamClusterEncryptedDoubleSnapshotBug(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterEncryptedTempl, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxAge: time.Second, Replicas: 3, }) require_NoError(t, err) numMsgs := 50 for i := 0; i < numMsgs; i++ { js.PublishAsync("foo", []byte("SNAP")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Perform a snapshot on a follower. nl := c.randomNonStreamLeader("$G", "TEST") mset, err := nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) err = mset.raftNode().InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) _, err = js.Publish("foo", []byte("SNAP2")) require_NoError(t, err) for _, seq := range []uint64{1, 11, 22, 51} { js.DeleteMsg("TEST", seq) } err = mset.raftNode().InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) _, err = js.Publish("foo", []byte("SNAP3")) require_NoError(t, err) } func TestJetStreamClusterRePublishUpdateSupported(t *testing.T) { test := func(t *testing.T, s *Server, stream string, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: stream, Storage: nats.MemoryStorage, Replicas: replicas, Subjects: []string{"foo.>"}, } _, err := js.AddStream(cfg) require_NoError(t, err) expectUpdate := func() { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Type != JSApiStreamUpdateResponseType { t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamUpdateResponseType) } if IsNatsErr(resp.Error, JSStreamInvalidConfigF) { t.Fatalf("Expected no error regarding config error, got %+v", resp.Error) } } expectRepublished := func(expectedRepub bool) { t.Helper() nc, js := jsClientConnect(t, s) defer nc.Close() // Create a subscriber for foo.> so that we can see // our published message being echoed back to us. sf, err := nc.SubscribeSync("foo.>") require_NoError(t, err) defer sf.Unsubscribe() // Create a subscriber for bar.> so that we can see // any potentially republished messages. sb, err := nc.SubscribeSync("bar.>") require_NoError(t, err) defer sf.Unsubscribe() // Publish a message, it will hit the foo.> stream and // may potentially be republished to the bar.> stream. _, err = js.Publish("foo."+stream, []byte("HELLO!")) require_NoError(t, err) // Wait for a little while so that we have enough time // to determine whether it's going to arrive on one or // both streams. checkSubsPending(t, sf, 1) if expectedRepub { checkSubsPending(t, sb, 1) } else { checkSubsPending(t, sb, 0) } } // At this point there's no republish config, so we should // only receive our published message on foo.>. expectRepublished(false) // Add a republish config so that everything on foo.> also // gets republished to bar.>. cfg.RePublish = &nats.RePublish{ Source: "foo.>", Destination: "bar.>", } expectUpdate() expectRepublished(true) // Now take the republish config away again, so we should go // back to only getting them on foo.>. cfg.RePublish = nil expectUpdate() expectRepublished(false) } t.Run("Single", func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() test(t, s, "single", 1) }) t.Run("Clustered", func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() test(t, c.randomNonLeader(), "clustered", 3) }) } func TestJetStreamClusterDirectGetFromLeafnode(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: CORE, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 19022, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) ln := c.createLeafNodeWithTemplate("LN-SPOKE", tmpl) defer ln.Shutdown() checkLeafNodeConnectedCount(t, ln, 2) nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "KV"}) require_NoError(t, err) _, err = kv.PutString("age", "22") require_NoError(t, err) // Now connect to the ln and make sure we can do a domain direct get. nc, _ = jsClientConnect(t, ln) defer nc.Close() js, err = nc.JetStream(nats.Domain("CORE")) require_NoError(t, err) kv, err = js.KeyValue("KV") require_NoError(t, err) entry, err := kv.Get("age") require_NoError(t, err) require_True(t, string(entry.Value()) == "22") } func TestJetStreamClusterUnknownReplicaOnClusterRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}, Replicas: 3}) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") lname := c.streamLeader(globalAccountName, "TEST").Name() sendStreamMsg(t, nc, "foo", "msg1") nc.Close() c.stopAll() // Restart the leader... for _, s := range c.servers { if s.Name() == lname { c.restartServer(s) } } // And one of the other servers for _, s := range c.servers { if s.Name() != lname { c.restartServer(s) break } } c.waitOnStreamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() sendStreamMsg(t, nc, "foo", "msg2") si, err := js.StreamInfo("TEST") require_NoError(t, err) if len(si.Cluster.Replicas) != 2 { t.Fatalf("Leader is %s - expected 2 peers, got %+v", si.Cluster.Leader, si.Cluster.Replicas[0]) } // However, since the leader does not know the name of the server // we should report an "unknown" name. var ok bool for _, r := range si.Cluster.Replicas { if strings.Contains(r.Name, "unknown") { // Check that it has no lag reported, and the it is not current. if r.Current { t.Fatal("Expected non started node to be marked as not current") } if r.Lag != 0 { t.Fatalf("Expected lag to not be set, was %v", r.Lag) } if r.Active != 0 { t.Fatalf("Expected active to not be set, was: %v", r.Active) } ok = true break } } if !ok { t.Fatalf("Should have had an unknown server name, did not: %+v - %+v", si.Cluster.Replicas[0], si.Cluster.Replicas[1]) } } func TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxAge: 5 * time.Second, Replicas: 3, }) require_NoError(t, err) sl := c.streamLeader("$G", "TEST") nl := c.randomNonStreamLeader("$G", "TEST") // Make sure we do not get disconnected when shutting the non-leader down. nc, js = jsClientConnect(t, sl) defer nc.Close() send1k := func() { t.Helper() for i := 0; i < 1000; i++ { js.PublishAsync("foo", []byte("SNAP")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } } // Send first 1000 to everyone. send1k() // Now shutdown a non-leader. c.waitOnStreamCurrent(nl, "$G", "TEST") nl.Shutdown() // Send another 1000. send1k() // Force snapshot on the leader. mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) err = mset.raftNode().InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) // Purge err = js.PurgeStream("TEST") require_NoError(t, err) // Send another 1000. send1k() // We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages. nc, _ = jsClientConnect(t, sl, nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() sub, err := nc.SubscribeSync("$JSC.R.>") require_NoError(t, err) // Now restart non-leader. nl = c.restartServer(nl) c.waitOnStreamCurrent(nl, "$G", "TEST") // Grab state directly from non-leader. mset, err = nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if state := mset.state(); state.FirstSeq != 2001 || state.LastSeq != 3000 { return fmt.Errorf("Incorrect state: %+v", state) } return nil }) // Make sure we only sent 2 sync catchup msgs. // This is for the delete range, and the EOF. nmsgs, _, _ := sub.Pending() if nmsgs != 2 { t.Fatalf("Expected only 2 sync catchup msgs to be sent signaling eof, but got %d", nmsgs) } msg, err := sub.NextMsg(0) require_NoError(t, err) mbuf := msg.Data[1:] dr, err := decodeDeleteRange(mbuf) require_NoError(t, err) require_Equal(t, dr.First, 1001) require_Equal(t, dr.Num, 1000) msg, err = sub.NextMsg(0) require_NoError(t, err) require_Equal(t, len(msg.Data), 0) } func TestJetStreamClusterStreamResetWithLargeFirstSeq(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxAge: 5 * time.Second, Replicas: 1, } _, err := js.AddStream(cfg) require_NoError(t, err) // Fake a very large first seq. sl := c.streamLeader("$G", "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.Lock() mset.store.Compact(1_000_000) mset.mu.Unlock() // Restart sl.Shutdown() sl = c.restartServer(sl) c.waitOnStreamLeader("$G", "TEST") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Make sure we have the correct state after restart. si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 1_000_000) // Now add in 10,000 messages. num := 10_000 for i := 0; i < num; i++ { js.PublishAsync("foo", []byte("SNAP")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 1_000_000) require_True(t, si.State.LastSeq == uint64(1_000_000+num-1)) // We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages. ncs, _ := jsClientConnect(t, sl, nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() sub, err := ncs.SubscribeSync("$JSC.R.>") require_NoError(t, err) // Now scale up to R3. cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) nl := c.randomNonStreamLeader("$G", "TEST") c.waitOnStreamCurrent(nl, "$G", "TEST") // Make sure we only sent the number of catchup msgs we expected. checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); nmsgs != (cfg.Replicas-1)*(num+1) { return fmt.Errorf("expected %d catchup msgs, but got %d", (cfg.Replicas-1)*(num+1), nmsgs) } return nil }) } func TestJetStreamClusterStreamCatchupInteriorNilMsgs(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, } _, err := js.AddStream(cfg) require_NoError(t, err) num := 100 for l := 0; l < 5; l++ { for i := 0; i < num-1; i++ { js.PublishAsync("foo", []byte("SNAP")) } // Blank msg. js.PublishAsync("foo", nil) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Make sure we have the correct state after restart. si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 500) // Now scale up to R3. cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) nl := c.randomNonStreamLeader("$G", "TEST") c.waitOnStreamCurrent(nl, "$G", "TEST") mset, err := nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() state := mset.store.State() mset.mu.RUnlock() require_True(t, state.Msgs == 500) } type captureCatchupWarnLogger struct { DummyLogger ch chan string } func (l *captureCatchupWarnLogger) Warnf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "simulate error") { select { case l.ch <- msg: default: } } } type catchupMockStore struct { StreamStore ch chan uint64 } func (s catchupMockStore) LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) { s.ch <- seq return s.StreamStore.LoadMsg(seq, sm) } func TestJetStreamClusterLeaderAbortsCatchupOnFollowerError(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, } _, err := js.AddStream(cfg) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") payload := string(make([]byte, 1024)) total := 100 for i := 0; i < total; i++ { sendStreamMsg(t, nc, "foo", payload) } c.waitOnAllCurrent() // Get the stream leader leader := c.streamLeader(globalAccountName, "TEST") mset, err := leader.GlobalAccount().lookupStream("TEST") require_NoError(t, err) var syncSubj string mset.mu.RLock() if mset.syncSub != nil { syncSubj = string(mset.syncSub.subject) } mset.mu.RUnlock() if syncSubj == _EMPTY_ { t.Fatal("Did not find the sync request subject") } // Setup the logger on the leader to make sure we capture the error and print // and also stop the runCatchup. l := &captureCatchupWarnLogger{ch: make(chan string, 10)} leader.SetLogger(l, false, false) // Set a fake message store that will allow us to verify // a few things. mset.mu.Lock() orgMS := mset.store ms := catchupMockStore{StreamStore: mset.store, ch: make(chan uint64)} mset.store = ms mset.mu.Unlock() // Need the system account to simulate the sync request that we are going to send. sysNC := natsConnect(t, c.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) defer sysNC.Close() // Setup a subscription to receive the messages sent by the leader. sub := natsSubSync(t, sysNC, nats.NewInbox()) req := &streamSyncRequest{ FirstSeq: 1, LastSeq: uint64(total), Peer: "bozo", // Should be one of the node name, but does not matter here } b, _ := json.Marshal(req) // Send the sync request and use our sub's subject for destination where leader // needs to send messages to. natsPubReq(t, sysNC, syncSubj, sub.Subject, b) // The mock store is blocked loading the first message, so we need to consume // the sequence before being able to receive the message in our sub. if seq := <-ms.ch; seq != 1 { t.Fatalf("Expected sequence to be 1, got %v", seq) } // Now consume and the leader should report the error and terminate runCatchup msg := natsNexMsg(t, sub, time.Second) msg.Respond([]byte("simulate error")) select { case <-l.ch: // OK case <-time.After(time.Second): t.Fatal("Did not get the expected error") } // The mock store should be blocked in seq==2 now, but after consuming, it should // abort the runCatchup. if seq := <-ms.ch; seq != 2 { t.Fatalf("Expected sequence to be 2, got %v", seq) } // We may have some more messages loaded as a race between when the sub will // indicate that the catchup should stop and the part where we send messages // in the batch, but we should likely not have sent all messages. loaded := 0 for done := false; !done; { select { case <-ms.ch: loaded++ case <-time.After(250 * time.Millisecond): done = true } } if loaded > 10 { t.Fatalf("Too many messages were sent after detecting remote is done: %v", loaded) } ch := make(chan string, 1) mset.mu.Lock() mset.store = orgMS leader.sysUnsubscribe(mset.syncSub) mset.syncSub = nil leader.systemSubscribe(syncSubj, _EMPTY_, false, mset.sysc, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) { var sreq streamSyncRequest if err := json.Unmarshal(msg, &sreq); err != nil { return } select { case ch <- reply: default: } }) mset.mu.Unlock() syncRepl := natsSubSync(t, sysNC, nats.NewInbox()+".>") // Make sure our sub is propagated time.Sleep(250 * time.Millisecond) if v := leader.gcbTotal(); v != 0 { t.Fatalf("Expected gcbTotal to be 0, got %v", v) } buf := make([]byte, 1_000_000) n := runtime.Stack(buf, true) if bytes.Contains(buf[:n], []byte("runCatchup")) { t.Fatalf("Looks like runCatchup is still running:\n%s", buf[:n]) } mset.mu.Lock() var state StreamState mset.store.FastState(&state) snapshot := &streamSnapshot{ Msgs: state.Msgs, Bytes: state.Bytes, FirstSeq: state.FirstSeq, LastSeq: state.LastSeq + 1, } b, _ = json.Marshal(snapshot) mset.node.SendSnapshot(b) mset.mu.Unlock() var sreqSubj string select { case sreqSubj = <-ch: case <-time.After(time.Second): t.Fatal("Did not receive sync request") } // Now send a message with a wrong sequence and expect to receive an error. em := encodeStreamMsg("foo", _EMPTY_, nil, []byte("fail"), 102, time.Now().UnixNano()) leader.sendInternalMsgLocked(sreqSubj, syncRepl.Subject, nil, em) msg = natsNexMsg(t, syncRepl, time.Second) if len(msg.Data) == 0 { t.Fatal("Expected err response from the remote") } } func TestJetStreamClusterStreamDirectGetNotTooSoon(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, _ := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "TEST", Storage: FileStorage, Subjects: []string{"foo"}, Replicas: 3, MaxMsgsPer: 1, AllowDirect: true, } addStream(t, nc, cfg) sendStreamMsg(t, nc, "foo", "bar") getSubj := fmt.Sprintf(JSDirectGetLastBySubjectT, "TEST", "foo") // Make sure we get all direct subs. checkForDirectSubs := func() { t.Helper() checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") if err != nil { return err } mset.mu.RLock() hasBoth := mset.directSub != nil && mset.lastBySub != nil mset.mu.RUnlock() if !hasBoth { return fmt.Errorf("%v does not have both direct subs registered", s) } } return nil }) } _, err := nc.Request(getSubj, nil, time.Second) require_NoError(t, err) checkForDirectSubs() // We want to make sure that when starting up we do not listen until we have a leader. nc.Close() c.stopAll() // Start just one.. s, opts := RunServerWithConfig(c.opts[0].ConfigFile) c.servers[0] = s c.opts[0] = opts nc, _ = jsClientConnect(t, s) defer nc.Close() _, err = nc.Request(getSubj, nil, time.Second) require_Error(t, err, nats.ErrNoResponders) // Now start all and make sure they all eventually have subs for direct access. c.restartAll() c.waitOnStreamLeader("$G", "TEST") _, err = nc.Request(getSubj, nil, time.Second) require_NoError(t, err) checkForDirectSubs() } func TestJetStreamClusterStaleReadsOnRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz"}, Replicas: 3, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) sl := c.streamLeader("$G", "TEST") r1 := c.randomNonStreamLeader("$G", "TEST") r1.Shutdown() nc.Close() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.Publish("bar", nil) require_NoError(t, err) _, err = js.Publish("baz", nil) require_NoError(t, err) r2 := c.randomNonStreamLeader("$G", "TEST") r2.Shutdown() sl.Shutdown() c.restartServer(r2) c.restartServer(r1) c.waitOnStreamLeader("$G", "TEST") nc.Close() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.Publish("foo", nil) require_NoError(t, err) _, err = js.Publish("bar", nil) require_NoError(t, err) _, err = js.Publish("baz", nil) require_NoError(t, err) c.restartServer(sl) c.waitOnAllCurrent() c.waitOnStreamLeader("$G", "TEST") // Grab expected from leader. var state StreamState sl = c.streamLeader("$G", "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.store.FastState(&state) checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { for _, s := range c.servers { if s.Running() { mset, err := s.GlobalAccount().lookupStream("TEST") if err != nil { return err } var fs StreamState mset.store.FastState(&fs) if !reflect.DeepEqual(fs, state) { return fmt.Errorf("States do not match, expected %+v but got %+v", state, fs) } } } return nil }) } func TestJetStreamClusterReplicasChangeStreamInfo(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() numStreams := 1 msgsPerStream := 10 for i := 0; i < numStreams; i++ { sname := fmt.Sprintf("TEST_%v", i) _, err := js.AddStream(&nats.StreamConfig{ Name: sname, Replicas: 3, }) require_NoError(t, err) for j := 0; j < msgsPerStream; j++ { sendStreamMsg(t, nc, sname, "msg") } } checkStreamInfo := func(js nats.JetStreamContext) { t.Helper() checkFor(t, 20*time.Second, 15*time.Millisecond, func() error { for i := 0; i < numStreams; i++ { si, err := js.StreamInfo(fmt.Sprintf("TEST_%v", i)) if err != nil { return err } if si.State.Msgs != uint64(msgsPerStream) || si.State.FirstSeq != 1 || si.State.LastSeq != uint64(msgsPerStream) { return fmt.Errorf("Invalid stream info for %s: %+v", si.Config.Name, si.State) } } return nil }) } checkStreamInfo(js) // Update replicas down to 1 for i := 0; i < numStreams; i++ { sname := fmt.Sprintf("TEST_%v", i) _, err := js.UpdateStream(&nats.StreamConfig{ Name: sname, Replicas: 1, }) require_NoError(t, err) } checkStreamInfo(js) // Back up to 3 for i := 0; i < numStreams; i++ { sname := fmt.Sprintf("TEST_%v", i) _, err := js.UpdateStream(&nats.StreamConfig{ Name: sname, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, sname) for _, s := range c.servers { c.waitOnStreamCurrent(s, globalAccountName, sname) } } checkStreamInfo(js) // Now shutdown the cluster and restart it nc.Close() c.stopAll() c.restartAll() for i := 0; i < numStreams; i++ { sname := fmt.Sprintf("TEST_%v", i) c.waitOnStreamLeader(globalAccountName, sname) for _, s := range c.servers { c.waitOnStreamCurrent(s, globalAccountName, sname) } } nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() checkStreamInfo(js) } func TestJetStreamClusterMaxOutstandingCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "MCB", 3) defer c.shutdown() for _, s := range c.servers { s.gcbMu.RLock() v := s.gcbOutMax s.gcbMu.RUnlock() if v != defaultMaxTotalCatchupOutBytes { t.Fatalf("Server %v, expected max_outstanding_catchup to be %v, got %v", s, defaultMaxTotalCatchupOutBytes, v) } } c.shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_outstanding_catchup: 1KB, domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf: { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` c = createJetStreamClusterWithTemplate(t, tmpl, "MCB", 3) defer c.shutdown() for _, s := range c.servers { s.gcbMu.RLock() v := s.gcbOutMax s.gcbMu.RUnlock() if v != 1024 { t.Fatalf("Server %v, expected max_outstanding_catchup to be 1KB, got %v", s, v) } } nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Close client now and will create new one nc.Close() c.waitOnStreamLeader(globalAccountName, "TEST") follower := c.randomNonStreamLeader(globalAccountName, "TEST") follower.Shutdown() c.waitOnStreamLeader(globalAccountName, "TEST") // Create new connection in case we would have been connected to follower. nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() payload := string(make([]byte, 2048)) for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "foo", payload) } // Cause snapshots on leader mset, err := c.streamLeader(globalAccountName, "TEST").GlobalAccount().lookupStream("TEST") require_NoError(t, err) err = mset.raftNode().InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) // Resart server and it should be able to catchup follower = c.restartServer(follower) c.waitOnStreamCurrent(follower, globalAccountName, "TEST") // Config reload not supported s := c.servers[0] cfile := s.getOpts().ConfigFile content, err := os.ReadFile(cfile) require_NoError(t, err) conf := string(content) conf = strings.ReplaceAll(conf, "max_outstanding_catchup: 1KB,", "max_outstanding_catchup: 1MB,") err = os.WriteFile(cfile, []byte(conf), 0644) require_NoError(t, err) err = s.Reload() require_Error(t, err, fmt.Errorf("config reload not supported for JetStreamMaxCatchup: old=1024, new=1048576")) } func TestJetStreamClusterCompressedStreamMessages(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // 32k (compress threshold ~4k) toSend, msg := 10_000, []byte(strings.Repeat("ABCD", 8*1024)) for i := 0; i < toSend; i++ { js.PublishAsync("foo", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } } // https://github.com/nats-io/nats-server/issues/5612 func TestJetStreamClusterWorkQueueLosingMessagesOnConsumerDelete(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"}, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) msg := []byte("test alskdjalksdjalskdjaksdjlaksjdlkajsdlakjsdlakjsdlakjdlakjsdlaksjdlj") for _, subj := range []string{"2", "5", "7", "9"} { for i := 0; i < 10; i++ { js.Publish(subj, msg) } } cfg := &nats.ConsumerConfig{ Name: "test", FilterSubjects: []string{"6", "7", "8", "9", "10"}, DeliverSubject: "bob", AckPolicy: nats.AckExplicitPolicy, AckWait: time.Minute, MaxAckPending: 1, } _, err = nc.SubscribeSync("bob") require_NoError(t, err) for i := 0; i < 5; i++ { _, err = js.AddConsumer("TEST", cfg) require_NoError(t, err) time.Sleep(time.Second) js.DeleteConsumer("TEST", "test") } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 40) } func TestJetStreamClusterR1ConsumerAdvisory(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.LimitsPolicy, Replicas: 3, }) require_NoError(t, err) sub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.CONSUMER.CREATED.>") require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "c1", AckPolicy: nats.AckExplicitPolicy, Replicas: 3, }) require_NoError(t, err) checkSubsPending(t, sub, 1) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "c2", AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_NoError(t, err) checkSubsPending(t, sub, 2) } // // DO NOT ADD NEW TESTS IN THIS FILE (unless to balance test times) // Add at the end of jetstream_cluster__test.go, with being the highest value. // nats-server-2.10.27/server/jetstream_cluster_3_test.go000066400000000000000000005215261477524627100230430ustar00rootroot00000000000000// Copyright 2022-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests_3 // +build !skip_js_tests,!skip_js_cluster_tests_3 package server import ( "bytes" "context" "encoding/json" "errors" "fmt" "math/rand" "net" "os" "path/filepath" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" ) func TestJetStreamClusterRemovePeerByID(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) require_NoError(t, err) // Wait for a leader c.waitOnStreamLeader(globalAccountName, "TEST") // Get the name of the one that is not restarted srvName := c.opts[2].ServerName // And its node ID peerID := c.servers[2].Node() nc.Close() // Now stop the whole cluster c.stopAll() // Restart all but one for i := 0; i < 2; i++ { opts := c.opts[i] s, o := RunServerWithConfig(opts.ConfigFile) c.servers[i] = s c.opts[i] = o } c.waitOnClusterReadyWithNumPeers(2) c.waitOnStreamLeader(globalAccountName, "TEST") // Now attempt to remove by name, this should fail because the cluster // was restarted and names are not persisted. ml := c.leader() nc, err = nats.Connect(ml.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() req := &JSApiMetaServerRemoveRequest{Server: srvName} jsreq, err := json.Marshal(req) require_NoError(t, err) rmsg, err := nc.Request(JSApiRemoveServer, jsreq, 2*time.Second) require_NoError(t, err) var resp JSApiMetaServerRemoveResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) require_True(t, resp.Error != nil) require_True(t, IsNatsErr(resp.Error, JSClusterServerNotMemberErr)) // Now try by ID, but first with an ID that does not match any peerID req.Peer = "some_bad_id" jsreq, err = json.Marshal(req) require_NoError(t, err) rmsg, err = nc.Request(JSApiRemoveServer, jsreq, 2*time.Second) require_NoError(t, err) resp = JSApiMetaServerRemoveResponse{} err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) require_True(t, resp.Error != nil) require_True(t, IsNatsErr(resp.Error, JSClusterServerNotMemberErr)) // Now with the proper peer ID req.Peer = peerID jsreq, err = json.Marshal(req) require_NoError(t, err) rmsg, err = nc.Request(JSApiRemoveServer, jsreq, 2*time.Second) require_NoError(t, err) resp = JSApiMetaServerRemoveResponse{} err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) require_True(t, resp.Error == nil) require_True(t, resp.Success) } func TestJetStreamClusterDiscardNewAndMaxMsgsPerSubject(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client for API requests. s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() for _, test := range []struct { name string storage StorageType replicas int }{ {"MEM-R1", MemoryStorage, 1}, {"FILE-R1", FileStorage, 1}, {"MEM-R3", MemoryStorage, 3}, {"FILE-R3", FileStorage, 3}, } { t.Run(test.name, func(t *testing.T) { js.DeleteStream("KV") // Make sure setting new without DiscardPolicy also being new is error. cfg := &StreamConfig{ Name: "KV", Subjects: []string{"KV.>"}, Storage: test.storage, AllowDirect: true, DiscardNewPer: true, MaxMsgs: 10, Replicas: test.replicas, } if _, apiErr := addStreamWithError(t, nc, cfg); apiErr == nil { t.Fatalf("Expected API error but got none") } else if apiErr.ErrCode != 10052 || !strings.Contains(apiErr.Description, "discard new per subject requires discard new policy") { t.Fatalf("Got wrong error: %+v", apiErr) } // Set broad discard new policy to engage DiscardNewPer cfg.Discard = DiscardNew // We should also error here since we have not setup max msgs per subject. if _, apiErr := addStreamWithError(t, nc, cfg); apiErr == nil { t.Fatalf("Expected API error but got none") } else if apiErr.ErrCode != 10052 || !strings.Contains(apiErr.Description, "discard new per subject requires max msgs per subject > 0") { t.Fatalf("Got wrong error: %+v", apiErr) } cfg.MaxMsgsPer = 1 addStream(t, nc, cfg) // We want to test that we reject new messages on a per subject basis if the // max msgs per subject limit has been hit, even if other limits have not. _, err := js.Publish("KV.foo", nil) require_NoError(t, err) _, err = js.Publish("KV.foo", nil) // Go client does not have const for this one. require_Error(t, err, errors.New("nats: maximum messages per subject exceeded")) }) } } func TestJetStreamClusterCreateConsumerWithReplicaOneGetsResponse(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C3", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "C3") // Update to scale down to R1, that should work (get a response) _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "C3", AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "C3") ci, err := js.ConsumerInfo("TEST", "C3") require_NoError(t, err) require_True(t, ci.Config.Replicas == 1) require_True(t, len(ci.Cluster.Replicas) == 0) } func TestJetStreamClusterMetaRecoveryLogic(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 1, }) require_NoError(t, err) err = js.DeleteStream("TEST") require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) err = js.DeleteStream("TEST") require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"baz"}, Replicas: 1, }) require_NoError(t, err) osi, err := js.StreamInfo("TEST") require_NoError(t, err) c.stopAll() c.restartAll() c.waitOnLeader() c.waitOnStreamLeader("$G", "TEST") s = c.randomNonLeader() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("TEST") require_NoError(t, err) if !reflect.DeepEqual(si.Config, osi.Config) { t.Fatalf("Expected %+v, but got %+v", osi.Config, si.Config) } } func TestJetStreamClusterDeleteConsumerWhileServerDown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomNonLeader()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "DC", AckPolicy: nats.AckExplicitPolicy, Replicas: 3, }) require_NoError(t, err) s := c.randomNonConsumerLeader("$G", "TEST", "DC") s.Shutdown() c.waitOnLeader() // In case that was metaleader. nc, js = jsClientConnect(t, c.randomNonLeader()) // In case we were connected there. defer nc.Close() err = js.DeleteConsumer("TEST", "DC") require_NoError(t, err) // Restart. s = c.restartServer(s) checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { hs := s.healthz(&HealthzOptions{ JSEnabledOnly: false, JSServerOnly: false, }) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) // Make sure we can not see it on the server that was down at the time of delete. mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) if o := mset.lookupConsumer("DC"); o != nil { t.Fatalf("Expected to not find consumer, but did") } // Now repeat but force a meta snapshot. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "DC", AckPolicy: nats.AckExplicitPolicy, Replicas: 3, }) require_NoError(t, err) s = c.randomNonConsumerLeader("$G", "TEST", "DC") s.Shutdown() c.waitOnLeader() // In case that was metaleader. nc, js = jsClientConnect(t, c.randomNonLeader()) // In case we were connected there. defer nc.Close() err = js.DeleteConsumer("TEST", "DC") require_NoError(t, err) err = c.leader().JetStreamSnapshotMeta() require_NoError(t, err) // Restart. s = c.restartServer(s) checkFor(t, time.Second*2, 200*time.Millisecond, func() error { hs := s.healthz(&HealthzOptions{ JSEnabledOnly: false, JSServerOnly: false, }) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) // Make sure we can not see it on the server that was down at the time of delete. mset, err = s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) if o := mset.lookupConsumer("DC"); o != nil { t.Fatalf("Expected to not find consumer, but did") } } func TestJetStreamClusterNegativeReplicas(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() testBadReplicas := func(t *testing.T, s *Server, name string) { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: name, Replicas: -1, }) require_Error(t, err, NewJSReplicasCountCannotBeNegativeError()) _, err = js.AddStream(&nats.StreamConfig{ Name: name, Replicas: 1, }) require_NoError(t, err) // Check update now. _, err = js.UpdateStream(&nats.StreamConfig{ Name: name, Replicas: -11, }) require_Error(t, err, NewJSReplicasCountCannotBeNegativeError()) // Now same for consumers durName := fmt.Sprintf("%s_dur", name) _, err = js.AddConsumer(name, &nats.ConsumerConfig{ Durable: durName, Replicas: -1, }) require_Error(t, err, NewJSReplicasCountCannotBeNegativeError()) _, err = js.AddConsumer(name, &nats.ConsumerConfig{ Durable: durName, Replicas: 1, }) require_NoError(t, err) // Check update now _, err = js.UpdateConsumer(name, &nats.ConsumerConfig{ Durable: durName, Replicas: -11, }) require_Error(t, err, NewJSReplicasCountCannotBeNegativeError()) } t.Run("Standalone", func(t *testing.T) { testBadReplicas(t, s, "TEST1") }) t.Run("Clustered", func(t *testing.T) { testBadReplicas(t, c.randomServer(), "TEST2") }) } func TestJetStreamClusterUserGivenConsName(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() test := func(t *testing.T, s *Server, stream string, replicas int, cons string) { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: stream, Replicas: replicas, }) require_NoError(t, err) cc := &CreateConsumerRequest{ Stream: stream, Config: ConsumerConfig{ Name: cons, FilterSubject: stream, InactiveThreshold: 10 * time.Second, }, } subj := fmt.Sprintf(JSApiConsumerCreateExT, stream, cons, stream) req, err := json.Marshal(cc) require_NoError(t, err) reply, err := nc.Request(subj, req, 2*time.Second) require_NoError(t, err) var cresp JSApiConsumerCreateResponse json.Unmarshal(reply.Data, &cresp) if cresp.Error != nil { t.Fatalf("Unexpected error: %v", cresp.Error) } require_Equal(t, cresp.Name, cons) require_Equal(t, cresp.Config.Name, cons) // Resend the add request but before change something that the server // should reject since the consumer already exist and we don't support // the update of the consumer that way. cc.Config.DeliverPolicy = DeliverNew req, err = json.Marshal(cc) require_NoError(t, err) reply, err = nc.Request(subj, req, 2*time.Second) require_NoError(t, err) cresp = JSApiConsumerCreateResponse{} json.Unmarshal(reply.Data, &cresp) require_Error(t, cresp.Error, NewJSConsumerCreateError(errors.New("deliver policy can not be updated"))) } t.Run("Standalone", func(t *testing.T) { test(t, s, "TEST", 1, "cons") }) t.Run("Clustered R1", func(t *testing.T) { test(t, c.randomServer(), "TEST2", 1, "cons2") }) t.Run("Clustered R3", func(t *testing.T) { test(t, c.randomServer(), "TEST3", 3, "cons3") }) } func TestJetStreamClusterUserGivenConsNameWithLeaderChange(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "msg") } consName := "myephemeral" cc := &CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Name: consName, FilterSubject: "foo", InactiveThreshold: time.Hour, Replicas: 3, }, } subj := fmt.Sprintf(JSApiConsumerCreateExT, "TEST", consName, "foo") req, err := json.Marshal(cc) require_NoError(t, err) reply, err := nc.Request(subj, req, 2*time.Second) require_NoError(t, err) var cresp JSApiConsumerCreateResponse json.Unmarshal(reply.Data, &cresp) if cresp.Error != nil { t.Fatalf("Unexpected error: %v", cresp.Error) } require_Equal(t, cresp.Name, consName) require_Equal(t, cresp.Config.Name, consName) // Consumer leader name clname := cresp.ConsumerInfo.Cluster.Leader nreq := &JSApiConsumerGetNextRequest{Batch: 1, Expires: time.Second} req, err = json.Marshal(nreq) require_NoError(t, err) sub := natsSubSync(t, nc, "xxx") rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", consName) err = nc.PublishRequest(rsubj, "xxx", req) require_NoError(t, err) msg := natsNexMsg(t, sub, time.Second) require_Equal(t, string(msg.Data), "msg") // Shutdown the consumer leader cl := c.serverByName(clname) cl.Shutdown() // Wait for a bit to be sure that we lost leadership time.Sleep(250 * time.Millisecond) // Wait for new leader c.waitOnStreamLeader(globalAccountName, "TEST") c.waitOnConsumerLeader(globalAccountName, "TEST", consName) // Make sure we can still consume. for i := 0; i < 2; i++ { err = nc.PublishRequest(rsubj, "xxx", req) require_NoError(t, err) msg = natsNexMsg(t, sub, time.Second) if len(msg.Data) == 0 { continue } require_Equal(t, string(msg.Data), "msg") return } t.Fatal("Did not receive message") } func TestJetStreamClusterMirrorCrossDomainOnLeadnodeNoSystemShare(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CORE", _EMPTY_, 3, 18033, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, "store_dir:", "domain: SPOKE, store_dir:", 1) ln := c.createLeafNodeWithTemplateNoSystem("LN-SPOKE", tmpl) defer ln.Shutdown() checkLeafNodeConnectedCount(t, ln, 1) // Create origin stream in hub. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPerSubject: 10, AllowDirect: true, }) require_NoError(t, err) // Now create the mirror on the leafnode. lnc, ljs := jsClientConnect(t, ln) defer lnc.Close() _, err = ljs.AddStream(&nats.StreamConfig{ Name: "M", MaxMsgsPerSubject: 10, AllowDirect: true, MirrorDirect: true, Mirror: &nats.StreamSource{ Name: "TEST", External: &nats.ExternalStream{ APIPrefix: "$JS.HUB.API", }, }, }) require_NoError(t, err) // Publish to the hub stream and make sure the mirror gets those messages. for i := 0; i < 20; i++ { js.Publish("foo", nil) } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 10) checkFor(t, time.Second, 200*time.Millisecond, func() error { si, err := ljs.StreamInfo("M") require_NoError(t, err) if si.State.Msgs == 10 { return nil } return fmt.Errorf("State not current: %+v", si.State) }) } func TestJetStreamClusterFirstSeqMismatch(t *testing.T) { c := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, "C", 3, func(serverName, clusterName, storeDir, conf string) string { tf := createTempFile(t, "") logName := tf.Name() tf.Close() return fmt.Sprintf("%s\nlogfile: '%s'", conf, logName) }) defer c.shutdown() rs := c.randomServer() nc, js := jsClientConnect(t, rs) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, MaxAge: 2 * time.Second, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") mset, err := c.streamLeader(globalAccountName, "TEST").GlobalAccount().lookupStream("TEST") require_NoError(t, err) node := mset.raftNode() nl := c.randomNonStreamLeader(globalAccountName, "TEST") if rs == nl { nc.Close() for _, s := range c.servers { if s != nl { nc, _ = jsClientConnect(t, s) defer nc.Close() break } } } wg := sync.WaitGroup{} wg.Add(1) ch := make(chan struct{}) go func() { defer wg.Done() for i := 0; ; i++ { sendStreamMsg(t, nc, "foo", "msg") select { case <-ch: return default: } } }() time.Sleep(2500 * time.Millisecond) nl.Shutdown() time.Sleep(500 * time.Millisecond) node.InstallSnapshot(mset.stateSnapshot()) time.Sleep(3500 * time.Millisecond) c.restartServer(nl) c.waitOnAllCurrent() close(ch) wg.Wait() log := nl.getOpts().LogFile nl.Shutdown() content, err := os.ReadFile(log) require_NoError(t, err) if bytes.Contains(content, []byte(errFirstSequenceMismatch.Error())) { t.Fatalf("First sequence mismatch occurred!") } } func TestJetStreamClusterConsumerInactiveThreshold(t *testing.T) { // Create a standalone, a cluster, and a super cluster s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() test := func(t *testing.T, c *cluster, s *Server, replicas int) { if c != nil { s = c.randomServer() } nc, js := jsClientConnect(t, s) defer nc.Close() sname := fmt.Sprintf("TEST%d", replicas) _, err := js.AddStream(&nats.StreamConfig{ Name: sname, Subjects: []string{sname}, Replicas: replicas, }) require_NoError(t, err) if c != nil { c.waitOnStreamLeader(globalAccountName, sname) } for i := 0; i < 10; i++ { js.PublishAsync(sname, []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } waitOnCleanup := func(ci *nats.ConsumerInfo) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { _, err := js.ConsumerInfo(ci.Stream, ci.Name) if err == nil { return fmt.Errorf("Consumer still present") } return nil }) } // Test to make sure inactive threshold is enforced for all types. // Ephemeral and Durable, both push and pull. // Ephemeral Push (no bind to deliver subject) ci, err := js.AddConsumer(sname, &nats.ConsumerConfig{ DeliverSubject: "_no_bind_", InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) // Ephemeral Pull ci, err = js.AddConsumer(sname, &nats.ConsumerConfig{ AckPolicy: nats.AckExplicitPolicy, InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) // Support InactiveThresholds for Durables as well. // Durable Push (no bind to deliver subject) ci, err = js.AddConsumer(sname, &nats.ConsumerConfig{ Durable: "d1", DeliverSubject: "_no_bind_", InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) // Durable Push (no bind to deliver subject) with an activity // threshold set after creation ci, err = js.AddConsumer(sname, &nats.ConsumerConfig{ Durable: "d2", DeliverSubject: "_no_bind_", }) require_NoError(t, err) if c != nil { c.waitOnConsumerLeader(globalAccountName, sname, "d2") } _, err = js.UpdateConsumer(sname, &nats.ConsumerConfig{ Durable: "d2", DeliverSubject: "_no_bind_", InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) // Durable Pull ci, err = js.AddConsumer(sname, &nats.ConsumerConfig{ Durable: "d3", AckPolicy: nats.AckExplicitPolicy, InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) // Durable Pull with an inactivity threshold set after creation ci, err = js.AddConsumer(sname, &nats.ConsumerConfig{ Durable: "d4", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) if c != nil { c.waitOnConsumerLeader(globalAccountName, sname, "d4") } _, err = js.UpdateConsumer(sname, &nats.ConsumerConfig{ Durable: "d4", AckPolicy: nats.AckExplicitPolicy, InactiveThreshold: 50 * time.Millisecond, }) require_NoError(t, err) waitOnCleanup(ci) } t.Run("standalone", func(t *testing.T) { test(t, nil, s, 1) }) t.Run("cluster-r1", func(t *testing.T) { test(t, c, nil, 1) }) t.Run("cluster-r3", func(t *testing.T) { test(t, c, nil, 3) }) t.Run("super-cluster-r1", func(t *testing.T) { test(t, sc.randomCluster(), nil, 1) }) t.Run("super-cluster-r3", func(t *testing.T) { test(t, sc.randomCluster(), nil, 3) }) } // To capture our false warnings for clustered stream lag. type testStreamLagWarnLogger struct { DummyLogger ch chan string } func (l *testStreamLagWarnLogger) Warnf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "has high message lag") { select { case l.ch <- msg: default: } } } // False triggering warnings on stream lag because not offsetting by failures. func TestJetStreamClusterStreamLagWarning(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) sl := c.streamLeader("$G", "TEST") l := &testStreamLagWarnLogger{ch: make(chan string, 10)} sl.SetLogger(l, false, false) // We only need to trigger post RAFT propose failures that increment mset.clfs. // Dedupe with msgIDs is one, so we will use that. m := nats.NewMsg("foo") m.Data = []byte("OK") m.Header.Set(JSMsgId, "zz") // Make sure we know we will trip the warning threshold. for i := 0; i < 2*streamLagWarnThreshold; i++ { js.PublishMsgAsync(m) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } select { case msg := <-l.ch: t.Fatalf("Unexpected msg lag warning seen: %s", msg) case <-time.After(100 * time.Millisecond): // OK } } // https://github.com/nats-io/nats-server/issues/3603 func TestJetStreamClusterSignalPullConsumersOnDelete(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Create 2 pull consumers. sub1, err := js.PullSubscribe("foo", "d1") require_NoError(t, err) sub2, err := js.PullSubscribe("foo", "d2") require_NoError(t, err) // We want to make sure we get kicked out prior to the timeout // when consumers are being deleted or the parent stream is being deleted. // Note this should be lower case, Go client needs to be updated. expectedErr := errors.New("nats: consumer deleted") // Queue up the delete for sub1 time.AfterFunc(250*time.Millisecond, func() { js.DeleteConsumer("TEST", "d1") }) start := time.Now() _, err = sub1.Fetch(1, nats.MaxWait(10*time.Second)) require_Error(t, err, expectedErr) // Check that we bailed early. if time.Since(start) > time.Second { t.Fatalf("Took to long to bail out on consumer delete") } time.AfterFunc(250*time.Millisecond, func() { js.DeleteStream("TEST") }) start = time.Now() _, err = sub2.Fetch(1, nats.MaxWait(10*time.Second)) require_Error(t, err, expectedErr) if time.Since(start) > time.Second { t.Fatalf("Took to long to bail out on stream delete") } } // https://github.com/nats-io/nats-server/issues/3559 func TestJetStreamClusterSourceWithOptStartTime(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() test := func(t *testing.T, c *cluster, s *Server) { replicas := 1 if c != nil { s = c.randomServer() replicas = 3 } nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: replicas, }) require_NoError(t, err) yesterday := time.Now().Add(-24 * time.Hour) _, err = js.AddStream(&nats.StreamConfig{ Name: "SOURCE", Replicas: replicas, Sources: []*nats.StreamSource{{ Name: "TEST", OptStartTime: &yesterday, }}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR", Replicas: replicas, Mirror: &nats.StreamSource{ Name: "TEST", OptStartTime: &yesterday, }, }) require_NoError(t, err) total := 10 for i := 0; i < total; i++ { sendStreamMsg(t, nc, "foo", "hello") } checkCount := func(sname string, expected int) { t.Helper() checkFor(t, 10*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo(sname) if err != nil { return err } if n := si.State.Msgs; n != uint64(expected) { return fmt.Errorf("Expected stream %q to have %v messages, got %v", sname, expected, n) } return nil }) } checkCount("TEST", 10) checkCount("SOURCE", 10) checkCount("MIRROR", 10) err = js.PurgeStream("SOURCE") require_NoError(t, err) err = js.PurgeStream("MIRROR") require_NoError(t, err) checkCount("TEST", 10) checkCount("SOURCE", 0) checkCount("MIRROR", 0) nc.Close() if c != nil { c.stopAll() c.restartAll() c.waitOnStreamLeader(globalAccountName, "TEST") c.waitOnStreamLeader(globalAccountName, "SOURCE") c.waitOnStreamLeader(globalAccountName, "MIRROR") s = c.randomServer() } else { sd := s.JetStreamConfig().StoreDir s.Shutdown() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() } // Wait a bit before checking because sync'ing (even with the defect) // would not happen right away. I tried with 1 sec and test would pass, // so need to be at least that much. time.Sleep(2 * time.Second) nc, js = jsClientConnect(t, s) defer nc.Close() checkCount("TEST", 10) checkCount("SOURCE", 0) checkCount("MIRROR", 0) } t.Run("standalone", func(t *testing.T) { test(t, nil, s) }) t.Run("cluster", func(t *testing.T) { test(t, c, nil) }) } type networkCableUnplugged struct { net.Conn sync.Mutex unplugged bool wb bytes.Buffer wg sync.WaitGroup } func (c *networkCableUnplugged) Write(b []byte) (int, error) { c.Lock() if c.unplugged { c.wb.Write(b) c.Unlock() return len(b), nil } else if c.wb.Len() > 0 { c.wb.Write(b) buf := c.wb.Bytes() c.wb.Reset() c.Unlock() if _, err := c.Conn.Write(buf); err != nil { return 0, err } return len(b), nil } c.Unlock() return c.Conn.Write(b) } func (c *networkCableUnplugged) Read(b []byte) (int, error) { c.Lock() wait := c.unplugged c.Unlock() if wait { c.wg.Wait() } return c.Conn.Read(b) } func TestJetStreamClusterScaleDownWhileNoQuorum(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) require_NoError(t, err) for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "foo", "msg") } // Let's have a server from this R2 stream be network partitionned. // We will take the leader, but doesn't have to be. // To simulate partition, we will replace all its routes with a // special connection that drops messages. sl := c.serverByName(si.Cluster.Leader) if s == sl { nc.Close() for s = c.randomServer(); s != sl; s = c.randomServer() { } nc, js = jsClientConnect(t, s) defer nc.Close() } sl.mu.Lock() sl.forEachRoute(func(r *client) { r.mu.Lock() ncu := &networkCableUnplugged{Conn: r.nc, unplugged: true} ncu.wg.Add(1) r.nc = ncu r.mu.Unlock() }) sl.mu.Unlock() // Wait for the stream info to fail checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return err } if si.Cluster.Leader == _EMPTY_ { return nil } return fmt.Errorf("stream still has a leader") }) // Make sure if meta leader was on same server as stream leader we make sure // it elects new leader to receive update request. c.waitOnLeader() // Now try to edit the stream by making it an R1. In some case we get // a context deadline error, in some no error. So don't check the returned error. js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }, nats.MaxWait(5*time.Second)) sl.mu.Lock() sl.forEachRoute(func(r *client) { r.mu.Lock() ncu := r.nc.(*networkCableUnplugged) ncu.Lock() ncu.unplugged = false ncu.wg.Done() ncu.Unlock() r.mu.Unlock() }) sl.mu.Unlock() checkClusterFormed(t, c.servers...) c.waitOnStreamLeader(globalAccountName, "TEST") } // We noticed that ha_assets enforcement seemed to not be upheld when assets created in a rapid fashion. func TestJetStreamClusterHAssetsEnforcement(t *testing.T) { tmpl := strings.Replace(jsClusterTempl, "store_dir:", "limits: {max_ha_assets: 2}, store_dir:", 1) c := createJetStreamClusterWithTemplateAndModHook(t, tmpl, "R3S", 3, nil) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST-1", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST-2", Subjects: []string{"bar"}, Replicas: 3, }) require_NoError(t, err) exceededErrs := []error{errors.New("system limit reached"), errors.New("no suitable peers")} // Should fail. _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST-3", Subjects: []string{"baz"}, Replicas: 3, }) require_Error(t, err, exceededErrs...) } func TestJetStreamClusterInterestStreamConsumer(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.InterestPolicy, Replicas: 3, }) require_NoError(t, err) var subs []*nats.Subscription ns := 5 for i := 0; i < ns; i++ { dn := fmt.Sprintf("d%d", i) sub, err := js.PullSubscribe("foo", dn) require_NoError(t, err) subs = append(subs, sub) } // Send 10 msgs n := 10 for i := 0; i < n; i++ { sendStreamMsg(t, nc, "foo", "msg") } // Collect all the messages. var msgs []*nats.Msg for _, sub := range subs { lmsgs := fetchMsgs(t, sub, n, time.Second) if len(lmsgs) != n { t.Fatalf("Did not receive all msgs: %d vs %d", len(lmsgs), n) } msgs = append(msgs, lmsgs...) } // Shuffle rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) for _, m := range msgs { m.AckSync() } // Make sure replicated acks are processed. time.Sleep(250 * time.Millisecond) si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 0 { t.Fatalf("Should not have any messages left: %d of %d", si.State.Msgs, n) } } func TestJetStreamClusterNoPanicOnStreamInfoWhenNoLeaderYet(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc := natsConnect(t, c.randomServer().ClientURL()) defer nc.Close() js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond)) wg := sync.WaitGroup{} wg.Add(1) ch := make(chan struct{}) go func() { defer wg.Done() for { js.StreamInfo("TEST") select { case <-ch: return case <-time.After(15 * time.Millisecond): } } }() time.Sleep(250 * time.Millisecond) // Don't care if this succeeds or not (could get a context deadline // due to the low MaxWait() when creating the context). js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) close(ch) wg.Wait() } func TestJetStreamClusterNoTimeoutOnStreamInfoOnPreferredLeader(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.StreamInfo("TEST") require_NoError(t, err) // Simulate the preferred stream leader to not have initialized the raft node yet. sl := c.streamLeader(globalAccountName, "TEST") acc, err := sl.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) sjs := sl.getJetStream() rg := mset.raftGroup() sjs.mu.Lock() rg.node = nil sjs.mu.Unlock() // Should not time out on the stream info during this condition. _, err = js.StreamInfo("TEST") require_NoError(t, err) } // Issue https://github.com/nats-io/nats-server/issues/3630 func TestJetStreamClusterPullConsumerAcksExtendInactivityThreshold(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) n := 10 for i := 0; i < n; i++ { sendStreamMsg(t, nc, "foo", "msg") } // Pull Consumer sub, err := js.PullSubscribe("foo", "d", nats.InactiveThreshold(time.Second)) require_NoError(t, err) fetchMsgs(t, sub, n/2, time.Second) // Will wait for .5s. time.Sleep(500 * time.Millisecond) msgs := fetchMsgs(t, sub, n/2, time.Second) if len(msgs) != n/2 { t.Fatalf("Did not receive msgs: %d vs %d", len(msgs), n/2) } // Wait for .5s. time.Sleep(500 * time.Millisecond) msgs[0].Ack() // Ack // Wait another .5s. time.Sleep(500 * time.Millisecond) msgs[1].Nak() // Nak // Wait another .5s. time.Sleep(500 * time.Millisecond) msgs[2].Term() // Term time.Sleep(500 * time.Millisecond) msgs[3].InProgress() // WIP // The above should have kept the consumer alive. _, err = js.ConsumerInfo("TEST", "d") require_NoError(t, err) // Make sure it gets cleaned up. time.Sleep(2 * time.Second) _, err = js.ConsumerInfo("TEST", "d") require_Error(t, err, nats.ErrConsumerNotFound) } // https://github.com/nats-io/nats-server/issues/3677 func TestJetStreamClusterParallelStreamCreation(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() np := 100 startCh := make(chan bool) errCh := make(chan error, np) wg := sync.WaitGroup{} wg.Add(np) start := sync.WaitGroup{} start.Add(np) for i := 0; i < np; i++ { go func() { defer wg.Done() // Individual connection nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Signal we are ready start.Done() // Make them all fire at once. <-startCh if _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"common.*.*"}, Replicas: 3, }); err != nil { errCh <- err } }() } start.Wait() close(startCh) wg.Wait() if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } // We had a bug during parallel stream creation as well that would overwrite the sync subject used for catchups, etc. // Test that here as well by shutting down a non-leader, adding a whole bunch of messages, and making sure on restart // we properly recover. nl := c.randomNonStreamLeader(globalAccountName, "TEST") nl.Shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() msg := bytes.Repeat([]byte("Z"), 128) for i := 0; i < 100; i++ { js.PublishAsync("common.foo.bar", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // We need to force the leader to do a snapshot so we kick in upper layer catchup which depends on syncSubject. sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) node := mset.raftNode() require_NotNil(t, node) node.InstallSnapshot(mset.stateSnapshot()) nl = c.restartServer(nl) c.waitOnServerCurrent(nl) mset, err = nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) // Check state directly. mset.mu.Lock() var state StreamState mset.store.FastState(&state) mset.mu.Unlock() require_Equal(t, state.Msgs, 100) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 100) } // In addition to test above, if streams were attempted to be created in parallel // it could be that multiple raft groups would be created for the same asset. func TestJetStreamClusterParallelStreamCreationDupeRaftGroups(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() np := 20 startCh := make(chan bool) wg := sync.WaitGroup{} wg.Add(np) for i := 0; i < np; i++ { go func() { defer wg.Done() // Individual connection nc, _ := jsClientConnect(t, c.randomServer()) js, _ := nc.JetStream(nats.MaxWait(time.Second)) defer nc.Close() // Make them all fire at once. <-startCh // Ignore errors in this test, care about raft group and metastate. js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"common.*.*"}, Replicas: 3, }) }() } close(startCh) wg.Wait() // Restart a server too. s := c.randomServer() s.Shutdown() s = c.restartServer(s) c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "TEST") // Check that this server has only two active raft nodes after restart. if nrn := s.numRaftNodes(); nrn != 2 { t.Fatalf("Expected only two active raft nodes, got %d", nrn) } // Make sure we only have 2 unique raft groups for all servers. // One for meta, one for stream. expected := 2 rg := make(map[string]struct{}) for _, s := range c.servers { s.rnMu.RLock() for _, ni := range s.raftNodes { n := ni.(*raft) rg[n.Group()] = struct{}{} } s.rnMu.RUnlock() } if len(rg) != expected { t.Fatalf("Expected only %d distinct raft groups for all servers, go %d", expected, len(rg)) } } func TestJetStreamClusterParallelConsumerCreation(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"common.*.*"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") np := 50 startCh := make(chan bool) errCh := make(chan error, np) cfg := &nats.ConsumerConfig{ Durable: "dlc", Replicas: 3, } wg := sync.WaitGroup{} swg := sync.WaitGroup{} wg.Add(np) swg.Add(np) for i := 0; i < np; i++ { go func() { defer wg.Done() // Individual connection nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() swg.Done() // Make them all fire at once. <-startCh if _, err := js.AddConsumer("TEST", cfg); err != nil { errCh <- err } }() } swg.Wait() close(startCh) wg.Wait() if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } // Make sure we only have 3 unique raft groups for all servers. // One for meta, one for stream, one for consumer. expected := 3 rg := make(map[string]struct{}) for _, s := range c.servers { s.rnMu.RLock() for _, ni := range s.raftNodes { n := ni.(*raft) rg[n.Group()] = struct{}{} } s.rnMu.RUnlock() } if len(rg) != expected { t.Fatalf("Expected only %d distinct raft groups for all servers, go %d", expected, len(rg)) } } func TestJetStreamClusterGhostEphemeralsAfterRestart(t *testing.T) { consumerNotActiveStartInterval = time.Second consumerNotActiveMaxInterval = time.Second t.Cleanup(func() { consumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval consumerNotActiveMaxInterval = defaultConsumerNotActiveMaxInterval }) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Add in 100 memory based ephemerals. for i := 0; i < 100; i++ { _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Replicas: 1, InactiveThreshold: time.Second, MemoryStorage: true, }) require_NoError(t, err) } // Grab random server. rs := c.randomServer() // Now shutdown cluster. c.stopAll() // Let the consumers all expire. time.Sleep(2 * time.Second) // Restart first and wait so that we know it will try cleanup without a metaleader. // It will fail as there's no metaleader at that time, it should keep retrying on an interval. c.restartServer(rs) time.Sleep(time.Second) c.restartAll() c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "TEST") nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() subj := fmt.Sprintf(JSApiConsumerListT, "TEST") checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { // Request will take at most 4 seconds if some consumers can't be found. m, err := nc.Request(subj, nil, 5*time.Second) if err != nil { return err } var resp JSApiConsumerListResponse err = json.Unmarshal(m.Data, &resp) require_NoError(t, err) if len(resp.Consumers) != 0 { return fmt.Errorf("Still have %d consumers", len(resp.Consumers)) } if len(resp.Missing) != 0 { return fmt.Errorf("Still have %d missing consumers", len(resp.Missing)) } return nil }) } func TestJetStreamClusterReplacementPolicyAfterPeerRemove(t *testing.T) { // R3 scenario where there is a redundant node in each unique cloud so removing a peer should result in // an immediate replacement also preserving cloud uniqueness. sc := createJetStreamClusterExplicit(t, "PR9", 9) sc.waitOnPeerCount(9) reset := func(s *Server) { s.mu.Lock() rch := s.sys.resetCh s.mu.Unlock() if rch != nil { rch <- struct{}{} } s.sendStatszUpdate() } tags := []string{"cloud:aws", "cloud:aws", "cloud:aws", "cloud:gcp", "cloud:gcp", "cloud:gcp", "cloud:az", "cloud:az", "cloud:az"} var serverUTags = make(map[string]string) for i, s := range sc.servers { s.optsMu.Lock() serverUTags[s.Name()] = tags[i] s.opts.Tags.Add(tags[i]) s.opts.JetStreamUniqueTag = "cloud" s.optsMu.Unlock() reset(s) } ml := sc.leader() js := ml.getJetStream() require_True(t, js != nil) js.mu.RLock() cc := js.cluster require_True(t, cc != nil) // Walk and make sure all tags are registered. expires := time.Now().Add(10 * time.Second) for time.Now().Before(expires) { allOK := true for _, p := range cc.meta.Peers() { si, ok := ml.nodeToInfo.Load(p.ID) require_True(t, ok) ni := si.(nodeInfo) if len(ni.tags) == 0 { allOK = false reset(sc.serverByName(ni.name)) } } if allOK { break } } js.mu.RUnlock() defer sc.shutdown() sc.waitOnClusterReadyWithNumPeers(9) s := sc.leader() nc, jsc := jsClientConnect(t, s) defer nc.Close() _, err := jsc.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) sc.waitOnStreamLeader(globalAccountName, "TEST") osi, err := jsc.StreamInfo("TEST") require_NoError(t, err) // Double check original placement honors unique_tag var uTags = make(map[string]struct{}) uTags[serverUTags[osi.Cluster.Leader]] = struct{}{} for _, replica := range osi.Cluster.Replicas { evalTag := serverUTags[replica.Name] if _, exists := uTags[evalTag]; !exists { uTags[evalTag] = struct{}{} continue } else { t.Fatalf("expected initial placement to honor unique_tag") } } // Remove a peer and select replacement 5 times to avoid false good for i := 0; i < 5; i++ { // Remove 1 peer replica (this will be random cloud region as initial placement was randomized ordering) // After each successful iteration, osi will reflect the current RG peers toRemove := osi.Cluster.Replicas[0].Name resp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "TEST"), []byte(`{"peer":"`+toRemove+`"}`), time.Second) require_NoError(t, err) var rpResp JSApiStreamRemovePeerResponse err = json.Unmarshal(resp.Data, &rpResp) require_NoError(t, err) require_True(t, rpResp.Success) sc.waitOnStreamLeader(globalAccountName, "TEST") checkFor(t, time.Second, 200*time.Millisecond, func() error { osi, err = jsc.StreamInfo("TEST") require_NoError(t, err) if len(osi.Cluster.Replicas) != 2 { return fmt.Errorf("expected R3, got R%d", len(osi.Cluster.Replicas)+1) } // STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred by // checking that the toRemove peer is gone. for _, replica := range osi.Cluster.Replicas { if replica.Name == toRemove { return fmt.Errorf("expected replaced replica, old replica still present") } } return nil }) // Validate that replacement with new peer still honors uTags = make(map[string]struct{}) //reset uTags[serverUTags[osi.Cluster.Leader]] = struct{}{} for _, replica := range osi.Cluster.Replicas { evalTag := serverUTags[replica.Name] if _, exists := uTags[evalTag]; !exists { uTags[evalTag] = struct{}{} continue } else { t.Fatalf("expected new peer and revised placement to honor unique_tag") } } } } func TestJetStreamClusterReplacementPolicyAfterPeerRemoveNoPlace(t *testing.T) { // R3 scenario where there are exactly three unique cloud nodes, so removing a peer should NOT // result in a new peer sc := createJetStreamClusterExplicit(t, "threeup", 3) sc.waitOnPeerCount(3) reset := func(s *Server) { s.mu.Lock() rch := s.sys.resetCh s.mu.Unlock() if rch != nil { rch <- struct{}{} } s.sendStatszUpdate() } tags := []string{"cloud:aws", "cloud:gcp", "cloud:az"} var serverUTags = make(map[string]string) for i, s := range sc.servers { s.optsMu.Lock() serverUTags[s.Name()] = tags[i] s.opts.Tags.Add(tags[i]) s.opts.JetStreamUniqueTag = "cloud" s.optsMu.Unlock() reset(s) } ml := sc.leader() js := ml.getJetStream() require_True(t, js != nil) js.mu.RLock() cc := js.cluster require_True(t, cc != nil) // Walk and make sure all tags are registered. expires := time.Now().Add(10 * time.Second) for time.Now().Before(expires) { allOK := true for _, p := range cc.meta.Peers() { si, ok := ml.nodeToInfo.Load(p.ID) require_True(t, ok) ni := si.(nodeInfo) if len(ni.tags) == 0 { allOK = false reset(sc.serverByName(ni.name)) } } if allOK { break } } js.mu.RUnlock() defer sc.shutdown() sc.waitOnClusterReadyWithNumPeers(3) s := sc.leader() nc, jsc := jsClientConnect(t, s) defer nc.Close() _, err := jsc.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) sc.waitOnStreamLeader(globalAccountName, "TEST") osi, err := jsc.StreamInfo("TEST") require_NoError(t, err) // Double check original placement honors unique_tag var uTags = make(map[string]struct{}) uTags[serverUTags[osi.Cluster.Leader]] = struct{}{} for _, replica := range osi.Cluster.Replicas { evalTag := serverUTags[replica.Name] if _, exists := uTags[evalTag]; !exists { uTags[evalTag] = struct{}{} continue } else { t.Fatalf("expected initial placement to honor unique_tag") } } // Remove 1 peer replica (this will be random cloud region as initial placement was randomized ordering) _, err = nc.Request("$JS.API.STREAM.PEER.REMOVE.TEST", []byte(`{"peer":"`+osi.Cluster.Replicas[0].Name+`"}`), time.Second*10) require_NoError(t, err) sc.waitOnStreamLeader(globalAccountName, "TEST") // Verify R2 since no eligible peer can replace the removed peer without braking unique constraint checkFor(t, time.Second, 200*time.Millisecond, func() error { osi, err = jsc.StreamInfo("TEST") require_NoError(t, err) if len(osi.Cluster.Replicas) != 1 { return fmt.Errorf("expected R2, got R%d", len(osi.Cluster.Replicas)+1) } return nil }) // Validate that remaining members still honor unique tags uTags = make(map[string]struct{}) //reset uTags[serverUTags[osi.Cluster.Leader]] = struct{}{} for _, replica := range osi.Cluster.Replicas { evalTag := serverUTags[replica.Name] if _, exists := uTags[evalTag]; !exists { uTags[evalTag] = struct{}{} continue } else { t.Fatalf("expected revised placement to honor unique_tag") } } } // https://github.com/nats-io/nats-server/issues/3191 func TestJetStreamClusterLeafnodeDuplicateConsumerMessages(t *testing.T) { // Cluster B c := createJetStreamCluster(t, jsClusterTempl, "B", _EMPTY_, 2, 22020, false) defer c.shutdown() // Cluster A // Domain is "A' lc := c.createLeafNodesWithStartPortAndDomain("A", 2, 22110, "A") defer lc.shutdown() lc.waitOnClusterReady() // We want A-S-1 connected to B-S-1 and A-S-2 connected to B-S-2 // So adjust if needed. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for i, ls := range lc.servers { ls.mu.RLock() var remoteServer string for _, rc := range ls.leafs { rc.mu.Lock() remoteServer = rc.leaf.remoteServer rc.mu.Unlock() break } ls.mu.RUnlock() wantedRemote := fmt.Sprintf("S-%d", i+1) if remoteServer != wantedRemote { ls.Shutdown() lc.restartServer(ls) return fmt.Errorf("Leafnode server %d not connected to %q", i+1, wantedRemote) } } return nil }) // Wait on ready again. lc.waitOnClusterReady() // Create a stream and a durable pull consumer on cluster A. lnc, ljs := jsClientConnect(t, lc.randomServer()) defer lnc.Close() _, err := ljs.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, }) require_NoError(t, err) // Make sure stream leader is on S-1 checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := ljs.StreamInfo("TEST") require_NoError(t, err) if si.Cluster.Leader == "A-S-1" { return nil } _, err = lnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) return fmt.Errorf("Stream leader not placed on A-S-1") }) _, err = ljs.StreamInfo("TEST") require_NoError(t, err) _, err = ljs.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", Replicas: 2, MaxDeliver: 1, AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) // Make sure consumer leader is on S-2 checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := ljs.ConsumerInfo("TEST", "dlc") require_NoError(t, err) if ci.Cluster.Leader == "A-S-2" { return nil } _, err = lnc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "dlc"), nil, time.Second) require_NoError(t, err) return fmt.Errorf("Stream leader not placed on A-S-1") }) _, err = ljs.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Send 2 messages. sendStreamMsg(t, lnc, "foo", "M-1") sendStreamMsg(t, lnc, "foo", "M-2") // Now bind apps to cluster B servers and bind to pull consumer. nc1, _ := jsClientConnect(t, c.servers[0]) defer nc1.Close() js1, err := nc1.JetStream(nats.Domain("A")) require_NoError(t, err) sub1, err := js1.PullSubscribe("foo", "dlc", nats.BindStream("TEST")) require_NoError(t, err) defer sub1.Unsubscribe() nc2, _ := jsClientConnect(t, c.servers[1]) defer nc2.Close() js2, err := nc2.JetStream(nats.Domain("A")) require_NoError(t, err) sub2, err := js2.PullSubscribe("foo", "dlc", nats.BindStream("TEST")) require_NoError(t, err) defer sub2.Unsubscribe() // Make sure we can properly get messages. msgs, err := sub1.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) require_True(t, string(msgs[0].Data) == "M-1") msgs, err = sub2.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) require_True(t, string(msgs[0].Data) == "M-2") // Make sure delivered state makes it to other server to not accidentally send M-2 again // and fail the test below. time.Sleep(250 * time.Millisecond) // Now let's introduce and event, where A-S-2 will now reconnect after a restart to B-S-2 checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ls := lc.servers[1] wantedRemote := "S-1" var remoteServer string ls.mu.RLock() for _, rc := range ls.leafs { rc.mu.Lock() remoteServer = rc.leaf.remoteServer rc.mu.Unlock() break } ls.mu.RUnlock() if remoteServer != wantedRemote { ls.Shutdown() lc.restartServer(ls) return fmt.Errorf("Leafnode server not connected to %q", wantedRemote) } return nil }) // Wait on ready again. lc.waitOnClusterReady() lc.waitOnStreamLeader(globalAccountName, "TEST") lc.waitOnConsumerLeader(globalAccountName, "TEST", "dlc") // Send 2 more messages. sendStreamMsg(t, lnc, "foo", "M-3") sendStreamMsg(t, lnc, "foo", "M-4") msgs, err = sub1.Fetch(2) require_NoError(t, err) require_True(t, len(msgs) == 2) require_True(t, string(msgs[0].Data) == "M-3") require_True(t, string(msgs[1].Data) == "M-4") // Send 2 more messages. sendStreamMsg(t, lnc, "foo", "M-5") sendStreamMsg(t, lnc, "foo", "M-6") msgs, err = sub2.Fetch(2) require_NoError(t, err) require_True(t, len(msgs) == 2) require_True(t, string(msgs[0].Data) == "M-5") require_True(t, string(msgs[1].Data) == "M-6") } func snapRGSet(pFlag bool, banner string, osi *nats.StreamInfo) *map[string]struct{} { var snapSet = make(map[string]struct{}) if pFlag { fmt.Println(banner) } if osi == nil { if pFlag { fmt.Printf("bonkers!\n") } return nil } snapSet[osi.Cluster.Leader] = struct{}{} if pFlag { fmt.Printf("Leader: %s\n", osi.Cluster.Leader) } for _, replica := range osi.Cluster.Replicas { snapSet[replica.Name] = struct{}{} if pFlag { fmt.Printf("Replica: %s\n", replica.Name) } } return &snapSet } func TestJetStreamClusterAfterPeerRemoveZeroState(t *testing.T) { // R3 scenario (w/messages) in a 4-node cluster. Peer remove from RG and add back to same RG later. // Validate that original peer brought no memory or issues from its previous RG tour of duty, specifically // that the restored peer has the correct filestore usage bytes for the asset. var err error sc := createJetStreamClusterExplicit(t, "cl4", 4) defer sc.shutdown() sc.waitOnClusterReadyWithNumPeers(4) s := sc.leader() nc, jsc := jsClientConnect(t, s) defer nc.Close() _, err = jsc.AddStream(&nats.StreamConfig{ Name: "foo", Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) sc.waitOnStreamLeader(globalAccountName, "foo") osi, err := jsc.StreamInfo("foo") require_NoError(t, err) // make sure 0 msgs require_True(t, osi.State.Msgs == 0) // load up messages toSend := 10000 // storage bytes with JS message overhead assetStoreBytesExpected := uint64(460000) for i := 1; i <= toSend; i++ { msg := []byte("Hello World") if _, err = jsc.Publish("foo.a", msg); err != nil { t.Fatalf("unexpected publish error: %v", err) } } osi, err = jsc.StreamInfo("foo") require_NoError(t, err) // make sure 10000 msgs require_True(t, osi.State.Msgs == uint64(toSend)) origSet := *snapRGSet(false, "== Orig RG Set ==", osi) // remove 1 peer replica (1 of 2 non-leaders) origPeer := osi.Cluster.Replicas[0].Name resp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "foo"), []byte(`{"peer":"`+origPeer+`"}`), time.Second) require_NoError(t, err) var rpResp JSApiStreamRemovePeerResponse err = json.Unmarshal(resp.Data, &rpResp) require_NoError(t, err) require_True(t, rpResp.Success) // validate the origPeer is removed with a replacement newPeer sc.waitOnStreamLeader(globalAccountName, "foo") checkFor(t, time.Second, 200*time.Millisecond, func() error { osi, err = jsc.StreamInfo("foo") require_NoError(t, err) if len(osi.Cluster.Replicas) != 2 { return fmt.Errorf("expected R3, got R%d", len(osi.Cluster.Replicas)+1) } // STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred for _, replica := range osi.Cluster.Replicas { if replica.Name == origPeer { return fmt.Errorf("expected replaced replica, old replica still present") } } return nil }) // identify the new peer var newPeer string osi, err = jsc.StreamInfo("foo") require_NoError(t, err) newSet := *snapRGSet(false, "== New RG Set ==", osi) for peer := range newSet { _, ok := origSet[peer] if !ok { newPeer = peer break } } require_True(t, newPeer != "") // kick out newPeer which will cause origPeer to be assigned to the RG again resp, err = nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "foo"), []byte(`{"peer":"`+newPeer+`"}`), time.Second) require_NoError(t, err) err = json.Unmarshal(resp.Data, &rpResp) require_NoError(t, err) require_True(t, rpResp.Success) // validate the newPeer is removed and R3 has reformed (with origPeer) sc.waitOnStreamLeader(globalAccountName, "foo") checkFor(t, time.Second, 200*time.Millisecond, func() error { osi, err = jsc.StreamInfo("foo") require_NoError(t, err) if len(osi.Cluster.Replicas) != 2 { return fmt.Errorf("expected R3, got R%d", len(osi.Cluster.Replicas)+1) } // STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred for _, replica := range osi.Cluster.Replicas { if replica.Name == newPeer { return fmt.Errorf("expected replaced replica, old replica still present") } } return nil }) osi, err = jsc.StreamInfo("foo") require_NoError(t, err) // make sure all msgs reported in stream at this point with original leader require_True(t, osi.State.Msgs == uint64(toSend)) snapRGSet(false, "== RG Set w/origPeer Back ==", osi) // get a handle to original peer server var origServer *Server = sc.serverByName(origPeer) if origServer == nil { t.Fatalf("expected to get a handle to original peer server by name") } checkFor(t, time.Second, 200*time.Millisecond, func() error { jszResult, err := origServer.Jsz(nil) require_NoError(t, err) if jszResult.Store != assetStoreBytesExpected { return fmt.Errorf("expected %d storage on orig peer, got %d", assetStoreBytesExpected, jszResult.Store) } return nil }) } func TestJetStreamClusterMemLeaderRestart(t *testing.T) { // Test if R3 clustered mem store asset leader server restarted, that asset remains stable with final quorum c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() ml := c.leader() nc, jsc := jsClientConnect(t, ml) defer nc.Close() _, err := jsc.AddStream(&nats.StreamConfig{ Name: "foo", Storage: nats.MemoryStorage, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // load up messages toSend := 10000 for i := 1; i <= toSend; i++ { msg := []byte("Hello World") if _, err = jsc.Publish("foo.a", msg); err != nil { t.Fatalf("unexpected publish error: %v", err) } } osi, err := jsc.StreamInfo("foo") require_NoError(t, err) // make sure 10000 msgs require_True(t, osi.State.Msgs == uint64(toSend)) // Shutdown the stream leader server rs := c.serverByName(osi.Cluster.Leader) rs.Shutdown() // Make sure that we have a META leader (there can always be a re-election) c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "foo") // Should still have quorum and a new leader checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { osi, err = jsc.StreamInfo("foo") if err != nil { return fmt.Errorf("expected healthy stream asset, got %s", err.Error()) } if osi.Cluster.Leader == _EMPTY_ { return fmt.Errorf("expected healthy stream asset with new leader") } if osi.State.Msgs != uint64(toSend) { return fmt.Errorf("expected healthy stream asset %d messages, got %d messages", toSend, osi.State.Msgs) } return nil }) // Now restart the old leader peer (old stream state) oldrs := rs rs, _ = RunServerWithConfig(rs.getOpts().ConfigFile) defer rs.Shutdown() // Replaced old with new server for i := 0; i < len(c.servers); i++ { if c.servers[i] == oldrs { c.servers[i] = rs } } // Wait for cluster to be formed checkClusterFormed(t, c.servers...) // Make sure that we have a leader (there can always be a re-election) c.waitOnLeader() // Can we get stream info after return osi, err = jsc.StreamInfo("foo") if err != nil { t.Fatalf("expected stream asset info return, got %s", err.Error()) } // When asset leader came back did we re-form with quorum if osi.Cluster.Leader == "" { t.Fatalf("expected a current leader after old leader restarted") } } // Customer reported R1 consumers that seemed to be ghosted after server restart. func TestJetStreamClusterLostConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "GHOST", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 10; i++ { for j := 0; j < 10; j++ { _, err := js.Publish(fmt.Sprintf("events.%d.%d", i, j), []byte("test")) require_NoError(t, err) } } s := c.randomServer() s.Shutdown() c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "TEST") nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() cc := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ AckPolicy: AckExplicit, Replicas: 1, }, } req, err := json.Marshal(cc) require_NoError(t, err) reqSubj := fmt.Sprintf(JSApiConsumerCreateT, "TEST") // Now create 50 consumers. Ensure they are successfully created, so they're included in our snapshot. for i := 0; i < 50; i++ { _, err = nc.Request(reqSubj, req, time.Second) require_NoError(t, err) } // Grab the meta leader. ml := c.leader() require_NoError(t, ml.JetStreamSnapshotMeta()) numConsumerAssignments := func(s *Server) int { t.Helper() js := s.getJetStream() js.mu.RLock() defer js.mu.RUnlock() cc := js.cluster for _, asa := range cc.streams { for _, sa := range asa { return len(sa.consumers) } } return 0 } checkFor(t, time.Second, 100*time.Millisecond, func() error { num := numConsumerAssignments(ml) if num == 50 { return nil } return fmt.Errorf("Consumers is only %d", num) }) // Restart the server we shutdown. We snapshotted to the snapshot // has to fill in the new consumers. // The bug would fail to add them to the meta state since the stream // existed. s = c.restartServer(s) checkFor(t, time.Second, 100*time.Millisecond, func() error { num := numConsumerAssignments(s) if num == 50 { return nil } return fmt.Errorf("Consumers is only %d", num) }) } // https://github.com/nats-io/nats-server/issues/3636 func TestJetStreamClusterScaleDownDuringServerOffline(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "hello") } s := c.randomNonStreamLeader(globalAccountName, "TEST") s.Shutdown() c.waitOnLeader() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) s = c.restartServer(s) checkFor(t, time.Second, 200*time.Millisecond, func() error { hs := s.healthz(nil) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) } // Reported by a customer manually upgrading their streams to support direct gets. // Worked if single replica but not in clustered mode. func TestJetStreamClusterDirectGetStreamUpgrade(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "KV_TEST", Subjects: []string{"$KV.TEST.>"}, Discard: nats.DiscardNew, MaxMsgsPerSubject: 1, DenyDelete: true, Replicas: 3, }) require_NoError(t, err) kv, err := js.KeyValue("TEST") require_NoError(t, err) _, err = kv.PutString("name", "derek") require_NoError(t, err) entry, err := kv.Get("name") require_NoError(t, err) require_True(t, string(entry.Value()) == "derek") // Now simulate a update to the stream to support direct gets. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "KV_TEST", Subjects: []string{"$KV.TEST.>"}, Discard: nats.DiscardNew, MaxMsgsPerSubject: 1, DenyDelete: true, AllowDirect: true, Replicas: 3, }) require_NoError(t, err) // Rebind to KV to make sure we DIRECT version of Get(). kv, err = js.KeyValue("TEST") require_NoError(t, err) // Make sure direct get works. entry, err = kv.Get("name") require_NoError(t, err) require_True(t, string(entry.Value()) == "derek") } // For interest (or workqueue) based streams its important to match the replication factor. // This was the case but now that more control over consumer creation is allowed its possible // to create a consumer where the replication factor does not match. This could cause // instability in the state between servers and cause problems on leader switches. func TestJetStreamClusterInterestPolicyStreamForConsumersToMatchRFactor(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.InterestPolicy, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "XX", AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_Error(t, err, NewJSConsumerReplicasShouldMatchStreamError()) } // https://github.com/nats-io/nats-server/issues/3791 func TestJetStreamClusterKVWatchersWithServerDown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, }) require_NoError(t, err) kv.PutString("foo", "bar") kv.PutString("foo", "baz") // Shutdown a follower. s := c.randomNonStreamLeader(globalAccountName, "KV_TEST") s.Shutdown() c.waitOnLeader() nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() js, err = nc.JetStream(nats.MaxWait(2 * time.Second)) require_NoError(t, err) kv, err = js.KeyValue("TEST") require_NoError(t, err) for i := 0; i < 100; i++ { w, err := kv.Watch("foo") require_NoError(t, err) w.Stop() } } // TestJetStreamClusterCurrentVsHealth is designed to show the // difference between "current" and "healthy" when async publishes // outpace the rate at which they can be applied. func TestJetStreamClusterCurrentVsHealth(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() server := c.randomNonLeader() nc, js := jsClientConnect(t, server) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) server = c.randomNonStreamLeader(globalAccountName, "TEST") stream, err := server.GlobalAccount().lookupStream("TEST") require_NoError(t, err) raft, ok := stream.raftGroup().node.(*raft) require_True(t, ok) for i := 0; i < 1000; i++ { _, err := js.PublishAsync("foo", []byte("bar")) require_NoError(t, err) raft.RLock() commit := raft.commit applied := raft.applied raft.RUnlock() current := raft.Current() healthy := raft.Healthy() if !current || !healthy || commit != applied { t.Logf( "%d | Current %v, healthy %v, commit %d, applied %d, pending %d", i, current, healthy, commit, applied, commit-applied, ) } } } // Several users and customers use this setup, but many times across leafnodes. // This should be allowed in same account since we are really protecting against // multiple pub acks with cycle detection. func TestJetStreamClusterActiveActiveSourcedStreams(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "A", Subjects: []string{"A.>"}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "B", Subjects: []string{"B.>"}, }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "A", Subjects: []string{"A.>"}, Sources: []*nats.StreamSource{{ Name: "B", FilterSubject: "B.>", }}, }) require_NoError(t, err) // Before this would fail. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "B", Subjects: []string{"B.>"}, Sources: []*nats.StreamSource{{ Name: "A", FilterSubject: "A.>", }}, }) require_NoError(t, err) } func TestJetStreamClusterUpdateConsumerShouldNotForceDeleteOnRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R7S", 7) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) require_NoError(t, err) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "D", DeliverSubject: "_no_bind_", }) require_NoError(t, err) // Shutdown a consumer follower. nc.Close() s := c.serverByName(ci.Cluster.Replicas[0].Name) s.Shutdown() c.waitOnLeader() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Change delivery subject. _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "D", DeliverSubject: "_d_", }) require_NoError(t, err) // Create interest in new and old deliver subject. _, err = nc.SubscribeSync("_d_") require_NoError(t, err) _, err = nc.SubscribeSync("_no_bind_") require_NoError(t, err) nc.Flush() c.restartServer(s) c.waitOnAllCurrent() // Wait on bad error that would cleanup consumer. time.Sleep(time.Second) _, err = js.ConsumerInfo("TEST", "D") require_NoError(t, err) } func TestJetStreamClusterInterestPolicyEphemeral(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() for _, test := range []struct { testName string stream string subject string durable string name string }{ {testName: "InterestWithDurable", durable: "eph", subject: "intdur", stream: "INT_DUR"}, {testName: "InterestWithName", name: "eph", subject: "inteph", stream: "INT_EPH"}, } { t.Run(test.testName, func(t *testing.T) { var err error nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.AddStream(&nats.StreamConfig{ Name: test.stream, Subjects: []string{test.subject}, Retention: nats.LimitsPolicy, Replicas: 3, }) require_NoError(t, err) const inactiveThreshold = time.Second _, err = js.AddConsumer(test.stream, &nats.ConsumerConfig{ DeliverSubject: nats.NewInbox(), AckPolicy: nats.AckExplicitPolicy, InactiveThreshold: inactiveThreshold, Durable: test.durable, Name: test.name, }) require_NoError(t, err) name := test.durable if test.durable == _EMPTY_ { name = test.name } const msgs = 5_000 done, count := make(chan bool, 1), 0 sub, err := js.Subscribe(_EMPTY_, func(msg *nats.Msg) { require_NoError(t, msg.Ack()) count++ if count >= msgs { select { case done <- true: default: } } }, nats.Bind(test.stream, name), nats.ManualAck()) require_NoError(t, err) // This happens only if we start publishing messages after consumer was created. pubDone := make(chan struct{}) go func(subject string) { for i := 0; i < msgs; i++ { js.Publish(subject, []byte("DATA")) } close(pubDone) }(test.subject) // Wait for inactive threshold to expire and all messages to be published and received // Bug is we clean up active consumers when we should not. time.Sleep(3 * inactiveThreshold / 2) select { case <-pubDone: case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } info, err := js.ConsumerInfo(test.stream, name) if err != nil { t.Fatalf("Expected to be able to retrieve consumer: %v", err) } require_True(t, info.Delivered.Stream == msgs) // Stop the subscription and remove the interest. err = sub.Unsubscribe() require_NoError(t, err) // Now wait for interest inactivity threshold to kick in. time.Sleep(3 * inactiveThreshold / 2) // Check if the consumer has been removed. _, err = js.ConsumerInfo(test.stream, name) require_Error(t, err, nats.ErrConsumerNotFound) }) } } // TestJetStreamClusterWALBuildupOnNoOpPull tests whether or not the consumer // RAFT log is being compacted when the stream is idle but we are performing // lots of fetches. Otherwise the disk usage just spirals out of control if // there are no other state changes to trigger a compaction. func TestJetStreamClusterWALBuildupOnNoOpPull(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) sub, err := js.PullSubscribe( "foo", "durable", nats.ConsumerReplicas(3), ) require_NoError(t, err) for i := 0; i < 10000; i++ { _, _ = sub.Fetch(1, nats.MaxWait(time.Microsecond)) } // Needs to be at least 10 seconds, otherwise we won't hit the // minSnapDelta that prevents us from snapshotting too often time.Sleep(time.Second * 11) for i := 0; i < 1024; i++ { _, _ = sub.Fetch(1, nats.MaxWait(time.Microsecond)) } time.Sleep(time.Second) server := c.randomNonConsumerLeader(globalAccountName, "TEST", "durable") stream, err := server.globalAccount().lookupStream("TEST") require_NoError(t, err) consumer := stream.lookupConsumer("durable") require_NotNil(t, consumer) entries, bytes := consumer.raftNode().Size() t.Log("new entries:", entries) t.Log("new bytes:", bytes) if max := uint64(1024); entries > max { t.Fatalf("got %d entries, expected less than %d entries", entries, max) } } // Found in https://github.com/nats-io/nats-server/issues/3848 // When Max Age was specified and stream was scaled up, new replicas // were expiring messages much later than the leader. func TestJetStreamClusterStreamMaxAgeScaleUp(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for _, test := range []struct { name string storage nats.StorageType stream string purge bool }{ {name: "file", storage: nats.FileStorage, stream: "A", purge: false}, {name: "memory", storage: nats.MemoryStorage, stream: "B", purge: false}, {name: "file with purge", storage: nats.FileStorage, stream: "C", purge: true}, {name: "memory with purge", storage: nats.MemoryStorage, stream: "D", purge: true}, } { t.Run(test.name, func(t *testing.T) { ttl := time.Second * 5 // Add stream with one replica and short MaxAge. _, err := js.AddStream(&nats.StreamConfig{ Name: test.stream, Replicas: 1, Subjects: []string{test.stream}, MaxAge: ttl, Storage: test.storage, }) require_NoError(t, err) // Add some messages. for i := 0; i < 10; i++ { sendStreamMsg(t, nc, test.stream, "HELLO") } // We need to also test if we properly set expiry // if first sequence is not 1. if test.purge { err = js.PurgeStream(test.stream) require_NoError(t, err) // Add some messages. for i := 0; i < 10; i++ { sendStreamMsg(t, nc, test.stream, "HELLO") } } // Mark the time when all messages were published. start := time.Now() // Sleep for half of the MaxAge time. time.Sleep(ttl / 2) // Scale up the Stream to 3 replicas. _, err = js.UpdateStream(&nats.StreamConfig{ Name: test.stream, Replicas: 3, Subjects: []string{test.stream}, MaxAge: ttl, Storage: test.storage, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, test.stream) // All messages should still be there. info, err := js.StreamInfo(test.stream) require_NoError(t, err) require_Equal(t, info.State.Msgs, 10) // Wait until MaxAge is reached. time.Sleep(ttl - time.Since(start) + (1 * time.Second)) // Check if all messages are expired. info, err = js.StreamInfo(test.stream) require_NoError(t, err) require_Equal(t, info.State.Msgs, 0) // Now switch leader to one of replicas _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, test.stream), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, test.stream) // and make sure that it also expired all messages info, err = js.StreamInfo(test.stream) require_NoError(t, err) require_Equal(t, info.State.Msgs, 0) }) } } func TestJetStreamClusterWorkQueueConsumerReplicatedAfterScaleUp(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 1, Subjects: []string{"WQ"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) // Create an ephemeral consumer. sub, err := js.SubscribeSync("WQ") require_NoError(t, err) // Scale up to R3. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"WQ"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") ci, err := sub.ConsumerInfo() require_NoError(t, err) require_True(t, ci.Config.Replicas == 0 || ci.Config.Replicas == 3) c.waitOnConsumerLeader(globalAccountName, "TEST", ci.Name) s := c.consumerLeader(globalAccountName, "TEST", ci.Name) require_NotNil(t, s) mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer(ci.Name) require_NotNil(t, o) require_NotNil(t, o.raftNode()) } // https://github.com/nats-io/nats-server/issues/3953 func TestJetStreamClusterWorkQueueAfterScaleUp(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 1, Subjects: []string{"WQ"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "d1", DeliverSubject: "d1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) wch := make(chan bool, 1) _, err = nc.Subscribe("d1", func(msg *nats.Msg) { msg.AckSync() wch <- true }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"WQ"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") sendStreamMsg(t, nc, "WQ", "SOME WORK") select { case <-wch: case <-time.After(5 * time.Second): t.Fatalf("Did not receive ack signal") } checkFor(t, time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == 0 { return nil } return fmt.Errorf("Still have %d msgs left", si.State.Msgs) }) } func TestJetStreamClusterInterestBasedStreamAndConsumerSnapshots(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"foo"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub, err := js.SubscribeSync("foo", nats.Durable("d22")) require_NoError(t, err) num := 200 for i := 0; i < num; i++ { js.PublishAsync("foo", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, num) // Shutdown one server. s := c.randomServer() s.Shutdown() c.waitOnStreamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Now ack all messages while the other server is down. for i := 0; i < num; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) m.AckSync() } // Wait for all message acks to be processed and all messages to be removed. checkFor(t, time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == 0 { return nil } return fmt.Errorf("Still have %d msgs left", si.State.Msgs) }) // Force a snapshot on the consumer leader before restarting the downed server. cl := c.consumerLeader(globalAccountName, "TEST", "d22") require_NotNil(t, cl) mset, err := cl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("d22") require_NotNil(t, o) snap, err := o.store.EncodedState() require_NoError(t, err) n := o.raftNode() require_NotNil(t, n) require_NoError(t, n.InstallSnapshot(snap)) // Now restart the downed server. s = c.restartServer(s) // Make the restarted server the eventual leader. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader(globalAccountName, "TEST") if sl := c.streamLeader(globalAccountName, "TEST"); sl != s { sl.JetStreamStepdownStream(globalAccountName, "TEST") return fmt.Errorf("Server %s is not leader yet", s) } return nil }) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 0) } func TestJetStreamClusterConsumerFollowerStoreStateAckFloorBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"foo"}, }) require_NoError(t, err) sub, err := js.PullSubscribe(_EMPTY_, "C", nats.BindStream("TEST"), nats.ManualAck()) require_NoError(t, err) num := 100 for i := 0; i < num; i++ { sendStreamMsg(t, nc, "foo", "data") } // This one prevents the state for pending from reaching 0 and resetting, which would not show the bug. sendStreamMsg(t, nc, "foo", "data") // Ack all but one and out of order and make sure all consumers have the same stored state. msgs, err := sub.Fetch(num, nats.MaxWait(time.Second)) require_NoError(t, err) require_True(t, len(msgs) == num) _, err = sub.Fetch(1, nats.MaxWait(time.Second)) require_NoError(t, err) rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) for _, m := range msgs { if err := m.AckSync(); err != nil { t.Fatalf("Ack failed :%+v", err) } } checkConsumerState := func(delivered, ackFloor nats.SequenceInfo, numAckPending int) error { expectedDelivered := uint64(num) + 1 if delivered.Stream != expectedDelivered || delivered.Consumer != expectedDelivered { return fmt.Errorf("Wrong delivered, expected %d got %+v", expectedDelivered, delivered) } expectedAck := uint64(num) if ackFloor.Stream != expectedAck || ackFloor.Consumer != expectedAck { return fmt.Errorf("Wrong ackFloor, expected %d got %+v", expectedAck, ackFloor) } if numAckPending != 1 { return errors.New("Expected num ack pending to be 1") } return nil } ci, err := js.ConsumerInfo("TEST", "C") require_NoError(t, err) require_NoError(t, checkConsumerState(ci.Delivered, ci.AckFloor, ci.NumAckPending)) // Check each consumer on each server for it's store state and make sure it matches as well. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") if err != nil { return err } if mset == nil { return errors.New("Mset should not be nil") } o := mset.lookupConsumer("C") if o == nil { return errors.New("Consumer should not be nil") } state, err := o.store.State() if err != nil { return err } delivered := nats.SequenceInfo{Stream: state.Delivered.Stream, Consumer: state.Delivered.Consumer} ackFloor := nats.SequenceInfo{Stream: state.AckFloor.Stream, Consumer: state.AckFloor.Consumer} if err := checkConsumerState(delivered, ackFloor, len(state.Pending)); err != nil { return err } } return nil }) // Now stepdown the consumer and move its leader and check the state after transition. // Make the restarted server the eventual leader. seen := make(map[*Server]bool) cl := c.consumerLeader(globalAccountName, "TEST", "C") require_NotNil(t, cl) seen[cl] = true allSeen := func() bool { for _, s := range c.servers { if !seen[s] { return false } } return true } checkAllLeaders := func() { t.Helper() checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "TEST", "C") if allSeen() { return nil } cl := c.consumerLeader(globalAccountName, "TEST", "C") seen[cl] = true ci, err := js.ConsumerInfo("TEST", "C") if err != nil { return err } if err := checkConsumerState(ci.Delivered, ci.AckFloor, ci.NumAckPending); err != nil { return err } cl.JetStreamStepdownConsumer(globalAccountName, "TEST", "C") return fmt.Errorf("Not all servers have been consumer leader yet") }) } checkAllLeaders() // No restart all servers and check again. c.stopAll() c.restartAll() c.waitOnLeader() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() seen = make(map[*Server]bool) checkAllLeaders() } func TestJetStreamClusterInterestLeakOnDisableJetStream(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.leader()) defer nc.Close() for i := 1; i <= 5; i++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("test_%d", i), Subjects: []string{fmt.Sprintf("test_%d", i)}, Replicas: 3, }) require_NoError(t, err) } c.waitOnAllCurrent() server := c.randomNonLeader() account := server.SystemAccount() server.DisableJetStream() checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { var sublist []*subscription account.sl.localSubs(&sublist, false) var danglingJSC, danglingRaft int for _, sub := range sublist { if strings.HasPrefix(string(sub.subject), "$JSC.") { danglingJSC++ } else if strings.HasPrefix(string(sub.subject), "$NRG.") { danglingRaft++ } } if danglingJSC > 0 || danglingRaft > 0 { return fmt.Errorf("unexpected dangling interests for JetStream assets after shutdown (%d $JSC, %d $NRG)", danglingJSC, danglingRaft) } return nil }) } func TestJetStreamClusterNoLeadersDuringLameDuck(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Grab the first server and set lameduck option directly. s := c.servers[0] s.optsMu.Lock() s.opts.LameDuckDuration = 5 * time.Second s.opts.LameDuckGracePeriod = -5 * time.Second s.optsMu.Unlock() // Connect to the third server. nc, js := jsClientConnect(t, c.servers[2]) defer nc.Close() allServersHaveLeaders := func() bool { haveLeader := make(map[*Server]bool) for _, s := range c.servers { s.rnMu.RLock() for _, n := range s.raftNodes { if n.Leader() { haveLeader[s] = true break } } s.rnMu.RUnlock() } return len(haveLeader) == len(c.servers) } // Create streams until we have a leader on all the servers. var index int checkFor(t, 10*time.Second, time.Millisecond, func() error { if allServersHaveLeaders() { return nil } index++ _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST_%d", index), Subjects: []string{fmt.Sprintf("foo.%d", index)}, Replicas: 3, }) require_NoError(t, err) return fmt.Errorf("All servers do not have at least one leader") }) // Put our server into lameduck mode. // Need a client. dummy, _ := jsClientConnect(t, s) defer dummy.Close() go s.lameDuckMode() // Wait for all leaders to move off. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { s.rnMu.RLock() defer s.rnMu.RUnlock() for _, n := range s.raftNodes { if n.Leader() { return fmt.Errorf("Server still has a leader") } } return nil }) // All leader evacuated. // Create a go routine that will create streams constantly. qch := make(chan bool) go func() { var index int for { select { case <-time.After(time.Millisecond): index++ _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("NEW_TEST_%d", index), Subjects: []string{fmt.Sprintf("bar.%d", index)}, Replicas: 3, }) if err != nil { return } case <-qch: return } } }() defer close(qch) // Make sure we do not have any leaders placed on the lameduck server. for s.isRunning() { var hasLeader bool s.rnMu.RLock() for _, n := range s.raftNodes { hasLeader = hasLeader || n.Leader() } s.rnMu.RUnlock() if hasLeader { t.Fatalf("Server had a leader when it should not due to lameduck mode") } } } func TestJetStreamClusterNoR1AssetsDuringLameDuck(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Grab the first server and set lameduck option directly. s := c.servers[0] s.optsMu.Lock() s.opts.LameDuckDuration = 5 * time.Second s.opts.LameDuckGracePeriod = -5 * time.Second s.optsMu.Unlock() // Connect to the server to keep it alive when we go into LDM. dummy, _ := jsClientConnect(t, s) defer dummy.Close() // Connect to the third server. nc, js := jsClientConnect(t, c.servers[2]) defer nc.Close() // Now put the first server into lame duck mode. go s.lameDuckMode() // Wait for news to arrive that the first server has gone into // lame duck mode and been marked offline. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { id := s.info.ID s := c.servers[2] s.mu.RLock() defer s.mu.RUnlock() var isOffline bool s.nodeToInfo.Range(func(_, v any) bool { ni := v.(nodeInfo) if ni.id == id { isOffline = ni.offline return false } return true }) if !isOffline { return fmt.Errorf("first node is still online unexpectedly") } return nil }) // Create a go routine that will create streams constantly. qch := make(chan bool) go func() { var index int for { select { case <-time.After(time.Millisecond * 25): index++ _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("NEW_TEST_%d", index), Subjects: []string{fmt.Sprintf("bar.%d", index)}, Replicas: 1, }) if err != nil { return } case <-qch: return } } }() defer close(qch) gacc := s.GlobalAccount() if gacc == nil { t.Fatalf("No global account") } // Make sure we do not have any R1 assets placed on the lameduck server. for s.isRunning() { if len(gacc.streams()) > 0 { t.Fatalf("Server had an R1 asset when it should not due to lameduck mode") } time.Sleep(15 * time.Millisecond) } s.WaitForShutdown() } // If a consumer has not been registered (possible in heavily loaded systems with lots of assets) // it could miss the signal of a message going away. If that message was pending and expires the // ack floor could fall below the stream first sequence. This test will force that condition and // make sure the system resolves itself. func TestJetStreamClusterConsumerAckFloorDrift(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, MaxAge: time.Second, MaxMsgs: 10, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "C") require_NoError(t, err) // Publish as many messages as the ack floor check threshold +5 (what we set ackfloor to later). totalMessages := 55 for i := 0; i < totalMessages; i++ { sendStreamMsg(t, nc, "foo", "HELLO") } // No-op but will surface as delivered. _, err = sub.Fetch(10) require_NoError(t, err) // We will initialize the state with delivered being 10 and ackfloor being 0 directly. // Fetch will asynchronously propagate this state, so can't reliably request this from the leader immediately. state := &ConsumerState{Delivered: SequencePair{Consumer: 10, Stream: 10}} // Now let messages expire. checkFor(t, 5*time.Second, time.Second, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == 0 { return nil } return fmt.Errorf("stream still has msgs") }) // Set state to ackfloor of 5 and no pending. state.AckFloor.Consumer = 5 state.AckFloor.Stream = 5 state.Pending = nil // Now put back the state underneath of the consumers. for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("C") require_NotNil(t, o) o.mu.Lock() o.applyState(state) cfs := o.store.(*consumerFileStore) o.mu.Unlock() // The lower layer will ignore, so set more directly. cfs.mu.Lock() cfs.state = *state cfs.mu.Unlock() // Also snapshot to remove any raft entries that could affect it. snap, err := o.store.EncodedState() require_NoError(t, err) require_NoError(t, o.raftNode().InstallSnapshot(snap)) } cl := c.consumerLeader(globalAccountName, "TEST", "C") require_NotNil(t, cl) err = cl.JetStreamStepdownConsumer(globalAccountName, "TEST", "C") require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "C") checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "C") if err != nil { return err } // Replicated state should stay the same. if ci.AckFloor.Stream != 5 && ci.AckFloor.Consumer != 5 { return fmt.Errorf("replicated AckFloor not correct, expected %d, got %+v", 5, ci.AckFloor) } cl = c.consumerLeader(globalAccountName, "TEST", "C") mset, err := cl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("C") require_NotNil(t, o) o.mu.RLock() defer o.mu.RUnlock() // Make sure we catch this and adjust. if o.asflr != uint64(totalMessages) && o.adflr != 10 { return fmt.Errorf("leader AckFloor not correct, expected %d, got %+v", 10, ci.AckFloor) } return nil }) } func TestJetStreamClusterInterestStreamFilteredConsumersWithNoInterest(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Retention: nats.InterestPolicy, Replicas: 3, }) require_NoError(t, err) // Create three subscribers. ackCb := func(m *nats.Msg) { m.Ack() } _, err = js.Subscribe("foo", ackCb, nats.BindStream("TEST"), nats.ManualAck()) require_NoError(t, err) _, err = js.Subscribe("bar", ackCb, nats.BindStream("TEST"), nats.ManualAck()) require_NoError(t, err) _, err = js.Subscribe("baz", ackCb, nats.BindStream("TEST"), nats.ManualAck()) require_NoError(t, err) // Now send 100 messages, randomly picking foo or bar, but never baz. for i := 0; i < 100; i++ { if rand.Intn(2) > 0 { sendStreamMsg(t, nc, "foo", "HELLO") } else { sendStreamMsg(t, nc, "bar", "WORLD") } } // Messages are expected to go to 0. checkFor(t, time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == 0 { return nil } return fmt.Errorf("stream still has msgs") }) } func TestJetStreamClusterChangeClusterAfterStreamCreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "foo", "HELLO") } _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 1, }) require_NoError(t, err) c.stopAll() c.name = "FOO" for _, o := range c.opts { buf, err := os.ReadFile(o.ConfigFile) require_NoError(t, err) nbuf := bytes.Replace(buf, []byte("name: NATS"), []byte("name: FOO"), 1) err = os.WriteFile(o.ConfigFile, nbuf, 0640) require_NoError(t, err) } c.restartAll() c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) // This should fail with no suitable peers, since the asset was created under the NATS cluster which has no peers. require_Error(t, err, errors.New("nats: no suitable peers for placement")) // Make sure we can swap the cluster. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Placement: &nats.Placement{Cluster: "FOO"}, }) require_NoError(t, err) } // The consumer info() call does not take into account whether a consumer // is a leader or not, so results would be very different when asking servers // that housed consumer followers vs leaders. func TestJetStreamClusterConsumerInfoForJszForFollowers(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "foo", "HELLO") } sub, err := js.PullSubscribe("foo", "d") require_NoError(t, err) fetch, ack := 122, 22 msgs, err := sub.Fetch(fetch, nats.MaxWait(10*time.Second)) require_NoError(t, err) require_True(t, len(msgs) == fetch) for _, m := range msgs[:ack] { m.AckSync() } // Let acks propagate. time.Sleep(100 * time.Millisecond) for _, s := range c.servers { jsz, err := s.Jsz(&JSzOptions{Accounts: true, Consumer: true}) require_NoError(t, err) require_True(t, len(jsz.AccountDetails) == 1) require_True(t, len(jsz.AccountDetails[0].Streams) == 1) require_True(t, len(jsz.AccountDetails[0].Streams[0].Consumer) == 1) consumer := jsz.AccountDetails[0].Streams[0].Consumer[0] if consumer.Delivered.Consumer != uint64(fetch) || consumer.Delivered.Stream != uint64(fetch) { t.Fatalf("Incorrect delivered for %v: %+v", s, consumer.Delivered) } if consumer.AckFloor.Consumer != uint64(ack) || consumer.AckFloor.Stream != uint64(ack) { t.Fatalf("Incorrect ackfloor for %v: %+v", s, consumer.AckFloor) } } } // Make sure that stopping a stream shutdowns down it's raft node. func TestJetStreamClusterStreamNodeShutdownBugOnStop(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "HELLO") } s := c.randomServer() numNodesStart := s.numRaftNodes() mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) node := mset.raftNode() require_NotNil(t, node) node.InstallSnapshot(mset.stateSnapshot()) // Stop the stream mset.stop(false, false) node.WaitForStop() if numNodes := s.numRaftNodes(); numNodes != numNodesStart-1 { t.Fatalf("RAFT nodes after stream stop incorrect: %d vs %d", numNodesStart, numNodes) } } func TestJetStreamClusterStreamAccountingOnStoreError(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, MaxBytes: 1 * 1024 * 1024 * 1024, Replicas: 3, }) require_NoError(t, err) msg := strings.Repeat("Z", 32*1024) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", msg) } s := c.randomServer() acc, err := s.LookupAccount("$U") require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) mset.mu.Lock() mset.store.Stop() sjs := mset.js mset.mu.Unlock() // Now delete the stream js.DeleteStream("TEST") // Wait for this to propgate. // The bug will have us not release reserved resources properly. checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { info, err := js.AccountInfo() require_NoError(t, err) // Default tier if info.Store != 0 { return fmt.Errorf("Expected store to be 0 but got %v", friendlyBytes(info.Store)) } return nil }) // Now check js from server directly regarding reserved. sjs.mu.RLock() reserved := sjs.storeReserved sjs.mu.RUnlock() // Under bug will show 1GB if reserved != 0 { t.Fatalf("Expected store reserved to be 0 after stream delete, got %v", friendlyBytes(reserved)) } } func TestJetStreamClusterStreamAccountingDriftFixups(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, MaxBytes: 2 * 1024 * 1024, Replicas: 3, }) require_NoError(t, err) msg := strings.Repeat("Z", 32*1024) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", msg) } err = js.PurgeStream("TEST") require_NoError(t, err) checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { info, err := js.AccountInfo() require_NoError(t, err) if info.Store != 0 { return fmt.Errorf("Store usage not 0: %d", info.Store) } return nil }) s := c.leader() jsz, err := s.Jsz(nil) require_NoError(t, err) require_True(t, jsz.JetStreamStats.Store == 0) acc, err := s.LookupAccount("$U") require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() jsa, tier, stype := mset.jsa, mset.tier, mset.stype mset.mu.RUnlock() // Drift the usage. jsa.updateUsage(tier, stype, -100) checkFor(t, time.Second, 200*time.Millisecond, func() error { info, err := js.AccountInfo() require_NoError(t, err) if info.Store != 0 { return fmt.Errorf("Store usage not 0: %d", info.Store) } return nil }) jsz, err = s.Jsz(nil) require_NoError(t, err) require_True(t, jsz.JetStreamStats.Store == 0) } // Some older streams seem to have been created or exist with no explicit cluster setting. // For server <= 2.9.16 you could not scale the streams up since we could not place them in another cluster. func TestJetStreamClusterStreamScaleUpNoGroupCluster(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, }) require_NoError(t, err) // Manually going to grab stream assignment and update it to be without the group cluster. s := c.streamLeader(globalAccountName, "TEST") mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) sa := mset.streamAssignment() require_NotNil(t, sa) // Make copy to not change stream's sa = sa.copyGroup() // Remove cluster and preferred. sa.Group.Cluster = _EMPTY_ sa.Group.Preferred = _EMPTY_ // Insert into meta layer. if sjs := s.getJetStream(); sjs != nil { sjs.mu.RLock() meta := sjs.cluster.meta sjs.mu.RUnlock() if meta != nil { meta.ForwardProposal(encodeUpdateStreamAssignment(sa)) } } // Make sure it got propagated.. checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { sa := mset.streamAssignment().copyGroup() require_NotNil(t, sa) if sa.Group.Cluster != _EMPTY_ { return fmt.Errorf("Cluster still not cleared") } return nil }) // Now we know it has been nil'd out. Make sure we can scale up. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) } // https://github.com/nats-io/nats-server/issues/4162 func TestJetStreamClusterStaleDirectGetOnRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, }) require_NoError(t, err) _, err = kv.PutString("foo", "bar") require_NoError(t, err) // Close client in case we were connected to server below. // We will recreate. nc.Close() // Shutdown a non-leader. s := c.randomNonStreamLeader(globalAccountName, "KV_TEST") s.Shutdown() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err = js.KeyValue("TEST") require_NoError(t, err) _, err = kv.PutString("foo", "baz") require_NoError(t, err) errCh := make(chan error, 100) done := make(chan struct{}) go func() { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.KeyValue("TEST") if err != nil { errCh <- err return } for { select { case <-done: return default: entry, err := kv.Get("foo") if err != nil { errCh <- err return } if v := string(entry.Value()); v != "baz" { errCh <- fmt.Errorf("Got wrong value: %q", v) } } } }() // Restart c.restartServer(s) // Wait for a bit to make sure as this server participates in direct gets // it does not server stale reads. time.Sleep(2 * time.Second) close(done) if len(errCh) > 0 { t.Fatalf("Expected no errors but got %v", <-errCh) } } // This test mimics a user's setup where there is a cloud cluster/domain, and one for eu and ap that are leafnoded into the // cloud cluster, and one for cn that is leafnoded into the ap cluster. // We broke basic connectivity in 2.9.17 from publishing in eu for delivery in cn on same account which is daisy chained through ap. // We will also test cross account delivery in this test as well. func TestJetStreamClusterLeafnodePlusDaisyChainSetup(t *testing.T) { var cloudTmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: CLOUD, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { F { jetstream: enabled users = [ { user: "F", pass: "pass" } ] exports [ { stream: "F.>" } ] } T { jetstream: enabled users = [ { user: "T", pass: "pass" } ] imports [ { stream: { account: F, subject: "F.>"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` // Now create the cloud and make sure we are connected. // Cloud c := createJetStreamCluster(t, cloudTmpl, "CLOUD", _EMPTY_, 3, 22020, false) defer c.shutdown() var lnTmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { F { jetstream: enabled users = [ { user: "F", pass: "pass" } ] exports [ { stream: "F.>" } ] } T { jetstream: enabled users = [ { user: "T", pass: "pass" } ] imports [ { stream: { account: F, subject: "F.>"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` var leafFrag = ` leaf { listen: 127.0.0.1:-1 remotes [ { urls: [ %s ], account: "T" }, { urls: [ %s ], account: "F" } ] }` genLeafTmpl := func(tmpl string, c *cluster) string { t.Helper() // Create our leafnode cluster template first. var lnt, lnf []string for _, s := range c.servers { if s.ClusterName() != c.name { continue } ln := s.getOpts().LeafNode lnt = append(lnt, fmt.Sprintf("nats://T:pass@%s:%d", ln.Host, ln.Port)) lnf = append(lnf, fmt.Sprintf("nats://F:pass@%s:%d", ln.Host, ln.Port)) } lntc := strings.Join(lnt, ", ") lnfc := strings.Join(lnf, ", ") return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(leafFrag, lntc, lnfc), 1) } // Cluster EU // Domain is "EU' tmpl := strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "EU"), 1) tmpl = genLeafTmpl(tmpl, c) lceu := createJetStreamCluster(t, tmpl, "EU", "EU-", 3, 22110, false) lceu.waitOnClusterReady() defer lceu.shutdown() for _, s := range lceu.servers { checkLeafNodeConnectedCount(t, s, 2) } // Cluster AP // Domain is "AP' tmpl = strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "AP"), 1) tmpl = genLeafTmpl(tmpl, c) lcap := createJetStreamCluster(t, tmpl, "AP", "AP-", 3, 22180, false) lcap.waitOnClusterReady() defer lcap.shutdown() for _, s := range lcap.servers { checkLeafNodeConnectedCount(t, s, 2) } // Cluster CN // Domain is "CN' // This one connects to AP, not the cloud hub. tmpl = strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "CN"), 1) tmpl = genLeafTmpl(tmpl, lcap) lccn := createJetStreamCluster(t, tmpl, "CN", "CN-", 3, 22280, false) lccn.waitOnClusterReady() defer lccn.shutdown() for _, s := range lccn.servers { checkLeafNodeConnectedCount(t, s, 2) } // Now connect to CN on account F and subscribe to data. nc, _ := jsClientConnect(t, lccn.randomServer(), nats.UserInfo("F", "pass")) defer nc.Close() fsub, err := nc.SubscribeSync("F.EU.>") require_NoError(t, err) // Same for account T where the import is. nc, _ = jsClientConnect(t, lccn.randomServer(), nats.UserInfo("T", "pass")) defer nc.Close() tsub, err := nc.SubscribeSync("F.EU.>") require_NoError(t, err) // Let sub propagate. time.Sleep(500 * time.Millisecond) // Now connect to EU on account F and generate data. nc, _ = jsClientConnect(t, lceu.randomServer(), nats.UserInfo("F", "pass")) defer nc.Close() num := 10 for i := 0; i < num; i++ { err := nc.Publish("F.EU.DATA", []byte(fmt.Sprintf("MSG-%d", i))) require_NoError(t, err) } checkSubsPending(t, fsub, num) // Since we export and import in each cluster, we will receive 4x. // First hop from EU -> CLOUD is 1F and 1T // Second hop from CLOUD -> AP is 1F, 1T and another 1T // Third hop from AP -> CN is 1F, 1T, 1T and 1T // Each cluster hop that has the export/import mapping will add another T message copy. checkSubsPending(t, tsub, num*4) // Create stream in cloud. nc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo("F", "pass")) defer nc.Close() _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"TEST.>"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, fmt.Sprintf("TEST.%d", i), "OK") } // Now connect to EU. nc, js = jsClientConnect(t, lceu.randomServer(), nats.UserInfo("F", "pass")) defer nc.Close() // Create a mirror. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{ Name: "TEST", Domain: "CLOUD", }, }) require_NoError(t, err) checkFor(t, time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("M") require_NoError(t, err) if si.State.Msgs == 100 { return nil } return fmt.Errorf("State not current: %+v", si.State) }) } // https://github.com/nats-io/nats-server/pull/4197 func TestJetStreamClusterPurgeExReplayAfterRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "P3F", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"TEST.>"}, Replicas: 3, }) require_NoError(t, err) sendStreamMsg(t, nc, "TEST.0", "OK") sendStreamMsg(t, nc, "TEST.1", "OK") sendStreamMsg(t, nc, "TEST.2", "OK") runTest := func(f func(js nats.JetStreamManager)) *nats.StreamInfo { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // install snapshot, then execute interior func, ensuring the purge will be recovered later fsl := c.streamLeader(globalAccountName, "TEST") fsl.JetStreamSnapshotStream(globalAccountName, "TEST") f(js) time.Sleep(250 * time.Millisecond) fsl.Shutdown() fsl.WaitForShutdown() fsl = c.restartServer(fsl) c.waitOnServerCurrent(fsl) nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() c.waitOnStreamLeader(globalAccountName, "TEST") sl := c.streamLeader(globalAccountName, "TEST") // keep stepping down so the stream leader matches the initial leader // we need to check if it restored from the snapshot properly for sl != fsl { _, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") sl = c.streamLeader(globalAccountName, "TEST") } si, err := js.StreamInfo("TEST") require_NoError(t, err) return si } si := runTest(func(js nats.JetStreamManager) { err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: "TEST.0"}) require_NoError(t, err) }) if si.State.Msgs != 2 { t.Fatalf("Expected 2 msgs after restart, got %d", si.State.Msgs) } if si.State.FirstSeq != 2 || si.State.LastSeq != 3 { t.Fatalf("Expected FirstSeq=2, LastSeq=3 after restart, got FirstSeq=%d, LastSeq=%d", si.State.FirstSeq, si.State.LastSeq) } si = runTest(func(js nats.JetStreamManager) { err = js.PurgeStream("TEST") require_NoError(t, err) // Send 2 more messages. sendStreamMsg(t, nc, "TEST.1", "OK") sendStreamMsg(t, nc, "TEST.2", "OK") }) if si.State.Msgs != 2 { t.Fatalf("Expected 2 msgs after restart, got %d", si.State.Msgs) } if si.State.FirstSeq != 4 || si.State.LastSeq != 5 { t.Fatalf("Expected FirstSeq=4, LastSeq=5 after restart, got FirstSeq=%d, LastSeq=%d", si.State.FirstSeq, si.State.LastSeq) } // Now test a keep si = runTest(func(js nats.JetStreamManager) { err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Keep: 1}) require_NoError(t, err) // Send 4 more messages. sendStreamMsg(t, nc, "TEST.1", "OK") sendStreamMsg(t, nc, "TEST.2", "OK") sendStreamMsg(t, nc, "TEST.3", "OK") sendStreamMsg(t, nc, "TEST.1", "OK") }) if si.State.Msgs != 5 { t.Fatalf("Expected 5 msgs after restart, got %d", si.State.Msgs) } if si.State.FirstSeq != 5 || si.State.LastSeq != 9 { t.Fatalf("Expected FirstSeq=5, LastSeq=9 after restart, got FirstSeq=%d, LastSeq=%d", si.State.FirstSeq, si.State.LastSeq) } // Now test a keep on a subject si = runTest(func(js nats.JetStreamManager) { err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: "TEST.1", Keep: 1}) require_NoError(t, err) // Send 3 more messages. sendStreamMsg(t, nc, "TEST.1", "OK") sendStreamMsg(t, nc, "TEST.2", "OK") sendStreamMsg(t, nc, "TEST.3", "OK") }) if si.State.Msgs != 7 { t.Fatalf("Expected 7 msgs after restart, got %d", si.State.Msgs) } if si.State.FirstSeq != 5 || si.State.LastSeq != 12 { t.Fatalf("Expected FirstSeq=5, LastSeq=12 after restart, got FirstSeq=%d, LastSeq=%d", si.State.FirstSeq, si.State.LastSeq) } } func TestJetStreamClusterConsumerCleanupWithSameName(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "UPDATES", Subjects: []string{"DEVICE.*"}, Replicas: 3, }) require_NoError(t, err) // Create a consumer that will be an R1 that we will auto-recreate but using the same name. // We want to make sure that the system does not continually try to cleanup the new one from the old one. // Track the sequence for restart etc. var seq atomic.Uint64 msgCB := func(msg *nats.Msg) { msg.AckSync() meta, err := msg.Metadata() require_NoError(t, err) seq.Store(meta.Sequence.Stream) } waitOnSeqDelivered := func(expected uint64) { checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { received := seq.Load() if received == expected { return nil } return fmt.Errorf("Seq is %d, want %d", received, expected) }) } doSub := func() { _, err = js.Subscribe( "DEVICE.22", msgCB, nats.ConsumerName("dlc"), nats.SkipConsumerLookup(), nats.StartSequence(seq.Load()+1), nats.MaxAckPending(1), // One at a time. nats.ManualAck(), nats.ConsumerReplicas(1), nats.ConsumerMemoryStorage(), nats.MaxDeliver(1), nats.InactiveThreshold(time.Second), nats.IdleHeartbeat(250*time.Millisecond), ) require_NoError(t, err) } // Track any errors for consumer not active so we can recreate the consumer. errCh := make(chan error, 10) nc.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) { if errors.Is(err, nats.ErrConsumerNotActive) { s.Unsubscribe() errCh <- err doSub() } }) doSub() sendStreamMsg(t, nc, "DEVICE.22", "update-1") sendStreamMsg(t, nc, "DEVICE.22", "update-2") sendStreamMsg(t, nc, "DEVICE.22", "update-3") waitOnSeqDelivered(3) // Shutdown the consumer's leader. s := c.consumerLeader(globalAccountName, "UPDATES", "dlc") s.Shutdown() c.waitOnStreamLeader(globalAccountName, "UPDATES") // In case our client connection was to the same server. nc, _ = jsClientConnect(t, c.randomServer()) defer nc.Close() sendStreamMsg(t, nc, "DEVICE.22", "update-4") sendStreamMsg(t, nc, "DEVICE.22", "update-5") sendStreamMsg(t, nc, "DEVICE.22", "update-6") // Wait for the consumer not active error. <-errCh // Now restart server with the old consumer. c.restartServer(s) // Wait on all messages delivered. waitOnSeqDelivered(6) // Make sure no other errors showed up require_True(t, len(errCh) == 0) } func TestJetStreamClusterConsumerActions(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() var err error _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"test"}, }) require_NoError(t, err) ecSubj := fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "CONSUMER", "test") crReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverPolicy: DeliverLast, FilterSubject: "test", AckPolicy: AckExplicit, }, } // A new consumer. Should not be an error. crReq.Action = ActionCreate req, err := json.Marshal(crReq) require_NoError(t, err) resp, err := nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected error: %v", ccResp.Error) } ccResp.Error = nil // Consumer exists, but config is the same, so should be ok resp, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected er response: %v", ccResp.Error) } ccResp.Error = nil // Consumer exists. Config is different, so should error crReq.Config.Description = "changed" req, err = json.Marshal(crReq) require_NoError(t, err) resp, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error == nil { t.Fatalf("Unexpected ok response") } ccResp.Error = nil // Consumer update, so update should be ok crReq.Action = ActionUpdate crReq.Config.Description = "changed again" req, err = json.Marshal(crReq) require_NoError(t, err) resp, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected error response: %v", ccResp.Error) } ecSubj = fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "NEW", "test") ccResp.Error = nil // Updating new consumer, so should error crReq.Config.Name = "NEW" req, err = json.Marshal(crReq) require_NoError(t, err) resp, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error == nil { t.Fatalf("Unexpected ok response") } } func TestJetStreamClusterSnapshotAndRestoreWithHealthz(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) toSend, msg := 1000, bytes.Repeat([]byte("Z"), 1024) for i := 0; i < toSend; i++ { _, err := js.PublishAsync("foo", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 512, } req, _ := json.Marshal(sreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) require_NoError(t, err) var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) require_True(t, resp.Error == nil) state := *resp.State cfg := *resp.Config var snapshot []byte done := make(chan bool) sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- true return } // Could be writing to a file here too. snapshot = append(snapshot, m.Data...) // Flow ack m.Respond(nil) }) defer sub.Unsubscribe() // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } // Delete before we try to restore. require_NoError(t, js.DeleteStream("TEST")) checkHealth := func() { for _, s := range c.servers { s.healthz(nil) } } var rresp JSApiStreamRestoreResponse rreq := &JSApiStreamRestoreRequest{ Config: cfg, State: state, } req, _ = json.Marshal(rreq) rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, "TEST"), req, 5*time.Second) require_NoError(t, err) rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) require_True(t, rresp.Error == nil) checkHealth() // We will now chunk the snapshot responses (and EOF). var chunk [1024]byte for i, r := 0, bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) i++ // We will call healthz for all servers half way through the restore. if i%100 == 0 { checkHealth() } } rmsg, err = nc.Request(rresp.DeliverSubject, nil, time.Second) require_NoError(t, err) rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) require_True(t, rresp.Error == nil) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == uint64(toSend)) // Make sure stepdown works, this would fail before the fix. _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, 5*time.Second) require_NoError(t, err) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == uint64(toSend)) // Now make sure if we try to restore to a single server that the artifact is cleaned up and the server returns ok for healthz. s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ = jsClientConnect(t, s) defer nc.Close() rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, "TEST"), req, 5*time.Second) require_NoError(t, err) rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) require_True(t, rresp.Error == nil) for i, r := 0, bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } _, err = nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) require_NoError(t, err) i++ } rmsg, err = nc.Request(rresp.DeliverSubject, nil, time.Second) require_NoError(t, err) rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) require_True(t, rresp.Error != nil) require_Equal(t, rresp.ApiResponse.Error.ErrCode, 10074) status := s.healthz(nil) require_Equal(t, status.StatusCode, 200) } func TestJetStreamClusterBinaryStreamSnapshotCapability(t *testing.T) { c := createJetStreamClusterExplicit(t, "NATS", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) mset, err := c.streamLeader(globalAccountName, "TEST").GlobalAccount().lookupStream("TEST") require_NoError(t, err) if !mset.supportsBinarySnapshot() { t.Fatalf("Expected to signal that we could support binary stream snapshots") } } func TestJetStreamClusterBadEncryptKey(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterEncryptedTempl, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Create 10 streams. for i := 0; i < 10; i++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST-%d", i), Replicas: 3, }) require_NoError(t, err) } // Grab random server. s := c.randomServer() s.Shutdown() s.WaitForShutdown() var opts *Options for i := 0; i < len(c.servers); i++ { if c.servers[i] == s { opts = c.opts[i] break } } require_NotNil(t, opts) // Replace key with an empty key. buf, err := os.ReadFile(opts.ConfigFile) require_NoError(t, err) nbuf := bytes.Replace(buf, []byte("key: \"s3cr3t!\""), []byte("key: \"\""), 1) err = os.WriteFile(opts.ConfigFile, nbuf, 0640) require_NoError(t, err) // Make sure trying to start the server now fails. s, err = NewServer(LoadConfig(opts.ConfigFile)) require_NoError(t, err) require_NotNil(t, s) s.Start() if err := s.readyForConnections(1 * time.Second); err == nil { t.Fatalf("Expected server not to start") } } func TestJetStreamClusterAccountUsageDrifts(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` opFrag := ` operator: %s system_account: %s resolver: { type: MEM } resolver_preload = { %s : %s %s : %s } ` sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: -1, Consumer: 1, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: -1, Consumer: 1, Streams: 1} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) template := tmpl + fmt.Sprintf(opFrag, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamClusterWithTemplate(t, template, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, MaxBytes: 1 * 1024 * 1024 * 1024, MaxMsgs: 1000, Replicas: 3, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"bar"}, }) require_NoError(t, err) // These expected store values can come directly from stream info's state bytes. // We will *= 3 for R3 checkAccount := func(r1u, r3u uint64) { t.Helper() r3u *= 3 // Remote usage updates can be delayed, so wait for a bit for values we want. checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { info, err := js.AccountInfo() require_NoError(t, err) require_True(t, len(info.Tiers) >= 2) // These can move. if u := info.Tiers["R1"].Store; u != r1u { return fmt.Errorf("Expected R1 to be %v, got %v", friendlyBytes(r1u), friendlyBytes(u)) } if u := info.Tiers["R3"].Store; u != r3u { return fmt.Errorf("Expected R3 to be %v, got %v", friendlyBytes(r3u), friendlyBytes(u)) } return nil }) } checkAccount(0, 0) // Now add in some R3 data. msg := bytes.Repeat([]byte("Z"), 32*1024) // 32k smallMsg := bytes.Repeat([]byte("Z"), 4*1024) // 4k for i := 0; i < 1000; i++ { js.Publish("foo", msg) } sir3, err := js.StreamInfo("TEST1") require_NoError(t, err) checkAccount(0, sir3.State.Bytes) // Now add in some R1 data. for i := 0; i < 100; i++ { js.Publish("bar", msg) } sir1, err := js.StreamInfo("TEST2") require_NoError(t, err) checkAccount(sir1.State.Bytes, sir3.State.Bytes) // We will now test a bunch of scenarios to see that we are doing accounting correctly. // Since our R3 has a limit of 1000 msgs, let's add in more msgs and drop older ones. for i := 0; i < 100; i++ { js.Publish("foo", smallMsg) } sir3, err = js.StreamInfo("TEST1") require_NoError(t, err) checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Move our R3 stream leader and make sure acounting is correct. _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST1"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(aExpPub, "TEST1") checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Now scale down. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, MaxBytes: 1 * 1024 * 1024 * 1024, MaxMsgs: 1000, Replicas: 1, }) require_NoError(t, err) checkAccount(sir1.State.Bytes+sir3.State.Bytes, 0) // Add in more msgs which will replace the older and bigger ones. for i := 0; i < 100; i++ { js.Publish("foo", smallMsg) } sir3, err = js.StreamInfo("TEST1") require_NoError(t, err) // Now scale back up. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, MaxBytes: 1 * 1024 * 1024 * 1024, MaxMsgs: 1000, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(aExpPub, "TEST1") checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Test Purge. err = js.PurgeStream("TEST1") require_NoError(t, err) checkAccount(sir1.State.Bytes, 0) for i := 0; i < 1000; i++ { js.Publish("foo", smallMsg) } sir3, err = js.StreamInfo("TEST1") require_NoError(t, err) checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Need system user here to move the leader. snc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(sysCreds)) defer snc.Close() requestLeaderStepDown := func() { ml := c.leader() checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { if cml := c.leader(); cml == ml { snc.Request(JSApiLeaderStepDown, nil, time.Second) return fmt.Errorf("Metaleader has not moved yet") } return nil }) } // Test meta leader stepdowns. for i := 0; i < len(c.servers); i++ { requestLeaderStepDown() checkAccount(sir1.State.Bytes, sir3.State.Bytes) } // Now test cluster reset operations where we internally reset the NRG and optionally the stream too. // Only applicable to TEST1 stream which is R3. nl := c.randomNonStreamLeader(aExpPub, "TEST1") acc, err := nl.LookupAccount(aExpPub) require_NoError(t, err) mset, err := acc.lookupStream("TEST1") require_NoError(t, err) // NRG only mset.resetClusteredState(nil) checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Need to re-lookup this stream since we will recreate from reset above. checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { mset, err = acc.lookupStream("TEST1") return err }) // Now NRG and Stream state itself. mset.resetClusteredState(errFirstSequenceMismatch) checkAccount(sir1.State.Bytes, sir3.State.Bytes) // Now test server restart for _, s := range c.servers { s.Shutdown() s.WaitForShutdown() s = c.restartServer(s) // Wait on healthz and leader etc. checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { if hs := s.healthz(nil); hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) c.waitOnLeader() c.waitOnStreamLeader(aExpPub, "TEST1") c.waitOnStreamLeader(aExpPub, "TEST2") // Now check account again. checkAccount(sir1.State.Bytes, sir3.State.Bytes) } } func TestJetStreamClusterStreamFailTracking(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) m := nats.NewMsg("foo") m.Data = []byte("OK") b, bsz := 0, 5 sendBatch := func() { for i := b * bsz; i < b*bsz+bsz; i++ { msgId := fmt.Sprintf("ID:%d", i) m.Header.Set(JSMsgId, msgId) // Send it twice on purpose. js.PublishMsg(m) js.PublishMsg(m) } b++ } sendBatch() _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") sendBatch() // Now stop one and restart. nl := c.randomNonStreamLeader(globalAccountName, "TEST") mset, err := nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) // Reset raft mset.resetClusteredState(nil) time.Sleep(100 * time.Millisecond) nl.Shutdown() nl.WaitForShutdown() sendBatch() nl = c.restartServer(nl) sendBatch() for { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") if nl == c.streamLeader(globalAccountName, "TEST") { break } } sendBatch() _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) // Make sure all in order. errCh := make(chan error, 100) var wg sync.WaitGroup wg.Add(1) expected, seen := b*bsz, 0 sub, err := js.Subscribe("foo", func(msg *nats.Msg) { expectedID := fmt.Sprintf("ID:%d", seen) if v := msg.Header.Get(JSMsgId); v != expectedID { errCh <- err wg.Done() msg.Sub.Unsubscribe() return } seen++ if seen >= expected { wg.Done() msg.Sub.Unsubscribe() } }) require_NoError(t, err) defer sub.Unsubscribe() wg.Wait() if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } } func TestJetStreamClusterStreamFailTrackingSnapshots(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) m := nats.NewMsg("foo") m.Data = []byte("OK") // Send 1000 a dupe every msgID. for i := 0; i < 1000; i++ { msgId := fmt.Sprintf("ID:%d", i) m.Header.Set(JSMsgId, msgId) // Send it twice on purpose. js.PublishMsg(m) js.PublishMsg(m) } // Now stop one. nl := c.randomNonStreamLeader(globalAccountName, "TEST") nl.Shutdown() nl.WaitForShutdown() // Now send more and make sure leader snapshots. for i := 1000; i < 2000; i++ { msgId := fmt.Sprintf("ID:%d", i) m.Header.Set(JSMsgId, msgId) // Send it twice on purpose. js.PublishMsg(m) js.PublishMsg(m) } sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) node := mset.raftNode() require_NotNil(t, node) node.InstallSnapshot(mset.stateSnapshot()) // Now restart nl nl = c.restartServer(nl) c.waitOnServerCurrent(nl) // Move leader to NL for { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") if nl == c.streamLeader(globalAccountName, "TEST") { break } } _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) // Make sure all in order. errCh := make(chan error, 100) var wg sync.WaitGroup wg.Add(1) expected, seen := 2000, 0 sub, err := js.Subscribe("foo", func(msg *nats.Msg) { expectedID := fmt.Sprintf("ID:%d", seen) if v := msg.Header.Get(JSMsgId); v != expectedID { errCh <- err wg.Done() msg.Sub.Unsubscribe() return } seen++ if seen >= expected { wg.Done() msg.Sub.Unsubscribe() } }) require_NoError(t, err) defer sub.Unsubscribe() wg.Wait() if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } } func TestJetStreamClusterOrphanConsumerSubjects(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>", "bar.>"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "consumer_foo", Durable: "consumer_foo", FilterSubject: "foo.something", }) require_NoError(t, err) for _, replicas := range []int{3, 1, 3} { _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"bar.>"}, Replicas: replicas, }) require_NoError(t, err) c.waitOnAllCurrent() } c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "consumer_foo") info, err := js.ConsumerInfo("TEST", "consumer_foo") require_NoError(t, err) require_True(t, info.Cluster != nil) require_NotEqual(t, info.Cluster.Leader, "") require_Equal(t, len(info.Cluster.Replicas), 2) } func TestJetStreamClusterDurableConsumerInactiveThresholdLeaderSwitch(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) // Queue a msg. sendStreamMsg(t, nc, "foo", "ok") thresh := 250 * time.Millisecond // This will start the timer. sub, err := js.PullSubscribe("foo", "dlc", nats.InactiveThreshold(thresh)) require_NoError(t, err) // Switch over leader. cl := c.consumerLeader(globalAccountName, "TEST", "dlc") cl.JetStreamStepdownConsumer(globalAccountName, "TEST", "dlc") c.waitOnConsumerLeader(globalAccountName, "TEST", "dlc") // Create activity on this consumer. msgs, err := sub.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) // This is consider activity as well. So we can watch now up to thresh to make sure consumer still active. msgs[0].AckSync() // The consumer should not disappear for next `thresh` interval unless old leader does so. timeout := time.Now().Add(thresh) for time.Now().Before(timeout) { _, err := js.ConsumerInfo("TEST", "dlc") if err == nats.ErrConsumerNotFound { t.Fatalf("Consumer deleted when it should not have been") } } } func TestJetStreamClusterConsumerMaxDeliveryNumAckPendingBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) // send 50 msgs for i := 0; i < 50; i++ { _, err := js.Publish("foo", []byte("ok")) require_NoError(t, err) } // File based. sub, err := js.PullSubscribe("foo", "file", nats.ManualAck(), nats.MaxDeliver(1), nats.AckWait(time.Second), nats.MaxAckPending(10), ) require_NoError(t, err) msgs, err := sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) // Let first batch expire. time.Sleep(1200 * time.Millisecond) cia, err := js.ConsumerInfo("TEST", "file") require_NoError(t, err) // Make sure followers will have exact same state. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "file"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "file") cib, err := js.ConsumerInfo("TEST", "file") require_NoError(t, err) // Want to compare sans cluster details which we know will change due to leader change. // Also last activity for delivered can be slightly off so nil out as well. checkConsumerInfo := func(a, b *nats.ConsumerInfo, replicated bool) { t.Helper() require_Equal(t, a.Delivered.Consumer, 10) require_Equal(t, a.Delivered.Stream, 10) // If replicated, agreed upon state is used. Otherwise, o.asflr and o.adflr would be skipped ahead for R1. if replicated { require_Equal(t, a.AckFloor.Consumer, 0) require_Equal(t, a.AckFloor.Stream, 0) } else { require_Equal(t, a.AckFloor.Consumer, 10) require_Equal(t, a.AckFloor.Stream, 10) } require_Equal(t, a.NumPending, 40) require_Equal(t, a.NumRedelivered, 10) a.Cluster, b.Cluster = nil, nil a.Delivered.Last, b.Delivered.Last = nil, nil if !reflect.DeepEqual(a, b) { t.Fatalf("ConsumerInfo do not match\n\t%+v\n\t%+v", a, b) } } checkConsumerInfo(cia, cib, true) // Memory based. sub, err = js.PullSubscribe("foo", "mem", nats.ManualAck(), nats.MaxDeliver(1), nats.AckWait(time.Second), nats.MaxAckPending(10), nats.ConsumerMemoryStorage(), ) require_NoError(t, err) msgs, err = sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) // Let first batch retry and expire. time.Sleep(1200 * time.Millisecond) cia, err = js.ConsumerInfo("TEST", "mem") require_NoError(t, err) // Make sure followers will have exact same state. _, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "TEST", "mem"), nil, time.Second) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "TEST", "mem") cib, err = js.ConsumerInfo("TEST", "mem") require_NoError(t, err) checkConsumerInfo(cia, cib, true) // Now file based but R1 and server restart. sub, err = js.PullSubscribe("foo", "r1", nats.ManualAck(), nats.MaxDeliver(1), nats.AckWait(time.Second), nats.MaxAckPending(10), nats.ConsumerReplicas(1), ) require_NoError(t, err) msgs, err = sub.Fetch(10) require_NoError(t, err) require_Equal(t, len(msgs), 10) // Let first batch retry and expire. time.Sleep(1200 * time.Millisecond) cia, err = js.ConsumerInfo("TEST", "r1") require_NoError(t, err) cl := c.consumerLeader(globalAccountName, "TEST", "r1") cl.Shutdown() cl.WaitForShutdown() cl = c.restartServer(cl) c.waitOnServerCurrent(cl) cib, err = js.ConsumerInfo("TEST", "r1") require_NoError(t, err) // Created can skew a small bit due to server restart, this is expected. now := time.Now() cia.Created, cib.Created = now, now checkConsumerInfo(cia, cib, false) } func TestJetStreamClusterConsumerDefaultsFromStream(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() streamTmpl := &StreamConfig{ Name: "test", Subjects: []string{"test.*"}, Storage: MemoryStorage, ConsumerLimits: StreamConsumerLimits{ MaxAckPending: 0, InactiveThreshold: 0, }, } // Since nats.go doesn't yet know about the consumer limits, craft // the stream configuration request by hand. streamCreate := func(maxAckPending int, inactiveThreshold time.Duration) (*StreamConfig, error) { cfg := streamTmpl cfg.ConsumerLimits = StreamConsumerLimits{ MaxAckPending: maxAckPending, InactiveThreshold: inactiveThreshold, } j, err := json.Marshal(cfg) if err != nil { return nil, err } msg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, "test"), j, time.Second*3) if err != nil { return nil, err } var resp JSApiStreamCreateResponse if err := json.Unmarshal(msg.Data, &resp); err != nil { return nil, err } if resp.StreamInfo == nil { return nil, resp.ApiResponse.ToError() } return &resp.Config, resp.ApiResponse.ToError() } streamUpdate := func(maxAckPending int, inactiveThreshold time.Duration) (*StreamConfig, error) { cfg := streamTmpl cfg.ConsumerLimits = StreamConsumerLimits{ MaxAckPending: maxAckPending, InactiveThreshold: inactiveThreshold, } j, err := json.Marshal(cfg) if err != nil { return nil, err } msg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, "test"), j, time.Second*3) if err != nil { return nil, err } var resp JSApiStreamUpdateResponse if err := json.Unmarshal(msg.Data, &resp); err != nil { return nil, err } if resp.StreamInfo == nil { return nil, resp.ApiResponse.ToError() } return &resp.Config, resp.ApiResponse.ToError() } if _, err := streamCreate(15, time.Second); err != nil { t.Fatalf("Failed to add stream: %v", err) } t.Run("InheritDefaultsFromStream", func(t *testing.T) { ci, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "InheritDefaultsFromStream", }) require_NoError(t, err) switch { case ci.Config.InactiveThreshold != time.Second: t.Fatalf("InactiveThreshold should be 1s, got %s", ci.Config.InactiveThreshold) case ci.Config.MaxAckPending != 15: t.Fatalf("MaxAckPending should be 15, got %d", ci.Config.MaxAckPending) } }) t.Run("CreateConsumerErrorOnExceedMaxAckPending", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "CreateConsumerErrorOnExceedMaxAckPending", MaxAckPending: 30, }) switch e := err.(type) { case *nats.APIError: if ErrorIdentifier(e.ErrorCode) != JSConsumerMaxPendingAckExcessErrF { t.Fatalf("invalid error code, got %d, wanted %d", e.ErrorCode, JSConsumerMaxPendingAckExcessErrF) } default: t.Fatalf("should have returned API error, got %T", e) } }) t.Run("CreateConsumerErrorOnExceedInactiveThreshold", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "CreateConsumerErrorOnExceedInactiveThreshold", InactiveThreshold: time.Second * 2, }) switch e := err.(type) { case *nats.APIError: if ErrorIdentifier(e.ErrorCode) != JSConsumerInactiveThresholdExcess { t.Fatalf("invalid error code, got %d, wanted %d", e.ErrorCode, JSConsumerInactiveThresholdExcess) } default: t.Fatalf("should have returned API error, got %T", e) } }) t.Run("UpdateStreamErrorOnViolateConsumerMaxAckPending", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "UpdateStreamErrorOnViolateConsumerMaxAckPending", MaxAckPending: 15, }) require_NoError(t, err) if _, err = streamUpdate(10, 0); err == nil { t.Fatalf("stream update should have errored but didn't") } }) t.Run("UpdateStreamErrorOnViolateConsumerInactiveThreshold", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "UpdateStreamErrorOnViolateConsumerInactiveThreshold", InactiveThreshold: time.Second, }) require_NoError(t, err) if _, err = streamUpdate(0, time.Second/2); err == nil { t.Fatalf("stream update should have errored but didn't") } }) } // Discovered that we are not properly setting certain default filestore blkSizes. func TestJetStreamClusterCheckFileStoreBlkSizes(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Normal Stream _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C3", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // KV _, err = js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, }) require_NoError(t, err) blkSize := func(fs *fileStore) uint64 { fs.mu.RLock() defer fs.mu.RUnlock() return fs.fcfg.BlockSize } // We will check now the following filestores. // meta // TEST stream and NRG // C3 NRG // KV_TEST stream and NRG for _, s := range c.servers { js, cc := s.getJetStreamCluster() // META js.mu.RLock() meta := cc.meta js.mu.RUnlock() require_True(t, meta != nil) fs := meta.(*raft).wal.(*fileStore) require_True(t, blkSize(fs) == defaultMetaFSBlkSize) // TEST STREAM mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() fs = mset.store.(*fileStore) mset.mu.RUnlock() require_True(t, blkSize(fs) == defaultLargeBlockSize) // KV STREAM // Now the KV which is different default size. kv, err := s.GlobalAccount().lookupStream("KV_TEST") require_NoError(t, err) kv.mu.RLock() fs = kv.store.(*fileStore) kv.mu.RUnlock() require_True(t, blkSize(fs) == defaultKVBlockSize) // Now check NRGs // TEST Stream n := mset.raftNode() require_True(t, n != nil) fs = n.(*raft).wal.(*fileStore) require_True(t, blkSize(fs) == defaultMediumBlockSize) // KV TEST Stream n = kv.raftNode() require_True(t, n != nil) fs = n.(*raft).wal.(*fileStore) require_True(t, blkSize(fs) == defaultMediumBlockSize) // Consumer o := mset.lookupConsumer("C3") require_True(t, o != nil) n = o.raftNode() require_True(t, n != nil) fs = n.(*raft).wal.(*fileStore) require_True(t, blkSize(fs) == defaultMediumBlockSize) } } func TestJetStreamClusterDetectOrphanNRGs(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Normal Stream _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "DC", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // We will force an orphan for a certain server. s := c.randomNonStreamLeader(globalAccountName, "TEST") mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) sgn := mset.raftNode().Group() mset.clearRaftNode() o := mset.lookupConsumer("DC") require_True(t, o != nil) ogn := o.raftNode().Group() o.clearRaftNode() require_NoError(t, js.DeleteStream("TEST")) // Check that we do in fact have orphans. require_True(t, s.numRaftNodes() > 1) // This function will detect orphans and clean them up. s.checkForNRGOrphans() // Should only be meta NRG left. require_True(t, s.numRaftNodes() == 1) s.rnMu.RLock() defer s.rnMu.RUnlock() require_True(t, s.lookupRaftNode(sgn) == nil) require_True(t, s.lookupRaftNode(ogn) == nil) } func TestJetStreamClusterRestartThenScaleStreamReplicas(t *testing.T) { t.Skip("This test takes too long, need to make shorter") c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() nc2, producer := jsClientConnect(t, s) defer nc2.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") ctx, cancel := context.WithCancel(context.Background()) defer cancel() end := time.Now().Add(2 * time.Second) for time.Now().Before(end) { producer.Publish("foo", []byte(strings.Repeat("A", 128))) time.Sleep(time.Millisecond) } var wg sync.WaitGroup for i := 0; i < 5; i++ { sub, err := js.PullSubscribe("foo", fmt.Sprintf("C-%d", i)) require_NoError(t, err) wg.Add(1) go func() { defer wg.Done() for range time.NewTicker(10 * time.Millisecond).C { select { case <-ctx.Done(): return default: } msgs, err := sub.Fetch(1) if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, nats.ErrConnectionClosed) { t.Logf("Pull Error: %v", err) } for _, msg := range msgs { msg.Ack() } } }() } c.lameDuckRestartAll() c.waitOnStreamLeader(globalAccountName, "TEST") // Swap the logger to try to detect the condition after the restart. loggers := make([]*captureDebugLogger, 3) for i, srv := range c.servers { l := &captureDebugLogger{dbgCh: make(chan string, 10)} loggers[i] = l srv.SetLogger(l, true, false) } condition := `Direct proposal ignored, not leader (state: CLOSED)` errCh := make(chan error, 10) wg.Add(1) go func() { defer wg.Done() for { select { case dl := <-loggers[0].dbgCh: if strings.Contains(dl, condition) { errCh <- errors.New(condition) } case dl := <-loggers[1].dbgCh: if strings.Contains(dl, condition) { errCh <- errors.New(condition) } case dl := <-loggers[2].dbgCh: if strings.Contains(dl, condition) { errCh <- errors.New(condition) } case <-ctx.Done(): return } } }() // Start publishing again for a while. end = time.Now().Add(2 * time.Second) for time.Now().Before(end) { producer.Publish("foo", []byte(strings.Repeat("A", 128))) time.Sleep(time.Millisecond) } // Try to do a stream edit back to R=1 after doing all the upgrade. info, _ := js.StreamInfo("TEST") sconfig := info.Config sconfig.Replicas = 1 _, err = js.UpdateStream(&sconfig) require_NoError(t, err) // Leave running for some time after the update. time.Sleep(2 * time.Second) info, _ = js.StreamInfo("TEST") sconfig = info.Config sconfig.Replicas = 3 _, err = js.UpdateStream(&sconfig) require_NoError(t, err) select { case e := <-errCh: t.Fatalf("Bad condition on raft node: %v", e) case <-time.After(2 * time.Second): // Done } // Stop goroutines and wait for them to exit. cancel() wg.Wait() } // https://github.com/nats-io/nats-server/issues/4732 func TestJetStreamClusterStreamLimitsOnScaleUpAndMove(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` opFrag := ` operator: %s system_account: %s resolver: { type: MEM } resolver_preload = { %s : %s %s : %s } ` _, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: -1, Consumer: -1, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 0, Consumer: -1, Streams: 1} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) template := tmpl + fmt.Sprintf(opFrag, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamCluster(t, template, "CLOUD", _EMPTY_, 3, 22020, true) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) toSend, msg := 100, bytes.Repeat([]byte("Z"), 1024) for i := 0; i < toSend; i++ { _, err := js.PublishAsync("foo", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Scale up should fail here since no R3 storage. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_Error(t, err, errors.New("insufficient storage resources")) } func TestJetStreamClusterAPIAccessViaSystemAccount(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Connect to system account. nc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}) require_Error(t, err, NewJSNotEnabledForAccountError()) // Make sure same behavior swith single server. tmpl := ` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js = jsClientConnect(t, s, nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() _, err = js.AddStream(&nats.StreamConfig{Name: "TEST"}) require_Error(t, err, NewJSNotEnabledForAccountError()) } func TestJetStreamClusterStreamResetPreacks(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.InterestPolicy, Replicas: 3, }) require_NoError(t, err) err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100_000_000}) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "dlc") require_NoError(t, err) // Put 20 msgs in. for i := 0; i < 20; i++ { _, err := js.Publish("foo", nil) require_NoError(t, err) } // Consume and ack 10. msgs, err := sub.Fetch(10, nats.MaxWait(time.Second)) require_NoError(t, err) require_Equal(t, len(msgs), 10) for _, msg := range msgs { msg.AckSync() } // Now grab a non-leader server. // We will shut it down and remove the stream data. nl := c.randomNonStreamLeader(globalAccountName, "TEST") mset, err := nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) fs := mset.store.(*fileStore) mdir := filepath.Join(fs.fcfg.StoreDir, msgDir) nl.Shutdown() // In case that was the consumer leader. c.waitOnConsumerLeader(globalAccountName, "TEST", "dlc") // Now consume the remaining 10 and ack. msgs, err = sub.Fetch(10, nats.MaxWait(time.Second)) require_NoError(t, err) require_Equal(t, len(msgs), 10) for _, msg := range msgs { msg.AckSync() } // Now remove the stream manually. require_NoError(t, os.RemoveAll(mdir)) nl = c.restartServer(nl) c.waitOnServerCurrent(nl) mset, err = nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { state := mset.state() if state.Msgs != 0 || state.FirstSeq != 100_000_020 { return fmt.Errorf("Not correct state yet: %+v", state) } return nil }) } func TestJetStreamClusterDomainAdvisory(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: NGS, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "R3S", _EMPTY_, 3, 18033, true) defer c.shutdown() // Connect to system account. nc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() sub, err := nc.SubscribeSync(JSAdvisoryDomainLeaderElected) require_NoError(t, err) // Ask meta leader to move and make sure we get an advisory. nc.Request(JSApiLeaderStepDown, nil, time.Second) c.waitOnLeader() checkSubsPending(t, sub, 1) m, err := sub.NextMsg(time.Second) require_NoError(t, err) var adv JSDomainLeaderElectedAdvisory require_NoError(t, json.Unmarshal(m.Data, &adv)) ml := c.leader() js, cc := ml.getJetStreamCluster() js.mu.RLock() peer := cc.meta.ID() js.mu.RUnlock() require_Equal(t, adv.Leader, peer) require_Equal(t, adv.Domain, "NGS") require_Equal(t, adv.Cluster, "R3S") require_Equal(t, len(adv.Replicas), 3) } func TestJetStreamClusterLimitsBasedStreamFileStoreDesync(t *testing.T) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } system_account: sys no_auth_user: js accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream = { store_max_stream_bytes = 3mb } users = [ { user: js, pass: js } ] } }` c := createJetStreamClusterWithTemplate(t, conf, "limits", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cnc, cjs := jsClientConnect(t, c.randomServer()) defer cnc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "LTEST", Subjects: []string{"messages.*"}, Replicas: 3, MaxAge: 10 * time.Minute, MaxMsgs: 100_000, }) require_NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() psub, err := cjs.PullSubscribe("messages.*", "consumer") require_NoError(t, err) var ( wg sync.WaitGroup received uint64 errCh = make(chan error, 100_000) receivedMap = make(map[string]*nats.Msg) ) wg.Add(1) go func() { tick := time.NewTicker(20 * time.Millisecond) for { select { case <-ctx.Done(): wg.Done() return case <-tick.C: msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { received++ receivedMap[msg.Subject] = msg if meta, _ := msg.Metadata(); meta.NumDelivered > 1 { t.Logf("GOT MSG: %s :: %+v :: %d", msg.Subject, meta, len(msg.Data)) } msg.Ack() } } } }() // Send 20_000 msgs at roughly 1 msg per msec shouldDrop := make(map[string]error) wg.Add(1) go func() { payload := []byte(strings.Repeat("A", 1024)) tick := time.NewTicker(1 * time.Millisecond) for i := 1; i < 100_000; { select { case <-ctx.Done(): wg.Done() return case <-tick.C: // This should run into 3MB quota and get errors right away // before the max msgs limit does. subject := fmt.Sprintf("messages.%d", i) _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) if err != nil { errCh <- err } i++ // Any message over this number should not be a success // since the stream should be full due to the quota. // Here we capture that the messages have failed to confirm. if err != nil && i > 1000 { shouldDrop[subject] = err } } } }() // Collect enough errors to cause things to get out of sync. var errCount int Setup: for { select { case err = <-errCh: errCount++ if errCount >= 20_000 { // Stop both producing and consuming. cancel() break Setup } case <-time.After(5 * time.Second): t.Fatalf("Timed out waiting for limits error") } } // Both goroutines should be exiting now.. wg.Wait() // Check messages that ought to have been dropped. for subject := range receivedMap { found, ok := shouldDrop[subject] if ok { t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) } } getStreamDetails := func(t *testing.T, srv *Server) *StreamDetail { t.Helper() jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) require_NoError(t, err) if len(jsz.AccountDetails) > 0 && len(jsz.AccountDetails[0].Streams) > 0 { details := jsz.AccountDetails[0] stream := details.Streams[0] return &stream } t.Error("Could not find account details") return nil } checkState := func(t *testing.T) error { t.Helper() leaderSrv := c.streamLeader("js", "LTEST") streamLeader := getStreamDetails(t, leaderSrv) // t.Logf("Stream Leader: %+v", streamLeader.State) errs := make([]error, 0) for _, srv := range c.servers { if srv == leaderSrv { // Skip self continue } stream := getStreamDetails(t, srv) if stream.State.Msgs != streamLeader.State.Msgs { err := fmt.Errorf("Leader %v has %d messages, Follower %v has %d messages", stream.Cluster.Leader, streamLeader.State.Msgs, srv.Name(), stream.State.Msgs, ) errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Confirm state of the leader. leaderSrv := c.streamLeader("js", "LTEST") streamLeader := getStreamDetails(t, leaderSrv) if streamLeader.State.Msgs != received { t.Errorf("Leader %v has %d messages stored but %d messages were received (delta: %d)", leaderSrv.Name(), streamLeader.State.Msgs, received, received-streamLeader.State.Msgs) } cinfo, err := psub.ConsumerInfo() require_NoError(t, err) if received != cinfo.Delivered.Consumer { t.Errorf("Unexpected consumer sequence. Got: %v, expected: %v", cinfo.Delivered.Consumer, received) } // Check whether there was a drift among the leader and followers. var ( lastErr error attempts int ) Check: for range time.NewTicker(1 * time.Second).C { lastErr = checkState(t) if attempts > 5 { break Check } attempts++ } // Read the stream psub2, err := cjs.PullSubscribe("messages.*", "") require_NoError(t, err) Consume2: for { msgs, err := psub2.Fetch(100) if err != nil { continue } for _, msg := range msgs { msg.Ack() meta, _ := msg.Metadata() if meta.NumPending == 0 { break Consume2 } } } cinfo2, err := psub2.ConsumerInfo() require_NoError(t, err) a := cinfo.Delivered.Consumer b := cinfo2.Delivered.Consumer if a != b { t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) } // Test is done but replicas were in sync so can stop testing at this point. if lastErr == nil { return } // Now we will cause a few step downs while out of sync to get different results. t.Errorf("Replicas are out of sync:\n%v", lastErr) stepDown := func() { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "LTEST"), nil, time.Second) } // Check StreamInfo in this state then trigger a few step downs. var prevLeaderMsgs uint64 leaderSrv = c.streamLeader("js", "LTEST") sinfo, err := js.StreamInfo("LTEST") prevLeaderMsgs = sinfo.State.Msgs for i := 0; i < 10; i++ { stepDown() time.Sleep(2 * time.Second) leaderSrv = c.streamLeader("js", "LTEST") sinfo, err = js.StreamInfo("LTEST") if err != nil { t.Logf("Error: %v", err) continue } if leaderSrv != nil && sinfo != nil { t.Logf("When leader is %v, Messages: %d", leaderSrv.Name(), sinfo.State.Msgs) // Leave the leader as the replica with less messages that was out of sync. if prevLeaderMsgs > sinfo.State.Msgs { break } } } t.Logf("Changed to use leader %v which has %d messages", leaderSrv.Name(), sinfo.State.Msgs) // Read the stream again psub3, err := cjs.PullSubscribe("messages.*", "") require_NoError(t, err) Consume3: for { msgs, err := psub3.Fetch(100) if err != nil { continue } for _, msg := range msgs { msg.Ack() meta, _ := msg.Metadata() if meta.NumPending == 0 { break Consume3 } } } cinfo3, err := psub3.ConsumerInfo() require_NoError(t, err) // Compare against consumer that was created before resource limits error // with one created before the step down. a = cinfo.Delivered.Consumer b = cinfo2.Delivered.Consumer if a != b { t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) } // Compare against consumer that was created before resource limits error // with one created AFTER the step down. a = cinfo.Delivered.Consumer b = cinfo3.Delivered.Consumer if a != b { t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) } // Compare consumers created after the resource limits error. a = cinfo2.Delivered.Consumer b = cinfo3.Delivered.Consumer if a != b { t.Errorf("Consumers to same stream are at different sequences: %d vs %d", a, b) } } func TestJetStreamClusterAccountFileStoreLimits(t *testing.T) { c := createJetStreamClusterExplicit(t, "limits", 3) defer c.shutdown() limits := map[string]JetStreamAccountLimits{ "R1": { MaxMemory: 1 << 10, MaxStore: 1 << 10, MaxStreams: -1, MaxConsumers: -1, }, "R3": { MaxMemory: 1 << 10, MaxStore: 1 << 10, MaxStreams: -1, MaxConsumers: -1, }, } // Update the limits in all servers. for _, s := range c.servers { acc := s.GlobalAccount() if err := acc.UpdateJetStreamLimits(limits); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } } nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for _, replicas := range []int64{1, 3} { sname := fmt.Sprintf("test-stream:%d", replicas) t.Run(sname, func(t *testing.T) { sconfig := &nats.StreamConfig{ Name: sname, Replicas: int(replicas), Storage: nats.FileStorage, Retention: nats.LimitsPolicy, } _, err := js.AddStream(sconfig) if err != nil { t.Fatalf("Unexpected error creating stream: %v", err) } data := []byte(strings.Repeat("A", 1<<8)) for i := 0; i < 30; i++ { if _, err = js.Publish(sname, data); err != nil && !strings.Contains(err.Error(), "resource limits exceeded for account") { t.Errorf("Error publishing random data (iteration %d): %v", i, err) } if err = nc.Flush(); err != nil { t.Fatalf("Unexpected error flushing connection: %v", err) } _, err = js.StreamInfo(sname) require_NoError(t, err) } si, err := js.StreamInfo(sname) require_NoError(t, err) st := si.State maxStore := limits[fmt.Sprintf("R%d", replicas)].MaxStore if int64(st.Bytes) > replicas*maxStore { t.Errorf("Unexpected size of stream: got %d, expected less than %d\nstate: %#v", st.Bytes, maxStore, st) } }) } } nats-server-2.10.27/server/jetstream_cluster_4_test.go000066400000000000000000003757331477524627100230530ustar00rootroot00000000000000// Copyright 2022-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests_4 // +build !skip_js_tests,!skip_js_cluster_tests_4 package server import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "math/rand" "os" "path" "path/filepath" "runtime" "slices" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" ) func TestJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T) { t.Run("max msgs", func(t *testing.T) { testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ Name: "WQTEST_MM", Subjects: []string{"messages.*"}, Replicas: 3, MaxAge: 10 * time.Minute, MaxMsgs: 100, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardNew, }) }) t.Run("max bytes", func(t *testing.T) { testJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{ Name: "WQTEST_MB", Subjects: []string{"messages.*"}, Replicas: 3, MaxAge: 10 * time.Minute, MaxBytes: 1 * 1024 * 1024, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardNew, }) }) } func testJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T, sc *nats.StreamConfig) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } system_account: sys no_auth_user: js accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream = enabled users = [ { user: js, pass: js } ] } }` c := createJetStreamClusterWithTemplate(t, conf, sc.Name, 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cnc, cjs := jsClientConnect(t, c.randomServer()) defer cnc.Close() _, err := js.AddStream(sc) require_NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() psub, err := cjs.PullSubscribe("messages.*", "consumer") require_NoError(t, err) stepDown := func() { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Name), nil, time.Second) } // Messages will be produced and consumed in parallel, then once there are // enough errors a leader election will be triggered. var ( wg sync.WaitGroup received uint64 errCh = make(chan error, 100_000) receivedMap = make(map[string]*nats.Msg) ) wg.Add(1) go func() { tick := time.NewTicker(20 * time.Millisecond) for { select { case <-ctx.Done(): wg.Done() return case <-tick.C: msgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond)) if err != nil { // The consumer will continue to timeout here eventually. continue } for _, msg := range msgs { received++ receivedMap[msg.Subject] = msg msg.Ack() } } } }() shouldDrop := make(map[string]error) wg.Add(1) go func() { payload := []byte(strings.Repeat("A", 1024)) tick := time.NewTicker(1 * time.Millisecond) for i := 1; ; i++ { select { case <-ctx.Done(): wg.Done() return case <-tick.C: subject := fmt.Sprintf("messages.%d", i) _, err := js.Publish(subject, payload, nats.RetryAttempts(0)) if err != nil { errCh <- err } // Capture the messages that have failed. if err != nil { shouldDrop[subject] = err } } } }() // Collect enough errors to cause things to get out of sync. var errCount int Setup: for { select { case err = <-errCh: errCount++ if errCount%500 == 0 { stepDown() } else if errCount >= 2000 { // Stop both producing and consuming. cancel() break Setup } case <-time.After(5 * time.Second): // Unblock the test and continue. cancel() break Setup } } // Both goroutines should be exiting now.. wg.Wait() // Let acks propagate for stream checks. time.Sleep(250 * time.Millisecond) // Check messages that ought to have been dropped. for subject := range receivedMap { found, ok := shouldDrop[subject] if ok { t.Errorf("Should have dropped message published on %q since got error: %v", subject, found) } } } // https://github.com/nats-io/nats-server/issues/5071 func TestJetStreamClusterStreamPlacementDistribution(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 5) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() for i := 1; i <= 10; i++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST:%d", i), Subjects: []string{fmt.Sprintf("foo.%d.*", i)}, Replicas: 3, }) require_NoError(t, err) } // 10 streams, 3 replicas div 5 servers. expectedStreams := 10 * 3 / 5 for _, s := range c.servers { jsz, err := s.Jsz(nil) require_NoError(t, err) require_Equal(t, jsz.Streams, expectedStreams) } } func TestJetStreamClusterSourceWorkingQueueWithLimit(t *testing.T) { c := createJetStreamClusterExplicit(t, "WQ3", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "test", Subjects: []string{"test"}, Replicas: 3}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "wq", MaxMsgs: 100, Discard: nats.DiscardNew, Retention: nats.WorkQueuePolicy, Sources: []*nats.StreamSource{{Name: "test"}}, Replicas: 3}) require_NoError(t, err) sendBatch := func(subject string, n int) { for i := 0; i < n; i++ { _, err = js.Publish(subject, []byte(strconv.Itoa(i))) require_NoError(t, err) } } // Populate each one. sendBatch("test", 300) checkFor(t, 3*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("wq") require_NoError(t, err) if si.State.Msgs != 100 { return fmt.Errorf("Expected 100 msgs, got state: %+v", si.State) } return nil }) _, err = js.AddConsumer("wq", &nats.ConsumerConfig{Durable: "wqc", FilterSubject: "test", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ss, err := js.PullSubscribe("test", "wqc", nats.Bind("wq", "wqc")) require_NoError(t, err) // we must have at least one message on the transformed subject name (ie no timeout) f := func(done chan bool) { for i := 0; i < 300; i++ { m, err := ss.Fetch(1, nats.MaxWait(3*time.Second)) require_NoError(t, err) p, err := strconv.Atoi(string(m[0].Data)) require_NoError(t, err) require_Equal(t, p, i) time.Sleep(11 * time.Millisecond) err = m[0].Ack() require_NoError(t, err) } done <- true } var doneChan = make(chan bool) go f(doneChan) checkFor(t, 6*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("wq") require_NoError(t, err) if si.State.Msgs > 0 && si.State.Msgs <= 100 { return fmt.Errorf("Expected 0 msgs, got: %d", si.State.Msgs) } else if si.State.Msgs > 100 { t.Fatalf("Got more than our 100 message limit: %+v", si.State) } return nil }) select { case <-doneChan: ss.Drain() case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } } func TestJetStreamClusterStreamOrphanMsgsAndReplicasDrifting(t *testing.T) { type testParams struct { restartAny bool restartLeader bool rolloutRestart bool ldmRestart bool restarts int checkHealthz bool reconnectRoutes bool reconnectClients bool } test := func(t *testing.T, params *testParams, sc *nats.StreamConfig) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } server_tags: ["test"] system_account: sys no_auth_user: js accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream = enabled users = [ { user: js, pass: js } ] } }` c := createJetStreamClusterWithTemplate(t, conf, sc.Name, 3) defer c.shutdown() // Update lame duck duration for all servers. for _, s := range c.servers { s.optsMu.Lock() s.opts.LameDuckDuration = 5 * time.Second s.opts.LameDuckGracePeriod = -5 * time.Second s.optsMu.Unlock() } nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cnc, cjs := jsClientConnect(t, c.randomServer()) defer cnc.Close() _, err := js.AddStream(sc) require_NoError(t, err) pctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() // Start producers var wg sync.WaitGroup // First call is just to create the pull subscribers. mp := nats.MaxAckPending(10000) mw := nats.PullMaxWaiting(1000) aw := nats.AckWait(5 * time.Second) for i := 0; i < 10; i++ { for _, partition := range []string{"EEEEE"} { subject := fmt.Sprintf("MSGS.%s.*.H.100XY.*.*.WQ.00000000000%d", partition, i) consumer := fmt.Sprintf("consumer:%s:%d", partition, i) _, err := cjs.PullSubscribe(subject, consumer, mp, mw, aw) require_NoError(t, err) } } // Create a single consumer that does no activity. // Make sure we still calculate low ack properly and cleanup etc. _, err = cjs.PullSubscribe("MSGS.ZZ.>", "consumer:ZZ:0", mp, mw, aw) require_NoError(t, err) subjects := []string{ "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000000", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000001", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000002", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000003", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000004", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000005", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000006", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000007", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000008", "MSGS.EEEEE.P.H.100XY.1.100Z.WQ.000000000009", } payload := []byte(strings.Repeat("A", 1024)) for i := 0; i < 50; i++ { wg.Add(1) go func() { pnc, pjs := jsClientConnect(t, c.randomServer()) defer pnc.Close() for i := 1; i < 200_000; i++ { select { case <-pctx.Done(): wg.Done() return default: } for _, subject := range subjects { // Send each message a few times. msgID := nats.MsgId(nuid.Next()) pjs.PublishAsync(subject, payload, msgID) pjs.Publish(subject, payload, msgID, nats.AckWait(250*time.Millisecond)) pjs.Publish(subject, payload, msgID, nats.AckWait(250*time.Millisecond)) } } }() } // Rogue publisher that sends the same msg ID everytime. for i := 0; i < 10; i++ { wg.Add(1) go func() { pnc, pjs := jsClientConnect(t, c.randomServer()) defer pnc.Close() msgID := nats.MsgId("1234567890") for i := 1; ; i++ { select { case <-pctx.Done(): wg.Done() return default: } for _, subject := range subjects { // Send each message a few times. pjs.PublishAsync(subject, payload, msgID, nats.RetryAttempts(0), nats.RetryWait(0)) pjs.Publish(subject, payload, msgID, nats.AckWait(1*time.Millisecond), nats.RetryAttempts(0), nats.RetryWait(0)) pjs.Publish(subject, payload, msgID, nats.AckWait(1*time.Millisecond), nats.RetryAttempts(0), nats.RetryWait(0)) } } }() } // Let enough messages into the stream then start consumers. time.Sleep(15 * time.Second) ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() for i := 0; i < 10; i++ { subject := fmt.Sprintf("MSGS.EEEEE.*.H.100XY.*.*.WQ.00000000000%d", i) consumer := fmt.Sprintf("consumer:EEEEE:%d", i) for n := 0; n < 5; n++ { cpnc, cpjs := jsClientConnect(t, c.randomServer()) defer cpnc.Close() psub, err := cpjs.PullSubscribe(subject, consumer, mp, mw, aw) require_NoError(t, err) time.AfterFunc(15*time.Second, func() { cpnc.Close() }) wg.Add(1) go func() { tick := time.NewTicker(1 * time.Millisecond) for { if cpnc.IsClosed() { wg.Done() return } select { case <-ctx.Done(): wg.Done() return case <-tick.C: // Fetch 1 first, then if no errors Fetch 100. msgs, err := psub.Fetch(1, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } msgs, err = psub.Fetch(100, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } msgs, err = psub.Fetch(1000, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } } } }() } } for i := 0; i < 10; i++ { subject := fmt.Sprintf("MSGS.EEEEE.*.H.100XY.*.*.WQ.00000000000%d", i) consumer := fmt.Sprintf("consumer:EEEEE:%d", i) for n := 0; n < 10; n++ { cpnc, cpjs := jsClientConnect(t, c.randomServer()) defer cpnc.Close() psub, err := cpjs.PullSubscribe(subject, consumer, mp, mw, aw) if err != nil { t.Logf("ERROR: %v", err) continue } wg.Add(1) go func() { tick := time.NewTicker(1 * time.Millisecond) for { select { case <-ctx.Done(): wg.Done() return case <-tick.C: // Fetch 1 first, then if no errors Fetch 100. msgs, err := psub.Fetch(1, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } msgs, err = psub.Fetch(100, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } msgs, err = psub.Fetch(1000, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } } } }() } } // Periodically disconnect routes from one of the servers. if params.reconnectRoutes { wg.Add(1) go func() { for range time.NewTicker(10 * time.Second).C { select { case <-ctx.Done(): wg.Done() return default: } // Force disconnecting routes from one of the servers. s := c.servers[rand.Intn(3)] var routes []*client t.Logf("Disconnecting routes from %v", s.Name()) s.mu.Lock() for _, conns := range s.routes { routes = append(routes, conns...) } s.mu.Unlock() for _, r := range routes { r.closeConnection(ClientClosed) } } }() } // Periodically reconnect clients. if params.reconnectClients { reconnectClients := func(s *Server) { for _, client := range s.clients { client.closeConnection(Kicked) } } wg.Add(1) go func() { for range time.NewTicker(10 * time.Second).C { select { case <-ctx.Done(): wg.Done() return default: } // Force reconnect clients from one of the servers. s := c.servers[rand.Intn(len(c.servers))] reconnectClients(s) } }() } // Restarts time.AfterFunc(10*time.Second, func() { for i := 0; i < params.restarts; i++ { switch { case params.restartLeader: // Find server leader of the stream and restart it. s := c.streamLeader("js", sc.Name) if params.ldmRestart { s.lameDuckMode() } else { s.Shutdown() } s.WaitForShutdown() c.restartServer(s) case params.restartAny: s := c.servers[rand.Intn(len(c.servers))] if params.ldmRestart { s.lameDuckMode() } else { s.Shutdown() } s.WaitForShutdown() c.restartServer(s) case params.rolloutRestart: for _, s := range c.servers { if params.ldmRestart { s.lameDuckMode() } else { s.Shutdown() } s.WaitForShutdown() c.restartServer(s) if params.checkHealthz { hctx, hcancel := context.WithTimeout(ctx, 15*time.Second) defer hcancel() for range time.NewTicker(2 * time.Second).C { select { case <-hctx.Done(): default: } status := s.healthz(nil) if status.StatusCode == 200 { break } } } } } c.waitOnClusterReady() } }) // Wait until context is done then check state. <-ctx.Done() getStreamDetails := func(t *testing.T, srv *Server) *StreamDetail { t.Helper() jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) require_NoError(t, err) if len(jsz.AccountDetails) > 0 && len(jsz.AccountDetails[0].Streams) > 0 { stream := jsz.AccountDetails[0].Streams[0] return &stream } t.Error("Could not find account details") return nil } checkState := func(t *testing.T) error { t.Helper() leaderSrv := c.streamLeader("js", sc.Name) if leaderSrv == nil { return fmt.Errorf("no leader found for stream") } streamLeader := getStreamDetails(t, leaderSrv) var errs []error for _, srv := range c.servers { if srv == leaderSrv { // Skip self continue } stream := getStreamDetails(t, srv) if stream == nil { return fmt.Errorf("stream not found") } if stream.State.Msgs != streamLeader.State.Msgs { err := fmt.Errorf("Leader %v has %d messages, Follower %v has %d messages", stream.Cluster.Leader, streamLeader.State.Msgs, srv, stream.State.Msgs, ) errs = append(errs, err) } if stream.State.FirstSeq != streamLeader.State.FirstSeq { err := fmt.Errorf("Leader %v FirstSeq is %d, Follower %v is at %d", stream.Cluster.Leader, streamLeader.State.FirstSeq, srv, stream.State.FirstSeq, ) errs = append(errs, err) } if stream.State.LastSeq != streamLeader.State.LastSeq { err := fmt.Errorf("Leader %v LastSeq is %d, Follower %v is at %d", stream.Cluster.Leader, streamLeader.State.LastSeq, srv, stream.State.LastSeq, ) errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } checkMsgsEqual := func(t *testing.T) { // These have already been checked to be the same for all streams. state := getStreamDetails(t, c.streamLeader("js", sc.Name)).State // Gather the stream mset from each replica. var msets []*stream for _, s := range c.servers { acc, err := s.LookupAccount("js") require_NoError(t, err) mset, err := acc.lookupStream(sc.Name) require_NoError(t, err) msets = append(msets, mset) } for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { var expectedErr error var msgId string var smv StoreMsg for _, mset := range msets { mset.mu.RLock() sm, err := mset.store.LoadMsg(seq, &smv) mset.mu.RUnlock() if err != nil || expectedErr != nil { // If one of the msets reports an error for LoadMsg for this // particular sequence, then the same error should be reported // by all msets for that seq to prove consistency across replicas. // If one of the msets either returns no error or doesn't return // the same error, then that replica has drifted. if expectedErr == nil { expectedErr = err } else { require_Error(t, err, expectedErr) } continue } if msgId == _EMPTY_ { msgId = string(sm.hdr) } else if msgId != string(sm.hdr) { t.Fatalf("MsgIds do not match for seq %d: %q vs %q", seq, msgId, sm.hdr) } } } } // Wait for test to finish before checking state. wg.Wait() // If clustered, check whether leader and followers have drifted. if sc.Replicas > 1 { // If we have drifted do not have to wait too long, usually it's stuck for good. checkFor(t, 5*time.Minute, time.Second, func() error { return checkState(t) }) // If we succeeded now let's check that all messages are also the same. // We may have no messages but for tests that do we make sure each msg is the same // across all replicas. checkMsgsEqual(t) } err = checkForErr(2*time.Minute, time.Second, func() error { var consumerPending int consumers := make(map[string]int) for i := 0; i < 10; i++ { consumerName := fmt.Sprintf("consumer:EEEEE:%d", i) ci, err := js.ConsumerInfo(sc.Name, consumerName) if err != nil { return err } pending := int(ci.NumPending) consumers[consumerName] = pending consumerPending += pending } // Only check if there are any pending messages. if consumerPending > 0 { // Check state of streams and consumers. si, err := js.StreamInfo(sc.Name, &nats.StreamInfoRequest{SubjectsFilter: ">"}) if err != nil { return err } streamPending := int(si.State.Msgs) // FIXME: Num pending can be out of sync from the number of stream messages in the subject. if streamPending != consumerPending { return fmt.Errorf("Unexpected number of pending messages, stream=%d, consumers=%d \n subjects: %+v\nconsumers: %+v", streamPending, consumerPending, si.State.Subjects, consumers) } } return nil }) if err != nil { t.Logf("WRN: %v", err) } } // Setting up test variations below: // // File based with single replica and discard old policy. t.Run("R1F", func(t *testing.T) { params := &testParams{ restartAny: true, ldmRestart: false, rolloutRestart: false, restarts: 1, } test(t, params, &nats.StreamConfig{ Name: "OWQTEST_R1F", Subjects: []string{"MSGS.>"}, Replicas: 1, MaxAge: 30 * time.Minute, Duplicates: 5 * time.Minute, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardOld, AllowRollup: true, Placement: &nats.Placement{ Tags: []string{"test"}, }, }) }) // Clustered memory based with discard new policy and max msgs limit. t.Run("R3M", func(t *testing.T) { params := &testParams{ restartAny: true, ldmRestart: true, rolloutRestart: false, restarts: 1, checkHealthz: true, } test(t, params, &nats.StreamConfig{ Name: "OWQTEST_R3M", Subjects: []string{"MSGS.>"}, Replicas: 3, MaxAge: 30 * time.Minute, MaxMsgs: 100_000, Duplicates: 5 * time.Minute, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardNew, AllowRollup: true, Storage: nats.MemoryStorage, Placement: &nats.Placement{ Tags: []string{"test"}, }, }) }) // Clustered file based with discard new policy and max msgs limit. t.Run("R3F_DN", func(t *testing.T) { params := &testParams{ restartAny: true, ldmRestart: true, rolloutRestart: false, restarts: 1, checkHealthz: true, } test(t, params, &nats.StreamConfig{ Name: "OWQTEST_R3F_DN", Subjects: []string{"MSGS.>"}, Replicas: 3, MaxAge: 30 * time.Minute, MaxMsgs: 100_000, Duplicates: 5 * time.Minute, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardNew, AllowRollup: true, Placement: &nats.Placement{ Tags: []string{"test"}, }, }) }) // Clustered file based with discard old policy and max msgs limit. t.Run("R3F_DO", func(t *testing.T) { params := &testParams{ restartAny: true, ldmRestart: true, rolloutRestart: false, restarts: 1, checkHealthz: true, } test(t, params, &nats.StreamConfig{ Name: "OWQTEST_R3F_DO", Subjects: []string{"MSGS.>"}, Replicas: 3, MaxAge: 30 * time.Minute, MaxMsgs: 100_000, Duplicates: 5 * time.Minute, Retention: nats.WorkQueuePolicy, Discard: nats.DiscardOld, AllowRollup: true, Placement: &nats.Placement{ Tags: []string{"test"}, }, }) }) // Clustered file based with discard old policy and no limits. t.Run("R3F_DO_NOLIMIT", func(t *testing.T) { params := &testParams{ restartAny: true, ldmRestart: true, rolloutRestart: false, restarts: 1, checkHealthz: true, } test(t, params, &nats.StreamConfig{ Name: "OWQTEST_R3F_DO_NOLIMIT", Subjects: []string{"MSGS.>"}, Replicas: 3, Duplicates: 30 * time.Second, Discard: nats.DiscardOld, Placement: &nats.Placement{ Tags: []string{"test"}, }, }) }) } func TestJetStreamClusterConsumerNRGCleanup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.MemoryStorage, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) // First call is just to create the pull subscribers. _, err = js.PullSubscribe("foo", "dlc") require_NoError(t, err) require_NoError(t, js.DeleteConsumer("TEST", "dlc")) // Now delete the stream. require_NoError(t, js.DeleteStream("TEST")) // Now make sure we cleaned up the NRG directories for the stream and consumer. var numConsumers, numStreams int for _, s := range c.servers { sd := s.JetStreamConfig().StoreDir nd := filepath.Join(sd, "$SYS", "_js_") f, err := os.Open(nd) require_NoError(t, err) dirs, err := f.ReadDir(-1) require_NoError(t, err) for _, fi := range dirs { if strings.HasPrefix(fi.Name(), "C-") { numConsumers++ } else if strings.HasPrefix(fi.Name(), "S-") { numStreams++ } } f.Close() } require_Equal(t, numConsumers, 0) require_Equal(t, numStreams, 0) } // https://github.com/nats-io/nats-server/issues/4878 func TestClusteredInterestConsumerFilterEdit(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "INTEREST", Retention: nats.InterestPolicy, Subjects: []string{"interest.>"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("INTEREST", &nats.ConsumerConfig{ Durable: "C0", FilterSubject: "interest.>", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish(fmt.Sprintf("interest.%d", i), []byte(strconv.Itoa(i))) require_NoError(t, err) } // we check we got 10 messages nfo, err := js.StreamInfo("INTEREST") require_NoError(t, err) if nfo.State.Msgs != 10 { t.Fatalf("expected 10 messages got %d", nfo.State.Msgs) } // now we lower the consumer interest from all subjects to 1, // then check the stream state and check if interest behavior still works _, err = js.UpdateConsumer("INTEREST", &nats.ConsumerConfig{ Durable: "C0", FilterSubject: "interest.1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // we should now have only one message left nfo, err = js.StreamInfo("INTEREST") require_NoError(t, err) if nfo.State.Msgs != 1 { t.Fatalf("expected 1 message got %d", nfo.State.Msgs) } } func TestJetStreamClusterDoubleAckRedelivery(t *testing.T) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } server_tags: ["test"] system_account: sys no_auth_user: js accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream = enabled users = [ { user: js, pass: js } ] } }` c := createJetStreamClusterWithTemplate(t, conf, "R3F", 3) defer c.shutdown() for _, s := range c.servers { s.optsMu.Lock() s.opts.LameDuckDuration = 15 * time.Second s.opts.LameDuckGracePeriod = -15 * time.Second s.optsMu.Unlock() } s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() sc, err := js.AddStream(&nats.StreamConfig{ Name: "LIMITS", Subjects: []string{"foo.>"}, Replicas: 3, Storage: nats.FileStorage, }) require_NoError(t, err) stepDown := func() { _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Config.Name), nil, time.Second) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var wg sync.WaitGroup producer := func(name string) { wg.Add(1) nc, js := jsClientConnect(t, s) defer nc.Close() defer wg.Done() i := 0 payload := []byte(strings.Repeat("Z", 1024)) for range time.NewTicker(1 * time.Millisecond).C { select { case <-ctx.Done(): return default: } msgID := nats.MsgId(fmt.Sprintf("%s:%d", name, i)) js.PublishAsync("foo.bar", payload, msgID, nats.RetryAttempts(10)) i++ } } go producer("A") go producer("B") go producer("C") sub, err := js.PullSubscribe("foo.bar", "ABC", nats.AckWait(5*time.Second), nats.MaxAckPending(1000), nats.PullMaxWaiting(1000)) if err != nil { t.Fatal(err) } type ackResult struct { ack *nats.Msg original *nats.Msg redelivered *nats.Msg } received := make(map[string]int64) acked := make(map[string]*ackResult) errors := make(map[string]error) extraRedeliveries := 0 wg.Add(1) go func() { nc, js = jsClientConnect(t, s) defer nc.Close() defer wg.Done() fetch := func(t *testing.T, batchSize int) { msgs, err := sub.Fetch(batchSize, nats.MaxWait(500*time.Millisecond)) if err != nil { return } for _, msg := range msgs { meta, err := msg.Metadata() if err != nil { t.Error(err) continue } msgID := msg.Header.Get(nats.MsgIdHdr) if err, ok := errors[msgID]; ok { t.Logf("Redelivery (num_delivered: %v) after failed Ack Sync: %+v - %+v - error: %v", meta.NumDelivered, msg.Reply, msg.Header, err) } if resp, ok := acked[msgID]; ok { t.Errorf("Redelivery (num_delivered: %v) after successful Ack Sync: msgID:%v - redelivered:%v - original:%+v - ack:%+v", meta.NumDelivered, msgID, msg.Reply, resp.original.Reply, resp.ack) resp.redelivered = msg extraRedeliveries++ } received[msgID]++ // Retry quickly a few times after there is a failed ack. Retries: for i := 0; i < 10; i++ { resp, err := nc.Request(msg.Reply, []byte("+ACK"), 500*time.Millisecond) if err != nil { t.Logf("Error: %v %v", msgID, err) errors[msgID] = err } else { acked[msgID] = &ackResult{resp, msg, nil} break Retries } } } } for { select { case <-ctx.Done(): return default: } fetch(t, 1) fetch(t, 50) } }() // Cause a couple of step downs before the restarts as well. time.AfterFunc(5*time.Second, func() { stepDown() }) time.AfterFunc(10*time.Second, func() { stepDown() }) // Let messages be produced, and then restart the servers. <-time.After(15 * time.Second) NextServer: for _, s := range c.servers { s.lameDuckMode() s.WaitForShutdown() s = c.restartServer(s) hctx, hcancel := context.WithTimeout(ctx, 60*time.Second) defer hcancel() for range time.NewTicker(2 * time.Second).C { select { case <-hctx.Done(): t.Logf("WRN: Timed out waiting for healthz from %s", s) continue NextServer default: } status := s.healthz(nil) if status.StatusCode == 200 { continue NextServer } } // Pause in-between server restarts. time.Sleep(10 * time.Second) } // Stop all producer and consumer goroutines to check results. cancel() select { case <-ctx.Done(): case <-time.After(10 * time.Second): } wg.Wait() if extraRedeliveries > 0 { t.Fatalf("Received %v redeliveries after a successful ack", extraRedeliveries) } } func TestJetStreamClusterBusyStreams(t *testing.T) { t.Skip("Too long for CI at the moment") type streamSetup struct { config *nats.StreamConfig consumers []*nats.ConsumerConfig subjects []string } type job func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) type testParams struct { cluster string streams []*streamSetup producers int consumers int restartAny bool restartWait time.Duration ldmRestart bool rolloutRestart bool restarts int checkHealthz bool jobs []job expect job duration time.Duration producerMsgs int producerMsgSize int } test := func(t *testing.T, test *testParams) { conf := ` listen: 127.0.0.1:-1 http: 127.0.0.1:-1 server_name: %s jetstream: { domain: "cloud" store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } server_tags: ["test"] system_account: sys no_auth_user: js accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream = enabled users = [ { user: js, pass: js } ] } }` c := createJetStreamClusterWithTemplate(t, conf, test.cluster, 3) defer c.shutdown() for _, s := range c.servers { s.optsMu.Lock() s.opts.LameDuckDuration = 15 * time.Second s.opts.LameDuckGracePeriod = -15 * time.Second s.optsMu.Unlock() } nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() var wg sync.WaitGroup for _, stream := range test.streams { stream := stream wg.Add(1) go func() { defer wg.Done() _, err := js.AddStream(stream.config) require_NoError(t, err) for _, consumer := range stream.consumers { _, err := js.AddConsumer(stream.config.Name, consumer) require_NoError(t, err) } }() } wg.Wait() ctx, cancel := context.WithTimeout(context.Background(), test.duration) defer cancel() for _, stream := range test.streams { payload := []byte(strings.Repeat("A", test.producerMsgSize)) stream := stream subjects := stream.subjects // Create publishers on different connections that sends messages // to all the consumers subjects. var n atomic.Uint64 for i := 0; i < test.producers; i++ { wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for range time.NewTicker(1 * time.Millisecond).C { select { case <-ctx.Done(): return default: } for _, subject := range subjects { _, err := js.Publish(subject, payload, nats.AckWait(200*time.Millisecond)) if err == nil { if nn := n.Add(1); int(nn) >= test.producerMsgs { return } } } } }() } // Create multiple parallel pull subscribers per consumer config. for i := 0; i < test.consumers; i++ { for _, consumer := range stream.consumers { wg.Add(1) consumer := consumer go func() { defer wg.Done() for attempts := 0; attempts < 60; attempts++ { _, err := js.ConsumerInfo(stream.config.Name, consumer.Name) if err != nil { t.Logf("WRN: Failed creating pull subscriber: %v - %v - %v - %v", consumer.FilterSubject, stream.config.Name, consumer.Name, err) } } sub, err := js.PullSubscribe(consumer.FilterSubject, "", nats.Bind(stream.config.Name, consumer.Name)) if err != nil { t.Logf("WRN: Failed creating pull subscriber: %v - %v - %v - %v", consumer.FilterSubject, stream.config.Name, consumer.Name, err) return } require_NoError(t, err) for range time.NewTicker(100 * time.Millisecond).C { select { case <-ctx.Done(): return default: } msgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } msgs, err = sub.Fetch(100, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } } }() } } } for _, job := range test.jobs { go job(t, nc, js, c) } if test.restarts > 0 { wg.Add(1) time.AfterFunc(test.restartWait, func() { defer wg.Done() for i := 0; i < test.restarts; i++ { switch { case test.restartAny: s := c.servers[rand.Intn(len(c.servers))] if test.ldmRestart { s.lameDuckMode() } else { s.Shutdown() } s.WaitForShutdown() c.restartServer(s) case test.rolloutRestart: for _, s := range c.servers { if test.ldmRestart { s.lameDuckMode() } else { s.Shutdown() } s.WaitForShutdown() s = c.restartServer(s) if test.checkHealthz { hctx, hcancel := context.WithTimeout(ctx, 15*time.Second) defer hcancel() Healthz: for range time.NewTicker(2 * time.Second).C { select { case <-hctx.Done(): break Healthz default: } status := s.healthz(nil) if status.StatusCode == 200 { break Healthz } } } } } c.waitOnClusterReady() } }) } test.expect(t, nc, js, c) cancel() wg.Wait() } stepDown := func(nc *nats.Conn, streamName string) { nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, streamName), nil, time.Second) } getStreamDetails := func(t *testing.T, c *cluster, accountName, streamName string) *StreamDetail { t.Helper() srv := c.streamLeader(accountName, streamName) jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) require_NoError(t, err) for _, acc := range jsz.AccountDetails { if acc.Name == accountName { for _, stream := range acc.Streams { if stream.Name == streamName { return &stream } } } } t.Error("Could not find account details") return nil } checkMsgsEqual := func(t *testing.T, c *cluster, accountName, streamName string) { state := getStreamDetails(t, c, accountName, streamName).State var msets []*stream for _, s := range c.servers { acc, err := s.LookupAccount(accountName) require_NoError(t, err) mset, err := acc.lookupStream(streamName) require_NoError(t, err) msets = append(msets, mset) } for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { var msgId string var smv StoreMsg for _, mset := range msets { mset.mu.RLock() sm, err := mset.store.LoadMsg(seq, &smv) mset.mu.RUnlock() require_NoError(t, err) if msgId == _EMPTY_ { msgId = string(sm.hdr) } else if msgId != string(sm.hdr) { t.Fatalf("MsgIds do not match for seq %d: %q vs %q", seq, msgId, sm.hdr) } } } } checkConsumer := func(t *testing.T, c *cluster, accountName, streamName, consumerName string) { t.Helper() var leader string for _, s := range c.servers { jsz, err := s.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) require_NoError(t, err) for _, acc := range jsz.AccountDetails { if acc.Name == accountName { for _, stream := range acc.Streams { if stream.Name == streamName { for _, consumer := range stream.Consumer { if leader == "" { leader = consumer.Cluster.Leader } else if leader != consumer.Cluster.Leader { t.Errorf("There are two leaders for %s/%s: %s vs %s", stream.Name, consumer.Name, leader, consumer.Cluster.Leader) } } } } } } } } t.Run("R1F/rescale/R3F/sources:10/limits", func(t *testing.T) { testDuration := 3 * time.Minute totalStreams := 10 streams := make([]*streamSetup, totalStreams) sources := make([]*nats.StreamSource, totalStreams) for i := 0; i < totalStreams; i++ { name := fmt.Sprintf("test:%d", i) st := &streamSetup{ config: &nats.StreamConfig{ Name: name, Subjects: []string{fmt.Sprintf("test.%d.*", i)}, Replicas: 1, Retention: nats.LimitsPolicy, }, } st.subjects = append(st.subjects, fmt.Sprintf("test.%d.0", i)) sources[i] = &nats.StreamSource{Name: name} streams[i] = st } // Create Source consumer. sourceSetup := &streamSetup{ config: &nats.StreamConfig{ Name: "source-test", Replicas: 1, Retention: nats.LimitsPolicy, Sources: sources, }, consumers: make([]*nats.ConsumerConfig, 0), } cc := &nats.ConsumerConfig{ Name: "A", Durable: "A", FilterSubject: "test.>", AckPolicy: nats.AckExplicitPolicy, } sourceSetup.consumers = append(sourceSetup.consumers, cc) streams = append(streams, sourceSetup) scale := func(replicas int, wait time.Duration) job { return func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) { config := sourceSetup.config time.AfterFunc(wait, func() { config.Replicas = replicas for i := 0; i < 10; i++ { _, err := js.UpdateStream(config) if err == nil { return } time.Sleep(1 * time.Second) } }) } } expect := func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) { // The source stream should not be stuck or be different from the other streams. time.Sleep(testDuration + 1*time.Minute) accName := "js" streamName := "source-test" // Check a few times to see if there are no changes in the number of messages. var changed bool var prevMsgs uint64 for i := 0; i < 10; i++ { sinfo, err := js.StreamInfo(streamName) if err != nil { t.Logf("Error: %v", err) time.Sleep(2 * time.Second) continue } prevMsgs = sinfo.State.Msgs } for i := 0; i < 10; i++ { sinfo, err := js.StreamInfo(streamName) if err != nil { t.Logf("Error: %v", err) time.Sleep(2 * time.Second) continue } changed = prevMsgs != sinfo.State.Msgs prevMsgs = sinfo.State.Msgs time.Sleep(2 * time.Second) } if !changed { // Doing a leader step down should not cause the messages to change. stepDown(nc, streamName) for i := 0; i < 10; i++ { sinfo, err := js.StreamInfo(streamName) if err != nil { t.Logf("Error: %v", err) time.Sleep(2 * time.Second) continue } changed = prevMsgs != sinfo.State.Msgs prevMsgs = sinfo.State.Msgs time.Sleep(2 * time.Second) } if changed { t.Error("Stream msgs changed after the step down") } } ///////////////////////////////////////////////////////////////////////////////////////// // // // The number of messages sourced should match the count from all the other streams. // // // ///////////////////////////////////////////////////////////////////////////////////////// var expectedMsgs uint64 for i := 0; i < totalStreams; i++ { name := fmt.Sprintf("test:%d", i) sinfo, err := js.StreamInfo(name) require_NoError(t, err) expectedMsgs += sinfo.State.Msgs } sinfo, err := js.StreamInfo(streamName) require_NoError(t, err) gotMsgs := sinfo.State.Msgs if gotMsgs != expectedMsgs { t.Errorf("stream with sources has %v messages, but total sourced messages should be %v", gotMsgs, expectedMsgs) } checkConsumer(t, c, accName, streamName, "A") checkMsgsEqual(t, c, accName, streamName) } test(t, &testParams{ cluster: t.Name(), streams: streams, producers: 10, consumers: 10, restarts: 1, rolloutRestart: true, ldmRestart: true, checkHealthz: true, // TODO(dlc) - If this overlaps with the scale jobs this test will fail. // Leaders will be elected with partial state. restartWait: 65 * time.Second, jobs: []job{ scale(3, 15*time.Second), scale(1, 30*time.Second), scale(3, 60*time.Second), }, expect: expect, duration: testDuration, producerMsgSize: 1024, producerMsgs: 100_000, }) }) t.Run("R3F/streams:30/limits", func(t *testing.T) { testDuration := 3 * time.Minute totalStreams := 30 consumersPerStream := 5 streams := make([]*streamSetup, totalStreams) for i := 0; i < totalStreams; i++ { name := fmt.Sprintf("test:%d", i) st := &streamSetup{ config: &nats.StreamConfig{ Name: name, Subjects: []string{fmt.Sprintf("test.%d.*", i)}, Replicas: 3, Retention: nats.LimitsPolicy, }, consumers: make([]*nats.ConsumerConfig, 0), } for j := 0; j < consumersPerStream; j++ { subject := fmt.Sprintf("test.%d.%d", i, j) name := fmt.Sprintf("A:%d:%d", i, j) cc := &nats.ConsumerConfig{ Name: name, Durable: name, FilterSubject: subject, AckPolicy: nats.AckExplicitPolicy, } st.consumers = append(st.consumers, cc) st.subjects = append(st.subjects, subject) } streams[i] = st } expect := func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) { time.Sleep(testDuration + 1*time.Minute) accName := "js" for i := 0; i < totalStreams; i++ { streamName := fmt.Sprintf("test:%d", i) checkMsgsEqual(t, c, accName, streamName) } } test(t, &testParams{ cluster: t.Name(), streams: streams, producers: 10, consumers: 10, restarts: 1, rolloutRestart: true, ldmRestart: true, checkHealthz: true, restartWait: 45 * time.Second, expect: expect, duration: testDuration, producerMsgSize: 1024, producerMsgs: 100_000, }) }) } // https://github.com/nats-io/nats-server/issues/5488 func TestJetStreamClusterSingleMaxConsumerUpdate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", MaxConsumers: 1, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "test_consumer", MaxAckPending: 1000, }) require_NoError(t, err) // This would previously return a "nats: maximum consumers limit // reached" (10026) error. _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Name: "test_consumer", MaxAckPending: 1001, }) require_NoError(t, err) } func TestJetStreamClusterStreamLastSequenceResetAfterStorageWipe(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // After bug was found, number of streams and wiping store directory really did not affect. numStreams := 50 var wg sync.WaitGroup wg.Add(numStreams) for i := 1; i <= numStreams; i++ { go func(n int) { defer wg.Done() _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST:%d", n), Retention: nats.InterestPolicy, Subjects: []string{fmt.Sprintf("foo.%d.*", n)}, Replicas: 3, }, nats.MaxWait(30*time.Second)) require_NoError(t, err) subj := fmt.Sprintf("foo.%d.bar", n) for i := 0; i < 222; i++ { js.Publish(subj, nil) } }(i) } wg.Wait() for i := 0; i < 5; i++ { // Walk the servers and shut each down, and wipe the storage directory. for _, s := range c.servers { sd := s.JetStreamConfig().StoreDir s.Shutdown() s.WaitForShutdown() os.RemoveAll(sd) s = c.restartServer(s) checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { hs := s.healthz(nil) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) } for _, s := range c.servers { for i := 1; i <= numStreams; i++ { stream := fmt.Sprintf("TEST:%d", i) mset, err := s.GlobalAccount().lookupStream(stream) require_NoError(t, err) var state StreamState checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { mset.store.FastState(&state) if state.LastSeq != 222 { return fmt.Errorf("%v Wrong last sequence %d for %q - State %+v", s, state.LastSeq, stream, state) } return nil }) } } } } func TestJetStreamClusterAckFloorBetweenLeaderAndFollowers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Retention: nats.InterestPolicy, Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo.*", "consumer") require_NoError(t, err) // Do 25 rounds. for i := 1; i <= 25; i++ { // Send 50 msgs. for x := 0; x < 50; x++ { _, err := js.Publish("foo.bar", nil) require_NoError(t, err) } msgs, err := sub.Fetch(50, nats.MaxWait(10*time.Second)) require_NoError(t, err) require_Equal(t, len(msgs), 50) // Randomize rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) for _, m := range msgs { m.AckSync() } time.Sleep(100 * time.Millisecond) for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) consumer := mset.lookupConsumer("consumer") require_NotEqual(t, consumer, nil) info := consumer.info() require_Equal(t, info.NumAckPending, 0) require_Equal(t, info.AckFloor.Consumer, uint64(i*50)) require_Equal(t, info.AckFloor.Stream, uint64(i*50)) } } } // https://github.com/nats-io/nats-server/pull/5600 func TestJetStreamClusterConsumerLeak(t *testing.T) { N := 2000 // runs in under 10s, but significant enough to see the difference. NConcurrent := 100 clusterConf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leafnodes { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(t, clusterConf, "Leak-test", 3) defer cl.shutdown() cl.waitOnLeader() s := cl.randomNonLeader() // Create the test stream. streamName := "LEAK_TEST_STREAM" nc, js := jsClientConnect(t, s, nats.UserInfo("one", "p")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: streamName, Subjects: []string{"$SOMETHING.>"}, Storage: nats.FileStorage, Retention: nats.InterestPolicy, Replicas: 3, }) if err != nil { t.Fatalf("Error creating stream: %v", err) } concurrent := make(chan struct{}, NConcurrent) for i := 0; i < NConcurrent; i++ { concurrent <- struct{}{} } errors := make(chan error, N) wg := sync.WaitGroup{} wg.Add(N) // Gather the stats for comparison. before := &runtime.MemStats{} runtime.GC() runtime.ReadMemStats(before) for i := 0; i < N; { // wait for a slot to open up <-concurrent i++ go func() { defer func() { concurrent <- struct{}{} wg.Done() }() nc, js := jsClientConnect(t, s, nats.UserInfo("one", "p")) defer nc.Close() consumerName := "sessid_" + nuid.Next() _, err := js.AddConsumer(streamName, &nats.ConsumerConfig{ DeliverSubject: "inbox", Durable: consumerName, AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, FilterSubject: "$SOMETHING.ELSE.subject", AckWait: 30 * time.Second, MaxAckPending: 1024, }) if err != nil { errors <- fmt.Errorf("Error on JetStream consumer creation: %v", err) return } err = js.DeleteConsumer(streamName, consumerName) if err != nil { errors <- fmt.Errorf("Error on JetStream consumer deletion: %v", err) } }() } wg.Wait() if len(errors) > 0 { for err := range errors { t.Fatalf("%v", err) } } after := &runtime.MemStats{} runtime.GC() runtime.ReadMemStats(after) // Before https://github.com/nats-io/nats-server/pull/5600 this test was // adding 180Mb+ to HeapInuse. Now it's under 40Mb (ran locally on a Mac) limit := before.HeapInuse + 100*1024*1024 // 100MB if after.HeapInuse > before.HeapInuse+limit { t.Fatalf("Extra memory usage too high: %v", after.HeapInuse-before.HeapInuse) } } func TestJetStreamClusterWQRoundRobinSubjectRetention(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "wq_stream", Subjects: []string{"something.>"}, Storage: nats.FileStorage, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 100; i++ { n := (i % 5) + 1 _, err := js.Publish(fmt.Sprintf("something.%d", n), nil) require_NoError(t, err) } sub, err := js.PullSubscribe( "something.5", "wq_consumer_5", nats.BindStream("wq_stream"), nats.ConsumerReplicas(3), ) require_NoError(t, err) for { msgs, _ := sub.Fetch(5) if len(msgs) == 0 { break } for _, msg := range msgs { require_NoError(t, msg.AckSync()) } } si, err := js.StreamInfo("wq_stream") require_NoError(t, err) require_Equal(t, si.State.Msgs, 80) require_Equal(t, si.State.NumDeleted, 20) require_Equal(t, si.State.NumSubjects, 4) } type captureLeafClusterSpacesLogger struct { DummyLogger ch chan string warnCh chan string } func (l *captureLeafClusterSpacesLogger) Warnf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, `Server name has spaces and used as the cluster name, leaf remotes may not connect properly`) { select { case l.warnCh <- msg: default: } } } func (l *captureLeafClusterSpacesLogger) Errorf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, `Leafnode Error 'cluster name cannot contain spaces or new lines'`) { select { case l.ch <- msg: default: } } } func TestJetStreamClusterAndNamesWithSpaces(t *testing.T) { gwConf := ` listen: 127.0.0.1:-1 http: 127.0.0.1:-1 server_name: 'SRV %s' jetstream: { store_dir: '%s', } cluster { name: '%s' listen: 127.0.0.1:%d routes = [%s] } server_tags: ["test"] system_account: sys no_auth_user: js leafnodes { host: "127.0.0.1" port: -1 } accounts { sys { users = [ { user: sys, pass: sys } ] } js { jetstream: enabled users = [ { user: js, pass: js } ] } } ` c := createJetStreamClusterAndModHook(t, gwConf, "S P A C E 1", "GW_1_", 3, 15022, false, func(serverName, clusterName, storeDir, conf string) string { conf += ` gateway { name: "S P A C E 1" listen: 127.0.0.1:-1 } ` return conf }) defer c.shutdown() c2 := createJetStreamClusterAndModHook(t, gwConf, "S P A C E 2", "GW_2_", 3, 16022, false, func(serverName, clusterName, storeDir, conf string) string { conf += fmt.Sprintf(` gateway { name: "S P A C E 2" listen: 127.0.0.1:-1 gateways: [{ name: "S P A C E 1" url: "nats://127.0.0.1:%d" }] } `, c.servers[0].opts.Gateway.Port) return conf }) defer c2.shutdown() c3 := createJetStreamClusterAndModHook(t, gwConf, "S P A C E 3", "GW_3_", 3, 17022, false, func(serverName, clusterName, storeDir, conf string) string { conf += fmt.Sprintf(` gateway { name: "S P A C E 3" listen: 127.0.0.1:-1 gateways: [{ name: "S P A C E 1" url: "nats://127.0.0.1:%d" }] } `, c.servers[0].opts.Gateway.Port) return conf }) defer c3.shutdown() for _, s := range c2.servers { waitForOutboundGateways(t, s, 2, 2*time.Second) } for _, s := range c3.servers { waitForOutboundGateways(t, s, 2, 2*time.Second) } // Leaf with spaces in name which becomes its cluster name as well. leafConfA := ` host: "127.0.0.1" port: -1 server_name: "L E A F S P A C E" leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfA = fmt.Sprintf(leafConfA, c.servers[0].opts.LeafNode.Port) sconfA := createConfFile(t, []byte(leafConfA)) oA := LoadConfig(sconfA) leafA, err := NewServer(oA) require_NoError(t, err) lA := &captureLeafClusterSpacesLogger{ch: make(chan string, 10), warnCh: make(chan string, 10)} leafA.SetLogger(lA, false, false) leafA.Start() defer leafA.Shutdown() // Leaf with spaces in name but with a valid cluster name is able to connect. leafConfB := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "L E A F 2" cluster { name: "LEAF" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfB = fmt.Sprintf(leafConfB, c.servers[0].opts.LeafNode.Port) sconfB := createConfFile(t, []byte(leafConfB)) leafB, _ := RunServerWithConfig(sconfB) lB := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafB.SetLogger(lB, false, false) defer leafB.Shutdown() // Leaf with valid server name but cluster name with spaces. leafConfC := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "LEAF3" cluster { name: "L E A F 3" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfC = fmt.Sprintf(leafConfC, c.servers[0].opts.LeafNode.Port) sconfC := createConfFile(t, []byte(leafConfC)) leafC, _ := RunServerWithConfig(sconfC) lC := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafC.SetLogger(lC, false, false) defer leafC.Shutdown() // Leafs with valid server name but using protocol special characters in cluster name. leafConfD := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "LEAF4" cluster { name: "LEAF 4" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfD = fmt.Sprintf(leafConfD, c.servers[0].opts.LeafNode.Port) sconfD := createConfFile(t, []byte(leafConfD)) leafD, _ := RunServerWithConfig(sconfD) lD := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafD.SetLogger(lD, false, false) defer leafD.Shutdown() leafConfD2 := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "LEAF42" cluster { name: "LEAF4\r2" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfD2 = fmt.Sprintf(leafConfD2, c.servers[0].opts.LeafNode.Port) sconfD2 := createConfFile(t, []byte(leafConfD2)) leafD2, _ := RunServerWithConfig(sconfD2) lD2 := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafD2.SetLogger(lD2, false, false) defer leafD2.Shutdown() leafConfD3 := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "LEAF43" cluster { name: "LEAF4\t3" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfD3 = fmt.Sprintf(leafConfD3, c.servers[0].opts.LeafNode.Port) sconfD3 := createConfFile(t, []byte(leafConfD3)) leafD3, _ := RunServerWithConfig(sconfD3) lD3 := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafD3.SetLogger(lD3, false, false) defer leafD3.Shutdown() // Leaf with valid configuration should be able to connect to GW cluster with spaces in names. leafConfE := ` host: "127.0.0.1" port: -1 http: 127.0.0.1:-1 server_name: "LEAF5" cluster { name: "LEAF5" } leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" remotes: [ { url: "nats://127.0.0.1:%d" } ] } ` leafConfE = fmt.Sprintf(leafConfE, c.servers[0].opts.LeafNode.Port) sconfE := createConfFile(t, []byte(leafConfE)) leafE, _ := RunServerWithConfig(sconfE) lE := &captureLeafClusterSpacesLogger{ch: make(chan string, 10)} leafE.SetLogger(lE, false, false) defer leafE.Shutdown() // Finally do a smoke test of connectivity among gateways and that JS is working // when using clusters with spaces still. nc1, js1 := jsClientConnect(t, c.servers[1]) defer nc1.Close() _, err = js1.AddStream(&nats.StreamConfig{ Name: "foo", Subjects: []string{"foo"}, }) require_NoError(t, err) c.waitOnStreamLeader("js", "foo") sub1, err := nc1.SubscribeSync("foo") require_NoError(t, err) nc1.Flush() // Check that invalid configs got the errors. select { case <-lA.ch: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for error") } select { case <-lC.ch: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for error") } select { case <-lD.ch: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for error") } select { case <-lD2.ch: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for error") } select { case <-lD3.ch: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for error") } // Check that we got a warning about the server name being reused // for the cluster name. select { case <-lA.warnCh: case <-time.After(5 * time.Second): t.Errorf("Timed out waiting for warning") } // Check that valid configs were ok still. select { case <-lB.ch: t.Errorf("Unexpected error from valid leafnode config") case <-lE.ch: t.Errorf("Unexpected error from valid leafnode config") case <-time.After(2 * time.Second): } nc2, js2 := jsClientConnect(t, c2.servers[1]) defer nc2.Close() nc2.Publish("foo", []byte("test")) nc2.Flush() time.Sleep(250 * time.Millisecond) msg, err := sub1.NextMsg(1 * time.Second) require_NoError(t, err) require_Equal(t, "test", string(msg.Data)) sinfo, err := js2.StreamInfo("foo") require_NoError(t, err) require_Equal(t, sinfo.State.Msgs, 1) } func TestJetStreamClusterMetaSyncOrphanCleanup(t *testing.T) { c := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, "R3S", 3, func(serverName, clusterName, storeDir, conf string) string { return fmt.Sprintf("%s\nserver_tags: [server:%s]", conf, serverName) }) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Create a bunch of streams on S1 for i := 0; i < 100; i++ { stream := fmt.Sprintf("TEST-%d", i) subject := fmt.Sprintf("TEST.%d", i) _, err := js.AddStream(&nats.StreamConfig{ Name: stream, Subjects: []string{subject}, Storage: nats.FileStorage, Placement: &nats.Placement{Tags: []string{"server:S-1"}}, }) require_NoError(t, err) // Put in 10 msgs to each for j := 0; j < 10; j++ { _, err := js.Publish(subject, nil) require_NoError(t, err) } } // Now we will shutdown S1 and remove all of its meta-data to trip the condition. s := c.serverByName("S-1") require_True(t, s != nil) sd := s.JetStreamConfig().StoreDir nd := filepath.Join(sd, "$SYS", "_js_", "_meta_") s.Shutdown() s.WaitForShutdown() os.RemoveAll(nd) s = c.restartServer(s) c.waitOnServerCurrent(s) jsz, err := s.Jsz(nil) require_NoError(t, err) require_Equal(t, jsz.Streams, 100) // These will be recreated by the meta layer, but if the orphan detection deleted them they will be empty, // so check all streams to make sure they still have data. acc := s.GlobalAccount() var state StreamState for i := 0; i < 100; i++ { mset, err := acc.lookupStream(fmt.Sprintf("TEST-%d", i)) require_NoError(t, err) mset.store.FastState(&state) require_Equal(t, state.Msgs, 10) } } func TestJetStreamClusterKeyValueDesyncAfterHardKill(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.serverByName("S-1")) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "inconsistency", Replicas: 3, }) require_NoError(t, err) // First create should succeed. revision, err := kv.Create("key.exists", []byte("1")) require_NoError(t, err) require_Equal(t, revision, 1) // Second create will be rejected but bump CLFS. _, err = kv.Create("key.exists", []byte("2")) require_Error(t, err) // Insert a new message, should only be applied once, even if we hard kill and replay afterward. revision, err = kv.Put("key.put", []byte("3")) require_NoError(t, err) require_Equal(t, revision, 2) // Restart a server s3 := c.serverByName("S-3") // We will remove the index.db file after we shutdown. mset, err := s3.GlobalAccount().lookupStream("KV_inconsistency") require_NoError(t, err) fs := mset.store.(*fileStore) ifile := filepath.Join(fs.fcfg.StoreDir, msgDir, "index.db") s3.Shutdown() s3.WaitForShutdown() // Remove the index.db file to simulate a hard kill where server can not write out the index.db file. require_NoError(t, os.Remove(ifile)) c.restartServer(s3) c.waitOnClusterReady() c.waitOnAllCurrent() err = checkState(t, c, "$G", "KV_inconsistency") require_NoError(t, err) } func TestJetStreamClusterKeyValueSync(t *testing.T) { t.Skip("Too long for CI at the moment") c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() for _, s := range c.servers { s.optsMu.Lock() s.opts.LameDuckDuration = 15 * time.Second s.opts.LameDuckGracePeriod = -15 * time.Second s.optsMu.Unlock() } s := c.randomNonLeader() connect := func(t *testing.T) (*nats.Conn, nats.JetStreamContext) { return jsClientConnect(t, s) } const accountName = "$G" const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" createData := func(n int) []byte { b := make([]byte, n) for i := range b { b[i] = letterBytes[rand.Intn(len(letterBytes))] } return b } getOrCreateKvStore := func(kvname string) (nats.KeyValue, error) { _, js := connect(t) kvExists := false existingKvnames := js.KeyValueStoreNames() for existingKvname := range existingKvnames { if existingKvname == kvname { kvExists = true break } } if !kvExists { return js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: kvname, Replicas: 3, Storage: nats.FileStorage, }) } else { return js.KeyValue(kvname) } } abs := func(x int64) int64 { if x < 0 { return -x } return x } var counter int64 var errorCounter int64 checkMsgsEqual := func(t *testing.T, accountName, streamName string) error { // Gather all the streams replicas and compare contents. msets := make(map[*Server]*stream) for _, s := range c.servers { acc, err := s.LookupAccount(accountName) if err != nil { return err } mset, err := acc.lookupStream(streamName) if err != nil { return err } msets[s] = mset } str := getStreamDetails(t, c, accountName, streamName) if str == nil { return fmt.Errorf("could not get stream leader state") } state := str.State for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { var msgId string var smv StoreMsg for replica, mset := range msets { mset.mu.RLock() sm, err := mset.store.LoadMsg(seq, &smv) mset.mu.RUnlock() if err != nil { if err == ErrStoreMsgNotFound || err == errDeletedMsg { // Skip these. } else { t.Logf("WRN: Error loading message (seq=%d) from stream %q on replica %q: %v", seq, streamName, replica, err) } continue } if msgId == _EMPTY_ { msgId = string(sm.hdr) } else if msgId != string(sm.hdr) { t.Errorf("MsgIds do not match for seq %d on stream %q: %q vs %q", seq, streamName, msgId, sm.hdr) } } } return nil } keyUpdater := func(ctx context.Context, cancel context.CancelFunc, kvname string, numKeys int) { kv, err := getOrCreateKvStore(kvname) if err != nil { t.Fatalf("[%s]:%v", kvname, err) } for i := 0; i < numKeys; i++ { key := fmt.Sprintf("key-%d", i) kv.Create(key, createData(160)) } lastData := make(map[string][]byte) revisions := make(map[string]uint64) for { select { case <-ctx.Done(): return default: } r := rand.Intn(numKeys) key := fmt.Sprintf("key-%d", r) for i := 0; i < 5; i++ { _, err := kv.Get(key) if err != nil { atomic.AddInt64(&errorCounter, 1) if err == nats.ErrKeyNotFound { t.Logf("WRN: Key not found! [%s/%s] - [%s]", kvname, key, err) cancel() } } } k, err := kv.Get(key) if err != nil { atomic.AddInt64(&errorCounter, 1) } else { if revisions[key] != 0 && abs(int64(k.Revision())-int64(revisions[key])) < 2 { lastDataVal, ok := lastData[key] if ok && k.Revision() == revisions[key] && slices.Compare(lastDataVal, k.Value()) != 0 { t.Logf("data loss [%s/%s][rev:%d] expected:[%v] is:[%v]", kvname, key, revisions[key], string(lastDataVal), string(k.Value())) } } newData := createData(160) revisions[key], err = kv.Update(key, newData, k.Revision()) if err != nil && err != nats.ErrTimeout { atomic.AddInt64(&errorCounter, 1) } else { lastData[key] = newData } atomic.AddInt64(&counter, 1) } } } streamCount := 50 keysCount := 100 streamPrefix := "IKV" ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() // The keyUpdaters will run for less time. kctx, kcancel := context.WithTimeout(context.Background(), 3*time.Minute) defer kcancel() var wg sync.WaitGroup var streams []string for i := 0; i < streamCount; i++ { streamName := fmt.Sprintf("%s-%d", streamPrefix, i) streams = append(streams, "KV_"+streamName) wg.Add(1) go func(i int) { defer wg.Done() keyUpdater(kctx, cancel, streamName, keysCount) }(i) } debug := false nc2, _ := jsClientConnect(t, s) if debug { go func() { for range time.NewTicker(5 * time.Second).C { select { case <-ctx.Done(): return default: } for _, str := range streams { leaderSrv := c.streamLeader(accountName, str) if leaderSrv == nil { continue } streamLeader := getStreamDetails(t, c, accountName, str) if streamLeader == nil { continue } t.Logf("|------------------------------------------------------------------------------------------------------------------------|") lstate := streamLeader.State t.Logf("| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d |", str, leaderSrv.String()+"*", lstate.Msgs, lstate.Bytes, lstate.NumDeleted, lstate.FirstSeq, lstate.LastSeq, ) for _, srv := range c.servers { if srv == leaderSrv { continue } acc, err := srv.LookupAccount(accountName) if err != nil { continue } stream, err := acc.lookupStream(str) if err != nil { t.Logf("Error looking up stream %s on %s replica", str, srv) continue } state := stream.state() unsynced := lstate.Msgs != state.Msgs || lstate.Bytes != state.Bytes || lstate.NumDeleted != state.NumDeleted || lstate.FirstSeq != state.FirstSeq || lstate.LastSeq != state.LastSeq var result string if unsynced { result = "UNSYNCED" } t.Logf("| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d | %s", str, srv, state.Msgs, state.Bytes, state.NumDeleted, state.FirstSeq, state.LastSeq, result, ) } } t.Logf("|------------------------------------------------------------------------------------------------------------------------| %v", nc2.ConnectedUrl()) } }() } checkStreams := func(t *testing.T) { for _, str := range streams { checkFor(t, time.Minute, 500*time.Millisecond, func() error { return checkState(t, c, accountName, str) }) checkFor(t, time.Minute, 500*time.Millisecond, func() error { return checkMsgsEqual(t, accountName, str) }) } } Loop: for range time.NewTicker(30 * time.Second).C { select { case <-ctx.Done(): break Loop default: } rollout := func(t *testing.T) { for _, s := range c.servers { // For graceful mode s.lameDuckMode() s.WaitForShutdown() s = c.restartServer(s) hctx, hcancel := context.WithTimeout(context.Background(), 15*time.Second) defer hcancel() Healthz: for range time.NewTicker(2 * time.Second).C { select { case <-hctx.Done(): default: } status := s.healthz(nil) if status.StatusCode == 200 { break Healthz } } c.waitOnClusterReady() checkStreams(t) } } rollout(t) checkStreams(t) } wg.Wait() checkStreams(t) } func TestJetStreamClusterKeyValueLastSeqMismatch(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for _, r := range []int{1, 3} { t.Run(fmt.Sprintf("R=%d", r), func(t *testing.T) { kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: fmt.Sprintf("mismatch_%v", r), Replicas: r, }) require_NoError(t, err) revision, err := kv.Create("foo", []byte("1")) require_NoError(t, err) require_Equal(t, revision, 1) revision, err = kv.Create("bar", []byte("2")) require_NoError(t, err) require_Equal(t, revision, 2) // Now say we want to update baz but iff last was revision 1. _, err = kv.Update("baz", []byte("3"), uint64(1)) require_Error(t, err) require_Equal(t, err.Error(), `nats: wrong last sequence: 0`) }) } } func TestJetStreamClusterPubAckSequenceDupe(t *testing.T) { c := createJetStreamClusterExplicit(t, "TEST_CLUSTER", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() type client struct { nc *nats.Conn js nats.JetStreamContext } clients := make([]client, len(c.servers)) for i, server := range c.servers { clients[i].nc, clients[i].js = jsClientConnect(t, server) defer clients[i].nc.Close() } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST_STREAM", Subjects: []string{"TEST_SUBJECT.*"}, Replicas: 3, Duplicates: 1 * time.Minute, }) require_NoError(t, err) msgData := []byte("...") for seq := uint64(1); seq < 10; seq++ { if seq%3 == 0 { c.restartAll() } msgSubject := "TEST_SUBJECT." + strconv.FormatUint(seq, 10) msgIdOpt := nats.MsgId(nuid.Next()) firstPublisherClient := &clients[rand.Intn(len(clients))] secondPublisherClient := &clients[rand.Intn(len(clients))] pubAck1, err := firstPublisherClient.js.Publish(msgSubject, msgData, msgIdOpt) require_NoError(t, err) require_Equal(t, seq, pubAck1.Sequence) require_False(t, pubAck1.Duplicate) pubAck2, err := secondPublisherClient.js.Publish(msgSubject, msgData, msgIdOpt) require_NoError(t, err) require_Equal(t, seq, pubAck2.Sequence) require_True(t, pubAck2.Duplicate) } } func TestJetStreamClusterConsumeWithStartSequence(t *testing.T) { const ( NumMessages = 10 ChosenSeq = 5 StreamName = "TEST" StreamSubject = "ORDERS.*" StreamSubjectPrefix = "ORDERS." ) for _, ClusterSize := range []int{ 1, // Single server 3, // 3-node cluster } { R := ClusterSize t.Run( fmt.Sprintf("Nodes:%d,Replicas:%d", ClusterSize, R), func(t *testing.T) { // This is the success condition for all sub-tests below var ExpectedMsgId = "" checkMessage := func(t *testing.T, msg *nats.Msg) { t.Helper() msgMeta, err := msg.Metadata() require_NoError(t, err) // Check sequence number require_Equal(t, msgMeta.Sequence.Stream, ChosenSeq) // Check message id require_NotEqual(t, ExpectedMsgId, "") require_Equal(t, msg.Header.Get(nats.MsgIdHdr), ExpectedMsgId) } checkRawMessage := func(t *testing.T, msg *nats.RawStreamMsg) { t.Helper() // Check sequence number require_Equal(t, msg.Sequence, ChosenSeq) // Check message id require_NotEqual(t, ExpectedMsgId, "") require_Equal(t, msg.Header.Get(nats.MsgIdHdr), ExpectedMsgId) } // Setup: start server or cluster, connect client var server *Server if ClusterSize == 1 { server = RunBasicJetStreamServer(t) defer server.Shutdown() } else { c := createJetStreamCluster(t, jsClusterTempl, "HUB", _EMPTY_, ClusterSize, 22020, true) defer c.shutdown() server = c.randomServer() } // Setup: connect var nc *nats.Conn var js nats.JetStreamContext nc, js = jsClientConnect(t, server) defer nc.Close() // Setup: create stream _, err := js.AddStream(&nats.StreamConfig{ Replicas: R, Name: StreamName, Subjects: []string{StreamSubject}, }) require_NoError(t, err) // Setup: populate stream buf := make([]byte, 100) for i := uint64(1); i <= NumMessages; i++ { msgId := nuid.Next() pubAck, err := js.Publish(StreamSubjectPrefix+strconv.Itoa(int(i)), buf, nats.MsgId(msgId)) require_NoError(t, err) // Verify assumption made in tests below require_Equal(t, pubAck.Sequence, i) if i == ChosenSeq { // Save the expected message id for the chosen message ExpectedMsgId = msgId } } // Setup: create subscriptions, needs to be after stream creation or OptStartSeq could be clipped var preCreatedSub, preCreatedSubDurable *nats.Subscription { preCreatedSub, err = js.PullSubscribe( StreamSubject, "", nats.StartSequence(ChosenSeq), ) require_NoError(t, err) defer func() { require_NoError(t, preCreatedSub.Unsubscribe()) }() const Durable = "dlc_pre_created" c, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{ Durable: Durable, DeliverPolicy: nats.DeliverByStartSequencePolicy, OptStartSeq: ChosenSeq, Replicas: R, }) require_NoError(t, err) defer func() { require_NoError(t, js.DeleteConsumer(c.Stream, c.Name)) }() preCreatedSubDurable, err = js.PullSubscribe( "", "", nats.Bind(StreamName, Durable), ) require_NoError(t, err) defer func() { require_NoError(t, preCreatedSubDurable.Unsubscribe()) }() } // Tests various ways to consume the stream starting at the ChosenSeq sequence t.Run( "DurableConsumer", func(t *testing.T) { const Durable = "dlc" c, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{ Durable: Durable, DeliverPolicy: nats.DeliverByStartSequencePolicy, OptStartSeq: ChosenSeq, Replicas: R, }) require_NoError(t, err) defer func() { require_NoError(t, js.DeleteConsumer(c.Stream, c.Name)) }() sub, err := js.PullSubscribe( StreamSubject, Durable, ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) checkMessage(t, msgs[0]) }, ) t.Run( "DurableConsumerWithBind", func(t *testing.T) { const Durable = "dlc_bind" c, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{ Durable: Durable, DeliverPolicy: nats.DeliverByStartSequencePolicy, OptStartSeq: ChosenSeq, Replicas: R, }) require_NoError(t, err) defer func() { require_NoError(t, js.DeleteConsumer(c.Stream, c.Name)) }() sub, err := js.PullSubscribe( "", "", nats.Bind(StreamName, Durable), ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) checkMessage(t, msgs[0]) }, ) t.Run( "PreCreatedDurableConsumerWithBind", func(t *testing.T) { msgs, err := preCreatedSubDurable.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) checkMessage(t, msgs[0]) }, ) t.Run( "PullConsumer", func(t *testing.T) { sub, err := js.PullSubscribe( StreamSubject, "", nats.StartSequence(ChosenSeq), ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) checkMessage(t, msgs[0]) }, ) t.Run( "PreCreatedPullConsumer", func(t *testing.T) { msgs, err := preCreatedSub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) checkMessage(t, msgs[0]) }, ) t.Run( "SynchronousConsumer", func(t *testing.T) { sub, err := js.SubscribeSync( StreamSubject, nats.StartSequence(ChosenSeq), ) if err != nil { return } require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() msg, err := sub.NextMsg(1 * time.Second) require_NoError(t, err) checkMessage(t, msg) }, ) t.Run( "CallbackSubscribe", func(t *testing.T) { var waitGroup sync.WaitGroup waitGroup.Add(1) // To be populated by callback var receivedMsg *nats.Msg sub, err := js.Subscribe( StreamSubject, func(msg *nats.Msg) { // Save first message received if receivedMsg == nil { receivedMsg = msg waitGroup.Done() } }, nats.StartSequence(ChosenSeq), ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() waitGroup.Wait() require_NotNil(t, receivedMsg) checkMessage(t, receivedMsg) }, ) t.Run( "ChannelSubscribe", func(t *testing.T) { msgChannel := make(chan *nats.Msg, 1) sub, err := js.ChanSubscribe( StreamSubject, msgChannel, nats.StartSequence(ChosenSeq), ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() msg := <-msgChannel checkMessage(t, msg) }, ) t.Run( "GetRawStreamMessage", func(t *testing.T) { rawMsg, err := js.GetMsg(StreamName, ChosenSeq) require_NoError(t, err) checkRawMessage(t, rawMsg) }, ) t.Run( "GetLastMessageBySubject", func(t *testing.T) { rawMsg, err := js.GetLastMsg( StreamName, fmt.Sprintf("ORDERS.%d", ChosenSeq), ) require_NoError(t, err) checkRawMessage(t, rawMsg) }, ) }, ) } } func TestJetStreamClusterAckDeleted(t *testing.T) { const ( NumMessages = 10 StreamName = "TEST" StreamSubject = "ORDERS.*" StreamSubjectPrefix = "ORDERS." ) for _, ClusterSize := range []int{ 1, // Single server 3, // 3-node cluster } { R := ClusterSize t.Run( fmt.Sprintf("Nodes:%d,Replicas:%d", ClusterSize, R), func(t *testing.T) { // Setup: start server or cluster, connect client var server *Server if ClusterSize == 1 { server = RunBasicJetStreamServer(t) defer server.Shutdown() } else { c := createJetStreamCluster(t, jsClusterTempl, "HUB", _EMPTY_, ClusterSize, 22020, true) defer c.shutdown() server = c.randomServer() } // Setup: connect var nc *nats.Conn var js nats.JetStreamContext nc, js = jsClientConnect(t, server) defer nc.Close() // Setup: create stream _, err := js.AddStream(&nats.StreamConfig{ Replicas: R, Name: StreamName, Subjects: []string{StreamSubject}, Retention: nats.LimitsPolicy, Discard: nats.DiscardOld, MaxMsgs: 1, // Only keep the latest message }) require_NoError(t, err) // Setup: create durable consumer and subscription const Durable = "dlc" c, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{ Durable: Durable, Replicas: R, AckPolicy: nats.AckExplicitPolicy, MaxAckPending: NumMessages, }) require_NoError(t, err) defer func() { require_NoError(t, js.DeleteConsumer(c.Stream, c.Name)) }() // Setup: create durable consumer subscription sub, err := js.PullSubscribe( "", "", nats.Bind(StreamName, Durable), ) require_NoError(t, err) defer func() { require_NoError(t, sub.Unsubscribe()) }() // Collect received and non-ACKed messages receivedMessages := make([]*nats.Msg, 0, NumMessages) buf := make([]byte, 100) for i := uint64(1); i <= NumMessages; i++ { // Publish one message msgId := nuid.Next() pubAck, err := js.Publish( StreamSubjectPrefix+strconv.Itoa(int(i)), buf, nats.MsgId(msgId), ) require_NoError(t, err) require_Equal(t, pubAck.Sequence, i) // Consume message msgs, err := sub.Fetch(1) require_NoError(t, err) require_Equal(t, len(msgs), 1) // Validate message msg := msgs[0] require_Equal(t, msgs[0].Header.Get(nats.MsgIdHdr), msgId) // Validate message metadata msgMeta, err := msg.Metadata() require_NoError(t, err) // Check sequence number require_Equal(t, msgMeta.Sequence.Stream, i) // Save for ACK later receivedMessages = append(receivedMessages, msg) } // Verify stream state, expecting a single message due to limits streamInfo, err := js.StreamInfo(StreamName) require_NoError(t, err) require_Equal(t, streamInfo.State.Msgs, 1) // Verify consumer state, expecting ack floor corresponding to messages dropped consumerInfo, err := js.ConsumerInfo(StreamName, Durable) require_NoError(t, err) require_Equal(t, consumerInfo.NumAckPending, 1) require_Equal(t, consumerInfo.AckFloor.Stream, 9) require_Equal(t, consumerInfo.AckFloor.Consumer, 9) // ACK all messages (all except last have been dropped from the stream) for _, message := range receivedMessages { err := message.AckSync() require_NoError(t, err) } // Verify consumer state, all messages ACKed consumerInfo, err = js.ConsumerInfo(StreamName, Durable) require_NoError(t, err) require_Equal(t, consumerInfo.NumAckPending, 0) require_Equal(t, consumerInfo.AckFloor.Stream, 10) require_Equal(t, consumerInfo.AckFloor.Consumer, 10) }, ) } } func TestJetStreamClusterAPILimitDefault(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() for _, s := range c.servers { s.optsMu.RLock() lim := s.opts.JetStreamRequestQueueLimit s.optsMu.RUnlock() require_Equal(t, lim, JSDefaultRequestQueueLimit) require_Equal(t, atomic.LoadInt64(&s.getJetStream().queueLimit), JSDefaultRequestQueueLimit) } } func TestJetStreamClusterAPILimitAdvisory(t *testing.T) { // Hit the limit straight away. const queueLimit = 1 config := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: 256MB max_file_store: 2GB store_dir: '%s' request_queue_limit: ` + fmt.Sprintf("%d", queueLimit) + ` } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := createJetStreamClusterWithTemplate(t, config, "R3S", 3) defer c.shutdown() c.waitOnLeader() s := c.randomNonLeader() for _, s := range c.servers { lim := atomic.LoadInt64(&s.getJetStream().queueLimit) require_Equal(t, lim, queueLimit) } nc, _ := jsClientConnect(t, s) defer nc.Close() snc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() sub, err := snc.SubscribeSync(JSAdvisoryAPILimitReached) require_NoError(t, err) // There's a very slim chance that a worker could pick up a request between // pushing to and draining the queue, so make sure we've sent enough of them // to reliably trigger a drain and advisory. inbox := nc.NewRespInbox() for i := 0; i < runtime.GOMAXPROCS(-1)*2; i++ { require_NoError(t, nc.PublishMsg(&nats.Msg{ Subject: fmt.Sprintf(JSApiConsumerListT, "TEST"), Reply: inbox, })) } // Wait for the advisory to come in. msg, err := sub.NextMsg(time.Second * 5) require_NoError(t, err) var advisory JSAPILimitReachedAdvisory require_NoError(t, json.Unmarshal(msg.Data, &advisory)) require_Equal(t, advisory.Domain, _EMPTY_) // No JetStream domain was set. require_True(t, advisory.Dropped >= 1) // We dropped at least something. } func TestJetStreamPendingRequestsInJsz(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() metaleader := c.leader() sjs := metaleader.getJetStream() sjs.mu.Lock() sub := &subscription{ subject: []byte("$JS.API.VERY_SLOW"), icb: func(sub *subscription, client *client, acc *Account, subject, reply string, rmsg []byte) { select { case <-client.srv.quitCh: case <-time.After(time.Second * 3): } }, } err := metaleader.getJetStream().apiSubs.Insert(sub) sjs.mu.Unlock() require_NoError(t, err) nc, _ := jsClientConnect(t, c.randomNonLeader()) defer nc.Close() inbox := nc.NewRespInbox() msg := &nats.Msg{ Subject: "$JS.API.VERY_SLOW", Reply: inbox, } // Fall short of hitting the API limit by a little bit, // otherwise the requests get drained away. for i := 0; i < JSDefaultRequestQueueLimit-10; i++ { require_NoError(t, nc.PublishMsg(msg)) } jsz, err := metaleader.Jsz(nil) require_NoError(t, err) require_True(t, jsz.Meta != nil) require_NotEqual(t, jsz.Meta.Pending, 0) snc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() ch := make(chan *nats.Msg, 1) ssub, err := snc.ChanSubscribe(fmt.Sprintf(serverStatsSubj, metaleader.ID()), ch) require_NoError(t, err) require_NoError(t, ssub.AutoUnsubscribe(1)) msg = require_ChanRead(t, ch, time.Second*5) var m ServerStatsMsg require_NoError(t, json.Unmarshal(msg.Data, &m)) require_True(t, m.Stats.JetStream != nil) require_NotEqual(t, m.Stats.JetStream.Meta.Pending, 0) } func TestJetStreamConsumerReplicasAfterScale(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomNonLeader()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 5, }) require_NoError(t, err) // Put some messages in to test consumer state transfer. for i := 0; i < 100; i++ { js.PublishAsync("foo", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Create four different consumers. // Normal where we inherit replicas from parent. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 0) require_Equal(t, len(ci.Cluster.Replicas), 4) // Ephemeral ci, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 0) // Legacy ephemeral is 0 here too. require_Equal(t, len(ci.Cluster.Replicas), 0) eName := ci.Name // R1 ci, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "r1", AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 1) require_Equal(t, len(ci.Cluster.Replicas), 0) // R3 ci, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "r3", AckPolicy: nats.AckExplicitPolicy, Replicas: 3, }) require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 3) require_Equal(t, len(ci.Cluster.Replicas), 2) // Now create some state on r1 consumer. sub, err := js.PullSubscribe("foo", "r1") require_NoError(t, err) fetch := rand.Intn(99) + 1 // Needs to be at least 1. msgs, err := sub.Fetch(fetch, nats.MaxWait(10*time.Second)) require_NoError(t, err) require_Equal(t, len(msgs), fetch) ack := rand.Intn(fetch) for i := 0; i <= ack; i++ { msgs[i].AckSync() } r1ci, err := js.ConsumerInfo("TEST", "r1") require_NoError(t, err) r1ci.Delivered.Last, r1ci.AckFloor.Last = nil, nil // Now scale stream to R3. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") // Now check each. c.waitOnConsumerLeader(globalAccountName, "TEST", "dur") ci, err = js.ConsumerInfo("TEST", "dur") require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 0) require_Equal(t, len(ci.Cluster.Replicas), 2) c.waitOnConsumerLeader(globalAccountName, "TEST", eName) ci, err = js.ConsumerInfo("TEST", eName) require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 0) require_Equal(t, len(ci.Cluster.Replicas), 0) c.waitOnConsumerLeader(globalAccountName, "TEST", "r1") ci, err = js.ConsumerInfo("TEST", "r1") require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 1) require_Equal(t, len(ci.Cluster.Replicas), 0) // Now check that state transferred correctly. ci.Delivered.Last, ci.AckFloor.Last = nil, nil if ci.Delivered != r1ci.Delivered { t.Fatalf("Delivered state for R1 incorrect, wanted %+v got %+v", r1ci.Delivered, ci.Delivered) } if ci.AckFloor != r1ci.AckFloor { t.Fatalf("AckFloor state for R1 incorrect, wanted %+v got %+v", r1ci.AckFloor, ci.AckFloor) } c.waitOnConsumerLeader(globalAccountName, "TEST", "r3") ci, err = js.ConsumerInfo("TEST", "r3") require_NoError(t, err) require_Equal(t, ci.Config.Replicas, 3) require_Equal(t, len(ci.Cluster.Replicas), 2) } func TestJetStreamClusterDesyncAfterQuitDuringCatchup(t *testing.T) { for title, test := range map[string]func(s *Server, rn RaftNode){ "RAFT": func(s *Server, rn RaftNode) { rn.Stop() rn.WaitForStop() }, "server": func(s *Server, rn RaftNode) { s.running.Store(false) }, } { t.Run(title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Wait for all servers to have applied everything up to this point. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) if err != nil { return err } mset, err := acc.lookupStream("TEST") if err != nil { return err } _, _, applied := mset.raftNode().Progress() if applied != 1 { return fmt.Errorf("expected applied to be %d, got %d", 1, applied) } } return nil }) rs := c.randomNonStreamLeader(globalAccountName, "TEST") acc, err := rs.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) rn := mset.raftNode() snap, err := json.Marshal(streamSnapshot{Msgs: 1, Bytes: 1, FirstSeq: 100, LastSeq: 100, Failed: 0, Deleted: nil}) require_NoError(t, err) esm := encodeStreamMsgAllowCompress("foo", _EMPTY_, nil, nil, 0, 0, false) // Lock stream so that we can go into processSnapshot but must wait for this to unlock. mset.mu.Lock() var unlocked bool defer func() { if !unlocked { mset.mu.Unlock() } }() rn.ApplyQ().push(newCommittedEntry(100, []*Entry{newEntry(EntrySnapshot, snap)})) rn.ApplyQ().push(newCommittedEntry(101, []*Entry{newEntry(EntryNormal, esm)})) // Waiting for the apply queue entry to be captured in monitorStream first. time.Sleep(time.Second) // Set commit to a very high number, just so that we allow upping Applied() n := rn.(*raft) n.Lock() n.commit = 1000 n.Unlock() // Now stop the underlying RAFT node/server so processSnapshot must exit because of it. test(rs, rn) mset.mu.Unlock() unlocked = true // Allow some time for the applied number to be updated, in which case it's an error. time.Sleep(time.Second) _, _, applied := mset.raftNode().Progress() require_Equal(t, applied, 1) }) } } func TestJetStreamClusterDesyncAfterErrorDuringCatchup(t *testing.T) { tests := []struct { title string onErrorCondition func(server *Server, mset *stream) }{ { title: "TooManyRetries", onErrorCondition: func(server *Server, mset *stream) { // Too many retries while processing snapshot is considered a cluster reset. // If a leader is temporarily unavailable we shouldn't blow away our state. require_True(t, isClusterResetErr(errCatchupTooManyRetries)) mset.resetClusteredState(errCatchupTooManyRetries) }, }, { title: "AbortedNoLeader", onErrorCondition: func(server *Server, mset *stream) { for _, n := range server.raftNodes { rn := n.(*raft) if rn.accName == "$G" { rn.Lock() rn.updateLeader(noLeader) rn.Unlock() } } // Processing a snapshot while there's no leader elected is considered a cluster reset. // If a leader is temporarily unavailable we shouldn't blow away our state. var snap StreamReplicatedState snap.LastSeq = 1_000 // ensure we can catchup based on the snapshot err := mset.processSnapshot(&snap) require_True(t, errors.Is(err, errCatchupAbortedNoLeader)) require_True(t, isClusterResetErr(err)) mset.resetClusteredState(err) }, }, } for _, test := range tests { t.Run(test.title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) streamLeader := si.Cluster.Leader streamLeaderServer := c.serverByName(streamLeader) nc.Close() nc, js = jsClientConnect(t, streamLeaderServer) defer nc.Close() servers := slices.DeleteFunc([]string{"S-1", "S-2", "S-3"}, func(s string) bool { return s == streamLeader }) // Publish 10 messages. for i := 0; i < 10; i++ { pubAck, err := js.Publish("foo", []byte("ok")) require_NoError(t, err) require_Equal(t, pubAck.Sequence, uint64(i+1)) } outdatedServerName := servers[0] clusterResetServerName := servers[1] outdatedServer := c.serverByName(outdatedServerName) outdatedServer.Shutdown() outdatedServer.WaitForShutdown() // Publish 10 more messages, one server will be behind. for i := 0; i < 10; i++ { pubAck, err := js.Publish("foo", []byte("ok")) require_NoError(t, err) require_Equal(t, pubAck.Sequence, uint64(i+11)) } // We will not need the client anymore. nc.Close() // Shutdown stream leader so one server remains. streamLeaderServer.Shutdown() streamLeaderServer.WaitForShutdown() clusterResetServer := c.serverByName(clusterResetServerName) acc, err := clusterResetServer.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) // Run error condition. test.onErrorCondition(clusterResetServer, mset) // Stream leader stays offline, we only start the server with missing stream data. // We expect that the reset server must not allow the outdated server to become leader, as that would result in desync. c.restartServer(outdatedServer) c.waitOnStreamLeader(globalAccountName, "TEST") // Outdated server must NOT become the leader. newStreamLeaderServer := c.streamLeader(globalAccountName, "TEST") require_Equal(t, newStreamLeaderServer.Name(), clusterResetServerName) }) } } func TestJetStreamClusterDontInstallSnapshotWhenStoppingStream(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) // Wait for all servers to have applied everything. var maxApplied uint64 checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { maxApplied = 0 for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) if err != nil { return err } mset, err := acc.lookupStream("TEST") if err != nil { return err } _, _, applied := mset.node.Progress() if maxApplied == 0 { maxApplied = applied } else if applied < maxApplied { return fmt.Errorf("applied not high enough, expected %d, got %d", applied, maxApplied) } else if applied > maxApplied { return fmt.Errorf("applied higher on one server, expected %d, got %d", applied, maxApplied) } } return nil }) // Install a snapshot on a follower. s := c.randomNonStreamLeader(globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) err = mset.node.InstallSnapshot(mset.stateSnapshotLocked()) require_NoError(t, err) // Validate the snapshot reflects applied. validateStreamState := func(snap *snapshot) { t.Helper() require_Equal(t, snap.lastIndex, maxApplied) ss, err := DecodeStreamState(snap.data) require_NoError(t, err) require_Equal(t, ss.FirstSeq, 1) require_Equal(t, ss.LastSeq, 1) } snap, err := mset.node.(*raft).loadLastSnapshot() require_NoError(t, err) validateStreamState(snap) // Simulate a message being stored, but not calling Applied yet. err = mset.processJetStreamMsg("foo", _EMPTY_, nil, nil, 1, time.Now().UnixNano()) require_NoError(t, err) // Simulate the stream being stopped before we're able to call Applied. // If we'd install a snapshot during this, which would be a race condition, // we'd store a snapshot with state that's ahead of applied. err = mset.stop(false, false) require_NoError(t, err) // Validate the snapshot is the same as before. snap, err = mset.node.(*raft).loadLastSnapshot() require_NoError(t, err) validateStreamState(snap) } func TestJetStreamClusterDontInstallSnapshotWhenStoppingConsumer(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "CONSUMER", Replicas: 3, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // Add a message and let the consumer ack it, this moves the consumer's RAFT applied up. _, err = js.Publish("foo", nil) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "CONSUMER") require_NoError(t, err) msgs, err := sub.Fetch(1) require_NoError(t, err) require_Len(t, len(msgs), 1) err = msgs[0].AckSync() require_NoError(t, err) // Wait for all servers to have applied everything. var maxApplied uint64 checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { maxApplied = 0 for _, s := range c.servers { acc, err := s.lookupAccount(globalAccountName) if err != nil { return err } mset, err := acc.lookupStream("TEST") if err != nil { return err } o := mset.lookupConsumer("CONSUMER") if o == nil { return errors.New("consumer not found") } _, _, applied := o.node.Progress() if maxApplied == 0 { maxApplied = applied } else if applied < maxApplied { return fmt.Errorf("applied not high enough, expected %d, got %d", applied, maxApplied) } else if applied > maxApplied { return fmt.Errorf("applied higher on one server, expected %d, got %d", applied, maxApplied) } } return nil }) // Install a snapshot on a follower. s := c.randomNonStreamLeader(globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) snapBytes, err := o.store.EncodedState() require_NoError(t, err) err = o.node.InstallSnapshot(snapBytes) require_NoError(t, err) // Validate the snapshot reflects applied. validateStreamState := func(snap *snapshot) { t.Helper() require_Equal(t, snap.lastIndex, maxApplied) state, err := decodeConsumerState(snap.data) require_NoError(t, err) require_Equal(t, state.Delivered.Consumer, 1) require_Equal(t, state.Delivered.Stream, 1) } snap, err := o.node.(*raft).loadLastSnapshot() require_NoError(t, err) validateStreamState(snap) // Simulate a message being delivered, but not calling Applied yet. err = o.store.UpdateDelivered(2, 2, 1, time.Now().UnixNano()) require_NoError(t, err) // Simulate the consumer being stopped before we're able to call Applied. // If we'd install a snapshot during this, which would be a race condition, // we'd store a snapshot with state that's ahead of applied. err = o.stop() require_NoError(t, err) // Validate the snapshot is the same as before. snap, err = o.node.(*raft).loadLastSnapshot() require_NoError(t, err) validateStreamState(snap) } func TestJetStreamClusterDesyncAfterRestartReplacesLeaderSnapshot(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) // Reconnect to the leader. leader := c.streamLeader(globalAccountName, "TEST") nc.Close() nc, js = jsClientConnect(t, leader) defer nc.Close() lookupStream := func(s *Server) *stream { t.Helper() acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) return mset } // Stop one follower so it lags behind. rs := c.randomNonStreamLeader(globalAccountName, "TEST") mset := lookupStream(rs) n := mset.node.(*raft) followerSnapshots := path.Join(n.sd, snapshotsDir) rs.Shutdown() rs.WaitForShutdown() // Move the stream forward so the follower requires a snapshot. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 10}) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) // Install a snapshot on the leader, ensuring RAFT entries are compacted and a snapshot remains. mset = lookupStream(leader) n = mset.node.(*raft) err = n.InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) c.stopAll() // Replace follower snapshot with the leader's. // This simulates the follower coming online, getting a snapshot from the leader after which it goes offline. leaderSnapshots := path.Join(n.sd, snapshotsDir) err = os.RemoveAll(followerSnapshots) require_NoError(t, err) err = copyDir(t, followerSnapshots, leaderSnapshots) require_NoError(t, err) // Start the follower, it will load the snapshot from the leader. rs = c.restartServer(rs) // Shutting down must check that the leader's snapshot is not overwritten. rs.Shutdown() rs.WaitForShutdown() // Now start all servers back up. c.restartAll() c.waitOnAllCurrent() checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { return checkState(t, c, globalAccountName, "TEST") }) } func TestJetStreamClusterKeepRaftStateIfStreamCreationFailedDuringShutdown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) nc.Close() // Capture RAFT storage directory and JetStream handle before shutdown. s := c.randomNonStreamLeader(globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) sd := mset.node.(*raft).sd jss := s.getJetStream() // Shutdown the server. // Normally there are no actions taken anymore after shutdown completes, // but still do so to simulate actions taken while shutdown is in progress. s.Shutdown() s.WaitForShutdown() // Check RAFT state is kept. files, err := os.ReadDir(sd) require_NoError(t, err) require_True(t, len(files) > 0) // Simulate server shutting down, JetStream being disabled and a stream being created. sa := &streamAssignment{ Config: &StreamConfig{Name: "TEST"}, Group: &raftGroup{node: &raft{}}, } jss.processClusterCreateStream(acc, sa) // Check RAFT state is not deleted due to failing stream creation. files, err = os.ReadDir(sd) require_NoError(t, err) require_True(t, len(files) > 0) } func TestJetStreamClusterMetaSnapshotMustNotIncludePendingConsumers(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) require_NoError(t, err) // We're creating an R3 consumer, just so we can copy its state and turn it into pending below. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Name: "consumer", Replicas: 3}) require_NoError(t, err) nc.Close() // Bypass normal API so we can simulate having a consumer pending to be created. // A snapshot should never create pending consumers, as that would result // in ghost consumers if the meta proposal failed. ml := c.leader() mjs := ml.getJetStream() cc := mjs.cluster consumers := cc.streams[globalAccountName]["TEST"].consumers sampleCa := *consumers["consumer"] sampleCa.Name, sampleCa.pending = "pending-consumer", true consumers[sampleCa.Name] = &sampleCa // Create snapshot, this should not contain pending consumers. snap, err := mjs.metaSnapshot() require_NoError(t, err) ru := &recoveryUpdates{ removeStreams: make(map[string]*streamAssignment), removeConsumers: make(map[string]map[string]*consumerAssignment), addStreams: make(map[string]*streamAssignment), updateStreams: make(map[string]*streamAssignment), updateConsumers: make(map[string]map[string]*consumerAssignment), } err = mjs.applyMetaSnapshot(snap, ru, true) require_NoError(t, err) require_Len(t, len(ru.updateStreams), 1) for _, sa := range ru.updateStreams { for _, ca := range sa.consumers { require_NotEqual(t, ca.Name, "pending-consumer") } } for _, cas := range ru.updateConsumers { for _, ca := range cas { require_NotEqual(t, ca.Name, "pending-consumer") } } } func TestJetStreamClusterDesyncAfterPublishToLeaderWithoutQuorum(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) streamLeader := si.Cluster.Leader streamLeaderServer := c.serverByName(streamLeader) nc.Close() nc, js = jsClientConnect(t, streamLeaderServer) defer nc.Close() servers := slices.DeleteFunc([]string{"S-1", "S-2", "S-3"}, func(s string) bool { return s == streamLeader }) // Stop followers so further publishes will not have quorum. followerName1 := servers[0] followerName2 := servers[1] followerServer1 := c.serverByName(followerName1) followerServer2 := c.serverByName(followerName2) followerServer1.Shutdown() followerServer2.Shutdown() followerServer1.WaitForShutdown() followerServer2.WaitForShutdown() // Although this request will time out, it will be added to the stream leader's WAL. _, err = js.Publish("foo", []byte("first"), nats.AckWait(time.Second)) require_NotNil(t, err) require_Equal(t, err, nats.ErrTimeout) // Now shut down the leader as well. nc.Close() streamLeaderServer.Shutdown() streamLeaderServer.WaitForShutdown() // Only restart the (previous) followers. followerServer1 = c.restartServer(followerServer1) c.restartServer(followerServer2) c.waitOnStreamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, followerServer1) defer nc.Close() // Publishing a message will now have quorum. pubAck, err := js.Publish("foo", []byte("first, this is a retry")) require_NoError(t, err) require_Equal(t, pubAck.Sequence, 1) // Bring up the previous stream leader. c.restartServer(streamLeaderServer) c.waitOnAllCurrent() c.waitOnStreamLeader(globalAccountName, "TEST") // Check all servers ended up with the last published message, which had quorum. for _, s := range c.servers { c.waitOnStreamCurrent(s, globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) state := mset.state() require_Equal(t, state.Msgs, 1) require_Equal(t, state.Bytes, 55) } } func TestJetStreamClusterPreserveWALDuringCatchupWithMatchingTerm(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, Replicas: 3, }) nc.Close() require_NoError(t, err) // Pick one server that will only store a part of the messages in its WAL. rs := c.randomNonStreamLeader(globalAccountName, "TEST") ts := time.Now().UnixNano() // Manually add 3 append entries to each node's WAL, except for one node who is one behind. var scratch [1024]byte for _, s := range c.servers { for _, n := range s.raftNodes { rn := n.(*raft) if rn.accName == globalAccountName { for i := uint64(0); i < 3; i++ { // One server will be one behind and need to catchup. if s.Name() == rs.Name() && i >= 2 { break } esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, i, ts, true) entries := []*Entry{newEntry(EntryNormal, esm)} rn.Lock() ae := rn.buildAppendEntry(entries) ae.buf, err = ae.encode(scratch[:]) require_NoError(t, err) err = rn.storeToWAL(ae) rn.Unlock() require_NoError(t, err) } } } } // Restart all. c.stopAll() c.restartAll() c.waitOnAllCurrent() c.waitOnStreamLeader(globalAccountName, "TEST") rs = c.serverByName(rs.Name()) // Check all servers ended up with all published messages, which had quorum. for _, s := range c.servers { c.waitOnStreamCurrent(s, globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) state := mset.state() require_Equal(t, state.Msgs, 3) require_Equal(t, state.Bytes, 99) } // Check that the first two published messages came from our WAL, and // the last came from a catchup by another leader. for _, n := range rs.raftNodes { rn := n.(*raft) if rn.accName == globalAccountName { ae, err := rn.loadEntry(2) require_NoError(t, err) require_True(t, ae.leader == rn.ID()) ae, err = rn.loadEntry(3) require_NoError(t, err) require_True(t, ae.leader == rn.ID()) ae, err = rn.loadEntry(4) require_NoError(t, err) require_True(t, ae.leader != rn.ID()) } } } func TestJetStreamClusterReservedResourcesAccountingAfterClusterReset(t *testing.T) { for _, clusterResetErr := range []error{errLastSeqMismatch, errFirstSequenceMismatch} { t.Run(clusterResetErr.Error(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() maxBytes := int64(1024 * 1024 * 1024) _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, MaxBytes: maxBytes, }) require_NoError(t, err) sl := c.streamLeader(globalAccountName, "TEST") mem, store, err := sl.JetStreamReservedResources() require_NoError(t, err) require_Equal(t, mem, 0) require_Equal(t, store, maxBytes) acc, err := sl.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) sjs := sl.getJetStream() rn := mset.raftNode() sa := mset.streamAssignment() sjs.mu.RLock() saGroupNode := sa.Group.node sjs.mu.RUnlock() require_NotNil(t, sa) require_Equal(t, rn, saGroupNode) require_True(t, mset.resetClusteredState(clusterResetErr)) checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { sjs.mu.RLock() defer sjs.mu.RUnlock() if sa.Group.node == nil || sa.Group.node == saGroupNode { return errors.New("waiting for reset to complete") } return nil }) mem, store, err = sl.JetStreamReservedResources() require_NoError(t, err) require_Equal(t, mem, 0) require_Equal(t, store, maxBytes) }) } } func TestJetStreamClusterHardKillAfterStreamAdd(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) copyDir := func(dst, src string) error { srcFS := os.DirFS(src) return fs.WalkDir(srcFS, ".", func(p string, d os.DirEntry, err error) error { if err != nil { return err } newPath := path.Join(dst, p) if d.IsDir() { return os.MkdirAll(newPath, defaultDirPerms) } r, err := srcFS.Open(p) if err != nil { return err } defer r.Close() w, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, defaultFilePerms) if err != nil { return err } defer w.Close() _, err = io.Copy(w, r) return err }) } // Simulate being hard killed by: // 1. copy directories before shutdown copyToSrcMap := make(map[string]string) for _, s := range c.servers { sd := s.StoreDir() copySd := path.Join(t.TempDir(), JetStreamStoreDir) err = copyDir(copySd, sd) require_NoError(t, err) copyToSrcMap[copySd] = sd } // 2. stop all nc.Close() c.stopAll() // 3. revert directories to before shutdown for cp, dest := range copyToSrcMap { err = os.RemoveAll(dest) require_NoError(t, err) err = copyDir(dest, cp) require_NoError(t, err) } // 4. restart c.restartAll() c.waitOnAllCurrent() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Stream should exist still and not be removed after hard killing all servers, so expect no error. _, err = js.StreamInfo("TEST") require_NoError(t, err) } func TestJetStreamClusterStreamConsumerStateResetAfterRecreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() stream := "test:0" config := &nats.StreamConfig{ Name: stream, Subjects: []string{"test.0.*"}, Replicas: 3, Retention: nats.WorkQueuePolicy, MaxMsgs: 100_000, Discard: nats.DiscardNew, Duplicates: 5 * time.Second, Storage: nats.MemoryStorage, } consumer := "A:0:0" subject := "test.0.0" var ( duration = 30 * time.Minute producerMsgs = 200_000 producerMsgSize = 1024 payload = []byte(strings.Repeat("A", producerMsgSize)) wg sync.WaitGroup n atomic.Uint64 canPublish atomic.Bool ) createStream := func(t *testing.T) { t.Helper() _, err := js.AddStream(config) require_NoError(t, err) consumer := &nats.ConsumerConfig{ Durable: consumer, Replicas: 3, MaxAckPending: 100_000, MaxWaiting: 100_000, FilterSubject: subject, AckPolicy: nats.AckExplicitPolicy, } _, err = js.AddConsumer(stream, consumer) require_NoError(t, err) } deleteStream := func(t *testing.T) { err := js.DeleteStream(stream) require_NoError(t, err) } stopPublishing := func() { canPublish.Store(false) } resumePublishing := func() { canPublish.Store(true) } // Setup stream ctx, cancel := context.WithTimeout(context.Background(), duration) defer cancel() createStream(t) // Setup producer resumePublishing() wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for range time.NewTicker(1 * time.Millisecond).C { select { case <-ctx.Done(): return default: } if !canPublish.Load() { continue } _, err := js.Publish("test.0.0", payload, nats.AckWait(200*time.Millisecond)) if err == nil { if nn := n.Add(1); int(nn) >= producerMsgs { return } } } }() // Setup consumer acked := make(chan struct{}, 100) wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() Attempts: for attempts := 0; attempts < 10; attempts++ { _, err := js.ConsumerInfo(stream, consumer) if err != nil { t.Logf("WRN: Failed creating pull subscriber: %v - %v - %v - %v", subject, stream, consumer, err) time.Sleep(200 * time.Millisecond) continue } break Attempts } sub, err := js.PullSubscribe(subject, "", nats.Bind(stream, consumer)) if err != nil { t.Logf("WRN: Failed creating pull subscriber: %v - %v - %v - %v", subject, stream, consumer, err) return } require_NoError(t, err) for range time.NewTicker(100 * time.Millisecond).C { select { case <-ctx.Done(): return default: } msgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { time.AfterFunc(3*time.Second, func() { select { case <-ctx.Done(): return default: } msg.Ack() acked <- struct{}{} }) } msgs, err = sub.Fetch(10, nats.MaxWait(200*time.Millisecond)) if err != nil { continue } for _, msg := range msgs { msg.Ack() } } }() // Let publish and consume to happen for a bit. time.Sleep(2 * time.Second) // Recreate the stream deleteStream(t) stopPublishing() createStream(t) for i := 0; i < 3; i++ { js.Publish("test.0.0", payload, nats.AckWait(200*time.Millisecond)) } select { case <-time.After(5 * time.Second): t.Fatal("Timed out waiting for ack") case <-acked: time.Sleep(2 * time.Second) } sinfo, err := js.StreamInfo(stream) require_NoError(t, err) cinfo, err := js.ConsumerInfo(stream, consumer) require_NoError(t, err) cancel() if cinfo.Delivered.Stream > sinfo.State.LastSeq { t.Fatalf("Consumer Stream sequence is ahead of Stream LastSeq: consumer=%d, stream=%d", cinfo.Delivered.Stream, sinfo.State.LastSeq) } wg.Wait() } func TestJetStreamClusterOnlyPublishAdvisoriesWhenInterest(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() subj := "$JS.ADVISORY.TEST" s1 := c.servers[0] s2 := c.servers[1] // On the first server, see if we think the advisory will be published. require_False(t, s1.publishAdvisory(s1.GlobalAccount(), subj, "test")) // On the second server, subscribe to the advisory subject. nc, _ := jsClientConnect(t, s2) defer nc.Close() _, err := nc.Subscribe(subj, func(_ *nats.Msg) {}) require_NoError(t, err) // Wait for the interest to propagate to the first server. checkFor(t, time.Second, 25*time.Millisecond, func() error { if !s1.GlobalAccount().sl.HasInterest(subj) { return fmt.Errorf("expected interest in %q, not yet found", subj) } return nil }) // On the first server, try and publish the advisory again. THis time // it should succeed. require_True(t, s1.publishAdvisory(s1.GlobalAccount(), subj, "test")) } func TestJetStreamClusterRoutedAPIRecoverPerformance(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomNonLeader()) defer nc.Close() // We only run 16 JetStream API workers. mp := runtime.GOMAXPROCS(0) if mp > 16 { mp = 16 } leader := c.leader() ljs := leader.js.Load() // Take the JS lock, which allows the JS API queue to build up. ljs.mu.Lock() defer ljs.mu.Unlock() count := JSDefaultRequestQueueLimit - 1 ch := make(chan *nats.Msg, count) inbox := nc.NewRespInbox() _, err := nc.ChanSubscribe(inbox, ch) require_NoError(t, err) // To ensure a fair starting line, we need to submit as many tasks as // there are JS workers whilst holding the JS lock. This will ensure that // each JS API worker is properly wedged. msg := &nats.Msg{ Subject: fmt.Sprintf(JSApiConsumerInfoT, "Doesnt", "Exist"), Reply: "no_one_here", } for i := 0; i < mp; i++ { require_NoError(t, nc.PublishMsg(msg)) } // Then we want to submit a fixed number of tasks, big enough to fill // the queue, so that we can measure them. msg = &nats.Msg{ Subject: fmt.Sprintf(JSApiConsumerInfoT, "Doesnt", "Exist"), Reply: inbox, } for i := 0; i < count; i++ { require_NoError(t, nc.PublishMsg(msg)) } checkFor(t, 5*time.Second, 25*time.Millisecond, func() error { if queued := leader.jsAPIRoutedReqs.len(); queued != count { return fmt.Errorf("expected %d queued requests, got %d", count, queued) } return nil }) // Now we're going to release the lock and start timing. The workers // will now race to clear the queues and we'll wait to see how long // it takes for them all to respond. start := time.Now() ljs.mu.Unlock() for i := 0; i < count; i++ { <-ch } ljs.mu.Lock() t.Logf("Took %s to clear %d items", time.Since(start), count) } func TestJetStreamClusterStreamAckMsgR1SignalsRemovedMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.WorkQueuePolicy, Replicas: 1, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "CONSUMER", Replicas: 1, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) s := c.streamLeader(globalAccountName, "TEST") acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) // Too high sequence, should register pre-ack and return true allowing for retries. require_True(t, mset.ackMsg(o, 100)) var smv StoreMsg sm, err := mset.store.LoadMsg(1, &smv) require_NoError(t, err) require_Equal(t, sm.subj, "foo") // Now do a proper ack, should immediately remove the message since it's R1. require_True(t, mset.ackMsg(o, 1)) _, err = mset.store.LoadMsg(1, &smv) require_Error(t, err, ErrStoreMsgNotFound) } func TestJetStreamClusterStreamAckMsgR3SignalsRemovedMsg(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.WorkQueuePolicy, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "CONSUMER", Replicas: 3, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) getStreamAndConsumer := func(s *Server) (*stream, *consumer, error) { t.Helper() acc, err := s.lookupAccount(globalAccountName) if err != nil { return nil, nil, err } mset, err := acc.lookupStream("TEST") if err != nil { return nil, nil, err } o := mset.lookupConsumer("CONSUMER") if err != nil { return nil, nil, err } return mset, o, nil } sl := c.consumerLeader(globalAccountName, "TEST", "CONSUMER") sf := c.randomNonConsumerLeader(globalAccountName, "TEST", "CONSUMER") msetL, ol, err := getStreamAndConsumer(sl) require_NoError(t, err) msetF, of, err := getStreamAndConsumer(sf) require_NoError(t, err) // Too high sequence, should register pre-ack and return true allowing for retries. require_True(t, msetL.ackMsg(ol, 100)) require_True(t, msetF.ackMsg(of, 100)) // Let all servers ack the message. var smv StoreMsg for _, s := range c.servers { mset, _, err := getStreamAndConsumer(s) require_NoError(t, err) require_True(t, mset.ackMsg(of, 1)) _, err = mset.store.LoadMsg(1, &smv) require_Error(t, err, ErrStoreMsgNotFound) } } nats-server-2.10.27/server/jetstream_consumer_test.go000066400000000000000000001413461477524627100227710ustar00rootroot00000000000000// Copyright 2022-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests // +build !skip_js_tests package server import ( "encoding/json" "errors" "fmt" "math/rand" "slices" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" ) func TestJetStreamConsumerMultipleFiltersRemoveFilters(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three"}, MaxAge: time.Second * 90, }) require_NoError(t, err) _, err = mset.addConsumer(&ConsumerConfig{ Durable: "consumer", FilterSubjects: []string{"one", "two"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "one", "data") sendStreamMsg(t, nc, "two", "data") sendStreamMsg(t, nc, "three", "data") consumer, err := js.PullSubscribe("", "consumer", nats.Bind("TEST", "consumer")) require_NoError(t, err) msgs, err := consumer.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) _, err = mset.addConsumer(&ConsumerConfig{ Durable: "consumer", FilterSubjects: []string{}, }) require_NoError(t, err) msgs, err = consumer.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) msgs, err = consumer.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) } func TestJetStreamConsumerMultipleFiltersRace(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three", "four"}, MaxAge: time.Second * 90, }) require_NoError(t, err) var seqs []uint64 var mu sync.Mutex total := 10_000 var wg sync.WaitGroup send := func(subj string) { defer wg.Done() for i := 0; i < total; i++ { sendStreamMsg(t, nc, subj, "data") } } wg.Add(4) go send("one") go send("two") go send("three") go send("four") wg.Wait() mset.addConsumer(&ConsumerConfig{ Durable: "consumer", FilterSubjects: []string{"one", "two", "three"}, AckPolicy: AckExplicit, }) done := make(chan struct{}) wg.Add(10) for i := 0; i < 10; i++ { go func(t *testing.T) { defer wg.Done() c, err := js.PullSubscribe(_EMPTY_, "consumer", nats.Bind("TEST", "consumer")) require_NoError(t, err) for { select { case <-done: return default: } msgs, err := c.Fetch(10, nats.MaxWait(2*time.Second)) // We don't want to stop before at expected number of messages, as we want // to also test against getting to many messages. // Because of that, we ignore timeout and connection closed errors. if err != nil && err != nats.ErrTimeout && err != nats.ErrConnectionClosed { t.Errorf("error while fetching messages: %v", err) } for _, msg := range msgs { info, err := msg.Metadata() require_NoError(t, err) mu.Lock() seqs = append(seqs, info.Sequence.Consumer) mu.Unlock() msg.Ack() } } }(t) } checkFor(t, 30*time.Second, 100*time.Millisecond, func() error { mu.Lock() defer mu.Unlock() if len(seqs) != 3*total { return fmt.Errorf("found %d messages instead of %d", len(seqs), 3*total) } slices.Sort(seqs) for i := 1; i < len(seqs); i++ { if seqs[i] != seqs[i-1]+1 { fmt.Printf("seqs: %+v\n", seqs) return fmt.Errorf("sequence mismatch at %v", i) } } return nil }) close(done) wg.Wait() } func TestJetStreamConsumerMultipleConsumersSingleFilter(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() // Setup few subjects with varying messages count. subjects := []struct { subject string messages int wc bool }{ {subject: "one", messages: 5000}, {subject: "two", messages: 7500}, {subject: "three", messages: 2500}, {subject: "four", messages: 1000}, {subject: "five.>", messages: 3000, wc: true}, } totalMsgs := 0 for _, subject := range subjects { totalMsgs += subject.messages } // Setup consumers, filtering some of the messages from the stream. consumers := []*struct { name string subjects []string expectedMsgs int delivered atomic.Int32 }{ {name: "C1", subjects: []string{"one"}, expectedMsgs: 5000}, {name: "C2", subjects: []string{"two"}, expectedMsgs: 7500}, {name: "C3", subjects: []string{"one"}, expectedMsgs: 5000}, {name: "C4", subjects: []string{"one"}, expectedMsgs: 5000}, } mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three", "four", "five.>"}, MaxAge: time.Second * 90, }) require_NoError(t, err) for c, consumer := range consumers { _, err := mset.addConsumer(&ConsumerConfig{ Durable: consumer.name, FilterSubjects: consumer.subjects, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, DeliverSubject: nc.NewInbox(), }) require_NoError(t, err) go func(c int, name string) { _, err = js.Subscribe("", func(m *nats.Msg) { require_NoError(t, m.Ack()) require_NoError(t, err) consumers[c].delivered.Add(1) }, nats.Bind("TEST", name)) require_NoError(t, err) }(c, consumer.name) } // Publish with random intervals, while consumers are active. var wg sync.WaitGroup for _, subject := range subjects { wg.Add(subject.messages) go func(subject string, messages int, wc bool) { nc, js := jsClientConnect(t, s) defer nc.Close() time.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Millisecond) for i := 0; i < messages; i++ { time.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Microsecond) // If subject has wildcard, add random last subject token. pubSubject := subject if wc { pubSubject = fmt.Sprintf("%v.%v", subject, rand.Int63n(10)) } _, err := js.PublishAsync(pubSubject, []byte("data")) require_NoError(t, err) wg.Done() } }(subject.subject, subject.messages, subject.wc) } wg.Wait() checkFor(t, time.Second*10, time.Millisecond*500, func() error { for _, consumer := range consumers { info, err := js.ConsumerInfo("TEST", consumer.name) require_NoError(t, err) if info.Delivered.Consumer != uint64(consumer.expectedMsgs) { return fmt.Errorf("%v:expected consumer delivered seq %v, got %v. actually delivered: %v", consumer.name, consumer.expectedMsgs, info.Delivered.Consumer, consumer.delivered.Load()) } if info.AckFloor.Consumer != uint64(consumer.expectedMsgs) { return fmt.Errorf("%v: expected consumer ack floor %v, got %v", consumer.name, totalMsgs, info.AckFloor.Consumer) } if consumer.delivered.Load() != int32(consumer.expectedMsgs) { return fmt.Errorf("%v: expected %v, got %v", consumer.name, consumer.expectedMsgs, consumer.delivered.Load()) } } return nil }) } func TestJetStreamConsumerMultipleConsumersMultipleFilters(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() // Setup few subjects with varying messages count. subjects := []struct { subject string messages int wc bool }{ {subject: "one", messages: 50}, {subject: "two", messages: 75}, {subject: "three", messages: 250}, {subject: "four", messages: 10}, {subject: "five.>", messages: 300, wc: true}, } totalMsgs := 0 for _, subject := range subjects { totalMsgs += subject.messages } // Setup consumers, filtering some of the messages from the stream. consumers := []*struct { name string subjects []string expectedMsgs int delivered atomic.Int32 }{ {name: "C1", subjects: []string{"one", "two"}, expectedMsgs: 125}, {name: "C2", subjects: []string{"two", "three"}, expectedMsgs: 325}, {name: "C3", subjects: []string{"one", "three"}, expectedMsgs: 300}, {name: "C4", subjects: []string{"one", "five.>"}, expectedMsgs: 350}, } mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three", "four", "five.>"}, MaxAge: time.Second * 90, }) require_NoError(t, err) for c, consumer := range consumers { _, err := mset.addConsumer(&ConsumerConfig{ Durable: consumer.name, FilterSubjects: consumer.subjects, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, DeliverSubject: nc.NewInbox(), }) require_NoError(t, err) go func(c int, name string) { _, err = js.Subscribe("", func(m *nats.Msg) { require_NoError(t, m.Ack()) require_NoError(t, err) consumers[c].delivered.Add(1) }, nats.Bind("TEST", name)) require_NoError(t, err) }(c, consumer.name) } // Publish with random intervals, while consumers are active. var wg sync.WaitGroup for _, subject := range subjects { wg.Add(subject.messages) go func(subject string, messages int, wc bool) { nc, js := jsClientConnect(t, s) defer nc.Close() time.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Millisecond) for i := 0; i < messages; i++ { time.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Microsecond) // If subject has wildcard, add random last subject token. pubSubject := subject if wc { pubSubject = fmt.Sprintf("%v.%v", subject, rand.Int63n(10)) } ack, err := js.PublishAsync(pubSubject, []byte("data")) require_NoError(t, err) go func() { ack.Ok() wg.Done() }() } }(subject.subject, subject.messages, subject.wc) } done := make(chan struct{}) go func() { wg.Wait() close(done) }() select { case <-time.After(time.Second * 15): t.Fatalf("Timed out waiting for acks") case <-done: } wg.Wait() checkFor(t, time.Second*15, time.Second*1, func() error { for _, consumer := range consumers { info, err := js.ConsumerInfo("TEST", consumer.name) require_NoError(t, err) if info.Delivered.Consumer != uint64(consumer.expectedMsgs) { return fmt.Errorf("%v:expected consumer delivered seq %v, got %v. actually delivered: %v", consumer.name, consumer.expectedMsgs, info.Delivered.Consumer, consumer.delivered.Load()) } if info.AckFloor.Consumer != uint64(consumer.expectedMsgs) { return fmt.Errorf("%v: expected consumer ack floor %v, got %v", consumer.name, totalMsgs, info.AckFloor.Consumer) } if consumer.delivered.Load() != int32(consumer.expectedMsgs) { return fmt.Errorf("%v: expected %v, got %v", consumer.name, consumer.expectedMsgs, consumer.delivered.Load()) } } return nil }) } func TestJetStreamConsumerMultipleFiltersSequence(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three", "four", "five.>"}, MaxAge: time.Second * 90, }) require_NoError(t, err) _, err = mset.addConsumer(&ConsumerConfig{ Durable: "DUR", FilterSubjects: []string{"one", "two"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, DeliverSubject: nc.NewInbox(), }) require_NoError(t, err) for i := 0; i < 20; i++ { sendStreamMsg(t, nc, "one", fmt.Sprintf("%d", i)) } for i := 20; i < 40; i++ { sendStreamMsg(t, nc, "two", fmt.Sprintf("%d", i)) } for i := 40; i < 60; i++ { sendStreamMsg(t, nc, "one", fmt.Sprintf("%d", i)) } sub, err := js.SubscribeSync("", nats.Bind("TEST", "DUR")) require_NoError(t, err) for i := 0; i < 60; i++ { msg, err := sub.NextMsg(time.Second * 1) require_NoError(t, err) require_True(t, string(msg.Data) == fmt.Sprintf("%d", i)) } } func TestJetStreamConsumerActions(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one", "two", "three", "four", "five.>"}, MaxAge: time.Second * 90, }) require_NoError(t, err) // Create Consumer. No consumers existed before, so should be fine. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "DUR", FilterSubjects: []string{"one", "two"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionCreate) require_NoError(t, err) // Create consumer again. Should be ok if action is CREATE but config is exactly the same. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "DUR", FilterSubjects: []string{"one", "two"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionCreate) require_NoError(t, err) // Create consumer again. Should error if action is CREATE. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "DUR", FilterSubjects: []string{"one"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionCreate) require_Error(t, err) // Update existing consumer. Should be fine, as consumer exists. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "DUR", FilterSubjects: []string{"one"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionUpdate) require_NoError(t, err) // Update consumer. Should error, as this consumer does not exist. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "NEW", FilterSubjects: []string{"one"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionUpdate) require_Error(t, err) // Create new ephemeral. Should be fine as the consumer doesn't exist already _, err = mset.addConsumerWithAction(&ConsumerConfig{ Name: "EPH", FilterSubjects: []string{"one"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionCreate) require_NoError(t, err) // Trying to create it again right away. Should error as it already exists (and hasn't been cleaned up yet) _, err = mset.addConsumerWithAction(&ConsumerConfig{ Name: "EPH", FilterSubjects: []string{"one"}, AckPolicy: AckExplicit, DeliverPolicy: DeliverAll, AckWait: time.Second * 30, }, ActionCreate) require_Error(t, err) } func TestJetStreamConsumerActionsOnWorkQueuePolicyStream(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: WorkQueuePolicy, Subjects: []string{"one", "two", "three", "four", "five.>"}, }) require_NoError(t, err) _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C1", FilterSubjects: []string{"one", "two"}, AckPolicy: AckExplicit, }, ActionCreate) require_NoError(t, err) _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C2", FilterSubjects: []string{"three", "four"}, AckPolicy: AckExplicit, }, ActionCreate) require_NoError(t, err) _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C3", FilterSubjects: []string{"five.*"}, AckPolicy: AckExplicit, }, ActionCreate) require_NoError(t, err) // Updating a consumer by removing a previous subject filter. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C1", FilterSubjects: []string{"one"}, // Remove a subject. AckPolicy: AckExplicit, }, ActionUpdate) require_NoError(t, err) // Updating a consumer without overlapping subjects. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C2", FilterSubjects: []string{"three", "four", "two"}, // Add previously removed subject. AckPolicy: AckExplicit, }, ActionUpdate) require_NoError(t, err) // Creating a consumer with overlapping subjects should return an error. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C4", FilterSubjects: []string{"one", "two", "three", "four"}, AckPolicy: AckExplicit, }, ActionCreate) require_Error(t, err) if !IsNatsErr(err, JSConsumerWQConsumerNotUniqueErr) { t.Errorf("want error %q, got %q", ApiErrors[JSConsumerWQConsumerNotUniqueErr], err) } // Updating a consumer with overlapping subjects should return an error. _, err = mset.addConsumerWithAction(&ConsumerConfig{ Durable: "C3", FilterSubjects: []string{"one", "two", "three", "four"}, AckPolicy: AckExplicit, }, ActionUpdate) require_Error(t, err) if !IsNatsErr(err, JSConsumerWQConsumerNotUniqueErr) { t.Errorf("want error %q, got %q", ApiErrors[JSConsumerWQConsumerNotUniqueErr], err) } } func TestJetStreamConsumerActionsViaAPI(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() _, err := acc.addStream(&StreamConfig{ Name: "TEST", Retention: LimitsPolicy, Subjects: []string{"one"}, MaxAge: time.Second * 90, }) require_NoError(t, err) // Update non-existing consumer, which should fail. request, err := json.Marshal(&CreateConsumerRequest{ Action: ActionUpdate, Config: ConsumerConfig{ Durable: "hello", }, Stream: "TEST", }) require_NoError(t, err) resp, err := nc.Request("$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello", []byte(request), time.Second*6) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) require_Error(t, ccResp.Error) // create non existing consumer - which should be fine. ccResp.Error = nil request, err = json.Marshal(&CreateConsumerRequest{ Action: ActionCreate, Config: ConsumerConfig{ Durable: "hello", }, Stream: "TEST", }) require_NoError(t, err) resp, err = nc.Request("$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello", []byte(request), time.Second*6) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("expected nil, got %v", ccResp.Error) } // re-create existing consumer - which should be an error. ccResp.Error = nil request, err = json.Marshal(&CreateConsumerRequest{ Action: ActionCreate, Config: ConsumerConfig{ Durable: "hello", FilterSubject: "one", }, Stream: "TEST", }) require_NoError(t, err) resp, err = nc.Request("$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello", []byte(request), time.Second*6) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error == nil { t.Fatalf("expected err, got nil") } // create a named ephemeral consumer ccResp.Error = nil request, err = json.Marshal(&CreateConsumerRequest{ Action: ActionCreate, Config: ConsumerConfig{ Name: "ephemeral", FilterSubject: "one", }, Stream: "TEST", }) require_NoError(t, err) resp, err = nc.Request("$JS.API.CONSUMER.CREATE.TEST.ephemeral", []byte(request), time.Second*6) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) // re-create existing consumer - which should be an error. ccResp.Error = nil request, err = json.Marshal(&CreateConsumerRequest{ Action: ActionCreate, Config: ConsumerConfig{ Name: "ephemeral", FilterSubject: "one", }, Stream: "TEST", }) require_NoError(t, err) resp, err = nc.Request("$JS.API.CONSUMER.CREATE.TEST.ephemeral", []byte(request), time.Second*6) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error == nil { t.Fatalf("expected err, got nil") } } func TestJetStreamConsumerActionsUnmarshal(t *testing.T) { tests := []struct { name string given []byte expected ConsumerAction expectErr bool }{ {name: "action create", given: []byte(`{"action": "create"}`), expected: ActionCreate}, {name: "action update", given: []byte(`{"action": "update"}`), expected: ActionUpdate}, {name: "no action", given: []byte("{}"), expected: ActionCreateOrUpdate}, {name: "unknown", given: []byte(`{"action": "unknown"}`), expected: ActionCreateOrUpdate, expectErr: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var request CreateConsumerRequest err := json.Unmarshal(test.given, &request) fmt.Printf("given: %v, expected: %v\n", test.expectErr, err) if !test.expectErr { require_NoError(t, err) } else { require_Error(t, err) } require_True(t, test.expected == request.Action) }) } } func TestJetStreamConsumerMultipleFiltersLastPerSubject(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, error := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"one", "two"}, Replicas: 3, }) require_NoError(t, error) sendStreamMsg(t, nc, "one", "1") sendStreamMsg(t, nc, "one", "2") sendStreamMsg(t, nc, "one", "3") sendStreamMsg(t, nc, "two", "1") sendStreamMsg(t, nc, "two", "2") sendStreamMsg(t, nc, "two", "3") _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "C", FilterSubjects: []string{"one", "two"}, DeliverPolicy: nats.DeliverLastPerSubjectPolicy, Replicas: 3, DeliverSubject: "deliver", }) require_NoError(t, err) consumer, err := js.SubscribeSync("", nats.Bind("TEST", "C")) require_NoError(t, err) // expect last message for subject "one" msg, err := consumer.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, "3", string(msg.Data)) require_Equal(t, "one", msg.Subject) // expect last message for subject "two" msg, err = consumer.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, "3", string(msg.Data)) require_Equal(t, "two", msg.Subject) } func consumerWithFilterSubjects(filterSubjects []string) *consumer { c := consumer{} for _, filter := range filterSubjects { sub := &subjectFilter{ subject: filter, hasWildcard: subjectHasWildcard(filter), tokenizedSubject: tokenizeSubjectIntoSlice(nil, filter), } c.subjf = append(c.subjf, sub) } return &c } func filterSubjects(n int) []string { fs := make([]string, 0, n) for { literals := []string{"foo", "bar", nuid.Next(), "xyz", "abcdef"} fs = append(fs, strings.Join(literals, ".")) if len(fs) == n { return fs } // Create more filterSubjects by going through the literals and replacing one with the '*' wildcard. l := len(literals) for i := 0; i < l; i++ { e := make([]string, l) for j := 0; j < l; j++ { if j == i { e[j] = "*" } else { e[j] = literals[j] } } fs = append(fs, strings.Join(e, ".")) if len(fs) == n { return fs } } } } func TestJetStreamConsumerIsFilteredMatch(t *testing.T) { for _, test := range []struct { name string filterSubjects []string subject string result bool }{ {"no filter", []string{}, "foo.bar", true}, {"literal match", []string{"foo.baz", "foo.bar"}, "foo.bar", true}, {"literal mismatch", []string{"foo.baz", "foo.bar"}, "foo.ban", false}, {"wildcard > match", []string{"bar.>", "foo.>"}, "foo.bar", true}, {"wildcard > match", []string{"bar.>", "foo.>"}, "bar.foo", true}, {"wildcard > mismatch", []string{"bar.>", "foo.>"}, "baz.foo", false}, {"wildcard * match", []string{"bar.*", "foo.*"}, "foo.bar", true}, {"wildcard * match", []string{"bar.*", "foo.*"}, "bar.foo", true}, {"wildcard * mismatch", []string{"bar.*", "foo.*"}, "baz.foo", false}, {"wildcard * match", []string{"foo.*.x", "foo.*.y"}, "foo.bar.x", true}, {"wildcard * match", []string{"foo.*.x", "foo.*.y", "foo.*.z"}, "foo.bar.z", true}, {"many mismatch", filterSubjects(100), "foo.bar.do.not.match.any.filter.subject", false}, {"many match", filterSubjects(100), "foo.bar.12345.xyz.abcdef", true}, // will be matched by "foo.bar.*.xyz.abcdef" } { test := test t.Run(test.name, func(t *testing.T) { t.Parallel() c := consumerWithFilterSubjects(test.filterSubjects) if res := c.isFilteredMatch(test.subject); res != test.result { t.Fatalf("Subject %q filtered match of %v, should be %v, got %v", test.subject, test.filterSubjects, test.result, res) } }) } } func TestJetStreamConsumerWorkQueuePolicyOverlap(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*.*"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "ConsumerA", FilterSubject: "foo.bar.*", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "ConsumerB", FilterSubject: "foo.*.bar", AckPolicy: nats.AckExplicitPolicy, }) require_Error(t, err) require_True(t, strings.Contains(err.Error(), "unique")) } func TestJetStreamConsumerIsEqualOrSubsetMatch(t *testing.T) { for _, test := range []struct { name string filterSubjects []string subject string result bool }{ {"no filter", []string{}, "foo.bar", false}, {"literal match", []string{"foo.baz", "foo.bar"}, "foo.bar", true}, {"literal mismatch", []string{"foo.baz", "foo.bar"}, "foo.ban", false}, {"literal match", []string{"bar.>", "foo.>"}, "foo.>", true}, {"subset match", []string{"bar.foo.>", "foo.bar.>"}, "bar.>", true}, {"subset mismatch", []string{"bar.>", "foo.>"}, "baz.foo.>", false}, {"literal match", filterSubjects(100), "foo.bar.*.xyz.abcdef", true}, {"subset match", filterSubjects(100), "foo.bar.>", true}, } { test := test t.Run(test.name, func(t *testing.T) { t.Parallel() c := consumerWithFilterSubjects(test.filterSubjects) if res := c.isEqualOrSubsetMatch(test.subject); res != test.result { t.Fatalf("Subject %q subset match of %v, should be %v, got %v", test.subject, test.filterSubjects, test.result, res) } }) } } func TestJetStreamConsumerBackOff(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() for _, test := range []struct { name string config nats.ConsumerConfig shouldErr bool }{ { name: "backoff_with_max_deliver", config: nats.ConsumerConfig{ MaxDeliver: 3, BackOff: []time.Duration{time.Second, time.Minute}, }, shouldErr: false, }, { name: "backoff_with_max_deliver_equal", config: nats.ConsumerConfig{ MaxDeliver: 3, BackOff: []time.Duration{time.Second, time.Minute, time.Hour}, }, shouldErr: false, }, { name: "backoff_with_max_deliver_equal_to_zero", config: nats.ConsumerConfig{ MaxDeliver: 0, BackOff: []time.Duration{}, }, shouldErr: false, }, { name: "backoff_with_max_deliver_smaller", config: nats.ConsumerConfig{ MaxDeliver: 2, BackOff: []time.Duration{time.Second, time.Minute, time.Hour}, }, shouldErr: true, }, { name: "backoff_with_default_max_deliver", config: nats.ConsumerConfig{ BackOff: []time.Duration{time.Second, time.Minute, time.Hour}, }, shouldErr: false, }, } { t.Run(test.name, func(t *testing.T) { _, err := js.AddStream(&nats.StreamConfig{ Name: test.name, Subjects: []string{test.name}, }) require_NoError(t, err) _, err = js.AddConsumer(test.name, &test.config) require_True(t, test.shouldErr == (err != nil)) if test.shouldErr { require_True(t, strings.Contains(err.Error(), "max deliver")) } // test if updating consumers works too. test.config.Durable = "consumer" _, err = js.AddConsumer(test.name, &nats.ConsumerConfig{ Durable: test.config.Durable, }) require_NoError(t, err) test.config.Description = "Updated" _, err = js.UpdateConsumer(test.name, &test.config) require_True(t, test.shouldErr == (err != nil)) if test.shouldErr { require_True(t, strings.Contains(err.Error(), "max deliver")) } }) } } func TestJetStreamConsumerDelete(t *testing.T) { tests := []struct { name string replicas int }{ {"single server", 1}, {"clustered", 3}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var s *Server if test.replicas == 1 { s = RunBasicJetStreamServer(t) defer s.Shutdown() } else { c := createJetStreamClusterExplicit(t, "R3S", test.replicas) defer c.shutdown() s = c.randomServer() } nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, MaxAge: time.Second * 90, Replicas: test.replicas, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "consumer", FilterSubject: "events.>", Replicas: test.replicas, }) require_NoError(t, err) js.Publish("events.1", []byte("hello")) cr := JSApiConsumerGetNextRequest{ Batch: 10, Expires: time.Second * 30, } crBytes, err := json.Marshal(cr) require_NoError(t, err) inbox := nats.NewInbox() consumerSub, err := nc.SubscribeSync(inbox) require_NoError(t, err) err = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, "TEST", "consumer"), inbox, crBytes) require_NoError(t, err) msg, err := consumerSub.NextMsg(time.Second * 30) require_NoError(t, err) require_Equal(t, "hello", string(msg.Data)) js.DeleteConsumer("TEST", "consumer") msg, err = consumerSub.NextMsg(time.Second * 30) require_NoError(t, err) if !strings.Contains(string(msg.Header.Get("Description")), "Consumer Deleted") { t.Fatalf("Expected exclusive consumer error, got %q", msg.Header.Get("Description")) } }) } } func TestJetStreamConsumerFetchWithDrain(t *testing.T) { t.Skip() test := func(t *testing.T, cc *nats.ConsumerConfig) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.LimitsPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", cc) require_NoError(t, err) const messages = 10_000 for i := 0; i < messages; i++ { sendStreamMsg(t, nc, "foo", fmt.Sprintf("%d", i+1)) } cr := JSApiConsumerGetNextRequest{ Batch: 100_000, Expires: 10 * time.Second, } crBytes, err := json.Marshal(cr) require_NoError(t, err) msgs := make(map[int]int) processMsg := func(t *testing.T, sub *nats.Subscription, msgs map[int]int) bool { msg, err := sub.NextMsg(time.Second) if err != nil { return false } metadata, err := msg.Metadata() require_NoError(t, err) require_NoError(t, msg.Ack()) v, err := strconv.Atoi(string(msg.Data)) require_NoError(t, err) require_Equal(t, uint64(v), metadata.Sequence.Stream) if _, ok := msgs[int(metadata.Sequence.Stream-1)]; !ok && len(msgs) > 0 { t.Logf("Stream Sequence gap detected: current %d", metadata.Sequence.Stream) } if _, ok := msgs[int(metadata.Sequence.Stream)]; ok { t.Fatalf("Message for seq %d has been seen before", metadata.Sequence.Stream) } // We do not expect official redeliveries here so this should always be 1. if metadata.NumDelivered != 1 { t.Errorf("Expected NumDelivered of 1, got %d for seq %d", metadata.NumDelivered, metadata.Sequence.Stream) } msgs[int(metadata.Sequence.Stream)] = int(metadata.NumDelivered) return true } for { inbox := nats.NewInbox() sub, err := nc.SubscribeSync(inbox) require_NoError(t, err) err = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, "TEST", "C"), inbox, crBytes) require_NoError(t, err) // Drain after first message processed. processMsg(t, sub, msgs) sub.Drain() for { if !processMsg(t, sub, msgs) { if len(msgs) == messages { return } break } } } } t.Run("no-backoff", func(t *testing.T) { test(t, &nats.ConsumerConfig{ Durable: "C", AckPolicy: nats.AckExplicitPolicy, AckWait: 20 * time.Second, }) }) t.Run("with-backoff", func(t *testing.T) { test(t, &nats.ConsumerConfig{ Durable: "C", AckPolicy: nats.AckExplicitPolicy, AckWait: 20 * time.Second, BackOff: []time.Duration{25 * time.Millisecond, 100 * time.Millisecond, 250 * time.Millisecond}, }) }) } func TestJetStreamConsumerLongSubjectHang(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() readSubj := "a1." purgeSubj := "a2." _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{readSubj + ">", purgeSubj + ">"}, AllowRollup: true, }) require_NoError(t, err) prefix := strings.Repeat("a", 22) for i := 0; i < 2; i++ { subj := readSubj + prefix + fmt.Sprintf("%d", i) _, err = js.Publish(subj, []byte("hello")) require_NoError(t, err) chunkSubj := purgeSubj + fmt.Sprintf("%d", i) _, err = js.Publish(chunkSubj, []byte("contents")) require_NoError(t, err) } err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: purgeSubj + ">"}) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) // we should have 2 msgs left after purge require_Equal(t, si.State.Msgs, 2) sub, err := js.SubscribeSync(readSubj+">", nats.OrderedConsumer(), nats.DeliverLastPerSubject()) require_NoError(t, err) defer sub.Unsubscribe() for i := 0; i < 2; i++ { m, err := sub.NextMsg(500 * time.Millisecond) require_NoError(t, err) require_True(t, string(m.Data) == "hello") } } func TestJetStreamConsumerStuckAckPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() type ActiveWorkItem struct { ID int Expiry time.Time } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST_ACTIVE_WORK_ITEMS", Discard: nats.DiscardOld, MaxMsgsPerSubject: 1, Subjects: []string{"TEST_ACTIVE_WORK_ITEMS.>"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST_ACTIVE_WORK_ITEMS", &nats.ConsumerConfig{ Durable: "testactiveworkitemsconsumer", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: -1, MaxWaiting: 20000, AckWait: 15 * time.Second, }) require_NoError(t, err) sub, err := js.PullSubscribe("TEST_ACTIVE_WORK_ITEMS.>", "testactiveworkitemsconsumer", nats.BindStream("TEST_ACTIVE_WORK_ITEMS")) require_NoError(t, err) errs := make(chan error) go func() { for { msgs, err := sub.Fetch(200) if err != nil { // test is done. stop the loop. if errors.Is(err, nats.ErrSubscriptionClosed) || errors.Is(err, nats.ErrConnectionClosed) { return } if !errors.Is(err, nats.ErrTimeout) { errs <- err return } continue } for _, msg := range msgs { msg := msg var workItem ActiveWorkItem if err := json.Unmarshal(msg.Data, &workItem); err != nil { errs <- err return } now := time.Now() // If the work item has not expired, nak it with the respective delay. if workItem.Expiry.After(now) { msg.NakWithDelay(workItem.Expiry.Sub(now)) } else { msg.Ack() } } } }() for i := 0; i < 25_000; i++ { // Publish item to TEST_ACTIVE_WORK_ITEMS stream with an expiry time. workItem := ActiveWorkItem{ID: i, Expiry: time.Now().Add(30 * time.Second)} data, err := json.Marshal(workItem) require_NoError(t, err) _, err = js.Publish(fmt.Sprintf("TEST_ACTIVE_WORK_ITEMS.%d", i), data) require_NoError(t, err) // Update expiry time and republish item to TEST_ACTIVE_WORK_ITEMS stream. workItem.Expiry = time.Now().Add(3 * time.Second) data, err = json.Marshal(workItem) require_NoError(t, err) _, err = js.Publish(fmt.Sprintf("TEST_ACTIVE_WORK_ITEMS.%d", i), data) require_NoError(t, err) } noChange := false lastNumAckPending := 0 checkFor(t, 60*time.Second, 3*time.Second, func() error { select { case err := <-errs: t.Fatalf("consumer goroutine failed: %v", err) default: } ci, err := js.ConsumerInfo("TEST_ACTIVE_WORK_ITEMS", "testactiveworkitemsconsumer") require_NoError(t, err) if lastNumAckPending != 0 && lastNumAckPending == ci.NumAckPending { noChange = true } lastNumAckPending = ci.NumAckPending // If we have no change since last check, we can fail the test before `totalWait` timeout. if ci.NumAckPending > 0 && ci.NumPending == 0 { if noChange { _, err := sub.Fetch(1) if err != nil && errors.Is(err, nats.ErrTimeout) { t.Fatalf("num ack pending: %d\t num pending: %v\n", ci.NumAckPending, ci.NumPending) } } return fmt.Errorf("num ack pending: %d\t num pending: %v\n", ci.NumAckPending, ci.NumPending) } return nil }) } func TestJetStreamConsumerMultipleFitersWithStartDate(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() past := time.Now().Add(-90 * time.Second) _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "events.foo", "msg-1") sendStreamMsg(t, nc, "events.bar", "msg-2") sendStreamMsg(t, nc, "events.baz", "msg-3") sendStreamMsg(t, nc, "events.biz", "msg-4") sendStreamMsg(t, nc, "events.faz", "msg-5") sendStreamMsg(t, nc, "events.foo", "msg-6") sendStreamMsg(t, nc, "events.biz", "msg-7") for _, test := range []struct { name string filterSubjects []string startTime time.Time expectedMessages uint64 expectedStreamSequence uint64 }{ {"Single-Filter-first-sequence", []string{"events.foo"}, past, 2, 0}, {"Multiple-Filter-first-sequence", []string{"events.foo", "events.bar", "events.baz"}, past, 4, 0}, {"Multiple-Filters-second-subject", []string{"events.bar", "events.baz"}, past, 2, 1}, {"Multiple-Filters-first-last-subject", []string{"events.foo", "events.biz"}, past, 4, 0}, {"Multiple-Filters-in-future", []string{"events.foo", "events.biz"}, time.Now().Add(1 * time.Minute), 0, 7}, } { t.Run(test.name, func(t *testing.T) { info, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: test.name, FilterSubjects: test.filterSubjects, DeliverPolicy: nats.DeliverByStartTimePolicy, OptStartTime: &test.startTime, }) require_NoError(t, err) require_Equal(t, test.expectedStreamSequence, info.Delivered.Stream) require_Equal(t, test.expectedMessages, info.NumPending) }) } } func Benchmark____JetStreamConsumerIsFilteredMatch(b *testing.B) { subject := "foo.bar.do.not.match.any.filter.subject" for n := 1; n <= 1024; n *= 2 { name := fmt.Sprintf("%d filter subjects", int(n)) c := consumerWithFilterSubjects(filterSubjects(int(n))) b.Run(name, func(b *testing.B) { c.isFilteredMatch(subject) }) } } // https://github.com/nats-io/nats-server/issues/6085 func TestJetStreamConsumerBackoffNotRespectedWithMultipleInflightRedeliveries(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, }) require_NoError(t, err) maxDeliver := 3 backoff := []time.Duration{2 * time.Second, 4 * time.Second} sub, err := js.SubscribeSync( "events.>", nats.MaxDeliver(maxDeliver), nats.BackOff(backoff), nats.AckExplicit(), ) require_NoError(t, err) calculateExpectedBackoff := func(numDelivered int) time.Duration { expectedBackoff := 500 * time.Millisecond for i := 0; i < numDelivered-1 && i < len(backoff); i++ { expectedBackoff += backoff[i] } return expectedBackoff } // We get one message to be redelivered using the final backoff duration. firstMsgSent := time.Now() sendStreamMsg(t, nc, "events.first", "msg-1") _, err = sub.NextMsg(time.Second) require_NoError(t, err) require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(1)) _, err = sub.NextMsg(5 * time.Second) require_NoError(t, err) require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(2)) // This message will be redelivered with the final/highest backoff below. // If we now send a new message, the pending timer should be reset to the first backoff. // Otherwise, if it remains at the final backoff duration we'll get this message redelivered too late. sendStreamMsg(t, nc, "events.second", "msg-2") for { msg, err := sub.NextMsg(5 * time.Second) require_NoError(t, err) if msg.Subject == "events.first" { require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(3)) continue } // We expect the second message to be redelivered using the specified backoff strategy. // Before, the first redelivery of the second message would be sent after the highest backoff duration. metadata, err := msg.Metadata() require_NoError(t, err) numDelivered := int(metadata.NumDelivered) expectedBackoff := calculateExpectedBackoff(numDelivered) require_LessThan(t, time.Since(metadata.Timestamp), expectedBackoff) // We've received all message, test passed. if numDelivered >= maxDeliver { break } } } func TestJetStreamConsumerBackoffWhenBackoffLengthIsEqualToMaxDeliverConfig(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, }) require_NoError(t, err) maxDeliver := 3 backoff := []time.Duration{time.Second, 2 * time.Second, 3 * time.Second} sub, err := js.SubscribeSync( "events.>", nats.MaxDeliver(maxDeliver), nats.BackOff(backoff), nats.AckExplicit(), ) require_NoError(t, err) calculateExpectedBackoff := func(numDelivered int) time.Duration { return backoff[numDelivered-1] + 50*time.Millisecond // 50ms of margin to system overhead } // message to be redelivered using backoff duration. firstMsgSent := time.Now() sendStreamMsg(t, nc, "events.first", "msg-1") _, err = sub.NextMsg(time.Second) require_NoError(t, err) require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(1)) _, err = sub.NextMsg(2 * time.Second) require_NoError(t, err) require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(2)) _, err = sub.NextMsg(3 * time.Second) require_NoError(t, err) require_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(3)) } func TestJetStreamConsumerRetryAckAfterTimeout(t *testing.T) { for _, ack := range []struct { title string policy nats.SubOpt }{ {title: "AckExplicit", policy: nats.AckExplicit()}, {title: "AckAll", policy: nats.AckAll()}, } { t.Run(ack.title, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "CONSUMER", ack.policy) require_NoError(t, err) msgs, err := sub.Fetch(1) require_NoError(t, err) require_Len(t, len(msgs), 1) msg := msgs[0] // Send core request so the client is unaware of the ack being sent. _, err = nc.Request(msg.Reply, nil, time.Second) require_NoError(t, err) // It could be we have already acked this specific message, but we haven't received the success response. // Retrying the ack should not time out and still signal success. err = msg.AckSync() require_NoError(t, err) }) } } func TestJetStreamConsumerSwitchLeaderDuringInflightAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 2_000; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } sub, err := js.PullSubscribe( "foo", "CONSUMER", nats.MaxAckPending(2_000), nats.ManualAck(), nats.AckExplicit(), nats.AckWait(2*time.Second), ) require_NoError(t, err) acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) msgs, err := sub.Fetch(2_000) require_NoError(t, err) require_Len(t, len(msgs), 2_000) // Simulate an ack being pushed, and o.setLeader(false) being called before the ack is processed and resets o.awl atomic.AddInt64(&o.awl, 1) o.setLeader(false) o.setLeader(true) msgs, err = sub.Fetch(1, nats.MaxWait(5*time.Second)) require_NoError(t, err) require_Len(t, len(msgs), 1) } func TestJetStreamConsumerMessageDeletedDuringRedelivery(t *testing.T) { storageTypes := []nats.StorageType{nats.MemoryStorage, nats.FileStorage} for _, storageType := range storageTypes { t.Run(storageType.String(), func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: storageType, }) require_NoError(t, err) for i := 0; i < 3; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } sub, err := js.PullSubscribe( "foo", "CONSUMER", nats.ManualAck(), nats.AckExplicit(), nats.AckWait(time.Second), ) require_NoError(t, err) acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) msgs, err := sub.Fetch(3) require_NoError(t, err) require_Len(t, len(msgs), 3) err = js.DeleteMsg("TEST", 2) require_NoError(t, err) // Wait for mset.storeUpdates to call into o.decStreamPending which runs // the o.processTerm goroutine, removing one message from pending. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { o.mu.RLock() defer o.mu.RUnlock() if len(o.pending) != 2 { return fmt.Errorf("expected 2 pending, but got %d", len(o.pending)) } return nil }) // Now empty the redelivery queue and reset the pending state. o.mu.Lock() for _, seq := range o.rdq { o.removeFromRedeliverQueue(seq) } o.pending = make(map[uint64]*Pending) o.pending[2] = &Pending{} o.addToRedeliverQueue(2) // Also reset delivery/ack floors to confirm they get corrected. o.adflr, o.asflr = 0, 0 o.dseq, o.sseq = 11, 11 // Getting the next message should skip seq 2, as that's deleted, but must not touch state. _, _, err = o.getNextMsg() o.mu.Unlock() require_Error(t, err, ErrStoreEOF) require_Len(t, len(o.pending), 1) // Simulate the o.processTerm goroutine running after a call to o.getNextMsg. // Pending state and delivery/ack floors should be corrected. o.processTerm(2, 2, 1, ackTermUnackedLimitsReason, _EMPTY_) o.mu.RLock() defer o.mu.RUnlock() require_Len(t, len(o.pending), 0) require_Equal(t, o.adflr, 10) require_Equal(t, o.asflr, 10) }) } } func TestJetStreamConsumerDeliveryCount(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 2; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } sub, err := js.PullSubscribe( "foo", "CONSUMER", nats.ManualAck(), nats.AckExplicit(), nats.AckWait(time.Second), nats.MaxDeliver(1), ) require_NoError(t, err) acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") require_NotNil(t, o) msgs, err := sub.Fetch(2) require_NoError(t, err) require_Len(t, len(msgs), 2) require_NoError(t, msgs[1].Nak()) require_Equal(t, o.deliveryCount(1), 1) require_Equal(t, o.deliveryCount(2), 1) // max deliver 1 so this will fail _, err = sub.Fetch(1, nats.MaxWait(250*time.Millisecond)) require_Error(t, err) // This would previously report delivery count 0, because o.rdc!=nil require_Equal(t, o.deliveryCount(1), 1) require_Equal(t, o.deliveryCount(2), 1) } nats-server-2.10.27/server/jetstream_errors.go000066400000000000000000000041001477524627100213750ustar00rootroot00000000000000package server import ( "fmt" ) type errOpts struct { err error } // ErrorOption configures a NATS Error helper type ErrorOption func(*errOpts) // Unless ensures that if err is a ApiErr that err will be returned rather than the one being created via the helper func Unless(err error) ErrorOption { return func(opts *errOpts) { opts.err = err } } func parseOpts(opts []ErrorOption) *errOpts { eopts := &errOpts{} for _, opt := range opts { opt(eopts) } return eopts } type ErrorIdentifier uint16 // IsNatsErr determines if an error matches ID, if multiple IDs are given if the error matches any of these the function will be true func IsNatsErr(err error, ids ...ErrorIdentifier) bool { if err == nil { return false } ce, ok := err.(*ApiError) if !ok || ce == nil { return false } for _, id := range ids { ae, ok := ApiErrors[id] if !ok || ae == nil { continue } if ce.ErrCode == ae.ErrCode { return true } } return false } // ApiError is included in all responses if there was an error. type ApiError struct { Code int `json:"code"` ErrCode uint16 `json:"err_code,omitempty"` Description string `json:"description,omitempty"` } // ErrorsData is the source data for generated errors as found in errors.json type ErrorsData struct { Constant string `json:"constant"` Code int `json:"code"` ErrCode uint16 `json:"error_code"` Description string `json:"description"` Comment string `json:"comment"` Help string `json:"help"` URL string `json:"url"` Deprecates string `json:"deprecates"` } func (e *ApiError) Error() string { return fmt.Sprintf("%s (%d)", e.Description, e.ErrCode) } func (e *ApiError) toReplacerArgs(replacements []any) []string { var ( ra []string key string ) for i, replacement := range replacements { if i%2 == 0 { key = replacement.(string) continue } switch v := replacement.(type) { case string: ra = append(ra, key, v) case error: ra = append(ra, key, v.Error()) default: ra = append(ra, key, fmt.Sprintf("%v", v)) } } return ra } nats-server-2.10.27/server/jetstream_errors_generated.go000066400000000000000000003032371477524627100234300ustar00rootroot00000000000000// Generated code, do not edit. See errors.json and run go generate to update package server import "strings" const ( // JSAccountResourcesExceededErr resource limits exceeded for account JSAccountResourcesExceededErr ErrorIdentifier = 10002 // JSBadRequestErr bad request JSBadRequestErr ErrorIdentifier = 10003 // JSClusterIncompleteErr incomplete results JSClusterIncompleteErr ErrorIdentifier = 10004 // JSClusterNoPeersErrF Error causing no peers to be available ({err}) JSClusterNoPeersErrF ErrorIdentifier = 10005 // JSClusterNotActiveErr JetStream not in clustered mode JSClusterNotActiveErr ErrorIdentifier = 10006 // JSClusterNotAssignedErr JetStream cluster not assigned to this server JSClusterNotAssignedErr ErrorIdentifier = 10007 // JSClusterNotAvailErr JetStream system temporarily unavailable JSClusterNotAvailErr ErrorIdentifier = 10008 // JSClusterNotLeaderErr JetStream cluster can not handle request JSClusterNotLeaderErr ErrorIdentifier = 10009 // JSClusterPeerNotMemberErr peer not a member JSClusterPeerNotMemberErr ErrorIdentifier = 10040 // JSClusterRequiredErr JetStream clustering support required JSClusterRequiredErr ErrorIdentifier = 10010 // JSClusterServerNotMemberErr server is not a member of the cluster JSClusterServerNotMemberErr ErrorIdentifier = 10044 // JSClusterTagsErr tags placement not supported for operation JSClusterTagsErr ErrorIdentifier = 10011 // JSClusterUnSupportFeatureErr not currently supported in clustered mode JSClusterUnSupportFeatureErr ErrorIdentifier = 10036 // JSConsumerAlreadyExists action CREATE is used for a existing consumer with a different config (consumer already exists) JSConsumerAlreadyExists ErrorIdentifier = 10148 // JSConsumerBadDurableNameErr durable name can not contain '.', '*', '>' JSConsumerBadDurableNameErr ErrorIdentifier = 10103 // JSConsumerConfigRequiredErr consumer config required JSConsumerConfigRequiredErr ErrorIdentifier = 10078 // JSConsumerCreateDurableAndNameMismatch Consumer Durable and Name have to be equal if both are provided JSConsumerCreateDurableAndNameMismatch ErrorIdentifier = 10132 // JSConsumerCreateErrF General consumer creation failure string ({err}) JSConsumerCreateErrF ErrorIdentifier = 10012 // JSConsumerCreateFilterSubjectMismatchErr Consumer create request did not match filtered subject from create subject JSConsumerCreateFilterSubjectMismatchErr ErrorIdentifier = 10131 // JSConsumerDeliverCycleErr consumer deliver subject forms a cycle JSConsumerDeliverCycleErr ErrorIdentifier = 10081 // JSConsumerDeliverToWildcardsErr consumer deliver subject has wildcards JSConsumerDeliverToWildcardsErr ErrorIdentifier = 10079 // JSConsumerDescriptionTooLongErrF consumer description is too long, maximum allowed is {max} JSConsumerDescriptionTooLongErrF ErrorIdentifier = 10107 // JSConsumerDirectRequiresEphemeralErr consumer direct requires an ephemeral consumer JSConsumerDirectRequiresEphemeralErr ErrorIdentifier = 10091 // JSConsumerDirectRequiresPushErr consumer direct requires a push based consumer JSConsumerDirectRequiresPushErr ErrorIdentifier = 10090 // JSConsumerDoesNotExist action UPDATE is used for a nonexisting consumer (consumer does not exist) JSConsumerDoesNotExist ErrorIdentifier = 10149 // JSConsumerDuplicateFilterSubjects consumer cannot have both FilterSubject and FilterSubjects specified JSConsumerDuplicateFilterSubjects ErrorIdentifier = 10136 // JSConsumerDurableNameNotInSubjectErr consumer expected to be durable but no durable name set in subject JSConsumerDurableNameNotInSubjectErr ErrorIdentifier = 10016 // JSConsumerDurableNameNotMatchSubjectErr consumer name in subject does not match durable name in request JSConsumerDurableNameNotMatchSubjectErr ErrorIdentifier = 10017 // JSConsumerDurableNameNotSetErr consumer expected to be durable but a durable name was not set JSConsumerDurableNameNotSetErr ErrorIdentifier = 10018 // JSConsumerEmptyFilter consumer filter in FilterSubjects cannot be empty JSConsumerEmptyFilter ErrorIdentifier = 10139 // JSConsumerEphemeralWithDurableInSubjectErr consumer expected to be ephemeral but detected a durable name set in subject JSConsumerEphemeralWithDurableInSubjectErr ErrorIdentifier = 10019 // JSConsumerEphemeralWithDurableNameErr consumer expected to be ephemeral but a durable name was set in request JSConsumerEphemeralWithDurableNameErr ErrorIdentifier = 10020 // JSConsumerExistingActiveErr consumer already exists and is still active JSConsumerExistingActiveErr ErrorIdentifier = 10105 // JSConsumerFCRequiresPushErr consumer flow control requires a push based consumer JSConsumerFCRequiresPushErr ErrorIdentifier = 10089 // JSConsumerFilterNotSubsetErr consumer filter subject is not a valid subset of the interest subjects JSConsumerFilterNotSubsetErr ErrorIdentifier = 10093 // JSConsumerHBRequiresPushErr consumer idle heartbeat requires a push based consumer JSConsumerHBRequiresPushErr ErrorIdentifier = 10088 // JSConsumerInactiveThresholdExcess consumer inactive threshold exceeds system limit of {limit} JSConsumerInactiveThresholdExcess ErrorIdentifier = 10153 // JSConsumerInvalidDeliverSubject invalid push consumer deliver subject JSConsumerInvalidDeliverSubject ErrorIdentifier = 10112 // JSConsumerInvalidPolicyErrF Generic delivery policy error ({err}) JSConsumerInvalidPolicyErrF ErrorIdentifier = 10094 // JSConsumerInvalidSamplingErrF failed to parse consumer sampling configuration: {err} JSConsumerInvalidSamplingErrF ErrorIdentifier = 10095 // JSConsumerMaxDeliverBackoffErr max deliver is required to be > length of backoff values JSConsumerMaxDeliverBackoffErr ErrorIdentifier = 10116 // JSConsumerMaxPendingAckExcessErrF consumer max ack pending exceeds system limit of {limit} JSConsumerMaxPendingAckExcessErrF ErrorIdentifier = 10121 // JSConsumerMaxPendingAckPolicyRequiredErr consumer requires ack policy for max ack pending JSConsumerMaxPendingAckPolicyRequiredErr ErrorIdentifier = 10082 // JSConsumerMaxRequestBatchExceededF consumer max request batch exceeds server limit of {limit} JSConsumerMaxRequestBatchExceededF ErrorIdentifier = 10125 // JSConsumerMaxRequestBatchNegativeErr consumer max request batch needs to be > 0 JSConsumerMaxRequestBatchNegativeErr ErrorIdentifier = 10114 // JSConsumerMaxRequestExpiresToSmall consumer max request expires needs to be >= 1ms JSConsumerMaxRequestExpiresToSmall ErrorIdentifier = 10115 // JSConsumerMaxWaitingNegativeErr consumer max waiting needs to be positive JSConsumerMaxWaitingNegativeErr ErrorIdentifier = 10087 // JSConsumerMetadataLengthErrF consumer metadata exceeds maximum size of {limit} JSConsumerMetadataLengthErrF ErrorIdentifier = 10135 // JSConsumerMultipleFiltersNotAllowed consumer with multiple subject filters cannot use subject based API JSConsumerMultipleFiltersNotAllowed ErrorIdentifier = 10137 // JSConsumerNameContainsPathSeparatorsErr Consumer name can not contain path separators JSConsumerNameContainsPathSeparatorsErr ErrorIdentifier = 10127 // JSConsumerNameExistErr consumer name already in use JSConsumerNameExistErr ErrorIdentifier = 10013 // JSConsumerNameTooLongErrF consumer name is too long, maximum allowed is {max} JSConsumerNameTooLongErrF ErrorIdentifier = 10102 // JSConsumerNotFoundErr consumer not found JSConsumerNotFoundErr ErrorIdentifier = 10014 // JSConsumerOfflineErr consumer is offline JSConsumerOfflineErr ErrorIdentifier = 10119 // JSConsumerOnMappedErr consumer direct on a mapped consumer JSConsumerOnMappedErr ErrorIdentifier = 10092 // JSConsumerOverlappingSubjectFilters consumer subject filters cannot overlap JSConsumerOverlappingSubjectFilters ErrorIdentifier = 10138 // JSConsumerPullNotDurableErr consumer in pull mode requires a durable name JSConsumerPullNotDurableErr ErrorIdentifier = 10085 // JSConsumerPullRequiresAckErr consumer in pull mode requires ack policy JSConsumerPullRequiresAckErr ErrorIdentifier = 10084 // JSConsumerPullWithRateLimitErr consumer in pull mode can not have rate limit set JSConsumerPullWithRateLimitErr ErrorIdentifier = 10086 // JSConsumerPushMaxWaitingErr consumer in push mode can not set max waiting JSConsumerPushMaxWaitingErr ErrorIdentifier = 10080 // JSConsumerReplacementWithDifferentNameErr consumer replacement durable config not the same JSConsumerReplacementWithDifferentNameErr ErrorIdentifier = 10106 // JSConsumerReplicasExceedsStream consumer config replica count exceeds parent stream JSConsumerReplicasExceedsStream ErrorIdentifier = 10126 // JSConsumerReplicasShouldMatchStream consumer config replicas must match interest retention stream's replicas JSConsumerReplicasShouldMatchStream ErrorIdentifier = 10134 // JSConsumerSmallHeartbeatErr consumer idle heartbeat needs to be >= 100ms JSConsumerSmallHeartbeatErr ErrorIdentifier = 10083 // JSConsumerStoreFailedErrF error creating store for consumer: {err} JSConsumerStoreFailedErrF ErrorIdentifier = 10104 // JSConsumerWQConsumerNotDeliverAllErr consumer must be deliver all on workqueue stream JSConsumerWQConsumerNotDeliverAllErr ErrorIdentifier = 10101 // JSConsumerWQConsumerNotUniqueErr filtered consumer not unique on workqueue stream JSConsumerWQConsumerNotUniqueErr ErrorIdentifier = 10100 // JSConsumerWQMultipleUnfilteredErr multiple non-filtered consumers not allowed on workqueue stream JSConsumerWQMultipleUnfilteredErr ErrorIdentifier = 10099 // JSConsumerWQRequiresExplicitAckErr workqueue stream requires explicit ack JSConsumerWQRequiresExplicitAckErr ErrorIdentifier = 10098 // JSConsumerWithFlowControlNeedsHeartbeats consumer with flow control also needs heartbeats JSConsumerWithFlowControlNeedsHeartbeats ErrorIdentifier = 10108 // JSInsufficientResourcesErr insufficient resources JSInsufficientResourcesErr ErrorIdentifier = 10023 // JSInvalidJSONErr invalid JSON JSInvalidJSONErr ErrorIdentifier = 10025 // JSMaximumConsumersLimitErr maximum consumers limit reached JSMaximumConsumersLimitErr ErrorIdentifier = 10026 // JSMaximumStreamsLimitErr maximum number of streams reached JSMaximumStreamsLimitErr ErrorIdentifier = 10027 // JSMemoryResourcesExceededErr insufficient memory resources available JSMemoryResourcesExceededErr ErrorIdentifier = 10028 // JSMirrorConsumerSetupFailedErrF generic mirror consumer setup failure string ({err}) JSMirrorConsumerSetupFailedErrF ErrorIdentifier = 10029 // JSMirrorInvalidStreamName mirrored stream name is invalid JSMirrorInvalidStreamName ErrorIdentifier = 10142 // JSMirrorInvalidSubjectFilter mirror subject filter is invalid JSMirrorInvalidSubjectFilter ErrorIdentifier = 10151 // JSMirrorMaxMessageSizeTooBigErr stream mirror must have max message size >= source JSMirrorMaxMessageSizeTooBigErr ErrorIdentifier = 10030 // JSMirrorMultipleFiltersNotAllowed mirror with multiple subject transforms cannot also have a single subject filter JSMirrorMultipleFiltersNotAllowed ErrorIdentifier = 10150 // JSMirrorOverlappingSubjectFilters mirror subject filters can not overlap JSMirrorOverlappingSubjectFilters ErrorIdentifier = 10152 // JSMirrorWithFirstSeqErr stream mirrors can not have first sequence configured JSMirrorWithFirstSeqErr ErrorIdentifier = 10143 // JSMirrorWithSourcesErr stream mirrors can not also contain other sources JSMirrorWithSourcesErr ErrorIdentifier = 10031 // JSMirrorWithStartSeqAndTimeErr stream mirrors can not have both start seq and start time configured JSMirrorWithStartSeqAndTimeErr ErrorIdentifier = 10032 // JSMirrorWithSubjectFiltersErr stream mirrors can not contain filtered subjects JSMirrorWithSubjectFiltersErr ErrorIdentifier = 10033 // JSMirrorWithSubjectsErr stream mirrors can not contain subjects JSMirrorWithSubjectsErr ErrorIdentifier = 10034 // JSNoAccountErr account not found JSNoAccountErr ErrorIdentifier = 10035 // JSNoLimitsErr no JetStream default or applicable tiered limit present JSNoLimitsErr ErrorIdentifier = 10120 // JSNoMessageFoundErr no message found JSNoMessageFoundErr ErrorIdentifier = 10037 // JSNotEmptyRequestErr expected an empty request payload JSNotEmptyRequestErr ErrorIdentifier = 10038 // JSNotEnabledErr JetStream not enabled JSNotEnabledErr ErrorIdentifier = 10076 // JSNotEnabledForAccountErr JetStream not enabled for account JSNotEnabledForAccountErr ErrorIdentifier = 10039 // JSPeerRemapErr peer remap failed JSPeerRemapErr ErrorIdentifier = 10075 // JSRaftGeneralErrF General RAFT error string ({err}) JSRaftGeneralErrF ErrorIdentifier = 10041 // JSReplicasCountCannotBeNegative replicas count cannot be negative JSReplicasCountCannotBeNegative ErrorIdentifier = 10133 // JSRestoreSubscribeFailedErrF JetStream unable to subscribe to restore snapshot {subject}: {err} JSRestoreSubscribeFailedErrF ErrorIdentifier = 10042 // JSSequenceNotFoundErrF sequence {seq} not found JSSequenceNotFoundErrF ErrorIdentifier = 10043 // JSSnapshotDeliverSubjectInvalidErr deliver subject not valid JSSnapshotDeliverSubjectInvalidErr ErrorIdentifier = 10015 // JSSourceConsumerSetupFailedErrF General source consumer setup failure string ({err}) JSSourceConsumerSetupFailedErrF ErrorIdentifier = 10045 // JSSourceDuplicateDetected source stream, filter and transform (plus external if present) must form a unique combination (duplicate source configuration detected) JSSourceDuplicateDetected ErrorIdentifier = 10140 // JSSourceInvalidStreamName sourced stream name is invalid JSSourceInvalidStreamName ErrorIdentifier = 10141 // JSSourceInvalidSubjectFilter source subject filter is invalid JSSourceInvalidSubjectFilter ErrorIdentifier = 10145 // JSSourceInvalidTransformDestination source transform destination is invalid JSSourceInvalidTransformDestination ErrorIdentifier = 10146 // JSSourceMaxMessageSizeTooBigErr stream source must have max message size >= target JSSourceMaxMessageSizeTooBigErr ErrorIdentifier = 10046 // JSSourceMultipleFiltersNotAllowed source with multiple subject transforms cannot also have a single subject filter JSSourceMultipleFiltersNotAllowed ErrorIdentifier = 10144 // JSSourceOverlappingSubjectFilters source filters can not overlap JSSourceOverlappingSubjectFilters ErrorIdentifier = 10147 // JSStorageResourcesExceededErr insufficient storage resources available JSStorageResourcesExceededErr ErrorIdentifier = 10047 // JSStreamAssignmentErrF Generic stream assignment error string ({err}) JSStreamAssignmentErrF ErrorIdentifier = 10048 // JSStreamCreateErrF Generic stream creation error string ({err}) JSStreamCreateErrF ErrorIdentifier = 10049 // JSStreamDeleteErrF General stream deletion error string ({err}) JSStreamDeleteErrF ErrorIdentifier = 10050 // JSStreamExternalApiOverlapErrF stream external api prefix {prefix} must not overlap with {subject} JSStreamExternalApiOverlapErrF ErrorIdentifier = 10021 // JSStreamExternalDelPrefixOverlapsErrF stream external delivery prefix {prefix} overlaps with stream subject {subject} JSStreamExternalDelPrefixOverlapsErrF ErrorIdentifier = 10022 // JSStreamGeneralErrorF General stream failure string ({err}) JSStreamGeneralErrorF ErrorIdentifier = 10051 // JSStreamHeaderExceedsMaximumErr header size exceeds maximum allowed of 64k JSStreamHeaderExceedsMaximumErr ErrorIdentifier = 10097 // JSStreamInfoMaxSubjectsErr subject details would exceed maximum allowed JSStreamInfoMaxSubjectsErr ErrorIdentifier = 10117 // JSStreamInvalidConfigF Stream configuration validation error string ({err}) JSStreamInvalidConfigF ErrorIdentifier = 10052 // JSStreamInvalidErr stream not valid JSStreamInvalidErr ErrorIdentifier = 10096 // JSStreamInvalidExternalDeliverySubjErrF stream external delivery prefix {prefix} must not contain wildcards JSStreamInvalidExternalDeliverySubjErrF ErrorIdentifier = 10024 // JSStreamLimitsErrF General stream limits exceeded error string ({err}) JSStreamLimitsErrF ErrorIdentifier = 10053 // JSStreamMaxBytesRequired account requires a stream config to have max bytes set JSStreamMaxBytesRequired ErrorIdentifier = 10113 // JSStreamMaxStreamBytesExceeded stream max bytes exceeds account limit max stream bytes JSStreamMaxStreamBytesExceeded ErrorIdentifier = 10122 // JSStreamMessageExceedsMaximumErr message size exceeds maximum allowed JSStreamMessageExceedsMaximumErr ErrorIdentifier = 10054 // JSStreamMirrorNotUpdatableErr stream mirror configuration can not be updated JSStreamMirrorNotUpdatableErr ErrorIdentifier = 10055 // JSStreamMismatchErr stream name in subject does not match request JSStreamMismatchErr ErrorIdentifier = 10056 // JSStreamMoveAndScaleErr can not move and scale a stream in a single update JSStreamMoveAndScaleErr ErrorIdentifier = 10123 // JSStreamMoveInProgressF stream move already in progress: {msg} JSStreamMoveInProgressF ErrorIdentifier = 10124 // JSStreamMoveNotInProgress stream move not in progress JSStreamMoveNotInProgress ErrorIdentifier = 10129 // JSStreamMsgDeleteFailedF Generic message deletion failure error string ({err}) JSStreamMsgDeleteFailedF ErrorIdentifier = 10057 // JSStreamNameContainsPathSeparatorsErr Stream name can not contain path separators JSStreamNameContainsPathSeparatorsErr ErrorIdentifier = 10128 // JSStreamNameExistErr stream name already in use with a different configuration JSStreamNameExistErr ErrorIdentifier = 10058 // JSStreamNameExistRestoreFailedErr stream name already in use, cannot restore JSStreamNameExistRestoreFailedErr ErrorIdentifier = 10130 // JSStreamNotFoundErr stream not found JSStreamNotFoundErr ErrorIdentifier = 10059 // JSStreamNotMatchErr expected stream does not match JSStreamNotMatchErr ErrorIdentifier = 10060 // JSStreamOfflineErr stream is offline JSStreamOfflineErr ErrorIdentifier = 10118 // JSStreamPurgeFailedF Generic stream purge failure error string ({err}) JSStreamPurgeFailedF ErrorIdentifier = 10110 // JSStreamReplicasNotSupportedErr replicas > 1 not supported in non-clustered mode JSStreamReplicasNotSupportedErr ErrorIdentifier = 10074 // JSStreamReplicasNotUpdatableErr Replicas configuration can not be updated JSStreamReplicasNotUpdatableErr ErrorIdentifier = 10061 // JSStreamRestoreErrF restore failed: {err} JSStreamRestoreErrF ErrorIdentifier = 10062 // JSStreamRollupFailedF Generic stream rollup failure error string ({err}) JSStreamRollupFailedF ErrorIdentifier = 10111 // JSStreamSealedErr invalid operation on sealed stream JSStreamSealedErr ErrorIdentifier = 10109 // JSStreamSequenceNotMatchErr expected stream sequence does not match JSStreamSequenceNotMatchErr ErrorIdentifier = 10063 // JSStreamSnapshotErrF snapshot failed: {err} JSStreamSnapshotErrF ErrorIdentifier = 10064 // JSStreamStoreFailedF Generic error when storing a message failed ({err}) JSStreamStoreFailedF ErrorIdentifier = 10077 // JSStreamSubjectOverlapErr subjects overlap with an existing stream JSStreamSubjectOverlapErr ErrorIdentifier = 10065 // JSStreamTemplateCreateErrF Generic template creation failed string ({err}) JSStreamTemplateCreateErrF ErrorIdentifier = 10066 // JSStreamTemplateDeleteErrF Generic stream template deletion failed error string ({err}) JSStreamTemplateDeleteErrF ErrorIdentifier = 10067 // JSStreamTemplateNotFoundErr template not found JSStreamTemplateNotFoundErr ErrorIdentifier = 10068 // JSStreamUpdateErrF Generic stream update error string ({err}) JSStreamUpdateErrF ErrorIdentifier = 10069 // JSStreamWrongLastMsgIDErrF wrong last msg ID: {id} JSStreamWrongLastMsgIDErrF ErrorIdentifier = 10070 // JSStreamWrongLastSequenceErrF wrong last sequence: {seq} JSStreamWrongLastSequenceErrF ErrorIdentifier = 10071 // JSTempStorageFailedErr JetStream unable to open temp storage for restore JSTempStorageFailedErr ErrorIdentifier = 10072 // JSTemplateNameNotMatchSubjectErr template name in subject does not match request JSTemplateNameNotMatchSubjectErr ErrorIdentifier = 10073 ) var ( ApiErrors = map[ErrorIdentifier]*ApiError{ JSAccountResourcesExceededErr: {Code: 400, ErrCode: 10002, Description: "resource limits exceeded for account"}, JSBadRequestErr: {Code: 400, ErrCode: 10003, Description: "bad request"}, JSClusterIncompleteErr: {Code: 503, ErrCode: 10004, Description: "incomplete results"}, JSClusterNoPeersErrF: {Code: 400, ErrCode: 10005, Description: "{err}"}, JSClusterNotActiveErr: {Code: 500, ErrCode: 10006, Description: "JetStream not in clustered mode"}, JSClusterNotAssignedErr: {Code: 500, ErrCode: 10007, Description: "JetStream cluster not assigned to this server"}, JSClusterNotAvailErr: {Code: 503, ErrCode: 10008, Description: "JetStream system temporarily unavailable"}, JSClusterNotLeaderErr: {Code: 500, ErrCode: 10009, Description: "JetStream cluster can not handle request"}, JSClusterPeerNotMemberErr: {Code: 400, ErrCode: 10040, Description: "peer not a member"}, JSClusterRequiredErr: {Code: 503, ErrCode: 10010, Description: "JetStream clustering support required"}, JSClusterServerNotMemberErr: {Code: 400, ErrCode: 10044, Description: "server is not a member of the cluster"}, JSClusterTagsErr: {Code: 400, ErrCode: 10011, Description: "tags placement not supported for operation"}, JSClusterUnSupportFeatureErr: {Code: 503, ErrCode: 10036, Description: "not currently supported in clustered mode"}, JSConsumerAlreadyExists: {Code: 400, ErrCode: 10148, Description: "consumer already exists"}, JSConsumerBadDurableNameErr: {Code: 400, ErrCode: 10103, Description: "durable name can not contain '.', '*', '>'"}, JSConsumerConfigRequiredErr: {Code: 400, ErrCode: 10078, Description: "consumer config required"}, JSConsumerCreateDurableAndNameMismatch: {Code: 400, ErrCode: 10132, Description: "Consumer Durable and Name have to be equal if both are provided"}, JSConsumerCreateErrF: {Code: 500, ErrCode: 10012, Description: "{err}"}, JSConsumerCreateFilterSubjectMismatchErr: {Code: 400, ErrCode: 10131, Description: "Consumer create request did not match filtered subject from create subject"}, JSConsumerDeliverCycleErr: {Code: 400, ErrCode: 10081, Description: "consumer deliver subject forms a cycle"}, JSConsumerDeliverToWildcardsErr: {Code: 400, ErrCode: 10079, Description: "consumer deliver subject has wildcards"}, JSConsumerDescriptionTooLongErrF: {Code: 400, ErrCode: 10107, Description: "consumer description is too long, maximum allowed is {max}"}, JSConsumerDirectRequiresEphemeralErr: {Code: 400, ErrCode: 10091, Description: "consumer direct requires an ephemeral consumer"}, JSConsumerDirectRequiresPushErr: {Code: 400, ErrCode: 10090, Description: "consumer direct requires a push based consumer"}, JSConsumerDoesNotExist: {Code: 400, ErrCode: 10149, Description: "consumer does not exist"}, JSConsumerDuplicateFilterSubjects: {Code: 400, ErrCode: 10136, Description: "consumer cannot have both FilterSubject and FilterSubjects specified"}, JSConsumerDurableNameNotInSubjectErr: {Code: 400, ErrCode: 10016, Description: "consumer expected to be durable but no durable name set in subject"}, JSConsumerDurableNameNotMatchSubjectErr: {Code: 400, ErrCode: 10017, Description: "consumer name in subject does not match durable name in request"}, JSConsumerDurableNameNotSetErr: {Code: 400, ErrCode: 10018, Description: "consumer expected to be durable but a durable name was not set"}, JSConsumerEmptyFilter: {Code: 400, ErrCode: 10139, Description: "consumer filter in FilterSubjects cannot be empty"}, JSConsumerEphemeralWithDurableInSubjectErr: {Code: 400, ErrCode: 10019, Description: "consumer expected to be ephemeral but detected a durable name set in subject"}, JSConsumerEphemeralWithDurableNameErr: {Code: 400, ErrCode: 10020, Description: "consumer expected to be ephemeral but a durable name was set in request"}, JSConsumerExistingActiveErr: {Code: 400, ErrCode: 10105, Description: "consumer already exists and is still active"}, JSConsumerFCRequiresPushErr: {Code: 400, ErrCode: 10089, Description: "consumer flow control requires a push based consumer"}, JSConsumerFilterNotSubsetErr: {Code: 400, ErrCode: 10093, Description: "consumer filter subject is not a valid subset of the interest subjects"}, JSConsumerHBRequiresPushErr: {Code: 400, ErrCode: 10088, Description: "consumer idle heartbeat requires a push based consumer"}, JSConsumerInactiveThresholdExcess: {Code: 400, ErrCode: 10153, Description: "consumer inactive threshold exceeds system limit of {limit}"}, JSConsumerInvalidDeliverSubject: {Code: 400, ErrCode: 10112, Description: "invalid push consumer deliver subject"}, JSConsumerInvalidPolicyErrF: {Code: 400, ErrCode: 10094, Description: "{err}"}, JSConsumerInvalidSamplingErrF: {Code: 400, ErrCode: 10095, Description: "failed to parse consumer sampling configuration: {err}"}, JSConsumerMaxDeliverBackoffErr: {Code: 400, ErrCode: 10116, Description: "max deliver is required to be > length of backoff values"}, JSConsumerMaxPendingAckExcessErrF: {Code: 400, ErrCode: 10121, Description: "consumer max ack pending exceeds system limit of {limit}"}, JSConsumerMaxPendingAckPolicyRequiredErr: {Code: 400, ErrCode: 10082, Description: "consumer requires ack policy for max ack pending"}, JSConsumerMaxRequestBatchExceededF: {Code: 400, ErrCode: 10125, Description: "consumer max request batch exceeds server limit of {limit}"}, JSConsumerMaxRequestBatchNegativeErr: {Code: 400, ErrCode: 10114, Description: "consumer max request batch needs to be > 0"}, JSConsumerMaxRequestExpiresToSmall: {Code: 400, ErrCode: 10115, Description: "consumer max request expires needs to be >= 1ms"}, JSConsumerMaxWaitingNegativeErr: {Code: 400, ErrCode: 10087, Description: "consumer max waiting needs to be positive"}, JSConsumerMetadataLengthErrF: {Code: 400, ErrCode: 10135, Description: "consumer metadata exceeds maximum size of {limit}"}, JSConsumerMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10137, Description: "consumer with multiple subject filters cannot use subject based API"}, JSConsumerNameContainsPathSeparatorsErr: {Code: 400, ErrCode: 10127, Description: "Consumer name can not contain path separators"}, JSConsumerNameExistErr: {Code: 400, ErrCode: 10013, Description: "consumer name already in use"}, JSConsumerNameTooLongErrF: {Code: 400, ErrCode: 10102, Description: "consumer name is too long, maximum allowed is {max}"}, JSConsumerNotFoundErr: {Code: 404, ErrCode: 10014, Description: "consumer not found"}, JSConsumerOfflineErr: {Code: 500, ErrCode: 10119, Description: "consumer is offline"}, JSConsumerOnMappedErr: {Code: 400, ErrCode: 10092, Description: "consumer direct on a mapped consumer"}, JSConsumerOverlappingSubjectFilters: {Code: 400, ErrCode: 10138, Description: "consumer subject filters cannot overlap"}, JSConsumerPullNotDurableErr: {Code: 400, ErrCode: 10085, Description: "consumer in pull mode requires a durable name"}, JSConsumerPullRequiresAckErr: {Code: 400, ErrCode: 10084, Description: "consumer in pull mode requires ack policy"}, JSConsumerPullWithRateLimitErr: {Code: 400, ErrCode: 10086, Description: "consumer in pull mode can not have rate limit set"}, JSConsumerPushMaxWaitingErr: {Code: 400, ErrCode: 10080, Description: "consumer in push mode can not set max waiting"}, JSConsumerReplacementWithDifferentNameErr: {Code: 400, ErrCode: 10106, Description: "consumer replacement durable config not the same"}, JSConsumerReplicasExceedsStream: {Code: 400, ErrCode: 10126, Description: "consumer config replica count exceeds parent stream"}, JSConsumerReplicasShouldMatchStream: {Code: 400, ErrCode: 10134, Description: "consumer config replicas must match interest retention stream's replicas"}, JSConsumerSmallHeartbeatErr: {Code: 400, ErrCode: 10083, Description: "consumer idle heartbeat needs to be >= 100ms"}, JSConsumerStoreFailedErrF: {Code: 500, ErrCode: 10104, Description: "error creating store for consumer: {err}"}, JSConsumerWQConsumerNotDeliverAllErr: {Code: 400, ErrCode: 10101, Description: "consumer must be deliver all on workqueue stream"}, JSConsumerWQConsumerNotUniqueErr: {Code: 400, ErrCode: 10100, Description: "filtered consumer not unique on workqueue stream"}, JSConsumerWQMultipleUnfilteredErr: {Code: 400, ErrCode: 10099, Description: "multiple non-filtered consumers not allowed on workqueue stream"}, JSConsumerWQRequiresExplicitAckErr: {Code: 400, ErrCode: 10098, Description: "workqueue stream requires explicit ack"}, JSConsumerWithFlowControlNeedsHeartbeats: {Code: 400, ErrCode: 10108, Description: "consumer with flow control also needs heartbeats"}, JSInsufficientResourcesErr: {Code: 503, ErrCode: 10023, Description: "insufficient resources"}, JSInvalidJSONErr: {Code: 400, ErrCode: 10025, Description: "invalid JSON"}, JSMaximumConsumersLimitErr: {Code: 400, ErrCode: 10026, Description: "maximum consumers limit reached"}, JSMaximumStreamsLimitErr: {Code: 400, ErrCode: 10027, Description: "maximum number of streams reached"}, JSMemoryResourcesExceededErr: {Code: 500, ErrCode: 10028, Description: "insufficient memory resources available"}, JSMirrorConsumerSetupFailedErrF: {Code: 500, ErrCode: 10029, Description: "{err}"}, JSMirrorInvalidStreamName: {Code: 400, ErrCode: 10142, Description: "mirrored stream name is invalid"}, JSMirrorInvalidSubjectFilter: {Code: 400, ErrCode: 10151, Description: "mirror subject filter is invalid"}, JSMirrorMaxMessageSizeTooBigErr: {Code: 400, ErrCode: 10030, Description: "stream mirror must have max message size >= source"}, JSMirrorMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10150, Description: "mirror with multiple subject transforms cannot also have a single subject filter"}, JSMirrorOverlappingSubjectFilters: {Code: 400, ErrCode: 10152, Description: "mirror subject filters can not overlap"}, JSMirrorWithFirstSeqErr: {Code: 400, ErrCode: 10143, Description: "stream mirrors can not have first sequence configured"}, JSMirrorWithSourcesErr: {Code: 400, ErrCode: 10031, Description: "stream mirrors can not also contain other sources"}, JSMirrorWithStartSeqAndTimeErr: {Code: 400, ErrCode: 10032, Description: "stream mirrors can not have both start seq and start time configured"}, JSMirrorWithSubjectFiltersErr: {Code: 400, ErrCode: 10033, Description: "stream mirrors can not contain filtered subjects"}, JSMirrorWithSubjectsErr: {Code: 400, ErrCode: 10034, Description: "stream mirrors can not contain subjects"}, JSNoAccountErr: {Code: 503, ErrCode: 10035, Description: "account not found"}, JSNoLimitsErr: {Code: 400, ErrCode: 10120, Description: "no JetStream default or applicable tiered limit present"}, JSNoMessageFoundErr: {Code: 404, ErrCode: 10037, Description: "no message found"}, JSNotEmptyRequestErr: {Code: 400, ErrCode: 10038, Description: "expected an empty request payload"}, JSNotEnabledErr: {Code: 503, ErrCode: 10076, Description: "JetStream not enabled"}, JSNotEnabledForAccountErr: {Code: 503, ErrCode: 10039, Description: "JetStream not enabled for account"}, JSPeerRemapErr: {Code: 503, ErrCode: 10075, Description: "peer remap failed"}, JSRaftGeneralErrF: {Code: 500, ErrCode: 10041, Description: "{err}"}, JSReplicasCountCannotBeNegative: {Code: 400, ErrCode: 10133, Description: "replicas count cannot be negative"}, JSRestoreSubscribeFailedErrF: {Code: 500, ErrCode: 10042, Description: "JetStream unable to subscribe to restore snapshot {subject}: {err}"}, JSSequenceNotFoundErrF: {Code: 400, ErrCode: 10043, Description: "sequence {seq} not found"}, JSSnapshotDeliverSubjectInvalidErr: {Code: 400, ErrCode: 10015, Description: "deliver subject not valid"}, JSSourceConsumerSetupFailedErrF: {Code: 500, ErrCode: 10045, Description: "{err}"}, JSSourceDuplicateDetected: {Code: 400, ErrCode: 10140, Description: "duplicate source configuration detected"}, JSSourceInvalidStreamName: {Code: 400, ErrCode: 10141, Description: "sourced stream name is invalid"}, JSSourceInvalidSubjectFilter: {Code: 400, ErrCode: 10145, Description: "source subject filter is invalid"}, JSSourceInvalidTransformDestination: {Code: 400, ErrCode: 10146, Description: "source transform destination is invalid"}, JSSourceMaxMessageSizeTooBigErr: {Code: 400, ErrCode: 10046, Description: "stream source must have max message size >= target"}, JSSourceMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10144, Description: "source with multiple subject transforms cannot also have a single subject filter"}, JSSourceOverlappingSubjectFilters: {Code: 400, ErrCode: 10147, Description: "source filters can not overlap"}, JSStorageResourcesExceededErr: {Code: 500, ErrCode: 10047, Description: "insufficient storage resources available"}, JSStreamAssignmentErrF: {Code: 500, ErrCode: 10048, Description: "{err}"}, JSStreamCreateErrF: {Code: 500, ErrCode: 10049, Description: "{err}"}, JSStreamDeleteErrF: {Code: 500, ErrCode: 10050, Description: "{err}"}, JSStreamExternalApiOverlapErrF: {Code: 400, ErrCode: 10021, Description: "stream external api prefix {prefix} must not overlap with {subject}"}, JSStreamExternalDelPrefixOverlapsErrF: {Code: 400, ErrCode: 10022, Description: "stream external delivery prefix {prefix} overlaps with stream subject {subject}"}, JSStreamGeneralErrorF: {Code: 500, ErrCode: 10051, Description: "{err}"}, JSStreamHeaderExceedsMaximumErr: {Code: 400, ErrCode: 10097, Description: "header size exceeds maximum allowed of 64k"}, JSStreamInfoMaxSubjectsErr: {Code: 500, ErrCode: 10117, Description: "subject details would exceed maximum allowed"}, JSStreamInvalidConfigF: {Code: 500, ErrCode: 10052, Description: "{err}"}, JSStreamInvalidErr: {Code: 500, ErrCode: 10096, Description: "stream not valid"}, JSStreamInvalidExternalDeliverySubjErrF: {Code: 400, ErrCode: 10024, Description: "stream external delivery prefix {prefix} must not contain wildcards"}, JSStreamLimitsErrF: {Code: 500, ErrCode: 10053, Description: "{err}"}, JSStreamMaxBytesRequired: {Code: 400, ErrCode: 10113, Description: "account requires a stream config to have max bytes set"}, JSStreamMaxStreamBytesExceeded: {Code: 400, ErrCode: 10122, Description: "stream max bytes exceeds account limit max stream bytes"}, JSStreamMessageExceedsMaximumErr: {Code: 400, ErrCode: 10054, Description: "message size exceeds maximum allowed"}, JSStreamMirrorNotUpdatableErr: {Code: 400, ErrCode: 10055, Description: "stream mirror configuration can not be updated"}, JSStreamMismatchErr: {Code: 400, ErrCode: 10056, Description: "stream name in subject does not match request"}, JSStreamMoveAndScaleErr: {Code: 400, ErrCode: 10123, Description: "can not move and scale a stream in a single update"}, JSStreamMoveInProgressF: {Code: 400, ErrCode: 10124, Description: "stream move already in progress: {msg}"}, JSStreamMoveNotInProgress: {Code: 400, ErrCode: 10129, Description: "stream move not in progress"}, JSStreamMsgDeleteFailedF: {Code: 500, ErrCode: 10057, Description: "{err}"}, JSStreamNameContainsPathSeparatorsErr: {Code: 400, ErrCode: 10128, Description: "Stream name can not contain path separators"}, JSStreamNameExistErr: {Code: 400, ErrCode: 10058, Description: "stream name already in use with a different configuration"}, JSStreamNameExistRestoreFailedErr: {Code: 400, ErrCode: 10130, Description: "stream name already in use, cannot restore"}, JSStreamNotFoundErr: {Code: 404, ErrCode: 10059, Description: "stream not found"}, JSStreamNotMatchErr: {Code: 400, ErrCode: 10060, Description: "expected stream does not match"}, JSStreamOfflineErr: {Code: 500, ErrCode: 10118, Description: "stream is offline"}, JSStreamPurgeFailedF: {Code: 500, ErrCode: 10110, Description: "{err}"}, JSStreamReplicasNotSupportedErr: {Code: 500, ErrCode: 10074, Description: "replicas > 1 not supported in non-clustered mode"}, JSStreamReplicasNotUpdatableErr: {Code: 400, ErrCode: 10061, Description: "Replicas configuration can not be updated"}, JSStreamRestoreErrF: {Code: 500, ErrCode: 10062, Description: "restore failed: {err}"}, JSStreamRollupFailedF: {Code: 500, ErrCode: 10111, Description: "{err}"}, JSStreamSealedErr: {Code: 400, ErrCode: 10109, Description: "invalid operation on sealed stream"}, JSStreamSequenceNotMatchErr: {Code: 503, ErrCode: 10063, Description: "expected stream sequence does not match"}, JSStreamSnapshotErrF: {Code: 500, ErrCode: 10064, Description: "snapshot failed: {err}"}, JSStreamStoreFailedF: {Code: 503, ErrCode: 10077, Description: "{err}"}, JSStreamSubjectOverlapErr: {Code: 400, ErrCode: 10065, Description: "subjects overlap with an existing stream"}, JSStreamTemplateCreateErrF: {Code: 500, ErrCode: 10066, Description: "{err}"}, JSStreamTemplateDeleteErrF: {Code: 500, ErrCode: 10067, Description: "{err}"}, JSStreamTemplateNotFoundErr: {Code: 404, ErrCode: 10068, Description: "template not found"}, JSStreamUpdateErrF: {Code: 500, ErrCode: 10069, Description: "{err}"}, JSStreamWrongLastMsgIDErrF: {Code: 400, ErrCode: 10070, Description: "wrong last msg ID: {id}"}, JSStreamWrongLastSequenceErrF: {Code: 400, ErrCode: 10071, Description: "wrong last sequence: {seq}"}, JSTempStorageFailedErr: {Code: 500, ErrCode: 10072, Description: "JetStream unable to open temp storage for restore"}, JSTemplateNameNotMatchSubjectErr: {Code: 400, ErrCode: 10073, Description: "template name in subject does not match request"}, } // ErrJetStreamNotClustered Deprecated by JSClusterNotActiveErr ApiError, use IsNatsError() for comparisons ErrJetStreamNotClustered = ApiErrors[JSClusterNotActiveErr] // ErrJetStreamNotAssigned Deprecated by JSClusterNotAssignedErr ApiError, use IsNatsError() for comparisons ErrJetStreamNotAssigned = ApiErrors[JSClusterNotAssignedErr] // ErrJetStreamNotLeader Deprecated by JSClusterNotLeaderErr ApiError, use IsNatsError() for comparisons ErrJetStreamNotLeader = ApiErrors[JSClusterNotLeaderErr] // ErrJetStreamConsumerAlreadyUsed Deprecated by JSConsumerNameExistErr ApiError, use IsNatsError() for comparisons ErrJetStreamConsumerAlreadyUsed = ApiErrors[JSConsumerNameExistErr] // ErrJetStreamResourcesExceeded Deprecated by JSInsufficientResourcesErr ApiError, use IsNatsError() for comparisons ErrJetStreamResourcesExceeded = ApiErrors[JSInsufficientResourcesErr] // ErrMemoryResourcesExceeded Deprecated by JSMemoryResourcesExceededErr ApiError, use IsNatsError() for comparisons ErrMemoryResourcesExceeded = ApiErrors[JSMemoryResourcesExceededErr] // ErrJetStreamNotEnabled Deprecated by JSNotEnabledErr ApiError, use IsNatsError() for comparisons ErrJetStreamNotEnabled = ApiErrors[JSNotEnabledErr] // ErrStorageResourcesExceeded Deprecated by JSStorageResourcesExceededErr ApiError, use IsNatsError() for comparisons ErrStorageResourcesExceeded = ApiErrors[JSStorageResourcesExceededErr] // ErrJetStreamStreamAlreadyUsed Deprecated by JSStreamNameExistErr ApiError, use IsNatsError() for comparisons ErrJetStreamStreamAlreadyUsed = ApiErrors[JSStreamNameExistErr] // ErrJetStreamStreamNotFound Deprecated by JSStreamNotFoundErr ApiError, use IsNatsError() for comparisons ErrJetStreamStreamNotFound = ApiErrors[JSStreamNotFoundErr] // ErrReplicasNotSupported Deprecated by JSStreamReplicasNotSupportedErr ApiError, use IsNatsError() for comparisons ErrReplicasNotSupported = ApiErrors[JSStreamReplicasNotSupportedErr] ) // NewJSAccountResourcesExceededError creates a new JSAccountResourcesExceededErr error: "resource limits exceeded for account" func NewJSAccountResourcesExceededError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSAccountResourcesExceededErr] } // NewJSBadRequestError creates a new JSBadRequestErr error: "bad request" func NewJSBadRequestError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSBadRequestErr] } // NewJSClusterIncompleteError creates a new JSClusterIncompleteErr error: "incomplete results" func NewJSClusterIncompleteError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterIncompleteErr] } // NewJSClusterNoPeersError creates a new JSClusterNoPeersErrF error: "{err}" func NewJSClusterNoPeersError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSClusterNoPeersErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSClusterNotActiveError creates a new JSClusterNotActiveErr error: "JetStream not in clustered mode" func NewJSClusterNotActiveError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterNotActiveErr] } // NewJSClusterNotAssignedError creates a new JSClusterNotAssignedErr error: "JetStream cluster not assigned to this server" func NewJSClusterNotAssignedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterNotAssignedErr] } // NewJSClusterNotAvailError creates a new JSClusterNotAvailErr error: "JetStream system temporarily unavailable" func NewJSClusterNotAvailError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterNotAvailErr] } // NewJSClusterNotLeaderError creates a new JSClusterNotLeaderErr error: "JetStream cluster can not handle request" func NewJSClusterNotLeaderError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterNotLeaderErr] } // NewJSClusterPeerNotMemberError creates a new JSClusterPeerNotMemberErr error: "peer not a member" func NewJSClusterPeerNotMemberError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterPeerNotMemberErr] } // NewJSClusterRequiredError creates a new JSClusterRequiredErr error: "JetStream clustering support required" func NewJSClusterRequiredError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterRequiredErr] } // NewJSClusterServerNotMemberError creates a new JSClusterServerNotMemberErr error: "server is not a member of the cluster" func NewJSClusterServerNotMemberError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterServerNotMemberErr] } // NewJSClusterTagsError creates a new JSClusterTagsErr error: "tags placement not supported for operation" func NewJSClusterTagsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterTagsErr] } // NewJSClusterUnSupportFeatureError creates a new JSClusterUnSupportFeatureErr error: "not currently supported in clustered mode" func NewJSClusterUnSupportFeatureError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSClusterUnSupportFeatureErr] } // NewJSConsumerAlreadyExistsError creates a new JSConsumerAlreadyExists error: "consumer already exists" func NewJSConsumerAlreadyExistsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerAlreadyExists] } // NewJSConsumerBadDurableNameError creates a new JSConsumerBadDurableNameErr error: "durable name can not contain '.', '*', '>'" func NewJSConsumerBadDurableNameError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerBadDurableNameErr] } // NewJSConsumerConfigRequiredError creates a new JSConsumerConfigRequiredErr error: "consumer config required" func NewJSConsumerConfigRequiredError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerConfigRequiredErr] } // NewJSConsumerCreateDurableAndNameMismatchError creates a new JSConsumerCreateDurableAndNameMismatch error: "Consumer Durable and Name have to be equal if both are provided" func NewJSConsumerCreateDurableAndNameMismatchError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerCreateDurableAndNameMismatch] } // NewJSConsumerCreateError creates a new JSConsumerCreateErrF error: "{err}" func NewJSConsumerCreateError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerCreateErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerCreateFilterSubjectMismatchError creates a new JSConsumerCreateFilterSubjectMismatchErr error: "Consumer create request did not match filtered subject from create subject" func NewJSConsumerCreateFilterSubjectMismatchError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerCreateFilterSubjectMismatchErr] } // NewJSConsumerDeliverCycleError creates a new JSConsumerDeliverCycleErr error: "consumer deliver subject forms a cycle" func NewJSConsumerDeliverCycleError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDeliverCycleErr] } // NewJSConsumerDeliverToWildcardsError creates a new JSConsumerDeliverToWildcardsErr error: "consumer deliver subject has wildcards" func NewJSConsumerDeliverToWildcardsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDeliverToWildcardsErr] } // NewJSConsumerDescriptionTooLongError creates a new JSConsumerDescriptionTooLongErrF error: "consumer description is too long, maximum allowed is {max}" func NewJSConsumerDescriptionTooLongError(max interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerDescriptionTooLongErrF] args := e.toReplacerArgs([]interface{}{"{max}", max}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerDirectRequiresEphemeralError creates a new JSConsumerDirectRequiresEphemeralErr error: "consumer direct requires an ephemeral consumer" func NewJSConsumerDirectRequiresEphemeralError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDirectRequiresEphemeralErr] } // NewJSConsumerDirectRequiresPushError creates a new JSConsumerDirectRequiresPushErr error: "consumer direct requires a push based consumer" func NewJSConsumerDirectRequiresPushError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDirectRequiresPushErr] } // NewJSConsumerDoesNotExistError creates a new JSConsumerDoesNotExist error: "consumer does not exist" func NewJSConsumerDoesNotExistError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDoesNotExist] } // NewJSConsumerDuplicateFilterSubjectsError creates a new JSConsumerDuplicateFilterSubjects error: "consumer cannot have both FilterSubject and FilterSubjects specified" func NewJSConsumerDuplicateFilterSubjectsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDuplicateFilterSubjects] } // NewJSConsumerDurableNameNotInSubjectError creates a new JSConsumerDurableNameNotInSubjectErr error: "consumer expected to be durable but no durable name set in subject" func NewJSConsumerDurableNameNotInSubjectError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDurableNameNotInSubjectErr] } // NewJSConsumerDurableNameNotMatchSubjectError creates a new JSConsumerDurableNameNotMatchSubjectErr error: "consumer name in subject does not match durable name in request" func NewJSConsumerDurableNameNotMatchSubjectError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDurableNameNotMatchSubjectErr] } // NewJSConsumerDurableNameNotSetError creates a new JSConsumerDurableNameNotSetErr error: "consumer expected to be durable but a durable name was not set" func NewJSConsumerDurableNameNotSetError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerDurableNameNotSetErr] } // NewJSConsumerEmptyFilterError creates a new JSConsumerEmptyFilter error: "consumer filter in FilterSubjects cannot be empty" func NewJSConsumerEmptyFilterError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerEmptyFilter] } // NewJSConsumerEphemeralWithDurableInSubjectError creates a new JSConsumerEphemeralWithDurableInSubjectErr error: "consumer expected to be ephemeral but detected a durable name set in subject" func NewJSConsumerEphemeralWithDurableInSubjectError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerEphemeralWithDurableInSubjectErr] } // NewJSConsumerEphemeralWithDurableNameError creates a new JSConsumerEphemeralWithDurableNameErr error: "consumer expected to be ephemeral but a durable name was set in request" func NewJSConsumerEphemeralWithDurableNameError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerEphemeralWithDurableNameErr] } // NewJSConsumerExistingActiveError creates a new JSConsumerExistingActiveErr error: "consumer already exists and is still active" func NewJSConsumerExistingActiveError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerExistingActiveErr] } // NewJSConsumerFCRequiresPushError creates a new JSConsumerFCRequiresPushErr error: "consumer flow control requires a push based consumer" func NewJSConsumerFCRequiresPushError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerFCRequiresPushErr] } // NewJSConsumerFilterNotSubsetError creates a new JSConsumerFilterNotSubsetErr error: "consumer filter subject is not a valid subset of the interest subjects" func NewJSConsumerFilterNotSubsetError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerFilterNotSubsetErr] } // NewJSConsumerHBRequiresPushError creates a new JSConsumerHBRequiresPushErr error: "consumer idle heartbeat requires a push based consumer" func NewJSConsumerHBRequiresPushError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerHBRequiresPushErr] } // NewJSConsumerInactiveThresholdExcessError creates a new JSConsumerInactiveThresholdExcess error: "consumer inactive threshold exceeds system limit of {limit}" func NewJSConsumerInactiveThresholdExcessError(limit interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerInactiveThresholdExcess] args := e.toReplacerArgs([]interface{}{"{limit}", limit}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerInvalidDeliverSubjectError creates a new JSConsumerInvalidDeliverSubject error: "invalid push consumer deliver subject" func NewJSConsumerInvalidDeliverSubjectError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerInvalidDeliverSubject] } // NewJSConsumerInvalidPolicyError creates a new JSConsumerInvalidPolicyErrF error: "{err}" func NewJSConsumerInvalidPolicyError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerInvalidPolicyErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerInvalidSamplingError creates a new JSConsumerInvalidSamplingErrF error: "failed to parse consumer sampling configuration: {err}" func NewJSConsumerInvalidSamplingError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerInvalidSamplingErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerMaxDeliverBackoffError creates a new JSConsumerMaxDeliverBackoffErr error: "max deliver is required to be > length of backoff values" func NewJSConsumerMaxDeliverBackoffError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMaxDeliverBackoffErr] } // NewJSConsumerMaxPendingAckExcessError creates a new JSConsumerMaxPendingAckExcessErrF error: "consumer max ack pending exceeds system limit of {limit}" func NewJSConsumerMaxPendingAckExcessError(limit interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerMaxPendingAckExcessErrF] args := e.toReplacerArgs([]interface{}{"{limit}", limit}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerMaxPendingAckPolicyRequiredError creates a new JSConsumerMaxPendingAckPolicyRequiredErr error: "consumer requires ack policy for max ack pending" func NewJSConsumerMaxPendingAckPolicyRequiredError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMaxPendingAckPolicyRequiredErr] } // NewJSConsumerMaxRequestBatchExceededError creates a new JSConsumerMaxRequestBatchExceededF error: "consumer max request batch exceeds server limit of {limit}" func NewJSConsumerMaxRequestBatchExceededError(limit interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerMaxRequestBatchExceededF] args := e.toReplacerArgs([]interface{}{"{limit}", limit}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerMaxRequestBatchNegativeError creates a new JSConsumerMaxRequestBatchNegativeErr error: "consumer max request batch needs to be > 0" func NewJSConsumerMaxRequestBatchNegativeError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMaxRequestBatchNegativeErr] } // NewJSConsumerMaxRequestExpiresToSmallError creates a new JSConsumerMaxRequestExpiresToSmall error: "consumer max request expires needs to be >= 1ms" func NewJSConsumerMaxRequestExpiresToSmallError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMaxRequestExpiresToSmall] } // NewJSConsumerMaxWaitingNegativeError creates a new JSConsumerMaxWaitingNegativeErr error: "consumer max waiting needs to be positive" func NewJSConsumerMaxWaitingNegativeError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMaxWaitingNegativeErr] } // NewJSConsumerMetadataLengthError creates a new JSConsumerMetadataLengthErrF error: "consumer metadata exceeds maximum size of {limit}" func NewJSConsumerMetadataLengthError(limit interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerMetadataLengthErrF] args := e.toReplacerArgs([]interface{}{"{limit}", limit}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerMultipleFiltersNotAllowedError creates a new JSConsumerMultipleFiltersNotAllowed error: "consumer with multiple subject filters cannot use subject based API" func NewJSConsumerMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerMultipleFiltersNotAllowed] } // NewJSConsumerNameContainsPathSeparatorsError creates a new JSConsumerNameContainsPathSeparatorsErr error: "Consumer name can not contain path separators" func NewJSConsumerNameContainsPathSeparatorsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerNameContainsPathSeparatorsErr] } // NewJSConsumerNameExistError creates a new JSConsumerNameExistErr error: "consumer name already in use" func NewJSConsumerNameExistError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerNameExistErr] } // NewJSConsumerNameTooLongError creates a new JSConsumerNameTooLongErrF error: "consumer name is too long, maximum allowed is {max}" func NewJSConsumerNameTooLongError(max interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerNameTooLongErrF] args := e.toReplacerArgs([]interface{}{"{max}", max}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerNotFoundError creates a new JSConsumerNotFoundErr error: "consumer not found" func NewJSConsumerNotFoundError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerNotFoundErr] } // NewJSConsumerOfflineError creates a new JSConsumerOfflineErr error: "consumer is offline" func NewJSConsumerOfflineError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerOfflineErr] } // NewJSConsumerOnMappedError creates a new JSConsumerOnMappedErr error: "consumer direct on a mapped consumer" func NewJSConsumerOnMappedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerOnMappedErr] } // NewJSConsumerOverlappingSubjectFiltersError creates a new JSConsumerOverlappingSubjectFilters error: "consumer subject filters cannot overlap" func NewJSConsumerOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerOverlappingSubjectFilters] } // NewJSConsumerPullNotDurableError creates a new JSConsumerPullNotDurableErr error: "consumer in pull mode requires a durable name" func NewJSConsumerPullNotDurableError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerPullNotDurableErr] } // NewJSConsumerPullRequiresAckError creates a new JSConsumerPullRequiresAckErr error: "consumer in pull mode requires ack policy" func NewJSConsumerPullRequiresAckError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerPullRequiresAckErr] } // NewJSConsumerPullWithRateLimitError creates a new JSConsumerPullWithRateLimitErr error: "consumer in pull mode can not have rate limit set" func NewJSConsumerPullWithRateLimitError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerPullWithRateLimitErr] } // NewJSConsumerPushMaxWaitingError creates a new JSConsumerPushMaxWaitingErr error: "consumer in push mode can not set max waiting" func NewJSConsumerPushMaxWaitingError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerPushMaxWaitingErr] } // NewJSConsumerReplacementWithDifferentNameError creates a new JSConsumerReplacementWithDifferentNameErr error: "consumer replacement durable config not the same" func NewJSConsumerReplacementWithDifferentNameError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerReplacementWithDifferentNameErr] } // NewJSConsumerReplicasExceedsStreamError creates a new JSConsumerReplicasExceedsStream error: "consumer config replica count exceeds parent stream" func NewJSConsumerReplicasExceedsStreamError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerReplicasExceedsStream] } // NewJSConsumerReplicasShouldMatchStreamError creates a new JSConsumerReplicasShouldMatchStream error: "consumer config replicas must match interest retention stream's replicas" func NewJSConsumerReplicasShouldMatchStreamError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerReplicasShouldMatchStream] } // NewJSConsumerSmallHeartbeatError creates a new JSConsumerSmallHeartbeatErr error: "consumer idle heartbeat needs to be >= 100ms" func NewJSConsumerSmallHeartbeatError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerSmallHeartbeatErr] } // NewJSConsumerStoreFailedError creates a new JSConsumerStoreFailedErrF error: "error creating store for consumer: {err}" func NewJSConsumerStoreFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSConsumerStoreFailedErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSConsumerWQConsumerNotDeliverAllError creates a new JSConsumerWQConsumerNotDeliverAllErr error: "consumer must be deliver all on workqueue stream" func NewJSConsumerWQConsumerNotDeliverAllError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerWQConsumerNotDeliverAllErr] } // NewJSConsumerWQConsumerNotUniqueError creates a new JSConsumerWQConsumerNotUniqueErr error: "filtered consumer not unique on workqueue stream" func NewJSConsumerWQConsumerNotUniqueError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerWQConsumerNotUniqueErr] } // NewJSConsumerWQMultipleUnfilteredError creates a new JSConsumerWQMultipleUnfilteredErr error: "multiple non-filtered consumers not allowed on workqueue stream" func NewJSConsumerWQMultipleUnfilteredError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerWQMultipleUnfilteredErr] } // NewJSConsumerWQRequiresExplicitAckError creates a new JSConsumerWQRequiresExplicitAckErr error: "workqueue stream requires explicit ack" func NewJSConsumerWQRequiresExplicitAckError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerWQRequiresExplicitAckErr] } // NewJSConsumerWithFlowControlNeedsHeartbeatsError creates a new JSConsumerWithFlowControlNeedsHeartbeats error: "consumer with flow control also needs heartbeats" func NewJSConsumerWithFlowControlNeedsHeartbeatsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSConsumerWithFlowControlNeedsHeartbeats] } // NewJSInsufficientResourcesError creates a new JSInsufficientResourcesErr error: "insufficient resources" func NewJSInsufficientResourcesError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSInsufficientResourcesErr] } // NewJSInvalidJSONError creates a new JSInvalidJSONErr error: "invalid JSON" func NewJSInvalidJSONError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSInvalidJSONErr] } // NewJSMaximumConsumersLimitError creates a new JSMaximumConsumersLimitErr error: "maximum consumers limit reached" func NewJSMaximumConsumersLimitError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMaximumConsumersLimitErr] } // NewJSMaximumStreamsLimitError creates a new JSMaximumStreamsLimitErr error: "maximum number of streams reached" func NewJSMaximumStreamsLimitError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMaximumStreamsLimitErr] } // NewJSMemoryResourcesExceededError creates a new JSMemoryResourcesExceededErr error: "insufficient memory resources available" func NewJSMemoryResourcesExceededError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMemoryResourcesExceededErr] } // NewJSMirrorConsumerSetupFailedError creates a new JSMirrorConsumerSetupFailedErrF error: "{err}" func NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSMirrorConsumerSetupFailedErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSMirrorInvalidStreamNameError creates a new JSMirrorInvalidStreamName error: "mirrored stream name is invalid" func NewJSMirrorInvalidStreamNameError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorInvalidStreamName] } // NewJSMirrorInvalidSubjectFilterError creates a new JSMirrorInvalidSubjectFilter error: "mirror subject filter is invalid" func NewJSMirrorInvalidSubjectFilterError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorInvalidSubjectFilter] } // NewJSMirrorMaxMessageSizeTooBigError creates a new JSMirrorMaxMessageSizeTooBigErr error: "stream mirror must have max message size >= source" func NewJSMirrorMaxMessageSizeTooBigError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorMaxMessageSizeTooBigErr] } // NewJSMirrorMultipleFiltersNotAllowedError creates a new JSMirrorMultipleFiltersNotAllowed error: "mirror with multiple subject transforms cannot also have a single subject filter" func NewJSMirrorMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorMultipleFiltersNotAllowed] } // NewJSMirrorOverlappingSubjectFiltersError creates a new JSMirrorOverlappingSubjectFilters error: "mirror subject filters can not overlap" func NewJSMirrorOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorOverlappingSubjectFilters] } // NewJSMirrorWithFirstSeqError creates a new JSMirrorWithFirstSeqErr error: "stream mirrors can not have first sequence configured" func NewJSMirrorWithFirstSeqError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorWithFirstSeqErr] } // NewJSMirrorWithSourcesError creates a new JSMirrorWithSourcesErr error: "stream mirrors can not also contain other sources" func NewJSMirrorWithSourcesError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorWithSourcesErr] } // NewJSMirrorWithStartSeqAndTimeError creates a new JSMirrorWithStartSeqAndTimeErr error: "stream mirrors can not have both start seq and start time configured" func NewJSMirrorWithStartSeqAndTimeError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorWithStartSeqAndTimeErr] } // NewJSMirrorWithSubjectFiltersError creates a new JSMirrorWithSubjectFiltersErr error: "stream mirrors can not contain filtered subjects" func NewJSMirrorWithSubjectFiltersError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorWithSubjectFiltersErr] } // NewJSMirrorWithSubjectsError creates a new JSMirrorWithSubjectsErr error: "stream mirrors can not contain subjects" func NewJSMirrorWithSubjectsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSMirrorWithSubjectsErr] } // NewJSNoAccountError creates a new JSNoAccountErr error: "account not found" func NewJSNoAccountError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNoAccountErr] } // NewJSNoLimitsError creates a new JSNoLimitsErr error: "no JetStream default or applicable tiered limit present" func NewJSNoLimitsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNoLimitsErr] } // NewJSNoMessageFoundError creates a new JSNoMessageFoundErr error: "no message found" func NewJSNoMessageFoundError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNoMessageFoundErr] } // NewJSNotEmptyRequestError creates a new JSNotEmptyRequestErr error: "expected an empty request payload" func NewJSNotEmptyRequestError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNotEmptyRequestErr] } // NewJSNotEnabledError creates a new JSNotEnabledErr error: "JetStream not enabled" func NewJSNotEnabledError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNotEnabledErr] } // NewJSNotEnabledForAccountError creates a new JSNotEnabledForAccountErr error: "JetStream not enabled for account" func NewJSNotEnabledForAccountError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSNotEnabledForAccountErr] } // NewJSPeerRemapError creates a new JSPeerRemapErr error: "peer remap failed" func NewJSPeerRemapError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSPeerRemapErr] } // NewJSRaftGeneralError creates a new JSRaftGeneralErrF error: "{err}" func NewJSRaftGeneralError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSRaftGeneralErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSReplicasCountCannotBeNegativeError creates a new JSReplicasCountCannotBeNegative error: "replicas count cannot be negative" func NewJSReplicasCountCannotBeNegativeError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSReplicasCountCannotBeNegative] } // NewJSRestoreSubscribeFailedError creates a new JSRestoreSubscribeFailedErrF error: "JetStream unable to subscribe to restore snapshot {subject}: {err}" func NewJSRestoreSubscribeFailedError(err error, subject interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSRestoreSubscribeFailedErrF] args := e.toReplacerArgs([]interface{}{"{err}", err, "{subject}", subject}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSSequenceNotFoundError creates a new JSSequenceNotFoundErrF error: "sequence {seq} not found" func NewJSSequenceNotFoundError(seq uint64, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSSequenceNotFoundErrF] args := e.toReplacerArgs([]interface{}{"{seq}", seq}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSSnapshotDeliverSubjectInvalidError creates a new JSSnapshotDeliverSubjectInvalidErr error: "deliver subject not valid" func NewJSSnapshotDeliverSubjectInvalidError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSnapshotDeliverSubjectInvalidErr] } // NewJSSourceConsumerSetupFailedError creates a new JSSourceConsumerSetupFailedErrF error: "{err}" func NewJSSourceConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSSourceConsumerSetupFailedErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSSourceDuplicateDetectedError creates a new JSSourceDuplicateDetected error: "duplicate source configuration detected" func NewJSSourceDuplicateDetectedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceDuplicateDetected] } // NewJSSourceInvalidStreamNameError creates a new JSSourceInvalidStreamName error: "sourced stream name is invalid" func NewJSSourceInvalidStreamNameError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceInvalidStreamName] } // NewJSSourceInvalidSubjectFilterError creates a new JSSourceInvalidSubjectFilter error: "source subject filter is invalid" func NewJSSourceInvalidSubjectFilterError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceInvalidSubjectFilter] } // NewJSSourceInvalidTransformDestinationError creates a new JSSourceInvalidTransformDestination error: "source transform destination is invalid" func NewJSSourceInvalidTransformDestinationError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceInvalidTransformDestination] } // NewJSSourceMaxMessageSizeTooBigError creates a new JSSourceMaxMessageSizeTooBigErr error: "stream source must have max message size >= target" func NewJSSourceMaxMessageSizeTooBigError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceMaxMessageSizeTooBigErr] } // NewJSSourceMultipleFiltersNotAllowedError creates a new JSSourceMultipleFiltersNotAllowed error: "source with multiple subject transforms cannot also have a single subject filter" func NewJSSourceMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceMultipleFiltersNotAllowed] } // NewJSSourceOverlappingSubjectFiltersError creates a new JSSourceOverlappingSubjectFilters error: "source filters can not overlap" func NewJSSourceOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSSourceOverlappingSubjectFilters] } // NewJSStorageResourcesExceededError creates a new JSStorageResourcesExceededErr error: "insufficient storage resources available" func NewJSStorageResourcesExceededError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStorageResourcesExceededErr] } // NewJSStreamAssignmentError creates a new JSStreamAssignmentErrF error: "{err}" func NewJSStreamAssignmentError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamAssignmentErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamCreateError creates a new JSStreamCreateErrF error: "{err}" func NewJSStreamCreateError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamCreateErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamDeleteError creates a new JSStreamDeleteErrF error: "{err}" func NewJSStreamDeleteError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamDeleteErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamExternalApiOverlapError creates a new JSStreamExternalApiOverlapErrF error: "stream external api prefix {prefix} must not overlap with {subject}" func NewJSStreamExternalApiOverlapError(prefix interface{}, subject interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamExternalApiOverlapErrF] args := e.toReplacerArgs([]interface{}{"{prefix}", prefix, "{subject}", subject}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamExternalDelPrefixOverlapsError creates a new JSStreamExternalDelPrefixOverlapsErrF error: "stream external delivery prefix {prefix} overlaps with stream subject {subject}" func NewJSStreamExternalDelPrefixOverlapsError(prefix interface{}, subject interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamExternalDelPrefixOverlapsErrF] args := e.toReplacerArgs([]interface{}{"{prefix}", prefix, "{subject}", subject}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamGeneralError creates a new JSStreamGeneralErrorF error: "{err}" func NewJSStreamGeneralError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamGeneralErrorF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamHeaderExceedsMaximumError creates a new JSStreamHeaderExceedsMaximumErr error: "header size exceeds maximum allowed of 64k" func NewJSStreamHeaderExceedsMaximumError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamHeaderExceedsMaximumErr] } // NewJSStreamInfoMaxSubjectsError creates a new JSStreamInfoMaxSubjectsErr error: "subject details would exceed maximum allowed" func NewJSStreamInfoMaxSubjectsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamInfoMaxSubjectsErr] } // NewJSStreamInvalidConfigError creates a new JSStreamInvalidConfigF error: "{err}" func NewJSStreamInvalidConfigError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamInvalidConfigF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamInvalidError creates a new JSStreamInvalidErr error: "stream not valid" func NewJSStreamInvalidError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamInvalidErr] } // NewJSStreamInvalidExternalDeliverySubjError creates a new JSStreamInvalidExternalDeliverySubjErrF error: "stream external delivery prefix {prefix} must not contain wildcards" func NewJSStreamInvalidExternalDeliverySubjError(prefix interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamInvalidExternalDeliverySubjErrF] args := e.toReplacerArgs([]interface{}{"{prefix}", prefix}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamLimitsError creates a new JSStreamLimitsErrF error: "{err}" func NewJSStreamLimitsError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamLimitsErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamMaxBytesRequiredError creates a new JSStreamMaxBytesRequired error: "account requires a stream config to have max bytes set" func NewJSStreamMaxBytesRequiredError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMaxBytesRequired] } // NewJSStreamMaxStreamBytesExceededError creates a new JSStreamMaxStreamBytesExceeded error: "stream max bytes exceeds account limit max stream bytes" func NewJSStreamMaxStreamBytesExceededError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMaxStreamBytesExceeded] } // NewJSStreamMessageExceedsMaximumError creates a new JSStreamMessageExceedsMaximumErr error: "message size exceeds maximum allowed" func NewJSStreamMessageExceedsMaximumError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMessageExceedsMaximumErr] } // NewJSStreamMirrorNotUpdatableError creates a new JSStreamMirrorNotUpdatableErr error: "stream mirror configuration can not be updated" func NewJSStreamMirrorNotUpdatableError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMirrorNotUpdatableErr] } // NewJSStreamMismatchError creates a new JSStreamMismatchErr error: "stream name in subject does not match request" func NewJSStreamMismatchError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMismatchErr] } // NewJSStreamMoveAndScaleError creates a new JSStreamMoveAndScaleErr error: "can not move and scale a stream in a single update" func NewJSStreamMoveAndScaleError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMoveAndScaleErr] } // NewJSStreamMoveInProgressError creates a new JSStreamMoveInProgressF error: "stream move already in progress: {msg}" func NewJSStreamMoveInProgressError(msg interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamMoveInProgressF] args := e.toReplacerArgs([]interface{}{"{msg}", msg}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamMoveNotInProgressError creates a new JSStreamMoveNotInProgress error: "stream move not in progress" func NewJSStreamMoveNotInProgressError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamMoveNotInProgress] } // NewJSStreamMsgDeleteFailedError creates a new JSStreamMsgDeleteFailedF error: "{err}" func NewJSStreamMsgDeleteFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamMsgDeleteFailedF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamNameContainsPathSeparatorsError creates a new JSStreamNameContainsPathSeparatorsErr error: "Stream name can not contain path separators" func NewJSStreamNameContainsPathSeparatorsError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamNameContainsPathSeparatorsErr] } // NewJSStreamNameExistError creates a new JSStreamNameExistErr error: "stream name already in use with a different configuration" func NewJSStreamNameExistError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamNameExistErr] } // NewJSStreamNameExistRestoreFailedError creates a new JSStreamNameExistRestoreFailedErr error: "stream name already in use, cannot restore" func NewJSStreamNameExistRestoreFailedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamNameExistRestoreFailedErr] } // NewJSStreamNotFoundError creates a new JSStreamNotFoundErr error: "stream not found" func NewJSStreamNotFoundError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamNotFoundErr] } // NewJSStreamNotMatchError creates a new JSStreamNotMatchErr error: "expected stream does not match" func NewJSStreamNotMatchError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamNotMatchErr] } // NewJSStreamOfflineError creates a new JSStreamOfflineErr error: "stream is offline" func NewJSStreamOfflineError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamOfflineErr] } // NewJSStreamPurgeFailedError creates a new JSStreamPurgeFailedF error: "{err}" func NewJSStreamPurgeFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamPurgeFailedF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamReplicasNotSupportedError creates a new JSStreamReplicasNotSupportedErr error: "replicas > 1 not supported in non-clustered mode" func NewJSStreamReplicasNotSupportedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamReplicasNotSupportedErr] } // NewJSStreamReplicasNotUpdatableError creates a new JSStreamReplicasNotUpdatableErr error: "Replicas configuration can not be updated" func NewJSStreamReplicasNotUpdatableError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamReplicasNotUpdatableErr] } // NewJSStreamRestoreError creates a new JSStreamRestoreErrF error: "restore failed: {err}" func NewJSStreamRestoreError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamRestoreErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamRollupFailedError creates a new JSStreamRollupFailedF error: "{err}" func NewJSStreamRollupFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamRollupFailedF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamSealedError creates a new JSStreamSealedErr error: "invalid operation on sealed stream" func NewJSStreamSealedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamSealedErr] } // NewJSStreamSequenceNotMatchError creates a new JSStreamSequenceNotMatchErr error: "expected stream sequence does not match" func NewJSStreamSequenceNotMatchError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamSequenceNotMatchErr] } // NewJSStreamSnapshotError creates a new JSStreamSnapshotErrF error: "snapshot failed: {err}" func NewJSStreamSnapshotError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamSnapshotErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamStoreFailedError creates a new JSStreamStoreFailedF error: "{err}" func NewJSStreamStoreFailedError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamStoreFailedF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamSubjectOverlapError creates a new JSStreamSubjectOverlapErr error: "subjects overlap with an existing stream" func NewJSStreamSubjectOverlapError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamSubjectOverlapErr] } // NewJSStreamTemplateCreateError creates a new JSStreamTemplateCreateErrF error: "{err}" func NewJSStreamTemplateCreateError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamTemplateCreateErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamTemplateDeleteError creates a new JSStreamTemplateDeleteErrF error: "{err}" func NewJSStreamTemplateDeleteError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamTemplateDeleteErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamTemplateNotFoundError creates a new JSStreamTemplateNotFoundErr error: "template not found" func NewJSStreamTemplateNotFoundError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSStreamTemplateNotFoundErr] } // NewJSStreamUpdateError creates a new JSStreamUpdateErrF error: "{err}" func NewJSStreamUpdateError(err error, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamUpdateErrF] args := e.toReplacerArgs([]interface{}{"{err}", err}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamWrongLastMsgIDError creates a new JSStreamWrongLastMsgIDErrF error: "wrong last msg ID: {id}" func NewJSStreamWrongLastMsgIDError(id interface{}, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamWrongLastMsgIDErrF] args := e.toReplacerArgs([]interface{}{"{id}", id}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSStreamWrongLastSequenceError creates a new JSStreamWrongLastSequenceErrF error: "wrong last sequence: {seq}" func NewJSStreamWrongLastSequenceError(seq uint64, opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } e := ApiErrors[JSStreamWrongLastSequenceErrF] args := e.toReplacerArgs([]interface{}{"{seq}", seq}) return &ApiError{ Code: e.Code, ErrCode: e.ErrCode, Description: strings.NewReplacer(args...).Replace(e.Description), } } // NewJSTempStorageFailedError creates a new JSTempStorageFailedErr error: "JetStream unable to open temp storage for restore" func NewJSTempStorageFailedError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSTempStorageFailedErr] } // NewJSTemplateNameNotMatchSubjectError creates a new JSTemplateNameNotMatchSubjectErr error: "template name in subject does not match request" func NewJSTemplateNameNotMatchSubjectError(opts ...ErrorOption) *ApiError { eopts := parseOpts(opts) if ae, ok := eopts.err.(*ApiError); ok { return ae } return ApiErrors[JSTemplateNameNotMatchSubjectErr] } nats-server-2.10.27/server/jetstream_errors_test.go000066400000000000000000000053731477524627100224510ustar00rootroot00000000000000package server import ( "errors" "testing" ) func TestIsNatsErr(t *testing.T) { if !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSNotEnabledForAccountErr) { t.Fatalf("Expected error match") } if IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr) { t.Fatalf("Expected error mismatch") } if IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSClusterNotAvailErr) { t.Fatalf("Expected error mismatch") } if !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSNotEnabledForAccountErr) { t.Fatalf("Expected error match") } if !IsNatsErr(&ApiError{ErrCode: 10039}, 1, JSClusterNotActiveErr, JSNotEnabledForAccountErr) { t.Fatalf("Expected error match") } if IsNatsErr(&ApiError{ErrCode: 10039}, 1, 2, JSClusterNotActiveErr) { t.Fatalf("Expected error mismatch") } if IsNatsErr(nil, JSClusterNotActiveErr) { t.Fatalf("Expected error mismatch") } if IsNatsErr(errors.New("x"), JSClusterNotActiveErr) { t.Fatalf("Expected error mismatch") } } func TestApiError_Error(t *testing.T) { if es := ApiErrors[JSClusterNotActiveErr].Error(); es != "JetStream not in clustered mode (10006)" { t.Fatalf("Expected 'JetStream not in clustered mode (10006)', got %q", es) } } func TestApiError_NewWithTags(t *testing.T) { ne := NewJSRestoreSubscribeFailedError(errors.New("failed error"), "the.subject") if ne.Description != "JetStream unable to subscribe to restore snapshot the.subject: failed error" { t.Fatalf("Expected 'JetStream unable to subscribe to restore snapshot the.subject: failed error' got %q", ne.Description) } if ne == ApiErrors[JSRestoreSubscribeFailedErrF] { t.Fatalf("Expected a new instance") } } func TestApiError_NewWithUnless(t *testing.T) { if ne := NewJSStreamRestoreError(errors.New("failed error"), Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) { t.Fatalf("Expected JSNotEnabledForAccountErr got %s", ne) } if ne := NewJSStreamRestoreError(errors.New("failed error")); !IsNatsErr(ne, JSStreamRestoreErrF) { t.Fatalf("Expected JSStreamRestoreErrF got %s", ne) } if ne := NewJSStreamRestoreError(errors.New("failed error"), Unless(errors.New("other error"))); !IsNatsErr(ne, JSStreamRestoreErrF) { t.Fatalf("Expected JSStreamRestoreErrF got %s", ne) } if ne := NewJSPeerRemapError(Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) { t.Fatalf("Expected JSNotEnabledForAccountErr got %s", ne) } if ne := NewJSPeerRemapError(Unless(nil)); !IsNatsErr(ne, JSPeerRemapErr) { t.Fatalf("Expected JSPeerRemapErr got %s", ne) } if ne := NewJSPeerRemapError(Unless(errors.New("other error"))); !IsNatsErr(ne, JSPeerRemapErr) { t.Fatalf("Expected JSPeerRemapErr got %s", ne) } } nats-server-2.10.27/server/jetstream_events.go000066400000000000000000000271501477524627100213770ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "encoding/json" "time" ) // publishAdvisory sends the given advisory into the account. Returns true if // it was sent, false if not (i.e. due to lack of interest or a marshal error). func (s *Server) publishAdvisory(acc *Account, subject string, adv any) bool { if acc == nil { acc = s.SystemAccount() if acc == nil { return false } } // If there is no one listening for this advisory then save ourselves the effort // and don't bother encoding the JSON or sending it. if sl := acc.sl; (sl != nil && !sl.HasInterest(subject)) && !s.hasGatewayInterest(acc.Name, subject) { return false } ej, err := json.Marshal(adv) if err == nil { err = s.sendInternalAccountMsg(acc, subject, ej) if err != nil { s.Warnf("Advisory could not be sent for account %q: %v", acc.Name, err) } } else { s.Warnf("Advisory could not be serialized for account %q: %v", acc.Name, err) } return err == nil } // JSAPIAudit is an advisory about administrative actions taken on JetStream type JSAPIAudit struct { TypedEvent Server string `json:"server"` Client *ClientInfo `json:"client"` Subject string `json:"subject"` Request string `json:"request,omitempty"` Response string `json:"response"` Domain string `json:"domain,omitempty"` } const JSAPIAuditType = "io.nats.jetstream.advisory.v1.api_audit" // ActionAdvisoryType indicates which action against a stream, consumer or template triggered an advisory type ActionAdvisoryType string const ( CreateEvent ActionAdvisoryType = "create" DeleteEvent ActionAdvisoryType = "delete" ModifyEvent ActionAdvisoryType = "modify" ) // JSStreamActionAdvisory indicates that a stream was created, edited or deleted type JSStreamActionAdvisory struct { TypedEvent Stream string `json:"stream"` Action ActionAdvisoryType `json:"action"` Template string `json:"template,omitempty"` Domain string `json:"domain,omitempty"` } const JSStreamActionAdvisoryType = "io.nats.jetstream.advisory.v1.stream_action" // JSConsumerActionAdvisory indicates that a consumer was created or deleted type JSConsumerActionAdvisory struct { TypedEvent Stream string `json:"stream"` Consumer string `json:"consumer"` Action ActionAdvisoryType `json:"action"` Domain string `json:"domain,omitempty"` } const JSConsumerActionAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_action" // JSConsumerAckMetric is a metric published when a user acknowledges a message, the // number of these that will be published is dependent on SampleFrequency type JSConsumerAckMetric struct { TypedEvent Stream string `json:"stream"` Consumer string `json:"consumer"` ConsumerSeq uint64 `json:"consumer_seq"` StreamSeq uint64 `json:"stream_seq"` Delay int64 `json:"ack_time"` Deliveries uint64 `json:"deliveries"` Domain string `json:"domain,omitempty"` } // JSConsumerAckMetricType is the schema type for JSConsumerAckMetricType const JSConsumerAckMetricType = "io.nats.jetstream.metric.v1.consumer_ack" // JSConsumerDeliveryExceededAdvisory is an advisory informing that a message hit // its MaxDeliver threshold and so might be a candidate for DLQ handling type JSConsumerDeliveryExceededAdvisory struct { TypedEvent Stream string `json:"stream"` Consumer string `json:"consumer"` StreamSeq uint64 `json:"stream_seq"` Deliveries uint64 `json:"deliveries"` Domain string `json:"domain,omitempty"` } // JSConsumerDeliveryExceededAdvisoryType is the schema type for JSConsumerDeliveryExceededAdvisory const JSConsumerDeliveryExceededAdvisoryType = "io.nats.jetstream.advisory.v1.max_deliver" // JSConsumerDeliveryNakAdvisory is an advisory informing that a message was // naked by the consumer type JSConsumerDeliveryNakAdvisory struct { TypedEvent Stream string `json:"stream"` Consumer string `json:"consumer"` ConsumerSeq uint64 `json:"consumer_seq"` StreamSeq uint64 `json:"stream_seq"` Deliveries uint64 `json:"deliveries"` Domain string `json:"domain,omitempty"` } // JSConsumerDeliveryNakAdvisoryType is the schema type for JSConsumerDeliveryNakAdvisory const JSConsumerDeliveryNakAdvisoryType = "io.nats.jetstream.advisory.v1.nak" // JSConsumerDeliveryTerminatedAdvisory is an advisory informing that a message was // terminated by the consumer, so might be a candidate for DLQ handling type JSConsumerDeliveryTerminatedAdvisory struct { TypedEvent Stream string `json:"stream"` Consumer string `json:"consumer"` ConsumerSeq uint64 `json:"consumer_seq"` StreamSeq uint64 `json:"stream_seq"` Deliveries uint64 `json:"deliveries"` Reason string `json:"reason,omitempty"` Domain string `json:"domain,omitempty"` } // JSConsumerDeliveryTerminatedAdvisoryType is the schema type for JSConsumerDeliveryTerminatedAdvisory const JSConsumerDeliveryTerminatedAdvisoryType = "io.nats.jetstream.advisory.v1.terminated" // JSSnapshotCreateAdvisory is an advisory sent after a snapshot is successfully started type JSSnapshotCreateAdvisory struct { TypedEvent Stream string `json:"stream"` State StreamState `json:"state"` Client *ClientInfo `json:"client"` Domain string `json:"domain,omitempty"` } // JSSnapshotCreatedAdvisoryType is the schema type for JSSnapshotCreateAdvisory const JSSnapshotCreatedAdvisoryType = "io.nats.jetstream.advisory.v1.snapshot_create" // JSSnapshotCompleteAdvisory is an advisory sent after a snapshot is successfully started type JSSnapshotCompleteAdvisory struct { TypedEvent Stream string `json:"stream"` Start time.Time `json:"start"` End time.Time `json:"end"` Client *ClientInfo `json:"client"` Domain string `json:"domain,omitempty"` } // JSSnapshotCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory const JSSnapshotCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.snapshot_complete" // JSRestoreCreateAdvisory is an advisory sent after a snapshot is successfully started type JSRestoreCreateAdvisory struct { TypedEvent Stream string `json:"stream"` Client *ClientInfo `json:"client"` Domain string `json:"domain,omitempty"` } // JSRestoreCreateAdvisoryType is the schema type for JSSnapshotCreateAdvisory const JSRestoreCreateAdvisoryType = "io.nats.jetstream.advisory.v1.restore_create" // JSRestoreCompleteAdvisory is an advisory sent after a snapshot is successfully started type JSRestoreCompleteAdvisory struct { TypedEvent Stream string `json:"stream"` Start time.Time `json:"start"` End time.Time `json:"end"` Bytes int64 `json:"bytes"` Client *ClientInfo `json:"client"` Domain string `json:"domain,omitempty"` } // JSRestoreCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory const JSRestoreCompleteAdvisoryType = "io.nats.jetstream.advisory.v1.restore_complete" // Clustering specific. // JSClusterLeaderElectedAdvisoryType is sent when the system elects a new meta leader. const JSDomainLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.domain_leader_elected" // JSClusterLeaderElectedAdvisory indicates that a domain has elected a new leader. type JSDomainLeaderElectedAdvisory struct { TypedEvent Leader string `json:"leader"` Replicas []*PeerInfo `json:"replicas"` Cluster string `json:"cluster"` Domain string `json:"domain,omitempty"` } // JSStreamLeaderElectedAdvisoryType is sent when the system elects a new leader for a stream. const JSStreamLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.stream_leader_elected" // JSStreamLeaderElectedAdvisory indicates that a stream has elected a new leader. type JSStreamLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` Stream string `json:"stream"` Leader string `json:"leader"` Replicas []*PeerInfo `json:"replicas"` Domain string `json:"domain,omitempty"` } // JSStreamQuorumLostAdvisoryType is sent when the system detects a clustered stream and // its consumers are stalled and unable to make progress. const JSStreamQuorumLostAdvisoryType = "io.nats.jetstream.advisory.v1.stream_quorum_lost" // JSStreamQuorumLostAdvisory indicates that a stream has lost quorum and is stalled. type JSStreamQuorumLostAdvisory struct { TypedEvent Account string `json:"account,omitempty"` Stream string `json:"stream"` Replicas []*PeerInfo `json:"replicas"` Domain string `json:"domain,omitempty"` } // JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer. const JSConsumerLeaderElectedAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_leader_elected" // JSConsumerLeaderElectedAdvisory indicates that a consumer has elected a new leader. type JSConsumerLeaderElectedAdvisory struct { TypedEvent Account string `json:"account,omitempty"` Stream string `json:"stream"` Consumer string `json:"consumer"` Leader string `json:"leader"` Replicas []*PeerInfo `json:"replicas"` Domain string `json:"domain,omitempty"` } // JSConsumerQuorumLostAdvisoryType is sent when the system detects a clustered consumer and // is stalled and unable to make progress. const JSConsumerQuorumLostAdvisoryType = "io.nats.jetstream.advisory.v1.consumer_quorum_lost" // JSConsumerQuorumLostAdvisory indicates that a consumer has lost quorum and is stalled. type JSConsumerQuorumLostAdvisory struct { TypedEvent Account string `json:"account,omitempty"` Stream string `json:"stream"` Consumer string `json:"consumer"` Replicas []*PeerInfo `json:"replicas"` Domain string `json:"domain,omitempty"` } // JSServerOutOfStorageAdvisoryType is sent when the server is out of storage space. const JSServerOutOfStorageAdvisoryType = "io.nats.jetstream.advisory.v1.server_out_of_space" // JSServerOutOfSpaceAdvisory indicates that a stream has lost quorum and is stalled. type JSServerOutOfSpaceAdvisory struct { TypedEvent Server string `json:"server"` ServerID string `json:"server_id"` Stream string `json:"stream,omitempty"` Cluster string `json:"cluster"` Domain string `json:"domain,omitempty"` } // JSServerRemovedAdvisoryType is sent when the server has been removed and JS disabled. const JSServerRemovedAdvisoryType = "io.nats.jetstream.advisory.v1.server_removed" // JSServerRemovedAdvisory indicates that a stream has lost quorum and is stalled. type JSServerRemovedAdvisory struct { TypedEvent Server string `json:"server"` ServerID string `json:"server_id"` Cluster string `json:"cluster"` Domain string `json:"domain,omitempty"` } // JSAPILimitReachedAdvisoryType is sent when the JS API request queue limit is reached. const JSAPILimitReachedAdvisoryType = "io.nats.jetstream.advisory.v1.api_limit_reached" // JSAPILimitReachedAdvisory is a advisory published when JetStream hits the queue length limit. type JSAPILimitReachedAdvisory struct { TypedEvent Server string `json:"server"` // Server that created the event, name or ID Domain string `json:"domain,omitempty"` // Domain the server belongs to Dropped int64 `json:"dropped"` // How many messages did we drop from the queue } nats-server-2.10.27/server/jetstream_helpers_test.go000066400000000000000000001454371477524627100226050ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Do not exlude this file with the !skip_js_tests since those helpers // are also used by MQTT. package server import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "math/rand" "net" "net/url" "os" "path" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "golang.org/x/time/rate" ) // Support functions func init() { // Speed up raft for tests. hbInterval = 50 * time.Millisecond minElectionTimeout = 1500 * time.Millisecond maxElectionTimeout = 3500 * time.Millisecond lostQuorumInterval = 2 * time.Second lostQuorumCheck = 4 * hbInterval // For statz and jetstream placement speedups as well. statszRateLimit = 0 } // Used to setup clusters of clusters for tests. type cluster struct { servers []*Server opts []*Options name string t testing.TB nproxies []*netProxy } // Used to setup superclusters for tests. type supercluster struct { t *testing.T clusters []*cluster nproxies []*netProxy } func (sc *supercluster) shutdown() { if sc == nil { return } for _, np := range sc.nproxies { np.stop() } for _, c := range sc.clusters { shutdownCluster(c) } } func (sc *supercluster) randomServer() *Server { return sc.randomCluster().randomServer() } func (sc *supercluster) serverByName(sname string) *Server { for _, c := range sc.clusters { if s := c.serverByName(sname); s != nil { return s } } return nil } func (sc *supercluster) waitOnStreamLeader(account, stream string) { sc.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { for _, c := range sc.clusters { if leader := c.streamLeader(account, stream); leader != nil { time.Sleep(200 * time.Millisecond) return } } time.Sleep(100 * time.Millisecond) } sc.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) } var jsClusterAccountsTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } websocket { listen: 127.0.0.1:-1 compression: true handshake_timeout: "5s" no_tls: true } no_auth_user: one accounts { ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } TWO { users = [ { user: "two", pass: "p" } ]; jetstream: enabled } NOJS { users = [ { user: "nojs", pass: "p" } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterEncryptedTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', key: "s3cr3t!"} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterMaxBytesTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: u accounts { $U { users = [ { user: "u", pass: "p" } ] jetstream: { max_mem: 128MB max_file: 18GB max_bytes: true // Forces streams to indicate max_bytes. } } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterMaxBytesAccountLimitTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 4GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: u accounts { $U { users = [ { user: "u", pass: "p" } ] jetstream: { max_mem: 128MB max_file: 3GB max_bytes: true // Forces streams to indicate max_bytes. } } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsSuperClusterTempl = ` %s gateway { name: %s listen: 127.0.0.1:%d gateways = [%s ] } system_account: "$SYS" ` var jsClusterLimitsTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: u accounts { ONE { users = [ { user: "u", pass: "s3cr3t!" } ] jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsMixedModeGlobalAccountTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts {$SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsGWTempl = `%s{name: %s, urls: [%s]}` var jsClusterAccountLimitsTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: js accounts { $JS { users = [ { user: "js", pass: "p" } ]; jetstream: {max_store: 1MB, max_mem: 0} } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` func createJetStreamTaggedSuperCluster(t *testing.T) *supercluster { return createJetStreamTaggedSuperClusterWithGWProxy(t, nil) } func createJetStreamTaggedSuperClusterWithGWProxy(t *testing.T, gwm gwProxyMap) *supercluster { sc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 3, 3, nil, gwm) sc.waitOnPeerCount(9) reset := func(s *Server) { s.mu.Lock() rch := s.sys.resetCh s.mu.Unlock() if rch != nil { rch <- struct{}{} } s.sendStatszUpdate() } // Make first cluster AWS, US country code. for _, s := range sc.clusterForName("C1").servers { s.optsMu.Lock() s.opts.Tags.Add("cloud:aws") s.opts.Tags.Add("country:us") s.optsMu.Unlock() reset(s) } // Make second cluster GCP, UK country code. for _, s := range sc.clusterForName("C2").servers { s.optsMu.Lock() s.opts.Tags.Add("cloud:gcp") s.opts.Tags.Add("country:uk") s.optsMu.Unlock() reset(s) } // Make third cluster AZ, JP country code. for _, s := range sc.clusterForName("C3").servers { s.optsMu.Lock() s.opts.Tags.Add("cloud:az") s.opts.Tags.Add("country:jp") s.optsMu.Unlock() reset(s) } ml := sc.leader() js := ml.getJetStream() require_True(t, js != nil) js.mu.RLock() defer js.mu.RUnlock() cc := js.cluster require_True(t, cc != nil) // Walk and make sure all tags are registered. expires := time.Now().Add(10 * time.Second) for time.Now().Before(expires) { allOK := true for _, p := range cc.meta.Peers() { si, ok := ml.nodeToInfo.Load(p.ID) require_True(t, ok) ni := si.(nodeInfo) if len(ni.tags) == 0 { allOK = false reset(sc.serverByName(ni.name)) } } if allOK { break } } return sc } func createJetStreamSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { return createJetStreamSuperClusterWithTemplate(t, jsClusterTempl, numServersPer, numClusters) } func createJetStreamSuperClusterWithTemplate(t *testing.T, tmpl string, numServersPer, numClusters int) *supercluster { return createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, numServersPer, numClusters, nil, nil) } // For doing proxyies in GWs. type gwProxy struct { rtt time.Duration up int down int } // For use in normal clusters. type clusterProxy = gwProxy // Maps cluster names to proxy settings. type gwProxyMap map[string]*gwProxy func createJetStreamSuperClusterWithTemplateAndModHook(t *testing.T, tmpl string, numServersPer, numClusters int, modify modifyCb, gwm gwProxyMap) *supercluster { t.Helper() if numServersPer < 1 { t.Fatalf("Number of servers must be >= 1") } if numClusters <= 1 { t.Fatalf("Number of clusters must be > 1") } startClusterPorts := []int{20_022, 22_022, 24_022} startGatewayPorts := []int{20_122, 22_122, 24_122} startClusterPort := startClusterPorts[rand.Intn(len(startClusterPorts))] startGWPort := startGatewayPorts[rand.Intn(len(startGatewayPorts))] // Make the GWs form faster for the tests. SetGatewaysSolicitDelay(10 * time.Millisecond) defer ResetGatewaysSolicitDelay() cp, gp := startClusterPort, startGWPort var clusters []*cluster var nproxies []*netProxy var gws []string // Build GWs first, will be same for all servers. for i, port := 1, gp; i <= numClusters; i++ { cn := fmt.Sprintf("C%d", i) var gwp *gwProxy if len(gwm) > 0 { gwp = gwm[cn] } var urls []string for n := 0; n < numServersPer; n++ { routeURL := fmt.Sprintf("nats-route://127.0.0.1:%d", port) if gwp != nil { np := createNetProxy(gwp.rtt, gwp.up, gwp.down, routeURL, false) nproxies = append(nproxies, np) routeURL = np.routeURL() } urls = append(urls, routeURL) port++ } gws = append(gws, fmt.Sprintf(jsGWTempl, "\n\t\t\t", cn, strings.Join(urls, ","))) } gwconf := strings.Join(gws, _EMPTY_) for i := 1; i <= numClusters; i++ { cn := fmt.Sprintf("C%d", i) // Go ahead and build configurations. c := &cluster{servers: make([]*Server, 0, numServersPer), opts: make([]*Options, 0, numServersPer), name: cn} // Build out the routes that will be shared with all configs. var routes []string for port := cp; port < cp+numServersPer; port++ { routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) } routeConfig := strings.Join(routes, ",") for si := 0; si < numServersPer; si++ { storeDir := t.TempDir() sn := fmt.Sprintf("%s-S%d", cn, si+1) bconf := fmt.Sprintf(tmpl, sn, storeDir, cn, cp+si, routeConfig) conf := fmt.Sprintf(jsSuperClusterTempl, bconf, cn, gp, gwconf) gp++ if modify != nil { conf = modify(sn, cn, storeDir, conf) } s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) } checkClusterFormed(t, c.servers...) clusters = append(clusters, c) cp += numServersPer c.t = t } // Start any proxies. for _, np := range nproxies { np.start() } // Wait for the supercluster to be formed. egws := numClusters - 1 for _, c := range clusters { for _, s := range c.servers { waitForOutboundGateways(t, s, egws, 10*time.Second) } } sc := &supercluster{t, clusters, nproxies} sc.waitOnLeader() sc.waitOnAllCurrent() // Wait for all the peer nodes to be registered. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { var peers []string if ml := sc.leader(); ml != nil { peers = ml.ActivePeers() if len(peers) == numClusters*numServersPer { return nil } } return fmt.Errorf("Not correct number of peers, expected %d, got %d", numClusters*numServersPer, len(peers)) }) if sc.leader() == nil { sc.t.Fatalf("Expected a cluster leader, got none") } return sc } func (sc *supercluster) createLeafNodes(clusterName string, numServers int) *cluster { sc.t.Helper() // Create our leafnode cluster template first. return sc.createLeafNodesWithDomain(clusterName, numServers, "") } func (sc *supercluster) createLeafNodesWithDomain(clusterName string, numServers int, domain string) *cluster { sc.t.Helper() // Create our leafnode cluster template first. return sc.randomCluster().createLeafNodes(clusterName, numServers, domain) } func (sc *supercluster) createSingleLeafNode(extend bool) *Server { sc.t.Helper() return sc.randomCluster().createLeafNode(extend) } func (sc *supercluster) leader() *Server { for _, c := range sc.clusters { if leader := c.leader(); leader != nil { return leader } } return nil } func (sc *supercluster) waitOnLeader() { sc.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { for _, c := range sc.clusters { if leader := c.leader(); leader != nil { time.Sleep(250 * time.Millisecond) return } } time.Sleep(25 * time.Millisecond) } sc.t.Fatalf("Expected a cluster leader, got none") } func (sc *supercluster) waitOnAccount(account string) { sc.t.Helper() expires := time.Now().Add(40 * time.Second) for time.Now().Before(expires) { found := true for _, c := range sc.clusters { for _, s := range c.servers { acc, err := s.fetchAccount(account) found = found && err == nil && acc != nil } } if found { return } time.Sleep(100 * time.Millisecond) continue } sc.t.Fatalf("Expected account %q to exist but didn't", account) } func (sc *supercluster) waitOnAllCurrent() { sc.t.Helper() for _, c := range sc.clusters { c.waitOnAllCurrent() } } func (sc *supercluster) clusterForName(name string) *cluster { for _, c := range sc.clusters { if c.name == name { return c } } return nil } func (sc *supercluster) randomCluster() *cluster { clusters := append(sc.clusters[:0:0], sc.clusters...) rand.Shuffle(len(clusters), func(i, j int) { clusters[i], clusters[j] = clusters[j], clusters[i] }) return clusters[0] } func (sc *supercluster) waitOnPeerCount(n int) { sc.t.Helper() sc.waitOnLeader() leader := sc.leader() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { peers := leader.JetStreamClusterPeers() if len(peers) == n { return } time.Sleep(100 * time.Millisecond) } sc.t.Fatalf("Expected a super cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) } var jsClusterMirrorSourceImportsTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: dlc accounts { JS { jetstream: enabled users = [ { user: "rip", pass: "pass" } ] exports [ { service: "$JS.API.CONSUMER.>" } # To create internal consumers to mirror/source. { stream: "RI.DELIVER.SYNC.>" } # For the mirror/source consumers sending to IA via delivery subject. { service: "$JS.FC.>" } ] } IA { jetstream: enabled users = [ { user: "dlc", pass: "pass" } ] imports [ { service: { account: JS, subject: "$JS.API.CONSUMER.>"}, to: "RI.JS.API.CONSUMER.>" } { stream: { account: JS, subject: "RI.DELIVER.SYNC.>"} } { service: {account: JS, subject: "$JS.FC.>" }} ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterImportsTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: dlc accounts { JS { jetstream: enabled users = [ { user: "rip", pass: "pass" } ] exports [ { service: "$JS.API.>", response: stream } { service: "TEST" } # For publishing to the stream. { service: "$JS.ACK.TEST.*.>" } ] } IA { users = [ { user: "dlc", pass: "pass" } ] imports [ { service: { subject: "$JS.API.>", account: JS }} { service: { subject: "TEST", account: JS }} { service: { subject: "$JS.ACK.TEST.*.>", account: JS }} ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` func createMixedModeCluster(t testing.TB, tmpl string, clusterName, snPre string, numJsServers, numNonServers int, doJSConfig bool) *cluster { t.Helper() if clusterName == _EMPTY_ || numJsServers < 0 || numNonServers < 1 { t.Fatalf("Bad params") } numServers := numJsServers + numNonServers const startClusterPort = 23232 // Build out the routes that will be shared with all configs. var routes []string for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) } routeConfig := strings.Join(routes, ",") // Go ahead and build configurations and start servers. c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { storeDir := t.TempDir() sn := fmt.Sprintf("%sS-%d", snPre, cp-startClusterPort+1) conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) // Disable JS here. if cp-startClusterPort >= numJsServers { // We can disable by commmenting it out, meaning no JS config, or can set the config up and just set disabled. // e.g. jetstream: {domain: "SPOKE", enabled: false} if doJSConfig { conf = strings.Replace(conf, "jetstream: {", "jetstream: { enabled: false, ", 1) } else { conf = strings.Replace(conf, "jetstream: ", "# jetstream: ", 1) } } s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) } c.t = t // Wait til we are formed and have a leader. c.checkClusterFormed() if numJsServers > 0 { c.waitOnPeerCount(numJsServers) } return c } // This will create a cluster that is explicitly configured for the routes, etc. // and also has a defined clustername. All configs for routes and cluster name will be the same. func createJetStreamClusterExplicit(t testing.TB, clusterName string, numServers int) *cluster { return createJetStreamClusterWithTemplate(t, jsClusterTempl, clusterName, numServers) } func createJetStreamClusterWithTemplate(t testing.TB, tmpl string, clusterName string, numServers int) *cluster { return createJetStreamClusterWithTemplateAndModHook(t, tmpl, clusterName, numServers, nil) } func createJetStreamClusterWithTemplateAndModHook(t testing.TB, tmpl string, clusterName string, numServers int, modify modifyCb) *cluster { startPorts := []int{7_022, 9_022, 11_022, 15_022} port := startPorts[rand.Intn(len(startPorts))] return createJetStreamClusterAndModHook(t, tmpl, clusterName, _EMPTY_, numServers, port, true, modify) } func createJetStreamCluster(t testing.TB, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool) *cluster { return createJetStreamClusterAndModHook(t, tmpl, clusterName, snPre, numServers, portStart, waitOnReady, nil) } type modifyCb func(serverName, clusterName, storeDir, conf string) string func createJetStreamClusterAndModHook(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, waitOnReady bool, modify modifyCb) *cluster { return createJetStreamClusterEx(t, tmpl, cName, snPre, numServers, portStart, waitOnReady, modify, nil) } func createJetStreamClusterWithNetProxy(t testing.TB, cName string, numServers int, cnp *clusterProxy) *cluster { startPorts := []int{7_122, 9_122, 11_122, 15_122} port := startPorts[rand.Intn(len(startPorts))] return createJetStreamClusterEx(t, jsClusterTempl, cName, _EMPTY_, numServers, port, true, nil, cnp) } func createJetStreamClusterEx(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, wait bool, modify modifyCb, cnp *clusterProxy) *cluster { t.Helper() if cName == _EMPTY_ || numServers < 1 { t.Fatalf("Bad params") } // Flaky test prevention: // Binding a socket to IP stack port 0 will bind an ephemeral port from an OS-specific range. // If someone passes in to us a port spec which would cover that range, the test would be flaky. // Adjust these ports to be the most inclusive across the port runner OSes. // Linux: /proc/sys/net/ipv4/ip_local_port_range : 32768:60999 // is useful, and shows there's no safe available range without OS-specific tuning. // Our tests are usually run on Linux. Folks who care about other OSes: if you can't tune your test-runner OS to match, please // propose a viable alternative. const prohibitedPortFirst = 32768 const prohibitedPortLast = 60999 if (portStart >= prohibitedPortFirst && portStart <= prohibitedPortLast) || (portStart+numServers-1 >= prohibitedPortFirst && portStart+numServers-1 <= prohibitedPortLast) { t.Fatalf("test setup failure: may not specify a cluster port range which falls within %d:%d", prohibitedPortFirst, prohibitedPortLast) } // Build out the routes that will be shared with all configs. var routes []string var nproxies []*netProxy for cp := portStart; cp < portStart+numServers; cp++ { routeURL := fmt.Sprintf("nats-route://127.0.0.1:%d", cp) if cnp != nil { np := createNetProxy(cnp.rtt, cnp.up, cnp.down, routeURL, false) nproxies = append(nproxies, np) routeURL = np.routeURL() } routes = append(routes, routeURL) } routeConfig := strings.Join(routes, ",") // Go ahead and build configurations and start servers. c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: cName, nproxies: nproxies} // Start any proxies. for _, np := range nproxies { np.start() } for cp := portStart; cp < portStart+numServers; cp++ { storeDir := t.TempDir() sn := fmt.Sprintf("%sS-%d", snPre, cp-portStart+1) conf := fmt.Sprintf(tmpl, sn, storeDir, cName, cp, routeConfig) if modify != nil { conf = modify(sn, cName, storeDir, conf) } s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) } c.t = t // Wait til we are formed and have a leader. c.checkClusterFormed() if wait { c.waitOnClusterReady() } return c } func (c *cluster) addInNewServer() *Server { c.t.Helper() sn := fmt.Sprintf("S-%d", len(c.servers)+1) storeDir := c.t.TempDir() seedRoute := fmt.Sprintf("nats-route://127.0.0.1:%d", c.opts[0].Cluster.Port) conf := fmt.Sprintf(jsClusterTempl, sn, storeDir, c.name, -1, seedRoute) s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) c.checkClusterFormed() return s } // This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. func (c *cluster) createSingleLeafNodeNoSystemAccount() *Server { as := c.randomServer() lno := as.getOpts().LeafNode ln1 := fmt.Sprintf("nats://one:p@%s:%d", lno.Host, lno.Port) ln2 := fmt.Sprintf("nats://two:p@%s:%d", lno.Host, lno.Port) conf := fmt.Sprintf(jsClusterSingleLeafNodeTempl, c.t.TempDir(), ln1, ln2) s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) checkLeafNodeConnectedCount(c.t, as, 2) return s } // This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStream() *Server { return c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(_EMPTY_, "nojs") } func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(domain, user string) *Server { tmpl := jsClusterSingleLeafNodeLikeNGSTempl if domain != _EMPTY_ { nsc := fmt.Sprintf("domain: %s, store_dir:", domain) tmpl = strings.Replace(jsClusterSingleLeafNodeLikeNGSTempl, "store_dir:", nsc, 1) } as := c.randomServer() lno := as.getOpts().LeafNode ln := fmt.Sprintf("nats://%s:p@%s:%d", user, lno.Host, lno.Port) conf := fmt.Sprintf(tmpl, c.t.TempDir(), ln) s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) checkLeafNodeConnectedCount(c.t, as, 1) return s } var jsClusterSingleLeafNodeLikeNGSTempl = ` listen: 127.0.0.1:-1 server_name: LNJS jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { remotes [ { urls: [ %s ] } ] } ` var jsClusterSingleLeafNodeTempl = ` listen: 127.0.0.1:-1 server_name: LNJS jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { remotes [ { urls: [ %s ], account: "JSY" } { urls: [ %s ], account: "JSN" } ] } accounts { JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } JSN { users = [ { user: "n", pass: "p" } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTemplWithLeafNode = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTemplWithLeafNodeNoJS = ` listen: 127.0.0.1:-1 server_name: %s # Need to keep below since it fills in the store dir by default so just comment out. # jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTemplWithSingleLeafNode = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTemplWithSingleFleetLeafNode = ` listen: 127.0.0.1:-1 server_name: %s cluster: { name: fleet } jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsClusterTemplWithSingleLeafNodeNoJS = ` listen: 127.0.0.1:-1 server_name: %s # jetstream: {store_dir: '%s'} {{leaf}} # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` var jsLeafFrag = ` leaf { remotes [ { urls: [ %s ] } { urls: [ %s ], account: "$SYS" } ] } ` var jsLeafNoSysFrag = ` leaf { remotes [ { urls: [ %s ] } ] } ` func (c *cluster) createLeafNodes(clusterName string, numServers int, domain string) *cluster { c.t.Helper() return c.createLeafNodesWithStartPortAndDomain(clusterName, numServers, 22111, domain) } func (c *cluster) createLeafNodesNoJS(clusterName string, numServers int) *cluster { c.t.Helper() return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNodeNoJS, clusterName, numServers, 21333) } func (c *cluster) createLeafNodesWithStartPortAndDomain(clusterName string, numServers int, portStart int, domain string) *cluster { c.t.Helper() if domain == _EMPTY_ { return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNode, clusterName, numServers, portStart) } tmpl := strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, domain), 1) return c.createLeafNodesWithTemplateAndStartPort(tmpl, clusterName, numServers, portStart) } func (c *cluster) createLeafNode(extend bool) *Server { c.t.Helper() if extend { return c.createLeafNodeWithTemplate("LNS", strings.ReplaceAll(jsClusterTemplWithSingleLeafNode, "store_dir:", " extension_hint: will_extend, store_dir:")) } else { return c.createLeafNodeWithTemplate("LNS", jsClusterTemplWithSingleLeafNode) } } func (c *cluster) createLeafNodeWithTemplate(name, template string) *Server { c.t.Helper() tmpl := c.createLeafSolicit(template) conf := fmt.Sprintf(tmpl, name, c.t.TempDir()) s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) return s } func (c *cluster) createLeafNodeWithTemplateNoSystem(name, template string) *Server { return c.createLeafNodeWithTemplateNoSystemWithProto(name, template, "nats") } func (c *cluster) createLeafNodeWithTemplateNoSystemWithProto(name, template, proto string) *Server { c.t.Helper() tmpl := c.createLeafSolicitNoSystemWithProto(template, proto) conf := fmt.Sprintf(tmpl, name, c.t.TempDir()) s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) c.servers = append(c.servers, s) c.opts = append(c.opts, o) return s } // Helper to generate the leaf solicit configs. func (c *cluster) createLeafSolicit(tmpl string) string { return c.createLeafSolicitWithProto(tmpl, "nats") } func (c *cluster) createLeafSolicitWithProto(tmpl, proto string) string { c.t.Helper() // Create our leafnode cluster template first. var lns, lnss []string for _, s := range c.servers { if s.ClusterName() != c.name { continue } ln := s.getOpts().LeafNode lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) lnss = append(lnss, fmt.Sprintf("%s://admin:s3cr3t!@%s:%d", proto, ln.Host, ln.Port)) } lnc := strings.Join(lns, ", ") lnsc := strings.Join(lnss, ", ") lconf := fmt.Sprintf(jsLeafFrag, lnc, lnsc) return strings.Replace(tmpl, "{{leaf}}", lconf, 1) } func (c *cluster) createLeafSolicitNoSystemWithProto(tmpl, proto string) string { c.t.Helper() // Create our leafnode cluster template first. var lns []string for _, s := range c.servers { if s.ClusterName() != c.name { continue } switch proto { case "nats", "tls": ln := s.getOpts().LeafNode lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) case "ws", "wss": ln := s.getOpts().Websocket lns = append(lns, fmt.Sprintf("%s://%s:%d", proto, ln.Host, ln.Port)) } } lnc := strings.Join(lns, ", ") return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(jsLeafNoSysFrag, lnc), 1) } func (c *cluster) createLeafNodesWithTemplateMixedMode(template, clusterName string, numJsServers, numNonServers int, doJSConfig bool) *cluster { c.t.Helper() // Create our leafnode cluster template first. tmpl := c.createLeafSolicit(template) pre := clusterName + "-" lc := createMixedModeCluster(c.t, tmpl, clusterName, pre, numJsServers, numNonServers, doJSConfig) for _, s := range lc.servers { checkLeafNodeConnectedCount(c.t, s, 2) } lc.waitOnClusterReadyWithNumPeers(numJsServers) return lc } func (c *cluster) createLeafNodesWithTemplateAndStartPort(template, clusterName string, numServers int, portStart int) *cluster { c.t.Helper() // Create our leafnode cluster template first. tmpl := c.createLeafSolicit(template) pre := clusterName + "-" lc := createJetStreamCluster(c.t, tmpl, clusterName, pre, numServers, portStart, false) for _, s := range lc.servers { checkLeafNodeConnectedCount(c.t, s, 2) } return lc } // Helper function to close and disable leafnodes. func (s *Server) closeAndDisableLeafnodes() { var leafs []*client s.mu.Lock() for _, ln := range s.leafs { leafs = append(leafs, ln) } // Disable leafnodes for now. s.leafDisableConnect = true s.mu.Unlock() for _, ln := range leafs { ln.closeConnection(Revocation) } } // Helper function to re-enable leafnode connections. func (s *Server) reEnableLeafnodes() { s.mu.Lock() // Re-enable leafnodes. s.leafDisableConnect = false s.mu.Unlock() } // Helper to set the remote migrate feature. func (s *Server) setJetStreamMigrateOnRemoteLeaf() { s.mu.Lock() for _, cfg := range s.leafRemoteCfgs { cfg.JetStreamClusterMigrate = true } s.mu.Unlock() } // Will add in the mapping for the account to each server. func (c *cluster) addSubjectMapping(account, src, dest string) { c.t.Helper() for _, s := range c.servers { if s.ClusterName() != c.name { continue } acc, err := s.LookupAccount(account) if err != nil { c.t.Fatalf("Unexpected error on %v: %v", s, err) } if err := acc.AddMapping(src, dest); err != nil { c.t.Fatalf("Error adding mapping: %v", err) } } // Make sure interest propagates. time.Sleep(200 * time.Millisecond) } // Adjust limits for the given account. func (c *cluster) updateLimits(account string, newLimits map[string]JetStreamAccountLimits) { c.t.Helper() for _, s := range c.servers { acc, err := s.LookupAccount(account) if err != nil { c.t.Fatalf("Unexpected error: %v", err) } if err := acc.UpdateJetStreamLimits(newLimits); err != nil { c.t.Fatalf("Unexpected error: %v", err) } } } // Hack for staticcheck var skip = func(t *testing.T) { t.SkipNow() } func jsClientConnect(t testing.TB, s *Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { t.Helper() nc, err := nats.Connect(s.ClientURL(), opts...) if err != nil { t.Fatalf("Failed to create client: %v", err) } js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } return nc, js } func jsClientConnectEx(t testing.TB, s *Server, jsOpts []nats.JSOpt, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { t.Helper() nc, err := nats.Connect(s.ClientURL(), opts...) if err != nil { t.Fatalf("Failed to create client: %v", err) } jo := []nats.JSOpt{nats.MaxWait(10 * time.Second)} if len(jsOpts) > 0 { jo = append(jo, jsOpts...) } js, err := nc.JetStream(jo...) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } return nc, js } func jsClientConnectURL(t testing.TB, url string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { t.Helper() nc, err := nats.Connect(url, opts...) if err != nil { t.Fatalf("Failed to create client: %v", err) } js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } return nc, js } func checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) { t.Helper() checkFor(t, 10*time.Second, 20*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } func fetchMsgs(t *testing.T, sub *nats.Subscription, numExpected int, totalWait time.Duration) []*nats.Msg { t.Helper() result := make([]*nats.Msg, 0, numExpected) for start, count, wait := time.Now(), numExpected, totalWait; len(result) != numExpected; { msgs, err := sub.Fetch(count, nats.MaxWait(wait)) if err != nil { t.Fatal(err) } result = append(result, msgs...) count -= len(msgs) if wait = totalWait - time.Since(start); wait < 0 { break } } if len(result) != numExpected { t.Fatalf("Unexpected msg count, got %d, want %d", len(result), numExpected) } return result } func (c *cluster) restartServer(rs *Server) *Server { c.t.Helper() index := -1 var opts *Options for i, s := range c.servers { if s == rs { index = i break } } if index < 0 { c.t.Fatalf("Could not find server %v to restart", rs) } opts = c.opts[index] s, o := RunServerWithConfig(opts.ConfigFile) c.servers[index] = s c.opts[index] = o return s } func (c *cluster) checkClusterFormed() { c.t.Helper() checkClusterFormed(c.t, c.servers...) } func (c *cluster) waitOnPeerCount(n int) { c.t.Helper() c.waitOnLeader() leader := c.leader() for leader == nil { c.waitOnLeader() leader = c.leader() } expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { if peers := leader.JetStreamClusterPeers(); len(peers) == n { return } time.Sleep(100 * time.Millisecond) leader = c.leader() for leader == nil { c.waitOnLeader() leader = c.leader() } } c.t.Fatalf("Expected a cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) } func (c *cluster) waitOnConsumerLeader(account, stream, consumer string) { c.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { if leader := c.consumerLeader(account, stream, consumer); leader != nil { time.Sleep(200 * time.Millisecond) return } time.Sleep(100 * time.Millisecond) } c.t.Fatalf("Expected a consumer leader for %q %q %q, got none", account, stream, consumer) } func (c *cluster) consumerLeader(account, stream, consumer string) *Server { c.t.Helper() for _, s := range c.servers { if s.JetStreamIsConsumerLeader(account, stream, consumer) { return s } } return nil } func (c *cluster) randomNonConsumerLeader(account, stream, consumer string) *Server { c.t.Helper() for _, s := range c.servers { if !s.JetStreamIsConsumerLeader(account, stream, consumer) { return s } } return nil } func (c *cluster) waitOnStreamLeader(account, stream string) { c.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { if leader := c.streamLeader(account, stream); leader != nil { time.Sleep(200 * time.Millisecond) return } time.Sleep(100 * time.Millisecond) } c.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) } func (c *cluster) randomNonStreamLeader(account, stream string) *Server { c.t.Helper() for _, s := range c.servers { if s.JetStreamIsStreamAssigned(account, stream) && !s.JetStreamIsStreamLeader(account, stream) { return s } } return nil } func (c *cluster) streamLeader(account, stream string) *Server { c.t.Helper() for _, s := range c.servers { if s.JetStreamIsStreamLeader(account, stream) { return s } } return nil } func (c *cluster) waitOnStreamCurrent(s *Server, account, stream string) { c.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { if s.JetStreamIsStreamCurrent(account, stream) { time.Sleep(100 * time.Millisecond) return } time.Sleep(100 * time.Millisecond) } c.t.Fatalf("Expected server %q to eventually be current for stream %q", s, stream) } func (c *cluster) waitOnServerHealthz(s *Server) { c.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { hs := s.healthz(nil) if hs.Status == "ok" && hs.Error == _EMPTY_ { return } time.Sleep(100 * time.Millisecond) } c.t.Fatalf("Expected server %q to eventually return healthz 'ok', but got %q", s, s.healthz(nil).Error) } func (c *cluster) waitOnServerCurrent(s *Server) { c.t.Helper() expires := time.Now().Add(30 * time.Second) for time.Now().Before(expires) { time.Sleep(100 * time.Millisecond) if !s.JetStreamEnabled() || s.JetStreamIsCurrent() { return } } c.t.Fatalf("Expected server %q to eventually be current", s) } func (c *cluster) waitOnAllCurrent() { c.t.Helper() for _, cs := range c.servers { c.waitOnServerCurrent(cs) } } func (c *cluster) serverByName(sname string) *Server { for _, s := range c.servers { if s.Name() == sname { return s } } return nil } func (c *cluster) randomNonLeader() *Server { // range should randomize.. but.. for _, s := range c.servers { if s.Running() && !s.JetStreamIsLeader() { return s } } return nil } func (c *cluster) leader() *Server { for _, s := range c.servers { if s.JetStreamIsLeader() { return s } } return nil } func (c *cluster) expectNoLeader() { c.t.Helper() expires := time.Now().Add(maxElectionTimeout) for time.Now().Before(expires) { if c.leader() == nil { return } time.Sleep(20 * time.Millisecond) } c.t.Fatalf("Expected no leader but have one") } func (c *cluster) waitOnLeader() { c.t.Helper() expires := time.Now().Add(40 * time.Second) for time.Now().Before(expires) { if leader := c.leader(); leader != nil { time.Sleep(100 * time.Millisecond) return } time.Sleep(10 * time.Millisecond) } c.t.Fatalf("Expected a cluster leader, got none") } func (c *cluster) waitOnAccount(account string) { c.t.Helper() expires := time.Now().Add(40 * time.Second) for time.Now().Before(expires) { found := true for _, s := range c.servers { acc, err := s.fetchAccount(account) found = found && err == nil && acc != nil } if found { return } time.Sleep(100 * time.Millisecond) continue } c.t.Fatalf("Expected account %q to exist but didn't", account) } // Helper function to check that a cluster is formed func (c *cluster) waitOnClusterReady() { c.t.Helper() c.waitOnClusterReadyWithNumPeers(len(c.servers)) } func (c *cluster) waitOnClusterReadyWithNumPeers(numPeersExpected int) { c.t.Helper() var leader *Server expires := time.Now().Add(40 * time.Second) // Make sure we have all peers, and take into account the meta leader could still change. for time.Now().Before(expires) { if leader = c.leader(); leader == nil { time.Sleep(50 * time.Millisecond) continue } if len(leader.JetStreamClusterPeers()) == numPeersExpected { time.Sleep(100 * time.Millisecond) return } time.Sleep(10 * time.Millisecond) } if leader == nil { c.shutdown() c.t.Fatalf("Failed to elect a meta-leader") } peersSeen := len(leader.JetStreamClusterPeers()) c.shutdown() if leader == nil { c.t.Fatalf("Expected a cluster leader and fully formed cluster, no leader") } else { c.t.Fatalf("Expected a fully formed cluster, only %d of %d peers seen", peersSeen, numPeersExpected) } } // Helper function to remove JetStream from a server. func (c *cluster) removeJetStream(s *Server) { c.t.Helper() index := -1 for i, cs := range c.servers { if cs == s { index = i break } } cf := c.opts[index].ConfigFile cb, _ := os.ReadFile(cf) var sb strings.Builder for _, l := range strings.Split(string(cb), "\n") { if !strings.HasPrefix(strings.TrimSpace(l), "jetstream") { sb.WriteString(l + "\n") } } if err := os.WriteFile(cf, []byte(sb.String()), 0644); err != nil { c.t.Fatalf("Error writing updated config file: %v", err) } if err := s.Reload(); err != nil { c.t.Fatalf("Error on server reload: %v", err) } time.Sleep(100 * time.Millisecond) } func (c *cluster) stopAll() { c.t.Helper() for _, s := range c.servers { s.Shutdown() } } func (c *cluster) restartAll() { c.t.Helper() for i, s := range c.servers { if !s.Running() { opts := c.opts[i] s, o := RunServerWithConfig(opts.ConfigFile) c.servers[i] = s c.opts[i] = o } } c.waitOnClusterReady() } func (c *cluster) lameDuckRestartAll() { c.t.Helper() for i, s := range c.servers { s.lameDuckMode() s.WaitForShutdown() if !s.Running() { opts := c.opts[i] s, o := RunServerWithConfig(opts.ConfigFile) c.servers[i] = s c.opts[i] = o } } c.waitOnClusterReady() } func (c *cluster) restartAllSamePorts() { c.t.Helper() for i, s := range c.servers { if !s.Running() { opts := c.opts[i] s := RunServer(opts) c.servers[i] = s } } c.waitOnClusterReady() } func (c *cluster) totalSubs() (total int) { c.t.Helper() for _, s := range c.servers { total += int(s.NumSubscriptions()) } return total } func (c *cluster) stableTotalSubs() (total int) { nsubs := -1 checkFor(c.t, 2*time.Second, 250*time.Millisecond, func() error { subs := c.totalSubs() if subs == nsubs { return nil } nsubs = subs return fmt.Errorf("Still stabilizing") }) return nsubs } func addStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo { t.Helper() si, err := addStreamWithError(t, nc, cfg) if err != nil { t.Fatalf("Unexpected error: %+v", err) } return si } func addStreamWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfig) (*StreamInfo, *ApiError) { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, 5*time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Type != JSApiStreamCreateResponseType { t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamCreateResponseType) } return resp.StreamInfo, resp.Error } func updateStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Type != JSApiStreamUpdateResponseType { t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamUpdateResponseType) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } return resp.StreamInfo } // setInActiveDeleteThreshold sets the delete threshold for how long to wait // before deleting an inactive consumer. func (o *consumer) setInActiveDeleteThreshold(dthresh time.Duration) error { o.mu.Lock() defer o.mu.Unlock() deleteWasRunning := o.dtmr != nil stopAndClearTimer(&o.dtmr) // Do not add jitter if set via here. o.dthresh = dthresh if deleteWasRunning { o.dtmr = time.AfterFunc(o.dthresh, func() { o.deleteNotActive() }) } return nil } // Net Proxy - For introducing RTT and BW constraints. type netProxy struct { sync.RWMutex listener net.Listener conns []net.Conn rtt time.Duration up int down int url string surl string } func newNetProxy(rtt time.Duration, upRate, downRate int, serverURL string) *netProxy { return createNetProxy(rtt, upRate, downRate, serverURL, true) } func createNetProxy(rtt time.Duration, upRate, downRate int, serverURL string, start bool) *netProxy { hp := net.JoinHostPort("127.0.0.1", "0") l, e := net.Listen("tcp", hp) if e != nil { panic(fmt.Sprintf("Error listening on port: %s, %q", hp, e)) } port := l.Addr().(*net.TCPAddr).Port proxy := &netProxy{ listener: l, rtt: rtt, up: upRate, down: downRate, url: fmt.Sprintf("nats://127.0.0.1:%d", port), surl: serverURL, } if start { proxy.start() } return proxy } func (np *netProxy) start() { u, err := url.Parse(np.surl) if err != nil { panic(fmt.Sprintf("Could not parse server URL: %v", err)) } host := u.Host go func() { for { client, err := np.listener.Accept() if err != nil { return } server, err := net.DialTimeout("tcp", host, time.Second) if err != nil { continue } np.conns = append(np.conns, client, server) go np.loop(np.up, client, server) go np.loop(np.down, server, client) } }() } func (np *netProxy) clientURL() string { return np.url } func (np *netProxy) routeURL() string { return strings.Replace(np.url, "nats", "nats-route", 1) } func (np *netProxy) loop(tbw int, r, w net.Conn) { const rbl = 8192 var buf [rbl]byte ctx := context.Background() rl := rate.NewLimiter(rate.Limit(tbw), rbl) for { n, err := r.Read(buf[:]) if err != nil { w.Close() return } // RTT delays np.RLock() delay := np.rtt / 2 np.RUnlock() if delay > 0 { time.Sleep(delay) } if err := rl.WaitN(ctx, n); err != nil { return } if _, err = w.Write(buf[:n]); err != nil { r.Close() return } } } func (np *netProxy) updateRTT(rtt time.Duration) { np.Lock() np.rtt = rtt np.Unlock() } func (np *netProxy) stop() { if np.listener != nil { np.listener.Close() np.listener = nil for _, c := range np.conns { c.Close() } } } // Bitset, aka bitvector, allows tracking of large number of bits efficiently type bitset struct { // Bit map storage bitmap []uint8 // Number of bits currently set to 1 currentCount uint64 // Number of bits stored size uint64 } func NewBitset(size uint64) *bitset { byteSize := (size + 7) / 8 //Round up to the nearest byte return &bitset{ bitmap: make([]uint8, int(byteSize)), size: size, currentCount: 0, } } func (b *bitset) get(index uint64) bool { if index >= b.size { panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size)) } byteIndex := index / 8 bitIndex := uint(index % 8) bit := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) return bit != 0 } func (b *bitset) set(index uint64, value bool) { if index >= b.size { panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size)) } byteIndex := index / 8 bitIndex := uint(index % 8) byteMask := uint8(1) << bitIndex isSet := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) != 0 if value { b.bitmap[byteIndex] |= byteMask if !isSet { b.currentCount += 1 } } else { b.bitmap[byteIndex] &= ^byteMask if isSet { b.currentCount -= 1 } } } func (b *bitset) count() uint64 { return b.currentCount } func (b *bitset) String() string { const block = 8 // 8 bytes, 64 bits per line sb := strings.Builder{} sb.WriteString(fmt.Sprintf("Bits set: %d/%d\n", b.currentCount, b.size)) for i := 0; i < len(b.bitmap); i++ { if i%block == 0 { if i > 0 { sb.WriteString("\n") } sb.WriteString(fmt.Sprintf("[%4d] ", i*8)) } for j := uint8(0); j < 8; j++ { if b.bitmap[i]&(1< 0 { sb.WriteString("1") } else { sb.WriteString("0") } } } sb.WriteString("\n") return sb.String() } func copyDir(t *testing.T, dst, src string) error { t.Helper() srcFS := os.DirFS(src) return fs.WalkDir(srcFS, ".", func(p string, d os.DirEntry, err error) error { if err != nil { return err } newPath := path.Join(dst, p) if d.IsDir() { return os.MkdirAll(newPath, defaultDirPerms) } r, err := srcFS.Open(p) if err != nil { return err } defer r.Close() w, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, defaultFilePerms) if err != nil { return err } defer w.Close() _, err = io.Copy(w, r) return err }) } func getStreamDetails(t *testing.T, c *cluster, accountName, streamName string) *StreamDetail { t.Helper() srv := c.streamLeader(accountName, streamName) if srv == nil { return nil } jsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true}) require_NoError(t, err) for _, acc := range jsz.AccountDetails { if acc.Name == accountName { for _, stream := range acc.Streams { if stream.Name == streamName { return &stream } } } } t.Error("Could not find account details") return nil } func checkState(t *testing.T, c *cluster, accountName, streamName string) error { t.Helper() leaderSrv := c.streamLeader(accountName, streamName) if leaderSrv == nil { return fmt.Errorf("no leader server found for stream %q", streamName) } streamLeader := getStreamDetails(t, c, accountName, streamName) if streamLeader == nil { return fmt.Errorf("no leader found for stream %q", streamName) } var errs []error for _, srv := range c.servers { if srv == leaderSrv { // Skip self continue } acc, err := srv.LookupAccount(accountName) require_NoError(t, err) stream, err := acc.lookupStream(streamName) require_NoError(t, err) state := stream.state() if state.Msgs != streamLeader.State.Msgs { err := fmt.Errorf("[%s] Leader %v has %d messages, Follower %v has %d messages", streamName, leaderSrv, streamLeader.State.Msgs, srv, state.Msgs, ) errs = append(errs, err) } if state.FirstSeq != streamLeader.State.FirstSeq { err := fmt.Errorf("[%s] Leader %v FirstSeq is %d, Follower %v is at %d", streamName, leaderSrv, streamLeader.State.FirstSeq, srv, state.FirstSeq, ) errs = append(errs, err) } if state.LastSeq != streamLeader.State.LastSeq { err := fmt.Errorf("[%s] Leader %v LastSeq is %d, Follower %v is at %d", streamName, leaderSrv, streamLeader.State.LastSeq, srv, state.LastSeq, ) errs = append(errs, err) } if state.NumDeleted != streamLeader.State.NumDeleted { err := fmt.Errorf("[%s] Leader %v NumDeleted is %d, Follower %v is at %d\nSTATE_A: %+v\nSTATE_B: %+v\n", streamName, leaderSrv, streamLeader.State.NumDeleted, srv, state.NumDeleted, streamLeader.State, state, ) errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } nats-server-2.10.27/server/jetstream_jwt_test.go000066400000000000000000001504151477524627100217370ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests // +build !skip_js_tests package server import ( "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" jwt "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func TestJetStreamJWTLimits(t *testing.T) { updateJwt := func(url string, creds string, pubKey string, jwt string) { t.Helper() c := natsConnect(t, url, nats.UserCredentials(creds)) defer c.Close() if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { t.Fatal("error not expected in this test", err) } else { content := make(map[string]any) if err := json.Unmarshal(msg.Data, &content); err != nil { t.Fatalf("%v", err) } else if _, ok := content["data"]; !ok { t.Fatalf("did not get an ok response got: %v", content) } } } require_IdenticalLimits := func(infoLim JetStreamAccountLimits, lim jwt.JetStreamLimits) { t.Helper() if int64(infoLim.MaxConsumers) != lim.Consumer || int64(infoLim.MaxStreams) != lim.Streams || infoLim.MaxMemory != lim.MemoryStorage || infoLim.MaxStore != lim.DiskStorage { t.Fatalf("limits do not match %v != %v", infoLim, lim) } } expect_JSDisabledForAccount := func(c *nats.Conn) { t.Helper() if _, err := c.Request("$JS.API.INFO", nil, time.Second); err != nats.ErrTimeout && err != nats.ErrNoResponders { t.Fatalf("Unexpected error: %v", err) } } expect_InfoError := func(c *nats.Conn) { t.Helper() var info JSApiAccountInfoResponse if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if err = json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("response1 %v got error %v", string(resp.Data), err) } else if info.Error == nil { t.Fatalf("expected error") } } validate_limits := func(c *nats.Conn, expectedLimits jwt.JetStreamLimits) { t.Helper() var info JSApiAccountInfoResponse if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if err = json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("response1 %v got error %v", string(resp.Data), err) } else { require_IdenticalLimits(info.Limits, expectedLimits) } } // create system account sysKp, _ := nkeys.CreateAccount() sysPub, _ := sysKp.PublicKey() sysUKp, _ := nkeys.CreateUser() sysUSeed, _ := sysUKp.Seed() uclaim := newJWTTestUserClaims() uclaim.Subject, _ = sysUKp.PublicKey() sysUserJwt, err := uclaim.Encode(sysKp) require_NoError(t, err) sysKp.Seed() sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) // limits to apply and check limits1 := jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1, Consumer: 2, MaxBytesRequired: true} // has valid limits that would fail when incorrectly applied twice limits2 := jwt.JetStreamLimits{MemoryStorage: 4096 * 1024, DiskStorage: 8192 * 1024, Streams: 3, Consumer: 4} // limits exceeding actual configured value of DiskStorage limitsExceeded := jwt.JetStreamLimits{MemoryStorage: 8192 * 1024, DiskStorage: 16384 * 1024, Streams: 5, Consumer: 6} // create account using jetstream with both limits akp, _ := nkeys.CreateAccount() aPub, _ := akp.PublicKey() claim := jwt.NewAccountClaims(aPub) claim.Limits.JetStreamLimits = limits1 aJwt1, err := claim.Encode(oKp) require_NoError(t, err) claim.Limits.JetStreamLimits = limits2 aJwt2, err := claim.Encode(oKp) require_NoError(t, err) claim.Limits.JetStreamLimits = limitsExceeded aJwtLimitsExceeded, err := claim.Encode(oKp) require_NoError(t, err) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{} // disabled aJwt4, err := claim.Encode(oKp) require_NoError(t, err) // account user uKp, _ := nkeys.CreateUser() uSeed, _ := uKp.Seed() uclaim = newJWTTestUserClaims() uclaim.Subject, _ = uKp.PublicKey() userJwt, err := uclaim.Encode(akp) require_NoError(t, err) userCreds := genCredsFile(t, userJwt, uSeed) dir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} operator: %s resolver: { type: full dir: '%s' } system_account: %s `, dir, ojwt, dir, sysPub))) s, opts := RunServerWithConfig(conf) defer s.Shutdown() port := opts.Port updateJwt(s.ClientURL(), sysCreds, aPub, aJwt1) c := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCreds), nats.ReconnectWait(200*time.Millisecond)) defer c.Close() validate_limits(c, limits1) // keep using the same connection updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) validate_limits(c, limits2) // keep using the same connection but do NOT CHANGE anything. // This tests if the jwt is applied a second time (would fail) updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) validate_limits(c, limits2) // keep using the same connection. This update EXCEEDS LIMITS updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) validate_limits(c, limits2) // disable test after failure updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) expect_InfoError(c) // re enable, again testing with a value that can't be applied twice updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) validate_limits(c, limits2) // disable test no prior failure updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) expect_InfoError(c) // Wrong limits form start updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) expect_JSDisabledForAccount(c) // enable js but exceed limits. Followed by fix via restart updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) validate_limits(c, limits2) updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) validate_limits(c, limits2) s.Shutdown() conf = createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:%d jetstream: {max_mem_store: 20Mb, max_file_store: 20Mb, store_dir: "%s"} operator: %s resolver: { type: full dir: '%s' } system_account: %s `, port, dir, ojwt, dir, sysPub))) s, _ = RunServerWithConfig(conf) defer s.Shutdown() c.Flush() // force client to discover the disconnect checkClientsCount(t, s, 1) validate_limits(c, limitsExceeded) s.Shutdown() // disable jetstream test conf = createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:%d operator: %s resolver: { type: full dir: '%s' } system_account: %s `, port, ojwt, dir, sysPub))) s, _ = RunServerWithConfig(conf) defer s.Shutdown() c.Flush() // force client to discover the disconnect checkClientsCount(t, s, 1) expect_JSDisabledForAccount(c) // test that it stays disabled updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) expect_JSDisabledForAccount(c) c.Close() } func TestJetStreamJWTDisallowBearer(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, err := nkeys.CreateAccount() require_NoError(t, err) accIdPub, err := accKp.PublicKey() require_NoError(t, err) aClaim := jwt.NewAccountClaims(accIdPub) accJwt1, err := aClaim.Encode(oKp) require_NoError(t, err) aClaim.Limits.DisallowBearer = true accJwt2, err := aClaim.Encode(oKp) require_NoError(t, err) uc := jwt.NewUserClaims("dummy") uc.BearerToken = true uOpt1 := createUserCredsEx(t, uc, accKp) uc.BearerToken = false uOpt2 := createUserCredsEx(t, uc, accKp) dir := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s system_account: %s resolver: { type: full dir: '%s/jwt' } resolver_preload = { %s : "%s" } `, ojwt, syspub, dir, syspub, sysJwt))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() updateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1) disconnectErrCh := make(chan error, 10) defer close(disconnectErrCh) nc1, err := nats.Connect(s.ClientURL(), uOpt1, nats.NoReconnect(), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { disconnectErrCh <- err })) require_NoError(t, err) defer nc1.Close() // update jwt and observe bearer token get disconnected updateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1) select { case err := <-disconnectErrCh: require_Contains(t, err.Error(), "authorization violation") case <-time.After(time.Second): t.Fatalf("expected error on disconnect") } // assure bearer token is not allowed to connect _, err = nats.Connect(s.ClientURL(), uOpt1) require_Error(t, err) // assure non bearer token can connect nc2, err := nats.Connect(s.ClientURL(), uOpt2) require_NoError(t, err) defer nc2.Close() } func TestJetStreamJWTMove(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, aExpPub := createKey(t) test := func(t *testing.T, replicas int, accClaim *jwt.AccountClaims) { accClaim.Name = "acc" accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 5, 2, func(serverName, clustername, storeDir, conf string) string { switch sname := serverName[strings.Index(serverName, "-")+1:]; sname { case "S1", "S2": conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") } return conf + fmt.Sprintf(` server_tags: [cloud:%s-tag] operator: %s system_account: %s resolver: { type: full dir: '%s/jwt' } resolver_preload = { %s : %s } `, clustername, ojwt, syspub, storeDir, syspub, sysJwt) }, nil) defer sc.shutdown() s := sc.serverByName("C1-S1") require_False(t, s.JetStreamEnabled()) updateJwt(t, s.ClientURL(), sysCreds, accJwt, 10) sc.waitOnAccount(aExpPub) s = sc.serverByName("C2-S1") require_False(t, s.JetStreamEnabled()) nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) ci, err := js.AddStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, Placement: &nats.Placement{Tags: []string{"cloud:C1-tag"}}}) require_NoError(t, err) require_Equal(t, ci.Cluster.Name, "C1") _, err = js.AddConsumer("MOVE-ME", &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.Publish("MOVE-ME", []byte("hello world")) require_NoError(t, err) // Perform actual move ci, err = js.UpdateStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, Placement: &nats.Placement{Tags: []string{"cloud:C2-tag"}}}) require_NoError(t, err) require_Equal(t, ci.Cluster.Name, "C1") sc.clusterForName("C2").waitOnStreamLeader(aExpPub, "MOVE-ME") checkFor(t, 30*time.Second, 250*time.Millisecond, func() error { if si, err := js.StreamInfo("MOVE-ME"); err != nil { return fmt.Errorf("stream: %v", err) } else if si.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } else if len(si.Cluster.Replicas) != replicas-1 { return fmt.Errorf("Expected %d replicas, got %d", replicas-1, len(si.Cluster.Replicas)) } else if si.State.Msgs != 1 { return fmt.Errorf("expected one message") } // Now make sure consumer has leader etc.. if ci, err := js.ConsumerInfo("MOVE-ME", "dur"); err != nil { return fmt.Errorf("stream: %v", err) } else if ci.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", ci.Cluster.Name) } else if ci.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } return nil }) sub, err := js.PullSubscribe("", "dur", nats.BindStream("MOVE-ME")) require_NoError(t, err) m, err := sub.Fetch(1) require_NoError(t, err) require_NoError(t, m[0].AckSync()) } t.Run("tiered", func(t *testing.T) { accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1100, Consumer: 1, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 3300, Consumer: 1, Streams: 1} t.Run("R3", func(t *testing.T) { test(t, 3, accClaim) }) t.Run("R1", func(t *testing.T) { test(t, 1, accClaim) }) }) t.Run("non-tiered", func(t *testing.T) { accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{ DiskStorage: 4400, Consumer: 2, Streams: 2} t.Run("R3", func(t *testing.T) { test(t, 3, accClaim) }) t.Run("R1", func(t *testing.T) { test(t, 1, accClaim) }) }) } func TestJetStreamJWTClusteredTiers(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) newUser(t, sysKp) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1100, Consumer: 2, Streams: 2} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 1100, Consumer: 1, Streams: 1} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` + fmt.Sprintf(` operator: %s system_account: %s resolver = MEMORY resolver_preload = { %s : %s %s : %s } `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) defer c.shutdown() nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test absent tiers _, err = js.AddStream(&nats.StreamConfig{Name: "testR2", Replicas: 2, Subjects: []string{"testR2"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") _, err = js.AddStream(&nats.StreamConfig{Name: "testR5", Replicas: 5, Subjects: []string{"testR5"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") // Test tiers up to stream limits _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-1", Replicas: 3, Subjects: []string{"testR3-1"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-2", Replicas: 1, Subjects: []string{"testR1-2"}}) require_NoError(t, err) // Test exceeding tiered stream limit _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum number of streams reached") _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-3", Replicas: 3, Subjects: []string{"testR3-3"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum number of streams reached") // Test tiers up to consumer limits _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.AddConsumer("testR3-1", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // test exceeding tiered consumer limits _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur4", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur5", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") // test tiered storage limit msg := [512]byte{} _, err = js.Publish("testR1-1", msg[:]) require_NoError(t, err) _, err = js.Publish("testR3-1", msg[:]) require_NoError(t, err) _, err = js.Publish("testR3-1", msg[:]) require_NoError(t, err) _, err = js.Publish("testR1-2", msg[:]) require_NoError(t, err) time.Sleep(2000 * time.Millisecond) // wait for update timer to synchronize totals // test exceeding tiered storage limit _, err = js.Publish("testR1-1", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") _, err = js.Publish("testR3-1", []byte("fail this message!")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") // retrieve limits var info JSApiAccountInfoResponse m, err := nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &info) require_NoError(t, err) require_True(t, info.Memory == 0) // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit // Alternative to checking both values is, prior to the info request, wait for another update require_True(t, info.Store == 4400 || info.Store == 4439) require_True(t, info.Streams == 3) require_True(t, info.Consumers == 3) require_True(t, info.Limits == JetStreamAccountLimits{}) r1 := info.Tiers["R1"] require_True(t, r1.Streams == 2) require_True(t, r1.Consumers == 2) // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit // Alternative to checking both values is, prior to the info request, wait for another update require_True(t, r1.Store == 1100 || r1.Store == 1139) require_True(t, r1.Memory == 0) require_True(t, r1.Limits == JetStreamAccountLimits{ MaxMemory: 0, MaxStore: 1100, MaxStreams: 2, MaxConsumers: 2, MaxAckPending: -1, MemoryMaxStreamBytes: -1, StoreMaxStreamBytes: -1, MaxBytesRequired: false, }) r3 := info.Tiers["R3"] require_True(t, r3.Streams == 1) require_True(t, r3.Consumers == 1) require_True(t, r3.Store == 3300) require_True(t, r3.Memory == 0) require_True(t, r3.Limits == JetStreamAccountLimits{ MaxMemory: 0, MaxStore: 1100, MaxStreams: 1, MaxConsumers: 1, MaxAckPending: -1, MemoryMaxStreamBytes: -1, StoreMaxStreamBytes: -1, MaxBytesRequired: false, }) } func TestJetStreamJWTClusteredTiersChange(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 500, MemoryStorage: 0, Consumer: 1, Streams: 1} accJwt1 := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) start := time.Now() tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, func(serverName, clustername, storeDir, conf string) string { return conf + fmt.Sprintf(` operator: %s system_account: %s resolver: { type: full dir: '%s/jwt' }`, ojwt, syspub, storeDir) }) defer c.shutdown() updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) c.waitOnAccount(aExpPub) nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test tiers up to stream limits cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} _, err = js.AddStream(cfg) require_NoError(t, err) cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_Error(t, err, errors.New("nats: insufficient storage resources available")) time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} accJwt2 := encodeClaim(t, accClaim, aExpPub) updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) var rBefore, rAfter JSApiAccountInfoResponse m, err := nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &rBefore) require_NoError(t, err) _, err = js.UpdateStream(cfg) require_NoError(t, err) m, err = nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &rAfter) require_NoError(t, err) require_True(t, rBefore.Tiers["R1"].Streams == 1) require_True(t, rBefore.Tiers["R1"].Streams == rAfter.Tiers["R3"].Streams) require_True(t, rBefore.Tiers["R3"].Streams == 0) require_True(t, rAfter.Tiers["R1"].Streams == 0) } func TestJetStreamJWTClusteredDeleteTierWithStreamAndMove(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} accJwt1 := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) start := time.Now() tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, func(serverName, clustername, storeDir, conf string) string { return conf + fmt.Sprintf(` operator: %s system_account: %s resolver: { type: full dir: '%s/jwt' }`, ojwt, syspub, storeDir) }) defer c.shutdown() updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) c.waitOnAccount(aExpPub) nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test tiers up to stream limits cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} _, err = js.AddStream(cfg) require_NoError(t, err) _, err = js.Publish("testR1-1", nil) require_NoError(t, err) time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes delete(accClaim.Limits.JetStreamTieredLimits, "R1") accJwt2 := encodeClaim(t, accClaim, aExpPub) updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) var respBefore JSApiAccountInfoResponse m, err := nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &respBefore) require_NoError(t, err) require_True(t, respBefore.JetStreamAccountStats.Tiers["R3"].Streams == 0) require_True(t, respBefore.JetStreamAccountStats.Tiers["R1"].Streams == 1) _, err = js.Publish("testR1-1", nil) require_Error(t, err) require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) // I noticed this taking > 5 seconds checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { _, err = js.Publish("testR1-1", nil) return err }) var respAfter JSApiAccountInfoResponse m, err = nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &respAfter) require_NoError(t, err) require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Streams == 1) require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Store > 0) _, ok := respAfter.JetStreamAccountStats.Tiers["R1"] require_True(t, !ok) } func TestJetStreamJWTSysAccUpdateMixedMode(t *testing.T) { skp, spub := createKey(t) sUsr := createUserCreds(t, nil, skp) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "SYS" sysJwt := encodeClaim(t, sysClaim, spub) encodeJwt1Time := time.Now() akp, apub := createKey(t) aUsr := createUserCreds(t, nil, akp) claim := jwt.NewAccountClaims(apub) claim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 claim.Limits.JetStreamLimits.Streams = 1 jwt1 := encodeClaim(t, claim, apub) basePath := "/ngs/v1/accounts/jwt/" reqCount := int32(0) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == basePath { w.Write([]byte("ok")) } else if strings.HasSuffix(r.URL.Path, spub) { w.Write([]byte(sysJwt)) } else if strings.HasSuffix(r.URL.Path, apub) { w.Write([]byte(jwt1)) } else { // only count requests that could be filled return } atomic.AddInt32(&reqCount, 1) })) defer ts.Close() tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, func(serverName, clusterName, storeDir, conf string) string { // create an ngs like setup, with connection and non connection server if clusterName == "C1" { conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") } return fmt.Sprintf(`%s operator: %s system_account: %s resolver: URL("%s%s")`, conf, ojwt, spub, ts.URL, basePath) }, nil) defer sc.shutdown() disconnectChan := make(chan struct{}, 100) defer close(disconnectChan) disconnectCb := nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { disconnectChan <- struct{}{} }) s := sc.clusterForName("C1").randomServer() sysNc := natsConnect(t, s.ClientURL(), sUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) defer sysNc.Close() aNc := natsConnect(t, s.ClientURL(), aUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) defer aNc.Close() js, err := aNc.JetStream() require_NoError(t, err) si, err := js.AddStream(&nats.StreamConfig{Name: "bar", Subjects: []string{"bar"}, Replicas: 3}) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "C2") _, err = js.AccountInfo() require_NoError(t, err) r, err := sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) require_NoError(t, err) respb := ServerAPIResponse{Data: &Accountz{}} require_NoError(t, json.Unmarshal(r.Data, &respb)) hasJSExp := func(resp *ServerAPIResponse) bool { found := false for _, e := range resp.Data.(*Accountz).Account.Exports { if e.Subject == jsAllAPI { found = true break } } return found } require_True(t, hasJSExp(&respb)) // make sure jti increased time.Sleep(time.Second - time.Since(encodeJwt1Time)) sysJwt2 := encodeClaim(t, sysClaim, spub) oldRcount := atomic.LoadInt32(&reqCount) _, err = sysNc.Request(fmt.Sprintf(accUpdateEventSubjNew, spub), []byte(sysJwt2), time.Second) require_NoError(t, err) // test to make sure connected client (aNc) was not kicked time.Sleep(200 * time.Millisecond) require_True(t, len(disconnectChan) == 0) // ensure nothing new has happened, lookup for account not found is skipped during inc require_True(t, atomic.LoadInt32(&reqCount) == oldRcount) // no responders _, err = aNc.Request("foo", nil, time.Second) require_Error(t, err) require_Equal(t, err.Error(), "nats: no responders available for request") nc2, js2 := jsClientConnect(t, sc.clusterForName("C2").randomServer(), aUsr) defer nc2.Close() _, err = js2.AccountInfo() require_NoError(t, err) r, err = sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) require_NoError(t, err) respa := ServerAPIResponse{Data: &Accountz{}} require_NoError(t, json.Unmarshal(r.Data, &respa)) require_True(t, hasJSExp(&respa)) _, err = js.AccountInfo() require_NoError(t, err) } func TestJetStreamJWTExpiredAccountNotCountedTowardLimits(t *testing.T) { op, _ := nkeys.CreateOperator() opPk, _ := op.PublicKey() sk, _ := nkeys.CreateOperator() skPk, _ := sk.PublicKey() opClaim := jwt.NewOperatorClaims(opPk) opClaim.SigningKeys.Add(skPk) opJwt, err := opClaim.Encode(op) require_NoError(t, err) createAccountAndUser := func(pubKey, jwt1, creds1 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} var err error *jwt1, err = claim.Encode(sk) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) } generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { t.Helper() opk, _ := kp.PublicKey() c := jwt.NewGenericClaims(opk) c.Data["accounts"] = accs cJwt, err := c.Encode(kp) if err != nil { t.Fatalf("Expected no error %v", err) } return []byte(cJwt) } var syspub, sysjwt, sysCreds string createAccountAndUser(&syspub, &sysjwt, &sysCreds) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} system_account: %s resolver: { type: full allow_delete: true dir: '%s' timeout: "500ms" } `, opJwt, dirSrv, syspub, dirSrv))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // update system account jwt updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) var apub, ajwt1, aCreds1 string createAccountAndUser(&apub, &ajwt1, &aCreds1) // push jwt (for full resolver) updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) defer ncA.Close() ai, err := jsA.AccountInfo() require_NoError(t, err) require_True(t, ai.Limits.MaxMemory == 7*1024*1024) ncA.Close() nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) defer nc.Close() resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) var apub2, ajwt2, aCreds2 string createAccountAndUser(&apub2, &ajwt2, &aCreds2) // push jwt (for full resolver) updateJwt(t, s.ClientURL(), sysCreds, ajwt2, 1) ncB, jsB := jsClientConnect(t, s, nats.UserCredentials(aCreds2)) defer ncB.Close() ai, err = jsB.AccountInfo() require_NoError(t, err) require_True(t, ai.Limits.MaxMemory == 7*1024*1024) } func TestJetStreamJWTDeletedAccountDoesNotLeakSubscriptions(t *testing.T) { op, _ := nkeys.CreateOperator() opPk, _ := op.PublicKey() sk, _ := nkeys.CreateOperator() skPk, _ := sk.PublicKey() opClaim := jwt.NewOperatorClaims(opPk) opClaim.SigningKeys.Add(skPk) opJwt, err := opClaim.Encode(op) require_NoError(t, err) createAccountAndUser := func(pubKey, jwt1, creds1 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} var err error *jwt1, err = claim.Encode(sk) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) } generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { t.Helper() opk, _ := kp.PublicKey() c := jwt.NewGenericClaims(opk) c.Data["accounts"] = accs cJwt, err := c.Encode(kp) if err != nil { t.Fatalf("Expected no error %v", err) } return []byte(cJwt) } var syspub, sysjwt, sysCreds string createAccountAndUser(&syspub, &sysjwt, &sysCreds) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: %v} system_account: %s resolver: { type: full allow_delete: true dir: '%s' timeout: "500ms" } `, opJwt, dirSrv, syspub, dirSrv))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() checkNumSubs := func(expected uint32) uint32 { t.Helper() // Wait a bit before capturing number of subs... time.Sleep(250 * time.Millisecond) var ns uint32 checkFor(t, time.Second, 50*time.Millisecond, func() error { subsz, err := s.Subsz(nil) if err != nil { return err } ns = subsz.NumSubs if expected > 0 && ns > expected { return fmt.Errorf("Expected num subs to be back at %v, got %v", expected, ns) } return nil }) return ns } beforeCreate := checkNumSubs(0) // update system account jwt updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) createAndDelete := func() { t.Helper() var apub, ajwt1, aCreds1 string createAccountAndUser(&apub, &ajwt1, &aCreds1) // push jwt (for full resolver) updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) defer ncA.Close() ai, err := jsA.AccountInfo() require_NoError(t, err) require_True(t, ai.Limits.MaxMemory == 7*1024*1024) ncA.Close() nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) defer nc.Close() resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) } // Create and delete multiple accounts for i := 0; i < 10; i++ { createAndDelete() } // There is a subscription on `_R_.>` that is created on the system account // and that will not go away, so discount it. checkNumSubs(beforeCreate + 1) } func TestJetStreamJWTDeletedAccountIsReEnabled(t *testing.T) { op, _ := nkeys.CreateOperator() opPk, _ := op.PublicKey() sk, _ := nkeys.CreateOperator() skPk, _ := sk.PublicKey() opClaim := jwt.NewOperatorClaims(opPk) opClaim.SigningKeys.Add(skPk) opJwt, err := opClaim.Encode(op) require_NoError(t, err) createAccountAndUser := func(pubKey, jwt1, creds1 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10} var err error *jwt1, err = claim.Encode(sk) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) } generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { t.Helper() opk, _ := kp.PublicKey() c := jwt.NewGenericClaims(opk) c.Data["accounts"] = accs cJwt, err := c.Encode(kp) if err != nil { t.Fatalf("Expected no error %v", err) } return []byte(cJwt) } // admin user var syspub, sysjwt, sysCreds string createAccountAndUser(&syspub, &sysjwt, &sysCreds) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} system_account: %s resolver: { type: full allow_delete: true dir: '%s' timeout: "500ms" } `, opJwt, dirSrv, syspub, dirSrv))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // update system account jwt updateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1) // create account var apub, ajwt1, aCreds1 string kp, _ := nkeys.CreateAccount() apub, _ = kp.PublicKey() claim := jwt.NewAccountClaims(apub) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10, } ajwt1, err = claim.Encode(sk) require_NoError(t, err) // user ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) aCreds1 = genCredsFile(t, ujwt1, seed) // push user account updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) ncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1)) defer ncA.Close() jsA.AddStream(&nats.StreamConfig{Name: "foo"}) jsA.Publish("foo", []byte("Hello World")) jsA.Publish("foo", []byte("Hello Again")) // JS should be working ai, err := jsA.AccountInfo() require_NoError(t, err) require_True(t, ai.Limits.MaxMemory == 7*1024*1024) require_True(t, ai.Limits.MaxStore == 7*1024*1024) require_True(t, ai.Tier.Streams == 1) // connect with a different connection and delete the account. nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) defer nc.Close() // delete account resp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) // account was disabled and now disconnected, this should get a connection is closed error. _, err = jsA.AccountInfo() if err == nil || !errors.Is(err, nats.ErrConnectionClosed) { t.Errorf("Expected connection closed error, got: %v", err) } ncA.Close() // re-enable, same claims would be detected updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) // expected to get authorization timeout at this time _, err = nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds1)) if !errors.Is(err, nats.ErrAuthorization) { t.Errorf("Expected authorization issue on connect, got: %v", err) } // edit the account and push again with updated claims to same account claim = jwt.NewAccountClaims(apub) claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ MemoryStorage: -1, DiskStorage: 10 * 1024 * 1024, Streams: 10, } ajwt1, err = claim.Encode(sk) require_NoError(t, err) updateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1) // reconnect with the updated account ncA, jsA = jsClientConnect(t, s, nats.UserCredentials(aCreds1)) defer ncA.Close() ai, err = jsA.AccountInfo() if err != nil { t.Fatal(err) } require_True(t, ai.Limits.MaxMemory == -1) require_True(t, ai.Limits.MaxStore == 10*1024*1024) require_True(t, ai.Tier.Streams == 1) // should be possible to get stream info again si, err := jsA.StreamInfo("foo") if err != nil { t.Fatal(err) } if si.State.Msgs != 2 { t.Fatal("Unexpected number of messages from recovered stream") } msg, err := jsA.GetMsg("foo", 1) if err != nil { t.Fatal(err) } if string(msg.Data) != "Hello World" { t.Error("Unexpected message") } ncA.Close() } // Make sure 100MB HA means 100MB of R3, not 33.3MB. func TestJetStreamJWTHAStorageLimitsAndAccounting(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) newUser(t, sysKp) maxFileStorage := int64(100 * 1024 * 1024) maxMemStorage := int64(2 * 1024 * 1024) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` + fmt.Sprintf(` operator: %s system_account: %s resolver = MEMORY resolver_preload = { %s : %s %s : %s } `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) defer c.shutdown() nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test max bytes first. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{"foo"}}) require_NoError(t, err) require_NoError(t, js.DeleteStream("TEST")) _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, Subjects: []string{"foo"}}) require_NoError(t, err) // Now test actual usage. // We should be able to send just over 200 of these. msg := [500 * 1024]byte{} for i := 0; i < 250; i++ { if _, err := js.Publish("foo", msg[:]); err != nil { require_Error(t, err, NewJSAccountResourcesExceededError()) require_True(t, i > 200) break } } si, err := js.StreamInfo("TEST") require_NoError(t, err) // Make sure we are no more then 1 msg below our max in terms of size. delta := maxFileStorage - int64(si.State.Bytes) require_True(t, int(delta) < len(msg)) // Now memory as well. require_NoError(t, js.DeleteStream("TEST")) // Test max bytes first. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxMemStorage, Storage: nats.MemoryStorage, Subjects: []string{"foo"}}) require_NoError(t, err) require_NoError(t, js.DeleteStream("TEST")) _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, Storage: nats.MemoryStorage, Subjects: []string{"foo"}}) require_NoError(t, err) // This is much smaller, so should only be able to send 4. for i := 0; i < 5; i++ { if _, err := js.Publish("foo", msg[:]); err != nil { require_Error(t, err, NewJSAccountResourcesExceededError()) require_Equal(t, i, 4) break } } si, err = js.StreamInfo("TEST") require_NoError(t, err) // Make sure we are no more then 1 msg below our max in terms of size. delta = maxMemStorage - int64(si.State.Bytes) require_True(t, int(delta) < len(msg)) } func TestJetStreamJWTHAStorageLimitsOnScaleAndUpdate(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) newUser(t, sysKp) maxFileStorage := int64(5 * 1024 * 1024) maxMemStorage := int64(1 * 1024 * 1024) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` + fmt.Sprintf(` operator: %s system_account: %s resolver = MEMORY resolver_preload = { %s : %s %s : %s } `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) defer c.shutdown() nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test max bytes first. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{"foo"}}) require_NoError(t, err) // Now delete require_NoError(t, js.DeleteStream("TEST")) // Now do 5 1MB streams. for i := 1; i <= 5; i++ { sname := fmt.Sprintf("TEST%d", i) _, err = js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3, MaxBytes: 1 * 1024 * 1024}) require_NoError(t, err) } // Should fail. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) require_Error(t, err, errors.New("insufficient storage resources")) // Update Test1 and Test2 to smaller reservations. _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST1", Replicas: 3, MaxBytes: 512 * 1024}) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST2", Replicas: 3, MaxBytes: 512 * 1024}) require_NoError(t, err) // Now make sure TEST6 succeeds. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) require_NoError(t, err) // Now delete the R3 version. require_NoError(t, js.DeleteStream("TEST6")) // Now do R1 version and then we will scale up. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST6", Replicas: 1, MaxBytes: 1 * 1024 * 1024}) require_NoError(t, err) // Now make sure scale up works. _, err = js.UpdateStream(&nats.StreamConfig{Name: "TEST6", Replicas: 3, MaxBytes: 1 * 1024 * 1024}) require_NoError(t, err) // Add in a few more streams to check reserved reporting in account info. _, err = js.AddStream(&nats.StreamConfig{Name: "TEST7", Replicas: 1, MaxBytes: 2 * 1024 * 1024}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "TEST8", Replicas: 1, MaxBytes: 256 * 1024, Storage: nats.MemoryStorage}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "TEST9", Replicas: 3, MaxBytes: 22 * 1024, Storage: nats.MemoryStorage}) require_NoError(t, err) // Now make sure we report reserved correctly. // Do this direct to server since client does not support it yet. var info JSApiAccountInfoResponse resp, err := nc.Request("$JS.API.INFO", nil, time.Second) require_NoError(t, err) require_NoError(t, json.Unmarshal(resp.Data, &info)) stats := info.JetStreamAccountStats r1, r3 := stats.Tiers["R1"], stats.Tiers["R3"] require_Equal(t, r1.ReservedMemory, 256*1024) // TEST8 require_Equal(t, r1.ReservedStore, 2*1024*1024) // TEST7 require_Equal(t, r3.ReservedMemory, 22*1024) // TEST9 require_Equal(t, r3.ReservedStore, 5*1024*1024) // TEST1-TEST6 } func TestJetStreamJWTClusteredTiersR3StreamWithR1ConsumersAndAccounting(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) newUser(t, sysKp) accKp, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1100, Consumer: 10, Streams: 1} accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ DiskStorage: 1100, Consumer: 1, Streams: 1} accJwt := encodeClaim(t, accClaim, aExpPub) accCreds := newUser(t, accKp) tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` + fmt.Sprintf(` operator: %s system_account: %s resolver = MEMORY resolver_preload = { %s : %s %s : %s } `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Replicas: 3, }) require_NoError(t, err) // Now make sure we can add in 10 R1 consumers. for i := 1; i <= 10; i++ { _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: fmt.Sprintf("C-%d", i), AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_NoError(t, err) } info, err := js.AccountInfo() require_NoError(t, err) // Make sure we account for these properly. r1 := info.Tiers["R1"] r3 := info.Tiers["R3"] require_Equal(t, r1.Streams, 0) require_Equal(t, r1.Consumers, 10) require_Equal(t, r3.Streams, 1) require_Equal(t, r3.Consumers, 0) } func TestJetStreamJWTUpdateWithPreExistingStream(t *testing.T) { updateJwt := func(url string, creds string, pubKey string, jwt string) { t.Helper() c := natsConnect(t, url, nats.UserCredentials(creds)) defer c.Close() if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { t.Fatal("error not expected in this test", err) } else { content := make(map[string]any) if err := json.Unmarshal(msg.Data, &content); err != nil { t.Fatalf("%v", err) } else if _, ok := content["data"]; !ok { t.Fatalf("did not get an ok response got: %v", content) } } } createUserCreds := func(akp nkeys.KeyPair) string { uKp1, _ := nkeys.CreateUser() uSeed1, _ := uKp1.Seed() uclaim := newJWTTestUserClaims() uclaim.Subject, _ = uKp1.PublicKey() userJwt1, err := uclaim.Encode(akp) require_NoError(t, err) return genCredsFile(t, userJwt1, uSeed1) } // Create system account. sysKp, _ := nkeys.CreateAccount() sysPub, _ := sysKp.PublicKey() sysUKp, _ := nkeys.CreateUser() sysUSeed, _ := sysUKp.Seed() uclaim := newJWTTestUserClaims() uclaim.Subject, _ = sysUKp.PublicKey() sysUserJwt, err := uclaim.Encode(sysKp) require_NoError(t, err) sysKp.Seed() sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) // Create exporting account. akpE, _ := nkeys.CreateAccount() aPubE, _ := akpE.PublicKey() claimE := jwt.NewAccountClaims(aPubE) aJwtE, err := claimE.Encode(oKp) require_NoError(t, err) // Create importing account. akpI, _ := nkeys.CreateAccount() aPubI, _ := akpI.PublicKey() claimI := jwt.NewAccountClaims(aPubI) claimI.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 1024 * 1024} claimI.Imports.Add(&jwt.Import{ Name: "import", Subject: "foo", Account: aPubE, Type: jwt.Stream, }) aJwtI, err := claimI.Encode(oKp) require_NoError(t, err) // Create users. userCredsE := createUserCreds(akpE) userCredsI := createUserCreds(akpI) // Start server and update JWTs. dir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"} operator: %s resolver: { type: full dir: '%s' } system_account: %s `, dir, ojwt, dir, sysPub))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() updateJwt(s.ClientURL(), sysCreds, aPubI, aJwtI) updateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE) // Create stream on importing account before we restart. nci, js := jsClientConnect(t, s, nats.UserCredentials(userCredsI)) defer nci.Close() _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) // Restart server. nci.Close() s.Shutdown() s.WaitForShutdown() s, _ = RunServerWithConfig(conf) defer s.Shutdown() // Reconnect and confirm stream is empty. nci, js = jsClientConnect(t, s, nats.UserCredentials(userCredsI)) defer nci.Close() si, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, si.State.Msgs, 0) // If an import/export gets added when the stream already existed on startup. // We should still be able to route those messages. claimE.Exports.Add(&jwt.Export{ Name: "export", Subject: "foo", Type: jwt.Stream, }) aJwtE, err = claimE.Encode(oKp) require_NoError(t, err) updateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE) // Connect to exporting account and publish a message that should be exported/imported. nce := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCredsE)) defer nce.Close() err = nce.Publish("foo", nil) require_NoError(t, err) // Confirm the message was captured by the stream on the importing account. checkFor(t, 2*time.Second, 500*time.Millisecond, func() error { if si, err = js.StreamInfo("TEST"); err != nil { return err } else if si.State.Msgs != 1 { return fmt.Errorf("expected 1 message in stream, got %d", si.State.Msgs) } return nil }) } nats-server-2.10.27/server/jetstream_leafnode_test.go000066400000000000000000001106701477524627100227070ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests // +build !skip_js_tests package server import ( "fmt" "os" "strings" "testing" "time" jwt "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func TestJetStreamLeafNodeUniqueServerNameCrossJSDomain(t *testing.T) { name := "NOT-UNIQUE" test := func(s *Server, sIdExpected string, srvs ...*Server) { ids := map[string]string{} for _, srv := range srvs { checkLeafNodeConnectedCount(t, srv, 2) ids[srv.ID()] = srv.opts.JetStreamDomain } // ensure that an update for every server was received sysNc := natsConnect(t, fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", s.opts.Port)) defer sysNc.Close() sub, err := sysNc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) require_NoError(t, err) for { m, err := sub.NextMsg(time.Second) require_NoError(t, err) tk := strings.Split(m.Subject, ".") if domain, ok := ids[tk[2]]; ok { delete(ids, tk[2]) require_Contains(t, string(m.Data), fmt.Sprintf(`"domain":"%s"`, domain)) } if len(ids) == 0 { break } } cnt := 0 s.nodeToInfo.Range(func(key, value any) bool { cnt++ require_Equal(t, value.(nodeInfo).name, name) require_Equal(t, value.(nodeInfo).id, sIdExpected) return true }) require_True(t, cnt == 1) } tmplA := ` listen: -1 server_name: %s jetstream { max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', domain: hub } accounts { JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } leaf { port: -1 } ` tmplL := ` listen: -1 server_name: %s jetstream { max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', domain: %s } accounts { JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } leaf { remotes [ { urls: [ %s ], account: "JSY" } { urls: [ %s ], account: "$SYS" } ] } ` t.Run("same-domain", func(t *testing.T) { confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir()))) sA, oA := RunServerWithConfig(confA) defer sA.Shutdown() // using same domain as sA confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), "hub", fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) sL, _ := RunServerWithConfig(confL) defer sL.Shutdown() // as server name uniqueness is violates, sL.ID() is the expected value test(sA, sL.ID(), sA, sL) }) t.Run("different-domain", func(t *testing.T) { confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir()))) sA, oA := RunServerWithConfig(confA) defer sA.Shutdown() // using different domain as sA confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), "spoke", fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) sL, _ := RunServerWithConfig(confL) defer sL.Shutdown() checkLeafNodeConnectedCount(t, sL, 2) checkLeafNodeConnectedCount(t, sA, 2) // ensure sA contains only sA.ID test(sA, sA.ID(), sA, sL) }) } func TestJetStreamLeafNodeJwtPermsAndJSDomains(t *testing.T) { createAcc := func(js bool) (string, string, nkeys.KeyPair) { kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() claim := jwt.NewAccountClaims(aPub) if js { claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ MemoryStorage: 1024 * 1024, DiskStorage: 1024 * 1024, Streams: 1, Consumer: 2} } aJwt, err := claim.Encode(oKp) require_NoError(t, err) return aPub, aJwt, kp } sysPub, sysJwt, sysKp := createAcc(false) accPub, accJwt, accKp := createAcc(true) noExpiration := time.Now().Add(time.Hour) // create user for acc to be used in leaf node. lnCreds := createUserWithLimit(t, accKp, noExpiration, func(j *jwt.UserPermissionLimits) { j.Sub.Deny.Add("subdeny") j.Pub.Deny.Add("pubdeny") }) unlimitedCreds := createUserWithLimit(t, accKp, noExpiration, nil) sysCreds := createUserWithLimit(t, sysKp, noExpiration, nil) tmplA := ` operator: %s system_account: %s resolver: MEMORY resolver_preload: { %s: %s %s: %s } listen: 127.0.0.1:-1 leafnodes: { listen: 127.0.0.1:-1 } jetstream :{ domain: "cluster" store_dir: '%s' max_mem: 100Mb max_file: 100Mb } ` tmplL := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS jetstream: { domain: ln1 store_dir: '%s' max_mem: 50Mb max_file: 50Mb } leafnodes:{ remotes:[{ url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, { url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] } ` confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, ojwt, sysPub, sysPub, sysJwt, accPub, accJwt, t.TempDir()))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, t.TempDir(), sA.opts.LeafNode.Port, lnCreds, sA.opts.LeafNode.Port, sysCreds))) sL, _ := RunServerWithConfig(confL) defer sL.Shutdown() checkLeafNodeConnectedCount(t, sA, 2) checkLeafNodeConnectedCount(t, sL, 2) ncA := natsConnect(t, sA.ClientURL(), nats.UserCredentials(unlimitedCreds)) defer ncA.Close() ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sL.opts.Port)) defer ncL.Close() test := func(subject string, cSub, cPub *nats.Conn, remoteServerForSub *Server, accName string, pass bool) { t.Helper() sub, err := cSub.SubscribeSync(subject) require_NoError(t, err) require_NoError(t, cSub.Flush()) // ensure the subscription made it across, or if not sent due to sub deny, make sure it could have made it. if remoteServerForSub == nil { time.Sleep(200 * time.Millisecond) } else { checkSubInterest(t, remoteServerForSub, accName, subject, time.Second) } require_NoError(t, cPub.Publish(subject, []byte("hello world"))) require_NoError(t, cPub.Flush()) m, err := sub.NextMsg(500 * time.Millisecond) if pass { require_NoError(t, err) require_True(t, m.Subject == subject) require_Equal(t, string(m.Data), "hello world") } else { require_True(t, err == nats.ErrTimeout) } } t.Run("sub-on-ln-pass", func(t *testing.T) { test("sub", ncL, ncA, sA, accPub, true) }) t.Run("sub-on-ln-fail", func(t *testing.T) { test("subdeny", ncL, ncA, nil, "", false) }) t.Run("pub-on-ln-pass", func(t *testing.T) { test("pub", ncA, ncL, sL, "A", true) }) t.Run("pub-on-ln-fail", func(t *testing.T) { test("pubdeny", ncA, ncL, nil, "A", false) }) } func TestJetStreamLeafNodeClusterExtensionWithSystemAccount(t *testing.T) { /* Topologies tested here same == true A <-> B ^ |\ | \ | proxy | \ LA <-> LB same == false A <-> B ^ ^ | | | proxy | | LA <-> LB The proxy is turned on later, such that the system account connection can be started later, in a controlled way This explicitly tests the system state before and after this happens. */ tmplA := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } jetstream :{ domain: "cluster" store_dir: '%s' max_mem: 100Mb max_file: 100Mb } server_name: A cluster: { name: clust1 listen: 127.0.0.1:20104 routes=[nats-route://127.0.0.1:20105] no_advertise: true } ` tmplB := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } jetstream: { domain: "cluster" store_dir: '%s' max_mem: 100Mb max_file: 100Mb } server_name: B cluster: { name: clust1 listen: 127.0.0.1:20105 routes=[nats-route://127.0.0.1:20104] no_advertise: true } ` tmplLA := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS jetstream: { domain: "cluster" store_dir: '%s' max_mem: 50Mb max_file: 50Mb %s } server_name: LA cluster: { name: clustL listen: 127.0.0.1:20106 routes=[nats-route://127.0.0.1:20107] no_advertise: true } leafnodes:{ no_advertise: true remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } ` tmplLB := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS jetstream: { domain: "cluster" store_dir: '%s' max_mem: 50Mb max_file: 50Mb %s } server_name: LB cluster: { name: clustL listen: 127.0.0.1:20107 routes=[nats-route://127.0.0.1:20106] no_advertise: true } leafnodes:{ no_advertise: true remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } ` for _, testCase := range []struct { // which topology to pick same bool // If leaf server should be operational and form a Js cluster prior to joining. // In this setup this would be an error as you give the wrong hint. // But this should work itself out regardless leafFunctionPreJoin bool }{ {true, true}, {true, false}, {false, true}, {false, false}} { t.Run(fmt.Sprintf("%t-%t", testCase.same, testCase.leafFunctionPreJoin), func(t *testing.T) { sd1 := t.TempDir() confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, sd1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() sd2 := t.TempDir() confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, sd2))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() checkClusterFormed(t, sA, sB) c := cluster{t: t, servers: []*Server{sA, sB}} c.waitOnLeader() // starting this will allow the second remote in tmplL to successfully connect. port := sB.opts.LeafNode.Port if testCase.same { port = sA.opts.LeafNode.Port } p := &proxyAcceptDetectFailureLate{acceptPort: port} defer p.close() lPort := p.runEx(t, true) hint := "" if testCase.leafFunctionPreJoin { hint = fmt.Sprintf("extension_hint: %s", strings.ToUpper(jsNoExtend)) } sd3 := t.TempDir() // deliberately pick server sA and proxy confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, sd3, hint, sA.opts.LeafNode.Port, lPort))) sLA, _ := RunServerWithConfig(confLA) defer sLA.Shutdown() sd4 := t.TempDir() // deliberately pick server sA and proxy confLB := createConfFile(t, []byte(fmt.Sprintf(tmplLB, sd4, hint, sA.opts.LeafNode.Port, lPort))) sLB, _ := RunServerWithConfig(confLB) defer sLB.Shutdown() checkClusterFormed(t, sLA, sLB) strmCfg := func(name, placementCluster string) *nats.StreamConfig { if placementCluster == "" { return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} } return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, Placement: &nats.Placement{Cluster: placementCluster}} } // Only after the system account is fully connected can streams be placed anywhere. testJSFunctions := func(pass bool) { ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sA.opts.Port)) defer ncA.Close() jsA, err := ncA.JetStream() require_NoError(t, err) _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA1-%t", pass), "")) require_NoError(t, err) _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA2-%t", pass), "clust1")) require_NoError(t, err) _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA3-%t", pass), "clustL")) if pass { require_NoError(t, err) } else { require_Error(t, err) require_Contains(t, err.Error(), "no suitable peers for placement") } ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) defer ncL.Close() jsL, err := ncL.JetStream() require_NoError(t, err) _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL1-%t", pass), "")) require_NoError(t, err) _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL2-%t", pass), "clustL")) require_NoError(t, err) _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL3-%t", pass), "clust1")) if pass { require_NoError(t, err) } else { require_Error(t, err) require_Contains(t, err.Error(), "no suitable peers for placement") } } clusterLnCnt := func(expected int) error { cnt := 0 for _, s := range c.servers { cnt += s.NumLeafNodes() } if cnt == expected { return nil } return fmt.Errorf("not enought leaf node connections, got %d needed %d", cnt, expected) } // Even though there are two remotes defined in tmplL, only one will be able to connect. checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(2) }) checkLeafNodeConnectedCount(t, sLA, 1) checkLeafNodeConnectedCount(t, sLB, 1) c.waitOnPeerCount(2) if testCase.leafFunctionPreJoin { cl := cluster{t: t, servers: []*Server{sLA, sLB}} cl.waitOnLeader() cl.waitOnPeerCount(2) testJSFunctions(false) } else { // In cases where the leaf nodes have to wait for the system account to connect, // JetStream should not be operational during that time ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) defer ncA.Close() jsA, err := ncA.JetStream() require_NoError(t, err) _, err = jsA.AddStream(strmCfg("fail-false", "")) require_Error(t, err) } // Starting the proxy will connect the system accounts. // After they are connected the clusters are merged. // Once this happened, all streams in test can be placed anywhere in the cluster. // Before that only the cluster the client is connected to can be used for placement p.start() // Even though there are two remotes defined in tmplL, only one will be able to connect. checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(4) }) checkLeafNodeConnectedCount(t, sLA, 2) checkLeafNodeConnectedCount(t, sLB, 2) // The leader will reside in the main cluster only c.waitOnPeerCount(4) testJSFunctions(true) }) } } func TestJetStreamLeafNodeClusterMixedModeExtensionWithSystemAccount(t *testing.T) { /* Topology used in this test: CLUSTER(A <-> B <-> C (NO JS)) ^ | LA */ // once every server is up, we expect these peers to be part of the JetStream meta cluster expectedJetStreamPeers := map[string]struct{}{ "A": {}, "B": {}, "LA": {}, } tmplA := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } server_name: A cluster: { name: clust1 listen: 127.0.0.1:20114 routes=[nats-route://127.0.0.1:20115,nats-route://127.0.0.1:20116] no_advertise: true } ` tmplB := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } server_name: B cluster: { name: clust1 listen: 127.0.0.1:20115 routes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20116] no_advertise: true } ` tmplC := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } jetstream: { enabled: false %s } server_name: C cluster: { name: clust1 listen: 127.0.0.1:20116 routes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20115] no_advertise: true } ` tmplLA := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS # the extension hint is to simplify this test. without it present we would need a cluster of size 2 jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb, extension_hint: will_extend } server_name: LA leafnodes:{ no_advertise: true remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } # add the cluster here so we can test placement cluster: { name: clustL } ` for _, withDomain := range []bool{true, false} { t.Run(fmt.Sprintf("with-domain:%t", withDomain), func(t *testing.T) { var jsDisabledDomainString string var jsEnabledDomainString string if withDomain { jsEnabledDomainString = `domain: "domain", ` jsDisabledDomainString = `domain: "domain"` } else { // in case no domain name is set, fall back to the extension hint. // since JS is disabled, the value of this does not clash with other uses. jsDisabledDomainString = "extension_hint: will_extend" } sd1 := t.TempDir() confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, jsEnabledDomainString, sd1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() sd2 := t.TempDir() confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, jsEnabledDomainString, sd2))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() confC := createConfFile(t, []byte(fmt.Sprintf(tmplC, jsDisabledDomainString))) sC, _ := RunServerWithConfig(confC) defer sC.Shutdown() checkClusterFormed(t, sA, sB, sC) c := cluster{t: t, servers: []*Server{sA, sB, sC}} c.waitOnPeerCount(2) sd3 := t.TempDir() // deliberately pick server sC (no JS) to connect to confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, jsEnabledDomainString, sd3, sC.opts.LeafNode.Port, sC.opts.LeafNode.Port))) sLA, _ := RunServerWithConfig(confLA) defer sLA.Shutdown() checkLeafNodeConnectedCount(t, sC, 2) checkLeafNodeConnectedCount(t, sLA, 2) c.waitOnPeerCount(3) peers := c.leader().JetStreamClusterPeers() for _, peer := range peers { if _, ok := expectedJetStreamPeers[peer]; !ok { t.Fatalf("Found unexpected peer %q", peer) } } // helper to create stream config with uniqe name and subject cnt := 0 strmCfg := func(placementCluster string) *nats.StreamConfig { name := fmt.Sprintf("s-%d", cnt) cnt++ if placementCluster == "" { return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} } return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, Placement: &nats.Placement{Cluster: placementCluster}} } test := func(port int, expectedDefPlacement string) { ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", port)) defer ncA.Close() jsA, err := ncA.JetStream() require_NoError(t, err) si, err := jsA.AddStream(strmCfg("")) require_NoError(t, err) require_Contains(t, si.Cluster.Name, expectedDefPlacement) si, err = jsA.AddStream(strmCfg("clust1")) require_NoError(t, err) require_Contains(t, si.Cluster.Name, "clust1") si, err = jsA.AddStream(strmCfg("clustL")) require_NoError(t, err) require_Contains(t, si.Cluster.Name, "clustL") } test(sA.opts.Port, "clust1") test(sB.opts.Port, "clust1") test(sC.opts.Port, "clust1") test(sLA.opts.Port, "clustL") }) } } func TestJetStreamLeafNodeCredsDenies(t *testing.T) { tmplL := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account = SYS jetstream: { domain: "cluster" store_dir: '%s' max_mem: 50Mb max_file: 50Mb } leafnodes:{ remotes:[{url:nats://a1:a1@127.0.0.1:20125, account: A, credentials: '%s' }, {url:nats://s1:s1@127.0.0.1:20125, account: SYS, credentials: '%s', deny_imports: foo, deny_exports: bar}] } ` akp, err := nkeys.CreateAccount() require_NoError(t, err) creds := createUserWithLimit(t, akp, time.Time{}, func(pl *jwt.UserPermissionLimits) { pl.Pub.Deny.Add(jsAllAPI) pl.Sub.Deny.Add(jsAllAPI) }) sd := t.TempDir() confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, sd, creds, creds))) opts := LoadConfig(confL) sL, err := NewServer(opts) require_NoError(t, err) l := captureNoticeLogger{} sL.SetLogger(&l, false, false) go sL.Start() defer sL.Shutdown() // wait till the notices got printed UNTIL_READY: for { <-time.After(50 * time.Millisecond) l.Lock() for _, n := range l.notices { if strings.Contains(n, "Server is ready") { l.Unlock() break UNTIL_READY } } l.Unlock() } l.Lock() cnt := 0 for _, n := range l.notices { if strings.Contains(n, "LeafNode Remote for Account A uses credentials file") || strings.Contains(n, "LeafNode Remote for System Account uses") || strings.Contains(n, "Remote for System Account uses restricted export permissions") || strings.Contains(n, "Remote for System Account uses restricted import permissions") { cnt++ } } l.Unlock() require_True(t, cnt == 4) } func TestJetStreamLeafNodeDefaultDomainCfg(t *testing.T) { tmplHub := ` listen: 127.0.0.1:%d accounts :{ A:{ jetstream: %s, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS jetstream : %s server_name: HUB leafnodes: { listen: 127.0.0.1:%d } %s ` tmplL := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: LEAF leafnodes: { remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},%s] } %s ` test := func(domain string, sysShared bool) { confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, "disabled", "disabled", -1, ""))) sHub, _ := RunServerWithConfig(confHub) defer sHub.Shutdown() noDomainFix := "" if domain == _EMPTY_ { noDomainFix = `default_js_domain:{A:""}` } sys := "" if sysShared { sys = fmt.Sprintf(`{url:nats://s1:s1@127.0.0.1:%d, account: SYS}`, sHub.opts.LeafNode.Port) } sdLeaf := t.TempDir() confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, sys, noDomainFix))) sLeaf, _ := RunServerWithConfig(confL) defer sLeaf.Shutdown() lnCnt := 1 if sysShared { lnCnt++ } checkLeafNodeConnectedCount(t, sHub, lnCnt) checkLeafNodeConnectedCount(t, sLeaf, lnCnt) ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sHub.opts.Port)) defer ncA.Close() jsA, err := ncA.JetStream() require_NoError(t, err) _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) require_True(t, err == nats.ErrNoResponders) // Add in default domain and restart server require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, sHub.opts.Port, "disabled", "disabled", sHub.opts.LeafNode.Port, fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) sHub.Shutdown() sHub.WaitForShutdown() checkLeafNodeConnectedCount(t, sLeaf, 0) sHubUpd1, _ := RunServerWithConfig(confHub) defer sHubUpd1.Shutdown() checkLeafNodeConnectedCount(t, sHubUpd1, lnCnt) checkLeafNodeConnectedCount(t, sLeaf, lnCnt) _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) require_NoError(t, err) // Enable jetstream in hub. sdHub := t.TempDir() jsEnabled := fmt.Sprintf(`{ domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }`, domain, sdHub) require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, sHubUpd1.opts.Port, "disabled", jsEnabled, sHubUpd1.opts.LeafNode.Port, fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) sHubUpd1.Shutdown() sHubUpd1.WaitForShutdown() checkLeafNodeConnectedCount(t, sLeaf, 0) sHubUpd2, _ := RunServerWithConfig(confHub) defer sHubUpd2.Shutdown() checkLeafNodeConnectedCount(t, sHubUpd2, lnCnt) checkLeafNodeConnectedCount(t, sLeaf, lnCnt) _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) require_NoError(t, err) // Enable jetstream in account A of hub // This is a mis config, as you can't have it both ways, local jetstream but default to another one require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, sHubUpd2.opts.Port, "enabled", jsEnabled, sHubUpd2.opts.LeafNode.Port, fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) if domain != _EMPTY_ { // in case no domain name exists there are no additional guard rails, hence no error // It is the users responsibility to get this edge case right sHubUpd2.Shutdown() sHubUpd2.WaitForShutdown() checkLeafNodeConnectedCount(t, sLeaf, 0) sHubUpd3, err := NewServer(LoadConfig(confHub)) sHubUpd3.Shutdown() require_Error(t, err) require_Contains(t, err.Error(), `default_js_domain contains account name "A" with enabled JetStream`) } } t.Run("with-domain-sys", func(t *testing.T) { test("domain", true) }) t.Run("with-domain-nosys", func(t *testing.T) { test("domain", false) }) t.Run("no-domain", func(t *testing.T) { test("", true) }) t.Run("no-domain", func(t *testing.T) { test("", false) }) } func TestJetStreamLeafNodeDefaultDomainJwtExplicit(t *testing.T) { tmplHub := ` listen: 127.0.0.1:%d operator: %s system_account: %s resolver: MEM resolver_preload: { %s:%s %s:%s } jetstream : disabled server_name: HUB leafnodes: { listen: 127.0.0.1:%d } %s ` tmplL := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: LEAF leafnodes: { remotes:[{url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, {url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] } %s ` test := func(domain string) { noDomainFix := "" if domain == _EMPTY_ { noDomainFix = `default_js_domain:{A:""}` } sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) aKp, aPub := createKey(t) aClaim := jwt.NewAccountClaims(aPub) aJwt := encodeClaim(t, aClaim, aPub) aCreds := newUser(t, aKp) confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, ojwt, syspub, syspub, sysJwt, aPub, aJwt, -1, ""))) sHub, _ := RunServerWithConfig(confHub) defer sHub.Shutdown() sdLeaf := t.TempDir() confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, aCreds, sHub.opts.LeafNode.Port, sysCreds, noDomainFix))) sLeaf, _ := RunServerWithConfig(confL) defer sLeaf.Shutdown() checkLeafNodeConnectedCount(t, sHub, 2) checkLeafNodeConnectedCount(t, sLeaf, 2) ncA := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", sHub.opts.Port), createUserCreds(t, nil, aKp)) defer ncA.Close() jsA, err := ncA.JetStream() require_NoError(t, err) _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) require_True(t, err == nats.ErrNoResponders) // Add in default domain and restart server require_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, sHub.opts.Port, ojwt, syspub, syspub, sysJwt, aPub, aJwt, sHub.opts.LeafNode.Port, fmt.Sprintf(`default_js_domain: {%s:"%s"}`, aPub, domain))), 0664)) sHub.Shutdown() sHub.WaitForShutdown() checkLeafNodeConnectedCount(t, sLeaf, 0) sHubUpd1, _ := RunServerWithConfig(confHub) defer sHubUpd1.Shutdown() checkLeafNodeConnectedCount(t, sHubUpd1, 2) checkLeafNodeConnectedCount(t, sLeaf, 2) _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) require_NoError(t, err) } t.Run("with-domain", func(t *testing.T) { test("domain") }) t.Run("no-domain", func(t *testing.T) { test("") }) } func TestJetStreamLeafNodeDefaultDomainClusterBothEnds(t *testing.T) { // test to ensure that default domain functions when both ends of the leaf node connection are clusters tmplHub1 := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} } jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: HUB1 cluster: { name: HUB listen: 127.0.0.1:20134 routes=[nats-route://127.0.0.1:20135] } leafnodes: { listen:127.0.0.1:-1 } ` tmplHub2 := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} } jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: HUB2 cluster: { name: HUB listen: 127.0.0.1:20135 routes=[nats-route://127.0.0.1:20134] } leafnodes: { listen:127.0.0.1:-1 } ` tmplL1 := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} } jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: LEAF1 cluster: { name: LEAF listen: 127.0.0.1:20136 routes=[nats-route://127.0.0.1:20137] } leafnodes: { remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] } default_js_domain: {B:"DHUB"} ` tmplL2 := ` listen: 127.0.0.1:-1 accounts :{ A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} } jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } server_name: LEAF2 cluster: { name: LEAF listen: 127.0.0.1:20137 routes=[nats-route://127.0.0.1:20136] } leafnodes: { remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] } default_js_domain: {B:"DHUB"} ` sd1 := t.TempDir() confHub1 := createConfFile(t, []byte(fmt.Sprintf(tmplHub1, sd1))) sHub1, _ := RunServerWithConfig(confHub1) defer sHub1.Shutdown() sd2 := t.TempDir() confHub2 := createConfFile(t, []byte(fmt.Sprintf(tmplHub2, sd2))) sHub2, _ := RunServerWithConfig(confHub2) defer sHub2.Shutdown() checkClusterFormed(t, sHub1, sHub2) c1 := cluster{t: t, servers: []*Server{sHub1, sHub2}} c1.waitOnPeerCount(2) sd3 := t.TempDir() confLeaf1 := createConfFile(t, []byte(fmt.Sprintf(tmplL1, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) sLeaf1, _ := RunServerWithConfig(confLeaf1) defer sLeaf1.Shutdown() confLeaf2 := createConfFile(t, []byte(fmt.Sprintf(tmplL2, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) sLeaf2, _ := RunServerWithConfig(confLeaf2) defer sLeaf2.Shutdown() checkClusterFormed(t, sLeaf1, sLeaf2) c2 := cluster{t: t, servers: []*Server{sLeaf1, sLeaf2}} c2.waitOnPeerCount(2) checkLeafNodeConnectedCount(t, sHub1, 4) checkLeafNodeConnectedCount(t, sLeaf1, 2) checkLeafNodeConnectedCount(t, sLeaf2, 2) ncB := natsConnect(t, fmt.Sprintf("nats://b1:b1@127.0.0.1:%d", sLeaf1.getOpts().Port)) defer ncB.Close() jsB1, err := ncB.JetStream() require_NoError(t, err) si, err := jsB1.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "HUB") jsB2, err := ncB.JetStream(nats.Domain("DHUB")) require_NoError(t, err) si, err = jsB2.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "HUB") } func TestJetStreamLeafNodeSvcImportExportCycle(t *testing.T) { accounts := ` accounts { SYS: { users: [{user: admin, password: admin}] } LEAF_USER: { users: [{user: leaf_user, password: leaf_user}] imports: [ {service: {account: LEAF_INGRESS, subject: "foo"}} {service: {account: LEAF_INGRESS, subject: "_INBOX.>"}} {service: {account: LEAF_INGRESS, subject: "$JS.leaf.API.>"}, to: "JS.leaf_ingress@leaf.API.>" } ] jetstream: enabled } LEAF_INGRESS: { users: [{user: leaf_ingress, password: leaf_ingress}] exports: [ {service: "foo", accounts: [LEAF_USER]} {service: "_INBOX.>", accounts: [LEAF_USER]} {service: "$JS.leaf.API.>", response_type: "stream", accounts: [LEAF_USER]} ] imports: [ ] jetstream: enabled } } system_account: SYS ` hconf := createConfFile(t, []byte(fmt.Sprintf(` %s listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" } `, accounts))) defer os.Remove(hconf) s, o := RunServerWithConfig(hconf) defer s.Shutdown() lconf := createConfFile(t, []byte(fmt.Sprintf(` %s server_name: leaf-server jetstream { store_dir: '%s' domain=leaf } listen: "127.0.0.1:-1" leafnodes { remotes = [ { urls: ["nats-leaf://leaf_ingress:leaf_ingress@127.0.0.1:%v"] account: "LEAF_INGRESS" } ] } `, accounts, t.TempDir(), o.LeafNode.Port))) defer os.Remove(lconf) sl, so := RunServerWithConfig(lconf) defer sl.Shutdown() checkLeafNodeConnected(t, sl) nc := natsConnect(t, fmt.Sprintf("nats://leaf_user:leaf_user@127.0.0.1:%v", so.Port)) defer nc.Close() js, _ := nc.JetStream(nats.APIPrefix("JS.leaf_ingress@leaf.API.")) _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.FileStorage, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("msg")) require_NoError(t, err) } func TestJetStreamLeafNodeJSClusterMigrateRecovery(t *testing.T) { tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: hub, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "hub", _EMPTY_, 3, 12232, true) defer c.shutdown() tmpl = strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", "domain: leaf, store_dir:", 1) lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "leaf", 3, 23913) defer lnc.shutdown() lnc.waitOnClusterReady() for _, s := range lnc.servers { s.setJetStreamMigrateOnRemoteLeaf() } nc, _ := jsClientConnect(t, lnc.randomServer()) defer nc.Close() ljs, err := nc.JetStream(nats.Domain("leaf")) require_NoError(t, err) // Create an asset in the leafnode cluster. si, err := ljs.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "leaf") require_NotEqual(t, si.Cluster.Leader, noLeader) require_Equal(t, len(si.Cluster.Replicas), 2) // Count how many remotes each server in the leafnode cluster is // supposed to have and then take them down. remotes := map[*Server]int{} for _, s := range lnc.servers { remotes[s] += len(s.leafRemoteCfgs) s.closeAndDisableLeafnodes() checkLeafNodeConnectedCount(t, s, 0) } // The Raft nodes in the leafnode cluster now need some time to // notice that they're no longer receiving AEs from a leader, as // they should have been forced into observer mode. Check that // this is the case. time.Sleep(maxElectionTimeout) for _, s := range lnc.servers { s.rnMu.RLock() for name, n := range s.raftNodes { // We don't expect the metagroup to have turned into an // observer but all other assets should have done. if name == defaultMetaGroupName { require_False(t, n.IsObserver()) } else { require_True(t, n.IsObserver()) } } s.rnMu.RUnlock() } // Bring the leafnode connections back up. for _, s := range lnc.servers { s.reEnableLeafnodes() checkLeafNodeConnectedCount(t, s, remotes[s]) } // Wait for nodes to notice they are no longer in observer mode // and to leave observer mode. time.Sleep(maxElectionTimeout) for _, s := range lnc.servers { s.rnMu.RLock() for _, n := range s.raftNodes { require_False(t, n.IsObserver()) } s.rnMu.RUnlock() } // Previously nodes would have left observer mode but then would // have failed to elect a stream leader as they were stuck on a // long election timer. Now this should work reliably. lnc.waitOnStreamLeader(globalAccountName, "TEST") } nats-server-2.10.27/server/jetstream_meta_benchmark_test.go000066400000000000000000000213701477524627100240700ustar00rootroot00000000000000// Copyright 2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 // +build !skip_js_tests,!skip_js_cluster_tests,!skip_js_cluster_tests_2 package server import ( "fmt" "sync" "sync/atomic" "testing" "github.com/nats-io/nats.go" ) func BenchmarkJetStreamCreate(b *testing.B) { const ( verbose = false resourcePrefix = "S" concurrency = 12 ) // Types of resource that this benchmark creates type ResourceType string const ( Stream ResourceType = "Stream" KVBucket ResourceType = "KVBucket" ObjectStore ResourceType = "ObjStore" ) resourceTypeCases := []ResourceType{ Stream, KVBucket, ObjectStore, } benchmarksCases := []struct { clusterSize int replicas int storage nats.StorageType }{ {1, 1, nats.MemoryStorage}, {3, 3, nats.MemoryStorage}, {3, 3, nats.FileStorage}, } for _, bc := range benchmarksCases { bName := fmt.Sprintf( "N=%d,R=%d,storage=%s,C=%d", bc.clusterSize, bc.replicas, bc.storage.String(), concurrency, ) b.Run( bName, func(b *testing.B) { for _, rt := range resourceTypeCases { //for _, bc := range benchmarksCases { rName := fmt.Sprintf("resource=%s", rt) b.Run( rName, func(b *testing.B) { if verbose { b.Logf( "Creating %d %s resources in cluster with %d nodes, R=%d, %s storage", b.N, string(rt), bc.clusterSize, bc.replicas, bc.storage, ) } // Setup server or cluster _, leaderServer, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize) defer shutdown() defer nc.Close() // All clients connect to cluster (meta) leader for lower variability connectURL := leaderServer.ClientURL() // Wait for all clients and main routine to be ready wgReady := sync.WaitGroup{} wgReady.Add(concurrency + 1) // Wait for all routines to complete wgComplete := sync.WaitGroup{} wgComplete.Add(concurrency) // Number of operations (divided amongst clients) opsLeft := atomic.Int64{} opsLeft.Store(int64(b.N)) totalErrors := atomic.Int64{} // Pre-create connections and JS contexts for i := 1; i <= concurrency; i++ { nc, js := jsClientConnectURL(b, connectURL) defer nc.Close() go func(clientId int, nc *nats.Conn, js nats.JetStreamContext) { defer wgComplete.Done() // Config struct (reused and modified in place for each call) streamConfig := nats.StreamConfig{ Name: "?", Storage: bc.storage, Replicas: bc.replicas, } kvConfig := nats.KeyValueConfig{ Bucket: "?", Storage: bc.storage, Replicas: bc.replicas, } objConfig := nats.ObjectStoreConfig{ Bucket: "?", Storage: bc.storage, Replicas: bc.replicas, } // Block until everyone is ready wgReady.Done() wgReady.Wait() errCount := int64(0) defer func() { // Roll up error count on completion totalErrors.Add(errCount) }() // Track per-client opCount (just for logging/debugging) opCount := 0 for opsLeft.Add(-1) >= 0 { var err error // Create unique resource name resourceName := fmt.Sprintf("%s_%d_%d", resourcePrefix, clientId, opCount) switch rt { case Stream: streamConfig.Name = resourceName _, err = js.AddStream(&streamConfig) case KVBucket: kvConfig.Bucket = resourceName _, err = js.CreateKeyValue(&kvConfig) case ObjectStore: objConfig.Bucket = resourceName _, err = js.CreateObjectStore(&objConfig) } opCount += 1 if err != nil { b.Logf("Error creating %s (%s): %s", rt, resourceName, err) errCount += 1 } } if verbose { b.Logf("Client %d completed %d operations", clientId, opCount) } }(i, nc, js) } // Wait for all clients to be ready wgReady.Done() wgReady.Wait() // Start benchmark clock b.ResetTimer() wgComplete.Wait() b.StopTimer() b.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), "%error") }, ) } }, ) } } func BenchmarkJetStreamCreateConsumers(b *testing.B) { const ( verbose = false streamName = "S" consumerPrefix = "C" concurrency = 12 ) benchmarksCases := []struct { clusterSize int consumerReplicas int consumerStorage nats.StorageType }{ {1, 1, nats.MemoryStorage}, {3, 3, nats.MemoryStorage}, {3, 3, nats.FileStorage}, } type ConsumerType string const ( Ephemeral ConsumerType = "Ephemeral" Durable ConsumerType = "Durable" ) consumerTypeCases := []ConsumerType{ Ephemeral, Durable, } for _, bc := range benchmarksCases { bName := fmt.Sprintf( "N=%d,R=%d,storage=%s,C=%d", bc.clusterSize, bc.consumerReplicas, bc.consumerStorage.String(), concurrency, ) b.Run( bName, func(b *testing.B) { for _, ct := range consumerTypeCases { cName := fmt.Sprintf("Consumer=%s", ct) b.Run( cName, func(b *testing.B) { if verbose { b.Logf( "Creating %d consumers in cluster with %d nodes, R=%d, %s storage", b.N, bc.clusterSize, bc.consumerReplicas, bc.consumerStorage, ) } // Setup server or cluster _, leaderServer, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize) defer shutdown() defer nc.Close() // All clients connect to cluster (meta) leader for lower variability connectURL := leaderServer.ClientURL() // Create stream streamConfig := nats.StreamConfig{ Name: streamName, Storage: nats.FileStorage, Replicas: bc.clusterSize, } _, err := js.AddStream(&streamConfig) if err != nil { b.Fatalf("Failed to create stream: %s", err) } // Wait for all clients and main routine to be ready wgReady := sync.WaitGroup{} wgReady.Add(concurrency + 1) // Wait for all routines to complete wgComplete := sync.WaitGroup{} wgComplete.Add(concurrency) // Number of operations (divided amongst clients) opsLeft := atomic.Int64{} opsLeft.Store(int64(b.N)) // Total number of errors totalErrors := atomic.Int64{} // Pre-create connections and JS contexts for i := 1; i <= concurrency; i++ { nc, js := jsClientConnectURL(b, connectURL) defer nc.Close() go func(clientId int, nc *nats.Conn, js nats.JetStreamContext) { defer wgComplete.Done() // Config struct (reused and modified in place for each call) cfg := nats.ConsumerConfig{ Durable: "", Name: "", Replicas: bc.consumerReplicas, MemoryStorage: bc.consumerStorage == nats.MemoryStorage, } // Block until everyone is ready wgReady.Done() wgReady.Wait() errCount := int64(0) opCount := 0 for opsLeft.Add(-1) >= 0 { var err error // Set unique consumer name cfg.Name = fmt.Sprintf("%s_%d_%d", consumerPrefix, clientId, opCount) if ct == Durable { cfg.Durable = cfg.Name } _, err = js.AddConsumer(streamName, &cfg) if err != nil { b.Logf("Failed to add consumer: %s", err) errCount += 1 } opCount += 1 } if verbose { b.Logf("Client %d completed %d operations", clientId, opCount) } totalErrors.Add(errCount) }(i, nc, js) } // Wait for all clients to be ready wgReady.Done() wgReady.Wait() // Start benchmark clock b.ResetTimer() wgComplete.Wait() b.StopTimer() b.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), "%error") }, ) } }, ) } } nats-server-2.10.27/server/jetstream_sourcing_scaling_test.go000066400000000000000000000201571477524627100244630ustar00rootroot00000000000000// Copyright 2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "github.com/nats-io/nats.go" "strconv" "testing" "time" ) var serverConfig1 = ` server_name: server1 listen: 127.0.0.1:4222 http: 8222 prof_port = 18222 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: my_cluster listen: 127.0.0.1:4248 routes: [nats://127.0.0.1:4249,nats://127.0.0.1:4250] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var serverConfig2 = ` server_name: server2 listen: 127.0.0.1:5222 http: 8223 prof_port = 18223 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: my_cluster listen: 127.0.0.1:4249 routes: [nats://127.0.0.1:4248,nats://127.0.0.1:4250] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var serverConfig3 = ` server_name: server3 listen: 127.0.0.1:6222 http: 8224 prof_port = 18224 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: my_cluster listen: 127.0.0.1:4250 routes: [nats://127.0.0.1:4248,nats://127.0.0.1:4249] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var connectURL = "nats://127.0.0.1:4222,nats://127.0.0.1:4223,nats://127.0.0.1:4224" func createMyLocalCluster(t *testing.T) *cluster { c := &cluster{servers: make([]*Server, 0, 3), opts: make([]*Options, 0, 3), name: "C3"} storeDir1 := t.TempDir() s1, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig1, storeDir1)))) c.servers = append(c.servers, s1) c.opts = append(c.opts, o) storeDir2 := t.TempDir() s2, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig2, storeDir2)))) c.servers = append(c.servers, s2) c.opts = append(c.opts, o) storeDir3 := t.TempDir() s3, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig3, storeDir3)))) c.servers = append(c.servers, s3) c.opts = append(c.opts, o) c.t = t // Wait til we are formed and have a leader. c.checkClusterFormed() c.waitOnClusterReady() return c } // This test is being skipped by CI as it takes too long to run and is meant to test the scalability of sourcing // rather than being a unit test. func TestStreamSourcingScalingSourcingManyBenchmark(t *testing.T) { t.Skip() var numSourced = 1000 var numMsgPerSource = 10_000 var batchSize = 200 var retries int var err error c := createMyLocalCluster(t) defer c.shutdown() nc, err := nats.Connect(connectURL) if err != nil { t.Fatalf("Failed to create client: %v", err) } js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) if err != nil { t.Fatalf("Unexpected error getting JetStream context: %v", err) } defer nc.Close() // create n streams to source from for i := 0; i < numSourced; i++ { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("sourced-%d", i), Subjects: []string{strconv.Itoa(i)}, Retention: nats.LimitsPolicy, Storage: nats.FileStorage, Discard: nats.DiscardOld, Replicas: 1, AllowDirect: true, MirrorDirect: false, DiscardNewPerSubject: false, }) require_NoError(t, err) } fmt.Printf("Streams created\n") // publish n messages for each sourced stream for j := 0; j < numMsgPerSource; j++ { start := time.Now() var pafs = make([]nats.PubAckFuture, numSourced) for i := 0; i < numSourced; i++ { var err error for { pafs[i], err = js.PublishAsync(strconv.Itoa(i), []byte(strconv.Itoa(j))) if err != nil { fmt.Printf("Error async publishing: %v, retrying\n", err) retries++ time.Sleep(10 * time.Millisecond) } else { break } } if i != 0 && i%batchSize == 0 { <-js.PublishAsyncComplete() } } <-js.PublishAsyncComplete() for i := 0; i < numSourced; i++ { select { case <-pafs[i].Ok(): case psae := <-pafs[i].Err(): fmt.Printf("Error on PubAckFuture: %v, retrying sync...\n", psae) retries++ _, err = js.Publish(strconv.Itoa(i), []byte(strconv.Itoa(j))) require_NoError(t, err) } } end := time.Now() if j%1000 == 0 { fmt.Printf("Published round %d, avg pub latency %v\n", j, end.Sub(start)/time.Duration(numSourced)) } } fmt.Printf("Messages published\n") // create the StreamSources streamSources := make([]*nats.StreamSource, numSourced) for i := 0; i < numSourced; i++ { streamSources[i] = &nats.StreamSource{Name: fmt.Sprintf("sourced-%d", i), FilterSubject: strconv.Itoa(i)} } // create a stream that sources from them _, err = js.AddStream(&nats.StreamConfig{ Name: "sourcing", Subjects: []string{"foo"}, Sources: streamSources, Retention: nats.LimitsPolicy, Storage: nats.FileStorage, Discard: nats.DiscardOld, Replicas: 3, AllowDirect: true, MirrorDirect: false, DiscardNewPerSubject: false, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "sourcing") sl := c.streamLeader(globalAccountName, "sourcing") fmt.Printf("Sourcing stream created leader is *** %s ***\n", sl.Name()) expectedSeq := make([]int, numSourced) start := time.Now() var lastMsgs uint64 mset, err := sl.GlobalAccount().lookupStream("sourcing") require_NoError(t, err) checkFor(t, 5*time.Minute, 1000*time.Millisecond, func() error { mset.mu.RLock() var state StreamState mset.store.FastState(&state) mset.mu.RUnlock() if state.Msgs == uint64(numMsgPerSource*numSourced) { fmt.Printf("👠Test passed: expected %d messages, got %d and took %v\n", uint64(numMsgPerSource*numSourced), state.Msgs, time.Since(start)) return nil } else if state.Msgs < uint64(numMsgPerSource*numSourced) { fmt.Printf("Current Rate %d per second - Received %d\n", state.Msgs-lastMsgs, state.Msgs) lastMsgs = state.Msgs return fmt.Errorf("Expected %d messages, got %d", uint64(numMsgPerSource*numSourced), state.Msgs) } else { fmt.Printf("Too many messages! expected %d (retries=%d), got %d\n", uint64(numMsgPerSource*numSourced), retries, state.Msgs) return fmt.Errorf("Too many messages: expected %d (retries=%d), got %d", uint64(numMsgPerSource*numSourced), retries, state.Msgs) } }) // Check that all the messages sourced in the stream are correct // Note: expects to see exactly increasing matching sequence numbers, so could theoretically fail if some messages // get recorded 'out of order' (according to the payload value) because asynchronous JS publication is used to // publish the messages. // However, that should not happen if the publish 'batch size' is not more than the number of streams being sourced. // create a consumer on sourcing _, err = js.AddConsumer("sourcing", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) syncSub, err := js.SubscribeSync("", nats.BindStream("sourcing")) require_NoError(t, err) start = time.Now() print("Checking the messages\n") for i := 0; i < numSourced*numMsgPerSource; i++ { msg, err := syncSub.NextMsg(30 * time.Second) require_NoError(t, err) sId, err := strconv.Atoi(msg.Subject) require_NoError(t, err) seq, err := strconv.Atoi(string(msg.Data)) require_NoError(t, err) if expectedSeq[sId] == seq { expectedSeq[sId]++ } else { t.Fatalf("Expected seq number %d got %d for source %d\n", expectedSeq[sId], seq, sId) } msg.Ack() if i%100_000 == 0 { now := time.Now() fmt.Printf("[%v] Checked %d messages: %f msgs/sec \n", now, i, 100_000/now.Sub(start).Seconds()) start = now } } print("👠Done. \n") } nats-server-2.10.27/server/jetstream_super_cluster_test.go000066400000000000000000003534541477524627100240420ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 && !skip_js_super_cluster_tests // +build !skip_js_tests,!skip_js_cluster_tests,!skip_js_cluster_tests_2,!skip_js_super_cluster_tests package server import ( "encoding/json" "errors" "fmt" "math/rand" "net/http" "net/http/httptest" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func TestJetStreamSuperClusterMetaPlacement(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // We want to influence where the meta leader will place itself when we ask the // current leader to stepdown. ml := sc.leader() cn := ml.ClusterName() var pcn string for _, c := range sc.clusters { if c.name != cn { pcn = c.name break } } // Client based API s := sc.randomServer() nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() stepdown := func(cn string) *JSApiLeaderStepDownResponse { req := &JSApiLeaderStepdownRequest{Placement: &Placement{Cluster: cn}} jreq, err := json.Marshal(req) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(JSApiLeaderStepDown, jreq, time.Second) if err != nil { t.Fatalf("Error on stepdown request: %v", err) } var sdr JSApiLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &sdr); err != nil { t.Fatalf("Unexpected error: %v", err) } return &sdr } // Make sure we get correct errors for tags and bad or unavailable cluster placement. sdr := stepdown("C22") if sdr.Error == nil || !strings.Contains(sdr.Error.Description, "no replacement peer connected") { t.Fatalf("Got incorrect error result: %+v", sdr.Error) } // Should work. sdr = stepdown(pcn) if sdr.Error != nil { t.Fatalf("Got an error on stepdown: %+v", sdr.Error) } sc.waitOnLeader() ml = sc.leader() cn = ml.ClusterName() if cn != pcn { t.Fatalf("Expected new metaleader to be in cluster %q, got %q", pcn, cn) } } func TestJetStreamSuperClusterUniquePlacementTag(t *testing.T) { tmlp := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', unique_tag: az} leaf {listen: 127.0.0.1:-1} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` s := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 5, 2, func(serverName, clustername, storeDir, conf string) string { azTag := map[string]string{ "C1-S1": "az:same", "C1-S2": "az:same", "C1-S3": "az:same", "C1-S4": "az:same", "C1-S5": "az:same", "C2-S1": "az:1", "C2-S2": "az:2", "C2-S3": "az:1", "C2-S4": "az:2", "C2-S5": "az:1", } return conf + fmt.Sprintf("\nserver_tags: [cloud:%s-tag, %s]\n", clustername, azTag[serverName]) }, nil) defer s.shutdown() inDifferentAz := func(ci *nats.ClusterInfo) (bool, error) { t.Helper() if len(ci.Replicas) == 0 { return true, nil } // if R2 (has replica, this setup does not support R3), test if the server in a cluster picked the same az, // as determined by modulo2 of server number which aligns with az dummy := 0 srvnum1 := 0 srvnum2 := 0 if n, _ := fmt.Sscanf(ci.Leader, "C%d-S%d", &dummy, &srvnum1); n != 2 { return false, fmt.Errorf("couldn't parse leader") } if n, _ := fmt.Sscanf(ci.Replicas[0].Name, "C%d-S%d", &dummy, &srvnum2); n != 2 { return false, fmt.Errorf("couldn't parse replica") } return srvnum1%2 != srvnum2%2, nil } nc := natsConnect(t, s.randomServer().ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) for i, test := range []struct { placement *nats.Placement replicas int fail bool cluster string }{ // these pass because replica count is 1 {&nats.Placement{Tags: []string{"az:same"}}, 1, false, "C1"}, {&nats.Placement{Tags: []string{"cloud:C1-tag", "az:same"}}, 1, false, "C1"}, {&nats.Placement{Tags: []string{"cloud:C1-tag"}}, 1, false, "C1"}, // pass because az is set, which disables the filter {&nats.Placement{Tags: []string{"az:same"}}, 2, false, "C1"}, {&nats.Placement{Tags: []string{"cloud:C1-tag", "az:same"}}, 2, false, "C1"}, // fails because this cluster only has the same az {&nats.Placement{Tags: []string{"cloud:C1-tag"}}, 2, true, ""}, // fails because no 3 unique tags exist {&nats.Placement{Tags: []string{"cloud:C2-tag"}}, 3, true, ""}, {nil, 3, true, ""}, // pass because replica count is low enough {nil, 2, false, "C2"}, {&nats.Placement{Tags: []string{"cloud:C2-tag"}}, 2, false, "C2"}, // pass because az is provided {&nats.Placement{Tags: []string{"az:1"}}, 3, false, "C2"}, {&nats.Placement{Tags: []string{"az:2"}}, 2, false, "C2"}, } { name := fmt.Sprintf("test-%d", i) t.Run(name, func(t *testing.T) { si, err := js.AddStream(&nats.StreamConfig{Name: name, Replicas: test.replicas, Placement: test.placement}) if test.fail { require_Error(t, err) require_Contains(t, err.Error(), "no suitable peers for placement", "server tag not unique") return } require_NoError(t, err) if test.cluster != _EMPTY_ { require_Equal(t, si.Cluster.Name, test.cluster) } // skip placement test if tags call for a particular az if test.placement != nil && len(test.placement.Tags) > 0 { for _, tag := range test.placement.Tags { if strings.HasPrefix(tag, "az:") { return } } } diff, err := inDifferentAz(si.Cluster) require_NoError(t, err) require_True(t, diff) }) } t.Run("scale-up-test", func(t *testing.T) { // create enough streams so we hit it eventually for i := 0; i < 10; i++ { cfg := &nats.StreamConfig{Name: fmt.Sprintf("scale-up-%d", i), Replicas: 1, Placement: &nats.Placement{Tags: []string{"cloud:C2-tag"}}} si, err := js.AddStream(cfg) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "C2") cfg.Replicas = 2 si, err = js.UpdateStream(cfg) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "C2") checkFor(t, 10, 250*time.Millisecond, func() error { if si, err := js.StreamInfo(cfg.Name); err != nil { return err } else if diff, err := inDifferentAz(si.Cluster); err != nil { return err } else if !diff { return fmt.Errorf("not in different AZ") } return nil }) } }) } func TestJetStreamSuperClusterBasics(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Client based API s := sc.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Check request origin placement. if si.Cluster.Name != s.ClusterName() { t.Fatalf("Expected stream to be placed in %q, but got %q", s.ClusterName(), si.Cluster.Name) } // Check consumers. sub, err := js.SubscribeSync("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, toSend) ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci.Delivered.Consumer != uint64(toSend) || ci.NumAckPending != toSend { t.Fatalf("ConsumerInfo is not correct: %+v", ci) } // Now check we can place a stream. pcn := "C3" scResp, err := js.AddStream(&nats.StreamConfig{ Name: "TEST2", Placement: &nats.Placement{Cluster: pcn}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.Cluster.Name != pcn { t.Fatalf("Expected the stream to be placed in %q, got %q", pcn, scResp.Cluster.Name) } } // Test that consumer interest across gateways and superclusters is properly identitifed in a remote cluster. func TestJetStreamSuperClusterCrossClusterConsumerInterest(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Since we need all of the peers accounted for to add the stream wait for all to be present. sc.waitOnPeerCount(9) // Client based API - Connect to Cluster C1. Stream and consumer will live in C2. s := sc.clusterForName("C1").randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() pcn := "C2" _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 3, Placement: &nats.Placement{Cluster: pcn}}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Pull based first. sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send a message. if _, err = js.Publish("foo", []byte("CCI")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } fetchMsgs(t, sub, 1, 5*time.Second) // Now check push based delivery. sub, err = js.SubscribeSync("foo", nats.Durable("rip")) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkSubsPending(t, sub, 1) // Send another message. if _, err = js.Publish("foo", []byte("CCI")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkSubsPending(t, sub, 2) } func TestJetStreamSuperClusterPeerReassign(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Client based API s := sc.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() pcn := "C2" // Create a stream in C2 that sources TEST _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Placement: &nats.Placement{Cluster: pcn}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send in 10 messages. msg, toSend := []byte("Hello JS Clustering"), 10 for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Now grab info for this stream. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "TEST" { t.Fatalf("StreamInfo is not correct %+v", si) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Check request origin placement. if si.Cluster.Name != pcn { t.Fatalf("Expected stream to be placed in %q, but got %q", s.ClusterName(), si.Cluster.Name) } // Now remove a peer that is assigned to the stream. rc := sc.clusterForName(pcn) rs := rc.randomNonStreamLeader("$G", "TEST") rc.removeJetStream(rs) // Check the stream info is eventually correct. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return fmt.Errorf("Could not fetch stream info: %v", err) } if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected 2 replicas, got %d", len(si.Cluster.Replicas)) } for _, peer := range si.Cluster.Replicas { if !peer.Current { return fmt.Errorf("Expected replica to be current: %+v", peer) } if !strings.HasPrefix(peer.Name, pcn) { t.Fatalf("Stream peer reassigned to wrong cluster: %q", peer.Name) } } return nil }) } func TestJetStreamSuperClusterInterestOnlyMode(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) template := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} accounts { one { jetstream: enabled users [{user: one, password: password}] } two { %s users [{user: two, password: password}] } } cluster { listen: 127.0.0.1:%d name: %s routes = ["nats://127.0.0.1:%d"] } gateway { name: %s listen: 127.0.0.1:%d gateways = [{name: %s, urls: ["nats://127.0.0.1:%d"]}] } ` storeDir1 := t.TempDir() conf1 := createConfFile(t, []byte(fmt.Sprintf(template, "S1", storeDir1, "", 23222, "A", 23222, "A", 11222, "B", 11223))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() storeDir2 := t.TempDir() conf2 := createConfFile(t, []byte(fmt.Sprintf(template, "S2", storeDir2, "", 23223, "B", 23223, "B", 11223, "A", 11222))) s2, o2 := RunServerWithConfig(conf2) defer s2.Shutdown() waitForInboundGateways(t, s1, 1, 2*time.Second) waitForInboundGateways(t, s2, 1, 2*time.Second) waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) nc1 := natsConnect(t, fmt.Sprintf("nats://two:password@127.0.0.1:%d", o1.Port)) defer nc1.Close() nc1.Publish("foo", []byte("some message")) nc1.Flush() nc2 := natsConnect(t, fmt.Sprintf("nats://two:password@127.0.0.1:%d", o2.Port)) defer nc2.Close() nc2.Publish("bar", []byte("some message")) nc2.Flush() checkMode := func(accName string, expectedMode GatewayInterestMode) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { servers := []*Server{s1, s2} for _, s := range servers { var gws []*client s.getInboundGatewayConnections(&gws) for _, gw := range gws { var mode GatewayInterestMode gw.mu.Lock() ie := gw.gw.insim[accName] if ie != nil { mode = ie.mode } gw.mu.Unlock() if ie == nil { return fmt.Errorf("Account %q not in map", accName) } if mode != expectedMode { return fmt.Errorf("Expected account %q mode to be %v, got: %v", accName, expectedMode, mode) } } } return nil }) } checkMode("one", InterestOnly) checkMode("two", Optimistic) // Now change account "two" to enable JS changeCurrentConfigContentWithNewContent(t, conf1, []byte(fmt.Sprintf(template, "S1", storeDir1, "jetstream: enabled", 23222, "A", 23222, "A", 11222, "B", 11223))) changeCurrentConfigContentWithNewContent(t, conf2, []byte(fmt.Sprintf(template, "S2", storeDir2, "jetstream: enabled", 23223, "B", 23223, "B", 11223, "A", 11222))) if err := s1.Reload(); err != nil { t.Fatalf("Error on s1 reload: %v", err) } if err := s2.Reload(); err != nil { t.Fatalf("Error on s2 reload: %v", err) } checkMode("one", InterestOnly) checkMode("two", InterestOnly) } func TestJetStreamSuperClusterConnectionCount(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 2) defer sc.shutdown() sysNc := natsConnect(t, sc.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) defer sysNc.Close() _, err := sysNc.Request(fmt.Sprintf(accDirectReqSubj, "ONE", "CONNS"), nil, 100*time.Millisecond) // this is a timeout as the server only responds when it has connections.... // not convinced this should be that way, but also not the issue to investigate. require_True(t, err == nats.ErrTimeout) for i := 1; i <= 2; i++ { func() { nc := natsConnect(t, sc.clusterForName(fmt.Sprintf("C%d", i)).randomServer().ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) name := fmt.Sprintf("foo%d", 1) _, err = js.AddStream(&nats.StreamConfig{ Name: name, Subjects: []string{name}, Replicas: 3}) require_NoError(t, err) }() } func() { nc := natsConnect(t, sc.clusterForName("C1").randomServer().ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "src", Sources: []*nats.StreamSource{{Name: "foo1"}, {Name: "foo2"}}, Replicas: 3}) require_NoError(t, err) }() func() { nc := natsConnect(t, sc.clusterForName("C2").randomServer().ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "mir", Mirror: &nats.StreamSource{Name: "foo2"}, Replicas: 3}) require_NoError(t, err) }() // There should be no active NATS CLIENT connections, but we still need // to wait a little bit... checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { _, err := sysNc.Request(fmt.Sprintf(accDirectReqSubj, "ONE", "CONNS"), nil, 100*time.Millisecond) if err != nats.ErrTimeout { return fmt.Errorf("Expected timeout, got %v", err) } return nil }) sysNc.Close() s := sc.randomServer() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { acc, err := s.lookupAccount("ONE") if err != nil { t.Fatalf("Could not look up account: %v", err) } if n := acc.NumConnections(); n != 0 { return fmt.Errorf("Expected no connections, got %d", n) } return nil }) } func TestJetStreamSuperClusterConsumersBrokenGateways(t *testing.T) { sc := createJetStreamSuperCluster(t, 1, 2) defer sc.shutdown() // Client based API s := sc.clusterForName("C1").randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // This will be in C1. _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Create a stream in C2 that sources TEST _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Placement: &nats.Placement{Cluster: "C2"}, Sources: []*nats.StreamSource{{Name: "TEST"}}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Wait for direct consumer to get registered and detect interest across GW. time.Sleep(time.Second) // Send 100 msgs over 100ms in separate Go routine. msg, toSend, done := []byte("Hello"), 100, make(chan bool) go func() { // Send in 10 messages. for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Errorf("Unexpected publish error: %v", err) } time.Sleep(500 * time.Microsecond) } done <- true }() breakGW := func() { s.gateway.Lock() gw := s.gateway.out["C2"] s.gateway.Unlock() if gw != nil { gw.closeConnection(ClientClosed) } } // Wait til about half way through. time.Sleep(20 * time.Millisecond) // Now break GW connection. breakGW() // Wait for GW to reform. for _, c := range sc.clusters { for _, s := range c.servers { waitForOutboundGateways(t, s, 1, 2*time.Second) } } select { case <-done: case <-time.After(2 * time.Second): t.Fatalf("Did not complete sending first batch of messages") } // Make sure we can deal with data loss at the end. checkFor(t, 20*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 100 { return fmt.Errorf("Expected to have %d messages, got %d", 100, si.State.Msgs) } return nil }) // Now send 100 more. Will aos break here in the middle. for i := 0; i < toSend; i++ { if _, err = js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } if i == 50 { breakGW() } } // Wait for GW to reform. for _, c := range sc.clusters { for _, s := range c.servers { waitForOutboundGateways(t, s, 1, 2*time.Second) } } si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 200 { t.Fatalf("Expected to have %d messages, got %d", 200, si.State.Msgs) } checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { return fmt.Errorf("Unexpected error: %v", err) } if si.State.Msgs != 200 { return fmt.Errorf("Expected to have %d messages, got %d", 200, si.State.Msgs) } return nil }) } func TestJetStreamSuperClusterLeafNodesWithSharedSystemAccountAndSameDomain(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() lnc := sc.createLeafNodes("LNC", 2) defer lnc.shutdown() // We want to make sure there is only one leader and its always in the supercluster. sc.waitOnLeader() if ml := lnc.leader(); ml != nil { t.Fatalf("Detected a meta-leader in the leafnode cluster: %s", ml) } // leafnodes should have been added into the overall peer count. sc.waitOnPeerCount(8) // Check here that we auto detect sharing system account as well and auto place the correct // deny imports and exports. ls := lnc.randomServer() if ls == nil { t.Fatalf("Expected a leafnode server, got none") } gacc := ls.globalAccount().GetName() ls.mu.Lock() var hasDE, hasDI bool for _, ln := range ls.leafs { ln.mu.Lock() if ln.leaf.remote.RemoteLeafOpts.LocalAccount == gacc { re := ln.perms.pub.deny.Match(jsAllAPI) hasDE = len(re.psubs)+len(re.qsubs) > 0 rs := ln.perms.sub.deny.Match(jsAllAPI) hasDI = len(rs.psubs)+len(rs.qsubs) > 0 } ln.mu.Unlock() } ls.mu.Unlock() if !hasDE { t.Fatalf("No deny export on global account") } if !hasDI { t.Fatalf("No deny import on global account") } // Make a stream by connecting to the leafnode cluster. Make sure placement is correct. // Client based API nc, js := jsClientConnect(t, lnc.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != "LNC" { t.Fatalf("Expected default placement to be %q, got %q", "LNC", si.Cluster.Name) } // Now make sure placement also works if we want to place in a cluster in the supercluster. pcn := "C2" si, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"baz"}, Replicas: 2, Placement: &nats.Placement{Cluster: pcn}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != pcn { t.Fatalf("Expected default placement to be %q, got %q", pcn, si.Cluster.Name) } } func TestJetStreamSuperClusterLeafNodesWithSharedSystemAccountAndDifferentDomain(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() lnc := sc.createLeafNodesWithDomain("LNC", 2, "LEAFDOMAIN") defer lnc.shutdown() // We want to make sure there is only one leader and its always in the supercluster. sc.waitOnLeader() lnc.waitOnLeader() // even though system account is shared, because domains differ, sc.waitOnPeerCount(6) lnc.waitOnPeerCount(2) // Check here that we auto detect sharing system account as well and auto place the correct // deny imports and exports. ls := lnc.randomServer() if ls == nil { t.Fatalf("Expected a leafnode server, got none") } gacc := ls.globalAccount().GetName() ls.mu.Lock() var hasDE, hasDI bool for _, ln := range ls.leafs { ln.mu.Lock() if ln.leaf.remote.RemoteLeafOpts.LocalAccount == gacc { re := ln.perms.pub.deny.Match(jsAllAPI) hasDE = len(re.psubs)+len(re.qsubs) > 0 rs := ln.perms.sub.deny.Match(jsAllAPI) hasDI = len(rs.psubs)+len(rs.qsubs) > 0 } ln.mu.Unlock() } ls.mu.Unlock() if !hasDE { t.Fatalf("No deny export on global account") } if !hasDI { t.Fatalf("No deny import on global account") } // Make a stream by connecting to the leafnode cluster. Make sure placement is correct. // Client based API nc, js := jsClientConnect(t, lnc.randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != "LNC" { t.Fatalf("Expected default placement to be %q, got %q", "LNC", si.Cluster.Name) } // Now make sure placement does not works for cluster in different domain pcn := "C2" _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"baz"}, Replicas: 2, Placement: &nats.Placement{Cluster: pcn}, }) if err == nil || !strings.Contains(err.Error(), "no suitable peers for placement") { t.Fatalf("Expected no suitable peers for placement, got: %v", err) } } func TestJetStreamSuperClusterSingleLeafNodeWithSharedSystemAccount(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() ln := sc.createSingleLeafNode(true) defer ln.Shutdown() // We want to make sure there is only one leader and its always in the supercluster. sc.waitOnLeader() // leafnodes should have been added into the overall peer count. sc.waitOnPeerCount(7) // Now make sure we can place a stream in the leaf node. // First connect to the leafnode server itself. nc, js := jsClientConnect(t, ln) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != "LNS" { t.Fatalf("Expected to be placed in leafnode with %q as cluster name, got %q", "LNS", si.Cluster.Name) } // Now check we can place on here as well but connect to the hub. nc, js = jsClientConnect(t, sc.randomServer()) defer nc.Close() si, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"bar"}, Placement: &nats.Placement{Cluster: "LNS"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.Cluster.Name != "LNS" { t.Fatalf("Expected to be placed in leafnode with %q as cluster name, got %q", "LNS", si.Cluster.Name) } } // Issue reported with superclusters and leafnodes where first few get next requests for pull subscribers // have the wrong subject. func TestJetStreamSuperClusterGetNextRewrite(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 2, 2) defer sc.shutdown() // Will connect the leafnode to cluster C1. We will then connect the "client" to cluster C2 to cross gateways. ln := sc.clusterForName("C1").createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain("C", "nojs") defer ln.Shutdown() c2 := sc.clusterForName("C2") nc, js := jsClientConnectEx(t, c2.randomServer(), []nats.JSOpt{nats.Domain("C")}, nats.UserInfo("nojs", "p")) defer nc.Close() // Create a stream and add messages. if _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}); err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 0; i < 10; i++ { if _, err := js.Publish("foo", []byte("ok")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Pull messages and make sure subject rewrite works. sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } for _, m := range fetchMsgs(t, sub, 5, time.Second) { if m.Subject != "foo" { t.Fatalf("Expected %q as subject but got %q", "foo", m.Subject) } } } func TestJetStreamSuperClusterEphemeralCleanup(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() // Create a stream in cluster 0 s := sc.clusters[0].randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() for _, test := range []struct { name string sourceInCluster int streamName string sourceName string }{ {"local", 0, "TEST1", "S1"}, {"remote", 1, "TEST2", "S2"}, } { t.Run(test.name, func(t *testing.T) { if _, err := js.AddStream(&nats.StreamConfig{Name: test.streamName, Replicas: 3}); err != nil { t.Fatalf("Error adding %q stream: %v", test.streamName, err) } if _, err := js.Publish(test.streamName, []byte("hello")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Now create a source for that stream, either in same or remote cluster. s2 := sc.clusters[test.sourceInCluster].randomServer() nc2, js2 := jsClientConnect(t, s2) defer nc2.Close() if _, err := js2.AddStream(&nats.StreamConfig{ Name: test.sourceName, Storage: nats.FileStorage, Sources: []*nats.StreamSource{{Name: test.streamName}}, Replicas: 1, }); err != nil { t.Fatalf("Error adding source stream: %v", err) } // Check that TEST(n) has 1 consumer and that S(n) is created and has 1 message. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo(test.sourceName) if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) // Get the consumer because we will want to artificially reduce // the delete threshold. leader := sc.clusters[0].streamLeader("$G", test.streamName) mset, err := leader.GlobalAccount().lookupStream(test.streamName) if err != nil { t.Fatalf("Expected to find a stream for %q, got %v", test.streamName, err) } cons := mset.getConsumers()[0] cons.mu.Lock() cons.dthresh = 1250 * time.Millisecond active := cons.active dtimerSet := cons.dtmr != nil deliver := cons.cfg.DeliverSubject cons.mu.Unlock() if !active || dtimerSet { t.Fatalf("Invalid values for active=%v dtimerSet=%v", active, dtimerSet) } // To add to the mix, let's create a local interest on the delivery subject // and stop it. This is to ensure that this does not stop timers that should // still be running and monitor the GW interest. sub := natsSubSync(t, nc, deliver) natsFlush(t, nc) natsUnsub(t, sub) natsFlush(t, nc) // Now remove the "S(n)" stream... if err := js2.DeleteStream(test.sourceName); err != nil { t.Fatalf("Error deleting stream: %v", err) } // Now check that the stream S(n) is really removed and that // the consumer is gone for stream TEST(n). checkFor(t, 5*time.Second, 25*time.Millisecond, func() error { // First, make sure that stream S(n) has disappeared. if _, err := js2.StreamInfo(test.sourceName); err == nil { return fmt.Errorf("Stream %q should no longer exist", test.sourceName) } if ndc := mset.numDirectConsumers(); ndc != 0 { return fmt.Errorf("Expected %q stream to have 0 consumers, got %v", test.streamName, ndc) } return nil }) }) } } func TestJetStreamSuperClusterGetNextSubRace(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 2, 2) defer sc.shutdown() // Will connect the leafnode to cluster C1. We will then connect the "client" to cluster C2 to cross gateways. ln := sc.clusterForName("C1").createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain("C", "nojs") defer ln.Shutdown() // Shutdown 1 of the server from C1, (the one LN is not connected to) for _, s := range sc.clusterForName("C1").servers { s.mu.Lock() if len(s.leafs) == 0 { s.mu.Unlock() s.Shutdown() break } s.mu.Unlock() } // Wait on meta leader in case shutdown of server above caused an election. sc.waitOnLeader() var c2Srv *Server // Take the server from C2 that has no inbound from C1. c2 := sc.clusterForName("C2") for _, s := range c2.servers { var gwsa [2]*client gws := gwsa[:0] s.getInboundGatewayConnections(&gws) if len(gws) == 0 { c2Srv = s break } } if c2Srv == nil { t.Fatalf("Both servers in C2 had an inbound GW connection!") } nc, js := jsClientConnectEx(t, c2Srv, []nats.JSOpt{nats.Domain("C")}, nats.UserInfo("nojs", "p")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) require_NoError(t, err) _, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "ok") } // Wait for all messages to appear in the consumer checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { ci, err := js.ConsumerInfo("foo", "dur") if err != nil { return err } if n := ci.NumPending; n != 100 { return fmt.Errorf("Expected 100 msgs, got %v", n) } return nil }) req := &JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second} jreq, err := json.Marshal(req) require_NoError(t, err) // Create this by hand here to make sure we create the subscription // on the reply subject for every single request nextSubj := fmt.Sprintf(JSApiRequestNextT, "foo", "dur") nextSubj = "$JS.C.API" + strings.TrimPrefix(nextSubj, "$JS.API") for i := 0; i < 100; i++ { inbox := nats.NewInbox() sub := natsSubSync(t, nc, inbox) natsPubReq(t, nc, nextSubj, inbox, jreq) msg := natsNexMsg(t, sub, time.Second) if len(msg.Header) != 0 && string(msg.Data) != "ok" { t.Fatalf("Unexpected message: header=%+v data=%s", msg.Header, msg.Data) } sub.Unsubscribe() } } func TestJetStreamSuperClusterPullConsumerAndHeaders(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() c1 := sc.clusterForName("C1") c2 := sc.clusterForName("C2") nc, js := jsClientConnect(t, c1.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "ORIGIN"}); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 50 for i := 0; i < toSend; i++ { if _, err := js.Publish("ORIGIN", []byte("ok")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } nc2, js2 := jsClientConnect(t, c2.randomServer()) defer nc2.Close() _, err := js2.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{Name: "ORIGIN"}}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Wait for them to be in the sourced stream. checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { if si, _ := js2.StreamInfo("S"); si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d msgs for %q, got %d", toSend, "S", si.State.Msgs) } return nil }) // Now create a pull consumer for the sourced stream. _, err = js2.AddConsumer("S", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now we will connect and request the next message from each server in C1 cluster and check that headers remain in place. for _, s := range c1.servers { nc, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer nc.Close() m, err := nc.Request("$JS.API.CONSUMER.MSG.NEXT.S.dlc", nil, 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if len(m.Header) != 1 { t.Fatalf("Expected 1 header element, got %+v", m.Header) } } } func TestJetStreamSuperClusterStatszActiveServers(t *testing.T) { sc := createJetStreamSuperCluster(t, 2, 2) defer sc.shutdown() checkActive := func(expected int) { t.Helper() checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { s := sc.randomServer() nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) if err != nil { t.Fatalf("Failed to create system client: %v", err) } defer nc.Close() resp, err := nc.Request(serverStatsPingReqSubj, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ssm ServerStatsMsg if err := json.Unmarshal(resp.Data, &ssm); err != nil { t.Fatalf("Unexpected error: %v", err) } if ssm.Stats.ActiveServers != expected { return fmt.Errorf("Wanted %d, got %d", expected, ssm.Stats.ActiveServers) } return nil }) } checkActive(4) c := sc.randomCluster() ss := c.randomServer() ss.Shutdown() checkActive(3) c.restartServer(ss) checkActive(4) } func TestJetStreamSuperClusterSourceAndMirrorConsumersLeaderChange(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() c1 := sc.clusterForName("C1") c2 := sc.clusterForName("C2") nc, js := jsClientConnect(t, c1.randomServer()) defer nc.Close() var sources []*nats.StreamSource numStreams := 10 for i := 1; i <= numStreams; i++ { name := fmt.Sprintf("O%d", i) sources = append(sources, &nats.StreamSource{Name: name}) if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Place our new stream that will source all the others in different cluster. nc, js = jsClientConnect(t, c2.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "S", Replicas: 2, Sources: sources, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Force leader change twice. nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "S"), nil, time.Second) c2.waitOnStreamLeader("$G", "S") nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "S"), nil, time.Second) c2.waitOnStreamLeader("$G", "S") // Now make sure we only have a single direct consumer on our origin streams. // Pick one at random. name := fmt.Sprintf("O%d", rand.Intn(numStreams-1)+1) c1.waitOnStreamLeader("$G", name) s := c1.streamLeader("$G", name) a, err := s.lookupAccount("$G") if err != nil { t.Fatalf("Unexpected error: %v", err) } mset, err := a.lookupStream(name) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { if ndc := mset.numDirectConsumers(); ndc != 1 { return fmt.Errorf("Stream %q wanted 1 direct consumer, got %d", name, ndc) } return nil }) // Now create a mirror of selected from above. Will test same scenario. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 2, Mirror: &nats.StreamSource{Name: name}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Force leader change twice. nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "M"), nil, time.Second) c2.waitOnStreamLeader("$G", "M") nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "M"), nil, time.Second) c2.waitOnStreamLeader("$G", "M") checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { if ndc := mset.numDirectConsumers(); ndc != 2 { return fmt.Errorf("Stream %q wanted 2 direct consumers, got %d", name, ndc) } return nil }) } func TestJetStreamSuperClusterPushConsumerInterest(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() for _, test := range []struct { name string queue string }{ {"non queue", _EMPTY_}, {"queue", "queue"}, } { t.Run(test.name, func(t *testing.T) { testInterest := func(s *Server) { t.Helper() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) var sub *nats.Subscription if test.queue != _EMPTY_ { sub, err = js.QueueSubscribeSync("foo", test.queue) } else { sub, err = js.SubscribeSync("foo", nats.Durable("dur")) } require_NoError(t, err) js.Publish("foo", []byte("msg1")) // Since the GW watcher is checking every 1sec, make sure we are // giving it enough time for the delivery to start. _, err = sub.NextMsg(2 * time.Second) require_NoError(t, err) } // Create the durable push consumer from cluster "0" testInterest(sc.clusters[0].servers[0]) // Now "move" to a server in cluster "1" testInterest(sc.clusters[1].servers[0]) }) } } func TestJetStreamSuperClusterOverflowPlacement(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterMaxBytesTempl, 3, 3) defer sc.shutdown() pcn := "C2" s := sc.clusterForName(pcn).randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // With this setup, we opted in for requiring MaxBytes, so this should error. _, err := js.AddStream(&nats.StreamConfig{ Name: "foo", Replicas: 3, }) require_Error(t, err, NewJSStreamMaxBytesRequiredError()) // R=2 on purpose to leave one server empty. _, err = js.AddStream(&nats.StreamConfig{ Name: "foo", Replicas: 2, MaxBytes: 2 * 1024 * 1024 * 1024, }) require_NoError(t, err) // Now try to add another that will overflow the current cluster's reservation. // Since we asked explicitly for the same cluster this should fail. // Note this will not be testing the peer picker since the update has probably not made it to the meta leader. _, err = js.AddStream(&nats.StreamConfig{ Name: "bar", Replicas: 3, MaxBytes: 2 * 1024 * 1024 * 1024, Placement: &nats.Placement{Cluster: pcn}, }) require_Contains(t, err.Error(), "nats: no suitable peers for placement") // Now test actual overflow placement. So try again with no placement designation. // This will test the peer picker's logic since they are updated at this point and the meta leader // knows it can not place it in C2. si, err := js.AddStream(&nats.StreamConfig{ Name: "bar", Replicas: 3, MaxBytes: 2 * 1024 * 1024 * 1024, }) require_NoError(t, err) // Make sure we did not get place into C2. falt := si.Cluster.Name if falt == pcn { t.Fatalf("Expected to be placed in another cluster besides %q, but got %q", pcn, falt) } // One more time that should spill over again to our last cluster. si, err = js.AddStream(&nats.StreamConfig{ Name: "baz", Replicas: 3, MaxBytes: 2 * 1024 * 1024 * 1024, }) require_NoError(t, err) // Make sure we did not get place into C2. if salt := si.Cluster.Name; salt == pcn || salt == falt { t.Fatalf("Expected to be placed in last cluster besides %q or %q, but got %q", pcn, falt, salt) } // Now place a stream of R1 into C2 which should have space. si, err = js.AddStream(&nats.StreamConfig{ Name: "dlc", MaxBytes: 2 * 1024 * 1024 * 1024, }) require_NoError(t, err) if si.Cluster.Name != pcn { t.Fatalf("Expected to be placed in our origin cluster %q, but got %q", pcn, si.Cluster.Name) } } func TestJetStreamSuperClusterConcurrentOverflow(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterMaxBytesTempl, 3, 3) defer sc.shutdown() pcn := "C2" startCh := make(chan bool) var wg sync.WaitGroup var swg sync.WaitGroup start := func(name string) { defer wg.Done() s := sc.clusterForName(pcn).randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() swg.Done() <-startCh _, err := js.AddStream(&nats.StreamConfig{ Name: name, Replicas: 3, MaxBytes: 2 * 1024 * 1024 * 1024, }) require_NoError(t, err) } wg.Add(2) swg.Add(2) go start("foo") go start("bar") swg.Wait() // Now start both at same time. close(startCh) wg.Wait() } func TestJetStreamSuperClusterStreamTagPlacement(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() placeOK := func(connectCluster string, tags []string, expectedCluster string) { t.Helper() nc, js := jsClientConnect(t, sc.clusterForName(connectCluster).randomServer()) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Placement: &nats.Placement{Tags: tags}, }) require_NoError(t, err) if si.Cluster.Name != expectedCluster { t.Fatalf("Failed to place properly in %q, got %q", expectedCluster, si.Cluster.Name) } js.DeleteStream("TEST") } placeOK("C2", []string{"cloud:aws"}, "C1") placeOK("C2", []string{"country:jp"}, "C3") placeOK("C1", []string{"cloud:gcp", "country:uk"}, "C2") // Case shoud not matter. placeOK("C1", []string{"cloud:GCP", "country:UK"}, "C2") placeOK("C2", []string{"Cloud:AwS", "Country:uS"}, "C1") placeErr := func(connectCluster string, tags []string) { t.Helper() nc, js := jsClientConnect(t, sc.clusterForName(connectCluster).randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Placement: &nats.Placement{Tags: tags}, }) require_Contains(t, err.Error(), "no suitable peers for placement", "tags not matched") require_Contains(t, err.Error(), tags...) } placeErr("C1", []string{"cloud:GCP", "country:US"}) placeErr("C1", []string{"country:DN"}) placeErr("C1", []string{"cloud:DO"}) } func TestJetStreamSuperClusterRemovedPeersAndStreamsListAndDelete(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() pcn := "C2" sc.waitOnLeader() ml := sc.leader() if ml.ClusterName() == pcn { pcn = "C1" } // Client based API nc, js := jsClientConnect(t, ml) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "GONE", Replicas: 3, Placement: &nats.Placement{Cluster: pcn}, }) require_NoError(t, err) _, err = js.AddConsumer("GONE", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Placement: &nats.Placement{Cluster: ml.ClusterName()}, }) require_NoError(t, err) // Put messages in.. num := 100 for i := 0; i < num; i++ { js.PublishAsync("GONE", []byte("SLS")) js.PublishAsync("TEST", []byte("SLS")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } c := sc.clusterForName(pcn) c.shutdown() // Grab Stream List.. start := time.Now() resp, err := nc.Request(JSApiStreamList, nil, 2*time.Second) require_NoError(t, err) if delta := time.Since(start); delta > 100*time.Millisecond { t.Fatalf("Stream list call took too long to return: %v", delta) } var list JSApiStreamListResponse err = json.Unmarshal(resp.Data, &list) require_NoError(t, err) if len(list.Missing) != 1 || list.Missing[0] != "GONE" { t.Fatalf("Wrong Missing: %+v", list) } // Check behavior of stream info as well. We want it to return the stream is offline and not just timeout. _, err = js.StreamInfo("GONE") // FIXME(dlc) - Go client not putting nats: prefix on for stream but does for consumer. require_Error(t, err, NewJSStreamOfflineError(), errors.New("nats: stream is offline")) // Same for Consumer start = time.Now() resp, err = nc.Request("$JS.API.CONSUMER.LIST.GONE", nil, 2*time.Second) require_NoError(t, err) if delta := time.Since(start); delta > 100*time.Millisecond { t.Fatalf("Consumer list call took too long to return: %v", delta) } var clist JSApiConsumerListResponse err = json.Unmarshal(resp.Data, &clist) require_NoError(t, err) if len(clist.Missing) != 1 || clist.Missing[0] != "dlc" { t.Fatalf("Wrong Missing: %+v", clist) } _, err = js.ConsumerInfo("GONE", "dlc") require_Error(t, err, NewJSConsumerOfflineError(), errors.New("nats: consumer is offline")) // Make sure delete works. err = js.DeleteConsumer("GONE", "dlc") require_NoError(t, err) err = js.DeleteStream("GONE") require_NoError(t, err) // Test it is really gone. _, err = js.StreamInfo("GONE") require_Error(t, err, nats.ErrStreamNotFound) } func TestJetStreamSuperClusterConsumerDeliverNewBug(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() pcn := "C2" sc.waitOnLeader() ml := sc.leader() if ml.ClusterName() == pcn { pcn = "C1" } // Client based API nc, js := jsClientConnect(t, ml) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "T", Replicas: 3, Placement: &nats.Placement{Cluster: pcn}, }) require_NoError(t, err) // Put messages in.. num := 100 for i := 0; i < num; i++ { js.PublishAsync("T", []byte("OK")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } ci, err := js.AddConsumer("T", &nats.ConsumerConfig{ Durable: "d", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, }) require_NoError(t, err) if ci.Delivered.Consumer != 0 || ci.Delivered.Stream != 100 { t.Fatalf("Incorrect consumer delivered info: %+v", ci.Delivered) } c := sc.clusterForName(pcn) for _, s := range c.servers { sd := s.JetStreamConfig().StoreDir s.Shutdown() removeDir(t, sd) s = c.restartServer(s) c.waitOnServerHealthz(s) c.waitOnConsumerLeader("$G", "T", "d") } c.waitOnConsumerLeader("$G", "T", "d") cl := c.consumerLeader(globalAccountName, "T", "d") mset, err := cl.GlobalAccount().lookupStream("T") require_NoError(t, err) o := mset.lookupConsumer("d") require_NotNil(t, o) o.mu.RLock() defer o.mu.RUnlock() if o.dseq-1 != 0 || o.sseq-1 != 100 { t.Fatalf("Incorrect consumer delivered info: dseq=%d, sseq=%d", o.dseq-1, o.sseq-1) } if np := o.checkNumPending(); np != 0 { t.Fatalf("Did not expect NumPending, got %d", np) } } // This will test our ability to move streams and consumers between clusters. func TestJetStreamSuperClusterMovingStreamsAndConsumers(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() for _, test := range []struct { name string replicas int }{ {"R1", 1}, {"R3", 3}, } { t.Run(test.name, func(t *testing.T) { replicas := test.replicas si, err := js.AddStream(&nats.StreamConfig{ Name: "MOVE", Replicas: replicas, Placement: &nats.Placement{Tags: []string{"cloud:aws"}}, }) require_NoError(t, err) defer js.DeleteStream("MOVE") if si.Cluster.Name != "C1" { t.Fatalf("Failed to place properly in %q, got %q", "C1", si.Cluster.Name) } for i := 0; i < 1000; i++ { _, err := js.PublishAsync("MOVE", []byte("Moving on up")) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Durable Push Consumer, so same R. dpushSub, err := js.SubscribeSync("MOVE", nats.Durable("dlc")) require_NoError(t, err) defer dpushSub.Unsubscribe() // Ephemeral Push Consumer, R1. epushSub, err := js.SubscribeSync("MOVE") require_NoError(t, err) defer epushSub.Unsubscribe() // Durable Pull Consumer, so same R. dpullSub, err := js.PullSubscribe("MOVE", "dlc-pull") require_NoError(t, err) defer dpullSub.Unsubscribe() // TODO(dlc) - Server supports ephemeral pulls but Go client does not yet. si, err = js.StreamInfo("MOVE") require_NoError(t, err) if si.State.Consumers != 3 { t.Fatalf("Expected 3 attached consumers, got %d", si.State.Consumers) } initialState := si.State checkSubsPending(t, dpushSub, int(initialState.Msgs)) checkSubsPending(t, epushSub, int(initialState.Msgs)) // Ack 100 toAck := 100 for i := 0; i < toAck; i++ { m, err := dpushSub.NextMsg(time.Second) require_NoError(t, err) m.AckSync() // Ephemeral m, err = epushSub.NextMsg(time.Second) require_NoError(t, err) m.AckSync() } // Do same with pull subscriber. for _, m := range fetchMsgs(t, dpullSub, toAck, 5*time.Second) { m.AckSync() } // First make sure we disallow move and replica changes in same update. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "MOVE", Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, Replicas: replicas + 1, }) require_Error(t, err, NewJSStreamMoveAndScaleError()) // Now move to new cluster. si, err = js.UpdateStream(&nats.StreamConfig{ Name: "MOVE", Replicas: replicas, Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, }) require_NoError(t, err) if si.Cluster.Name != "C1" { t.Fatalf("Expected cluster of %q but got %q", "C1", si.Cluster.Name) } // Make sure we can not move an inflight stream and consumers, should error. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "MOVE", Replicas: replicas, Placement: &nats.Placement{Tags: []string{"cloud:aws"}}, }) require_Contains(t, err.Error(), "stream move already in progress") checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("MOVE", nats.MaxWait(500*time.Millisecond)) if err != nil { return err } // We should see 2X peers. numPeers := len(si.Cluster.Replicas) if si.Cluster.Leader != _EMPTY_ { numPeers++ } if numPeers != 2*replicas { // The move can happen very quick now, so we might already be done. if si.Cluster.Name == "C2" { return nil } return fmt.Errorf("Expected to see %d replicas, got %d", 2*replicas, numPeers) } return nil }) // Expect a new leader to emerge and replicas to drop as a leader is elected. // We have to check fast or it might complete and we will not see intermediate steps. sc.waitOnStreamLeader("$G", "MOVE") checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("MOVE", nats.MaxWait(500*time.Millisecond)) if err != nil { return err } if len(si.Cluster.Replicas) >= 2*replicas { return fmt.Errorf("Expected <%d replicas, got %d", 2*replicas, len(si.Cluster.Replicas)) } return nil }) // Should see the cluster designation and leader switch to C2. // We should also shrink back down to original replica count. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { si, err := js.StreamInfo("MOVE", nats.MaxWait(500*time.Millisecond)) if err != nil { return err } if si.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } // Now we want to see that we shrink back to original. if len(si.Cluster.Replicas) != replicas-1 { return fmt.Errorf("Expected %d replicas, got %d", replicas-1, len(si.Cluster.Replicas)) } return nil }) // Check moved state is same as initial state. si, err = js.StreamInfo("MOVE") require_NoError(t, err) if !reflect.DeepEqual(si.State, initialState) { t.Fatalf("States do not match after migration:\n%+v\nvs\n%+v", si.State, initialState) } // Make sure we can still send messages. addN := toAck for i := 0; i < addN; i++ { _, err := js.Publish("MOVE", []byte("Done Moved")) require_NoError(t, err) } si, err = js.StreamInfo("MOVE") require_NoError(t, err) expectedPushMsgs := initialState.Msgs + uint64(addN) expectedPullMsgs := uint64(addN) if si.State.Msgs != expectedPushMsgs { t.Fatalf("Expected to be able to send new messages") } // Now check consumers, make sure the state is correct and that they transferred state and reflect the new messages. // We Ack'd 100 and sent another 100, so should be same. checkConsumer := func(sub *nats.Subscription, isPull bool) { t.Helper() checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { ci, err := sub.ConsumerInfo() if err != nil { return err } var expectedMsgs uint64 if isPull { expectedMsgs = expectedPullMsgs } else { expectedMsgs = expectedPushMsgs } if ci.Delivered.Consumer != expectedMsgs || ci.Delivered.Stream != expectedMsgs { return fmt.Errorf("Delivered for %q is not correct: %+v", ci.Name, ci.Delivered) } if ci.AckFloor.Consumer != uint64(toAck) || ci.AckFloor.Stream != uint64(toAck) { return fmt.Errorf("AckFloor for %q is not correct: %+v", ci.Name, ci.AckFloor) } if isPull && ci.NumAckPending != 0 { return fmt.Errorf("NumAckPending for %q is not correct: %v", ci.Name, ci.NumAckPending) } else if !isPull && ci.NumAckPending != int(initialState.Msgs) { return fmt.Errorf("NumAckPending for %q is not correct: %v", ci.Name, ci.NumAckPending) } // Make sure the replicas etc are back to what is expected. si, err := js.StreamInfo("MOVE") if err != nil { return err } numExpected := si.Config.Replicas if ci.Config.Durable == _EMPTY_ { numExpected = 1 } numPeers := len(ci.Cluster.Replicas) if ci.Cluster.Leader != _EMPTY_ { numPeers++ } if numPeers != numExpected { return fmt.Errorf("Expected %d peers, got %d", numExpected, numPeers) } // If we are push check sub pending. if !isPull { checkSubsPending(t, sub, int(expectedPushMsgs)-toAck) } return nil }) } checkPushConsumer := func(sub *nats.Subscription) { t.Helper() checkConsumer(sub, false) } checkPullConsumer := func(sub *nats.Subscription) { t.Helper() checkConsumer(sub, true) } checkPushConsumer(dpushSub) checkPushConsumer(epushSub) checkPullConsumer(dpullSub) // Cleanup err = js.DeleteStream("MOVE") require_NoError(t, err) }) } } func TestJetStreamSuperClusterMovingStreamsWithMirror(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "SOURCE", Subjects: []string{"foo", "bar"}, Replicas: 3, Placement: &nats.Placement{Tags: []string{"cloud:aws"}}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR", Replicas: 1, Mirror: &nats.StreamSource{Name: "SOURCE"}, Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, }) require_NoError(t, err) done := make(chan struct{}) exited := make(chan struct{}) errors := make(chan error, 1) numNoResp := uint64(0) // We will run a separate routine and send at 100hz go func() { nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() defer close(exited) for { select { case <-done: return case <-time.After(10 * time.Millisecond): _, err := js.Publish("foo", []byte("100HZ")) if err == nil { } else if err == nats.ErrNoStreamResponse { atomic.AddUint64(&numNoResp, 1) continue } if err != nil { errors <- err return } } } }() // Let it get going. time.Sleep(1500 * time.Millisecond) // Now move the source to a new cluster. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "SOURCE", Subjects: []string{"foo", "bar"}, Replicas: 3, Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, }) require_NoError(t, err) checkFor(t, 30*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("SOURCE") if err != nil { return err } if si.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } // Now we want to see that we shrink back to original. if len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Expected %d replicas, got %d", 2, len(si.Cluster.Replicas)) } // Let's get to 50+ msgs. if si.State.Msgs < 50 { return fmt.Errorf("Only see %d msgs", si.State.Msgs) } return nil }) close(done) <-exited if nnr := atomic.LoadUint64(&numNoResp); nnr > 0 { if nnr > 5 { t.Fatalf("Expected no or very few failed message publishes, got %d", nnr) } else { t.Logf("Got a few failed publishes: %d", nnr) } } checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("SOURCE") require_NoError(t, err) mi, err := js.StreamInfo("MIRROR") require_NoError(t, err) if !reflect.DeepEqual(si.State, mi.State) { return fmt.Errorf("Expected mirror to be the same, got %+v vs %+v", mi.State, si.State) } return nil }) } func TestJetStreamSuperClusterMovingStreamAndMoveBack(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() for _, test := range []struct { name string replicas int }{ {"R1", 1}, {"R3", 3}, } { t.Run(test.name, func(t *testing.T) { js.DeleteStream("TEST") _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: test.replicas, Placement: &nats.Placement{Tags: []string{"cloud:aws"}}, }) require_NoError(t, err) toSend := 10_000 for i := 0; i < toSend; i++ { _, err := js.Publish("TEST", []byte("HELLO WORLD")) require_NoError(t, err) } _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Replicas: test.replicas, Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, }) require_NoError(t, err) checkMove := func(cluster string) { t.Helper() sc.waitOnStreamLeader("$G", "TEST") checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return err } if si.Cluster.Name != cluster { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } else if !strings.HasPrefix(si.Cluster.Leader, cluster) { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } // Now we want to see that we shrink back to original. if len(si.Cluster.Replicas) != test.replicas-1 { return fmt.Errorf("Expected %d replicas, got %d", test.replicas-1, len(si.Cluster.Replicas)) } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Only see %d msgs", si.State.Msgs) } return nil }) } checkMove("C2") // The move could be completed when looking at the stream info, but the meta leader could still // deny move updates for a short time while state is cleaned up. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Replicas: test.replicas, Placement: &nats.Placement{Tags: []string{"cloud:aws"}}, }) return err }) checkMove("C1") }) } } func TestJetStreamSuperClusterImportConsumerStreamSubjectRemap(t *testing.T) { template := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: HUB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts: { JS: { jetstream: enabled users: [ {user: js, password: pwd} ] exports [ # This is streaming to a delivery subject for a push based consumer. { stream: "deliver.ORDERS.*" } # This is to ack received messages. This is a service to support sync ack. { service: "$JS.ACK.ORDERS.*.>" } # To support ordered consumers, flow control. { service: "$JS.FC.>" } ] }, IM: { users: [ {user: im, password: pwd} ] imports [ { stream: { account: JS, subject: "deliver.ORDERS.route" }} { stream: { account: JS, subject: "deliver.ORDERS.gateway" }} { stream: { account: JS, subject: "deliver.ORDERS.leaf1" }} { stream: { account: JS, subject: "deliver.ORDERS.leaf2" }} { service: {account: JS, subject: "$JS.FC.>" }} ] }, $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }, } leaf { listen: 127.0.0.1:-1 }` test := func(t *testing.T, queue bool) { c := createJetStreamSuperClusterWithTemplate(t, template, 3, 2) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s, nats.UserInfo("js", "pwd")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"foo"}, // The JS subject. Replicas: 3, Placement: &nats.Placement{Cluster: "C1"}, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("OK")) require_NoError(t, err) for dur, deliver := range map[string]string{ "dur-route": "deliver.ORDERS.route", "dur-gateway": "deliver.ORDERS.gateway", "dur-leaf-1": "deliver.ORDERS.leaf1", "dur-leaf-2": "deliver.ORDERS.leaf2", } { cfg := &nats.ConsumerConfig{ Durable: dur, DeliverSubject: deliver, AckPolicy: nats.AckExplicitPolicy, } if queue { cfg.DeliverGroup = "queue" } _, err = js.AddConsumer("ORDERS", cfg) require_NoError(t, err) } testCase := func(t *testing.T, s *Server, dSubj string) { nc2, err := nats.Connect(s.ClientURL(), nats.UserInfo("im", "pwd")) require_NoError(t, err) defer nc2.Close() var sub *nats.Subscription if queue { sub, err = nc2.QueueSubscribeSync(dSubj, "queue") } else { sub, err = nc2.SubscribeSync(dSubj) } require_NoError(t, err) m, err := sub.NextMsg(time.Second) require_NoError(t, err) if m.Subject != "foo" { t.Fatalf("Subject not mapped correctly across account boundary, expected %q got %q", "foo", m.Subject) } require_False(t, strings.Contains(m.Reply, "@")) } t.Run("route", func(t *testing.T) { // pick random non consumer leader so we receive via route s := c.clusterForName("C1").randomNonConsumerLeader("JS", "ORDERS", "dur-route") testCase(t, s, "deliver.ORDERS.route") }) t.Run("gateway", func(t *testing.T) { // pick server with inbound gateway from consumer leader, so we receive from gateway and have no route in between scl := c.clusterForName("C1").consumerLeader("JS", "ORDERS", "dur-gateway") var sfound *Server for _, s := range c.clusterForName("C2").servers { s.mu.Lock() for _, c := range s.gateway.in { if c.GetName() == scl.info.ID { sfound = s break } } s.mu.Unlock() if sfound != nil { break } } testCase(t, sfound, "deliver.ORDERS.gateway") }) t.Run("leaf-post-export", func(t *testing.T) { // create leaf node server connected post export/import scl := c.clusterForName("C1").consumerLeader("JS", "ORDERS", "dur-leaf-1") cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { remotes [ { url: "nats://im:pwd@127.0.0.1:%d" } ] } authorization: { user: im, password: pwd } `, scl.getOpts().LeafNode.Port))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() checkLeafNodeConnected(t, scl) testCase(t, s, "deliver.ORDERS.leaf1") }) t.Run("leaf-pre-export", func(t *testing.T) { // create leaf node server connected pre export, perform export/import on leaf node server scl := c.clusterForName("C1").consumerLeader("JS", "ORDERS", "dur-leaf-2") cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { remotes [ { url: "nats://js:pwd@127.0.0.1:%d", account: JS2 } ] } accounts: { JS2: { users: [ {user: js, password: pwd} ] exports [ # This is streaming to a delivery subject for a push based consumer. { stream: "deliver.ORDERS.leaf2" } # This is to ack received messages. This is a service to support sync ack. { service: "$JS.ACK.ORDERS.*.>" } # To support ordered consumers, flow control. { service: "$JS.FC.>" } ] }, IM2: { users: [ {user: im, password: pwd} ] imports [ { stream: { account: JS2, subject: "deliver.ORDERS.leaf2" }} { service: {account: JS2, subject: "$JS.FC.>" }} ] }, } `, scl.getOpts().LeafNode.Port))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() checkLeafNodeConnected(t, scl) testCase(t, s, "deliver.ORDERS.leaf2") }) } t.Run("noQueue", func(t *testing.T) { test(t, false) }) t.Run("queue", func(t *testing.T) { test(t, true) }) } func TestJetStreamSuperClusterMaxHaAssets(t *testing.T) { sc := createJetStreamSuperClusterWithTemplateAndModHook(t, ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', limits: {max_ha_assets: 2}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } `, 3, 2, func(serverName, clusterName, storeDir, conf string) string { return conf }, nil) defer sc.shutdown() // speed up statsz reporting for _, c := range sc.clusters { for _, s := range c.servers { s.mu.Lock() s.sys.statsz = 10 * time.Millisecond s.sys.cstatsz = s.sys.statsz s.sys.stmr.Reset(s.sys.statsz) s.mu.Unlock() } } nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() ncSys := natsConnect(t, sc.randomServer().ClientURL(), nats.UserInfo("admin", "s3cr3t!")) defer ncSys.Close() statszSub, err := ncSys.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) require_NoError(t, err) require_NoError(t, ncSys.Flush()) waitStatsz := func(peers, haassets int) { t.Helper() for peersWithExactHaAssets := 0; peersWithExactHaAssets < peers; { m, err := statszSub.NextMsg(time.Second) require_NoError(t, err) var statsz ServerStatsMsg err = json.Unmarshal(m.Data, &statsz) require_NoError(t, err) if statsz.Stats.JetStream == nil { continue } if haassets == statsz.Stats.JetStream.Stats.HAAssets { peersWithExactHaAssets++ } } } waitStatsz(6, 1) // counts _meta_ _, err = js.AddStream(&nats.StreamConfig{Name: "S0", Replicas: 1, Placement: &nats.Placement{Cluster: "C1"}}) require_NoError(t, err) waitStatsz(6, 1) _, err = js.AddStream(&nats.StreamConfig{Name: "S1", Replicas: 3, Placement: &nats.Placement{Cluster: "C1"}}) require_NoError(t, err) waitStatsz(3, 2) waitStatsz(3, 1) _, err = js.AddStream(&nats.StreamConfig{Name: "S2", Replicas: 3, Placement: &nats.Placement{Cluster: "C1"}}) require_NoError(t, err) waitStatsz(3, 3) waitStatsz(3, 1) _, err = js.AddStream(&nats.StreamConfig{Name: "S3", Replicas: 3, Placement: &nats.Placement{Cluster: "C1"}}) require_Error(t, err) require_Contains(t, err.Error(), "nats: no suitable peers for placement") require_Contains(t, err.Error(), "miscellaneous issue") require_NoError(t, js.DeleteStream("S1")) waitStatsz(3, 2) waitStatsz(3, 1) _, err = js.AddConsumer("S2", &nats.ConsumerConfig{Durable: "DUR1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) waitStatsz(3, 3) waitStatsz(3, 1) _, err = js.AddConsumer("S2", &nats.ConsumerConfig{Durable: "DUR2", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: insufficient resources") _, err = js.AddConsumer("S2", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) waitStatsz(3, 3) waitStatsz(3, 1) _, err = js.UpdateStream(&nats.StreamConfig{Name: "S2", Replicas: 3, Description: "foobar"}) require_NoError(t, err) waitStatsz(3, 3) waitStatsz(3, 1) si, err := js.AddStream(&nats.StreamConfig{Name: "S4", Replicas: 3}) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "C2") waitStatsz(3, 3) waitStatsz(3, 2) si, err = js.AddStream(&nats.StreamConfig{Name: "S5", Replicas: 3}) require_NoError(t, err) require_Equal(t, si.Cluster.Name, "C2") waitStatsz(6, 3) _, err = js.AddConsumer("S4", &nats.ConsumerConfig{Durable: "DUR2", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: insufficient resources") _, err = js.UpdateStream(&nats.StreamConfig{Name: "S2", Replicas: 3, Placement: &nats.Placement{Cluster: "C2"}}) require_Error(t, err) require_Contains(t, err.Error(), "nats: no suitable peers for placement") require_Contains(t, err.Error(), "miscellaneous issue") } func TestJetStreamSuperClusterStreamAlternates(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() // C1 _, err := js.AddStream(&nats.StreamConfig{ Name: "SOURCE", Subjects: []string{"foo", "bar", "baz"}, Replicas: 3, Placement: &nats.Placement{Tags: []string{"cloud:aws", "country:us"}}, }) require_NoError(t, err) // C2 _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR-1", Replicas: 1, Mirror: &nats.StreamSource{Name: "SOURCE"}, Placement: &nats.Placement{Tags: []string{"cloud:gcp", "country:uk"}}, }) require_NoError(t, err) // C3 _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR-2", Replicas: 2, Mirror: &nats.StreamSource{Name: "SOURCE"}, Placement: &nats.Placement{Tags: []string{"cloud:az", "country:jp"}}, }) require_NoError(t, err) // No client support yet, so do by hand. getStreamInfo := func(nc *nats.Conn, expected string) { t.Helper() resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "SOURCE"), nil, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) require_True(t, len(si.Alternates) == 3) require_True(t, si.Alternates[0].Cluster == expected) seen := make(map[string]struct{}) for _, alt := range si.Alternates { seen[alt.Cluster] = struct{}{} } require_True(t, len(seen) == 3) } // Connect to different clusters to check ordering. nc, _ = jsClientConnect(t, sc.clusterForName("C1").randomServer()) defer nc.Close() getStreamInfo(nc, "C1") nc, _ = jsClientConnect(t, sc.clusterForName("C2").randomServer()) defer nc.Close() getStreamInfo(nc, "C2") nc, _ = jsClientConnect(t, sc.clusterForName("C3").randomServer()) defer nc.Close() getStreamInfo(nc, "C3") } // We had a scenario where a consumer would not recover properly on restart due to // the cluster state not being set properly when checking source subjects. func TestJetStreamSuperClusterStateOnRestartPreventsConsumerRecovery(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() // C1 _, err := js.AddStream(&nats.StreamConfig{ Name: "SOURCE", Subjects: []string{"foo", "bar"}, Replicas: 3, Placement: &nats.Placement{Tags: []string{"cloud:aws", "country:us"}}, }) require_NoError(t, err) // C2 _, err = js.AddStream(&nats.StreamConfig{ Name: "DS", Subjects: []string{"baz"}, Replicas: 3, Sources: []*nats.StreamSource{{Name: "SOURCE"}}, Placement: &nats.Placement{Tags: []string{"cloud:gcp", "country:uk"}}, }) require_NoError(t, err) // Bind to DS and match filter subject of SOURCE. _, err = js.AddConsumer("DS", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", DeliverSubject: "d", }) require_NoError(t, err) // Send a few messages. for i := 0; i < 100; i++ { _, err := js.Publish("foo", []byte("HELLO")) require_NoError(t, err) } sub := natsSubSync(t, nc, "d") natsNexMsg(t, sub, 5*time.Second) c := sc.clusterForName("C2") cl := c.consumerLeader("$G", "DS", "dlc") // Pull source out from underneath the downstream stream. err = js.DeleteStream("SOURCE") require_NoError(t, err) cl.Shutdown() cl = c.restartServer(cl) c.waitOnServerHealthz(cl) // Now make sure the consumer is still on this server and has restarted properly. mset, err := cl.GlobalAccount().lookupStream("DS") require_NoError(t, err) if o := mset.lookupConsumer("dlc"); o == nil { t.Fatalf("Consumer was not properly restarted") } } // We allow mirrors to opt-in to direct get in a distributed queue group. func TestJetStreamSuperClusterStreamDirectGetMirrorQueueGroup(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() // C1 // Do by hand for now. cfg := &StreamConfig{ Name: "SOURCE", Subjects: []string{"kv.>"}, MaxMsgsPer: 1, Placement: &Placement{Tags: []string{"cloud:aws", "country:us"}}, AllowDirect: true, Replicas: 3, Storage: MemoryStorage, } addStream(t, nc, cfg) num := 100 for i := 0; i < num; i++ { js.PublishAsync(fmt.Sprintf("kv.%d", i), []byte("VAL")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // C2 cfg = &StreamConfig{ Name: "M1", Mirror: &StreamSource{Name: "SOURCE"}, Placement: &Placement{Tags: []string{"cloud:gcp", "country:uk"}}, MirrorDirect: true, Storage: MemoryStorage, } addStream(t, nc, cfg) // C3 (clustered) cfg = &StreamConfig{ Name: "M2", Mirror: &StreamSource{Name: "SOURCE"}, Replicas: 3, Placement: &Placement{Tags: []string{"country:jp"}}, MirrorDirect: true, Storage: MemoryStorage, } addStream(t, nc, cfg) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("M2") require_NoError(t, err) if si.State.Msgs != uint64(num) { return fmt.Errorf("Expected %d msgs, got state: %d", num, si.State.Msgs) } return nil }) // Since last one was an R3, check and wait for the direct subscription. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { sl := sc.clusterForName("C3").streamLeader("$G", "M2") if mset, err := sl.GlobalAccount().lookupStream("M2"); err == nil { mset.mu.RLock() ok := mset.mirror.dsub != nil mset.mu.RUnlock() if ok { return nil } } return fmt.Errorf("No dsub yet") }) // Always do a direct get to the source, but check that we are getting answers from the mirrors when connected to their cluster. getSubj := fmt.Sprintf(JSDirectMsgGetT, "SOURCE") req := []byte(`{"last_by_subj":"kv.22"}`) getMsg := func(c *nats.Conn) *nats.Msg { m, err := c.Request(getSubj, req, time.Second) require_NoError(t, err) require_True(t, string(m.Data) == "VAL") require_True(t, m.Header.Get(JSSequence) == "23") require_True(t, m.Header.Get(JSSubject) == "kv.22") return m } // C1 -> SOURCE nc, _ = jsClientConnect(t, sc.clusterForName("C1").randomServer()) defer nc.Close() m := getMsg(nc) require_True(t, m.Header.Get(JSStream) == "SOURCE") // C2 -> M1 nc, _ = jsClientConnect(t, sc.clusterForName("C2").randomServer()) defer nc.Close() m = getMsg(nc) require_True(t, m.Header.Get(JSStream) == "M1") // C3 -> M2 nc, _ = jsClientConnect(t, sc.clusterForName("C3").randomServer()) defer nc.Close() m = getMsg(nc) require_True(t, m.Header.Get(JSStream) == "M2") } func TestJetStreamSuperClusterTagInducedMoveCancel(t *testing.T) { server := map[string]struct{}{} sc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2, func(serverName, clusterName, storeDir, conf string) string { server[serverName] = struct{}{} return fmt.Sprintf("%s\nserver_tags: [%s]", conf, clusterName) }, nil) defer sc.shutdown() // Client based API c := sc.randomCluster() srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Placement: &nats.Placement{Tags: []string{"C1"}}, Replicas: 3, } siCreate, err := js.AddStream(cfg) require_NoError(t, err) require_Equal(t, siCreate.Cluster.Name, "C1") toSend := uint64(1_000) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } ncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer ncsys.Close() // cause a move by altering placement tags cfg.Placement.Tags = []string{"C2"} _, err = js.UpdateStream(cfg) require_NoError(t, err) rmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamCancelMoveT, "$G", "TEST"), nil, 5*time.Second) require_NoError(t, err) var cancelResp JSApiStreamUpdateResponse require_NoError(t, json.Unmarshal(rmsg.Data, &cancelResp)) if cancelResp.Error != nil && ErrorIdentifier(cancelResp.Error.ErrCode) == JSStreamMoveNotInProgress { t.Skip("This can happen with delays, when Move completed before Cancel", cancelResp.Error) return } require_True(t, cancelResp.Error == nil) checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.Config.Placement != nil { return fmt.Errorf("expected placement to be cleared got: %+v", si.Config.Placement) } return nil }) } func TestJetStreamSuperClusterMoveCancel(t *testing.T) { usageTickOld := usageTick usageTick = 250 * time.Millisecond defer func() { usageTick = usageTickOld }() server := map[string]struct{}{} sc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2, func(serverName, clusterName, storeDir, conf string) string { server[serverName] = struct{}{} return fmt.Sprintf("%s\nserver_tags: [%s]", conf, serverName) }, nil) defer sc.shutdown() // Client based API c := sc.randomCluster() srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() siCreate, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) streamPeerSrv := []string{siCreate.Cluster.Leader, siCreate.Cluster.Replicas[0].Name, siCreate.Cluster.Replicas[1].Name} // determine empty server for _, s := range streamPeerSrv { delete(server, s) } // pick left over server in same cluster as other server emptySrv := _EMPTY_ for s := range server { // server name is prefixed with cluster name if strings.HasPrefix(s, c.name) { emptySrv = s break } } expectedPeers := map[string]struct{}{ getHash(streamPeerSrv[0]): {}, getHash(streamPeerSrv[1]): {}, getHash(streamPeerSrv[2]): {}, } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "DUR", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{InactiveThreshold: time.Hour, AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ephName := ci.Name toSend := uint64(1_000) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } serverEmpty := func(fromSrv string) error { if jszAfter, err := c.serverByName(fromSrv).Jsz(nil); err != nil { return fmt.Errorf("could not fetch JS info for server: %v", err) } else if jszAfter.Streams != 0 { return fmt.Errorf("empty server still has %d streams", jszAfter.Streams) } else if jszAfter.Consumers != 0 { return fmt.Errorf("empty server still has %d consumers", jszAfter.Consumers) } else if jszAfter.Bytes != 0 { return fmt.Errorf("empty server still has %d storage", jszAfter.Store) } return nil } checkSrvInvariant := func(s *Server, expectedPeers map[string]struct{}) error { js, cc := s.getJetStreamCluster() js.mu.Lock() defer js.mu.Unlock() if sa, ok := cc.streams["$G"]["TEST"]; !ok { return fmt.Errorf("stream not found") } else if len(sa.Group.Peers) != len(expectedPeers) { return fmt.Errorf("stream peer group size not %d, but %d", len(expectedPeers), len(sa.Group.Peers)) } else if da, ok := sa.consumers["DUR"]; !ok { return fmt.Errorf("durable not found") } else if len(da.Group.Peers) != len(expectedPeers) { return fmt.Errorf("durable peer group size not %d, but %d", len(expectedPeers), len(da.Group.Peers)) } else if ea, ok := sa.consumers[ephName]; !ok { return fmt.Errorf("ephemeral not found") } else if len(ea.Group.Peers) != 1 { return fmt.Errorf("ephemeral peer group size not 1, but %d", len(ea.Group.Peers)) } else if _, ok := expectedPeers[ea.Group.Peers[0]]; !ok { return fmt.Errorf("ephemeral peer not an expected peer") } else { for _, p := range sa.Group.Peers { if _, ok := expectedPeers[p]; !ok { return fmt.Errorf("peer not expected") } found := false for _, dp := range da.Group.Peers { if p == dp { found = true break } } if !found { t.Logf("durable peer group does not match stream peer group") } } } return nil } ncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer ncsys.Close() time.Sleep(2 * usageTick) aiBefore, err := js.AccountInfo() require_NoError(t, err) for _, moveFromSrv := range streamPeerSrv { moveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{Server: moveFromSrv, Tags: []string{emptySrv}}) require_NoError(t, err) rmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, "$G", "TEST"), moveReq, 5*time.Second) require_NoError(t, err) var moveResp JSApiStreamUpdateResponse require_NoError(t, json.Unmarshal(rmsg.Data, &moveResp)) require_True(t, moveResp.Error == nil) rmsg, err = ncsys.Request(fmt.Sprintf(JSApiServerStreamCancelMoveT, "$G", "TEST"), nil, 5*time.Second) require_NoError(t, err) var cancelResp JSApiStreamUpdateResponse require_NoError(t, json.Unmarshal(rmsg.Data, &cancelResp)) if cancelResp.Error != nil && ErrorIdentifier(cancelResp.Error.ErrCode) == JSStreamMoveNotInProgress { t.Skip("This can happen with delays, when Move completed before Cancel", cancelResp.Error) return } require_True(t, cancelResp.Error == nil) for _, sExpected := range streamPeerSrv { s := c.serverByName(sExpected) require_True(t, s.JetStreamIsStreamAssigned("$G", "TEST")) checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { return checkSrvInvariant(s, expectedPeers) }) } checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { return serverEmpty(emptySrv) }) checkFor(t, 3*usageTick, 100*time.Millisecond, func() error { if aiAfter, err := js.AccountInfo(); err != nil { return err } else if aiAfter.Store != aiBefore.Store { return fmt.Errorf("store before %d and after %d don't match", aiBefore.Store, aiAfter.Store) } else { return nil } }) } } func TestJetStreamSuperClusterDoubleStreamMove(t *testing.T) { server := map[string]struct{}{} sc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2, func(serverName, clusterName, storeDir, conf string) string { server[serverName] = struct{}{} return fmt.Sprintf("%s\nserver_tags: [%s]", conf, serverName) }, nil) defer sc.shutdown() // Client based API c := sc.randomCluster() srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() siCreate, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) srvMoveList := []string{siCreate.Cluster.Leader, siCreate.Cluster.Replicas[0].Name, siCreate.Cluster.Replicas[1].Name} // determine empty server for _, s := range srvMoveList { delete(server, s) } // pick left over server in same cluster as other server for s := range server { // server name is prefixed with cluster name if strings.HasPrefix(s, c.name) { srvMoveList = append(srvMoveList, s) break } } servers := []*Server{ c.serverByName(srvMoveList[0]), c.serverByName(srvMoveList[1]), c.serverByName(srvMoveList[2]), c.serverByName(srvMoveList[3]), // starts out empty } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "DUR", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{InactiveThreshold: time.Hour, AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ephName := ci.Name toSend := uint64(100) for i := uint64(0); i < toSend; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } ncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer ncsys.Close() move := func(fromSrv string, toTags ...string) { sEmpty := c.serverByName(fromSrv) jszBefore, err := sEmpty.Jsz(nil) require_NoError(t, err) require_True(t, jszBefore.Streams == 1) moveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{ Server: fromSrv, Tags: toTags}) require_NoError(t, err) rmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, "$G", "TEST"), moveReq, 100*time.Second) require_NoError(t, err) var moveResp JSApiStreamUpdateResponse require_NoError(t, json.Unmarshal(rmsg.Data, &moveResp)) require_True(t, moveResp.Error == nil) } serverEmpty := func(fromSrv string) error { if jszAfter, err := c.serverByName(fromSrv).Jsz(nil); err != nil { return fmt.Errorf("could not fetch JS info for server: %v", err) } else if jszAfter.Streams != 0 { return fmt.Errorf("empty server still has %d streams", jszAfter.Streams) } else if jszAfter.Consumers != 0 { return fmt.Errorf("empty server still has %d consumers", jszAfter.Consumers) } else if jszAfter.Store != 0 { return fmt.Errorf("empty server still has %d storage", jszAfter.Store) } return nil } moveComplete := func(toSrv string, expectedSet ...string) error { eSet := map[string]int{} foundInExpected := false for i, sExpected := range expectedSet { eSet[sExpected] = i s := c.serverByName(sExpected) if !s.JetStreamIsStreamAssigned("$G", "TEST") { return fmt.Errorf("expected stream to be assigned to %s", sExpected) } // test list order invariant js, cc := s.getJetStreamCluster() sExpHash := getHash(sExpected) js.mu.Lock() if sa, ok := cc.streams["$G"]["TEST"]; !ok { js.mu.Unlock() return fmt.Errorf("stream not found in cluster") } else if len(sa.Group.Peers) != 3 { js.mu.Unlock() return fmt.Errorf("peers not reset") } else if sa.Group.Peers[i] != sExpHash { js.mu.Unlock() return fmt.Errorf("stream: expected peer %s on index %d, got %s/%s", sa.Group.Peers[i], i, sExpHash, sExpected) } else if ca, ok := sa.consumers["DUR"]; !ok { js.mu.Unlock() return fmt.Errorf("durable not found in stream") } else { found := false for _, peer := range ca.Group.Peers { if peer == sExpHash { found = true break } } if !found { js.mu.Unlock() return fmt.Errorf("consumer expected peer %s/%s bud didn't find in %+v", sExpHash, sExpected, ca.Group.Peers) } if ephA, ok := sa.consumers[ephName]; ok { if len(ephA.Group.Peers) != 1 { return fmt.Errorf("ephemeral peers not reset") } foundInExpected = foundInExpected || (ephA.Group.Peers[0] == cc.meta.ID()) } } js.mu.Unlock() } if len(expectedSet) > 0 && !foundInExpected { return fmt.Errorf("ephemeral peer not expected") } for _, s := range servers { if jszAfter, err := c.serverByName(toSrv).Jsz(nil); err != nil { return fmt.Errorf("could not fetch JS info for server: %v", err) } else if jszAfter.Messages != toSend { return fmt.Errorf("messages not yet copied, got %d, expected %d", jszAfter.Messages, toSend) } nc, js := jsClientConnect(t, s) defer nc.Close() if si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)); err != nil { return fmt.Errorf("could not fetch stream info: %v", err) } else if len(si.Cluster.Replicas)+1 != si.Config.Replicas { return fmt.Errorf("not yet downsized replica should be empty has: %d %s", len(si.Cluster.Replicas), si.Cluster.Leader) } else if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("leader not found") } else if len(expectedSet) > 0 { if _, ok := eSet[si.Cluster.Leader]; !ok { return fmt.Errorf("leader %s not in expected set %+v", si.Cluster.Leader, eSet) } else if _, ok := eSet[si.Cluster.Replicas[0].Name]; !ok { return fmt.Errorf("leader %s not in expected set %+v", si.Cluster.Replicas[0].Name, eSet) } else if _, ok := eSet[si.Cluster.Replicas[1].Name]; !ok { return fmt.Errorf("leader %s not in expected set %+v", si.Cluster.Replicas[1].Name, eSet) } } nc.Close() } return nil } moveAndCheck := func(from, to string, expectedSet ...string) { t.Helper() move(from, to) checkFor(t, 40*time.Second, 100*time.Millisecond, func() error { return moveComplete(to, expectedSet...) }) checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { return serverEmpty(from) }) } checkFor(t, 20*time.Second, 1000*time.Millisecond, func() error { return serverEmpty(srvMoveList[3]) }) // first iteration establishes order of server 0-2 (the internal order in the server could be 1,0,2) moveAndCheck(srvMoveList[0], srvMoveList[3]) moveAndCheck(srvMoveList[1], srvMoveList[0]) moveAndCheck(srvMoveList[2], srvMoveList[1]) moveAndCheck(srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[1], srvMoveList[2]) // second iteration iterates in order moveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[1], srvMoveList[2], srvMoveList[3]) moveAndCheck(srvMoveList[1], srvMoveList[0], srvMoveList[2], srvMoveList[3], srvMoveList[0]) moveAndCheck(srvMoveList[2], srvMoveList[1], srvMoveList[3], srvMoveList[0], srvMoveList[1]) moveAndCheck(srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[1], srvMoveList[2]) // iterate in the opposite direction and establish order 2-0 moveAndCheck(srvMoveList[2], srvMoveList[3], srvMoveList[0], srvMoveList[1], srvMoveList[3]) moveAndCheck(srvMoveList[1], srvMoveList[2], srvMoveList[0], srvMoveList[3], srvMoveList[2]) moveAndCheck(srvMoveList[0], srvMoveList[1], srvMoveList[3], srvMoveList[2], srvMoveList[1]) moveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0]) // move server in the middle of list moveAndCheck(srvMoveList[1], srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[3]) moveAndCheck(srvMoveList[0], srvMoveList[1], srvMoveList[2], srvMoveList[3], srvMoveList[1]) moveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0]) // repeatedly use end moveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[2], srvMoveList[1], srvMoveList[3]) moveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0]) moveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[2], srvMoveList[1], srvMoveList[3]) moveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0]) } func TestJetStreamSuperClusterPeerEvacuationAndStreamReassignment(t *testing.T) { s := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2, func(serverName, clusterName, storeDir, conf string) string { return fmt.Sprintf("%s\nserver_tags: [cluster:%s, server:%s]", conf, clusterName, serverName) }, nil) defer s.shutdown() c := s.clusterForName("C1") // Client based API srv := c.randomNonLeader() nc, js := jsClientConnect(t, srv) defer nc.Close() test := func(t *testing.T, r int, moveTags []string, targetCluster string, testMigrateTo bool, listFrom bool) { si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: r, }) require_NoError(t, err) defer js.DeleteStream("TEST") startSet := map[string]struct{}{ si.Cluster.Leader: {}, } for _, p := range si.Cluster.Replicas { startSet[p.Name] = struct{}{} } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "DUR", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) sub, err := js.SubscribeSync("foo") require_NoError(t, err) for i := 0; i < 100; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } toMoveFrom := si.Cluster.Leader if !listFrom { toMoveFrom = _EMPTY_ } sLdr := c.serverByName(si.Cluster.Leader) jszBefore, err := sLdr.Jsz(nil) require_NoError(t, err) require_True(t, jszBefore.Streams == 1) require_True(t, jszBefore.Consumers >= 1) require_True(t, jszBefore.Store != 0) migrateToServer := _EMPTY_ if testMigrateTo { // find an empty server for _, s := range c.servers { name := s.Name() found := si.Cluster.Leader == name if !found { for _, r := range si.Cluster.Replicas { if r.Name == name { found = true break } } } if !found { migrateToServer = name break } } jszAfter, err := c.serverByName(migrateToServer).Jsz(nil) require_NoError(t, err) require_True(t, jszAfter.Streams == 0) moveTags = append(moveTags, fmt.Sprintf("server:%s", migrateToServer)) } ncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer ncsys.Close() moveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{Server: toMoveFrom, Tags: moveTags}) require_NoError(t, err) rmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, "$G", "TEST"), moveReq, 100*time.Second) require_NoError(t, err) var moveResp JSApiStreamUpdateResponse require_NoError(t, json.Unmarshal(rmsg.Data, &moveResp)) require_True(t, moveResp.Error == nil) // test move to particular server if testMigrateTo { toSrv := c.serverByName(migrateToServer) checkFor(t, 20*time.Second, 1000*time.Millisecond, func() error { jszAfter, err := toSrv.Jsz(nil) if err != nil { return fmt.Errorf("could not fetch JS info for server: %v", err) } if jszAfter.Streams != 1 { return fmt.Errorf("server expected to have one stream, has %d", jszAfter.Streams) } return nil }) } // Now wait until the stream is now current. checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("could not fetch stream info: %v", err) } if si.Cluster.Leader == toMoveFrom { return fmt.Errorf("peer not removed yet: %+v", toMoveFrom) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("no leader yet") } if len(si.Cluster.Replicas) != r-1 { return fmt.Errorf("not yet downsized replica should be %d has: %d", r-1, len(si.Cluster.Replicas)) } if si.Config.Replicas != r { return fmt.Errorf("bad replica count %d", si.Config.Replicas) } if si.Cluster.Name != targetCluster { return fmt.Errorf("stream expected in %s but found in %s", si.Cluster.Name, targetCluster) } sNew := s.serverByName(si.Cluster.Leader) if jszNew, err := sNew.Jsz(nil); err != nil { return err } else if jszNew.Streams != 1 { return fmt.Errorf("new leader has %d streams, not one", jszNew.Streams) } else if jszNew.Store != jszBefore.Store { return fmt.Errorf("new leader has %d storage, should have %d", jszNew.Store, jszBefore.Store) } return nil }) // test draining checkFor(t, 20*time.Second, time.Second, func() error { if !listFrom { // when needed determine which server move moved away from si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return fmt.Errorf("could not fetch stream info: %v", err) } for n := range startSet { if n != si.Cluster.Leader { var found bool for _, p := range si.Cluster.Replicas { if p.Name == n { found = true break } } if !found { toMoveFrom = n } } } } if toMoveFrom == _EMPTY_ { return fmt.Errorf("server to move away from not found") } sEmpty := c.serverByName(toMoveFrom) jszAfter, err := sEmpty.Jsz(nil) if err != nil { return fmt.Errorf("could not fetch JS info for server: %v", err) } if jszAfter.Streams != 0 { return fmt.Errorf("empty server still has %d streams", jszAfter.Streams) } if jszAfter.Consumers != 0 { return fmt.Errorf("empty server still has %d consumers", jszAfter.Consumers) } if jszAfter.Store != 0 { return fmt.Errorf("empty server still has %d storage", jszAfter.Store) } return nil }) // consume messages from ephemeral consumer for i := 0; i < 100; i++ { _, err := sub.NextMsg(time.Second) require_NoError(t, err) } } for i := 1; i <= 3; i++ { t.Run(fmt.Sprintf("r%d", i), func(t *testing.T) { test(t, i, nil, "C1", false, true) }) t.Run(fmt.Sprintf("r%d-explicit", i), func(t *testing.T) { test(t, i, nil, "C1", true, true) }) t.Run(fmt.Sprintf("r%d-nosrc", i), func(t *testing.T) { test(t, i, nil, "C1", false, false) }) } t.Run("r3-cluster-move", func(t *testing.T) { test(t, 3, []string{"cluster:C2"}, "C2", false, false) }) t.Run("r3-cluster-move-nosrc", func(t *testing.T) { test(t, 3, []string{"cluster:C2"}, "C2", false, true) }) } func TestJetStreamSuperClusterMirrorInheritsAllowDirect(t *testing.T) { sc := createJetStreamTaggedSuperCluster(t) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "KV", Subjects: []string{"key.*"}, Placement: &nats.Placement{Tags: []string{"cloud:aws", "country:us"}}, MaxMsgsPerSubject: 1, AllowDirect: true, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "KV"}, Placement: &nats.Placement{Tags: []string{"cloud:gcp", "country:uk"}}, }) require_NoError(t, err) // Do direct grab for now. resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "M"), nil, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) if !si.Config.MirrorDirect { t.Fatalf("Expected MirrorDirect to be inherited as true") } } func TestJetStreamSuperClusterSystemLimitsPlacement(t *testing.T) { const largeSystemLimit = 1024 const smallSystemLimit = 512 tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: _MAXMEM_ max_file_store: _MAXFILE_ store_dir: '%s', } server_tags: [ _TAG_ ] leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` storeCnf := func(serverName, clusterName, storeDir, conf string) string { switch { case strings.HasPrefix(serverName, "C1"): conf = strings.Replace(conf, "_MAXMEM_", fmt.Sprint(largeSystemLimit), 1) conf = strings.Replace(conf, "_MAXFILE_", fmt.Sprint(largeSystemLimit), 1) return strings.Replace(conf, "_TAG_", serverName, 1) case strings.HasPrefix(serverName, "C2"): conf = strings.Replace(conf, "_MAXMEM_", fmt.Sprint(smallSystemLimit), 1) conf = strings.Replace(conf, "_MAXFILE_", fmt.Sprint(smallSystemLimit), 1) return strings.Replace(conf, "_TAG_", serverName, 1) default: return conf } } sCluster := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, storeCnf, nil) defer sCluster.shutdown() requestLeaderStepDown := func(clientURL string) error { nc, err := nats.Connect(clientURL, nats.UserInfo("admin", "s3cr3t!")) if err != nil { return err } defer nc.Close() ncResp, err := nc.Request(JSApiLeaderStepDown, nil, time.Second) if err != nil { return err } var resp JSApiLeaderStepDownResponse if err := json.Unmarshal(ncResp.Data, &resp); err != nil { return err } if resp.Error != nil { return resp.Error } if !resp.Success { return fmt.Errorf("leader step down request not successful") } return nil } // Force large cluster to be leader var largeLeader *Server err := checkForErr(15*time.Second, 500*time.Millisecond, func() error { // Range over cluster A, which is the large cluster. servers := sCluster.clusters[0].servers for _, s := range servers { if s.JetStreamIsLeader() { largeLeader = s return nil } } if err := requestLeaderStepDown(servers[0].ClientURL()); err != nil { return fmt.Errorf("failed to request leader step down: %s", err) } return fmt.Errorf("leader is not in large cluster") }) if err != nil { t.Skipf("failed to get desired layout: %s", err) } getStreams := func(jsm nats.JetStreamManager) []string { var streams []string for s := range jsm.StreamNames() { streams = append(streams, s) } return streams } nc, js := jsClientConnect(t, largeLeader) defer nc.Close() cases := []struct { name string storage nats.StorageType createMaxBytes int64 serverTag string wantErr bool }{ { name: "file create large stream on small cluster b0", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S1", wantErr: true, }, { name: "memory create large stream on small cluster b0", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S1", wantErr: true, }, { name: "file create large stream on small cluster b1", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S2", wantErr: true, }, { name: "memory create large stream on small cluster b1", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S2", wantErr: true, }, { name: "file create large stream on small cluster b2", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S3", wantErr: true, }, { name: "memory create large stream on small cluster b2", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C2-S3", wantErr: true, }, { name: "file create large stream on large cluster a0", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S1", }, { name: "memory create large stream on large cluster a0", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S1", }, { name: "file create large stream on large cluster a1", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S2", }, { name: "memory create large stream on large cluster a1", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S2", }, { name: "file create large stream on large cluster a2", storage: nats.FileStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S3", }, { name: "memory create large stream on large cluster a2", storage: nats.MemoryStorage, createMaxBytes: smallSystemLimit + 1, serverTag: "C1-S3", }, } for i := 0; i < len(cases) && !t.Failed(); i++ { c := cases[i] t.Run(c.name, func(st *testing.T) { var clusterName string if strings.HasPrefix(c.serverTag, "a") { clusterName = "cluster-a" } else if strings.HasPrefix(c.serverTag, "b") { clusterName = "cluster-b" } if s := getStreams(js); len(s) != 0 { st.Fatalf("unexpected stream count, got=%d, want=0", len(s)) } streamName := fmt.Sprintf("TEST-%s", c.serverTag) si, err := js.AddStream(&nats.StreamConfig{ Name: streamName, Subjects: []string{"foo"}, Storage: c.storage, MaxBytes: c.createMaxBytes, Placement: &nats.Placement{ Cluster: clusterName, Tags: []string{c.serverTag}, }, }) if c.wantErr && err == nil { if s := getStreams(js); len(s) != 1 { st.Logf("unexpected stream count, got=%d, want=1, streams=%v", len(s), s) } cfg := si.Config st.Fatalf("unexpected success, maxBytes=%d, cluster=%s, tags=%v", cfg.MaxBytes, cfg.Placement.Cluster, cfg.Placement.Tags) } else if !c.wantErr && err != nil { if s := getStreams(js); len(s) != 0 { st.Logf("unexpected stream count, got=%d, want=0, streams=%v", len(s), s) } require_NoError(st, err) } if err == nil { if s := getStreams(js); len(s) != 1 { st.Fatalf("unexpected stream count, got=%d, want=1", len(s)) } } // Delete regardless. js.DeleteStream(streamName) }) } } func TestJetStreamSuperClusterMixedModeSwitchToInterestOnlyStaticConfig(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf: { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { ONE { users = [ { user: "one", pass: "pwd" } ] jetstream: enabled } TWO { users = [ { user: "two", pass: "pwd" } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 5, 3, func(serverName, clusterName, storeDir, conf string) string { sname := serverName[strings.Index(serverName, "-")+1:] switch sname { case "S4", "S5": conf = strings.ReplaceAll(conf, "jetstream: { ", "#jetstream: { ") default: conf = strings.ReplaceAll(conf, "leaf: { ", "#leaf: { ") } return conf }, nil) defer sc.shutdown() // Connect our client to a non JS server c := sc.randomCluster() var s *Server for _, as := range c.servers { if !as.JetStreamEnabled() { s = as break } } if s == nil { t.Fatal("Did not find a non JS server!") } nc, js := jsClientConnect(t, s, nats.UserInfo("one", "pwd")) defer nc.Close() // Just create a stream and then make sure that all gateways have switched // to interest-only mode. si, err := js.AddStream(&nats.StreamConfig{Name: "interest", Replicas: 3}) require_NoError(t, err) sc.waitOnStreamLeader("ONE", "interest") check := func(accName string) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { // Check only JS servers outbound GW connections if !s.JetStreamEnabled() { continue } opts := s.getOpts() for _, gw := range opts.Gateway.Gateways { if gw.Name == opts.Gateway.Name { continue } checkGWInterestOnlyMode(t, s, gw.Name, accName) } } } } // Starting v2.9.0, all accounts should be switched to interest-only mode check("ONE") check("TWO") var gwsa [16]*client gws := gwsa[:0] s = sc.serverByName(si.Cluster.Leader) // Get the GW outbound connections s.getOutboundGatewayConnections(&gws) for _, gwc := range gws { gwc.mu.Lock() gwc.nc.Close() gwc.mu.Unlock() } waitForOutboundGateways(t, s, 2, 5*time.Second) check("ONE") check("TWO") } func TestJetStreamSuperClusterMixedModeSwitchToInterestOnlyOperatorConfig(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) skp, _ := nkeys.CreateAccount() spub, _ := skp.PublicKey() nac := jwt.NewAccountClaims(spub) sjwt, err := nac.Encode(kp) require_NoError(t, err) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac = jwt.NewAccountClaims(apub) // Set some limits to enable JS. nac.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 nac.Limits.JetStreamLimits.Streams = 10 ajwt, err := nac.Encode(kp) require_NoError(t, err) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, spub) { w.Write([]byte(sjwt)) } else { w.Write([]byte(ajwt)) } })) defer ts.Close() operator := fmt.Sprintf(` operator: %s resolver: URL("%s/ngs/v1/accounts/jwt/") `, ojwt, ts.URL) tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf: { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` + operator sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 5, 3, func(serverName, clusterName, storeDir, conf string) string { conf = strings.ReplaceAll(conf, "system_account: \"$SYS\"", fmt.Sprintf("system_account: \"%s\"", spub)) sname := serverName[strings.Index(serverName, "-")+1:] switch sname { case "S4", "S5": conf = strings.ReplaceAll(conf, "jetstream: { ", "#jetstream: { ") default: conf = strings.ReplaceAll(conf, "leaf: { ", "#leaf: { ") } return conf }, nil) defer sc.shutdown() // Connect our client to a non JS server c := sc.randomCluster() var s *Server for _, as := range c.servers { if !as.JetStreamEnabled() { s = as break } } if s == nil { t.Fatal("Did not find a non JS server!") } nc, js := jsClientConnect(t, s, createUserCreds(t, nil, akp)) defer nc.Close() // Just create a stream and then make sure that all gateways have switched // to interest-only mode. si, err := js.AddStream(&nats.StreamConfig{Name: "interest", Replicas: 3}) require_NoError(t, err) sc.waitOnStreamLeader(apub, "interest") check := func(s *Server) { opts := s.getOpts() for _, gw := range opts.Gateway.Gateways { if gw.Name == opts.Gateway.Name { continue } checkGWInterestOnlyMode(t, s, gw.Name, apub) } } s = sc.serverByName(si.Cluster.Leader) check(s) // Let's cause a leadership change and verify that it still works. _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "interest"), nil, time.Second) require_NoError(t, err) sc.waitOnStreamLeader(apub, "interest") si, err = js.StreamInfo("interest") require_NoError(t, err) s = sc.serverByName(si.Cluster.Leader) check(s) var gwsa [16]*client gws := gwsa[:0] // Get the GW outbound connections s.getOutboundGatewayConnections(&gws) for _, gwc := range gws { gwc.mu.Lock() gwc.nc.Close() gwc.mu.Unlock() } waitForOutboundGateways(t, s, 2, 5*time.Second) check(s) } type captureGWRewriteLogger struct { DummyLogger ch chan string } func (l *captureGWRewriteLogger) Tracef(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "$JS.SNAPSHOT.ACK.TEST") && strings.Contains(msg, gwReplyPrefix) { select { case l.ch <- msg: default: } } } func TestJetStreamSuperClusterGWReplyRewrite(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() nc, js := jsClientConnect(t, sc.serverByName("C1-S1")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) sc.waitOnStreamLeader(globalAccountName, "TEST") for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", "msg") } nc2, _ := jsClientConnect(t, sc.serverByName("C2-S2")) defer nc2.Close() s := sc.clusters[0].streamLeader(globalAccountName, "TEST") var gws []*client s.getOutboundGatewayConnections(&gws) for _, gw := range gws { gw.mu.Lock() gw.trace = true gw.mu.Unlock() } l := &captureGWRewriteLogger{ch: make(chan string, 1)} s.SetLogger(l, false, true) // Send a request through the gateway sreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 512, } natsSub(t, nc2, sreq.DeliverSubject, func(m *nats.Msg) { m.Respond(nil) }) natsFlush(t, nc2) req, _ := json.Marshal(sreq) rmsg, err := nc2.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) require_NoError(t, err) var resp JSApiStreamSnapshotResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } // Now we just want to make sure that the reply has the gateway prefix select { case <-l.ch: case <-time.After(10 * time.Second): } } func TestJetStreamSuperClusterGWOfflineSatus(t *testing.T) { orgEventsHBInterval := eventsHBInterval eventsHBInterval = 500 * time.Millisecond //time.Second defer func() { eventsHBInterval = orgEventsHBInterval }() tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} gateway { name: "local" listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { SYS { users [{user: sys, password: pwd}] } ONE { jetstream: enabled users [{user: one, password: pwd}] } } system_account=SYS ` c := createJetStreamClusterWithTemplate(t, tmpl, "local", 3) defer c.shutdown() var gwURLs string for i, s := range c.servers { if i > 0 { gwURLs += "," } gwURLs += `"nats://` + s.GatewayAddr().String() + `"` } tmpl2 := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} gateway { name: "remote" listen: 127.0.0.1:-1 __remote__ } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { SYS { users [{user: sys, password: pwd}] } ONE { jetstream: enabled users [{user: one, password: pwd}] } } system_account=SYS ` c2 := createJetStreamClusterAndModHook(t, tmpl2, "remote", "R", 2, 16022, false, func(serverName, clusterName, storeDir, conf string) string { conf = strings.Replace(conf, "__remote__", fmt.Sprintf("gateways [ { name: 'local', urls: [%s] } ]", gwURLs), 1) return conf }) defer c2.shutdown() for _, s := range c.servers { waitForOutboundGateways(t, s, 1, 2*time.Second) } for _, s := range c2.servers { waitForOutboundGateways(t, s, 1, 2*time.Second) } c.waitOnPeerCount(5) // Simulate going offline without sending shutdown protocol for _, s := range c2.servers { c := s.getOutboundGatewayConnection("local") c.setNoReconnect() c.mu.Lock() c.nc.Close() c.mu.Unlock() } c2.shutdown() checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { var ok int for _, s := range c.servers { jsz, err := s.Jsz(nil) if err != nil { return err } for _, r := range jsz.Meta.Replicas { if r.Name == "RS-1" && r.Offline { ok++ } else if r.Name == "RS-2" && r.Offline { ok++ } } } if ok != 2 { return fmt.Errorf("RS-1 or RS-2 still marked as online") } return nil }) } func TestJetStreamSuperClusterMovingR1Stream(t *testing.T) { // Make C2 have some latency. gwm := gwProxyMap{ "C2": &gwProxy{ rtt: 10 * time.Millisecond, up: 1 * 1024 * 1024 * 1024, // 1gbit down: 1 * 1024 * 1024 * 1024, // 1gbit }, } sc := createJetStreamTaggedSuperClusterWithGWProxy(t, gwm) defer sc.shutdown() nc, js := jsClientConnect(t, sc.clusterForName("C1").randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", }) require_NoError(t, err) toSend := 10_000 for i := 0; i < toSend; i++ { _, err := js.PublishAsync("TEST", []byte("HELLO WORLD")) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Have it move to GCP. _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Placement: &nats.Placement{Tags: []string{"cloud:gcp"}}, }) require_NoError(t, err) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { sc.waitOnStreamLeader(globalAccountName, "TEST") si, err := js.StreamInfo("TEST") if err != nil { return err } if si.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } else if !strings.HasPrefix(si.Cluster.Leader, "C2") { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } // Now we want to see that we shrink back to original. if len(si.Cluster.Replicas) != 0 { return fmt.Errorf("Expected 0 replicas, got %d", len(si.Cluster.Replicas)) } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Only see %d msgs", si.State.Msgs) } return nil }) } // https://github.com/nats-io/nats-server/issues/4396 func TestJetStreamSuperClusterR1StreamPeerRemove(t *testing.T) { sc := createJetStreamSuperCluster(t, 1, 3) defer sc.shutdown() nc, js := jsClientConnect(t, sc.serverByName("C1-S1")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, }) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) // Call peer remove on the only peer the leader. resp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, "TEST"), []byte(`{"peer":"`+si.Cluster.Leader+`"}`), time.Second) require_NoError(t, err) var rpr JSApiStreamRemovePeerResponse require_NoError(t, json.Unmarshal(resp.Data, &rpr)) require_False(t, rpr.Success) require_True(t, rpr.Error.ErrCode == 10075) // Stream should still be in place and useable. _, err = js.StreamInfo("TEST") require_NoError(t, err) } nats-server-2.10.27/server/jetstream_test.go000066400000000000000000023663651477524627100210720ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_js_tests // +build !skip_js_tests package server import ( "bytes" "context" crand "crypto/rand" "encoding/base64" "encoding/json" "errors" "fmt" "io" "math" "math/rand" "net/http" "net/url" "os" "path/filepath" "reflect" "runtime" "runtime/debug" "slices" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/sysmem" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) func TestJetStreamBasicNilConfig(t *testing.T) { s := RunRandClientPortServer(t) defer s.Shutdown() if err := s.EnableJetStream(nil); err != nil { t.Fatalf("Expected no error, got %v", err) } if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } if s.SystemAccount() == nil { t.Fatalf("Expected system account to be created automatically") } // Grab our config since it was dynamically generated. config := s.JetStreamConfig() if config == nil { t.Fatalf("Expected non-nil config") } // Check dynamic max memory. hwMem := sysmem.Memory() if hwMem != 0 { // Check if memory being limited via GOMEMLIMIT if being set. if gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 { hwMem = gml } // Make sure its about 75% est := hwMem / 4 * 3 if config.MaxMemory != est { t.Fatalf("Expected memory to be 80 percent of system memory, got %v vs %v", config.MaxMemory, est) } } // Make sure it was created. stat, err := os.Stat(config.StoreDir) if err != nil { t.Fatalf("Expected the store directory to be present, %v", err) } if stat == nil || !stat.IsDir() { t.Fatalf("Expected a directory") } } func RunBasicJetStreamServer(t testing.TB) *Server { opts := DefaultTestOptions opts.Port = -1 opts.JetStream = true opts.StoreDir = t.TempDir() return RunServer(&opts) } func RunJetStreamServerOnPort(port int, sd string) *Server { opts := DefaultTestOptions opts.Port = port opts.JetStream = true opts.StoreDir = filepath.Dir(sd) return RunServer(&opts) } func clientConnectToServer(t *testing.T, s *Server) *nats.Conn { t.Helper() nc, err := nats.Connect(s.ClientURL(), nats.Name("JS-TEST"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1)) if err != nil { t.Fatalf("Failed to create client: %v", err) } return nc } func clientConnectWithOldRequest(t *testing.T, s *Server) *nats.Conn { nc, err := nats.Connect(s.ClientURL(), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Failed to create client: %v", err) } return nc } func TestJetStreamEnableAndDisableAccount(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Global in simple setup should be enabled already. if !s.GlobalAccount().JetStreamEnabled() { t.Fatalf("Expected to have jetstream enabled on global account") } if na := s.JetStreamNumAccounts(); na != 1 { t.Fatalf("Expected 1 account, got %d", na) } if err := s.GlobalAccount().DisableJetStream(); err != nil { t.Fatalf("Did not expect error on disabling account: %v", err) } if na := s.JetStreamNumAccounts(); na != 0 { t.Fatalf("Expected no accounts, got %d", na) } // Make sure we unreserved resources. if rm, rd, err := s.JetStreamReservedResources(); err != nil { t.Fatalf("Unexpected error requesting jetstream reserved resources: %v", err) } else if rm != 0 || rd != 0 { t.Fatalf("Expected reserved memory and store to be 0, got %v and %v", friendlyBytes(rm), friendlyBytes(rd)) } acc, _ := s.LookupOrRegisterAccount("$FOO") if err := acc.EnableJetStream(nil); err != nil { t.Fatalf("Did not expect error on enabling account: %v", err) } if na := s.JetStreamNumAccounts(); na != 1 { t.Fatalf("Expected 1 account, got %d", na) } if err := acc.DisableJetStream(); err != nil { t.Fatalf("Did not expect error on disabling account: %v", err) } if na := s.JetStreamNumAccounts(); na != 0 { t.Fatalf("Expected no accounts, got %d", na) } // We should get error if disabling something not enabled. acc, _ = s.LookupOrRegisterAccount("$BAR") if err := acc.DisableJetStream(); err == nil { t.Fatalf("Expected error on disabling account that was not enabled") } // Should get an error for trying to enable a non-registered account. acc = NewAccount("$BAZ") if err := acc.EnableJetStream(nil); err == nil { t.Fatalf("Expected error on enabling account that was not registered") } } func TestJetStreamAddStream(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() js.Publish("foo", []byte("Hello World!")) state := mset.state() if state.Msgs != 1 { t.Fatalf("Expected 1 message, got %d", state.Msgs) } if state.Bytes == 0 { t.Fatalf("Expected non-zero bytes") } js.Publish("foo", []byte("Hello World Again!")) state = mset.state() if state.Msgs != 2 { t.Fatalf("Expected 2 messages, got %d", state.Msgs) } if err := mset.delete(); err != nil { t.Fatalf("Got an error deleting the stream: %v", err) } }) } } func TestJetStreamAddStreamDiscardNew(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", MaxMsgs: 10, MaxBytes: 4096, Discard: DiscardNew, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", MaxMsgs: 10, MaxBytes: 4096, Discard: DiscardNew, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() subj := "foo" toSend := 10 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, subj, fmt.Sprintf("MSG: %d", i+1)) } // We expect this one to fail due to discard policy. resp, _ := nc.Request(subj, []byte("discard me"), 100*time.Millisecond) if resp == nil { t.Fatalf("No response, possible timeout?") } if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != "maximum messages exceeded" || pa.Stream != "foo" { t.Fatalf("Expected to get an error about maximum messages, got %q", resp.Data) } // Now do bytes. mset.purge(nil) big := make([]byte, 8192) resp, _ = nc.Request(subj, big, 100*time.Millisecond) if resp == nil { t.Fatalf("No response, possible timeout?") } if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != "maximum bytes exceeded" || pa.Stream != "foo" { t.Fatalf("Expected to get an error about maximum bytes, got %q", resp.Data) } }) } } func TestJetStreamAutoTuneFSConfig(t *testing.T) { s := RunRandClientPortServer(t) defer s.Shutdown() jsconfig := &JetStreamConfig{MaxMemory: -1, MaxStore: 128 * 1024 * 1024, StoreDir: t.TempDir()} if err := s.EnableJetStream(jsconfig); err != nil { t.Fatalf("Expected no error, got %v", err) } maxMsgSize := int32(512) streamConfig := func(name string, maxMsgs, maxBytes int64) *StreamConfig { t.Helper() cfg := &StreamConfig{Name: name, MaxMsgSize: maxMsgSize, Storage: FileStorage} if maxMsgs > 0 { cfg.MaxMsgs = maxMsgs } if maxBytes > 0 { cfg.MaxBytes = maxBytes } return cfg } acc := s.GlobalAccount() testBlkSize := func(subject string, maxMsgs, maxBytes int64, expectedBlkSize uint64) { t.Helper() mset, err := acc.addStream(streamConfig(subject, maxMsgs, maxBytes)) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() fsCfg, err := mset.fileStoreConfig() if err != nil { t.Fatalf("Unexpected error retrieving file store: %v", err) } if fsCfg.BlockSize != expectedBlkSize { t.Fatalf("Expected auto tuned block size to be %d, got %d", expectedBlkSize, fsCfg.BlockSize) } } testBlkSize("foo", 1, 0, FileStoreMinBlkSize) testBlkSize("foo", 1, 512, FileStoreMinBlkSize) testBlkSize("foo", 1, 1024*1024, defaultMediumBlockSize) testBlkSize("foo", 1, 8*1024*1024, defaultMediumBlockSize) testBlkSize("foo_bar_baz", -1, 32*1024*1024, FileStoreMaxBlkSize) } func TestJetStreamConsumerAndStreamDescriptions(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() descr := "foo asset" acc := s.GlobalAccount() // Check stream's first. mset, err := acc.addStream(&StreamConfig{Name: "foo", Description: descr}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } if cfg := mset.config(); cfg.Description != descr { t.Fatalf("Expected a description of %q, got %q", descr, cfg.Description) } // Now consumer edescr := "analytics" o, err := mset.addConsumer(&ConsumerConfig{ Description: edescr, DeliverSubject: "to", AckPolicy: AckNone}) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } if cfg := o.config(); cfg.Description != edescr { t.Fatalf("Expected a description of %q, got %q", edescr, cfg.Description) } // Test max. data := make([]byte, JSMaxDescriptionLen+1) crand.Read(data) bigDescr := base64.StdEncoding.EncodeToString(data) _, err = acc.addStream(&StreamConfig{Name: "bar", Description: bigDescr}) if err == nil || !strings.Contains(err.Error(), "description is too long") { t.Fatalf("Expected an error but got none") } _, err = mset.addConsumer(&ConsumerConfig{ Description: bigDescr, DeliverSubject: "to", AckPolicy: AckNone}) if err == nil || !strings.Contains(err.Error(), "description is too long") { t.Fatalf("Expected an error but got none") } } func TestJetStreamConsumerWithNameAndDurable(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() descr := "foo asset" name := "name" durable := "durable" acc := s.GlobalAccount() // Check stream's first. mset, err := acc.addStream(&StreamConfig{Name: "foo", Description: descr}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } if cfg := mset.config(); cfg.Description != descr { t.Fatalf("Expected a description of %q, got %q", descr, cfg.Description) } // it's ok to specify both durable and name, but they have to be the same. _, err = mset.addConsumer(&ConsumerConfig{ DeliverSubject: "to", Durable: "consumer", Name: "consumer", AckPolicy: AckNone}) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } // if they're not the same, expect error _, err = mset.addConsumer(&ConsumerConfig{ DeliverSubject: "to", Durable: durable, Name: name, AckPolicy: AckNone}) if !strings.Contains(err.Error(), "Consumer Durable and Name have to be equal") { t.Fatalf("Wrong error while adding consumer with not matching Name and Durable: %v", err) } } func TestJetStreamPubAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sname := "PUBACK" acc := s.GlobalAccount() mconfig := &StreamConfig{Name: sname, Subjects: []string{"foo"}, Storage: MemoryStorage} mset, err := acc.addStream(mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() checkRespDetails := func(resp *nats.Msg, err error, seq uint64) { if err != nil { t.Fatalf("Unexpected error from send stream msg: %v", err) } if resp == nil { t.Fatalf("No response from send stream msg") } pa := getPubAckResponse(resp.Data) if pa == nil || pa.Error != nil { t.Fatalf("Expected a valid JetStreamPubAck, got %q", resp.Data) } if pa.Stream != sname { t.Fatalf("Expected %q for stream name, got %q", sname, pa.Stream) } if pa.Sequence != seq { t.Fatalf("Expected %d for sequence, got %d", seq, pa.Sequence) } } // Send messages and make sure pubAck details are correct. for i := uint64(1); i <= 1000; i++ { resp, err := nc.Request("foo", []byte("HELLO"), 100*time.Millisecond) checkRespDetails(resp, err, i) } } func TestJetStreamConsumerWithStartTime(t *testing.T) { subj := "my_stream" cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: subj, Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: subj, Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() fsCfg := &FileStoreConfig{BlockSize: 100} mset, err := s.GlobalAccount().addStreamWithStore(c.mconfig, fsCfg) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 250 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, subj, fmt.Sprintf("MSG: %d", i+1)) } time.Sleep(10 * time.Millisecond) startTime := time.Now().UTC() for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, subj, fmt.Sprintf("MSG: %d", i+1)) } if msgs := mset.state().Msgs; msgs != uint64(toSend*2) { t.Fatalf("Expected %d messages, got %d", toSend*2, msgs) } o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d", DeliverPolicy: DeliverByStartTime, OptStartTime: &startTime, AckPolicy: AckExplicit, }) require_NoError(t, err) defer o.delete() msg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) require_NoError(t, err) sseq, dseq, _, _, _ := replyInfo(msg.Reply) if dseq != 1 { t.Fatalf("Expected delivered seq of 1, got %d", dseq) } if sseq != uint64(toSend+1) { t.Fatalf("Expected to get store seq of %d, got %d", toSend+1, sseq) } }) } } // Test for https://github.com/nats-io/jetstream/issues/143 func TestJetStreamConsumerWithMultipleStartOptions(t *testing.T) { subj := "my_stream" cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: subj, Subjects: []string{"foo.>"}, Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: subj, Subjects: []string{"foo.>"}, Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() obsReq := CreateConsumerRequest{ Stream: subj, Config: ConsumerConfig{ Durable: "d", DeliverPolicy: DeliverLast, FilterSubject: "foo.22", AckPolicy: AckExplicit, }, } req, err := json.Marshal(obsReq) require_NoError(t, err) _, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, subj), req, time.Second) require_NoError(t, err) nc.Close() s.Shutdown() }) } } func TestJetStreamConsumerMaxDeliveries(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Queue up our work item. sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() maxDeliver := 5 ackWait := 10 * time.Millisecond o, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: sub.Subject, AckPolicy: AckExplicit, AckWait: ackWait, MaxDeliver: maxDeliver, }) require_NoError(t, err) defer o.delete() // Wait for redeliveries to pile up. checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != maxDeliver { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, maxDeliver) } return nil }) // Now wait a bit longer and make sure we do not have more than maxDeliveries. time.Sleep(2 * ackWait) if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxDeliver { t.Fatalf("Did not receive correct number of messages: %d vs %d", nmsgs, maxDeliver) } }) } } func TestJetStreamNextReqFromMsg(t *testing.T) { bef := time.Now() expires, _, _, _, _, _, err := nextReqFromMsg([]byte(`{"expires":5000000000}`)) // nanoseconds require_NoError(t, err) now := time.Now() if expires.Before(bef.Add(5*time.Second)) || expires.After(now.Add(5*time.Second)) { t.Fatal("Expires out of expected range") } } func TestJetStreamPullConsumerDelayedFirstPullWithReplayOriginal(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Queue up our work item. sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d", AckPolicy: AckExplicit, ReplayPolicy: ReplayOriginal, }) require_NoError(t, err) defer o.delete() // Force delay here which triggers the bug. time.Sleep(250 * time.Millisecond) if _, err = nc.Request(o.requestNextMsgSubject(), nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } }) } } func TestJetStreamConsumerAckFloorFill(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MQ", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "MQ", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() for i := 1; i <= 4; i++ { sendStreamMsg(t, nc, c.mconfig.Name, fmt.Sprintf("msg-%d", i)) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d", DeliverSubject: sub.Subject, AckPolicy: AckExplicit, }) require_NoError(t, err) defer o.delete() var first *nats.Msg for i := 1; i <= 3; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving message %d: %v", i, err) } // Don't ack 1 or 4. if i == 1 { first = m } else if i == 2 || i == 3 { m.Respond(nil) } } nc.Flush() if info := o.info(); info.AckFloor.Consumer != 0 { t.Fatalf("Expected the ack floor to be 0, got %d", info.AckFloor.Consumer) } // Now ack first, should move ack floor to 3. first.Respond(nil) nc.Flush() checkFor(t, time.Second, 50*time.Millisecond, func() error { if info := o.info(); info.AckFloor.Consumer != 3 { return fmt.Errorf("Expected the ack floor to be 3, got %d", info.AckFloor.Consumer) } return nil }) }) } } func TestJetStreamNoPanicOnRaceBetweenShutdownAndConsumerDelete(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_STREAM", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "MY_STREAM", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() var cons []*consumer for i := 0; i < 100; i++ { o, err := mset.addConsumer(&ConsumerConfig{ Durable: fmt.Sprintf("d%d", i), AckPolicy: AckExplicit, }) require_NoError(t, err) defer o.delete() cons = append(cons, o) } wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for _, c := range cons { c.delete() } }() time.Sleep(10 * time.Millisecond) s.Shutdown() }) } } func TestJetStreamAddStreamMaxMsgSize(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, MaxMsgSize: 22, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, MaxMsgSize: 22, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() if _, err := nc.Request("foo", []byte("Hello World!"), time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } tooBig := []byte("1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ") resp, err := nc.Request("foo", tooBig, time.Second) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != "message size exceeds maximum allowed" { t.Fatalf("Expected to get an error for maximum message size, got %q", pa.Error) } }) } } func TestJetStreamAddStreamCanonicalNames(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() expectErr := func(_ *stream, err error) { t.Helper() if !IsNatsErr(err, JSStreamInvalidConfigF) { t.Fatalf("Expected error but got none") } } expectErr(acc.addStream(&StreamConfig{Name: "foo.bar"})) expectErr(acc.addStream(&StreamConfig{Name: "foo.bar."})) expectErr(acc.addStream(&StreamConfig{Name: "foo.*"})) expectErr(acc.addStream(&StreamConfig{Name: "foo.>"})) expectErr(acc.addStream(&StreamConfig{Name: "*"})) expectErr(acc.addStream(&StreamConfig{Name: ">"})) expectErr(acc.addStream(&StreamConfig{Name: "*>"})) } func TestJetStreamAddStreamBadSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() expectAPIErr := func(cfg StreamConfig) { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) resp, _ := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) var scResp JSApiStreamCreateResponse if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } require_Error(t, scResp.ToError(), NewJSStreamInvalidConfigError(fmt.Errorf("invalid subject"))) } expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{"foo.bar."}}) expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{".."}}) expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{".*"}}) expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{".>"}}) expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{" x"}}) expectAPIErr(StreamConfig{Name: "MyStream", Storage: MemoryStorage, Subjects: []string{"y "}}) } func TestJetStreamMaxConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "MAXC", Storage: nats.MemoryStorage, Subjects: []string{"in.maxc.>"}, MaxConsumers: 2, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } si, err := js.StreamInfo("MAXC") require_NoError(t, err) if si.Config.MaxConsumers != 2 { t.Fatalf("Expected max of 2, got %d", si.Config.MaxConsumers) } // Make sure we get the right error. // This should succeed. if _, err := js.PullSubscribe("in.maxc.foo", "maxc_foo"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Create for the same consumer must be idempotent, and not trigger limit. if _, err := js.PullSubscribe("in.maxc.foo", "maxc_foo"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Create (with explicit create API) for the same consumer must be idempotent, and not trigger limit. obsReq := CreateConsumerRequest{ Stream: "MAXC", Config: ConsumerConfig{Durable: "maxc_baz"}, Action: ActionCreate, } req, err := json.Marshal(obsReq) require_NoError(t, err) msg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "MAXC", "maxc_baz"), req, time.Second) require_NoError(t, err) var resp JSApiConsumerInfoResponse require_NoError(t, json.Unmarshal(msg.Data, &resp)) if resp.Error != nil { t.Fatalf("Unexpected error: %v", resp.Error) } msg, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, "MAXC", "maxc_baz"), req, time.Second) require_NoError(t, err) var resp2 JSApiConsumerInfoResponse require_NoError(t, json.Unmarshal(msg.Data, &resp2)) if resp2.Error != nil { t.Fatalf("Unexpected error: %v", resp2.Error) } // Exceeds limit. if _, err := js.SubscribeSync("in.maxc.bar"); err == nil { t.Fatalf("Expected error but got none") } } func TestJetStreamAddStreamOverlappingSubjects(t *testing.T) { mconfig := &StreamConfig{ Name: "ok", Storage: MemoryStorage, Subjects: []string{"foo", "bar", "baz.*", "foo.bar.baz.>"}, } s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() mset, err := acc.addStream(mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() expectErr := func(_ *stream, err error) { t.Helper() if err == nil || !strings.Contains(err.Error(), "subjects overlap") { t.Fatalf("Expected error but got none") } } // Test that any overlapping subjects will fail. expectErr(acc.addStream(&StreamConfig{Name: "foo"})) expectErr(acc.addStream(&StreamConfig{Name: "a", Subjects: []string{"baz", "bar"}})) expectErr(acc.addStream(&StreamConfig{Name: "b", Subjects: []string{">"}, NoAck: true})) expectErr(acc.addStream(&StreamConfig{Name: "c", Subjects: []string{"baz.33"}})) expectErr(acc.addStream(&StreamConfig{Name: "d", Subjects: []string{"*.33"}})) expectErr(acc.addStream(&StreamConfig{Name: "e", Subjects: []string{"*.>"}})) expectErr(acc.addStream(&StreamConfig{Name: "f", Subjects: []string{"foo.bar", "*.bar.>"}})) } func TestJetStreamAddStreamOverlapWithJSAPISubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() expectErr := func(_ *stream, err error) { t.Helper() if err == nil || !strings.Contains(err.Error(), "subjects that overlap with jetstream api") { t.Fatalf("Expected error but got none") } } // Test that any overlapping subjects with our JSAPI should fail. expectErr(acc.addStream(&StreamConfig{Name: "a", Subjects: []string{"$JS.API.foo", "$JS.API.bar"}})) expectErr(acc.addStream(&StreamConfig{Name: "b", Subjects: []string{"$JS.API.>"}})) expectErr(acc.addStream(&StreamConfig{Name: "c", Subjects: []string{"$JS.API.*"}})) // Events and Advisories etc should be ok. if _, err := acc.addStream(&StreamConfig{Name: "a", Subjects: []string{"$JS.EVENT.>"}, NoAck: true}); err != nil { t.Fatalf("Expected this to work: %v", err) } } func TestJetStreamAddStreamSameConfigOK(t *testing.T) { mconfig := &StreamConfig{ Name: "ok", Subjects: []string{"foo", "bar", "baz.*", "foo.bar.baz.>"}, Storage: MemoryStorage, } s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() mset, err := acc.addStream(mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Adding again with same config should be idempotent. if _, err = acc.addStream(mconfig); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } } func sendStreamMsg(t *testing.T, nc *nats.Conn, subject, msg string) *PubAck { t.Helper() resp, _ := nc.Request(subject, []byte(msg), 500*time.Millisecond) if resp == nil { t.Fatalf("No response for %q, possible timeout?", msg) } pa := getPubAckResponse(resp.Data) if pa == nil || pa.Error != nil { t.Fatalf("Expected a valid JetStreamPubAck, got %q", resp.Data) } return pa.PubAck } func TestJetStreamBasicAckPublish(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "foo", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "foo", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() for i := 0; i < 50; i++ { sendStreamMsg(t, nc, "foo.bar", "Hello World!") } state := mset.state() if state.Msgs != 50 { t.Fatalf("Expected 50 messages, got %d", state.Msgs) } }) } } func TestJetStreamStateTimestamps(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "foo", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "foo", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() start := time.Now() delay := 250 * time.Millisecond sendStreamMsg(t, nc, "foo.bar", "Hello World!") time.Sleep(delay) sendStreamMsg(t, nc, "foo.bar", "Hello World Again!") state := mset.state() if state.FirstTime.Before(start) { t.Fatalf("Unexpected first message timestamp: %v", state.FirstTime) } if state.LastTime.Before(start.Add(delay)) { t.Fatalf("Unexpected last message timestamp: %v", state.LastTime) } }) } } func TestJetStreamNoAckStream(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "foo", Storage: MemoryStorage, NoAck: true}}, {"FileStore", &StreamConfig{Name: "foo", Storage: FileStorage, NoAck: true}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // We can use NoAck to suppress acks even when reply subjects are present. mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() if _, err := nc.Request("foo", []byte("Hello World!"), 25*time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected a timeout error and no response with acks suppressed") } state := mset.state() if state.Msgs != 1 { t.Fatalf("Expected 1 message, got %d", state.Msgs) } }) } } func TestJetStreamCreateConsumer(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "foo", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}, Retention: WorkQueuePolicy}}, {"FileStore", &StreamConfig{Name: "foo", Storage: FileStorage, Subjects: []string{"foo", "bar"}, Retention: WorkQueuePolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Check for basic errors. if _, err := mset.addConsumer(nil); err == nil { t.Fatalf("Expected an error for no config") } // No deliver subject, meaning its in pull mode, work queue mode means it is required to // do explicit ack. if _, err := mset.addConsumer(&ConsumerConfig{}); err == nil { t.Fatalf("Expected an error on work queue / pull mode without explicit ack mode") } // Check for delivery subject errors. // Literal delivery subject required. if _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: "foo.*"}); err == nil { t.Fatalf("Expected an error on bad delivery subject") } // Check for cycles if _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: "foo"}); err == nil { t.Fatalf("Expected an error on delivery subject that forms a cycle") } if _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: "bar"}); err == nil { t.Fatalf("Expected an error on delivery subject that forms a cycle") } if _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: "*"}); err == nil { t.Fatalf("Expected an error on delivery subject that forms a cycle") } // StartPosition conflicts now := time.Now().UTC() if _, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: "A", OptStartSeq: 1, OptStartTime: &now, }); err == nil { t.Fatalf("Expected an error on start position conflicts") } if _, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: "A", OptStartTime: &now, }); err == nil { t.Fatalf("Expected an error on start position conflicts") } // Non-Durables need to have subscription to delivery subject. delivery := nats.NewInbox() nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(delivery) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } if err := mset.deleteConsumer(o); err != nil { t.Fatalf("Expected no error on delete, got %v", err) } // Now let's check that durables can be created and a duplicate call to add will be ok. dcfg := &ConsumerConfig{ Durable: "ddd", DeliverSubject: delivery, AckPolicy: AckExplicit, } if _, err = mset.addConsumer(dcfg); err != nil { t.Fatalf("Unexpected error creating consumer: %v", err) } if _, err = mset.addConsumer(dcfg); err != nil { t.Fatalf("Unexpected error creating second identical consumer: %v", err) } // Not test that we can change the delivery subject if that is only thing that has not // changed and we are not active. sub.Unsubscribe() sub, _ = nc.SubscribeSync("d.d.d") nc.Flush() defer sub.Unsubscribe() dcfg.DeliverSubject = "d.d.d" if _, err = mset.addConsumer(dcfg); err != nil { t.Fatalf("Unexpected error creating third consumer with just deliver subject changed: %v", err) } }) } } func TestJetStreamBasicDeliverSubject(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MSET", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "MSET", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 100 sendSubj := "foo.bar" for i := 1; i <= toSend; i++ { sendStreamMsg(t, nc, sendSubj, strconv.Itoa(i)) } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } // Now create an consumer. Use different connection. nc2 := clientConnectToServer(t, s) defer nc2.Close() sub, _ := nc2.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc2.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() // Check for our messages. checkMsgs := func(seqOff int) { t.Helper() checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) // Now let's check the messages for i := 0; i < toSend; i++ { m, _ := sub.NextMsg(time.Second) // JetStream will have the subject match the stream subject, not delivery subject. if m.Subject != sendSubj { t.Fatalf("Expected original subject of %q, but got %q", sendSubj, m.Subject) } // Now check that reply subject exists and has a sequence as the last token. if seq := o.seqFromReply(m.Reply); seq != uint64(i+seqOff) { t.Fatalf("Expected sequence of %d , got %d", i+seqOff, seq) } // Ack the message here. m.Respond(nil) } } checkMsgs(1) // Now send more and make sure delivery picks back up. for i := toSend + 1; i <= toSend*2; i++ { sendStreamMsg(t, nc, sendSubj, strconv.Itoa(i)) } state = mset.state() if state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d messages, got %d", toSend*2, state.Msgs) } checkMsgs(101) checkSubEmpty := func() { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 { t.Fatalf("Expected sub to have no pending") } } checkSubEmpty() o.delete() // Now check for deliver last, deliver new and deliver by seq. o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverPolicy: DeliverLast}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Did not get expected message, got %v", err) } // All Consumers start with sequence #1. if seq := o.seqFromReply(m.Reply); seq != 1 { t.Fatalf("Expected sequence to be 1, but got %d", seq) } // Check that is is the last msg we sent though. if mseq, _ := strconv.Atoi(string(m.Data)); mseq != 200 { t.Fatalf("Expected messag sequence to be 200, but got %d", mseq) } checkSubEmpty() o.delete() // Make sure we only got one message. if m, err := sub.NextMsg(5 * time.Millisecond); err == nil { t.Fatalf("Expected no msg, got %+v", m) } checkSubEmpty() o.delete() // Now try by sequence number. o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverPolicy: DeliverByStartSequence, OptStartSeq: 101}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() checkMsgs(1) // Now do push based queue-subscribers sub, _ = nc2.QueueSubscribeSync("_qg_", "dev") defer sub.Unsubscribe() nc2.Flush() o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverGroup: "dev"}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() // Since we sent another batch need check to be looking for 2x. toSend *= 2 checkMsgs(1) }) } } func workerModeConfig(name string) *ConsumerConfig { return &ConsumerConfig{Durable: name, AckPolicy: AckExplicit} } func TestJetStreamBasicWorkQueue(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(workerModeConfig(oname)) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() if o.nextSeq() != 1 { t.Fatalf("Expected to be starting at sequence 1") } nc := clientConnectWithOldRequest(t, s) defer nc.Close() // Now load up some messages. toSend := 100 sendSubj := "bar" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } getNext := func(seqno int) { t.Helper() nextMsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error for seq %d: %v", seqno, err) } if nextMsg.Subject != "bar" { t.Fatalf("Expected subject of %q, got %q", "bar", nextMsg.Subject) } if seq := o.seqFromReply(nextMsg.Reply); seq != uint64(seqno) { t.Fatalf("Expected sequence of %d , got %d", seqno, seq) } } // Make sure we can get the messages already there. for i := 1; i <= toSend; i++ { getNext(i) } // Now we want to make sure we can get a message that is published to the message // set as we are waiting for it. nextDelay := 50 * time.Millisecond go func() { time.Sleep(nextDelay) sendStreamMsg(t, nc, sendSubj, "Hello World!") }() start := time.Now() getNext(toSend + 1) if time.Since(start) < nextDelay { t.Fatalf("Received message too quickly") } // Now do same thing but combine waiting for new ones with sending. go func() { time.Sleep(nextDelay) for i := 0; i < toSend; i++ { nc.Request(sendSubj, []byte("Hello World!"), 50*time.Millisecond) } }() for i := toSend + 2; i < toSend*2+2; i++ { getNext(i) } }) } } func TestJetStreamWorkQueueMaxWaiting(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Make sure these cases fail cfg := &ConsumerConfig{Durable: "foo", AckPolicy: AckExplicit, MaxWaiting: 10, DeliverSubject: "_INBOX.22"} if _, err := mset.addConsumer(cfg); err == nil { t.Fatalf("Expected an error with MaxWaiting set on non-pull based consumer") } cfg = &ConsumerConfig{Durable: "foo", AckPolicy: AckExplicit, MaxWaiting: -1} if _, err := mset.addConsumer(cfg); err == nil { t.Fatalf("Expected an error with MaxWaiting being negative") } // Create basic work queue mode consumer. wcfg := workerModeConfig("MAXWQ") o, err := mset.addConsumer(wcfg) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() // Make sure we set default correctly. if cfg := o.config(); cfg.MaxWaiting != JSWaitQueueDefaultMax { t.Fatalf("Expected default max waiting to have been set to %d, got %d", JSWaitQueueDefaultMax, cfg.MaxWaiting) } expectWaiting := func(expected int) { t.Helper() checkFor(t, time.Second, 25*time.Millisecond, func() error { if oi := o.info(); oi.NumWaiting != expected { return fmt.Errorf("Expected %d waiting, got %d", expected, oi.NumWaiting) } return nil }) } nc := clientConnectWithOldRequest(t, s) defer nc.Close() // Like muxed new INBOX. sub, _ := nc.SubscribeSync("req.*") defer sub.Unsubscribe() nc.Flush() checkSubPending := func(numExpected int) { t.Helper() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } getSubj := o.requestNextMsgSubject() // Queue up JSWaitQueueDefaultMax requests. for i := 0; i < JSWaitQueueDefaultMax; i++ { nc.PublishRequest(getSubj, fmt.Sprintf("req.%d", i), nil) } expectWaiting(JSWaitQueueDefaultMax) // We are at the max, so we should get a 409 saying that we have // exceeded the number of pull requests. m, err := nc.Request(getSubj, nil, 100*time.Millisecond) require_NoError(t, err) // Make sure this is the 409 if v := m.Header.Get("Status"); v != "409" { t.Fatalf("Expected a 409 status code, got %q", v) } // The sub for the other requests should not have received anything checkSubPending(0) // Now send some messages that should make some of the requests complete sendStreamMsg(t, nc, "foo", "Hello World!") sendStreamMsg(t, nc, "bar", "Hello World!") expectWaiting(JSWaitQueueDefaultMax - 2) }) } } func TestJetStreamWorkQueueWrapWaiting(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() maxWaiting := 8 wcfg := workerModeConfig("WRAP") wcfg.MaxWaiting = maxWaiting o, err := mset.addConsumer(wcfg) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() getSubj := o.requestNextMsgSubject() expectWaiting := func(expected int) { t.Helper() checkFor(t, time.Second, 25*time.Millisecond, func() error { if oi := o.info(); oi.NumWaiting != expected { return fmt.Errorf("Expected %d waiting, got %d", expected, oi.NumWaiting) } return nil }) } nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync("req.*") defer sub.Unsubscribe() nc.Flush() // Fill up waiting. for i := 0; i < maxWaiting; i++ { nc.PublishRequest(getSubj, fmt.Sprintf("req.%d", i), nil) } expectWaiting(maxWaiting) // Now use 1/2 of the waiting. for i := 0; i < maxWaiting/2; i++ { sendStreamMsg(t, nc, "foo", "Hello World!") } expectWaiting(maxWaiting / 2) // Now add in two (2) more pull requests. for i := maxWaiting; i < maxWaiting+2; i++ { nc.PublishRequest(getSubj, fmt.Sprintf("req.%d", i), nil) } expectWaiting(maxWaiting/2 + 2) // Now use second 1/2 of the waiting and the 2 extra. for i := 0; i < maxWaiting/2+2; i++ { sendStreamMsg(t, nc, "bar", "Hello World!") } expectWaiting(0) checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxWaiting+2 { return fmt.Errorf("Expected sub to have %d pending, got %d", maxWaiting+2, nmsgs) } return nil }) }) } } func TestJetStreamWorkQueueRequest(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() o, err := mset.addConsumer(workerModeConfig("WRAP")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 25 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "bar", "Hello World!") } reply := "_.consumer._" sub, _ := nc.SubscribeSync(reply) defer sub.Unsubscribe() getSubj := o.requestNextMsgSubject() checkSubPending := func(numExpected int) { t.Helper() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } // Create a formal request object. req := &JSApiConsumerGetNextRequest{Batch: toSend} jreq, _ := json.Marshal(req) nc.PublishRequest(getSubj, reply, jreq) checkSubPending(toSend) // Now check that we can ask for NoWait req.Batch = 1 req.NoWait = true jreq, _ = json.Marshal(req) resp, err := nc.Request(getSubj, jreq, 100*time.Millisecond) require_NoError(t, err) if status := resp.Header.Get("Status"); !strings.HasPrefix(status, "404") { t.Fatalf("Expected status code of 404") } // Load up more messages. for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo", "Hello World!") } // Now we will ask for a batch larger then what is queued up. req.Batch = toSend + 10 req.NoWait = true jreq, _ = json.Marshal(req) nc.PublishRequest(getSubj, reply, jreq) // We should now have 2 * toSend + the 404 message. checkSubPending(2*toSend + 1) for i := 0; i < 2*toSend+1; i++ { sub.NextMsg(time.Millisecond) } checkSubPending(0) mset.purge(nil) // Now do expiration req.Batch = 1 req.NoWait = false req.Expires = 100 * time.Millisecond jreq, _ = json.Marshal(req) nc.PublishRequest(getSubj, reply, jreq) // Let it expire time.Sleep(200 * time.Millisecond) // Send a few more messages. These should not be delivered to the sub. sendStreamMsg(t, nc, "foo", "Hello World!") sendStreamMsg(t, nc, "bar", "Hello World!") time.Sleep(100 * time.Millisecond) // Expect the request timed out message. checkSubPending(1) if resp, _ = sub.NextMsg(time.Millisecond); resp == nil { t.Fatalf("Expected an expired status message") } if status := resp.Header.Get("Status"); !strings.HasPrefix(status, "408") { t.Fatalf("Expected status code of 408") } // Send a new request, we should not get the 408 because our previous request // should have expired. nc.PublishRequest(getSubj, reply, jreq) checkSubPending(1) sub.NextMsg(time.Second) checkSubPending(0) }) } } func TestJetStreamSubjectFiltering(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MSET", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "MSET", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 50 subjA := "foo.A" subjB := "foo.B" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, subjA, "Hello World!") sendStreamMsg(t, nc, subjB, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d messages, got %d", toSend*2, state.Msgs) } delivery := nats.NewInbox() sub, _ := nc.SubscribeSync(delivery) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: subjB}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() // Now let's check the messages for i := 1; i <= toSend; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) // JetStream will have the subject match the stream subject, not delivery subject. // We want these to only be subjB. if m.Subject != subjB { t.Fatalf("Expected original subject of %q, but got %q", subjB, m.Subject) } // Now check that reply subject exists and has a sequence as the last token. if seq := o.seqFromReply(m.Reply); seq != uint64(i) { t.Fatalf("Expected sequence of %d , got %d", i, seq) } // Ack the message here. m.Respond(nil) } if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 { t.Fatalf("Expected sub to have no pending") } }) } } func TestJetStreamWorkQueueSubjectFiltering(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 50 subjA := "foo.A" subjB := "foo.B" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, subjA, "Hello World!") sendStreamMsg(t, nc, subjB, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d messages, got %d", toSend*2, state.Msgs) } oname := "WQ" o, err := mset.addConsumer(&ConsumerConfig{Durable: oname, FilterSubject: subjA, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() if o.nextSeq() != 1 { t.Fatalf("Expected to be starting at sequence 1") } getNext := func(seqno int) { t.Helper() nextMsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) require_NoError(t, err) if nextMsg.Subject != subjA { t.Fatalf("Expected subject of %q, got %q", subjA, nextMsg.Subject) } if seq := o.seqFromReply(nextMsg.Reply); seq != uint64(seqno) { t.Fatalf("Expected sequence of %d , got %d", seqno, seq) } nextMsg.Respond(nil) } // Make sure we can get the messages already there. for i := 1; i <= toSend; i++ { getNext(i) } }) } } func TestJetStreamWildcardSubjectFiltering(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "ORDERS", Storage: MemoryStorage, Subjects: []string{"orders.*.*"}}}, {"FileStore", &StreamConfig{Name: "ORDERS", Storage: FileStorage, Subjects: []string{"orders.*.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 100 for i := 1; i <= toSend; i++ { subj := fmt.Sprintf("orders.%d.%s", i, "NEW") sendStreamMsg(t, nc, subj, "new order") } // Randomly move 25 to shipped. toShip := 25 shipped := make(map[int]bool) for i := 0; i < toShip; { orderId := rand.Intn(toSend-1) + 1 if shipped[orderId] { continue } subj := fmt.Sprintf("orders.%d.%s", orderId, "SHIPPED") sendStreamMsg(t, nc, subj, "shipped order") shipped[orderId] = true i++ } state := mset.state() if state.Msgs != uint64(toSend+toShip) { t.Fatalf("Expected %d messages, got %d", toSend+toShip, state.Msgs) } delivery := nats.NewInbox() sub, _ := nc.SubscribeSync(delivery) defer sub.Unsubscribe() nc.Flush() // Get all shipped. o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: "orders.*.SHIPPED"}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() checkFor(t, time.Second, 25*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toShip { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toShip) } return nil }) for nmsgs, _, _ := sub.Pending(); nmsgs > 0; nmsgs, _, _ = sub.Pending() { sub.NextMsg(time.Second) } if nmsgs, _, _ := sub.Pending(); nmsgs != 0 { t.Fatalf("Expected no pending, got %d", nmsgs) } // Get all new o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: "orders.*.NEW"}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() checkFor(t, time.Second, 25*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) for nmsgs, _, _ := sub.Pending(); nmsgs > 0; nmsgs, _, _ = sub.Pending() { sub.NextMsg(time.Second) } if nmsgs, _, _ := sub.Pending(); nmsgs != 0 { t.Fatalf("Expected no pending, got %d", nmsgs) } // Now grab a single orderId that has shipped, so we should have two messages. var orderId int for orderId = range shipped { break } subj := fmt.Sprintf("orders.%d.*", orderId) o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: subj}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() checkFor(t, time.Second, 25*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 2 { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, 2) } return nil }) }) } } func TestJetStreamWorkQueueAckAndNext(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(workerModeConfig(oname)) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() if o.nextSeq() != 1 { t.Fatalf("Expected to be starting at sequence 1") } nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 sendSubj := "bar" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() // Kick things off. // For normal work queue semantics, you send requests to the subject with stream and consumer name. // We will do this to start it off then use ack+next to get other messages. nc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, nil) for i := 0; i < toSend; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error waiting for messages: %v", err) } if !bytes.Equal(m.Data, []byte("Hello World!")) { t.Fatalf("Got an invalid message from the stream: %q", m.Data) } nc.PublishRequest(m.Reply, sub.Subject, AckNext) } }) } } func TestJetStreamWorkQueueRequestBatch(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_MSG_SET", Storage: MemoryStorage, Subjects: []string{"foo", "bar"}}}, {"FileStore", &StreamConfig{Name: "MY_MSG_SET", Storage: FileStorage, Subjects: []string{"foo", "bar"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(workerModeConfig(oname)) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() if o.nextSeq() != 1 { t.Fatalf("Expected to be starting at sequence 1") } nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 sendSubj := "bar" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() // For normal work queue semantics, you send requests to the subject with stream and consumer name. // We will do this to start it off then use ack+next to get other messages. // Kick things off with batch size of 50. batchSize := 50 nc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte(strconv.Itoa(batchSize))) // We should receive batchSize with no acks or additional requests. checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != batchSize { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, batchSize) } return nil }) // Now queue up the request without messages and add them after. sub, _ = nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() mset.purge(nil) nc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte(strconv.Itoa(batchSize))) nc.Flush() // Make sure its registered. for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } // We should receive batchSize with no acks or additional requests. checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != batchSize { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, batchSize) } return nil }) }) } } func TestJetStreamWorkQueueRetentionStream(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "MWQ", Storage: MemoryStorage, Subjects: []string{"MY_WORK_QUEUE.>"}, Retention: WorkQueuePolicy}, }, {name: "FileStore", mconfig: &StreamConfig{ Name: "MWQ", Storage: FileStorage, Subjects: []string{"MY_WORK_QUEUE.>"}, Retention: WorkQueuePolicy}, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // This type of stream has restrictions which we will test here. // DeliverAll is only start mode allowed. if _, err := mset.addConsumer(&ConsumerConfig{DeliverPolicy: DeliverLast}); err == nil { t.Fatalf("Expected an error with anything but DeliverAll") } // We will create a non-partitioned consumer. This should succeed. o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit}) require_NoError(t, err) defer o.delete() // Now if we create another this should fail, only can have one non-partitioned. if _, err := mset.addConsumer(&ConsumerConfig{}); err == nil { t.Fatalf("Expected an error on attempt for second consumer for a workqueue") } o.delete() if numo := mset.numConsumers(); numo != 0 { t.Fatalf("Expected to have zero consumers, got %d", numo) } // Now add in an consumer that has a partition. pindex := 1 pConfig := func(pname string) *ConsumerConfig { dname := fmt.Sprintf("PPBO-%d", pindex) pindex += 1 return &ConsumerConfig{Durable: dname, FilterSubject: pname, AckPolicy: AckExplicit} } o, err = mset.addConsumer(pConfig("MY_WORK_QUEUE.A")) require_NoError(t, err) defer o.delete() // Now creating another with separate partition should work. o2, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.B")) require_NoError(t, err) defer o2.delete() // Anything that would overlap should fail though. if _, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.A")); err == nil { t.Fatalf("Expected an error on attempt for partitioned consumer for a workqueue") } if _, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.B")); err == nil { t.Fatalf("Expected an error on attempt for partitioned consumer for a workqueue") } o3, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.C")) require_NoError(t, err) o.delete() o2.delete() o3.delete() // Test with wildcards, first from wider to narrower o, err = mset.addConsumer(pConfig("MY_WORK_QUEUE.>")) require_NoError(t, err) if _, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.*.BAR")); err == nil { t.Fatalf("Expected an error on attempt for partitioned consumer for a workqueue") } o.delete() // Now from narrower to wider o, err = mset.addConsumer(pConfig("MY_WORK_QUEUE.*.BAR")) require_NoError(t, err) if _, err := mset.addConsumer(pConfig("MY_WORK_QUEUE.>")); err == nil { t.Fatalf("Expected an error on attempt for partitioned consumer for a workqueue") } o.delete() // Push based will be allowed now, including ephemerals. // They can not overlap etc meaning same rules as above apply. o4, err := mset.addConsumer(&ConsumerConfig{ Durable: "DURABLE", DeliverSubject: "SOME.SUBJ", AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Unexpected Error: %v", err) } defer o4.delete() // Now try to create an ephemeral nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() // This should fail at first due to conflict above. ephCfg := &ConsumerConfig{DeliverSubject: sub.Subject, AckPolicy: AckExplicit} if _, err := mset.addConsumer(ephCfg); err == nil { t.Fatalf("Expected an error ") } // Delete of o4 should clear. o4.delete() o5, err := mset.addConsumer(ephCfg) if err != nil { t.Fatalf("Unexpected Error: %v", err) } defer o5.delete() }) } } func TestJetStreamAckAllRedelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_S22", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "MY_S22", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: sub.Subject, AckWait: 50 * time.Millisecond, AckPolicy: AckAll, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() // Wait for messages. // We will do 5 redeliveries. for i := 1; i <= 5; i++ { checkFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend*i { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend*i) } return nil }) } // Stop redeliveries. o.delete() // Now make sure that they are all redelivered in order for each redelivered batch. for l := 1; l <= 5; l++ { for i := 1; i <= toSend; i++ { m, _ := sub.NextMsg(time.Second) if seq := o.streamSeqFromReply(m.Reply); seq != uint64(i) { t.Fatalf("Expected stream sequence of %d, got %d", i, seq) } } } }) } } func TestJetStreamAckReplyStreamPending(t *testing.T) { msc := StreamConfig{ Name: "MY_WQ", Subjects: []string{"foo.*"}, Storage: MemoryStorage, MaxAge: 1 * time.Second, Retention: WorkQueuePolicy, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.1", "Hello World!") } nc.Flush() state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit}) require_NoError(t, err) defer o.delete() expectPending := func(ep int) { t.Helper() // Now check consumer info. checkFor(t, time.Second, 10*time.Millisecond, func() error { if info, pep := o.info(), ep+1; int(info.NumPending) != pep { return fmt.Errorf("Expected consumer info pending of %d, got %d", pep, info.NumPending) } return nil }) m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, _, _, _, pending := replyInfo(m.Reply) if pending != uint64(ep) { t.Fatalf("Expected ack reply pending of %d, got %d - reply: %q", ep, pending, m.Reply) } } expectPending(toSend - 1) // Send some more while we are connected. for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.1", "Hello World!") } nc.Flush() expectPending(toSend*2 - 2) // Purge and send a new one. mset.purge(nil) nc.Flush() sendStreamMsg(t, nc, "foo.1", "Hello World!") expectPending(0) for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.22", "Hello World!") } expectPending(toSend - 1) // 201 // Test that delete will not register for consumed messages. mset.removeMsg(mset.state().FirstSeq) expectPending(toSend - 2) // 202 // Now remove one that has not been delivered. mset.removeMsg(250) expectPending(toSend - 4) // 203 // Test Expiration. mset.purge(nil) for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.1", "Hello World!") } nc.Flush() // Wait for expiration to kick in. checkFor(t, 5*time.Second, time.Second, func() error { if state := mset.state(); state.Msgs != 0 { return fmt.Errorf("Stream still has messages") } return nil }) sendStreamMsg(t, nc, "foo.33", "Hello World!") expectPending(0) // Now do filtered consumers. o.delete() o, err = mset.addConsumer(&ConsumerConfig{Durable: "PBO-FILTERED", AckPolicy: AckExplicit, FilterSubject: "foo.22"}) require_NoError(t, err) defer o.delete() for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.33", "Hello World!") } nc.Flush() if info := o.info(); info.NumPending != 0 { t.Fatalf("Expected no pending, got %d", info.NumPending) } // Now send one message that will match us. sendStreamMsg(t, nc, "foo.22", "Hello World!") expectPending(0) sendStreamMsg(t, nc, "foo.22", "Hello World!") // 504 sendStreamMsg(t, nc, "foo.22", "Hello World!") // 505 sendStreamMsg(t, nc, "foo.22", "Hello World!") // 506 sendStreamMsg(t, nc, "foo.22", "Hello World!") // 507 expectPending(3) mset.removeMsg(506) expectPending(1) for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.22", "Hello World!") } nc.Flush() expectPending(100) mset.purge(nil) sendStreamMsg(t, nc, "foo.22", "Hello World!") expectPending(0) }) } } func TestJetStreamAckReplyStreamPendingWithAcks(t *testing.T) { msc := StreamConfig{ Name: "MY_STREAM", Subjects: []string{"foo", "bar", "baz"}, Storage: MemoryStorage, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 500 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo", "Hello Foo!") sendStreamMsg(t, nc, "bar", "Hello Bar!") sendStreamMsg(t, nc, "baz", "Hello Baz!") } state := mset.state() if state.Msgs != uint64(toSend*3) { t.Fatalf("Expected %d messages, got %d", toSend*3, state.Msgs) } dsubj := "_d_" o, err := mset.addConsumer(&ConsumerConfig{ Durable: "D-1", AckPolicy: AckExplicit, FilterSubject: "foo", DeliverSubject: dsubj, }) require_NoError(t, err) defer o.delete() if info := o.info(); int(info.NumPending) != toSend { t.Fatalf("Expected consumer info pending of %d, got %d", toSend, info.NumPending) } sub, _ := nc.SubscribeSync(dsubj) defer sub.Unsubscribe() checkFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) // Should be zero. if info := o.info(); int(info.NumPending) != 0 { t.Fatalf("Expected consumer info pending of %d, got %d", 0, info.NumPending) } else if info.NumAckPending != toSend { t.Fatalf("Expected %d to be pending acks, got %d", toSend, info.NumAckPending) } }) } } func TestJetStreamWorkQueueAckWaitRedelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage, Retention: WorkQueuePolicy}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage, Retention: WorkQueuePolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } ackWait := 100 * time.Millisecond o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit, AckWait: ackWait}) require_NoError(t, err) defer o.delete() sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() reqNextMsgSubj := o.requestNextMsgSubject() // Consume all the messages. But do not ack. for i := 0; i < toSend; i++ { nc.PublishRequest(reqNextMsgSubj, sub.Subject, nil) if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error waiting for messages: %v", err) } } if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 { t.Fatalf("Did not consume all messages, still have %d", nmsgs) } // All messages should still be there. state = mset.state() if int(state.Msgs) != toSend { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } // Now consume and ack. for i := 1; i <= toSend; i++ { nc.PublishRequest(reqNextMsgSubj, sub.Subject, nil) m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error waiting for message[%d]: %v", i, err) } sseq, dseq, dcount, _, _ := replyInfo(m.Reply) if sseq != uint64(i) { t.Fatalf("Expected set sequence of %d , got %d", i, sseq) } // Delivery sequences should always increase. if dseq != uint64(toSend+i) { t.Fatalf("Expected delivery sequence of %d , got %d", toSend+i, dseq) } if dcount == 1 { t.Fatalf("Expected these to be marked as redelivered") } // Ack the message here. m.AckSync() } if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 { t.Fatalf("Did not consume all messages, still have %d", nmsgs) } // Flush acks nc.Flush() // Now check the mset as well, since we have a WorkQueue retention policy this should be empty. if state := mset.state(); state.Msgs != 0 { t.Fatalf("Expected no messages, got %d", state.Msgs) } }) } } func TestJetStreamWorkQueueNakRedelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage, Retention: WorkQueuePolicy}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage, Retention: WorkQueuePolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 10 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit}) require_NoError(t, err) defer o.delete() getMsg := func(sseq, dseq int) *nats.Msg { t.Helper() m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rsseq, rdseq, _, _, _ := replyInfo(m.Reply) if rdseq != uint64(dseq) { t.Fatalf("Expected delivered sequence of %d , got %d", dseq, rdseq) } if rsseq != uint64(sseq) { t.Fatalf("Expected store sequence of %d , got %d", sseq, rsseq) } return m } for i := 1; i <= 5; i++ { m := getMsg(i, i) // Ack the message here. m.Respond(nil) } // Grab #6 m := getMsg(6, 6) // NAK this one and make sure its processed. m.Respond(AckNak) nc.Flush() // When we request again should be store sequence 6 again. getMsg(6, 7) // Then we should get 7, 8, etc. getMsg(7, 8) getMsg(8, 9) }) } } func TestJetStreamWorkQueueWorkingIndicator(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage, Retention: WorkQueuePolicy}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage, Retention: WorkQueuePolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 2 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } ackWait := 100 * time.Millisecond o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit, AckWait: ackWait}) require_NoError(t, err) defer o.delete() getMsg := func(sseq, dseq int) *nats.Msg { t.Helper() m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rsseq, rdseq, _, _, _ := replyInfo(m.Reply) if rdseq != uint64(dseq) { t.Fatalf("Expected delivered sequence of %d , got %d", dseq, rdseq) } if rsseq != uint64(sseq) { t.Fatalf("Expected store sequence of %d , got %d", sseq, rsseq) } return m } getMsg(1, 1) // Now wait past ackWait time.Sleep(ackWait * 2) // We should get 1 back. m := getMsg(1, 2) // Now let's take longer than ackWait to process but signal we are working on the message. timeout := time.Now().Add(3 * ackWait) for time.Now().Before(timeout) { m.Respond(AckProgress) nc.Flush() time.Sleep(ackWait / 5) } // We should get 2 here, not 1 since we have indicated we are working on it. m2 := getMsg(2, 3) time.Sleep(ackWait / 2) m2.Respond(AckProgress) // Now should get 1 back then 2. m = getMsg(1, 4) m.Respond(nil) getMsg(2, 5) }) } } func TestJetStreamWorkQueueTerminateDelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "MY_WQ", Storage: MemoryStorage, Retention: WorkQueuePolicy}}, {"FileStore", &StreamConfig{Name: "MY_WQ", Storage: FileStorage, Retention: WorkQueuePolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 22 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, c.mconfig.Name, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } ackWait := 25 * time.Millisecond o, err := mset.addConsumer(&ConsumerConfig{Durable: "PBO", AckPolicy: AckExplicit, AckWait: ackWait}) require_NoError(t, err) defer o.delete() getMsg := func(sseq, dseq int) *nats.Msg { t.Helper() m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rsseq, rdseq, _, _, _ := replyInfo(m.Reply) if rdseq != uint64(dseq) { t.Fatalf("Expected delivered sequence of %d , got %d", dseq, rdseq) } if rsseq != uint64(sseq) { t.Fatalf("Expected store sequence of %d , got %d", sseq, rsseq) } return m } // Make sure we get the correct advisory sub, _ := nc.SubscribeSync(JSAdvisoryConsumerMsgTerminatedPre + ".>") defer sub.Unsubscribe() getMsg(1, 1) // Now wait past ackWait time.Sleep(ackWait * 2) // We should get 1 back. m := getMsg(1, 2) // Now terminate m.Respond([]byte(fmt.Sprintf("%s with reason", string(AckTerm)))) time.Sleep(ackWait * 2) // We should get 2 here, not 1 since we have indicated we wanted to terminate. getMsg(2, 3) // Check advisory was delivered. am, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var adv JSConsumerDeliveryTerminatedAdvisory json.Unmarshal(am.Data, &adv) if adv.Stream != "MY_WQ" { t.Fatalf("Expected stream of %s, got %s", "MY_WQ", adv.Stream) } if adv.Consumer != "PBO" { t.Fatalf("Expected consumer of %s, got %s", "PBO", adv.Consumer) } if adv.StreamSeq != 1 { t.Fatalf("Expected stream sequence of %d, got %d", 1, adv.StreamSeq) } if adv.ConsumerSeq != 2 { t.Fatalf("Expected consumer sequence of %d, got %d", 2, adv.ConsumerSeq) } if adv.Deliveries != 2 { t.Fatalf("Expected delivery count of %d, got %d", 2, adv.Deliveries) } if adv.Reason != "with reason" { t.Fatalf("Advisory did not have a reason") } }) } } func TestJetStreamConsumerAckAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "ACK-ACK" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() o, err := mset.addConsumer(&ConsumerConfig{Durable: "worker", AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() rqn := o.requestNextMsgSubject() nc := clientConnectToServer(t, s) defer nc.Close() // 4 for number of ack protocols to test them all. for i := 0; i < 4; i++ { sendStreamMsg(t, nc, mname, "Hello World!") } testAck := func(ackType []byte) { m, err := nc.Request(rqn, nil, 10*time.Millisecond) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send a request for the ack and make sure the server "ack's" the ack. if _, err := nc.Request(m.Reply, ackType, 10*time.Millisecond); err != nil { t.Fatalf("Unexpected error on ack/ack: %v", err) } } testAck(AckAck) testAck(AckNak) testAck(AckProgress) testAck(AckTerm) } func TestJetStreamAckNext(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "ACKNXT" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() o, err := mset.addConsumer(&ConsumerConfig{Durable: "worker", AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nc := clientConnectToServer(t, s) defer nc.Close() for i := 0; i < 12; i++ { sendStreamMsg(t, nc, mname, fmt.Sprintf("msg %d", i)) } q := make(chan *nats.Msg, 10) sub, err := nc.ChanSubscribe(nats.NewInbox(), q) if err != nil { t.Fatalf("SubscribeSync failed: %s", err) } nc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte("1")) // normal next should imply 1 msg := <-q err = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: AckNext}) if err != nil { t.Fatalf("RespondMsg failed: %s", err) } // read 1 message and check ack was done etc msg = <-q if len(q) != 0 { t.Fatalf("Expected empty q got %d", len(q)) } if o.info().AckFloor.Stream != 1 { t.Fatalf("First message was not acknowledged") } if !bytes.Equal(msg.Data, []byte("msg 1")) { t.Fatalf("wrong message received, expected: msg 1 got %q", msg.Data) } // now ack and request 5 more using a naked number err = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: append(AckNext, []byte(" 5")...)}) if err != nil { t.Fatalf("RespondMsg failed: %s", err) } getMsgs := func(start, count int) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() for i := start; i < count+1; i++ { select { case msg := <-q: expect := fmt.Sprintf("msg %d", i+1) if !bytes.Equal(msg.Data, []byte(expect)) { t.Fatalf("wrong message received, expected: %s got %#v", expect, msg) } case <-ctx.Done(): t.Fatalf("did not receive all messages") } } } getMsgs(1, 5) // now ack and request 5 more using the full request err = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: append(AckNext, []byte(`{"batch": 5}`)...)}) if err != nil { t.Fatalf("RespondMsg failed: %s", err) } getMsgs(6, 10) if o.info().AckFloor.Stream != 2 { t.Fatalf("second message was not acknowledged") } } func TestJetStreamPublishDeDupe(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "DeDupe" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage, MaxAge: time.Hour, Subjects: []string{"foo.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Check Duplicates setting. duplicates := mset.config().Duplicates if duplicates != StreamDefaultDuplicatesWindow { t.Fatalf("Expected a default of %v, got %v", StreamDefaultDuplicatesWindow, duplicates) } cfg := mset.config() // Make sure can't be negative. cfg.Duplicates = -25 * time.Millisecond if err := mset.update(&cfg); err == nil { t.Fatalf("Expected an error but got none") } // Make sure can't be longer than age if its set. cfg.Duplicates = 2 * time.Hour if err := mset.update(&cfg); err == nil { t.Fatalf("Expected an error but got none") } nc := clientConnectToServer(t, s) defer nc.Close() sendMsg := func(seq uint64, id, msg string) *PubAck { t.Helper() m := nats.NewMsg(fmt.Sprintf("foo.%d", seq)) m.Header.Add(JSMsgId, id) m.Data = []byte(msg) resp, _ := nc.RequestMsg(m, 100*time.Millisecond) if resp == nil { t.Fatalf("No response for %q, possible timeout?", msg) } pa := getPubAckResponse(resp.Data) if pa == nil || pa.Error != nil { t.Fatalf("Expected a JetStreamPubAck, got %q", resp.Data) } if pa.Sequence != seq { t.Fatalf("Did not get correct sequence in PubAck, expected %d, got %d", seq, pa.Sequence) } return pa.PubAck } expect := func(n uint64) { t.Helper() state := mset.state() if state.Msgs != n { t.Fatalf("Expected %d messages, got %d", n, state.Msgs) } } sendMsg(1, "AA", "Hello DeDupe!") sendMsg(2, "BB", "Hello DeDupe!") sendMsg(3, "CC", "Hello DeDupe!") sendMsg(4, "ZZ", "Hello DeDupe!") expect(4) sendMsg(1, "AA", "Hello DeDupe!") sendMsg(2, "BB", "Hello DeDupe!") sendMsg(4, "ZZ", "Hello DeDupe!") expect(4) cfg = mset.config() cfg.Duplicates = 100 * time.Millisecond if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } nmids := func(expected int) { t.Helper() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nids := mset.numMsgIds(); nids != expected { return fmt.Errorf("Expected %d message ids, got %d", expected, nids) } return nil }) } nmids(4) time.Sleep(cfg.Duplicates * 2) sendMsg(5, "AAA", "Hello DeDupe!") sendMsg(6, "BBB", "Hello DeDupe!") sendMsg(7, "CCC", "Hello DeDupe!") sendMsg(8, "DDD", "Hello DeDupe!") sendMsg(9, "ZZZ", "Hello DeDupe!") nmids(5) // Eventually will drop to zero. nmids(0) // Now test server restart cfg.Duplicates = 30 * time.Minute if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } mset.purge(nil) // Send 5 new messages. sendMsg(10, "AAAA", "Hello DeDupe!") sendMsg(11, "BBBB", "Hello DeDupe!") sendMsg(12, "CCCC", "Hello DeDupe!") sendMsg(13, "DDDD", "Hello DeDupe!") sendMsg(14, "EEEE", "Hello DeDupe!") // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc = clientConnectToServer(t, s) defer nc.Close() mset, _ = s.GlobalAccount().lookupStream(mname) if nms := mset.state().Msgs; nms != 5 { t.Fatalf("Expected 5 restored messages, got %d", nms) } nmids(5) // Send same and make sure duplicate detection still works. // Send 5 duplicate messages. sendMsg(10, "AAAA", "Hello DeDupe!") sendMsg(11, "BBBB", "Hello DeDupe!") sendMsg(12, "CCCC", "Hello DeDupe!") sendMsg(13, "DDDD", "Hello DeDupe!") sendMsg(14, "EEEE", "Hello DeDupe!") if nms := mset.state().Msgs; nms != 5 { t.Fatalf("Expected 5 restored messages, got %d", nms) } nmids(5) // Check we set duplicate properly. pa := sendMsg(10, "AAAA", "Hello DeDupe!") if !pa.Duplicate { t.Fatalf("Expected duplicate to be set") } // Purge should NOT wipe the msgIds. They should still persist. mset.purge(nil) nmids(5) } func getPubAckResponse(msg []byte) *JSPubAckResponse { var par JSPubAckResponse if err := json.Unmarshal(msg, &par); err != nil { return nil } return &par } func TestJetStreamPublishExpect(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "EXPECT" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage, MaxAge: time.Hour, Subjects: []string{"foo.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Test that we get no error when expected stream is correct. m := nats.NewMsg("foo.bar") m.Data = []byte("HELLO") m.Header.Set(JSExpectedStream, mname) resp, err := nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error != nil { t.Fatalf("Expected a valid JetStreamPubAck, got %q", resp.Data) } // Now test that we get an error back when expecting a different stream. m.Header.Set(JSExpectedStream, "ORDERS") resp, err = nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil { t.Fatalf("Expected an error, got %q", resp.Data) } // Now test that we get an error back when expecting a different sequence number. m.Header.Set(JSExpectedStream, mname) m.Header.Set(JSExpectedLastSeq, "10") resp, err = nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil { t.Fatalf("Expected an error, got %q", resp.Data) } // Or if we expect that there are no messages by setting "0" as the expected last seq m.Header.Set(JSExpectedLastSeq, "0") resp, err = nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil { t.Fatalf("Expected an error, got %q", resp.Data) } // Now send a message with a message ID and make sure we can match that. m = nats.NewMsg("foo.bar") m.Data = []byte("HELLO") m.Header.Set(JSMsgId, "AAA") if _, err = nc.RequestMsg(m, 100*time.Millisecond); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now try again with new message ID but require last one to be 'BBB' m.Header.Set(JSMsgId, "ZZZ") m.Header.Set(JSExpectedLastMsgId, "BBB") resp, err = nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil { t.Fatalf("Expected an error, got %q", resp.Data) } // Restart the server and make sure we remember/rebuild last seq and last msgId. // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc = clientConnectToServer(t, s) defer nc.Close() // Our last sequence was 2 and last msgId was "AAA" m = nats.NewMsg("foo.baz") m.Data = []byte("HELLO AGAIN") m.Header.Set(JSExpectedLastSeq, "2") m.Header.Set(JSExpectedLastMsgId, "AAA") m.Header.Set(JSMsgId, "BBB") resp, err = nc.RequestMsg(m, 100*time.Millisecond) require_NoError(t, err) if pa := getPubAckResponse(resp.Data); pa == nil || pa.Error != nil { t.Fatalf("Expected a valid JetStreamPubAck, got %q", resp.Data) } } func TestJetStreamPullConsumerRemoveInterest(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MYS-PULL" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() wcfg := &ConsumerConfig{Durable: "worker", AckPolicy: AckExplicit} o, err := mset.addConsumer(wcfg) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } rqn := o.requestNextMsgSubject() defer o.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Ask for a message even though one is not there. This will queue us up for waiting. if _, err := nc.Request(rqn, nil, 10*time.Millisecond); err == nil { t.Fatalf("Expected an error, got none") } // This is using new style request mechanism. so drop the connection itself to get rid of interest. nc.Close() // Wait for client cleanup checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if n := s.NumClients(); err != nil || n != 0 { return fmt.Errorf("Still have %d clients", n) } return nil }) nc = clientConnectToServer(t, s) defer nc.Close() // Send a message sendStreamMsg(t, nc, mname, "Hello World!") msg, err := nc.Request(rqn, nil, time.Second) require_NoError(t, err) _, dseq, dc, _, _ := replyInfo(msg.Reply) if dseq != 1 { t.Fatalf("Expected consumer sequence of 1, got %d", dseq) } if dc != 1 { t.Fatalf("Expected delivery count of 1, got %d", dc) } // Now do old school request style and more than one waiting. nc = clientConnectWithOldRequest(t, s) defer nc.Close() // Now queue up 10 waiting via failed requests. for i := 0; i < 10; i++ { if _, err := nc.Request(rqn, nil, 1*time.Millisecond); err == nil { t.Fatalf("Expected an error, got none") } } // Send a second message sendStreamMsg(t, nc, mname, "Hello World!") msg, err = nc.Request(rqn, nil, time.Second) require_NoError(t, err) _, dseq, dc, _, _ = replyInfo(msg.Reply) if dseq != 2 { t.Fatalf("Expected consumer sequence of 2, got %d", dseq) } if dc != 1 { t.Fatalf("Expected delivery count of 1, got %d", dc) } } func TestJetStreamConsumerRateLimit(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "RATELIMIT" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() msgSize := 128 * 1024 msg := make([]byte, msgSize) crand.Read(msg) // 10MB totalSize := 10 * 1024 * 1024 toSend := totalSize / msgSize for i := 0; i < toSend; i++ { nc.Publish(mname, msg) } nc.Flush() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { state := mset.state() if state.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d messages, got %d", toSend, state.Msgs) } return nil }) // 100Mbit rateLimit := uint64(100 * 1024 * 1024) // Make sure if you set a rate with a pull based consumer it errors. _, err = mset.addConsumer(&ConsumerConfig{Durable: "to", AckPolicy: AckExplicit, RateLimit: rateLimit}) if err == nil { t.Fatalf("Expected an error, got none") } // Now create one and measure the rate delivered. o, err := mset.addConsumer(&ConsumerConfig{ Durable: "rate", DeliverSubject: "to", RateLimit: rateLimit, AckPolicy: AckNone}) require_NoError(t, err) defer o.delete() var received int done := make(chan bool) start := time.Now() nc.Subscribe("to", func(m *nats.Msg) { received++ if received >= toSend { done <- true } }) nc.Flush() select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive all the messages in time") } tt := time.Since(start) rate := float64(8*toSend*msgSize) / tt.Seconds() if rate > float64(rateLimit)*1.25 { t.Fatalf("Exceeded desired rate of %d mbps, got %0.f mbps", rateLimit/(1024*1024), rate/(1024*1024)) } } func TestJetStreamEphemeralConsumerRecoveryAfterServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MYS" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: sub.Subject, AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } defer o.delete() // Snapshot our name. oname := o.String() // Send 100 messages for i := 0; i < 100; i++ { sendStreamMsg(t, nc, mname, "Hello World!") } if state := mset.state(); state.Msgs != 100 { t.Fatalf("Expected %d messages, got %d", 100, state.Msgs) } // Read 6 messages for i := 0; i <= 6; i++ { if m, err := sub.NextMsg(time.Second); err == nil { m.Respond(nil) } else { t.Fatalf("Unexpected error: %v", err) } } // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) restartServer := func() { t.Helper() // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) } // Do twice for i := 0; i < 2; i++ { // Restart. restartServer() defer s.Shutdown() mset, err = s.GlobalAccount().lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } o = mset.lookupConsumer(oname) if o == nil { t.Fatalf("Error looking up consumer %q", oname) } // Make sure config does not have durable. if cfg := o.config(); cfg.Durable != _EMPTY_ { t.Fatalf("Expected no durable to be set") } // Wait for it to become active checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if !o.isActive() { return fmt.Errorf("Consumer not active") } return nil }) } // Now close the connection. Make sure this acts like an ephemeral and goes away. o.setInActiveDeleteThreshold(10 * time.Millisecond) nc.Close() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if o := mset.lookupConsumer(oname); o != nil { return fmt.Errorf("Consumer still active") } return nil }) } func TestJetStreamConsumerMaxDeliveryAndServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MYS" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() streamCreated := mset.createdTime() dsubj := "D.TO" max := 3 o, err := mset.addConsumer(&ConsumerConfig{ Durable: "TO", DeliverSubject: dsubj, AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, MaxDeliver: max, }) defer o.delete() consumerCreated := o.createdTime() // For calculation of consumer created times below. time.Sleep(5 * time.Millisecond) nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(dsubj) nc.Flush() defer sub.Unsubscribe() // Send one message. sendStreamMsg(t, nc, mname, "order-1") checkSubPending := func(numExpected int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } checkNumMsgs := func(numExpected uint64) { t.Helper() mset, err = s.GlobalAccount().lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } state := mset.state() if state.Msgs != numExpected { t.Fatalf("Expected %d msgs, got %d", numExpected, state.Msgs) } } // Wait til we know we have max queued up. checkSubPending(max) // Once here we have gone over the limit for the 1st message for max deliveries. // Send second sendStreamMsg(t, nc, mname, "order-2") // Just wait for first delivery + one redelivery. checkSubPending(max + 2) // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) restartServer := func() { t.Helper() sd := s.JetStreamConfig().StoreDir // Stop current s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) } waitForClientReconnect := func() { checkFor(t, 2500*time.Millisecond, 5*time.Millisecond, func() error { if !nc.IsConnected() { return fmt.Errorf("Not connected") } return nil }) } // Restart. restartServer() defer s.Shutdown() checkNumMsgs(2) // Wait for client to be reconnected. waitForClientReconnect() // Once we are here send third order. sendStreamMsg(t, nc, mname, "order-3") checkNumMsgs(3) // Restart. restartServer() defer s.Shutdown() checkNumMsgs(3) // Wait for client to be reconnected. waitForClientReconnect() // Now we should have max times three on our sub. checkSubPending(max * 3) // Now do some checks on created timestamps. mset, err = s.GlobalAccount().lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } if mset.createdTime() != streamCreated { t.Fatalf("Stream creation time not restored, wanted %v, got %v", streamCreated, mset.createdTime()) } o = mset.lookupConsumer("TO") if o == nil { t.Fatalf("Error looking up consumer: %v", err) } // Consumer created times can have a very small skew. delta := o.createdTime().Sub(consumerCreated) if delta > 5*time.Millisecond { t.Fatalf("Consumer creation time not restored, wanted %v, got %v", consumerCreated, o.createdTime()) } } func TestJetStreamDeleteConsumerAndServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sendSubj := "MYQ" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sendSubj, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(workerModeConfig(oname)) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } // Now delete and then we will restart the o.delete() if numo := mset.numConsumers(); numo != 0 { t.Fatalf("Expected to have zero consumers, got %d", numo) } // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) sd := s.JetStreamConfig().StoreDir // Stop current s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) defer s.Shutdown() mset, err = s.GlobalAccount().lookupStream(sendSubj) if err != nil { t.Fatalf("Expected to find a stream for %q", sendSubj) } if numo := mset.numConsumers(); numo != 0 { t.Fatalf("Expected to have zero consumers, got %d", numo) } } func TestJetStreamRedeliveryAfterServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sendSubj := "MYQ" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sendSubj, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 25 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "TO", DeliverSubject: sub.Subject, AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, }) require_NoError(t, err) defer o.delete() checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) sd := s.JetStreamConfig().StoreDir // Stop current s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) defer s.Shutdown() // Don't wait for reconnect from old client. nc = clientConnectToServer(t, s) defer nc.Close() sub, _ = nc.SubscribeSync(sub.Subject) defer sub.Unsubscribe() checkFor(t, time.Second, 50*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) } func TestJetStreamSnapshots(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MY-STREAM" subjects := []string{"foo", "bar", "baz"} cfg := StreamConfig{ Name: mname, Storage: FileStorage, Subjects: subjects, MaxMsgs: 1000, } acc := s.GlobalAccount() mset, err := acc.addStream(&cfg) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() // Make sure we send some as floor. toSend := rand.Intn(200) + 22 for i := 1; i <= toSend; i++ { msg := fmt.Sprintf("Hello World %d", i) subj := subjects[rand.Intn(len(subjects))] sendStreamMsg(t, nc, subj, msg) } // Create up to 10 consumers. numConsumers := rand.Intn(10) + 1 var obs []obsi for i := 1; i <= numConsumers; i++ { cname := fmt.Sprintf("WQ-%d", i) o, err := mset.addConsumer(workerModeConfig(cname)) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now grab some messages. toReceive := rand.Intn(toSend/2) + 1 for r := 0; r < toReceive; r++ { resp, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp != nil { resp.Respond(nil) } } obs = append(obs, obsi{o.config(), toReceive}) } nc.Flush() // Snapshot state of the stream and consumers. info := info{mset.config(), mset.state(), obs} sr, err := mset.snapshot(5*time.Second, false, true) if err != nil { t.Fatalf("Error getting snapshot: %v", err) } zr := sr.Reader snapshot, err := io.ReadAll(zr) if err != nil { t.Fatalf("Error reading snapshot") } // Try to restore from snapshot with current stream present, should error. r := bytes.NewReader(snapshot) if _, err := acc.RestoreStream(&info.cfg, r); err == nil { t.Fatalf("Expected an error trying to restore existing stream") } else if !strings.Contains(err.Error(), "name already in use") { t.Fatalf("Incorrect error received: %v", err) } // Now delete so we can restore. pusage := acc.JetStreamUsage() mset.delete() r.Reset(snapshot) mset, err = acc.RestoreStream(&info.cfg, r) require_NoError(t, err) // Now compare to make sure they are equal. if nusage := acc.JetStreamUsage(); !reflect.DeepEqual(nusage, pusage) { t.Fatalf("Usage does not match after restore: %+v vs %+v", nusage, pusage) } if state := mset.state(); !reflect.DeepEqual(state, info.state) { t.Fatalf("State does not match: %+v vs %+v", state, info.state) } if cfg := mset.config(); !reflect.DeepEqual(cfg, info.cfg) { t.Fatalf("Configs do not match: %+v vs %+v", cfg, info.cfg) } // Consumers. if mset.numConsumers() != len(info.obs) { t.Fatalf("Number of consumers do not match: %d vs %d", mset.numConsumers(), len(info.obs)) } for _, oi := range info.obs { if o := mset.lookupConsumer(oi.cfg.Durable); o != nil { if uint64(oi.ack+1) != o.nextSeq() { t.Fatalf("[%v] Consumer next seq is not correct: %d vs %d", o.String(), oi.ack+1, o.nextSeq()) } } else { t.Fatalf("Expected to get an consumer") } } // Now try restoring to a different s2 := RunBasicJetStreamServer(t) defer s2.Shutdown() acc = s2.GlobalAccount() r.Reset(snapshot) mset, err = acc.RestoreStream(&info.cfg, r) require_NoError(t, err) o := mset.lookupConsumer("WQ-1") if o == nil { t.Fatalf("Could not lookup consumer") } nc2 := clientConnectToServer(t, s2) defer nc2.Close() // Make sure we can read messages. if _, err := nc2.Request(o.requestNextMsgSubject(), nil, 5*time.Second); err != nil { t.Fatalf("Unexpected error getting next message: %v", err) } } func TestJetStreamSnapshotsAPI(t *testing.T) { lopts := DefaultTestOptions lopts.ServerName = "LS" lopts.Port = -1 lopts.LeafNode.Host = lopts.Host lopts.LeafNode.Port = -1 ls := RunServer(&lopts) defer ls.Shutdown() opts := DefaultTestOptions opts.ServerName = "S" opts.Port = -1 tdir := t.TempDir() opts.JetStream = true opts.JetStreamDomain = "domain" opts.StoreDir = tdir maxStore := int64(1024 * 1024 * 1024) opts.maxStoreSet = true opts.JetStreamMaxStore = maxStore rurl, _ := url.Parse(fmt.Sprintf("nats-leaf://%s:%d", lopts.LeafNode.Host, lopts.LeafNode.Port)) opts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{rurl}}} s := RunServer(&opts) defer s.Shutdown() checkLeafNodeConnected(t, s) mname := "MY-STREAM" subjects := []string{"foo", "bar", "baz"} cfg := StreamConfig{ Name: mname, Storage: FileStorage, Subjects: subjects, MaxMsgs: 1000, } acc := s.GlobalAccount() mset, err := acc.addStreamWithStore(&cfg, &FileStoreConfig{BlockSize: 128}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() toSend := rand.Intn(100) + 1 for i := 1; i <= toSend; i++ { msg := fmt.Sprintf("Hello World %d", i) subj := subjects[rand.Intn(len(subjects))] sendStreamMsg(t, nc, subj, msg) } o, err := mset.addConsumer(workerModeConfig("WQ")) require_NoError(t, err) // Now grab some messages. toReceive := rand.Intn(toSend) + 1 for r := 0; r < toReceive; r++ { resp, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp != nil { resp.Respond(nil) } } // Make sure we get proper error for non-existent request, streams,etc, rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "foo"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) if resp.Error == nil || resp.Error.Code != 400 || resp.Error.Description != "bad request" { t.Fatalf("Did not get correct error response: %+v", resp.Error) } sreq := &JSApiStreamSnapshotRequest{} req, _ := json.Marshal(sreq) rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "foo"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } json.Unmarshal(rmsg.Data, &resp) if resp.Error == nil || resp.Error.Code != 404 || resp.Error.Description != "stream not found" { t.Fatalf("Did not get correct error response: %+v", resp.Error) } rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } json.Unmarshal(rmsg.Data, &resp) if resp.Error == nil || resp.Error.Code != 400 || resp.Error.Description != "deliver subject not valid" { t.Fatalf("Did not get correct error response: %+v", resp.Error) } // Set delivery subject, do not subscribe yet. Want this to be an ok pattern. sreq.DeliverSubject = nats.NewInbox() // Just for test, usually left alone. sreq.ChunkSize = 1024 req, _ = json.Marshal(sreq) rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } resp.Error = nil json.Unmarshal(rmsg.Data, &resp) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } // Check that we have the config and the state. if resp.Config == nil { t.Fatalf("Expected a stream config in the response, got %+v\n", resp) } if resp.State == nil { t.Fatalf("Expected a stream state in the response, got %+v\n", resp) } // Grab state for comparison. state := *resp.State config := *resp.Config // Setup to process snapshot chunks. var snapshot []byte done := make(chan bool) sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- true return } // Could be writing to a file here too. snapshot = append(snapshot, m.Data...) // Flow ack m.Respond(nil) }) defer sub.Unsubscribe() // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } // Now make sure this snapshot is legit. var rresp JSApiStreamRestoreResponse rreq := &JSApiStreamRestoreRequest{ Config: config, State: state, } req, _ = json.Marshal(rreq) // Make sure we get an error since stream still exists. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } json.Unmarshal(rmsg.Data, &rresp) if !IsNatsErr(rresp.Error, JSStreamNameExistRestoreFailedErr) { t.Fatalf("Did not get correct error response: %+v", rresp.Error) } // Delete this stream. mset.delete() // Sending no request message will error now. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), nil, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Make sure to clear. rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error == nil || rresp.Error.Code != 400 || rresp.Error.Description != "bad request" { t.Fatalf("Did not get correct error response: %+v", rresp.Error) } // This should work. rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Make sure to clear. rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } // Can be any size message. var chunk [512]byte for r := bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) } nc.Request(rresp.DeliverSubject, nil, time.Second) mset, err = acc.lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } if !reflect.DeepEqual(mset.state(), state) { t.Fatalf("Did not match states, %+v vs %+v", mset.state(), state) } // Now ask that the stream be checked first. sreq.ChunkSize = 0 sreq.CheckMsgs = true snapshot = snapshot[:0] req, _ = json.Marshal(sreq) if _, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, 5*time.Second); err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } // Now connect through a cluster server and make sure we can get things to work this way as well. // This client, connecting to a leaf without shared system account and domain needs to provide the domain explicitly. nc2 := clientConnectToServer(t, ls) defer nc2.Close() // Wait a bit for interest to propagate. time.Sleep(100 * time.Millisecond) snapshot = snapshot[:0] req, _ = json.Marshal(sreq) rmsg, err = nc2.Request(fmt.Sprintf(strings.ReplaceAll(JSApiStreamSnapshotT, JSApiPrefix, "$JS.domain.API"), mname), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } resp.Error = nil json.Unmarshal(rmsg.Data, &resp) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } // Wait to receive the snapshot. select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("Did not receive our snapshot in time") } // Now do a restore through the new client connection. // Delete this stream first. mset, err = acc.lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } state = mset.state() mset.delete() req, _ = json.Marshal(rreq) rmsg, err = nc2.Request(strings.ReplaceAll(JSApiStreamRestoreT, JSApiPrefix, "$JS.domain.API"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Make sure to clear. rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } // Make sure when we send something without a reply subject the subscription is shutoff. r := bytes.NewReader(snapshot) n, _ := r.Read(chunk[:]) nc2.Publish(rresp.DeliverSubject, chunk[:n]) nc2.Flush() n, _ = r.Read(chunk[:]) if _, err := nc2.Request(rresp.DeliverSubject, chunk[:n], 100*time.Millisecond); err == nil { t.Fatalf("Expected restore subscription to be closed") } req, _ = json.Marshal(rreq) rmsg, err = nc2.Request(strings.ReplaceAll(JSApiStreamRestoreT, JSApiPrefix, "$JS.domain.API"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Make sure to clear. rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } for r := bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } // Make sure other side responds to reply subjects for ack flow. Optional. if _, err := nc2.Request(rresp.DeliverSubject, chunk[:n], time.Second); err != nil { t.Fatalf("Restore not honoring reply subjects for ack flow") } } // For EOF this will send back stream info or an error. si, err := nc2.Request(rresp.DeliverSubject, nil, time.Second) if err != nil { t.Fatalf("Got an error restoring stream: %v", err) } var scResp JSApiStreamCreateResponse if err := json.Unmarshal(si.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.Error != nil { t.Fatalf("Got an unexpected error from EOF on restore: %+v", scResp.Error) } if !reflect.DeepEqual(scResp.StreamInfo.State, state) { t.Fatalf("Did not match states, %+v vs %+v", scResp.StreamInfo.State, state) } // Now make sure that if we try to change the name/identity of the stream we get an error. mset, err = acc.lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } mset.state() mset.delete() rreq.Config.Name = "NEW_STREAM" req, _ = json.Marshal(rreq) rmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, rreq.Config.Name), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } // Make sure to clear. rresp.Error = nil json.Unmarshal(rmsg.Data, &rresp) // We should not get an error here. if rresp.Error != nil { t.Fatalf("Got an unexpected error response: %+v", rresp.Error) } for r := bytes.NewReader(snapshot); ; { n, err := r.Read(chunk[:]) if err != nil { break } nc.Request(rresp.DeliverSubject, chunk[:n], time.Second) } si, err = nc2.Request(rresp.DeliverSubject, nil, time.Second) require_NoError(t, err) scResp.Error = nil if err := json.Unmarshal(si.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.Error == nil { t.Fatalf("Expected an error but got none") } expect := "names do not match" if !strings.Contains(scResp.Error.Description, expect) { t.Fatalf("Expected description of %q, got %q", expect, scResp.Error.Description) } } func TestJetStreamPubAckPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "foo", Storage: nats.MemoryStorage}); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 1_000_000 start := time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } func TestJetStreamPubPerfWithFullStream(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() toSend, msg := 1_000_000, []byte("OK") _, err := js.AddStream(&nats.StreamConfig{Name: "foo", MaxMsgs: int64(toSend)}) require_NoError(t, err) start := time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", msg) } <-js.PublishAsyncComplete() tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) // Now do same amount but knowing we are at our limit. start = time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", msg) } <-js.PublishAsyncComplete() tt = time.Since(start) fmt.Printf("\ntime is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } func TestJetStreamSnapshotsAPIPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() cfg := StreamConfig{ Name: "snap-perf", Storage: FileStorage, } acc := s.GlobalAccount() if _, err := acc.addStream(&cfg); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() msg := make([]byte, 128*1024) // If you don't give gzip some data will spend too much time compressing everything to zero. crand.Read(msg) for i := 0; i < 10000; i++ { nc.Publish("snap-perf", msg) } nc.Flush() sreq := &JSApiStreamSnapshotRequest{DeliverSubject: nats.NewInbox()} req, _ := json.Marshal(sreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "snap-perf"), req, time.Second) if err != nil { t.Fatalf("Unexpected error on snapshot request: %v", err) } var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) if resp.Error != nil { t.Fatalf("Did not get correct error response: %+v", resp.Error) } done := make(chan bool) total := 0 sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { m.Sub.Unsubscribe() done <- true return } // We don't do anything with the snapshot, just take // note of the size. total += len(m.Data) // Flow ack m.Respond(nil) }) defer sub.Unsubscribe() start := time.Now() // Wait to receive the snapshot. select { case <-done: case <-time.After(30 * time.Second): t.Fatalf("Did not receive our snapshot in time") } td := time.Since(start) fmt.Printf("Received %d bytes in %v\n", total, td) fmt.Printf("Rate %.0f MB/s\n", float64(total)/td.Seconds()/(1024*1024)) } func TestJetStreamActiveDelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "ADS", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "ADS", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Now load up some messages. toSend := 100 sendSubj := "foo.22" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, state.Msgs) } o, err := mset.addConsumer(&ConsumerConfig{Durable: "to", DeliverSubject: "d"}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() // We have no active interest above. So consumer will be considered inactive. Let's subscribe and make sure // we get the messages instantly. This will test that we hook interest activation correctly. sub, _ := nc.SubscribeSync("d") defer sub.Unsubscribe() nc.Flush() checkFor(t, 100*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) }) } } func TestJetStreamEphemeralConsumers(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "EP", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "EP", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !o.isActive() { t.Fatalf("Expected the consumer to be considered active") } if numo := mset.numConsumers(); numo != 1 { t.Fatalf("Expected number of consumers to be 1, got %d", numo) } // Set our delete threshold to something low for testing purposes. o.setInActiveDeleteThreshold(100 * time.Millisecond) // Make sure works now. nc.Request("foo.22", nil, 100*time.Millisecond) checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 1 { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, 1) } return nil }) // Now close the subscription, this should trip active state on the ephemeral consumer. sub.Unsubscribe() checkFor(t, time.Second, 10*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Expected the ephemeral consumer to be considered inactive") } return nil }) // The reason for this still being 1 is that we give some time in case of a reconnect scenario. // We detect right away on the interest change but we wait for interest to be re-established. // This is in case server goes away but app is fine, we do not want to recycle those consumers. if numo := mset.numConsumers(); numo != 1 { t.Fatalf("Expected number of consumers to be 1, got %d", numo) } // We should delete this one after the delete threshold. checkFor(t, time.Second, 100*time.Millisecond, func() error { if numo := mset.numConsumers(); numo != 0 { return fmt.Errorf("Expected number of consumers to be 0, got %d", numo) } return nil }) }) } } func TestJetStreamConsumerReconnect(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "ET", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "ET", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() // Capture the subscription. delivery := sub.Subject o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !o.isActive() { t.Fatalf("Expected the consumer to be considered active") } if numo := mset.numConsumers(); numo != 1 { t.Fatalf("Expected number of consumers to be 1, got %d", numo) } // We will simulate reconnect by unsubscribing on one connection and forming // the same on another. Once we have cluster tests we will do more testing on // reconnect scenarios. getMsg := func(seqno int) *nats.Msg { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error for %d: %v", seqno, err) } if seq := o.seqFromReply(m.Reply); seq != uint64(seqno) { t.Fatalf("Expected sequence of %d , got %d", seqno, seq) } m.Respond(nil) return m } sendMsg := func() { t.Helper() if err := nc.Publish("foo.22", []byte("OK!")); err != nil { return } } checkForInActive := func() { checkFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Consumer is still active") } return nil }) } // Send and Pull first message. sendMsg() // 1 getMsg(1) // Cancel first one. sub.Unsubscribe() // Re-establish new sub on same subject. sub, _ = nc.SubscribeSync(delivery) nc.Flush() // We should be getting 2 here. sendMsg() // 2 getMsg(2) sub.Unsubscribe() checkForInActive() // send 3-10 for i := 0; i <= 7; i++ { sendMsg() } // Make sure they are all queued up with no interest. nc.Flush() // Restablish again. sub, _ = nc.SubscribeSync(delivery) nc.Flush() // We should be getting 3-10 here. for i := 3; i <= 10; i++ { getMsg(i) } }) } } func TestJetStreamDurableConsumerReconnect(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DT", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "DT", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() dname := "d22" subj1 := nats.NewInbox() o, err := mset.addConsumer(&ConsumerConfig{ Durable: dname, DeliverSubject: subj1, AckPolicy: AckExplicit, AckWait: 50 * time.Millisecond}) if err != nil { t.Fatalf("Unexpected error: %v", err) } sendMsg := func() { t.Helper() if err := nc.Publish("foo.22", []byte("OK!")); err != nil { return } } // Send 10 msgs toSend := 10 for i := 0; i < toSend; i++ { sendMsg() } sub, _ := nc.SubscribeSync(subj1) defer sub.Unsubscribe() checkFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) getMsg := func(seqno int) *nats.Msg { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if seq := o.streamSeqFromReply(m.Reply); seq != uint64(seqno) { t.Fatalf("Expected sequence of %d , got %d", seqno, seq) } m.Respond(nil) return m } // Ack first half for i := 1; i <= toSend/2; i++ { m := getMsg(i) m.Respond(nil) } // Now unsubscribe and wait to become inactive sub.Unsubscribe() checkFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Consumer is still active") } return nil }) // Now we should be able to replace the delivery subject. subj2 := nats.NewInbox() sub, _ = nc.SubscribeSync(subj2) defer sub.Unsubscribe() nc.Flush() o, err = mset.addConsumer(&ConsumerConfig{ Durable: dname, DeliverSubject: subj2, AckPolicy: AckExplicit, AckWait: 50 * time.Millisecond}) if err != nil { t.Fatalf("Unexpected error trying to add a new durable consumer: %v", err) } // We should get the remaining messages here. for i := toSend/2 + 1; i <= toSend; i++ { m := getMsg(i) m.Respond(nil) } }) } } func TestJetStreamDurableConsumerReconnectWithOnlyPending(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DT", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "DT", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() dname := "d22" subj1 := nats.NewInbox() o, err := mset.addConsumer(&ConsumerConfig{ Durable: dname, DeliverSubject: subj1, AckPolicy: AckExplicit, AckWait: 25 * time.Millisecond}) if err != nil { t.Fatalf("Unexpected error: %v", err) } sendMsg := func(payload string) { t.Helper() if err := nc.Publish("foo.22", []byte(payload)); err != nil { return } } sendMsg("1") sub, _ := nc.SubscribeSync(subj1) defer sub.Unsubscribe() checkFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 1 { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, 1) } return nil }) // Now unsubscribe and wait to become inactive sub.Unsubscribe() checkFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Consumer is still active") } return nil }) // Send the second message while delivery subscriber is not running sendMsg("2") // Now we should be able to replace the delivery subject. subj2 := nats.NewInbox() o, err = mset.addConsumer(&ConsumerConfig{ Durable: dname, DeliverSubject: subj2, AckPolicy: AckExplicit, AckWait: 25 * time.Millisecond}) if err != nil { t.Fatalf("Unexpected error trying to add a new durable consumer: %v", err) } sub, _ = nc.SubscribeSync(subj2) defer sub.Unsubscribe() nc.Flush() // We should get msg "1" and "2" delivered. They will be reversed. for i := 0; i < 2; i++ { msg, err := sub.NextMsg(500 * time.Millisecond) if err != nil { t.Fatalf("Unexpected error: %v", err) } sseq, _, dc, _, _ := replyInfo(msg.Reply) if sseq == 1 && dc == 1 { t.Fatalf("Expected a redelivery count greater then 1 for sseq 1, got %d", dc) } if sseq != 1 && sseq != 2 { t.Fatalf("Expected stream sequence of 1 or 2 but got %d", sseq) } } }) } } func TestJetStreamDurableFilteredSubjectConsumerReconnect(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DT", Storage: MemoryStorage, Subjects: []string{"foo.*"}}}, {"FileStore", &StreamConfig{Name: "DT", Storage: FileStorage, Subjects: []string{"foo.*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() sendMsgs := func(toSend int) { for i := 0; i < toSend; i++ { var subj string if i%2 == 0 { subj = "foo.AA" } else { subj = "foo.ZZ" } _, err := js.Publish(subj, []byte("OK!")) require_NoError(t, err) } } // Send 50 msgs toSend := 50 sendMsgs(toSend) dname := "d33" dsubj := nats.NewInbox() // Now create an consumer for foo.AA, only requesting the last one. _, err = mset.addConsumer(&ConsumerConfig{ Durable: dname, DeliverSubject: dsubj, FilterSubject: "foo.AA", DeliverPolicy: DeliverLast, AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, _ := nc.SubscribeSync(dsubj) defer sub.Unsubscribe() // Used to calculate difference between store seq and delivery seq. storeBaseOff := 47 getMsg := func(seq int) *nats.Msg { t.Helper() sseq := 2*seq + storeBaseOff m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } rsseq, roseq, dcount, _, _ := replyInfo(m.Reply) if roseq != uint64(seq) { t.Fatalf("Expected consumer sequence of %d , got %d", seq, roseq) } if rsseq != uint64(sseq) { t.Fatalf("Expected stream sequence of %d , got %d", sseq, rsseq) } if dcount != 1 { t.Fatalf("Expected message to not be marked as redelivered") } return m } getRedeliveredMsg := func(seq int) *nats.Msg { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, roseq, dcount, _, _ := replyInfo(m.Reply) if roseq != uint64(seq) { t.Fatalf("Expected consumer sequence of %d , got %d", seq, roseq) } if dcount < 2 { t.Fatalf("Expected message to be marked as redelivered") } // Ack this message. m.Respond(nil) return m } // All consumers start at 1 and always have increasing sequence numbers. m := getMsg(1) m.Respond(nil) // Now send 50 more, so 100 total, 26 (last + 50/2) for this consumer. sendMsgs(toSend) state := mset.state() if state.Msgs != uint64(toSend*2) { t.Fatalf("Expected %d messages, got %d", toSend*2, state.Msgs) } // For tracking next expected. nextSeq := 2 noAcks := 0 for i := 0; i < toSend/2; i++ { m := getMsg(nextSeq) if i%2 == 0 { m.Respond(nil) // Ack evens. } else { noAcks++ } nextSeq++ } // We should now get those redelivered. for i := 0; i < noAcks; i++ { getRedeliveredMsg(nextSeq) nextSeq++ } // Now send 50 more. sendMsgs(toSend) storeBaseOff -= noAcks * 2 for i := 0; i < toSend/2; i++ { m := getMsg(nextSeq) m.Respond(nil) nextSeq++ } }) } } func TestJetStreamConsumerInactiveNoDeadlock(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send lots of msgs and have them queued up. for i := 0; i < 10000; i++ { js.Publish("DC", []byte("OK!")) } if state := mset.state(); state.Msgs != 10000 { t.Fatalf("Expected %d messages, got %d", 10000, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) sub.SetPendingLimits(-1, -1) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() for i := 0; i < 10; i++ { if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Force us to become inactive but we want to make sure we do not lock up // the internal sendq. sub.Unsubscribe() nc.Flush() }) } } func TestJetStreamMetadata(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Retention: WorkQueuePolicy, Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Retention: WorkQueuePolicy, Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() for i := 0; i < 10; i++ { nc.Publish("DC", []byte("OK!")) nc.Flush() time.Sleep(time.Millisecond) } if state := mset.state(); state.Msgs != 10 { t.Fatalf("Expected %d messages, got %d", 10, state.Msgs) } o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() for i := uint64(1); i <= 10; i++ { m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } sseq, dseq, dcount, ts, _ := replyInfo(m.Reply) mreq := &JSApiMsgGetRequest{Seq: sseq} req, err := json.Marshal(mreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Load the original message from the stream to verify ReplyInfo ts against stored message smsgj, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, c.mconfig.Name), req, time.Second) if err != nil { t.Fatalf("Could not retrieve stream message: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(smsgj.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Message == nil || resp.Error != nil { t.Fatalf("Did not receive correct response") } smsg := resp.Message if ts != smsg.Time.UnixNano() { t.Fatalf("Wrong timestamp in ReplyInfo for msg %d, expected %v got %v", i, ts, smsg.Time.UnixNano()) } if sseq != i { t.Fatalf("Expected set sequence of %d, got %d", i, sseq) } if dseq != i { t.Fatalf("Expected delivery sequence of %d, got %d", i, dseq) } if dcount != 1 { t.Fatalf("Expected delivery count to be 1, got %d", dcount) } m.Respond(AckAck) } // Now make sure we get right response when message is missing. mreq := &JSApiMsgGetRequest{Seq: 1} req, err := json.Marshal(mreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Load the original message from the stream to verify ReplyInfo ts against stored message rmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, c.mconfig.Name), req, time.Second) if err != nil { t.Fatalf("Could not retrieve stream message: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(rmsg.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Error == nil || resp.Error.Code != 404 || resp.Error.Description != "no message found" { t.Fatalf("Did not get correct error response: %+v", resp.Error) } }) } } func TestJetStreamRedeliverCount(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err = js.Publish("DC", []byte("OK!")); err != nil { t.Fatal(err) } checkFor(t, time.Second, time.Millisecond*250, func() error { if state := mset.state(); state.Msgs != 1 { return fmt.Errorf("Expected %d messages, got %d", 1, state.Msgs) } return nil }) o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() for i := uint64(1); i <= 10; i++ { m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } sseq, dseq, dcount, _, _ := replyInfo(m.Reply) // Make sure we keep getting stream sequence #1 if sseq != 1 { t.Fatalf("Expected set sequence of 1, got %d", sseq) } if dseq != i { t.Fatalf("Expected delivery sequence of %d, got %d", i, dseq) } // Now make sure dcount is same as dseq (or i). if dcount != i { t.Fatalf("Expected delivery count to be %d, got %d", i, dcount) } // Make sure it keeps getting sent back. m.Respond(AckNak) nc.Flush() } }) } } // We want to make sure that for pull based consumers that if we ack // late with no interest the redelivery attempt is removed and we do // not get the message back. func TestJetStreamRedeliverAndLateAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "LA", Storage: MemoryStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() o, err := mset.addConsumer(&ConsumerConfig{Durable: "DDD", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Queue up message sendStreamMsg(t, nc, "LA", "Hello World!") nextSubj := o.requestNextMsgSubject() msg, err := nc.Request(nextSubj, nil, time.Second) require_NoError(t, err) // Wait for past ackwait time time.Sleep(150 * time.Millisecond) // Now ack! msg.AckSync() // We should not get this back. if _, err := nc.Request(nextSubj, nil, 10*time.Millisecond); err == nil { t.Fatalf("Message should not have been sent back") } } // https://github.com/nats-io/nats-server/issues/1502 func TestJetStreamPendingNextTimer(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "NT", Storage: MemoryStorage, Subjects: []string{"ORDERS.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "DDD", AckPolicy: AckExplicit, FilterSubject: "ORDERS.test", AckWait: 100 * time.Millisecond, }) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() sendAndReceive := func() { nc := clientConnectToServer(t, s) defer nc.Close() // Queue up message sendStreamMsg(t, nc, "ORDERS.test", "Hello World! #1") sendStreamMsg(t, nc, "ORDERS.test", "Hello World! #2") nextSubj := o.requestNextMsgSubject() for i := 0; i < 2; i++ { if _, err := nc.Request(nextSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } } nc.Close() time.Sleep(200 * time.Millisecond) } sendAndReceive() sendAndReceive() sendAndReceive() } func TestJetStreamCanNotNakAckd(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send 10 msgs for i := 0; i < 10; i++ { js.Publish("DC", []byte("OK!")) } if state := mset.state(); state.Msgs != 10 { t.Fatalf("Expected %d messages, got %d", 10, state.Msgs) } o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() for i := uint64(1); i <= 10; i++ { m, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Ack evens. if i%2 == 0 { m.Respond(nil) } } nc.Flush() // Fake these for now. ackReplyT := "$JS.A.DC.WQ.1.%d.%d" checkBadNak := func(seq int) { t.Helper() if err := nc.Publish(fmt.Sprintf(ackReplyT, seq, seq), AckNak); err != nil { t.Fatalf("Error sending nak: %v", err) } nc.Flush() if _, err := nc.Request(o.requestNextMsgSubject(), nil, 10*time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Did not expect new delivery on nak of %d", seq) } } // If the nak took action it will deliver another message, incrementing the next delivery seq. // We ack evens above, so these should fail for i := 2; i <= 10; i += 2 { checkBadNak(i) } // Now check we can not nak something we do not have. checkBadNak(22) }) } } func TestJetStreamStreamPurge(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send 100 msgs for i := 0; i < 100; i++ { js.Publish("DC", []byte("OK!")) } if state := mset.state(); state.Msgs != 100 { t.Fatalf("Expected %d messages, got %d", 100, state.Msgs) } mset.purge(nil) state := mset.state() if state.Msgs != 0 { t.Fatalf("Expected %d messages, got %d", 0, state.Msgs) } // Make sure first timestamp are reset. if !state.FirstTime.IsZero() { t.Fatalf("Expected the state's first time to be zero after purge") } time.Sleep(10 * time.Millisecond) now := time.Now() js.Publish("DC", []byte("OK!")) state = mset.state() if state.Msgs != 1 { t.Fatalf("Expected %d message, got %d", 1, state.Msgs) } if state.FirstTime.Before(now) { t.Fatalf("First time is incorrect after adding messages back in") } if state.FirstTime != state.LastTime { t.Fatalf("Expected first and last times to be the same for only message") } }) } } func TestJetStreamStreamPurgeWithConsumer(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send 100 msgs for i := 0; i < 100; i++ { js.Publish("DC", []byte("OK!")) } if state := mset.state(); state.Msgs != 100 { t.Fatalf("Expected %d messages, got %d", 100, state.Msgs) } // Now create an consumer and make sure it functions properly. o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nextSubj := o.requestNextMsgSubject() for i := 0; i < 50; i++ { msg, err := nc.Request(nextSubj, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Ack. msg.Respond(nil) } // Now grab next 25 without ack. for i := 0; i < 25; i++ { if _, err := nc.Request(nextSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } } state := o.info() if state.AckFloor.Consumer != 50 { t.Fatalf("Expected ack floor of 50, got %d", state.AckFloor.Consumer) } if state.NumAckPending != 25 { t.Fatalf("Expected len(pending) to be 25, got %d", state.NumAckPending) } // Now do purge. mset.purge(nil) if state := mset.state(); state.Msgs != 0 { t.Fatalf("Expected %d messages, got %d", 0, state.Msgs) } // Now re-acquire state and check that we did the right thing. // Pending should be cleared, and stream sequences should have been set // to the total messages before purge + 1. state = o.info() if state.NumAckPending != 0 { t.Fatalf("Expected no pending, got %d", state.NumAckPending) } if state.Delivered.Stream != 100 { t.Fatalf("Expected to have setseq now at next seq of 100, got %d", state.Delivered.Stream) } // Check AckFloors which should have also been adjusted. if state.AckFloor.Stream != 100 { t.Fatalf("Expected ackfloor for setseq to be 100, got %d", state.AckFloor.Stream) } if state.AckFloor.Consumer != 75 { t.Fatalf("Expected ackfloor for obsseq to be 75, got %d", state.AckFloor.Consumer) } // Also make sure we can get new messages correctly. js.Publish("DC", []byte("OK-22")) if msg, err := nc.Request(nextSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if string(msg.Data) != "OK-22" { t.Fatalf("Received wrong message, wanted 'OK-22', got %q", msg.Data) } }) } } func TestJetStreamStreamPurgeWithConsumerAndRedelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send 100 msgs for i := 0; i < 100; i++ { js.Publish("DC", []byte("OK!")) } if state := mset.state(); state.Msgs != 100 { t.Fatalf("Expected %d messages, got %d", 100, state.Msgs) } // Now create an consumer and make sure it functions properly. // This will test redelivery state and purge of the stream. wcfg := &ConsumerConfig{ Durable: "WQ", AckPolicy: AckExplicit, AckWait: 20 * time.Millisecond, } o, err := mset.addConsumer(wcfg) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nextSubj := o.requestNextMsgSubject() for i := 0; i < 50; i++ { // Do not ack these. if _, err := nc.Request(nextSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Now wait to make sure we are in a redelivered state. time.Sleep(wcfg.AckWait * 2) // Now do purge. mset.purge(nil) if state := mset.state(); state.Msgs != 0 { t.Fatalf("Expected %d messages, got %d", 0, state.Msgs) } // Now get the state and check that we did the right thing. // Pending should be cleared, and stream sequences should have been set // to the total messages before purge + 1. state := o.info() if state.NumAckPending != 0 { t.Fatalf("Expected no pending, got %d", state.NumAckPending) } if state.Delivered.Stream != 100 { t.Fatalf("Expected to have setseq now at next seq of 100, got %d", state.Delivered.Stream) } // Check AckFloors which should have also been adjusted. if state.AckFloor.Stream != 100 { t.Fatalf("Expected ackfloor for setseq to be 100, got %d", state.AckFloor.Stream) } if state.AckFloor.Consumer != 50 { t.Fatalf("Expected ackfloor for obsseq to be 75, got %d", state.AckFloor.Consumer) } // Also make sure we can get new messages correctly. js.Publish("DC", []byte("OK-22")) if msg, err := nc.Request(nextSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if string(msg.Data) != "OK-22" { t.Fatalf("Received wrong message, wanted 'OK-22', got %q", msg.Data) } }) } } func TestJetStreamInterestRetentionStream(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage, Retention: InterestPolicy}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage, Retention: InterestPolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() // Send 100 msgs totalMsgs := 100 for i := 0; i < totalMsgs; i++ { js.Publish("DC", []byte("OK!")) } checkNumMsgs := func(numExpected int) { t.Helper() checkFor(t, time.Second, 15*time.Millisecond, func() error { if state := mset.state(); state.Msgs != uint64(numExpected) { return fmt.Errorf("Expected %d messages, got %d", numExpected, state.Msgs) } return nil }) } // Since we had no interest this should be 0. checkNumMsgs(0) syncSub := func() *nats.Subscription { sub, _ := nc.SubscribeSync(nats.NewInbox()) nc.Flush() return sub } // Now create three consumers. // 1. AckExplicit // 2. AckAll // 3. AckNone sub1 := syncSub() mset.addConsumer(&ConsumerConfig{DeliverSubject: sub1.Subject, AckPolicy: AckExplicit}) sub2 := syncSub() mset.addConsumer(&ConsumerConfig{DeliverSubject: sub2.Subject, AckPolicy: AckAll}) sub3 := syncSub() mset.addConsumer(&ConsumerConfig{DeliverSubject: sub3.Subject, AckPolicy: AckNone}) for i := 0; i < totalMsgs; i++ { js.Publish("DC", []byte("OK!")) } checkNumMsgs(totalMsgs) // Wait for all messsages to be pending for each sub. for i, sub := range []*nats.Subscription{sub1, sub2, sub3} { checkFor(t, 5*time.Second, 25*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); nmsgs != totalMsgs { return fmt.Errorf("Did not receive correct number of messages: %d vs %d for sub %d", nmsgs, totalMsgs, i+1) } return nil }) } getAndAck := func(sub *nats.Subscription) { t.Helper() if m, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else { m.Respond(nil) } nc.Flush() } // Ack evens for the explicit ack sub. var odds []*nats.Msg for i := 1; i <= totalMsgs; i++ { if m, err := sub1.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if i%2 == 0 { m.Respond(nil) // Ack evens. } else { odds = append(odds, m) } } nc.Flush() checkNumMsgs(totalMsgs) // Now ack first for AckAll sub2 getAndAck(sub2) // We should be at the same number since we acked 1, explicit acked 2 checkNumMsgs(totalMsgs) // Now ack second for AckAll sub2 getAndAck(sub2) // We should now have 1 removed. checkNumMsgs(totalMsgs - 1) // Now ack third for AckAll sub2 getAndAck(sub2) // We should still only have 1 removed. checkNumMsgs(totalMsgs - 1) // Now ack odds from explicit. for _, m := range odds { m.Respond(nil) // Ack } nc.Flush() // we should have 1, 2, 3 acks now. checkNumMsgs(totalMsgs - 3) nm, _, _ := sub2.Pending() // Now ack last ackAll message. This should clear all of them. for i := 1; i <= nm; i++ { if m, err := sub2.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if i == nm { m.Respond(nil) } } nc.Flush() // Should be zero now. checkNumMsgs(0) }) } } func TestJetStreamInterestRetentionStreamWithFilteredConsumers(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Subjects: []string{"*"}, Storage: MemoryStorage, Retention: InterestPolicy}}, {"FileStore", &StreamConfig{Name: "DC", Subjects: []string{"*"}, Storage: FileStorage, Retention: InterestPolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, js := jsClientConnect(t, s) defer nc.Close() fsub, err := js.SubscribeSync("foo") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer fsub.Unsubscribe() bsub, err := js.SubscribeSync("bar") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer bsub.Unsubscribe() msg := []byte("FILTERED") sendMsg := func(subj string) { t.Helper() if _, err = js.Publish(subj, msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } getAndAck := func(sub *nats.Subscription) { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error getting msg: %v", err) } m.AckSync() } checkState := func(expected uint64) { t.Helper() si, err := js.StreamInfo("DC") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != expected { t.Fatalf("Expected %d msgs, got %d", expected, si.State.Msgs) } } sendMsg("foo") checkState(1) getAndAck(fsub) checkState(0) sendMsg("bar") sendMsg("foo") checkState(2) getAndAck(bsub) checkState(1) getAndAck(fsub) checkState(0) }) } } func TestJetStreamInterestRetentionWithWildcardsAndFilteredConsumers(t *testing.T) { msc := StreamConfig{ Name: "DCWC", Subjects: []string{"foo.*"}, Storage: MemoryStorage, Retention: InterestPolicy, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Send 10 msgs for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo.bar", "Hello World!") } if state := mset.state(); state.Msgs != 0 { t.Fatalf("Expected %d messages, got %d", 0, state.Msgs) } cfg := &ConsumerConfig{Durable: "ddd", FilterSubject: "foo.bar", AckPolicy: AckExplicit} o, err := mset.addConsumer(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() sendStreamMsg(t, nc, "foo.bar", "Hello World!") if state := mset.state(); state.Msgs != 1 { t.Fatalf("Expected %d message, got %d", 1, state.Msgs) } else if state.FirstSeq != 11 { t.Fatalf("Expected %d for first seq, got %d", 11, state.FirstSeq) } // Now send to foo.baz, which has no interest, so we should not hold onto this message. sendStreamMsg(t, nc, "foo.baz", "Hello World!") if state := mset.state(); state.Msgs != 1 { t.Fatalf("Expected %d message, got %d", 1, state.Msgs) } }) } } func TestJetStreamInterestRetentionStreamWithDurableRestart(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "IK", Storage: MemoryStorage, Retention: InterestPolicy}}, {"FileStore", &StreamConfig{Name: "IK", Storage: FileStorage, Retention: InterestPolicy}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() checkNumMsgs := func(numExpected int) { t.Helper() checkFor(t, time.Second, 50*time.Millisecond, func() error { if state := mset.state(); state.Msgs != uint64(numExpected) { return fmt.Errorf("Expected %d messages, got %d", numExpected, state.Msgs) } return nil }) } nc := clientConnectToServer(t, s) defer nc.Close() sub, _ := nc.SubscribeSync(nats.NewInbox()) nc.Flush() cfg := &ConsumerConfig{Durable: "ivan", DeliverPolicy: DeliverNew, DeliverSubject: sub.Subject, AckPolicy: AckNone} o, _ := mset.addConsumer(cfg) sendStreamMsg(t, nc, "IK", "M1") sendStreamMsg(t, nc, "IK", "M2") checkSubPending := func(numExpected int) { t.Helper() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } checkSubPending(2) checkNumMsgs(0) // Now stop the subscription. sub.Unsubscribe() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Still active consumer") } return nil }) sendStreamMsg(t, nc, "IK", "M3") sendStreamMsg(t, nc, "IK", "M4") checkNumMsgs(2) // Now restart the durable. sub, _ = nc.SubscribeSync(nats.NewInbox()) nc.Flush() cfg.DeliverSubject = sub.Subject if o, err = mset.addConsumer(cfg); err != nil { t.Fatalf("Error re-establishing the durable consumer: %v", err) } checkSubPending(2) for _, expected := range []string{"M3", "M4"} { if m, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else if string(m.Data) != expected { t.Fatalf("Expected %q, got %q", expected, m.Data) } } // Should all be gone now. checkNumMsgs(0) // Now restart again and make sure we do not get any messages. sub.Unsubscribe() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Still active consumer") } return nil }) o.delete() sub, _ = nc.SubscribeSync(nats.NewInbox()) nc.Flush() cfg.DeliverSubject = sub.Subject cfg.AckPolicy = AckExplicit // Set ack if o, err = mset.addConsumer(cfg); err != nil { t.Fatalf("Error re-establishing the durable consumer: %v", err) } time.Sleep(100 * time.Millisecond) checkSubPending(0) checkNumMsgs(0) // Now queue up some messages. for i := 1; i <= 10; i++ { sendStreamMsg(t, nc, "IK", fmt.Sprintf("M%d", i)) } checkNumMsgs(10) checkSubPending(10) // Create second consumer sub2, _ := nc.SubscribeSync(nats.NewInbox()) nc.Flush() cfg.DeliverSubject = sub2.Subject cfg.Durable = "derek" o2, err := mset.addConsumer(cfg) if err != nil { t.Fatalf("Error creating second durable consumer: %v", err) } // Now queue up some messages. for i := 11; i <= 20; i++ { sendStreamMsg(t, nc, "IK", fmt.Sprintf("M%d", i)) } checkNumMsgs(20) checkSubPending(20) // Now make sure deleting the consumers will remove messages from // the stream since we are interest retention based. o.delete() checkNumMsgs(10) o2.delete() checkNumMsgs(0) }) } } func TestJetStreamConsumerReplayRate(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Send 10 msgs totalMsgs := 10 var gaps []time.Duration lst := time.Now() for i := 0; i < totalMsgs; i++ { gaps = append(gaps, time.Since(lst)) lst = time.Now() nc.Publish("DC", []byte("OK!")) // Calculate a gap between messages. gap := 10*time.Millisecond + time.Duration(rand.Intn(20))*time.Millisecond time.Sleep(gap) } if state := mset.state(); state.Msgs != uint64(totalMsgs) { t.Fatalf("Expected %d messages, got %d", totalMsgs, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() // Firehose/instant which is default. last := time.Now() for i := 0; i < totalMsgs; i++ { if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } now := time.Now() // Delivery from addConsumer starts in a go routine, so be // more tolerant for the first message. limit := 5 * time.Millisecond if i == 0 { limit = 10 * time.Millisecond } if now.Sub(last) > limit { t.Fatalf("Expected firehose/instant delivery, got message gap of %v", now.Sub(last)) } last = now } // Now do replay rate to match original. o, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, ReplayPolicy: ReplayOriginal}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() // Original rate messsages were received for push based consumer. for i := 0; i < totalMsgs; i++ { start := time.Now() if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } gap := time.Since(start) // 15ms is high but on macs time.Sleep(delay) does not sleep only delay. // Also on travis if things get bogged down this could be delayed. gl, gh := gaps[i]-10*time.Millisecond, gaps[i]+15*time.Millisecond if gap < gl || gap > gh { t.Fatalf("Gap is off for %d, expected %v got %v", i, gaps[i], gap) } } // Now create pull based. oc := workerModeConfig("PM") oc.ReplayPolicy = ReplayOriginal o, err = mset.addConsumer(oc) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() for i := 0; i < totalMsgs; i++ { start := time.Now() if _, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } gap := time.Since(start) // 10ms is high but on macs time.Sleep(delay) does not sleep only delay. gl, gh := gaps[i]-5*time.Millisecond, gaps[i]+10*time.Millisecond if gap < gl || gap > gh { t.Fatalf("Gap is incorrect for %d, expected %v got %v", i, gaps[i], gap) } } }) } } func TestJetStreamConsumerReplayRateNoAck(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Send 10 msgs totalMsgs := 10 for i := 0; i < totalMsgs; i++ { nc.Request("DC", []byte("Hello World"), time.Second) time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond) } if state := mset.state(); state.Msgs != uint64(totalMsgs) { t.Fatalf("Expected %d messages, got %d", totalMsgs, state.Msgs) } subj := "d.dc" o, err := mset.addConsumer(&ConsumerConfig{ Durable: "derek", DeliverSubject: subj, AckPolicy: AckNone, ReplayPolicy: ReplayOriginal, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer o.delete() // Sleep a random amount of time. time.Sleep(time.Duration(rand.Intn(20)) * time.Millisecond) sub, _ := nc.SubscribeSync(subj) nc.Flush() checkFor(t, time.Second, 25*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != totalMsgs { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, totalMsgs) } return nil }) }) } } func TestJetStreamConsumerReplayQuit(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{Name: "DC", Storage: MemoryStorage}}, {"FileStore", &StreamConfig{Name: "DC", Storage: FileStorage}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Send 2 msgs nc.Request("DC", []byte("OK!"), time.Second) time.Sleep(100 * time.Millisecond) nc.Request("DC", []byte("OK!"), time.Second) if state := mset.state(); state.Msgs != 2 { t.Fatalf("Expected %d messages, got %d", 2, state.Msgs) } sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() // Now do replay rate to match original. o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, ReplayPolicy: ReplayOriginal}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Allow loop and deliver / replay go routine to spin up time.Sleep(50 * time.Millisecond) base := runtime.NumGoroutine() o.delete() checkFor(t, 100*time.Millisecond, 10*time.Millisecond, func() error { if runtime.NumGoroutine() >= base { return fmt.Errorf("Consumer go routines still running") } return nil }) }) } } func TestJetStreamSystemLimits(t *testing.T) { s := RunRandClientPortServer(t) defer s.Shutdown() if _, _, err := s.JetStreamReservedResources(); err == nil { t.Fatalf("Expected error requesting jetstream reserved resources when not enabled") } // Create some accounts. facc, _ := s.LookupOrRegisterAccount("FOO") bacc, _ := s.LookupOrRegisterAccount("BAR") zacc, _ := s.LookupOrRegisterAccount("BAZ") jsconfig := &JetStreamConfig{MaxMemory: 1024, MaxStore: 8192, StoreDir: t.TempDir()} if err := s.EnableJetStream(jsconfig); err != nil { t.Fatalf("Expected no error, got %v", err) } if rm, rd, err := s.JetStreamReservedResources(); err != nil { t.Fatalf("Unexpected error requesting jetstream reserved resources: %v", err) } else if rm != 0 || rd != 0 { t.Fatalf("Expected reserved memory and store to be 0, got %d and %d", rm, rd) } limits := func(mem int64, store int64) map[string]JetStreamAccountLimits { return map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: mem, MaxStore: store, MaxStreams: -1, MaxConsumers: -1, }, } } if err := facc.EnableJetStream(limits(24, 192)); err != nil { t.Fatalf("Unexpected error: %v", err) } // Use up rest of our resources in memory if err := bacc.EnableJetStream(limits(1000, 0)); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now ask for more memory. Should error. if err := zacc.EnableJetStream(limits(1000, 0)); err == nil { t.Fatalf("Expected an error when exhausting memory resource limits") } // Disk too. if err := zacc.EnableJetStream(limits(0, 10000)); err == nil { t.Fatalf("Expected an error when exhausting memory resource limits") } facc.DisableJetStream() bacc.DisableJetStream() zacc.DisableJetStream() // Make sure we unreserved resources. if rm, rd, err := s.JetStreamReservedResources(); err != nil { t.Fatalf("Unexpected error requesting jetstream reserved resources: %v", err) } else if rm != 0 || rd != 0 { t.Fatalf("Expected reserved memory and store to be 0, got %v and %v", friendlyBytes(rm), friendlyBytes(rd)) } if err := facc.EnableJetStream(limits(24, 192)); err != nil { t.Fatalf("Unexpected error: %v", err) } // Test Adjust lim := limits(jsconfig.MaxMemory, jsconfig.MaxStore) l := lim[_EMPTY_] l.MaxStreams = 10 l.MaxConsumers = 10 lim[_EMPTY_] = l if err := facc.UpdateJetStreamLimits(lim); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } var msets []*stream // Now test max streams and max consumers. Note max consumers is per stream. for i := 0; i < 10; i++ { mname := fmt.Sprintf("foo.%d", i) mset, err := facc.addStream(&StreamConfig{Name: strconv.Itoa(i), Storage: MemoryStorage, Subjects: []string{mname}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } msets = append(msets, mset) } // Remove them all for _, mset := range msets { mset.delete() } // Now try to add one with bytes limit that would exceed the account limit. if _, err := facc.addStream(&StreamConfig{Name: "22", Storage: MemoryStorage, MaxBytes: jsconfig.MaxStore * 2}); err == nil { t.Fatalf("Expected error adding stream over limit") } // Replicas can't be > 1 if _, err := facc.addStream(&StreamConfig{Name: "22", Storage: MemoryStorage, Replicas: 10}); err == nil { t.Fatalf("Expected error adding stream over limit") } // Test consumers limit against account limit when the stream does not set a limit mset, err := facc.addStream(&StreamConfig{Name: "22", Storage: MemoryStorage, Subjects: []string{"foo.22"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } for i := 0; i < 10; i++ { oname := fmt.Sprintf("O:%d", i) _, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } // This one should fail. if _, err := mset.addConsumer(&ConsumerConfig{Durable: "O:22", AckPolicy: AckExplicit}); err == nil { t.Fatalf("Expected error adding consumer over the limit") } // Test consumer limit against stream limit mset.delete() mset, err = facc.addStream(&StreamConfig{Name: "22", Storage: MemoryStorage, Subjects: []string{"foo.22"}, MaxConsumers: 5}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } for i := 0; i < 5; i++ { oname := fmt.Sprintf("O:%d", i) _, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } // This one should fail. if _, err := mset.addConsumer(&ConsumerConfig{Durable: "O:22", AckPolicy: AckExplicit}); err == nil { t.Fatalf("Expected error adding consumer over the limit") } // Test the account having smaller limits than the stream mset.delete() mset, err = facc.addStream(&StreamConfig{Name: "22", Storage: MemoryStorage, Subjects: []string{"foo.22"}, MaxConsumers: 10}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } l.MaxConsumers = 5 lim[_EMPTY_] = l if err := facc.UpdateJetStreamLimits(lim); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } for i := 0; i < 5; i++ { oname := fmt.Sprintf("O:%d", i) _, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } // This one should fail. if _, err := mset.addConsumer(&ConsumerConfig{Durable: "O:22", AckPolicy: AckExplicit}); err == nil { t.Fatalf("Expected error adding consumer over the limit") } } func TestJetStreamSystemLimitsPlacement(t *testing.T) { const smallSystemLimit = 128 const mediumSystemLimit = smallSystemLimit * 2 const largeSystemLimit = smallSystemLimit * 3 tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: _MAXMEM_ max_file_store: _MAXFILE_ store_dir: '%s' } server_tags: [ _TAG_ ] leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` storeCnf := func(serverName, clusterName, storeDir, conf string) string { switch serverName { case "S-1": conf = strings.Replace(conf, "_MAXMEM_", fmt.Sprint(smallSystemLimit), 1) conf = strings.Replace(conf, "_MAXFILE_", fmt.Sprint(smallSystemLimit), 1) return strings.Replace(conf, "_TAG_", "small", 1) case "S-2": conf = strings.Replace(conf, "_MAXMEM_", fmt.Sprint(mediumSystemLimit), 1) conf = strings.Replace(conf, "_MAXFILE_", fmt.Sprint(mediumSystemLimit), 1) return strings.Replace(conf, "_TAG_", "medium", 1) case "S-3": conf = strings.Replace(conf, "_MAXMEM_", fmt.Sprint(largeSystemLimit), 1) conf = strings.Replace(conf, "_MAXFILE_", fmt.Sprint(largeSystemLimit), 1) return strings.Replace(conf, "_TAG_", "large", 1) default: return conf } } cluster := createJetStreamClusterWithTemplateAndModHook(t, tmpl, "cluster-a", 3, storeCnf) defer cluster.shutdown() requestLeaderStepDown := func(clientURL string) error { nc, err := nats.Connect(clientURL, nats.UserInfo("admin", "s3cr3t!")) if err != nil { return err } defer nc.Close() ncResp, err := nc.Request(JSApiLeaderStepDown, nil, 3*time.Second) if err != nil { return err } var resp JSApiLeaderStepDownResponse if err := json.Unmarshal(ncResp.Data, &resp); err != nil { return err } if resp.Error != nil { return resp.Error } if !resp.Success { return fmt.Errorf("leader step down request not successful") } return nil } largeSrv := cluster.servers[2] // Force large server to be leader err := checkForErr(15*time.Second, 500*time.Millisecond, func() error { if largeSrv.JetStreamIsLeader() { return nil } if err := requestLeaderStepDown(largeSrv.ClientURL()); err != nil { return err } return fmt.Errorf("large server is not leader") }) if err != nil { t.Skipf("failed to get desired layout: %s", err) } nc, js := jsClientConnect(t, largeSrv) defer nc.Close() cases := []struct { name string storage nats.StorageType createMaxBytes int64 serverTag string wantErr bool }{ { name: "file create large stream on small server", storage: nats.FileStorage, createMaxBytes: largeSystemLimit, serverTag: "small", wantErr: true, }, { name: "memory create large stream on small server", storage: nats.MemoryStorage, createMaxBytes: largeSystemLimit, serverTag: "small", wantErr: true, }, { name: "file create large stream on medium server", storage: nats.FileStorage, createMaxBytes: largeSystemLimit, serverTag: "medium", wantErr: true, }, { name: "memory create large stream on medium server", storage: nats.MemoryStorage, createMaxBytes: largeSystemLimit, serverTag: "medium", wantErr: true, }, { name: "file create large stream on large server", storage: nats.FileStorage, createMaxBytes: largeSystemLimit, serverTag: "large", }, { name: "memory create large stream on large server", storage: nats.MemoryStorage, createMaxBytes: largeSystemLimit, serverTag: "large", }, } for i := 0; i < len(cases) && !t.Failed(); i++ { c := cases[i] t.Run(c.name, func(st *testing.T) { _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: c.storage, MaxBytes: c.createMaxBytes, Placement: &nats.Placement{ Cluster: "cluster-a", Tags: []string{c.serverTag}, }, }) if c.wantErr && err == nil { st.Fatalf("unexpected stream create success, maxBytes=%d, tag=%s", c.createMaxBytes, c.serverTag) } else if !c.wantErr && err != nil { st.Fatalf("unexpected error: %s", err) } if err == nil { err = js.DeleteStream("TEST") require_NoError(st, err) } }) } // These next two tests should fail because although the stream fits in the // large and medium server, it doesn't fit on the small server. si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.FileStorage, MaxBytes: smallSystemLimit + 1, Replicas: 3, }) if err == nil { t.Fatalf("unexpected file stream create success, maxBytes=%d, replicas=%d", si.Config.MaxBytes, si.Config.Replicas) } si, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.MemoryStorage, MaxBytes: smallSystemLimit + 1, Replicas: 3, }) if err == nil { t.Fatalf("unexpected memory stream create success, maxBytes=%d, replicas=%d", si.Config.MaxBytes, si.Config.Replicas) } t.Run("meta info placement in request - empty request", func(t *testing.T) { nc, err = nats.Connect(largeSrv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() var resp JSApiLeaderStepDownResponse ncResp, err := nc.Request(JSApiLeaderStepDown, []byte("{}"), 3*time.Second) require_NoError(t, err) err = json.Unmarshal(ncResp.Data, &resp) require_NoError(t, err) require_True(t, resp.Error == nil) require_True(t, resp.Success) }) t.Run("meta info placement in request - uninitialized fields", func(t *testing.T) { nc, err = nats.Connect(largeSrv.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() cluster.waitOnClusterReadyWithNumPeers(3) var resp JSApiLeaderStepDownResponse req, err := json.Marshal(JSApiLeaderStepdownRequest{Placement: nil}) require_NoError(t, err) ncResp, err := nc.Request(JSApiLeaderStepDown, req, 10*time.Second) require_NoError(t, err) err = json.Unmarshal(ncResp.Data, &resp) require_NoError(t, err) require_True(t, resp.Error == nil) }) } func TestJetStreamStreamLimitUpdate(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() err := s.GlobalAccount().UpdateJetStreamLimits(map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 128, MaxStore: 128, MaxStreams: 1, }, }) require_NoError(t, err) nc, js := jsClientConnect(t, s) defer nc.Close() for _, storage := range []nats.StorageType{nats.MemoryStorage, nats.FileStorage} { cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: storage, MaxBytes: 32, } _, err = js.AddStream(cfg) require_NoError(t, err) // Create with same config is idempotent, and must not exceed max streams as it already exists. _, err = js.AddStream(cfg) require_NoError(t, err) cfg.MaxBytes = 16 _, err = js.UpdateStream(cfg) require_NoError(t, err) require_NoError(t, js.DeleteStream("TEST")) } } func TestJetStreamStreamStorageTrackingAndLimits(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() gacc := s.GlobalAccount() al := map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 8192, MaxStore: -1, MaxStreams: -1, MaxConsumers: -1, }, } if err := gacc.UpdateJetStreamLimits(al); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } mset, err := gacc.addStream(&StreamConfig{Name: "LIMITS", Storage: MemoryStorage, Retention: WorkQueuePolicy}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "LIMITS", "Hello World!") } state := mset.state() usage := gacc.JetStreamUsage() // Make sure these are working correctly. if state.Bytes != usage.Memory { t.Fatalf("Expected to have stream bytes match memory usage, %d vs %d", state.Bytes, usage.Memory) } if usage.Streams != 1 { t.Fatalf("Expected to have 1 stream, got %d", usage.Streams) } // Do second stream. mset2, err := gacc.addStream(&StreamConfig{Name: "NUM22", Storage: MemoryStorage, Retention: WorkQueuePolicy}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset2.delete() for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "NUM22", "Hello World!") } stats2 := mset2.state() usage = gacc.JetStreamUsage() if usage.Memory != (state.Bytes + stats2.Bytes) { t.Fatalf("Expected to track both streams, account is %v, stream1 is %v, stream2 is %v", usage.Memory, state.Bytes, stats2.Bytes) } // Make sure delete works. mset2.delete() stats2 = mset2.state() usage = gacc.JetStreamUsage() if usage.Memory != (state.Bytes + stats2.Bytes) { t.Fatalf("Expected to track both streams, account is %v, stream1 is %v, stream2 is %v", usage.Memory, state.Bytes, stats2.Bytes) } // Now drain the first one by consuming the messages. o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() for i := 0; i < toSend; i++ { msg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg.Respond(nil) } nc.Flush() state = mset.state() checkFor(t, time.Second, 15*time.Millisecond, func() error { usage = gacc.JetStreamUsage() if usage.Memory != 0 { return fmt.Errorf("Expected usage memory to be 0, got %d", usage.Memory) } return nil }) // Now send twice the number of messages. Should receive an error at some point, and we will check usage against limits. var errSeen string for i := 0; i < toSend*2; i++ { resp, _ := nc.Request("LIMITS", []byte("The quick brown fox jumped over the..."), 50*time.Millisecond) if string(resp.Data) != OK { errSeen = string(resp.Data) break } } if errSeen == "" { t.Fatalf("Expected to see an error when exceeding the account limits") } state = mset.state() var lim JetStreamAccountLimits checkFor(t, time.Second, 15*time.Millisecond, func() error { usage = gacc.JetStreamUsage() lim = al[_EMPTY_] if usage.Memory > uint64(lim.MaxMemory) { return fmt.Errorf("Expected memory to not exceed limit of %d, got %d", lim.MaxMemory, usage.Memory) } return nil }) // make sure that unlimited accounts work lim.MaxMemory = -1 if err := gacc.UpdateJetStreamLimits(al); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "LIMITS", "Hello World!") } } func TestJetStreamStreamFileTrackingAndLimits(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() gacc := s.GlobalAccount() al := map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: 8192, MaxStore: 9600, MaxStreams: -1, MaxConsumers: -1, }, } if err := gacc.UpdateJetStreamLimits(al); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } mconfig := &StreamConfig{Name: "LIMITS", Storage: FileStorage, Retention: WorkQueuePolicy} mset, err := gacc.addStream(mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "LIMITS", "Hello World!") } state := mset.state() usage := gacc.JetStreamUsage() // Make sure these are working correctly. if usage.Store != state.Bytes { t.Fatalf("Expected to have stream bytes match the store usage, %d vs %d", usage.Store, state.Bytes) } if usage.Streams != 1 { t.Fatalf("Expected to have 1 stream, got %d", usage.Streams) } // Do second stream. mconfig2 := &StreamConfig{Name: "NUM22", Storage: FileStorage, Retention: WorkQueuePolicy} mset2, err := gacc.addStream(mconfig2) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset2.delete() for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "NUM22", "Hello World!") } stats2 := mset2.state() usage = gacc.JetStreamUsage() if usage.Store != (state.Bytes + stats2.Bytes) { t.Fatalf("Expected to track both streams, usage is %v, stream1 is %v, stream2 is %v", usage.Store, state.Bytes, stats2.Bytes) } // Make sure delete works. mset2.delete() stats2 = mset2.state() usage = gacc.JetStreamUsage() if usage.Store != (state.Bytes + stats2.Bytes) { t.Fatalf("Expected to track both streams, account is %v, stream1 is %v, stream2 is %v", usage.Store, state.Bytes, stats2.Bytes) } // Now drain the first one by consuming the messages. o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() for i := 0; i < toSend; i++ { msg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg.Respond(nil) } nc.Flush() state = mset.state() usage = gacc.JetStreamUsage() if usage.Memory != 0 { t.Fatalf("Expected usage memeory to be 0, got %d", usage.Memory) } // Now send twice the number of messages. Should receive an error at some point, and we will check usage against limits. var errSeen string for i := 0; i < toSend*2; i++ { resp, _ := nc.Request("LIMITS", []byte("The quick brown fox jumped over the..."), 50*time.Millisecond) if string(resp.Data) != OK { errSeen = string(resp.Data) break } } if errSeen == "" { t.Fatalf("Expected to see an error when exceeding the account limits") } state = mset.state() usage = gacc.JetStreamUsage() lim := al[_EMPTY_] if usage.Memory > uint64(lim.MaxMemory) { t.Fatalf("Expected memory to not exceed limit of %d, got %d", lim.MaxMemory, usage.Memory) } } func TestJetStreamTieredLimits(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() gacc := s.GlobalAccount() tFail := map[string]JetStreamAccountLimits{ "nottheer": { MaxMemory: 8192, MaxStore: 9600, MaxStreams: -1, MaxConsumers: -1, }, } if err := gacc.UpdateJetStreamLimits(tFail); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } mconfig := &StreamConfig{Name: "LIMITS", Storage: FileStorage, Retention: WorkQueuePolicy} mset, err := gacc.addStream(mconfig) defer mset.delete() require_Error(t, err) require_Contains(t, err.Error(), "no JetStream default or applicable tiered limit present") tPass := map[string]JetStreamAccountLimits{ "R1": { MaxMemory: 8192, MaxStore: 9600, MaxStreams: -1, MaxConsumers: -1, }, } if err := gacc.UpdateJetStreamLimits(tPass); err != nil { t.Fatalf("Unexpected error updating jetstream account limits: %v", err) } } type obsi struct { cfg ConsumerConfig ack int } type info struct { cfg StreamConfig state StreamState obs []obsi } func TestJetStreamSimpleFileRecovery(t *testing.T) { base := runtime.NumGoroutine() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() ostate := make(map[string]info) nid := nuid.New() randomSubject := func() string { nid.RandomizePrefix() return fmt.Sprintf("SUBJ.%s", nid.Next()) } nc := clientConnectToServer(t, s) defer nc.Close() numStreams := 10 for i := 1; i <= numStreams; i++ { msetName := fmt.Sprintf("MMS-%d", i) subjects := []string{randomSubject(), randomSubject(), randomSubject()} msetConfig := StreamConfig{ Name: msetName, Storage: FileStorage, Subjects: subjects, MaxMsgs: 100, } mset, err := acc.addStream(&msetConfig) if err != nil { t.Fatalf("Unexpected error adding stream %q: %v", msetName, err) } toSend := rand.Intn(100) + 1 for n := 1; n <= toSend; n++ { msg := fmt.Sprintf("Hello %d", n*i) subj := subjects[rand.Intn(len(subjects))] sendStreamMsg(t, nc, subj, msg) } // Create up to 5 consumers. numObs := rand.Intn(5) + 1 var obs []obsi for n := 1; n <= numObs; n++ { oname := fmt.Sprintf("WQ-%d-%d", i, n) o, err := mset.addConsumer(workerModeConfig(oname)) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now grab some messages. toReceive := rand.Intn(toSend) + 1 rsubj := o.requestNextMsgSubject() for r := 0; r < toReceive; r++ { resp, err := nc.Request(rsubj, nil, time.Second) require_NoError(t, err) if resp != nil { resp.Respond(nil) } } obs = append(obs, obsi{o.config(), toReceive}) } ostate[msetName] = info{mset.config(), mset.state(), obs} } pusage := acc.JetStreamUsage() nc.Flush() // Shutdown and restart and make sure things come back. sd := s.JetStreamConfig().StoreDir s.Shutdown() checkFor(t, 2*time.Second, 200*time.Millisecond, func() error { delta := (runtime.NumGoroutine() - base) if delta > 3 { return fmt.Errorf("%d Go routines still exist post Shutdown()", delta) } return nil }) s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() acc = s.GlobalAccount() nusage := acc.JetStreamUsage() if !reflect.DeepEqual(nusage, pusage) { t.Fatalf("Usage does not match after restore: %+v vs %+v", nusage, pusage) } for mname, info := range ostate { mset, err := acc.lookupStream(mname) if err != nil { t.Fatalf("Expected to find a stream for %q", mname) } if state := mset.state(); !reflect.DeepEqual(state, info.state) { t.Fatalf("State does not match: %+v vs %+v", state, info.state) } if cfg := mset.config(); !reflect.DeepEqual(cfg, info.cfg) { t.Fatalf("Configs do not match: %+v vs %+v", cfg, info.cfg) } // Consumers. if mset.numConsumers() != len(info.obs) { t.Fatalf("Number of consumers do not match: %d vs %d", mset.numConsumers(), len(info.obs)) } for _, oi := range info.obs { if o := mset.lookupConsumer(oi.cfg.Durable); o != nil { if uint64(oi.ack+1) != o.nextSeq() { t.Fatalf("Consumer next seq is not correct: %d vs %d", oi.ack+1, o.nextSeq()) } } else { t.Fatalf("Expected to get an consumer") } } } } func TestJetStreamPushConsumerFlowControl(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := nc.SubscribeSync(nats.NewInbox()) require_NoError(t, err) defer sub.Unsubscribe() obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "dlc", DeliverSubject: sub.Subject, FlowControl: true, Heartbeat: 5 * time.Second, }, } req, err := json.Marshal(obsReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "dlc"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error != nil { t.Fatalf("Unexpected error: %+v", ccResp.Error) } // Grab the low level consumer so we can manually set the fc max. if mset, err := s.GlobalAccount().lookupStream("TEST"); err != nil { t.Fatalf("Error looking up stream: %v", err) } else if obs := mset.lookupConsumer("dlc"); obs == nil { t.Fatalf("Error looking up stream: %v", err) } else { obs.mu.Lock() obs.setMaxPendingBytes(16 * 1024) obs.mu.Unlock() } msgSize := 1024 msg := make([]byte, msgSize) crand.Read(msg) sendBatch := func(n int) { for i := 0; i < n; i++ { if _, err := js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkSubPending := func(numExpected int) { t.Helper() checkFor(t, time.Second, 100*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } sendBatch(100) checkSubPending(2) // First four data and flowcontrol from slow start pause. var n int for m, err := sub.NextMsg(time.Second); err == nil; m, err = sub.NextMsg(time.Second) { if m.Subject == "TEST" { n++ } else { // This should be a FC control message. if m.Header.Get("Status") != "100" { t.Fatalf("Expected a 100 status code, got %q", m.Header.Get("Status")) } if m.Header.Get("Description") != "FlowControl Request" { t.Fatalf("Wrong description, got %q", m.Header.Get("Description")) } m.Respond(nil) } } if n != 100 { t.Fatalf("Expected to receive all 100 messages but got %d", n) } } func TestJetStreamFlowControlRequiresHeartbeats(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", DeliverSubject: nats.NewInbox(), FlowControl: true, }); err == nil || IsNatsErr(err, JSConsumerWithFlowControlNeedsHeartbeats) { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamPushConsumerIdleHeartbeats(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := nc.SubscribeSync(nats.NewInbox()) require_NoError(t, err) defer sub.Unsubscribe() // Test errors first obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverSubject: sub.Subject, Heartbeat: time.Millisecond, }, } req, err := json.Marshal(obsReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "TEST"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error == nil { t.Fatalf("Expected an error, got none") } // Set acceptable heartbeat. obsReq.Config.Heartbeat = 100 * time.Millisecond req, err = json.Marshal(obsReq) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "TEST"), req, time.Second) require_NoError(t, err) ccResp.Error = nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, time.Second, 20*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs < 9 { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, 9) } return nil }) m, _ := sub.NextMsg(0) if m.Header.Get("Status") != "100" { t.Fatalf("Expected a 100 status code, got %q", m.Header.Get("Status")) } if m.Header.Get("Description") != "Idle Heartbeat" { t.Fatalf("Wrong description, got %q", m.Header.Get("Description")) } } func TestJetStreamPushConsumerIdleHeartbeatsWithFilterSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo", "bar"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } hbC := make(chan *nats.Msg, 8) sub, err := nc.ChanSubscribe(nats.NewInbox(), hbC) require_NoError(t, err) defer sub.Unsubscribe() obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverSubject: sub.Subject, FilterSubject: "bar", Heartbeat: 100 * time.Millisecond, }, } req, err := json.Marshal(obsReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "TEST"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } st := time.NewTicker(10 * time.Millisecond) defer st.Stop() done := time.NewTimer(time.Second) defer done.Stop() for { select { case <-st.C: js.Publish("foo", []byte("HELLO FOO")) case <-done.C: t.Fatalf("Expected to have seen idle heartbeats for consumer") case <-hbC: return } } } func TestJetStreamPushConsumerIdleHeartbeatsWithNoInterest(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } dsubj := "d.22" hbC := make(chan *nats.Msg, 8) sub, err := nc.ChanSubscribe("d.>", hbC) require_NoError(t, err) defer sub.Unsubscribe() obsReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverSubject: dsubj, Heartbeat: 100 * time.Millisecond, }, } req, err := json.Marshal(obsReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "TEST"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error != nil { t.Fatalf("Unexpected error: %+v", ccResp.Error) } done := time.NewTimer(400 * time.Millisecond) defer done.Stop() for { select { case <-done.C: return case m := <-hbC: if m.Header.Get("Status") == "100" { t.Fatalf("Did not expect to see a heartbeat with no formal interest") } } } } func TestJetStreamInfoAPIWithHeaders(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() m := nats.NewMsg(JSApiAccountInfo) m.Header.Add("Accept-Encoding", "json") m.Header.Add("Authorization", "s3cr3t") m.Data = []byte("HELLO-JS!") resp, err := nc.RequestMsg(m, time.Second) require_NoError(t, err) var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Error != nil { t.Fatalf("Received an error: %+v", info.Error) } } func TestJetStreamRequestAPI(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() // This will get the current information about usage and limits for this account. resp, err := nc.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now create a stream. msetCfg := StreamConfig{ Name: "MSET22", Storage: FileStorage, Subjects: []string{"foo", "bar", "baz"}, MaxMsgs: 100, } req, err := json.Marshal(msetCfg) require_NoError(t, err) resp, _ = nc.Request(fmt.Sprintf(JSApiStreamCreateT, msetCfg.Name), req, time.Second) var scResp JSApiStreamCreateResponse if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.StreamInfo == nil || scResp.Error != nil { t.Fatalf("Did not receive correct response: %+v", scResp.Error) } if time.Since(scResp.Created) > time.Second { t.Fatalf("Created time seems wrong: %v\n", scResp.Created) } // Check that the name in config has to match the name in the subject resp, _ = nc.Request(fmt.Sprintf(JSApiStreamCreateT, "BOB"), req, time.Second) scResp.Error, scResp.StreamInfo = nil, nil if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, scResp.Error, JSStreamMismatchErr) // Check that update works. msetCfg.Subjects = []string{"foo", "bar", "baz"} msetCfg.MaxBytes = 2222222 req, err = json.Marshal(msetCfg) require_NoError(t, err) resp, _ = nc.Request(fmt.Sprintf(JSApiStreamUpdateT, msetCfg.Name), req, time.Second) scResp.Error, scResp.StreamInfo = nil, nil if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.StreamInfo == nil || scResp.Error != nil { t.Fatalf("Did not receive correct response: %+v", scResp.Error) } // Check that updating a non existing stream fails cfg := StreamConfig{ Name: "UNKNOWN_STREAM", Storage: FileStorage, Subjects: []string{"foo"}, } req, err = json.Marshal(cfg) require_NoError(t, err) resp, _ = nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second) scResp.Error, scResp.StreamInfo = nil, nil if err := json.Unmarshal(resp.Data, &scResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if scResp.StreamInfo != nil || scResp.Error == nil || scResp.Error.Code != 404 { t.Fatalf("Unexpected error: %+v", scResp.Error) } // Now lookup info again and see that we can see the new stream. resp, err = nc.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) if err = json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Streams != 1 { t.Fatalf("Expected to see 1 Stream, got %d", info.Streams) } // Make sure list names works. resp, err = nc.Request(JSApiStreams, nil, time.Second) require_NoError(t, err) var namesResponse JSApiStreamNamesResponse if err = json.Unmarshal(resp.Data, &namesResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(namesResponse.Streams) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(namesResponse.Streams)) } if namesResponse.Total != 1 { t.Fatalf("Expected total to be 1 but got %d", namesResponse.Total) } if namesResponse.Offset != 0 { t.Fatalf("Expected offset to be 0 but got %d", namesResponse.Offset) } if namesResponse.Limit != JSApiNamesLimit { t.Fatalf("Expected limit to be %d but got %d", JSApiNamesLimit, namesResponse.Limit) } if namesResponse.Streams[0] != msetCfg.Name { t.Fatalf("Expected to get %q, but got %q", msetCfg.Name, namesResponse.Streams[0]) } // Now do detailed version. resp, err = nc.Request(JSApiStreamList, nil, time.Second) require_NoError(t, err) var listResponse JSApiStreamListResponse if err = json.Unmarshal(resp.Data, &listResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(listResponse.Streams) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(listResponse.Streams)) } if listResponse.Total != 1 { t.Fatalf("Expected total to be 1 but got %d", listResponse.Total) } if listResponse.Offset != 0 { t.Fatalf("Expected offset to be 0 but got %d", listResponse.Offset) } if listResponse.Limit != JSApiListLimit { t.Fatalf("Expected limit to be %d but got %d", JSApiListLimit, listResponse.Limit) } if listResponse.Streams[0].Config.Name != msetCfg.Name { t.Fatalf("Expected to get %q, but got %q", msetCfg.Name, listResponse.Streams[0].Config.Name) } // Now send some messages, then we can poll for info on this stream. toSend := 10 for i := 0; i < toSend; i++ { nc.Request("foo", []byte("WELCOME JETSTREAM"), time.Second) } resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, msetCfg.Name), nil, time.Second) require_NoError(t, err) var msi StreamInfo if err = json.Unmarshal(resp.Data, &msi); err != nil { t.Fatalf("Unexpected error: %v", err) } if msi.State.Msgs != uint64(toSend) { t.Fatalf("Expected to get %d msgs, got %d", toSend, msi.State.Msgs) } if time.Since(msi.Created) > time.Second { t.Fatalf("Created time seems wrong: %v\n", msi.Created) } // Looking up one that is not there should yield an error. resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "BOB"), nil, time.Second) require_NoError(t, err) var bResp JSApiStreamInfoResponse if err = json.Unmarshal(resp.Data, &bResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, bResp.Error, JSStreamNotFoundErr) // Now create a consumer. delivery := nats.NewInbox() obsReq := CreateConsumerRequest{ Stream: msetCfg.Name, Config: ConsumerConfig{DeliverSubject: delivery}, } req, err = json.Marshal(obsReq) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, msetCfg.Name), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } // Ephemerals are now not rejected when there is no interest. if ccResp.ConsumerInfo == nil || ccResp.Error != nil { t.Fatalf("Got a bad response %+v", ccResp) } if time.Since(ccResp.Created) > time.Second { t.Fatalf("Created time seems wrong: %v\n", ccResp.Created) } // Now create subscription and make sure we get proper response. sub, _ := nc.SubscribeSync(delivery) nc.Flush() checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, toSend) } return nil }) // Check that we get an error if the stream name in the subject does not match the config. resp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "BOB"), req, time.Second) require_NoError(t, err) ccResp.Error, ccResp.ConsumerInfo = nil, nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } // Since we do not have interest this should have failed. checkNatsError(t, ccResp.Error, JSStreamMismatchErr) // Get the list of all of the consumers for our stream. resp, err = nc.Request(fmt.Sprintf(JSApiConsumersT, msetCfg.Name), nil, time.Second) require_NoError(t, err) var clResponse JSApiConsumerNamesResponse if err = json.Unmarshal(resp.Data, &clResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(clResponse.Consumers) != 1 { t.Fatalf("Expected only 1 consumer but got %d", len(clResponse.Consumers)) } // Now let's get info about our consumer. cName := clResponse.Consumers[0] resp, err = nc.Request(fmt.Sprintf(JSApiConsumerInfoT, msetCfg.Name, cName), nil, time.Second) require_NoError(t, err) var oinfo ConsumerInfo if err = json.Unmarshal(resp.Data, &oinfo); err != nil { t.Fatalf("Unexpected error: %v", err) } // Do some sanity checking. // Must match consumer.go const randConsumerNameLen = 8 if len(oinfo.Name) != randConsumerNameLen { t.Fatalf("Expected ephemeral name, got %q", oinfo.Name) } if len(oinfo.Config.Durable) != 0 { t.Fatalf("Expected no durable name, but got %q", oinfo.Config.Durable) } if oinfo.Config.DeliverSubject != delivery { t.Fatalf("Expected to have delivery subject of %q, got %q", delivery, oinfo.Config.DeliverSubject) } if oinfo.Delivered.Consumer != 10 { t.Fatalf("Expected consumer delivered sequence of 10, got %d", oinfo.Delivered.Consumer) } if oinfo.AckFloor.Consumer != 10 { t.Fatalf("Expected ack floor to be 10, got %d", oinfo.AckFloor.Consumer) } // Now delete the consumer. resp, _ = nc.Request(fmt.Sprintf(JSApiConsumerDeleteT, msetCfg.Name, cName), nil, time.Second) var cdResp JSApiConsumerDeleteResponse if err = json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !cdResp.Success || cdResp.Error != nil { t.Fatalf("Got a bad response %+v", ccResp) } // Make sure we can't create a durable using the ephemeral API endpoint. obsReq = CreateConsumerRequest{ Stream: msetCfg.Name, Config: ConsumerConfig{Durable: "myd", DeliverSubject: delivery}, } req, err = json.Marshal(obsReq) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, msetCfg.Name), req, time.Second) require_NoError(t, err) ccResp.Error, ccResp.ConsumerInfo = nil, nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, ccResp.Error, JSConsumerEphemeralWithDurableNameErr) // Now make sure we can create a durable on the subject with the proper name. resp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, msetCfg.Name, obsReq.Config.Durable), req, time.Second) ccResp.Error, ccResp.ConsumerInfo = nil, nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.ConsumerInfo == nil || ccResp.Error != nil { t.Fatalf("Did not receive correct response") } // Make sure empty durable in cfg does not work obsReq2 := CreateConsumerRequest{ Stream: msetCfg.Name, Config: ConsumerConfig{DeliverSubject: delivery}, } req2, err := json.Marshal(obsReq2) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, msetCfg.Name, obsReq.Config.Durable), req2, time.Second) require_NoError(t, err) ccResp.Error, ccResp.ConsumerInfo = nil, nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, ccResp.Error, JSConsumerDurableNameNotSetErr) // Now delete a msg. dreq := JSApiMsgDeleteRequest{Seq: 2} dreqj, err := json.Marshal(dreq) require_NoError(t, err) resp, _ = nc.Request(fmt.Sprintf(JSApiMsgDeleteT, msetCfg.Name), dreqj, time.Second) var delMsgResp JSApiMsgDeleteResponse if err = json.Unmarshal(resp.Data, &delMsgResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !delMsgResp.Success || delMsgResp.Error != nil { t.Fatalf("Got a bad response %+v", delMsgResp.Error) } // Now purge the stream. resp, _ = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, msetCfg.Name), nil, time.Second) var pResp JSApiStreamPurgeResponse if err = json.Unmarshal(resp.Data, &pResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !pResp.Success || pResp.Error != nil { t.Fatalf("Got a bad response %+v", pResp) } if pResp.Purged != 9 { t.Fatalf("Expected 9 purged, got %d", pResp.Purged) } // Now delete the stream. resp, _ = nc.Request(fmt.Sprintf(JSApiStreamDeleteT, msetCfg.Name), nil, time.Second) var dResp JSApiStreamDeleteResponse if err = json.Unmarshal(resp.Data, &dResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !dResp.Success || dResp.Error != nil { t.Fatalf("Got a bad response %+v", dResp.Error) } // Now grab stats again. // This will get the current information about usage and limits for this account. resp, err = nc.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Streams != 0 { t.Fatalf("Expected no remaining streams, got %d", info.Streams) } // Now do templates. mcfg := &StreamConfig{ Subjects: []string{"kv.*"}, Retention: LimitsPolicy, MaxAge: time.Hour, MaxMsgs: 4, Storage: MemoryStorage, Replicas: 1, } template := &StreamTemplateConfig{ Name: "kv", Config: mcfg, MaxStreams: 4, } req, err = json.Marshal(template) require_NoError(t, err) // Check that the name in config has to match the name in the subject resp, _ = nc.Request(fmt.Sprintf(JSApiTemplateCreateT, "BOB"), req, time.Second) var stResp JSApiStreamTemplateCreateResponse if err = json.Unmarshal(resp.Data, &stResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, stResp.Error, JSTemplateNameNotMatchSubjectErr) resp, _ = nc.Request(fmt.Sprintf(JSApiTemplateCreateT, template.Name), req, time.Second) stResp.Error, stResp.StreamTemplateInfo = nil, nil if err = json.Unmarshal(resp.Data, &stResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if stResp.StreamTemplateInfo == nil || stResp.Error != nil { t.Fatalf("Did not receive correct response") } // Create a second one. template.Name = "ss" template.Config.Subjects = []string{"foo", "bar"} req, err = json.Marshal(template) if err != nil { t.Fatalf("Unexpected error: %s", err) } resp, _ = nc.Request(fmt.Sprintf(JSApiTemplateCreateT, template.Name), req, time.Second) stResp.Error, stResp.StreamTemplateInfo = nil, nil if err = json.Unmarshal(resp.Data, &stResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if stResp.StreamTemplateInfo == nil || stResp.Error != nil { t.Fatalf("Did not receive correct response") } // Now grab the list of templates var tListResp JSApiStreamTemplateNamesResponse resp, err = nc.Request(JSApiTemplates, nil, time.Second) if err = json.Unmarshal(resp.Data, &tListResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(tListResp.Templates) != 2 { t.Fatalf("Expected 2 templates but got %d", len(tListResp.Templates)) } slices.Sort(tListResp.Templates) if tListResp.Templates[0] != "kv" { t.Fatalf("Expected to get %q, but got %q", "kv", tListResp.Templates[0]) } if tListResp.Templates[1] != "ss" { t.Fatalf("Expected to get %q, but got %q", "ss", tListResp.Templates[1]) } // Now delete one. // Test bad name. resp, _ = nc.Request(fmt.Sprintf(JSApiTemplateDeleteT, "bob"), nil, time.Second) var tDeleteResp JSApiStreamTemplateDeleteResponse if err = json.Unmarshal(resp.Data, &tDeleteResp); err != nil { t.Fatalf("Unexpected error: %v", err) } checkNatsError(t, tDeleteResp.Error, JSStreamTemplateNotFoundErr) resp, _ = nc.Request(fmt.Sprintf(JSApiTemplateDeleteT, "ss"), nil, time.Second) tDeleteResp.Error = nil if err = json.Unmarshal(resp.Data, &tDeleteResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !tDeleteResp.Success || tDeleteResp.Error != nil { t.Fatalf("Did not receive correct response: %+v", tDeleteResp.Error) } resp, err = nc.Request(JSApiTemplates, nil, time.Second) tListResp.Error, tListResp.Templates = nil, nil if err = json.Unmarshal(resp.Data, &tListResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(tListResp.Templates) != 1 { t.Fatalf("Expected 1 template but got %d", len(tListResp.Templates)) } if tListResp.Templates[0] != "kv" { t.Fatalf("Expected to get %q, but got %q", "kv", tListResp.Templates[0]) } // First create a stream from the template sendStreamMsg(t, nc, "kv.22", "derek") // Last do info resp, err = nc.Request(fmt.Sprintf(JSApiTemplateInfoT, "kv"), nil, time.Second) require_NoError(t, err) var ti StreamTemplateInfo if err = json.Unmarshal(resp.Data, &ti); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(ti.Streams) != 1 { t.Fatalf("Expected 1 stream, got %d", len(ti.Streams)) } if ti.Streams[0] != canonicalName("kv.22") { t.Fatalf("Expected stream with name %q, but got %q", canonicalName("kv.22"), ti.Streams[0]) } // Test that we can send nil or an empty legal json for requests that take no args. // We know this stream does not exist, this just checking request processing. checkEmptyReqArg := func(arg string) { t.Helper() var req []byte if len(arg) > 0 { req = []byte(arg) } resp, err = nc.Request(fmt.Sprintf(JSApiStreamDeleteT, "foo_bar_baz"), req, time.Second) var dResp JSApiStreamDeleteResponse if err = json.Unmarshal(resp.Data, &dResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if dResp.Error == nil || dResp.Error.Code != 404 { t.Fatalf("Got a bad response, expected a 404 response %+v", dResp.Error) } } checkEmptyReqArg("") checkEmptyReqArg("{}") checkEmptyReqArg(" {} ") checkEmptyReqArg(" { } ") } func TestJetStreamFilteredStreamNames(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() // Create some streams. var snid int createStream := func(subjects []string) { t.Helper() snid++ name := fmt.Sprintf("S-%d", snid) sc := &StreamConfig{Name: name, Subjects: subjects} if _, err := s.GlobalAccount().addStream(sc); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } } createStream([]string{"foo"}) // S1 createStream([]string{"bar"}) // S2 createStream([]string{"baz"}) // S3 createStream([]string{"foo.*", "bar.*"}) // S4 createStream([]string{"foo-1.22", "bar-1.33"}) // S5 expectStreams := func(filter string, streams []string) { t.Helper() req, _ := json.Marshal(&JSApiStreamNamesRequest{Subject: filter}) r, _ := nc.Request(JSApiStreams, req, time.Second) var resp JSApiStreamNamesResponse if err := json.Unmarshal(r.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(resp.Streams) != len(streams) { t.Fatalf("Expected %d results, got %d", len(streams), len(resp.Streams)) } } expectStreams("foo", []string{"S1"}) expectStreams("bar", []string{"S2"}) expectStreams("baz", []string{"S3"}) expectStreams("*", []string{"S1", "S2", "S3"}) expectStreams(">", []string{"S1", "S2", "S3", "S4", "S5"}) expectStreams("*.*", []string{"S4", "S5"}) expectStreams("*.22", []string{"S4", "S5"}) } func TestJetStreamUpdateStream(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Test basic updates. We allow changing the subjects, limits, and no_ack along with replicas(TBD w/ cluster) cfg := *c.mconfig // Can't change name. cfg.Name = "bar" if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "name must match") { t.Fatalf("Expected error trying to update name") } // Can't change max consumers for now. cfg = *c.mconfig cfg.MaxConsumers = 10 if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "can not change") { t.Fatalf("Expected error trying to change MaxConsumers") } // Can't change storage types. cfg = *c.mconfig if cfg.Storage == FileStorage { cfg.Storage = MemoryStorage } else { cfg.Storage = FileStorage } if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "can not change") { t.Fatalf("Expected error trying to change Storage") } // Can't change replicas > 1 for now. cfg = *c.mconfig cfg.Replicas = 10 if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "maximum replicas") { t.Fatalf("Expected error trying to change Replicas") } // Can't have a template set for now. cfg = *c.mconfig cfg.Template = "baz" if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "template") { t.Fatalf("Expected error trying to change Template owner") } // Can't change limits policy. cfg = *c.mconfig cfg.Retention = WorkQueuePolicy if err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), "can not change") { t.Fatalf("Expected error trying to change Retention") } // Now test changing limits. nc := clientConnectToServer(t, s) defer nc.Close() pending := uint64(100) for i := uint64(0); i < pending; i++ { sendStreamMsg(t, nc, "foo", "0123456789") } pendingBytes := mset.state().Bytes checkPending := func(msgs, bts uint64) { t.Helper() state := mset.state() if state.Msgs != msgs { t.Fatalf("Expected %d messages, got %d", msgs, state.Msgs) } if state.Bytes != bts { t.Fatalf("Expected %d bytes, got %d", bts, state.Bytes) } } checkPending(pending, pendingBytes) // Update msgs to higher. cfg = *c.mconfig cfg.MaxMsgs = int64(pending * 2) if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } if mset.config().MaxMsgs != cfg.MaxMsgs { t.Fatalf("Expected the change to take effect, %d vs %d", mset.config().MaxMsgs, cfg.MaxMsgs) } checkPending(pending, pendingBytes) // Update msgs to lower. cfg = *c.mconfig cfg.MaxMsgs = int64(pending / 2) if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } if mset.config().MaxMsgs != cfg.MaxMsgs { t.Fatalf("Expected the change to take effect, %d vs %d", mset.config().MaxMsgs, cfg.MaxMsgs) } checkPending(pending/2, pendingBytes/2) // Now do bytes. cfg = *c.mconfig cfg.MaxBytes = int64(pendingBytes / 4) if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } if mset.config().MaxBytes != cfg.MaxBytes { t.Fatalf("Expected the change to take effect, %d vs %d", mset.config().MaxBytes, cfg.MaxBytes) } checkPending(pending/4, pendingBytes/4) // Now do age. cfg = *c.mconfig cfg.MaxAge = time.Second if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } // Just wait a bit for expiration. time.Sleep(2 * time.Second) if mset.config().MaxAge != cfg.MaxAge { t.Fatalf("Expected the change to take effect, %d vs %d", mset.config().MaxAge, cfg.MaxAge) } checkPending(0, 0) // Now put back to original. cfg = *c.mconfig if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } for i := uint64(0); i < pending; i++ { sendStreamMsg(t, nc, "foo", "0123456789") } // subject changes. // Add in a subject first. cfg = *c.mconfig cfg.Subjects = []string{"foo", "bar"} if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } // Make sure we can still send to foo. sendStreamMsg(t, nc, "foo", "0123456789") // And we can now send to bar. sendStreamMsg(t, nc, "bar", "0123456789") // Now delete both and change to baz only. cfg.Subjects = []string{"baz"} if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } // Make sure we do not get response acks for "foo" or "bar". if resp, err := nc.Request("foo", nil, 25*time.Millisecond); err == nil || resp != nil { t.Fatalf("Expected no response from jetstream for deleted subject: %q", "foo") } if resp, err := nc.Request("bar", nil, 25*time.Millisecond); err == nil || resp != nil { t.Fatalf("Expected no response from jetstream for deleted subject: %q", "bar") } // Make sure we can send to "baz" sendStreamMsg(t, nc, "baz", "0123456789") if nmsgs := mset.state().Msgs; nmsgs != pending+3 { t.Fatalf("Expected %d msgs, got %d", pending+3, nmsgs) } // FileStore restarts for config save. cfg = *c.mconfig if cfg.Storage == FileStorage { cfg.Subjects = []string{"foo", "bar"} cfg.MaxMsgs = 2222 cfg.MaxBytes = 3333333 cfg.MaxAge = 22 * time.Hour if err := mset.update(&cfg); err != nil { t.Fatalf("Unexpected error %v", err) } // Pull since certain defaults etc are set in processing. cfg = mset.config() // Restart the // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) defer s.Shutdown() mset, err = s.GlobalAccount().lookupStream(cfg.Name) if err != nil { t.Fatalf("Expected to find a stream for %q", cfg.Name) } restored_cfg := mset.config() if !reflect.DeepEqual(cfg, restored_cfg) { t.Fatalf("restored configuration does not match: \n%+v\n vs \n%+v", restored_cfg, cfg) } } }) } } func TestJetStreamDeleteMsg(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc, js := jsClientConnect(t, s) defer nc.Close() pubTen := func() { t.Helper() for i := 0; i < 10; i++ { js.Publish("foo", []byte("Hello World!")) } } pubTen() state := mset.state() if state.Msgs != 10 { t.Fatalf("Expected 10 messages, got %d", state.Msgs) } bytesPerMsg := state.Bytes / 10 if bytesPerMsg == 0 { t.Fatalf("Expected non-zero bytes for msg size") } deleteAndCheck := func(seq, expectedFirstSeq uint64) { t.Helper() beforeState := mset.state() if removed, _ := mset.deleteMsg(seq); !removed { t.Fatalf("Expected the delete of sequence %d to succeed", seq) } expectedState := beforeState expectedState.Msgs-- expectedState.Bytes -= bytesPerMsg expectedState.FirstSeq = expectedFirstSeq sm, err := mset.getMsg(expectedFirstSeq) if err != nil { t.Fatalf("Error fetching message for seq: %d - %v", expectedFirstSeq, err) } expectedState.FirstTime = sm.Time expectedState.Deleted = nil expectedState.NumDeleted = 0 afterState := mset.state() afterState.Deleted = nil afterState.NumDeleted = 0 // Ignore first time in this test. if !reflect.DeepEqual(afterState, expectedState) { t.Fatalf("Stats not what we expected. Expected %+v, got %+v\n", expectedState, afterState) } } // Delete one from the middle deleteAndCheck(5, 1) // Now make sure sequences are updated properly. // Delete first msg. deleteAndCheck(1, 2) // Now last deleteAndCheck(10, 2) // Now gaps. deleteAndCheck(3, 2) deleteAndCheck(2, 4) mset.purge(nil) // Put ten more one. pubTen() deleteAndCheck(11, 12) deleteAndCheck(15, 12) deleteAndCheck(16, 12) deleteAndCheck(20, 12) // Only file storage beyond here. if c.mconfig.Storage == MemoryStorage { return } // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) sd := s.JetStreamConfig().StoreDir // Shutdown the s.Shutdown() s = RunJetStreamServerOnPort(port, sd) defer s.Shutdown() mset, err = s.GlobalAccount().lookupStream("foo") if err != nil { t.Fatalf("Expected to get the stream back") } expected := StreamState{Msgs: 6, Bytes: 6 * bytesPerMsg, FirstSeq: 12, LastSeq: 20, NumSubjects: 1} state = mset.state() state.FirstTime, state.LastTime, state.Deleted, state.NumDeleted = time.Time{}, time.Time{}, nil, 0 if !reflect.DeepEqual(expected, state) { t.Fatalf("State not what we expected. Expected %+v, got %+v\n", expected, state) } // Now create an consumer and make sure we get the right sequence. nc = clientConnectToServer(t, s) defer nc.Close() delivery := nats.NewInbox() sub, _ := nc.SubscribeSync(delivery) nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: "foo"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } expectedStoreSeq := []uint64{12, 13, 14, 17, 18, 19} for i := 0; i < 6; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if o.streamSeqFromReply(m.Reply) != expectedStoreSeq[i] { t.Fatalf("Expected store seq of %d, got %d", expectedStoreSeq[i], o.streamSeqFromReply(m.Reply)) } } }) } } // https://github.com/nats-io/jetstream/issues/396 func TestJetStreamLimitLockBug(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxMsgs: 10, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxMsgs: 10, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "ok") } state := mset.state() if state.Msgs != 10 { t.Fatalf("Expected 10 messages, got %d", state.Msgs) } }) } } func TestJetStreamNextMsgNoInterest(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() cfg := &StreamConfig{Name: "foo", Storage: FileStorage} mset, err := s.GlobalAccount().addStream(cfg) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectWithOldRequest(t, s) defer nc.Close() // Now create an consumer and make sure it functions properly. o, err := mset.addConsumer(workerModeConfig("WQ")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nextSubj := o.requestNextMsgSubject() // Queue up a worker but use a short time out. if _, err := nc.Request(nextSubj, nil, time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected a timeout error and no response with acks suppressed") } // Now send a message, the worker from above will still be known but we want to make // sure the system detects that so we will do a request for next msg right behind it. nc.Publish("foo", []byte("OK")) if msg, err := nc.Request(nextSubj, nil, 5*time.Millisecond); err != nil { t.Fatalf("Unexpected error: %v", err) } else { msg.Respond(nil) // Ack } // Now queue up 10 workers. for i := 0; i < 10; i++ { if _, err := nc.Request(nextSubj, nil, time.Microsecond); err != nats.ErrTimeout { t.Fatalf("Expected a timeout error and no response with acks suppressed") } } // Now publish ten messages. for i := 0; i < 10; i++ { nc.Publish("foo", []byte("OK")) } nc.Flush() for i := 0; i < 10; i++ { if msg, err := nc.Request(nextSubj, nil, 10*time.Millisecond); err != nil { t.Fatalf("Unexpected error for %d: %v", i, err) } else { msg.Respond(nil) // Ack } } nc.Flush() checkFor(t, time.Second, 10*time.Millisecond, func() error { ostate := o.info() if ostate.AckFloor.Stream != 11 || ostate.NumAckPending > 0 { return fmt.Errorf("Inconsistent ack state: %+v", ostate) } return nil }) }) } } func TestJetStreamMsgHeaders(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {name: "MemoryStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: MemoryStorage, Replicas: 1, }}, {name: "FileStore", mconfig: &StreamConfig{ Name: "foo", Retention: LimitsPolicy, MaxAge: time.Hour, Storage: FileStorage, Replicas: 1, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() m := nats.NewMsg("foo") m.Header.Add("Accept-Encoding", "json") m.Header.Add("Authorization", "s3cr3t") m.Data = []byte("Hello JetStream Headers - #1!") nc.PublishMsg(m) nc.Flush() checkFor(t, time.Second*2, time.Millisecond*250, func() error { state := mset.state() if state.Msgs != 1 { return fmt.Errorf("Expected 1 message, got %d", state.Msgs) } if state.Bytes == 0 { return fmt.Errorf("Expected non-zero bytes") } return nil }) // Now access raw from stream. sm, err := mset.getMsg(1) if err != nil { t.Fatalf("Unexpected error getting stored message: %v", err) } // Calculate the []byte version of the headers. var b bytes.Buffer b.WriteString("NATS/1.0\r\n") http.Header(m.Header).Write(&b) b.WriteString("\r\n") hdr := b.Bytes() if !bytes.Equal(sm.Header, hdr) { t.Fatalf("Message headers do not match, %q vs %q", hdr, sm.Header) } if !bytes.Equal(sm.Data, m.Data) { t.Fatalf("Message data do not match, %q vs %q", m.Data, sm.Data) } // Now do consumer based. sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() cm, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } // Check the message. // Check out original headers. if cm.Header.Get("Accept-Encoding") != "json" || cm.Header.Get("Authorization") != "s3cr3t" { t.Fatalf("Original headers not present") } if !bytes.Equal(m.Data, cm.Data) { t.Fatalf("Message payloads are not the same: %q vs %q", cm.Data, m.Data) } }) } } func TestJetStreamTemplateBasics(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() mcfg := &StreamConfig{ Subjects: []string{"kv.*"}, Retention: LimitsPolicy, MaxAge: time.Hour, MaxMsgs: 4, Storage: MemoryStorage, Replicas: 1, } template := &StreamTemplateConfig{ Name: "kv", Config: mcfg, MaxStreams: 4, } if _, err := acc.addStreamTemplate(template); err != nil { t.Fatalf("Unexpected error: %v", err) } if templates := acc.templates(); len(templates) != 1 { t.Fatalf("Expected to get array of 1 template, got %d", len(templates)) } if err := acc.deleteStreamTemplate("foo"); err == nil { t.Fatalf("Expected an error for non-existent template") } if err := acc.deleteStreamTemplate(template.Name); err != nil { t.Fatalf("Unexpected error: %v", err) } if templates := acc.templates(); len(templates) != 0 { t.Fatalf("Expected to get array of no templates, got %d", len(templates)) } // Add it back in and test basics if _, err := acc.addStreamTemplate(template); err != nil { t.Fatalf("Unexpected error: %v", err) } // Connect a client and send a message which should trigger the stream creation. nc := clientConnectToServer(t, s) defer nc.Close() sendStreamMsg(t, nc, "kv.22", "derek") sendStreamMsg(t, nc, "kv.33", "cat") sendStreamMsg(t, nc, "kv.44", "sam") sendStreamMsg(t, nc, "kv.55", "meg") if nms := acc.numStreams(); nms != 4 { t.Fatalf("Expected 4 auto-created streams, got %d", nms) } // This one should fail due to max. if resp, err := nc.Request("kv.99", nil, 100*time.Millisecond); err == nil { t.Fatalf("Expected this to fail, but got %q", resp.Data) } // Now delete template and make sure the underlying streams go away too. if err := acc.deleteStreamTemplate(template.Name); err != nil { t.Fatalf("Unexpected error: %v", err) } if nms := acc.numStreams(); nms != 0 { t.Fatalf("Expected no auto-created streams to remain, got %d", nms) } } func TestJetStreamTemplateFileStoreRecovery(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() mcfg := &StreamConfig{ Subjects: []string{"kv.*"}, Retention: LimitsPolicy, MaxAge: time.Hour, MaxMsgs: 50, Storage: FileStorage, Replicas: 1, } template := &StreamTemplateConfig{ Name: "kv", Config: mcfg, MaxStreams: 100, } if _, err := acc.addStreamTemplate(template); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we can not add in a stream on our own with a template owner. badCfg := *mcfg badCfg.Name = "bad" badCfg.Template = "kv" if _, err := acc.addStream(&badCfg); err == nil { t.Fatalf("Expected error adding stream with direct template owner") } // Connect a client and send a message which should trigger the stream creation. nc := clientConnectToServer(t, s) defer nc.Close() for i := 1; i <= 100; i++ { subj := fmt.Sprintf("kv.%d", i) for x := 0; x < 50; x++ { sendStreamMsg(t, nc, subj, "Hello") } } nc.Flush() if nms := acc.numStreams(); nms != 100 { t.Fatalf("Expected 100 auto-created streams, got %d", nms) } // Capture port since it was dynamic. u, _ := url.Parse(s.ClientURL()) port, _ := strconv.Atoi(u.Port()) restartServer := func() { t.Helper() sd := s.JetStreamConfig().StoreDir // Stop current s.Shutdown() // Restart. s = RunJetStreamServerOnPort(port, sd) } // Restart. restartServer() defer s.Shutdown() acc = s.GlobalAccount() if nms := acc.numStreams(); nms != 100 { t.Fatalf("Expected 100 auto-created streams, got %d", nms) } tmpl, err := acc.lookupStreamTemplate(template.Name) require_NoError(t, err) // Make sure t.delete() survives restart. tmpl.delete() // Restart. restartServer() defer s.Shutdown() acc = s.GlobalAccount() if nms := acc.numStreams(); nms != 0 { t.Fatalf("Expected no auto-created streams, got %d", nms) } if _, err := acc.lookupStreamTemplate(template.Name); err == nil { t.Fatalf("Expected to not find the template after restart") } } // This will be testing our ability to conditionally rewrite subjects for last mile // when working with JetStream. Consumers receive messages that have their subjects // rewritten to match the original subject. NATS routing is all subject based except // for the last mile to the client. func TestJetStreamSingleInstanceRemoteAccess(t *testing.T) { ca := createClusterWithName(t, "A", 1) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 1, ca) defer shutdownCluster(cb) // Connect our leafnode server to cluster B. opts := cb.opts[rand.Intn(len(cb.opts))] s, _ := runSolicitLeafServer(opts) defer s.Shutdown() checkLeafNodeConnected(t, s) if err := s.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()}); err != nil { t.Fatalf("Expected no error, got %v", err) } mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "foo", Storage: MemoryStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() toSend := 10 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo", "Hello World!") } // Now create a push based consumer. Connected to the non-jetstream server via a random server on cluster A. sl := ca.servers[rand.Intn(len(ca.servers))] nc2 := clientConnectToServer(t, sl) defer nc2.Close() sub, _ := nc2.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() // Need to wait for interest to propagate across GW. nc2.Flush() time.Sleep(25 * time.Millisecond) o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject}) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() checkSubPending := func(numExpected int) { t.Helper() checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } checkSubPending(toSend) checkMsg := func(m *nats.Msg, err error, i int) { t.Helper() if err != nil { t.Fatalf("Got an error checking message: %v", err) } if m.Subject != "foo" { t.Fatalf("Expected original subject of %q, but got %q", "foo", m.Subject) } // Now check that reply subject exists and has a sequence as the last token. if seq := o.seqFromReply(m.Reply); seq != uint64(i) { t.Fatalf("Expected sequence of %d , got %d", i, seq) } } // Now check the subject to make sure its the original one. for i := 1; i <= toSend; i++ { m, err := sub.NextMsg(time.Second) checkMsg(m, err, i) } // Now do a pull based consumer. o, err = mset.addConsumer(workerModeConfig("p")) if err != nil { t.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() nextMsg := o.requestNextMsgSubject() for i := 1; i <= toSend; i++ { m, err := nc.Request(nextMsg, nil, time.Second) checkMsg(m, err, i) } } func clientConnectToServerWithUP(t *testing.T, opts *Options, user, pass string) *nats.Conn { curl := fmt.Sprintf("nats://%s:%s@%s:%d", user, pass, opts.Host, opts.Port) nc, err := nats.Connect(curl, nats.Name("JS-UP-TEST"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1)) if err != nil { t.Fatalf("Failed to create client: %v", err) } return nc } func TestJetStreamCanNotEnableOnSystemAccount(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sa := s.SystemAccount() if err := sa.EnableJetStream(nil); err == nil { t.Fatalf("Expected an error trying to enable on the system account") } } func TestJetStreamMultipleAccountsBasics(t *testing.T) { tdir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts: { A: { jetstream: enabled users: [ {user: ua, password: pwd} ] }, B: { jetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k} users: [ {user: ub, password: pwd} ] }, C: { users: [ {user: uc, password: pwd} ] }, } `, tdir))) s, opts := RunServerWithConfig(conf) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } nca := clientConnectToServerWithUP(t, opts, "ua", "pwd") defer nca.Close() ncb := clientConnectToServerWithUP(t, opts, "ub", "pwd") defer ncb.Close() resp, err := ncb.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } limits := info.Limits if limits.MaxStreams != 10 { t.Fatalf("Expected 10 for MaxStreams, got %d", limits.MaxStreams) } if limits.MaxConsumers != 1000 { t.Fatalf("Expected MaxConsumers of %d, got %d", 1000, limits.MaxConsumers) } gb := int64(1024 * 1024 * 1024) if limits.MaxMemory != gb { t.Fatalf("Expected MaxMemory to be 1GB, got %d", limits.MaxMemory) } if limits.MaxStore != 1024*gb { t.Fatalf("Expected MaxStore to be 1TB, got %d", limits.MaxStore) } ncc := clientConnectToServerWithUP(t, opts, "uc", "pwd") defer ncc.Close() expectNotEnabled := func(resp *nats.Msg, err error) { t.Helper() if err != nil { t.Fatalf("Unexpected error requesting enabled status: %v", err) } if resp == nil { t.Fatalf("No response, possible timeout?") } var iResp JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &iResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if iResp.Error == nil { t.Fatalf("Expected an error on not enabled account") } } // Check C is not enabled. We expect a negative response, not a timeout. expectNotEnabled(ncc.Request(JSApiAccountInfo, nil, 250*time.Millisecond)) // Now do simple reload and check that we do the right thing. Testing enable and disable and also change in limits newConf := []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts: { A: { jetstream: disabled users: [ {user: ua, password: pwd} ] }, B: { jetstream: {max_mem: 32GB, max_store: 512GB, max_streams: 100, max_consumers: 4k} users: [ {user: ub, password: pwd} ] }, C: { jetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k} users: [ {user: uc, password: pwd} ] }, } `, tdir)) if err := os.WriteFile(conf, newConf, 0600); err != nil { t.Fatalf("Error rewriting server's config file: %v", err) } if err := s.Reload(); err != nil { t.Fatalf("Error on server reload: %v", err) } expectNotEnabled(nca.Request(JSApiAccountInfo, nil, 250*time.Millisecond)) resp, _ = ncb.Request(JSApiAccountInfo, nil, 250*time.Millisecond) if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Error != nil { t.Fatalf("Expected JetStream to be enabled, got %+v", info.Error) } resp, _ = ncc.Request(JSApiAccountInfo, nil, 250*time.Millisecond) if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Error != nil { t.Fatalf("Expected JetStream to be enabled, got %+v", info.Error) } // Now check that limits have been updated. // Account B resp, err = ncb.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } limits = info.Limits if limits.MaxStreams != 100 { t.Fatalf("Expected 100 for MaxStreams, got %d", limits.MaxStreams) } if limits.MaxConsumers != 4000 { t.Fatalf("Expected MaxConsumers of %d, got %d", 4000, limits.MaxConsumers) } if limits.MaxMemory != 32*gb { t.Fatalf("Expected MaxMemory to be 32GB, got %d", limits.MaxMemory) } if limits.MaxStore != 512*gb { t.Fatalf("Expected MaxStore to be 512GB, got %d", limits.MaxStore) } // Account C resp, err = ncc.Request(JSApiAccountInfo, nil, time.Second) require_NoError(t, err) if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } limits = info.Limits if limits.MaxStreams != 10 { t.Fatalf("Expected 10 for MaxStreams, got %d", limits.MaxStreams) } if limits.MaxConsumers != 1000 { t.Fatalf("Expected MaxConsumers of %d, got %d", 1000, limits.MaxConsumers) } if limits.MaxMemory != gb { t.Fatalf("Expected MaxMemory to be 1GB, got %d", limits.MaxMemory) } if limits.MaxStore != 1024*gb { t.Fatalf("Expected MaxStore to be 1TB, got %d", limits.MaxStore) } } func TestJetStreamServerResourcesConfig(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 2GB, max_file_store: 1TB, store_dir: %q} `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } gb := int64(1024 * 1024 * 1024) jsc := s.JetStreamConfig() if jsc.MaxMemory != 2*gb { t.Fatalf("Expected MaxMemory to be %d, got %d", 2*gb, jsc.MaxMemory) } if jsc.MaxStore != 1024*gb { t.Fatalf("Expected MaxStore to be %d, got %d", 1024*gb, jsc.MaxStore) } } // From 2.2.2 to 2.2.3 we fixed a bug that would not consistently place a jetstream directory // under the store directory configured. However there were some cases where the directory was // created that way and therefore 2.2.3 would start and not recognize the existing accounts, // streams and consumers. func TestJetStreamStoreDirectoryFix(t *testing.T) { sd := filepath.Join(os.TempDir(), "sd_test") defer removeDir(t, sd) conf := createConfFile(t, []byte(fmt.Sprintf("listen: 127.0.0.1:-1\njetstream: {store_dir: %q}\n", sd))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.Publish("TEST", []byte("TSS")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Push based. sub, err := js.SubscribeSync("TEST", nats.Durable("dlc")) require_NoError(t, err) defer sub.Unsubscribe() // Now shutdown the server. nc.Close() s.Shutdown() // Now move stuff up from the jetstream directory etc. jssd := filepath.Join(sd, JetStreamStoreDir) fis, _ := os.ReadDir(jssd) // This will be accounts, move them up one directory. for _, fi := range fis { os.Rename(filepath.Join(jssd, fi.Name()), filepath.Join(sd, fi.Name())) } removeDir(t, jssd) // Restart our server. Make sure our assets got moved. s, _ = RunServerWithConfig(conf) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() var names []string for name := range js.StreamNames() { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 stream but got %d", len(names)) } names = names[:0] for name := range js.ConsumerNames("TEST") { names = append(names, name) } if len(names) != 1 { t.Fatalf("Expected only 1 consumer but got %d", len(names)) } } func TestJetStreamPushConsumersPullError(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.Publish("TEST", []byte("TSS")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } // Push based. sub, err := js.SubscribeSync("TEST") require_NoError(t, err) defer sub.Unsubscribe() ci, err := sub.ConsumerInfo() require_NoError(t, err) // Now do a pull. Make sure we get an error. m, err := nc.Request(fmt.Sprintf(JSApiRequestNextT, "TEST", ci.Name), nil, time.Second) require_NoError(t, err) if m.Header.Get("Status") != "409" { t.Fatalf("Expected a 409 status code, got %q", m.Header.Get("Status")) } } func TestJetStreamPullConsumerMaxWaitingOfOne(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"TEST.A"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", MaxWaiting: 1, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // First check that a request can timeout (we had an issue where this was // not the case for MaxWaiting of 1). req := JSApiConsumerGetNextRequest{Batch: 1, Expires: 250 * time.Millisecond} reqb, _ := json.Marshal(req) msg, err := nc.Request("$JS.API.CONSUMER.MSG.NEXT.TEST.dur", reqb, 13000*time.Millisecond) require_NoError(t, err) if v := msg.Header.Get("Status"); v != "408" { t.Fatalf("Expected 408, got: %s", v) } // Now have a request waiting... req = JSApiConsumerGetNextRequest{Batch: 1} reqb, _ = json.Marshal(req) // Send the request, but do not block since we want then to send an extra // request that should be rejected. sub := natsSubSync(t, nc, nats.NewInbox()) err = nc.PublishRequest("$JS.API.CONSUMER.MSG.NEXT.TEST.dur", sub.Subject, reqb) require_NoError(t, err) // Send a new request, this should be rejected as a 409. req = JSApiConsumerGetNextRequest{Batch: 1, Expires: 250 * time.Millisecond} reqb, _ = json.Marshal(req) msg, err = nc.Request("$JS.API.CONSUMER.MSG.NEXT.TEST.dur", reqb, 300*time.Millisecond) require_NoError(t, err) if v := msg.Header.Get("Status"); v != "409" { t.Fatalf("Expected 409, got: %s", v) } if v := msg.Header.Get("Description"); v != "Exceeded MaxWaiting" { t.Fatalf("Expected error about exceeded max waiting, got: %s", v) } } func TestJetStreamPullConsumerMaxWaiting(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"test.*"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, MaxWaiting: 10, }) require_NoError(t, err) // Cannot be updated. _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, MaxWaiting: 1, }) if !strings.Contains(err.Error(), "can not be updated") { t.Fatalf(`expected "cannot be updated" error, got %s`, err) } } func TestJetStreamChangeConsumerType(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"test.*"}}) require_NoError(t, err) // create pull consumer _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "pull", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // cannot update pull -> push _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Name: "pull", AckPolicy: nats.AckExplicitPolicy, DeliverSubject: "foo", }) require_Contains(t, err.Error(), "can not update pull consumer to push based") // create push consumer _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "push", AckPolicy: nats.AckExplicitPolicy, DeliverSubject: "foo", }) require_NoError(t, err) // cannot change push -> pull _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Name: "push", AckPolicy: nats.AckExplicitPolicy, }) require_Contains(t, err.Error(), "can not update push consumer to pull based") } //////////////////////////////////////// // Benchmark placeholders // TODO(dlc) - move //////////////////////////////////////// func TestJetStreamPubPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() msetConfig := StreamConfig{ Name: "sr22", Storage: FileStorage, Subjects: []string{"foo"}, } if _, err := acc.addStream(&msetConfig); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() toSend := 5_000_000 numProducers := 5 payload := []byte("Hello World") startCh := make(chan bool) var wg sync.WaitGroup for n := 0; n < numProducers; n++ { wg.Add(1) go func() { defer wg.Done() <-startCh for i := 0; i < int(toSend)/numProducers; i++ { nc.Publish("foo", payload) } nc.Flush() }() } // Wait for Go routines. time.Sleep(20 * time.Millisecond) start := time.Now() close(startCh) wg.Wait() tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. start = time.Now() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() fmt.Printf("Took %v to restart!\n", time.Since(start)) } func TestJetStreamPubWithAsyncResponsePerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() msetConfig := StreamConfig{ Name: "sr33", Storage: FileStorage, Subjects: []string{"foo"}, } if _, err := acc.addStream(&msetConfig); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() toSend := 1_000_000 payload := []byte("Hello World") start := time.Now() for i := 0; i < toSend; i++ { nc.PublishRequest("foo", "bar", payload) } nc.Flush() tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } func TestJetStreamPubWithSyncPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) require_NoError(t, err) toSend := 1_000_000 payload := []byte("Hello World") start := time.Now() for i := 0; i < toSend; i++ { js.Publish("foo", payload) } tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } func TestJetStreamConsumerPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() msetConfig := StreamConfig{ Name: "sr22", Storage: MemoryStorage, Subjects: []string{"foo"}, } mset, err := acc.addStream(&msetConfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() payload := []byte("Hello World") toStore := 2000000 for i := 0; i < toStore; i++ { nc.Publish("foo", payload) } nc.Flush() _, err = mset.addConsumer(&ConsumerConfig{ Durable: "d", DeliverSubject: "d", AckPolicy: AckNone, }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } var received int done := make(chan bool) nc.Subscribe("d", func(m *nats.Msg) { received++ if received >= toStore { done <- true } }) start := time.Now() nc.Flush() <-done tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) } func TestJetStreamConsumerAckFileStorePerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() msetConfig := StreamConfig{ Name: "sr22", Storage: FileStorage, Subjects: []string{"foo"}, } mset, err := acc.addStream(&msetConfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() payload := []byte("Hello World") toStore := uint64(200000) for i := uint64(0); i < toStore; i++ { nc.Publish("foo", payload) } nc.Flush() if msgs := mset.state().Msgs; msgs != uint64(toStore) { t.Fatalf("Expected %d messages, got %d", toStore, msgs) } o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d", DeliverSubject: "d", AckPolicy: AckExplicit, AckWait: 10 * time.Minute, }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } defer o.stop() var received uint64 done := make(chan bool) sub, _ := nc.Subscribe("d", func(m *nats.Msg) { m.Respond(nil) // Ack received++ if received >= toStore { done <- true } }) sub.SetPendingLimits(-1, -1) start := time.Now() nc.Flush() <-done tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toStore)/tt.Seconds()) } func TestJetStreamPubSubPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() msetConfig := StreamConfig{ Name: "MSET22", Storage: FileStorage, Subjects: []string{"foo"}, } mset, err := acc.addStream(&msetConfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() var toSend = 1_000_000 var received int done := make(chan bool) delivery := "d" nc.Subscribe(delivery, func(m *nats.Msg) { received++ if received >= toSend { done <- true } }) nc.Flush() _, err = mset.addConsumer(&ConsumerConfig{ DeliverSubject: delivery, AckPolicy: AckNone, }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } payload := []byte("Hello World") start := time.Now() for i := 0; i < toSend; i++ { nc.Publish("foo", payload) } <-done tt := time.Since(start) fmt.Printf("time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } func TestJetStreamAckExplicitMsgRemoval(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{ Name: "MY_STREAM", Storage: MemoryStorage, Subjects: []string{"foo.*"}, Retention: InterestPolicy, }}, {"FileStore", &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.*"}, Retention: InterestPolicy, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc1 := clientConnectToServer(t, s) defer nc1.Close() nc2 := clientConnectToServer(t, s) defer nc2.Close() // Create two durable consumers on the same subject sub1, _ := nc1.SubscribeSync(nats.NewInbox()) defer sub1.Unsubscribe() nc1.Flush() o1, err := mset.addConsumer(&ConsumerConfig{ Durable: "dur1", DeliverSubject: sub1.Subject, FilterSubject: "foo.bar", AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o1.delete() sub2, _ := nc2.SubscribeSync(nats.NewInbox()) defer sub2.Unsubscribe() nc2.Flush() o2, err := mset.addConsumer(&ConsumerConfig{ Durable: "dur2", DeliverSubject: sub2.Subject, FilterSubject: "foo.bar", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o2.delete() // Send 2 messages toSend := 2 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc1, "foo.bar", fmt.Sprintf("msg%v", i+1)) } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %v messages, got %d", toSend, state.Msgs) } // Receive the messages and ack them. subs := []*nats.Subscription{sub1, sub2} for _, sub := range subs { for i := 0; i < toSend; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error acking message: %v", err) } m.Respond(nil) } } // To make sure acks are processed for checking state after sending new ones. checkFor(t, time.Second, 25*time.Millisecond, func() error { if state = mset.state(); state.Msgs != 0 { return fmt.Errorf("Stream still has messages") } return nil }) // Now close the 2nd subscription... sub2.Unsubscribe() nc2.Flush() // Send 2 more new messages for i := 0; i < toSend; i++ { sendStreamMsg(t, nc1, "foo.bar", fmt.Sprintf("msg%v", 2+i+1)) } state = mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %v messages, got %d", toSend, state.Msgs) } // first subscription should get it and will ack it. for i := 0; i < toSend; i++ { m, err := sub1.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message to ack: %v", err) } m.Respond(nil) } // For acks from m.Respond above nc1.Flush() // Now recreate the subscription for the 2nd JS consumer sub2, _ = nc2.SubscribeSync(nats.NewInbox()) defer sub2.Unsubscribe() o2, err = mset.addConsumer(&ConsumerConfig{ Durable: "dur2", DeliverSubject: sub2.Subject, FilterSubject: "foo.bar", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o2.delete() // Those messages should be redelivered to the 2nd consumer for i := 1; i <= toSend; i++ { m, err := sub2.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving message %d: %v", i, err) } m.Respond(nil) sseq := o2.streamSeqFromReply(m.Reply) // Depending on timing from above we could receive stream sequences out of order but // we know we want 3 & 4. if sseq != 3 && sseq != 4 { t.Fatalf("Expected stream sequence of 3 or 4 but got %d", sseq) } } }) } } // This test is in support fo clients that want to match on subject, they // can set the filter subject always. We always store the subject so that // should the stream later be edited to expand into more subjects the consumer // still gets what was actually requested func TestJetStreamConsumerFilterSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sc := &StreamConfig{Name: "MY_STREAM", Subjects: []string{"foo"}} mset, err := s.GlobalAccount().addStream(sc) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() cfg := &ConsumerConfig{ Durable: "d", DeliverSubject: "A", AckPolicy: AckExplicit, FilterSubject: "foo", } o, err := mset.addConsumer(cfg) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() if o.info().Config.FilterSubject != "foo" { t.Fatalf("Expected the filter to be stored") } // Now use the original cfg with updated delivery subject and make sure that works ok. cfg = &ConsumerConfig{ Durable: "d", DeliverSubject: "B", AckPolicy: AckExplicit, FilterSubject: "foo", } o, err = mset.addConsumer(cfg) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() } func TestJetStreamStoredMsgsDontDisappearAfterCacheExpiration(t *testing.T) { sc := &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.>"}, Retention: InterestPolicy, } s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStreamWithStore(sc, &FileStoreConfig{BlockSize: 128, CacheExpire: 15 * time.Millisecond}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc1 := clientConnectWithOldRequest(t, s) defer nc1.Close() // Create a durable consumers sub, _ := nc1.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc1.Flush() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "dur", DeliverSubject: sub.Subject, FilterSubject: "foo.bar", DeliverPolicy: DeliverNew, AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() nc2 := clientConnectWithOldRequest(t, s) defer nc2.Close() sendStreamMsg(t, nc2, "foo.bar", "msg1") msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Did not get message: %v", err) } if string(msg.Data) != "msg1" { t.Fatalf("Unexpected message: %q", msg.Data) } nc1.Close() // Get the message from the stream getMsgSeq := func(seq uint64) { t.Helper() mreq := &JSApiMsgGetRequest{Seq: seq} req, err := json.Marshal(mreq) if err != nil { t.Fatalf("Unexpected error: %v", err) } smsgj, err := nc2.Request(fmt.Sprintf(JSApiMsgGetT, sc.Name), req, time.Second) if err != nil { t.Fatalf("Could not retrieve stream message: %v", err) } if strings.Contains(string(smsgj.Data), "code") { t.Fatalf("Error: %q", smsgj.Data) } } getMsgSeq(1) time.Sleep(time.Second) sendStreamMsg(t, nc2, "foo.bar", "msg2") sendStreamMsg(t, nc2, "foo.bar", "msg3") getMsgSeq(1) getMsgSeq(2) getMsgSeq(3) } func TestJetStreamConsumerUpdateRedelivery(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{ Name: "MY_STREAM", Storage: MemoryStorage, Subjects: []string{"foo.>"}, Retention: InterestPolicy, }}, {"FileStore", &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.>"}, Retention: InterestPolicy, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Create a durable consumer. sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "dur22", DeliverSubject: sub.Subject, FilterSubject: "foo.bar", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, MaxDeliver: 3, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() // Send 20 messages toSend := 20 for i := 1; i <= toSend; i++ { sendStreamMsg(t, nc, "foo.bar", fmt.Sprintf("msg-%v", i)) } state := mset.state() if state.Msgs != uint64(toSend) { t.Fatalf("Expected %v messages, got %d", toSend, state.Msgs) } // Receive the messages and ack only every 4th for i := 0; i < toSend; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } seq, _, _, _, _ := replyInfo(m.Reply) // 4, 8, 12, 16, 20 if seq%4 == 0 { m.Respond(nil) } } // Now close the sub and open a new one and update the consumer. sub.Unsubscribe() // Wait for it to become inactive checkFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error { if o.isActive() { return fmt.Errorf("Consumer still active") } return nil }) // Send 20 more messages. for i := toSend; i < toSend*2; i++ { sendStreamMsg(t, nc, "foo.bar", fmt.Sprintf("msg-%v", i)) } // Create new subscription. sub, _ = nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err = mset.addConsumer(&ConsumerConfig{ Durable: "dur22", DeliverSubject: sub.Subject, FilterSubject: "foo.bar", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond, MaxDeliver: 3, }) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } defer o.delete() expect := toSend + toSend - 5 // mod 4 acks checkFor(t, time.Second, 5*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expect { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, expect) } return nil }) for i, eseq := 0, uint64(1); i < expect; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } // Skip the ones we ack'd from above. We should not get them back here. if eseq <= uint64(toSend) && eseq%4 == 0 { eseq++ } seq, _, dc, _, _ := replyInfo(m.Reply) if seq != eseq { t.Fatalf("Expected stream sequence of %d, got %d", eseq, seq) } if seq <= uint64(toSend) && dc != 2 { t.Fatalf("Expected delivery count of 2 for sequence of %d, got %d", seq, dc) } if seq > uint64(toSend) && dc != 1 { t.Fatalf("Expected delivery count of 1 for sequence of %d, got %d", seq, dc) } if seq > uint64(toSend) { m.Respond(nil) // Ack } eseq++ } // We should get the second half back since we did not ack those from above. expect = toSend - 5 checkFor(t, time.Second, 5*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expect { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, expect) } return nil }) for i, eseq := 0, uint64(1); i < expect; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } // Skip the ones we ack'd from above. We should not get them back here. if eseq <= uint64(toSend) && eseq%4 == 0 { eseq++ } seq, _, dc, _, _ := replyInfo(m.Reply) if seq != eseq { t.Fatalf("Expected stream sequence of %d, got %d", eseq, seq) } if dc != 3 { t.Fatalf("Expected delivery count of 3 for sequence of %d, got %d", seq, dc) } eseq++ } }) } } func TestJetStreamConsumerMaxAckPending(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{ Name: "MY_STREAM", Storage: MemoryStorage, Subjects: []string{"foo.*"}, }}, {"FileStore", &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.*"}, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Do error scenarios. _, err = mset.addConsumer(&ConsumerConfig{ Durable: "d22", DeliverSubject: nats.NewInbox(), AckPolicy: AckNone, MaxAckPending: 1, }) if err == nil { t.Fatalf("Expected error, MaxAckPending only applicable to ack != AckNone") } // Queue up 100 messages. toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.bar", fmt.Sprintf("MSG: %d", i+1)) } // Limit to 33 maxAckPending := 33 o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d22", DeliverSubject: nats.NewInbox(), AckPolicy: AckExplicit, MaxAckPending: maxAckPending, }) require_NoError(t, err) defer o.delete() sub, _ := nc.SubscribeSync(o.info().Config.DeliverSubject) defer sub.Unsubscribe() checkSubPending := func(numExpected int) { t.Helper() checkFor(t, time.Second, 20*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } checkSubPending(maxAckPending) // We hit the limit, double check we stayed there. if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending { t.Fatalf("Too many messages received: %d vs %d", nmsgs, maxAckPending) } // Now ack them all. for i := 0; i < maxAckPending; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving message %d: %v", i, err) } m.Respond(nil) } checkSubPending(maxAckPending) o.stop() mset.purge(nil) // Now test a consumer that is live while we publish messages to the stream. o, err = mset.addConsumer(&ConsumerConfig{ Durable: "d33", DeliverSubject: nats.NewInbox(), AckPolicy: AckExplicit, MaxAckPending: maxAckPending, }) require_NoError(t, err) defer o.delete() sub, _ = nc.SubscribeSync(o.info().Config.DeliverSubject) defer sub.Unsubscribe() nc.Flush() checkSubPending(0) // Now stream more then maxAckPending. for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.baz", fmt.Sprintf("MSG: %d", i+1)) } checkSubPending(maxAckPending) // We hit the limit, double check we stayed there. if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending { t.Fatalf("Too many messages received: %d vs %d", nmsgs, maxAckPending) } }) } } func TestJetStreamPullConsumerMaxAckPending(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{ Name: "MY_STREAM", Storage: MemoryStorage, Subjects: []string{"foo.*"}, }}, {"FileStore", &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.*"}, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Queue up 100 messages. toSend := 100 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.bar", fmt.Sprintf("MSG: %d", i+1)) } // Limit to 33 maxAckPending := 33 o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d22", AckPolicy: AckExplicit, MaxAckPending: maxAckPending, }) require_NoError(t, err) defer o.delete() getSubj := o.requestNextMsgSubject() var toAck []*nats.Msg for i := 0; i < maxAckPending; i++ { if m, err := nc.Request(getSubj, nil, time.Second); err != nil { t.Fatalf("Unexpected error: %v", err) } else { toAck = append(toAck, m) } } // Now ack them all. for _, m := range toAck { m.Respond(nil) } // Now do batch above the max. sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() checkSubPending := func(numExpected int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) } return nil }) } req := &JSApiConsumerGetNextRequest{Batch: maxAckPending} jreq, _ := json.Marshal(req) nc.PublishRequest(getSubj, sub.Subject, jreq) checkSubPending(maxAckPending) // We hit the limit, double check we stayed there. if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending { t.Fatalf("Too many messages received: %d vs %d", nmsgs, maxAckPending) } }) } } func TestJetStreamPullConsumerMaxAckPendingRedeliveries(t *testing.T) { cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &StreamConfig{ Name: "MY_STREAM", Storage: MemoryStorage, Subjects: []string{"foo.*"}, }}, {"FileStore", &StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.*"}, }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() // Queue up 10 messages. toSend := 10 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "foo.bar", fmt.Sprintf("MSG: %d", i+1)) } // Limit to 1 maxAckPending := 1 ackWait := 20 * time.Millisecond expSeq := uint64(4) o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d22", DeliverPolicy: DeliverByStartSequence, OptStartSeq: expSeq, AckPolicy: AckExplicit, AckWait: ackWait, MaxAckPending: maxAckPending, }) require_NoError(t, err) defer o.delete() getSubj := o.requestNextMsgSubject() delivery := uint64(1) getNext := func() { t.Helper() m, err := nc.Request(getSubj, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } sseq, dseq, dcount, _, pending := replyInfo(m.Reply) if sseq != expSeq { t.Fatalf("Expected stream sequence of %d, got %d", expSeq, sseq) } if dseq != delivery { t.Fatalf("Expected consumer sequence of %d, got %d", delivery, dseq) } if dcount != delivery { t.Fatalf("Expected delivery count of %d, got %d", delivery, dcount) } if pending != uint64(toSend)-expSeq { t.Fatalf("Expected pending to be %d, got %d", uint64(toSend)-expSeq, pending) } delivery++ } getNext() getNext() getNext() getNext() getNext() }) } } func TestJetStreamDeliveryAfterServerRestart(t *testing.T) { opts := DefaultTestOptions opts.Port = -1 opts.JetStream = true opts.StoreDir = t.TempDir() s := RunServer(&opts) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{ Name: "MY_STREAM", Storage: FileStorage, Subjects: []string{"foo.>"}, Retention: InterestPolicy, }) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc := clientConnectToServer(t, s) defer nc.Close() inbox := nats.NewInbox() o, err := mset.addConsumer(&ConsumerConfig{ Durable: "dur", DeliverSubject: inbox, DeliverPolicy: DeliverNew, AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer o.delete() sub, err := nc.SubscribeSync(inbox) if err != nil { t.Fatalf("Error on subscribe: %v", err) } nc.Flush() // Send 1 message sendStreamMsg(t, nc, "foo.bar", "msg1") // Make sure we receive it and ack it. msg, err := sub.NextMsg(250 * time.Millisecond) if err != nil { t.Fatalf("Did not get message: %v", err) } // Ack it! msg.Respond(nil) nc.Flush() // Shutdown client and server nc.Close() dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) s.Shutdown() opts.Port = -1 opts.StoreDir = dir s = RunServer(&opts) defer s.Shutdown() // Lookup stream. mset, err = s.GlobalAccount().lookupStream("MY_STREAM") if err != nil { t.Fatalf("Error looking up stream: %v", err) } // Update consumer's deliver subject with new inbox inbox = nats.NewInbox() o, err = mset.addConsumer(&ConsumerConfig{ Durable: "dur", DeliverSubject: inbox, DeliverPolicy: DeliverNew, AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer o.delete() nc = clientConnectToServer(t, s) defer nc.Close() // Send 2nd message sendStreamMsg(t, nc, "foo.bar", "msg2") // Start sub on new inbox sub, err = nc.SubscribeSync(inbox) if err != nil { t.Fatalf("Error on subscribe: %v", err) } nc.Flush() // Should receive message 2. if _, err := sub.NextMsg(500 * time.Millisecond); err != nil { t.Fatalf("Did not get message: %v", err) } } // This is for the basics of importing the ability to send to a stream and consume // from a consumer that is pull based on push based on a well known delivery subject. func TestJetStreamAccountImportBasics(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 no_auth_user: rip jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts: { JS: { jetstream: enabled users: [ {user: dlc, password: foo} ] exports [ # This is for sending into a stream from other accounts. { service: "ORDERS.*" } # This is for accessing a pull based consumer. { service: "$JS.API.CONSUMER.MSG.NEXT.*.*" } # This is streaming to a delivery subject for a push based consumer. { stream: "deliver.ORDERS" } # This is to ack received messages. This is a service to ack acks.. { service: "$JS.ACK.ORDERS.*.>" } ] }, IU: { users: [ {user: rip, password: bar} ] imports [ { service: { subject: "ORDERS.*", account: JS }, to: "my.orders.$1" } { service: { subject: "$JS.API.CONSUMER.MSG.NEXT.ORDERS.d", account: JS }, to: "nxt.msg" } { stream: { subject: "deliver.ORDERS", account: JS }, to: "d" } { service: { subject: "$JS.ACK.ORDERS.*.>", account: JS } } ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() acc, err := s.LookupAccount("JS") if err != nil { t.Fatalf("Unexpected error looking up account: %v", err) } mset, err := acc.addStream(&StreamConfig{Name: "ORDERS", Subjects: []string{"ORDERS.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // This should be the rip user, the one that imports some JS. nc := clientConnectToServer(t, s) defer nc.Close() // Simple publish to a stream. pubAck := sendStreamMsg(t, nc, "my.orders.foo", "ORDERS-1") if pubAck.Stream != "ORDERS" || pubAck.Sequence != 1 { t.Fatalf("Bad pubAck received: %+v", pubAck) } if msgs := mset.state().Msgs; msgs != 1 { t.Fatalf("Expected 1 message, got %d", msgs) } total := 2 for i := 2; i <= total; i++ { sendStreamMsg(t, nc, "my.orders.bar", fmt.Sprintf("ORDERS-%d", i)) } if msgs := mset.state().Msgs; msgs != uint64(total) { t.Fatalf("Expected %d messages, got %d", total, msgs) } // Now test access to a pull based consumer, e.g. workqueue. o, err := mset.addConsumer(&ConsumerConfig{ Durable: "d", AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer o.delete() // We mapped the next message request, "$JS.API.CONSUMER.MSG.NEXT.ORDERS.d" -> "nxt.msg" m, err := nc.Request("nxt.msg", nil, time.Second) require_NoError(t, err) if string(m.Data) != "ORDERS-1" { t.Fatalf("Expected to receive %q, got %q", "ORDERS-1", m.Data) } // Now test access to a push based consumer o, err = mset.addConsumer(&ConsumerConfig{ Durable: "p", DeliverSubject: "deliver.ORDERS", AckPolicy: AckExplicit, }) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer o.delete() // We remapped from above, deliver.ORDERS -> d sub, _ := nc.SubscribeSync("d") defer sub.Unsubscribe() checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != total { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, total) } return nil }) m, _ = sub.NextMsg(time.Second) // Make sure we remapped subject correctly across the account boundary. if m.Subject != "ORDERS.foo" { t.Fatalf("Expected subject of %q, got %q", "ORDERS.foo", m.Subject) } // Now make sure we can ack messages correctly. m.Respond(AckAck) nc.Flush() if info := o.info(); info.AckFloor.Consumer != 1 { t.Fatalf("Did not receive the ack properly") } // Grab second one now. m, _ = sub.NextMsg(time.Second) // Make sure we remapped subject correctly across the account boundary. if m.Subject != "ORDERS.bar" { t.Fatalf("Expected subject of %q, got %q", "ORDERS.bar", m.Subject) } // Now make sure we can ack messages and get back an ack as well. resp, _ := nc.Request(m.Reply, nil, 100*time.Millisecond) if resp == nil { t.Fatalf("No response, possible timeout?") } if info := o.info(); info.AckFloor.Consumer != 2 { t.Fatalf("Did not receive the ack properly") } } // This tests whether we are able to aggregate all JetStream advisory events // from all accounts into a single account. Config for this test uses // service imports and exports as that allows for gathering all events // without having to know the account name and without separate entries // for each account in aggregate account config. // This test fails as it is not receiving the api audit event ($JS.EVENT.ADVISORY.API). func TestJetStreamAccountImportJSAdvisoriesAsService(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen=127.0.0.1:-1 no_auth_user: pp jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts { JS { jetstream: enabled users: [ {user: pp, password: foo} ] imports [ { service: { account: AGG, subject: '$JS.EVENT.ADVISORY.ACC.JS.>' }, to: '$JS.EVENT.ADVISORY.>' } ] } AGG { users: [ {user: agg, password: foo} ] exports: [ { service: '$JS.EVENT.ADVISORY.ACC.*.>', response: Singleton, account_token_position: 5 } ] } } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // This should be the pp user, one which manages JetStream assets ncJS, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Unexpected error during connect: %v", err) } defer ncJS.Close() // This is the agg user, which should aggregate all JS advisory events. ncAgg, err := nats.Connect(s.ClientURL(), nats.UserInfo("agg", "foo")) if err != nil { t.Fatalf("Unexpected error during connect: %v", err) } defer ncAgg.Close() js, err := ncJS.JetStream() if err != nil { t.Fatalf("Unexpected error: %v", err) } // user from JS account should receive events on $JS.EVENT.ADVISORY.> subject subJS, err := ncJS.SubscribeSync("$JS.EVENT.ADVISORY.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer subJS.Unsubscribe() // user from AGG account should receive events on mapped $JS.EVENT.ADVISORY.ACC.JS.> subject (with account name) subAgg, err := ncAgg.SubscribeSync("$JS.EVENT.ADVISORY.ACC.JS.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } // add stream using JS account // this should trigger 2 events: // - an action event on $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS // - an api audit event on $JS.EVENT.ADVISORY.API _, err = js.AddStream(&nats.StreamConfig{Name: "ORDERS", Subjects: []string{"ORDERS.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } gotEvents := map[string]int{} for i := 0; i < 2; i++ { msg, err := subJS.NextMsg(time.Second * 2) if err != nil { t.Fatalf("Unexpected error: %v", err) } gotEvents[msg.Subject]++ } if c := gotEvents["$JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS"]; c != 1 { t.Fatalf("Should have received one advisory from $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS but got %d", c) } if c := gotEvents["$JS.EVENT.ADVISORY.API"]; c != 1 { t.Fatalf("Should have received one advisory from $JS.EVENT.ADVISORY.API but got %d", c) } // same set of events should be received by AGG account // on subjects containing account name (ACC.JS) gotEvents = map[string]int{} for i := 0; i < 2; i++ { msg, err := subAgg.NextMsg(time.Second * 2) require_NoError(t, err) var adv JSAPIAudit require_NoError(t, json.Unmarshal(msg.Data, &adv)) // Make sure we have full fidelity info via implicit share. if adv.Client != nil { require_True(t, adv.Client.Host != _EMPTY_) require_True(t, adv.Client.User != _EMPTY_) require_True(t, adv.Client.Lang != _EMPTY_) } gotEvents[msg.Subject]++ } if c := gotEvents["$JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS"]; c != 1 { t.Fatalf("Should have received one advisory from $JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS but got %d", c) } if c := gotEvents["$JS.EVENT.ADVISORY.ACC.JS.API"]; c != 1 { t.Fatalf("Should have received one advisory from $JS.EVENT.ADVISORY.ACC.JS.API but got %d", c) } } // This tests whether we are able to aggregate all JetStream advisory events // from all accounts into a single account. Config for this test uses // stream imports and exports as that allows for gathering all events // as long as there is a separate stream import entry for each account // in aggregate account config. func TestJetStreamAccountImportJSAdvisoriesAsStream(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen=127.0.0.1:-1 no_auth_user: pp jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts { JS { jetstream: enabled users: [ {user: pp, password: foo} ] exports [ { stream: '$JS.EVENT.ADVISORY.>' } ] } AGG { users: [ {user: agg, password: foo} ] imports: [ { stream: { account: JS, subject: '$JS.EVENT.ADVISORY.>' }, to: '$JS.EVENT.ADVISORY.ACC.JS.>' } ] } } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // This should be the pp user, one which manages JetStream assets ncJS, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Unexpected error during connect: %v", err) } defer ncJS.Close() // This is the agg user, which should aggregate all JS advisory events. ncAgg, err := nats.Connect(s.ClientURL(), nats.UserInfo("agg", "foo")) if err != nil { t.Fatalf("Unexpected error during connect: %v", err) } defer ncAgg.Close() js, err := ncJS.JetStream() if err != nil { t.Fatalf("Unexpected error: %v", err) } // user from JS account should receive events on $JS.EVENT.ADVISORY.> subject subJS, err := ncJS.SubscribeSync("$JS.EVENT.ADVISORY.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer subJS.Unsubscribe() // user from AGG account should receive events on mapped $JS.EVENT.ADVISORY.ACC.JS.> subject (with account name) subAgg, err := ncAgg.SubscribeSync("$JS.EVENT.ADVISORY.ACC.JS.>") if err != nil { t.Fatalf("Unexpected error: %v", err) } // add stream using JS account // this should trigger 2 events: // - an action event on $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS // - an api audit event on $JS.EVENT.ADVISORY.API _, err = js.AddStream(&nats.StreamConfig{Name: "ORDERS", Subjects: []string{"ORDERS.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } var gotAPIAdvisory, gotCreateAdvisory bool for i := 0; i < 2; i++ { msg, err := subJS.NextMsg(time.Second * 2) if err != nil { t.Fatalf("Unexpected error on JS account: %v", err) } switch msg.Subject { case "$JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS": gotCreateAdvisory = true case "$JS.EVENT.ADVISORY.API": gotAPIAdvisory = true default: t.Fatalf("Unexpected subject: %q", msg.Subject) } } if !gotAPIAdvisory || !gotCreateAdvisory { t.Fatalf("Expected to have received both advisories on JS account (API advisory %v, create advisory %v)", gotAPIAdvisory, gotCreateAdvisory) } // same set of events should be received by AGG account // on subjects containing account name (ACC.JS) gotAPIAdvisory, gotCreateAdvisory = false, false for i := 0; i < 2; i++ { msg, err := subAgg.NextMsg(time.Second * 2) if err != nil { t.Fatalf("Unexpected error on AGG account: %v", err) } switch msg.Subject { case "$JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS": gotCreateAdvisory = true case "$JS.EVENT.ADVISORY.ACC.JS.API": gotAPIAdvisory = true default: t.Fatalf("Unexpected subject: %q", msg.Subject) } } if !gotAPIAdvisory || !gotCreateAdvisory { t.Fatalf("Expected to have received both advisories on AGG account (API advisory %v, create advisory %v)", gotAPIAdvisory, gotCreateAdvisory) } } // This is for importing all of JetStream into another account for admin purposes. func TestJetStreamAccountImportAll(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 no_auth_user: rip jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts: { JS: { jetstream: enabled users: [ {user: dlc, password: foo} ] exports [ { service: "$JS.API.>" } ] }, IU: { users: [ {user: rip, password: bar} ] imports [ { service: { subject: "$JS.API.>", account: JS }, to: "jsapi.>"} ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() acc, err := s.LookupAccount("JS") if err != nil { t.Fatalf("Unexpected error looking up account: %v", err) } mset, err := acc.addStream(&StreamConfig{Name: "ORDERS", Subjects: []string{"ORDERS.*"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // This should be the rip user, the one that imports all of JS. nc := clientConnectToServer(t, s) defer nc.Close() mapSubj := func(subject string) string { return strings.Replace(subject, "$JS.API.", "jsapi.", 1) } // This will get the current information about usage and limits for this account. resp, err := nc.Request(mapSubj(JSApiAccountInfo), nil, time.Second) require_NoError(t, err) var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } if info.Error != nil { t.Fatalf("Unexpected error: %+v", info.Error) } // Lookup streams. resp, err = nc.Request(mapSubj(JSApiStreams), nil, time.Second) require_NoError(t, err) var namesResponse JSApiStreamNamesResponse if err = json.Unmarshal(resp.Data, &namesResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if namesResponse.Error != nil { t.Fatalf("Unexpected error: %+v", namesResponse.Error) } } // https://github.com/nats-io/nats-server/issues/1736 func TestJetStreamServerReload(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q } accounts: { A: { users: [ {user: ua, password: pwd} ] }, B: { jetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k} users: [ {user: ub, password: pwd} ] }, SYS: { users: [ {user: uc, password: pwd} ] }, } no_auth_user: ub system_account: SYS `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() checkJSAccount := func() { t.Helper() resp, err := nc.Request(JSApiAccountInfo, nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var info JSApiAccountInfoResponse if err := json.Unmarshal(resp.Data, &info); err != nil { t.Fatalf("Unexpected error: %v", err) } } checkJSAccount() acc, err := s.LookupAccount("B") if err != nil { t.Fatalf("Unexpected error looking up account: %v", err) } mset, err := acc.addStream(&StreamConfig{Name: "22"}) require_NoError(t, err) toSend := 10 for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, "22", fmt.Sprintf("MSG: %d", i+1)) } if msgs := mset.state().Msgs; msgs != uint64(toSend) { t.Fatalf("Expected %d messages, got %d", toSend, msgs) } if err := s.Reload(); err != nil { t.Fatalf("Error on server reload: %v", err) } // Wait to get reconnected. checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { if !nc.IsConnected() { return fmt.Errorf("Not connected") } return nil }) checkJSAccount() sendStreamMsg(t, nc, "22", "MSG: 22") } func TestJetStreamConfigReloadWithGlobalAccount(t *testing.T) { tdir := t.TempDir() template := ` listen: 127.0.0.1:-1 authorization { users [ {user: anonymous} {user: user1, password: %s} ] } no_auth_user: anonymous jetstream { store_dir = %q } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, "pwd", tdir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() checkJSAccount := func() { t.Helper() if _, err := js.AccountInfo(); err != nil { t.Fatalf("Unexpected error: %v", err) } } checkJSAccount() if _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10 for i := 0; i < toSend; i++ { if _, err := js.Publish("foo", []byte(fmt.Sprintf("MSG: %d", i+1))); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } si, err := js.StreamInfo("foo") require_NoError(t, err) if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs after restart, got %d", toSend, si.State.Msgs) } if err := os.WriteFile(conf, []byte(fmt.Sprintf(template, "pwd2", tdir)), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatalf("Error during config reload: %v", err) } nc, js = jsClientConnect(t, s) defer nc.Close() // Try to add a new stream to the global account if _, err := js.AddStream(&nats.StreamConfig{Name: "bar"}); err != nil { t.Fatalf("Unexpected error: %v", err) } checkJSAccount() } // Test that we properly enforce per subject msg limits. func TestJetStreamMaxMsgsPerSubject(t *testing.T) { const maxPer = 5 msc := StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz.*"}, Storage: MemoryStorage, MaxMsgsPer: maxPer, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() pubAndCheck := func(subj string, num int, expectedNumMsgs uint64) { t.Helper() for i := 0; i < num; i++ { if _, err = js.Publish(subj, []byte("TSLA")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != expectedNumMsgs { t.Fatalf("Expected %d msgs, got %d", expectedNumMsgs, si.State.Msgs) } } pubAndCheck("foo", 1, 1) pubAndCheck("foo", 4, 5) // Now make sure our per subject limits kick in.. pubAndCheck("foo", 2, 5) pubAndCheck("baz.22", 5, 10) pubAndCheck("baz.33", 5, 15) // We are maxed so totals should be same no matter what we add here. pubAndCheck("baz.22", 5, 15) pubAndCheck("baz.33", 5, 15) // Now purge and make sure all is still good. mset.purge(nil) pubAndCheck("foo", 1, 1) pubAndCheck("foo", 4, 5) pubAndCheck("baz.22", 5, 10) pubAndCheck("baz.33", 5, 15) }) } } func TestJetStreamGetLastMsgBySubject(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: st, Replicas: 2, MaxMsgsPer: 20, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) si, err := js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "KV" { t.Fatalf("StreamInfo is not correct %+v", si) } for i := 0; i < 1000; i++ { msg := []byte(fmt.Sprintf("VAL-%d", i+1)) js.PublishAsync("kv.foo", msg) js.PublishAsync("kv.bar", msg) js.PublishAsync("kv.baz", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Check that if both set that errors. b, _ := json.Marshal(JSApiMsgGetRequest{LastFor: "kv.foo", Seq: 950}) rmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, "KV"), b, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(rmsg.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Error == nil { t.Fatalf("Expected an error when both are set, got %+v", resp.Error) } // Need to do stream GetMsg by hand for now until Go client support lands. getLast := func(subject string) *StoredMsg { t.Helper() req := &JSApiMsgGetRequest{LastFor: subject} b, _ := json.Marshal(req) rmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, "KV"), b, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(rmsg.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Message == nil || resp.Error != nil { t.Fatalf("Did not receive correct response: %+v", resp.Error) } return resp.Message } // Do basic checks. basicCheck := func(subject string, expectedSeq uint64) { sm := getLast(subject) if sm == nil { t.Fatalf("Expected a message but got none") } else if string(sm.Data) != "VAL-1000" { t.Fatalf("Wrong message payload, wanted %q but got %q", "VAL-1000", sm.Data) } else if sm.Sequence != expectedSeq { t.Fatalf("Wrong message sequence, wanted %d but got %d", expectedSeq, sm.Sequence) } else if !subjectIsSubsetMatch(sm.Subject, subject) { t.Fatalf("Wrong subject, wanted %q but got %q", subject, sm.Subject) } } basicCheck("kv.foo", 2998) basicCheck("kv.bar", 2999) basicCheck("kv.baz", 3000) basicCheck("kv.*", 3000) basicCheck(">", 3000) }) } } // https://github.com/nats-io/nats-server/issues/2329 func TestJetStreamGetLastMsgBySubjectAfterUpdate(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sc := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 2, } if _, err := js.AddStream(sc); err != nil { t.Fatalf("Unexpected error: %v", err) } // Now Update and add in other subjects. sc.Subjects = append(sc.Subjects, "bar", "baz") if _, err := js.UpdateStream(sc); err != nil { t.Fatalf("Unexpected error: %v", err) } js.Publish("foo", []byte("OK1")) // 1 js.Publish("bar", []byte("OK1")) // 2 js.Publish("foo", []byte("OK2")) // 3 js.Publish("bar", []byte("OK2")) // 4 // Need to do stream GetMsg by hand for now until Go client support lands. getLast := func(subject string) *StoredMsg { t.Helper() req := &JSApiMsgGetRequest{LastFor: subject} b, _ := json.Marshal(req) rmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, "TEST"), b, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var resp JSApiMsgGetResponse err = json.Unmarshal(rmsg.Data, &resp) if err != nil { t.Fatalf("Could not parse stream message: %v", err) } if resp.Message == nil || resp.Error != nil { t.Fatalf("Did not receive correct response: %+v", resp.Error) } return resp.Message } // Do basic checks. basicCheck := func(subject string, expectedSeq uint64) { sm := getLast(subject) if sm == nil { t.Fatalf("Expected a message but got none") } else if sm.Sequence != expectedSeq { t.Fatalf("Wrong message sequence, wanted %d but got %d", expectedSeq, sm.Sequence) } else if !subjectIsSubsetMatch(sm.Subject, subject) { t.Fatalf("Wrong subject, wanted %q but got %q", subject, sm.Subject) } } basicCheck("foo", 3) basicCheck("bar", 4) } func TestJetStreamLastSequenceBySubject(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: st, Replicas: 3, MaxMsgsPer: 1, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do manually for now. m, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) si, err := js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v, respmsg: %q", err, string(m.Data)) } if si == nil || si.Config.Name != "KV" { t.Fatalf("StreamInfo is not correct %+v", si) } js.PublishAsync("kv.foo", []byte("1")) js.PublishAsync("kv.bar", []byte("2")) js.PublishAsync("kv.baz", []byte("3")) select { case <-js.PublishAsyncComplete(): case <-time.After(time.Second): t.Fatalf("Did not receive completion signal") } // Now make sure we get an error if the last sequence is not correct per subject. pubAndCheck := func(subj, seq string, ok bool) { t.Helper() m := nats.NewMsg(subj) m.Data = []byte("HELLO") m.Header.Set(JSExpectedLastSubjSeq, seq) _, err := js.PublishMsg(m) if ok && err != nil { t.Fatalf("Unexpected error: %v", err) } if !ok && err == nil { t.Fatalf("Expected to get an error and got none") } } pubAndCheck("kv.foo", "1", true) // So last is now 4. pubAndCheck("kv.foo", "1", false) // This should fail. pubAndCheck("kv.bar", "2", true) pubAndCheck("kv.bar", "5", true) pubAndCheck("kv.xxx", "5", false) }) } } func TestJetStreamFilteredConsumersWithWiderFilter(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz", "N.*"}, }) require_NoError(t, err) // Add in some messages. js.Publish("foo", []byte("OK")) js.Publish("bar", []byte("OK")) js.Publish("baz", []byte("OK")) for i := 0; i < 12; i++ { js.Publish(fmt.Sprintf("N.%d", i+1), []byte("OK")) } checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 15 { return fmt.Errorf("Expected 15 msgs, got state: %+v", si.State) } return nil }) checkWider := func(subj string, numExpected int) { sub, err := js.SubscribeSync(subj) require_NoError(t, err) defer sub.Unsubscribe() checkSubsPending(t, sub, numExpected) } checkWider("*", 3) checkWider("N.*", 12) checkWider("*.*", 12) checkWider("N.>", 12) checkWider(">", 15) } func TestJetStreamMirrorAndSourcesFilteredConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz.*"}, }) require_NoError(t, err) // Create Mirror now. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, }) require_NoError(t, err) dsubj := nats.NewInbox() nc.SubscribeSync(dsubj) nc.Flush() createConsumer := func(sn, fs string) { t.Helper() _, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs}) require_NoError(t, err) } createConsumer("M", "foo") createConsumer("M", "bar") createConsumer("M", "baz.foo") // Now do some sources. if _, err := js.AddStream(&nats.StreamConfig{Name: "O1", Subjects: []string{"foo.*"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "O2", Subjects: []string{"bar.*"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Create Mirror now. _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{Name: "O1"}, {Name: "O2"}}, }) require_NoError(t, err) createConsumer("S", "foo.1") createConsumer("S", "bar.1") // Chaining // Create Mirror now. _, err = js.AddStream(&nats.StreamConfig{ Name: "M2", Mirror: &nats.StreamSource{Name: "M"}, }) require_NoError(t, err) createConsumer("M2", "foo") createConsumer("M2", "bar") createConsumer("M2", "baz.foo") } func TestJetStreamMirrorBasics(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() createStream := func(cfg *nats.StreamConfig) (*nats.StreamInfo, error) { return js.AddStream(cfg) } createStreamOk := func(cfg *nats.StreamConfig) { t.Helper() if _, err := createStream(cfg); err != nil { t.Fatalf("Expected no error, got %+v", err) } } // Test we get right config errors etc. cfg := &nats.StreamConfig{ Name: "M1", Subjects: []string{"foo", "bar", "baz"}, Mirror: &nats.StreamSource{Name: "S1"}, } _, err := createStream(cfg) if err == nil || !strings.Contains(err.Error(), "stream mirrors can not") { t.Fatalf("Expected error, got %+v", err) } // Clear subjects. cfg.Subjects = nil // Mirrored scfg := &nats.StreamConfig{ Name: "S1", Subjects: []string{"foo", "bar", "baz"}, } // Create mirrored stream createStreamOk(scfg) // Now create our mirror stream. createStreamOk(cfg) // For now wait for the consumer state to register. time.Sleep(250 * time.Millisecond) // Send 100 messages. for i := 0; i < 100; i++ { if _, err := js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Faster timeout since we loop below checking for condition. js2, err := nc.JetStream(nats.MaxWait(500 * time.Millisecond)) require_NoError(t, err) checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { si, err := js2.StreamInfo("M1") require_NoError(t, err) if si.State.Msgs != 100 { return fmt.Errorf("Expected 100 msgs, got state: %+v", si.State) } return nil }) // Purge the mirrored stream. if err := js.PurgeStream("S1"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } // Send 50 more msgs now. for i := 0; i < 50; i++ { if _, err := js.Publish("bar", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } cfg = &nats.StreamConfig{ Name: "M2", Storage: nats.FileStorage, Mirror: &nats.StreamSource{Name: "S1"}, } // Now create our second mirror stream. createStreamOk(cfg) checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { si, err := js2.StreamInfo("M2") require_NoError(t, err) if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 101 { return fmt.Errorf("Expected start seq of 101, got state: %+v", si.State) } return nil }) // Send 100 more msgs now. Should be 150 total, 101 first. for i := 0; i < 100; i++ { if _, err := js.Publish("baz", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } cfg = &nats.StreamConfig{ Name: "M3", Mirror: &nats.StreamSource{Name: "S1", OptStartSeq: 150}, } createStreamOk(cfg) checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { si, err := js2.StreamInfo("M3") require_NoError(t, err) if si.State.Msgs != 101 { return fmt.Errorf("Expected 101 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 150 { return fmt.Errorf("Expected start seq of 150, got state: %+v", si.State) } return nil }) // Make sure setting time works ok. start := time.Now().UTC().Add(-2 * time.Hour) cfg = &nats.StreamConfig{ Name: "M4", Mirror: &nats.StreamSource{Name: "S1", OptStartTime: &start}, } createStreamOk(cfg) checkFor(t, 5*time.Second, 250*time.Millisecond, func() error { si, err := js2.StreamInfo("M4") require_NoError(t, err) if si.State.Msgs != 150 { return fmt.Errorf("Expected 150 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 101 { return fmt.Errorf("Expected start seq of 101, got state: %+v", si.State) } return nil }) // Test subject filtering and transformation createStreamServerStreamConfig := func(cfg *StreamConfig, errToCheck uint16) { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse if err := json.Unmarshal(rm.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if errToCheck == 0 { if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } } else { if resp.Error.ErrCode != errToCheck { t.Fatalf("Expected error %+v, got: %+v", errToCheck, resp.Error) } } } // check for errors createStreamServerStreamConfig(&StreamConfig{ Name: "MBAD", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", FilterSubject: "foo", SubjectTransforms: []SubjectTransformConfig{{Source: "foo", Destination: "foo3"}}}, }, ApiErrors[JSMirrorMultipleFiltersNotAllowed].ErrCode) createStreamServerStreamConfig(&StreamConfig{ Name: "MBAD", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", SubjectTransforms: []SubjectTransformConfig{{Source: ".*.", Destination: "foo3"}}}, }, ApiErrors[JSMirrorInvalidSubjectFilter].ErrCode) createStreamServerStreamConfig(&StreamConfig{ Name: "MBAD", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", SubjectTransforms: []SubjectTransformConfig{{Source: "*", Destination: "{{wildcard(2)}}"}}}, }, ApiErrors[JSStreamCreateErrF].ErrCode) createStreamServerStreamConfig(&StreamConfig{ Name: "MBAD", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", SubjectTransforms: []SubjectTransformConfig{{Source: "foo", Destination: ""}, {Source: "foo", Destination: "bar"}}}, }, ApiErrors[JSMirrorOverlappingSubjectFilters].ErrCode) createStreamServerStreamConfig(&StreamConfig{ Name: "M5", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", SubjectTransforms: []SubjectTransformConfig{{Source: "foo", Destination: "foo2"}}}, }, 0) createStreamServerStreamConfig(&StreamConfig{ Name: "M6", Storage: FileStorage, Mirror: &StreamSource{Name: "S1", SubjectTransforms: []SubjectTransformConfig{{Source: "bar", Destination: "bar2"}, {Source: "baz", Destination: "baz2"}}}, }, 0) // Send 100 messages on foo (there should already be 50 messages on bar and 100 on baz in the stream) for i := 0; i < 100; i++ { if _, err := js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } var f = func(streamName string, subject string, subjectNumMsgs uint64, streamNumMsg uint64, firstSeq uint64, lastSeq uint64) func() error { return func() error { si, err := js2.StreamInfo(streamName, &nats.StreamInfoRequest{SubjectsFilter: ">"}) require_NoError(t, err) if ss, ok := si.State.Subjects[subject]; !ok { return fmt.Errorf("expected messages with the transformed subject %s", subject) } else { if ss != subjectNumMsgs { return fmt.Errorf("expected %d messages on the transformed subject %s but got %d", subjectNumMsgs, subject, ss) } } if si.State.Msgs != streamNumMsg { return fmt.Errorf("expected %d stream messages, got state: %+v", streamNumMsg, si.State) } if si.State.FirstSeq != firstSeq || si.State.LastSeq != lastSeq { return fmt.Errorf("expected first sequence=%d and last sequence=%d, but got state: %+v", firstSeq, lastSeq, si.State) } return nil } } checkFor(t, 10*time.Second, 500*time.Millisecond, f("M5", "foo2", 100, 100, 251, 350)) checkFor(t, 10*time.Second, 500*time.Millisecond, f("M6", "bar2", 50, 150, 101, 250)) checkFor(t, 10*time.Second, 500*time.Millisecond, f("M6", "baz2", 100, 150, 101, 250)) } func TestJetStreamMirrorUpdatePreventsSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "ORIGINAL"}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "MIRROR", Mirror: &nats.StreamSource{Name: "ORIGINAL"}}) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{Name: "MIRROR", Mirror: &nats.StreamSource{Name: "ORIGINAL"}, Subjects: []string{"x"}}) if err == nil || err.Error() != "nats: stream mirrors can not contain subjects" { t.Fatalf("Expected to not be able to put subjects on a stream, got: %+v", err) } } func TestJetStreamSourceBasics(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() createStream := func(cfg *StreamConfig) { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse if err := json.Unmarshal(rm.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } } if _, err := js.AddStream(&nats.StreamConfig{Name: "test", Sources: []*nats.StreamSource{{Name: ""}}}); err.Error() == "source stream name is invalid" { t.Fatal("Expected a source stream name is invalid error") } for _, sname := range []string{"foo", "bar", "baz"} { if _, err := js.AddStream(&nats.StreamConfig{Name: sname}); err != nil { t.Fatalf("Unexpected error: %v", err) } } sendBatch := func(subject string, n int) { for i := 0; i < n; i++ { if _, err := js.Publish(subject, []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } // Populate each one. sendBatch("foo", 10) sendBatch("bar", 15) sendBatch("baz", 25) cfg := &StreamConfig{ Name: "MS", Storage: FileStorage, Sources: []*StreamSource{ {Name: "foo", SubjectTransforms: []SubjectTransformConfig{{Source: ">", Destination: "foo2.>"}}}, {Name: "bar"}, {Name: "baz"}, }, } createStream(cfg) // Faster timeout since we loop below checking for condition. js2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond)) require_NoError(t, err) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("MS") require_NoError(t, err) if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } return nil }) ss, err := js.SubscribeSync("foo2.foo", nats.BindStream("MS")) require_NoError(t, err) // we must have at least one message on the transformed subject name (ie no timeout) _, err = ss.NextMsg(time.Millisecond) require_NoError(t, err) ss.Drain() // Test Source Updates ncfg := &nats.StreamConfig{ Name: "MS", Sources: []*nats.StreamSource{ // Keep foo, bar, remove baz, add dlc {Name: "foo"}, {Name: "bar"}, {Name: "dlc"}, }, } if _, err := js.UpdateStream(ncfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // Test optional start times, filtered subjects etc. if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"dlc", "rip", "jnm"}}); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("dlc", 20) sendBatch("rip", 20) sendBatch("dlc", 10) sendBatch("jnm", 10) cfg = &StreamConfig{ Name: "FMS", Storage: FileStorage, Sources: []*StreamSource{ {Name: "TEST", OptStartSeq: 26}, }, } createStream(cfg) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("FMS") require_NoError(t, err) if si.State.Msgs != 35 { return fmt.Errorf("Expected 35 msgs, got state: %+v", si.State) } return nil }) // Double check first starting. m, err := js.GetMsg("FMS", 1) require_NoError(t, err) if shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ { t.Fatalf("Expected a header, got none") } else if _, _, sseq := streamAndSeq(shdr); sseq != 26 { t.Fatalf("Expected header sequence of 26, got %d", sseq) } // Test Filters cfg = &StreamConfig{ Name: "FMS2", Storage: FileStorage, Sources: []*StreamSource{ {Name: "TEST", OptStartSeq: 11, SubjectTransforms: []SubjectTransformConfig{{Source: "dlc", Destination: "dlc2"}}}, }, } createStream(cfg) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("FMS2") require_NoError(t, err) if si.State.Msgs != 20 { return fmt.Errorf("Expected 20 msgs, got state: %+v", si.State) } return nil }) // Double check first starting. if m, err = js.GetMsg("FMS2", 1); err != nil { t.Fatalf("Unexpected error: %v", err) } if shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ { t.Fatalf("Expected a header, got none") } else if _, _, sseq := streamAndSeq(shdr); sseq != 11 { t.Fatalf("Expected header sequence of 11, got %d", sseq) } if m.Subject != "dlc2" { t.Fatalf("Expected transformed subject dlc2, but got %s instead", m.Subject) } // Test Filters cfg = &StreamConfig{ Name: "FMS3", Storage: FileStorage, Sources: []*StreamSource{ {Name: "TEST", SubjectTransforms: []SubjectTransformConfig{{Source: "dlc", Destination: "dlc2"}, {Source: "rip", Destination: ""}}}, }, } createStream(cfg) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("FMS3") require_NoError(t, err) if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } return nil }) // Double check first message if m, err = js.GetMsg("FMS3", 1); err != nil { t.Fatalf("Unexpected error: %v", err) } if shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ { t.Fatalf("Expected a header, got none") } else if m.Subject != "dlc2" { t.Fatalf("Expected subject 'dlc2' and got %s", m.Subject) } // Double check first message with the other subject if m, err = js.GetMsg("FMS3", 21); err != nil { t.Fatalf("Unexpected error: %v", err) } if shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ { t.Fatalf("Expected a header, got none") } else if m.Subject != "rip" { t.Fatalf("Expected subject 'rip' and got %s", m.Subject) } checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("FMS3") require_NoError(t, err) if si.State.Subjects["jnm"] != 0 { return fmt.Errorf("Unexpected messages from the source found") } return nil }) // pre 2.10 backwards compatibility transformConfig := nats.SubjectTransformConfig{Source: "B.*", Destination: "A.{{Wildcard(1)}}"} aConfig := nats.StreamConfig{Name: "A", Subjects: []string{"B.*"}, SubjectTransform: &transformConfig} if _, err := js.AddStream(&aConfig); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch("B.A", 1) sendBatch("B.B", 1) bConfig := nats.StreamConfig{Name: "B", Subjects: []string{"A.*"}} if _, err := js.AddStream(&bConfig); err != nil { t.Fatalf("Unexpected error: %v", err) } // fake a message that would have been sourced with pre 2.10 msg := nats.NewMsg("A.A") // pre 2.10 header format just stream name and sequence number msg.Header.Set(JSStreamSource, "A 1") msg.Data = []byte("OK") if _, err := js.PublishMsg(msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } bConfig.Sources = []*nats.StreamSource{{Name: "A"}} if _, err := js.UpdateStream(&bConfig); err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("B") require_NoError(t, err) if si.State.Msgs != 2 { return fmt.Errorf("Expected 2 msgs, got state: %+v", si.State) } return nil }) } func TestJetStreamSourceWorkingQueueWithLimit(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "test", Subjects: []string{"test"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "wq", MaxMsgs: 100, Discard: nats.DiscardNew, Retention: nats.WorkQueuePolicy, Sources: []*nats.StreamSource{{Name: "test"}}}) require_NoError(t, err) sendBatch := func(subject string, n int) { for i := 0; i < n; i++ { _, err = js.Publish(subject, []byte("OK")) require_NoError(t, err) } } // Populate each one. sendBatch("test", 300) checkFor(t, 3*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("wq") require_NoError(t, err) if si.State.Msgs != 100 { return fmt.Errorf("Expected 100 msgs, got state: %+v", si.State) } return nil }) _, err = js.AddConsumer("wq", &nats.ConsumerConfig{Durable: "wqc", FilterSubject: "test", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ss, err := js.PullSubscribe("test", "wqc", nats.Bind("wq", "wqc")) require_NoError(t, err) // we must have at least one message on the transformed subject name (ie no timeout) f := func(done chan bool) { for i := 0; i < 300; i++ { m, err := ss.Fetch(1, nats.MaxWait(3*time.Second)) require_NoError(t, err) time.Sleep(11 * time.Millisecond) err = m[0].Ack() require_NoError(t, err) } done <- true } var doneChan = make(chan bool) go f(doneChan) checkFor(t, 6*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("wq") require_NoError(t, err) if si.State.Msgs > 0 && si.State.Msgs <= 100 { return fmt.Errorf("Expected 0 msgs, got: %d", si.State.Msgs) } else if si.State.Msgs > 100 { t.Fatalf("Got more than our 100 message limit: %+v", si.State) } return nil }) select { case <-doneChan: ss.Drain() case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } } func TestJetStreamStreamSourceFromKV(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API reuqests. nc, js := jsClientConnect(t, s) defer nc.Close() // Create a kv store kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test"}) require_NoError(t, err) // Create a stream with a source from the kv store _, err = js.AddStream(&nats.StreamConfig{Name: "test", Retention: nats.InterestPolicy, Sources: []*nats.StreamSource{{Name: "KV_" + kv.Bucket()}}}) require_NoError(t, err) // Create a interested consumer _, err = js.AddConsumer("test", &nats.ConsumerConfig{Durable: "durable", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) ss, err := js.PullSubscribe("", "", nats.Bind("test", "durable")) require_NoError(t, err) rev1, err := kv.Create("key", []byte("value1")) require_NoError(t, err) m, err := ss.Fetch(1, nats.MaxWait(2*time.Second)) require_NoError(t, err) require_NoError(t, m[0].Ack()) if string(m[0].Data) != "value1" { t.Fatalf("Expected value1, got %s", m[0].Data) } rev2, err := kv.Update("key", []byte("value2"), rev1) require_NoError(t, err) _, err = kv.Update("key", []byte("value3"), rev2) require_NoError(t, err) m, err = ss.Fetch(1, nats.MaxWait(500*time.Millisecond)) require_NoError(t, err) require_NoError(t, m[0].Ack()) if string(m[0].Data) != "value2" { t.Fatalf("Expected value2, got %s", m[0].Data) } m, err = ss.Fetch(1, nats.MaxWait(500*time.Millisecond)) require_NoError(t, err) require_NoError(t, m[0].Ack()) if string(m[0].Data) != "value3" { t.Fatalf("Expected value3, got %s", m[0].Data) } } func TestJetStreamInputTransform(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() createStream := func(cfg *StreamConfig) { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse if err := json.Unmarshal(rm.Data, &resp); err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.Error != nil { t.Fatalf("Unexpected error: %+v", resp.Error) } } createStream(&StreamConfig{Name: "T1", Subjects: []string{"foo"}, SubjectTransform: &SubjectTransformConfig{Source: ">", Destination: "transformed.>"}, Storage: MemoryStorage}) // publish a message if _, err := js.Publish("foo", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } m, err := js.GetMsg("T1", 1) require_NoError(t, err) if m.Subject != "transformed.foo" { t.Fatalf("Expected message subject transformed.foo, got %s", m.Subject) } } func TestJetStreamOperatorAccounts(t *testing.T) { s, _ := RunServerWithConfig("./configs/js-op.conf") if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserCredentials("./configs/one.creds")) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST"}); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 100 for i := 0; i < toSend; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Close our user for account one. nc.Close() // Restart the server. s.Shutdown() s, _ = RunServerWithConfig("./configs/js-op.conf") defer s.Shutdown() jsz, err := s.Jsz(nil) require_NoError(t, err) if jsz.Streams != 1 { t.Fatalf("Expected jsz to report our stream on restart") } if jsz.Messages != uint64(toSend) { t.Fatalf("Expected jsz to report our %d messages on restart, got %d", toSend, jsz.Messages) } } func TestJetStreamServerDomainBadConfig(t *testing.T) { shouldFail := func(domain string) { t.Helper() opts := DefaultTestOptions opts.JetStreamDomain = domain if err := validateOptions(&opts); err == nil || !strings.Contains(err.Error(), "invalid domain name") { t.Fatalf("Expected bad domain error, got %v", err) } } shouldFail("HU..B") shouldFail("HU B") shouldFail(" ") shouldFail("\t") shouldFail("CORE.") shouldFail(".CORE") shouldFail("C.*.O. RE") shouldFail("C.ORE") } func TestJetStreamServerDomainConfig(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {domain: "HUB", store_dir: %q} `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } config := s.JetStreamConfig() if config.Domain != "HUB" { t.Fatalf("Expected %q as domain name, got %q", "HUB", config.Domain) } } func TestJetStreamServerDomainConfigButDisabled(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 jetstream: {domain: "HUB", enabled: false} `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() if s.JetStreamEnabled() { t.Fatalf("Expected JetStream NOT to be enabled") } opts := s.getOpts() if opts.JetStreamDomain != "HUB" { t.Fatalf("Expected %q as opts domain name, got %q", "HUB", opts.JetStreamDomain) } } func TestJetStreamDomainInPubAck(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {domain: "HUB", store_dir: %q} `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Storage: nats.MemoryStorage, Subjects: []string{"foo"}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // Check by hand for now til it makes its way into Go client. am, err := nc.Request("foo", nil, time.Second) require_NoError(t, err) var pa PubAck json.Unmarshal(am.Data, &pa) if pa.Domain != "HUB" { t.Fatalf("Expected PubAck to have domain of %q, got %q", "HUB", pa.Domain) } } // Issue #2213 func TestJetStreamDirectConsumersBeingReported(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "S", Sources: []*nats.StreamSource{{ Name: "TEST", }}, }) require_NoError(t, err) if _, err = js.Publish("foo", nil); err != nil { t.Fatalf("Unexpected publish error: %v", err) } checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { return fmt.Errorf("Could not get stream info: %v", err) } if si.State.Msgs != 1 { return fmt.Errorf("Expected 1 msg, got state: %+v", si.State) } return nil }) si, err := js.StreamInfo("TEST") require_NoError(t, err) // Direct consumers should not be reported if si.State.Consumers != 0 { t.Fatalf("Did not expect any consumers, got %d", si.State.Consumers) } // Now check for consumer in consumer names list. var names []string for name := range js.ConsumerNames("TEST") { names = append(names, name) } if len(names) != 0 { t.Fatalf("Expected no consumers but got %+v", names) } // Now check detailed list. var cis []*nats.ConsumerInfo for ci := range js.ConsumersInfo("TEST") { cis = append(cis, ci) } if len(cis) != 0 { t.Fatalf("Expected no consumers but got %+v", cis) } } // https://github.com/nats-io/nats-server/issues/2290 func TestJetStreamTemplatedErrorsBug(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.PullSubscribe("foo", "") if err != nil && strings.Contains(err.Error(), "{err}") { t.Fatalf("Error is not filled in: %v", err) } } func TestJetStreamServerEncryption(t *testing.T) { cases := []struct { name string cstr string cipher StoreCipher }{ {"Default", _EMPTY_, ChaCha}, {"ChaCha", ", cipher: chacha", ChaCha}, {"AES", ", cipher: aes", AES}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { tmpl := ` server_name: S22 listen: 127.0.0.1:-1 jetstream: {key: $JS_KEY, store_dir: '%s' %s} ` storeDir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, c.cstr))) os.Setenv("JS_KEY", "s3cr3t!!") defer os.Unsetenv("JS_KEY") s, _ := RunServerWithConfig(conf) defer s.Shutdown() config := s.JetStreamConfig() if config == nil { t.Fatalf("Expected config but got none") } defer removeDir(t, config.StoreDir) // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz"}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } msg := []byte("ENCRYPTED PAYLOAD!!") sendMsg := func(subj string) { t.Helper() if _, err := js.Publish(subj, msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } // Send 10 msgs for i := 0; i < 10; i++ { sendMsg("foo") } // Now create a consumer. sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } for i, m := range fetchMsgs(t, sub, 10, 5*time.Second) { if i < 5 { m.AckSync() } } // Grab our state to compare after restart. si, _ := js.StreamInfo("TEST") ci, _ := js.ConsumerInfo("TEST", "dlc") // Quick check to make sure everything not just plaintext still. sdir := filepath.Join(config.StoreDir, "$G", "streams", "TEST") // Make sure we can not find any plaintext strings in the target file. checkFor := func(fn string, strs ...string) { t.Helper() data, err := os.ReadFile(fn) if err != nil { t.Fatalf("Unexpected error: %v", err) } for _, str := range strs { if bytes.Contains(data, []byte(str)) { t.Fatalf("Found %q in body of file contents", str) } } } checkKeyFile := func(fn string) { t.Helper() if _, err := os.Stat(fn); err != nil { t.Fatalf("Expected a key file at %q", fn) } } // Check stream meta. checkEncrypted := func() { t.Helper() checkKeyFile(filepath.Join(sdir, JetStreamMetaFileKey)) checkFor(filepath.Join(sdir, JetStreamMetaFile), "TEST", "foo", "bar", "baz", "max_msgs", "max_bytes") // Check a message block. checkKeyFile(filepath.Join(sdir, "msgs", "1.key")) checkFor(filepath.Join(sdir, "msgs", "1.blk"), "ENCRYPTED PAYLOAD!!", "foo", "bar", "baz") // Check consumer meta and state. checkKeyFile(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFileKey)) checkFor(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFile), "TEST", "dlc", "foo", "bar", "baz", "max_msgs", "ack_policy") // Load and see if we can parse the consumer state. state, err := os.ReadFile(filepath.Join(sdir, "obs", "dlc", "o.dat")) require_NoError(t, err) if _, err := decodeConsumerState(state); err == nil { t.Fatalf("Expected decoding consumer state to fail") } } // Stop current s.Shutdown() checkEncrypted() // Restart. s, _ = RunServerWithConfig(conf) defer s.Shutdown() // Connect again. nc, js = jsClientConnect(t, s) defer nc.Close() si2, err := js.StreamInfo("TEST") require_NoError(t, err) if !reflect.DeepEqual(si, si2) { t.Fatalf("Stream infos did not match\n%+v\nvs\n%+v", si, si2) } ci2, _ := js.ConsumerInfo("TEST", "dlc") // Consumer create times can be slightly off after restore from disk. now := time.Now() ci.Created, ci2.Created = now, now ci.Delivered.Last, ci2.Delivered.Last = nil, nil ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil // Also clusters will be different. ci.Cluster, ci2.Cluster = nil, nil if !reflect.DeepEqual(ci, ci2) { t.Fatalf("Consumer infos did not match\n%+v\nvs\n%+v", ci, ci2) } // Send 10 more msgs for i := 0; i < 10; i++ { sendMsg("foo") } if si, err = js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 20 { t.Fatalf("Expected 20 msgs total, got %d", si.State.Msgs) } // Now test snapshots etc. acc := s.GlobalAccount() mset, err := acc.lookupStream("TEST") require_NoError(t, err) scfg := mset.config() sr, err := mset.snapshot(5*time.Second, false, true) if err != nil { t.Fatalf("Error getting snapshot: %v", err) } snapshot, err := io.ReadAll(sr.Reader) if err != nil { t.Fatalf("Error reading snapshot") } // Run new server w/o encryption. Make sure we can restore properly (meaning encryption was stripped etc). ns := RunBasicJetStreamServer(t) defer ns.Shutdown() nacc := ns.GlobalAccount() r := bytes.NewReader(snapshot) mset, err = nacc.RestoreStream(&scfg, r) require_NoError(t, err) ss := mset.store.State() if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq { t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State) } // Now restore to our encrypted server as well. if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } acc = s.GlobalAccount() r.Reset(snapshot) mset, err = acc.RestoreStream(&scfg, r) require_NoError(t, err) ss = mset.store.State() if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq { t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State) } // Check that all is encrypted like above since we know we need to convert since snapshots always plaintext. checkEncrypted() }) } } // User report of bug. func TestJetStreamConsumerBadNumPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"orders.*"}, }) require_NoError(t, err) newOrders := func(n int) { // Queue up new orders. for i := 0; i < n; i++ { js.Publish("orders.created", []byte("NEW")) } } newOrders(10) // Create to subscribers. process := func(m *nats.Msg) { js.Publish("orders.approved", []byte("APPROVED")) } op, err := js.Subscribe("orders.created", process) require_NoError(t, err) defer op.Unsubscribe() mon, err := js.SubscribeSync("orders.*") require_NoError(t, err) defer mon.Unsubscribe() waitForMsgs := func(n uint64) { t.Helper() checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("ORDERS") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != n { return fmt.Errorf("Expected %d msgs, got state: %+v", n, si.State) } return nil }) } checkForNoPending := func(sub *nats.Subscription) { t.Helper() if ci, err := sub.ConsumerInfo(); err != nil || ci == nil || ci.NumPending != 0 { if ci != nil && ci.NumPending != 0 { t.Fatalf("Bad consumer NumPending, expected 0 but got %d", ci.NumPending) } else { t.Fatalf("Bad consumer info: %+v", ci) } } } waitForMsgs(20) checkForNoPending(op) checkForNoPending(mon) newOrders(10) waitForMsgs(40) checkForNoPending(op) checkForNoPending(mon) } func TestJetStreamDeliverLastPerSubject(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: st, MaxMsgsPer: 5, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) si, err := js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "KV" { t.Fatalf("StreamInfo is not correct %+v", si) } // Interleave them on purpose. for i := 1; i <= 11; i++ { msg := []byte(fmt.Sprintf("%d", i)) js.PublishAsync("kv.b1.foo", msg) js.PublishAsync("kv.b2.foo", msg) js.PublishAsync("kv.b1.bar", msg) js.PublishAsync("kv.b2.bar", msg) js.PublishAsync("kv.b1.baz", msg) js.PublishAsync("kv.b2.baz", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(2 * time.Second): t.Fatalf("Did not receive completion signal") } // Do quick check that config needs FilteredSubjects otherwise bad config. badReq := CreateConsumerRequest{ Stream: "KV", Config: ConsumerConfig{ DeliverSubject: "b", DeliverPolicy: DeliverLastPerSubject, }, } req, err = json.Marshal(badReq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "KV"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ccResp JSApiConsumerCreateResponse if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.Error == nil || !strings.Contains(ccResp.Error.Description, "filter subject is not set") { t.Fatalf("Expected an error, got none") } // Now let's consume these via last per subject. obsReq := CreateConsumerRequest{ Stream: "KV", Config: ConsumerConfig{ DeliverSubject: "d", DeliverPolicy: DeliverLastPerSubject, FilterSubject: "kv.b1.*", }, } req, err = json.Marshal(obsReq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, "KV"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } ccResp.Error = nil if err = json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } sub, _ := nc.SubscribeSync("d") defer sub.Unsubscribe() // Helper to check messages are correct. checkNext := func(subject string, sseq uint64, v string) { t.Helper() m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if m.Subject != subject { t.Fatalf("Expected subject %q but got %q", subject, m.Subject) } meta, err := m.Metadata() if err != nil { t.Fatalf("didn't get metadata: %s", err) } if meta.Sequence.Stream != sseq { t.Fatalf("Expected stream seq %d but got %d", sseq, meta.Sequence.Stream) } if string(m.Data) != v { t.Fatalf("Expected data of %q but got %q", v, m.Data) } } checkSubsPending(t, sub, 3) // Now make sure they are what we expect. checkNext("kv.b1.foo", 61, "11") checkNext("kv.b1.bar", 63, "11") checkNext("kv.b1.baz", 65, "11") msg := []byte(fmt.Sprintf("%d", 22)) js.Publish("kv.b1.bar", msg) js.Publish("kv.b2.foo", msg) // Not filtered through.. checkSubsPending(t, sub, 1) checkNext("kv.b1.bar", 67, "22") }) } } func TestJetStreamDeliverLastPerSubjectNumPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{ Name: "KV", Subjects: []string{"KV.>"}, MaxMsgsPerSubject: 5, Replicas: 1, }); err != nil { t.Fatalf("Error adding stream: %v", err) } for i := 0; i < 5; i++ { msg := []byte(fmt.Sprintf("msg%d", i)) js.Publish("KV.foo", msg) js.Publish("KV.bar", msg) js.Publish("KV.baz", msg) js.Publish("KV.bat", msg) } // Delete some messages js.DeleteMsg("KV", 2) js.DeleteMsg("KV", 5) ci, err := js.AddConsumer("KV", &nats.ConsumerConfig{ DeliverSubject: nats.NewInbox(), AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverLastPerSubjectPolicy, FilterSubject: "KV.>", }) if err != nil { t.Fatalf("Error adding consumer: %v", err) } if ci.NumPending != 4 { t.Fatalf("Expected 4 pending msgs, got %v", ci.NumPending) } } // We had a report of a consumer delete crashing the server when in interest retention mode. // This I believe is only really possible in clustered mode, but we will force the issue here. func TestJetStreamConsumerCleanupWithRetentionPolicy(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"orders.*"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub, err := js.SubscribeSync("orders.*") require_NoError(t, err) payload := []byte("Hello World") for i := 0; i < 10; i++ { subj := fmt.Sprintf("orders.%d", i+1) js.Publish(subj, payload) } checkSubsPending(t, sub, 10) for i := 0; i < 10; i++ { m, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } m.AckSync() } ci, err := sub.ConsumerInfo() if err != nil { t.Fatalf("Unexpected error getting consumer info: %v", err) } acc := s.GlobalAccount() mset, err := acc.lookupStream("ORDERS") require_NoError(t, err) o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } lseq := mset.lastSeq() o.mu.Lock() // Force boundary condition here. o.asflr = lseq + 2 o.mu.Unlock() sub.Unsubscribe() // Make sure server still available. if _, err := js.StreamInfo("ORDERS"); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Issue #2392 func TestJetStreamPurgeEffectsConsumerDelivery(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, }) require_NoError(t, err) js.Publish("foo.a", []byte("show once")) sub, err := js.SubscribeSync("foo.*", nats.AckWait(250*time.Millisecond), nats.DeliverAll(), nats.AckExplicit()) require_NoError(t, err) defer sub.Unsubscribe() checkSubsPending(t, sub, 1) // Do not ack. if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Error receiving message: %v", err) } // Now purge stream. if err := js.PurgeStream("TEST"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } js.Publish("foo.b", []byte("show twice?")) // Do not ack again, should show back up. if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Error receiving message: %v", err) } // Make sure we get it back. if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Error receiving message: %v", err) } } // Issue #2403 func TestJetStreamExpireCausesDeadlock(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Storage: nats.MemoryStorage, MaxMsgs: 10, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub, err := js.SubscribeSync("foo.bar") require_NoError(t, err) defer sub.Unsubscribe() // Publish from two connections to get the write lock request wedged in between // having the RLock and wanting it again deeper in the stack. nc2, js2 := jsClientConnect(t, s) defer nc2.Close() for i := 0; i < 1000; i++ { js.PublishAsync("foo.bar", []byte("HELLO")) js2.PublishAsync("foo.bar", []byte("HELLO")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // If we deadlocked then we will not be able to get stream info. if _, err := js.StreamInfo("TEST"); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamConsumerPendingBugWithKV(t *testing.T) { msc := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: MemoryStorage, MaxMsgsPer: 1, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() // Not in Go client under server yet. mset, err := s.GlobalAccount().addStream(c.mconfig) if err != nil { t.Fatalf("Unexpected error: %v", err) } js.Publish("kv.1", []byte("1")) js.Publish("kv.2", []byte("2")) js.Publish("kv.3", []byte("3")) js.Publish("kv.1", []byte("4")) si, err := js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 3 { t.Fatalf("Expected 3 total msgs, got %d", si.State.Msgs) } o, err := mset.addConsumer(&ConsumerConfig{ Durable: "dlc", DeliverSubject: "xxx", DeliverPolicy: DeliverLastPerSubject, FilterSubject: ">", }) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ci := o.info(); ci.NumPending != 3 { t.Fatalf("Expected pending of 3, got %d", ci.NumPending) } }) } } // Issue #2420 func TestJetStreamDefaultMaxMsgsPer(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Storage: nats.MemoryStorage, MaxMsgs: 10, }) require_NoError(t, err) if si.Config.MaxMsgsPerSubject != -1 { t.Fatalf("Expected default of -1, got %d", si.Config.MaxMsgsPerSubject) } } // Issue #2423 func TestJetStreamBadConsumerCreateErr(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Storage: nats.MemoryStorage, }) require_NoError(t, err) // When adding a consumer with both deliver subject and max wait (push vs pull), // we got the wrong err about deliver subject having a wildcard. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "nowcerr", DeliverSubject: "X", MaxWaiting: 100, }) if err == nil { t.Fatalf("Expected an error but got none") } if !strings.Contains(err.Error(), "push mode can not set max waiting") { t.Fatalf("Incorrect error returned: %v", err) } } func TestJetStreamConsumerPushBound(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Storage: nats.MemoryStorage, Subjects: []string{"foo"}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } // We want to test extended consumer info for push based consumers. // We need to do these by hand for now. createConsumer := func(name, deliver string) { t.Helper() creq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: name, DeliverSubject: deliver, AckPolicy: AckExplicit, }, } req, err := json.Marshal(creq) if err != nil { t.Fatalf("Unexpected error: %v", err) } resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", name), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var ccResp JSApiConsumerCreateResponse if err := json.Unmarshal(resp.Data, &ccResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if ccResp.ConsumerInfo == nil || ccResp.Error != nil { t.Fatalf("Got a bad response %+v", ccResp) } } consumerInfo := func(name string) *ConsumerInfo { t.Helper() resp, err := nc.Request(fmt.Sprintf(JSApiConsumerInfoT, "TEST", name), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cinfo JSApiConsumerInfoResponse if err := json.Unmarshal(resp.Data, &cinfo); err != nil { t.Fatalf("Unexpected error: %v", err) } if cinfo.ConsumerInfo == nil || cinfo.Error != nil { t.Fatalf("Got a bad response %+v", cinfo) } return cinfo.ConsumerInfo } // First create a durable push and make sure we show now active status. createConsumer("dlc", "d.X") if ci := consumerInfo("dlc"); ci.PushBound { t.Fatalf("Expected push bound to be false") } // Now bind the deliver subject. sub, _ := nc.SubscribeSync("d.X") nc.Flush() // Make sure it registers. // Check that its reported. if ci := consumerInfo("dlc"); !ci.PushBound { t.Fatalf("Expected push bound to be set") } sub.Unsubscribe() nc.Flush() // Make sure it registers. if ci := consumerInfo("dlc"); ci.PushBound { t.Fatalf("Expected push bound to be false") } // Now make sure we have queue groups indictated as needed. createConsumer("ik", "d.Z") // Now bind the deliver subject with a queue group. sub, _ = nc.QueueSubscribeSync("d.Z", "g22") defer sub.Unsubscribe() nc.Flush() // Make sure it registers. // Check that queue group is not reported. if ci := consumerInfo("ik"); ci.PushBound { t.Fatalf("Expected push bound to be false") } sub.Unsubscribe() nc.Flush() // Make sure it registers. if ci := consumerInfo("ik"); ci.PushBound { t.Fatalf("Expected push bound to be false") } // Make sure pull consumers report PushBound as false by default. createConsumer("rip", _EMPTY_) if ci := consumerInfo("rip"); ci.PushBound { t.Fatalf("Expected push bound to be false") } } // Got a report of memory leaking, tracked it to internal clients for consumers. func TestJetStreamConsumerInternalClientLeak(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Storage: nats.MemoryStorage, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } ga, sa := s.GlobalAccount(), s.SystemAccount() ncb, nscb := ga.NumConnections(), sa.NumConnections() // Create 10 consumers for i := 0; i < 10; i++ { ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{DeliverSubject: "x"}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Accelerate ephemeral cleanup. mset, err := ga.lookupStream("TEST") if err != nil { t.Fatalf("Expected to find a stream for %q", "TEST") } o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } o.setInActiveDeleteThreshold(500 * time.Millisecond) } // Wait for them to all go away. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Consumers == 0 { return nil } return fmt.Errorf("Consumers still present") }) // Make sure we are not leaking clients/connections. // Server does not see these so need to look at account. if nca := ga.NumConnections(); nca != ncb { t.Fatalf("Leaked clients in global account: %d vs %d", ncb, nca) } if nsca := sa.NumConnections(); nsca != nscb { t.Fatalf("Leaked clients in system account: %d vs %d", nscb, nsca) } } func TestJetStreamConsumerEventingRaceOnShutdown(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.NoReconnect()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.MemoryStorage, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { if _, err := js.SubscribeSync("foo", nats.BindStream("TEST")); err != nil { return } } }() time.Sleep(50 * time.Millisecond) s.Shutdown() wg.Wait() } // Got a report of streams that expire all messages while the server is down report errors when clients reconnect // and try to send new messages. func TestJetStreamExpireAllWhileServerDown(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", MaxAge: 250 * time.Millisecond, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10_000 for i := 0; i < toSend; i++ { js.PublishAsync("TEST", []byte("OK")) } select { case <-js.PublishAsyncComplete(): case <-time.After(time.Second): t.Fatalf("Did not receive completion signal") } sd := s.JetStreamConfig().StoreDir s.Shutdown() time.Sleep(300 * time.Millisecond) // Restart after expire. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() if si, err := js.StreamInfo("TEST"); err != nil || si.State.Msgs != 0 { t.Fatalf("Unexpected stream info state: %+v", si) } for i := 0; i < 10; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected error: %v", err) } } if si, err := js.StreamInfo("TEST"); err != nil || si.State.Msgs != 10 { t.Fatalf("Unexpected stream info state: %+v", si) } } func TestJetStreamLongStreamNamesAndPubAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: strings.Repeat("ZABC", 256/4)[:255], Subjects: []string{"foo"}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } js.Publish("foo", []byte("HELLO")) } func TestJetStreamPerSubjectPending(t *testing.T) { for _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} { t.Run(st.String(), func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "KV_X", Subjects: []string{"$KV.X.>"}, MaxMsgsPerSubject: 5, Storage: st, }) if err != nil { t.Fatalf("add stream failed: %s", err) } // the message we will care for _, err = js.Publish("$KV.X.x.y.z", []byte("hello world")) if err != nil { t.Fatalf("publish failed: %s", err) } // make sure there's some unrelated message after _, err = js.Publish("$KV.X.1", []byte("hello world")) if err != nil { t.Fatalf("publish failed: %s", err) } // we expect the wildcard filter subject to match only the one message and so pending will be 0 sub, err := js.SubscribeSync("$KV.X.x.>", nats.DeliverLastPerSubject()) if err != nil { t.Fatalf("subscribe failed: %s", err) } msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("next failed: %s", err) } meta, err := msg.Metadata() if err != nil { t.Fatalf("meta failed: %s", err) } // with DeliverLastPerSubject set this is never 0, but without setting that its 0 correctly if meta.NumPending != 0 { t.Fatalf("expected numpending 0 got %d", meta.NumPending) } }) } } func TestJetStreamPublishExpectNoMsg(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "KV", Subjects: []string{"KV.>"}, MaxMsgsPerSubject: 5, }) if err != nil { t.Fatalf("add stream failed: %s", err) } if _, err = js.Publish("KV.22", []byte("hello world")); err != nil { t.Fatalf("Unexpected error: %v", err) } // This should succeed. m := nats.NewMsg("KV.33") m.Header.Set(JSExpectedLastSubjSeq, "0") if _, err := js.PublishMsg(m); err != nil { t.Fatalf("Unexpected error: %v", err) } // This should fail. m = nats.NewMsg("KV.22") m.Header.Set(JSExpectedLastSubjSeq, "0") if _, err := js.PublishMsg(m); err == nil { t.Fatalf("Expected error: %v", err) } if err := js.PurgeStream("KV"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } // This should succeed now. if _, err := js.PublishMsg(m); err != nil { t.Fatalf("Unexpected error: %v", err) } } func TestJetStreamPullLargeBatchExpired(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) if err != nil { t.Fatalf("add stream failed: %s", err) } sub, err := js.PullSubscribe("foo", "dlc", nats.PullMaxWaiting(10), nats.MaxAckPending(10*50_000_000)) if err != nil { t.Fatalf("Error creating pull subscriber: %v", err) } // Queue up 10 batch requests with timeout. rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") req := &JSApiConsumerGetNextRequest{Batch: 50_000_000, Expires: 100 * time.Millisecond} jreq, _ := json.Marshal(req) for i := 0; i < 10; i++ { nc.PublishRequest(rsubj, "bar", jreq) } nc.Flush() // Let them all expire. time.Sleep(150 * time.Millisecond) // Now do another and measure how long to timeout and shutdown the server. start := time.Now() sub.Fetch(1, nats.MaxWait(100*time.Millisecond)) s.Shutdown() if delta := time.Since(start); delta > 200*time.Millisecond { t.Fatalf("Took too long to expire: %v", delta) } } func TestJetStreamNegativeDupeWindow(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // we incorrectly set MaxAge to -1 which then as a side effect sets dupe window to -1 which should fail _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: nil, Retention: nats.WorkQueuePolicy, MaxConsumers: 1, MaxMsgs: -1, MaxBytes: -1, Discard: nats.DiscardNew, MaxAge: -1, MaxMsgsPerSubject: -1, MaxMsgSize: -1, Storage: nats.FileStorage, Replicas: 1, NoAck: false, }) if err == nil || err.Error() != "nats: duplicates window can not be negative" { t.Fatalf("Expected dupe window error got: %v", err) } } // Issue #2551 func TestJetStreamMirroredConsumerFailAfterRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "S1", Storage: nats.FileStorage, Subjects: []string{"foo", "bar", "baz"}, }) if err != nil { t.Fatalf("create failed: %s", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "M1", Storage: nats.FileStorage, Mirror: &nats.StreamSource{Name: "S1"}, }) if err != nil { t.Fatalf("create failed: %s", err) } _, err = js.AddConsumer("M1", &nats.ConsumerConfig{ Durable: "C1", FilterSubject: ">", AckPolicy: nats.AckExplicitPolicy, }) if err != nil { t.Fatalf("consumer create failed: %s", err) } // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() s.WaitForShutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() _, err = js.StreamInfo("M1") if err != nil { t.Fatalf("%s did not exist after start: %s", "M1", err) } _, err = js.ConsumerInfo("M1", "C1") if err != nil { t.Fatalf("C1 did not exist after start: %s", err) } } func TestJetStreamDisabledLimitsEnforcementJWT(t *testing.T) { updateJwt := func(url string, akp nkeys.KeyPair, pubKey string, jwt string) { t.Helper() c := natsConnect(t, url, createUserCreds(t, nil, akp)) defer c.Close() if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { t.Fatal("error not expected in this test", err) } else { content := make(map[string]any) if err := json.Unmarshal(msg.Data, &content); err != nil { t.Fatalf("%v", err) } else if _, ok := content["data"]; !ok { t.Fatalf("did not get an ok response got: %v", content) } } } // create system account sysKp, _ := nkeys.CreateAccount() sysPub, _ := sysKp.PublicKey() // limits to apply and check limits1 := jwt.JetStreamLimits{MemoryStorage: 1024, DiskStorage: 0, Streams: 1, Consumer: 2} akp, _ := nkeys.CreateAccount() aPub, _ := akp.PublicKey() claim := jwt.NewAccountClaims(aPub) claim.Limits.JetStreamLimits = limits1 aJwt1, err := claim.Encode(oKp) require_NoError(t, err) dir := t.TempDir() storeDir1 := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: -1 jetstream: {store_dir: '%s'} operator: %s resolver: { type: full dir: '%s' } system_account: %s `, storeDir1, ojwt, dir, sysPub))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() updateJwt(s.ClientURL(), sysKp, aPub, aJwt1) c := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp), nats.ReconnectWait(200*time.Millisecond)) defer c.Close() // keep using the same connection js, err := c.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "disk", Storage: nats.FileStorage, Subjects: []string{"disk"}, }) require_Error(t, err) } func TestJetStreamDisabledLimitsEnforcement(t *testing.T) { storeDir1 := t.TempDir() conf1 := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} accounts { one { jetstream: { mem: 1024 disk: 0 streams: 1 consumers: 2 } users [{user: one, password: password}] } } no_auth_user: one `, storeDir1))) s, _ := RunServerWithConfig(conf1) defer s.Shutdown() c := natsConnect(t, s.ClientURL()) defer c.Close() // keep using the same connection js, err := c.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "disk", Storage: nats.FileStorage, Subjects: []string{"disk"}, }) require_Error(t, err) } func TestJetStreamConsumerNoMsgPayload(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "S"}) require_NoError(t, err) msg := nats.NewMsg("S") msg.Header.Set("name", "derek") msg.Data = bytes.Repeat([]byte("A"), 128) for i := 0; i < 10; i++ { msg.Reply = _EMPTY_ // Fixed in Go client but not in embdedded on yet. _, err = js.PublishMsgAsync(msg) require_NoError(t, err) } mset, err := s.GlobalAccount().lookupStream("S") require_NoError(t, err) // Now create our consumer with no payload option. _, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: "_d_", Durable: "d22", HeadersOnly: true}) require_NoError(t, err) sub, err := js.SubscribeSync("S", nats.Durable("d22")) require_NoError(t, err) for i := 0; i < 10; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) if len(m.Data) > 0 { t.Fatalf("Expected no payload") } if ms := m.Header.Get(JSMsgSize); ms != "128" { t.Fatalf("Expected a header with msg size, got %q", ms) } } } // Issue #2607 func TestJetStreamPurgeAndFilteredConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "S", Subjects: []string{"FOO.*"}}) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish("FOO.adam", []byte("M")) require_NoError(t, err) _, err = js.Publish("FOO.eve", []byte("F")) require_NoError(t, err) } ci, err := js.AddConsumer("S", &nats.ConsumerConfig{ Durable: "adam", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "FOO.adam", }) require_NoError(t, err) if ci.NumPending != 10 { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } ci, err = js.AddConsumer("S", &nats.ConsumerConfig{ Durable: "eve", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "FOO.eve", }) require_NoError(t, err) if ci.NumPending != 10 { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } // Also check unfiltered with interleaving messages. _, err = js.AddConsumer("S", &nats.ConsumerConfig{ Durable: "all", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // Now purge only adam. jr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "FOO.adam"}) _, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "S"), jr, time.Second) require_NoError(t, err) si, err := js.StreamInfo("S") require_NoError(t, err) if si.State.Msgs != 10 { t.Fatalf("Expected 10 messages after purge, got %d", si.State.Msgs) } ci, err = js.ConsumerInfo("S", "eve") require_NoError(t, err) if ci.NumPending != 10 { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } ci, err = js.ConsumerInfo("S", "adam") require_NoError(t, err) if ci.NumPending != 0 { t.Fatalf("Expected NumPending to be 0, got %d", ci.NumPending) } if ci.AckFloor.Stream != 20 { t.Fatalf("Expected AckFloor for stream to be 20, got %d", ci.AckFloor.Stream) } ci, err = js.ConsumerInfo("S", "all") require_NoError(t, err) if ci.NumPending != 10 { t.Fatalf("Expected NumPending to be 10, got %d", ci.NumPending) } } // Issue #2662 func TestJetStreamLargeExpiresAndServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() maxAge := 2 * time.Second _, err := js.AddStream(&nats.StreamConfig{ Name: "S", Subjects: []string{"foo"}, MaxAge: maxAge, }) require_NoError(t, err) start := time.Now() _, err = js.Publish("foo", []byte("ok")) require_NoError(t, err) // Wait total of maxAge - 1s. time.Sleep(maxAge - time.Since(start) - time.Second) // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { si, err := js.StreamInfo("S") require_NoError(t, err) if si.State.Msgs != 0 { return fmt.Errorf("Expected no messages, got %d", si.State.Msgs) } return nil }) if waited := time.Since(start); waited > maxAge+time.Second { t.Fatalf("Waited to long %v vs %v for messages to expire", waited, maxAge) } } // Bug that was reported showing memstore not handling max per subject of 1. func TestJetStreamMessagePerSubjectKeepBug(t *testing.T) { test := func(t *testing.T, keep int64, store nats.StorageType) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", MaxMsgsPerSubject: keep, Storage: store, }) require_NoError(t, err) for i := 0; i < 100; i++ { _, err = js.Publish("TEST", []byte(fmt.Sprintf("test %d", i))) require_NoError(t, err) } nfo, err := js.StreamInfo("TEST") require_NoError(t, err) if nfo.State.Msgs != uint64(keep) { t.Fatalf("Expected %d message got %d", keep, nfo.State.Msgs) } } t.Run("FileStore", func(t *testing.T) { t.Run("Keep 10", func(t *testing.T) { test(t, 10, nats.FileStorage) }) t.Run("Keep 1", func(t *testing.T) { test(t, 1, nats.FileStorage) }) }) t.Run("MemStore", func(t *testing.T) { t.Run("Keep 10", func(t *testing.T) { test(t, 10, nats.MemoryStorage) }) t.Run("Keep 1", func(t *testing.T) { test(t, 1, nats.MemoryStorage) }) }) } func TestJetStreamInvalidDeliverSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{DeliverSubject: " x"}) require_Error(t, err, NewJSConsumerInvalidDeliverSubjectError()) } func TestJetStreamMemoryCorruption(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() errCh := make(chan error, 10) nc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) { select { case errCh <- e: default: } }) // The storage has to be MemoryStorage to show the issue kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "bucket", Storage: nats.MemoryStorage}) require_NoError(t, err) w1, err := kv.WatchAll() require_NoError(t, err) w2, err := kv.WatchAll(nats.MetaOnly()) require_NoError(t, err) kv.Put("key1", []byte("aaa")) kv.Put("key1", []byte("aab")) kv.Put("key2", []byte("zza")) kv.Put("key2", []byte("zzb")) kv.Delete("key1") kv.Delete("key2") kv.Put("key1", []byte("aac")) kv.Put("key2", []byte("zzc")) kv.Delete("key1") kv.Delete("key2") kv.Purge("key1") kv.Purge("key2") checkUpdates := func(updates <-chan nats.KeyValueEntry) { t.Helper() count := 0 for { select { case <-updates: count++ if count == 13 { return } case <-time.After(time.Second): t.Fatal("Did not receive all updates") } } } checkUpdates(w1.Updates()) checkUpdates(w2.Updates()) select { case e := <-errCh: t.Fatal(e) case <-time.After(250 * time.Millisecond): // OK } } func TestJetStreamRecoverBadStreamSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) sd := s.JetStreamConfig().StoreDir s.Shutdown() f := filepath.Join(sd, "$G", "streams", "TEST") fs, err := newFileStore(FileStoreConfig{StoreDir: f}, StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", " baz "}, // baz has spaces Storage: FileStorage, }) require_NoError(t, err) fs.Stop() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("TEST") require_NoError(t, err) if len(si.Config.Subjects) != 3 { t.Fatalf("Expected to recover all subjects") } } func TestJetStreamRecoverBadMirrorConfigWithSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sd := s.JetStreamConfig().StoreDir // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "S", Subjects: []string{"foo"}, }) require_NoError(t, err) s.Shutdown() f := filepath.Join(sd, "$G", "streams", "M") fs, err := newFileStore(FileStoreConfig{StoreDir: f}, StreamConfig{ Name: "M", Subjects: []string{"foo", "bar", "baz"}, // Mirrors should not have spaces. Mirror: &StreamSource{Name: "S"}, Storage: FileStorage, }) require_NoError(t, err) fs.Stop() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("M") require_NoError(t, err) if len(si.Config.Subjects) != 0 { t.Fatalf("Expected to have NO subjects on mirror") } } func TestJetStreamCrossAccountsDeliverSubjectInterest(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q} accounts: { A: { jetstream: enabled users: [ {user: a, password: pwd} ] exports [ { stream: "_d_" } # For the delivery subject for the consumer ] }, B: { users: [ {user: b, password: pwd} ] imports [ { stream: { account: A, subject: "_d_"}, to: "foo" } ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserInfo("a", "pwd")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) msg, toSend := []byte("OK"), 100 for i := 0; i < toSend; i++ { if _, err := js.PublishAsync("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Now create the consumer as well here manually that we will want to reference from Account B. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", DeliverSubject: "_d_"}) require_NoError(t, err) // Wait to see if the stream import signals to deliver messages with no real subscriber interest. time.Sleep(200 * time.Millisecond) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure we have not delivered any messages based on the import signal alone. if ci.NumPending != uint64(toSend) || ci.Delivered.Consumer != 0 { t.Fatalf("Bad consumer info, looks like we started delivering: %+v", ci) } // Now create interest in the delivery subject through the import on account B. nc, _ = jsClientConnect(t, s, nats.UserInfo("b", "pwd")) defer nc.Close() sub, err := nc.SubscribeSync("foo") require_NoError(t, err) checkSubsPending(t, sub, toSend) ci, err = js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure our consumer info reflects we delivered the messages. if ci.NumPending != 0 || ci.Delivered.Consumer != uint64(toSend) { t.Fatalf("Bad consumer info, looks like we did not deliver: %+v", ci) } } func TestJetStreamPullConsumerRequestCleanup(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "T", Storage: nats.MemoryStorage}) require_NoError(t, err) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) req := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 100 * time.Millisecond} jreq, err := json.Marshal(req) require_NoError(t, err) // Need interest otherwise the requests will be recycled based on that. _, err = nc.SubscribeSync("xx") require_NoError(t, err) // Queue up 100 requests. rsubj := fmt.Sprintf(JSApiRequestNextT, "T", "dlc") for i := 0; i < 100; i++ { err = nc.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) } // Wait to expire time.Sleep(200 * time.Millisecond) ci, err := js.ConsumerInfo("T", "dlc") require_NoError(t, err) if ci.NumWaiting != 0 { t.Fatalf("Expected to see no waiting requests, got %d", ci.NumWaiting) } } func TestJetStreamPullConsumerRequestMaximums(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() // Need to do this via server for now. acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{Name: "TEST", Storage: MemoryStorage}) require_NoError(t, err) _, err = mset.addConsumer(&ConsumerConfig{ Durable: "dlc", MaxRequestBatch: 10, MaxRequestMaxBytes: 10_000, MaxRequestExpires: time.Second, AckPolicy: AckExplicit, }) require_NoError(t, err) genReq := func(b, mb int, e time.Duration) []byte { req := &JSApiConsumerGetNextRequest{Batch: b, Expires: e, MaxBytes: mb} jreq, err := json.Marshal(req) require_NoError(t, err) return jreq } rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") // Exceeds max batch size. resp, err := nc.Request(rsubj, genReq(11, 0, 100*time.Millisecond), time.Second) require_NoError(t, err) if status := resp.Header.Get("Status"); status != "409" { t.Fatalf("Expected a 409 status code, got %q", status) } // Exceeds max expires. resp, err = nc.Request(rsubj, genReq(1, 0, 10*time.Minute), time.Second) require_NoError(t, err) if status := resp.Header.Get("Status"); status != "409" { t.Fatalf("Expected a 409 status code, got %q", status) } // Exceeds max bytes. resp, err = nc.Request(rsubj, genReq(10, 10_000*2, 10*time.Minute), time.Second) require_NoError(t, err) if status := resp.Header.Get("Status"); status != "409" { t.Fatalf("Expected a 409 status code, got %q", status) } } func TestJetStreamEphemeralPullConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "EC", Storage: nats.MemoryStorage}) require_NoError(t, err) ci, err := js.AddConsumer("EC", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) mset, err := s.GlobalAccount().lookupStream("EC") require_NoError(t, err) o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } err = o.setInActiveDeleteThreshold(50 * time.Millisecond) require_NoError(t, err) time.Sleep(100 * time.Millisecond) // Should no longer be around. if o := mset.lookupConsumer(ci.Name); o != nil { t.Fatalf("Expected consumer to be closed and removed") } // Make sure timer keeps firing etc. and does not delete until interest is gone. ci, err = js.AddConsumer("EC", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) if o = mset.lookupConsumer(ci.Name); o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } err = o.setInActiveDeleteThreshold(50 * time.Millisecond) require_NoError(t, err) // Need interest otherwise the requests will be recycled based on no real interest. sub, err := nc.SubscribeSync("xx") require_NoError(t, err) req := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 250 * time.Millisecond} jreq, err := json.Marshal(req) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "EC", ci.Name) err = nc.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) nc.Flush() time.Sleep(100 * time.Millisecond) // Should still be alive here. if o := mset.lookupConsumer(ci.Name); o == nil { t.Fatalf("Expected consumer to still be active") } // Remove interest. sub.Unsubscribe() // Make sure this EPC goes away now. checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { if o := mset.lookupConsumer(ci.Name); o != nil { return fmt.Errorf("Consumer still present") } return nil }) } func TestJetStreamEphemeralPullConsumersInactiveThresholdAndNoWait(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "ECIT", Storage: nats.MemoryStorage}) require_NoError(t, err) ci, err := js.AddConsumer("ECIT", &nats.ConsumerConfig{ AckPolicy: nats.AckExplicitPolicy, InactiveThreshold: 100 * time.Millisecond, }) require_NoError(t, err) // Send 10 no_wait requests every 25ms and consumer should still be present. req := &JSApiConsumerGetNextRequest{Batch: 10, NoWait: true} jreq, err := json.Marshal(req) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "ECIT", ci.Name) for i := 0; i < 10; i++ { err = nc.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) nc.Flush() time.Sleep(25 * time.Millisecond) } _, err = js.ConsumerInfo("ECIT", ci.Name) require_NoError(t, err) } func TestJetStreamPullConsumerCrossAccountExpires(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q} accounts: { JS: { jetstream: enabled users: [ {user: dlc, password: foo} ] exports [ { service: "$JS.API.CONSUMER.MSG.NEXT.>", response: stream } ] }, IU: { users: [ {user: mh, password: bar} ] imports [ { service: { subject: "$JS.API.CONSUMER.MSG.NEXT.*.*", account: JS } }] # Re-export for dasiy chain test. exports [ { service: "$JS.API.CONSUMER.MSG.NEXT.>", response: stream } ] }, IU2: { users: [ {user: ik, password: bar} ] imports [ { service: { subject: "$JS.API.CONSUMER.MSG.NEXT.*.*", account: IU } } ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Connect to JS account and create stream, put some messages into it. nc, js := jsClientConnect(t, s, nats.UserInfo("dlc", "foo")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "PC", Subjects: []string{"foo"}}) require_NoError(t, err) toSend := 50 for i := 0; i < toSend; i++ { _, err := js.Publish("foo", []byte("OK")) require_NoError(t, err) } // Now create pull consumer. _, err = js.AddConsumer("PC", &nats.ConsumerConfig{Durable: "PC", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Now access from the importing account. nc2, _ := jsClientConnect(t, s, nats.UserInfo("mh", "bar")) defer nc2.Close() // Make sure batch request works properly with stream response. req := &JSApiConsumerGetNextRequest{Batch: 10} jreq, err := json.Marshal(req) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "PC", "PC") // Make sure we can get a batch correctly etc. // This requires response stream above in the export definition. sub, err := nc2.SubscribeSync("xx") require_NoError(t, err) err = nc2.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) checkSubsPending(t, sub, 10) // Now let's queue up a bunch of requests and then delete interest to make sure the system // removes those requests. // Purge stream err = js.PurgeStream("PC") require_NoError(t, err) // Queue up 10 requests for i := 0; i < 10; i++ { err = nc2.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) } // Since using different connection, flush to make sure processed. nc2.Flush() ci, err := js.ConsumerInfo("PC", "PC") require_NoError(t, err) if ci.NumWaiting != 10 { t.Fatalf("Expected to see 10 waiting requests, got %d", ci.NumWaiting) } // Now remove interest and make sure requests are removed. sub.Unsubscribe() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { ci, err := js.ConsumerInfo("PC", "PC") require_NoError(t, err) if ci.NumWaiting != 0 { return fmt.Errorf("Requests still present") } return nil }) // Now let's test that ephemerals will go away as well when interest etc is no longer around. ci, err = js.AddConsumer("PC", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Set the inactivity threshold by hand for now. jsacc, err := s.LookupAccount("JS") require_NoError(t, err) mset, err := jsacc.lookupStream("PC") require_NoError(t, err) o := mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } err = o.setInActiveDeleteThreshold(50 * time.Millisecond) require_NoError(t, err) rsubj = fmt.Sprintf(JSApiRequestNextT, "PC", ci.Name) sub, err = nc2.SubscribeSync("zz") require_NoError(t, err) err = nc2.PublishRequest(rsubj, "zz", jreq) require_NoError(t, err) // Wait past inactive threshold. time.Sleep(100 * time.Millisecond) // Make sure it is still there.. ci, err = js.ConsumerInfo("PC", ci.Name) require_NoError(t, err) if ci.NumWaiting != 1 { t.Fatalf("Expected to see 1 waiting request, got %d", ci.NumWaiting) } // Now release interest. sub.Unsubscribe() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { _, err := js.ConsumerInfo("PC", ci.Name) if err == nil { return fmt.Errorf("Consumer still present") } return nil }) // Now test daisy chained. toSend = 10 for i := 0; i < toSend; i++ { _, err := js.Publish("foo", []byte("OK")) require_NoError(t, err) } ci, err = js.AddConsumer("PC", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Set the inactivity threshold by hand for now. o = mset.lookupConsumer(ci.Name) if o == nil { t.Fatalf("Error looking up consumer %q", ci.Name) } // Make this one longer so we test request purge and ephemerals in same test. err = o.setInActiveDeleteThreshold(500 * time.Millisecond) require_NoError(t, err) // Now access from the importing account. nc3, _ := jsClientConnect(t, s, nats.UserInfo("ik", "bar")) defer nc3.Close() sub, err = nc3.SubscribeSync("yy") require_NoError(t, err) rsubj = fmt.Sprintf(JSApiRequestNextT, "PC", ci.Name) err = nc3.PublishRequest(rsubj, "yy", jreq) require_NoError(t, err) checkSubsPending(t, sub, 10) // Purge stream err = js.PurgeStream("PC") require_NoError(t, err) // Queue up 10 requests for i := 0; i < 10; i++ { err = nc3.PublishRequest(rsubj, "yy", jreq) require_NoError(t, err) } // Since using different connection, flush to make sure processed. nc3.Flush() ci, err = js.ConsumerInfo("PC", ci.Name) require_NoError(t, err) if ci.NumWaiting != 10 { t.Fatalf("Expected to see 10 waiting requests, got %d", ci.NumWaiting) } // Now remove interest and make sure requests are removed. sub.Unsubscribe() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { ci, err := js.ConsumerInfo("PC", ci.Name) require_NoError(t, err) if ci.NumWaiting != 0 { return fmt.Errorf("Requests still present") } return nil }) // Now make sure the ephemeral goes away too. // Ephemerals have jitter by default of up to 1s. checkFor(t, 6*time.Second, 10*time.Millisecond, func() error { _, err := js.ConsumerInfo("PC", ci.Name) if err == nil { return fmt.Errorf("Consumer still present") } return nil }) } func TestJetStreamPullConsumerCrossAccountExpiresNoDataRace(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q} accounts: { JS: { jetstream: enabled users: [ {user: dlc, password: foo} ] exports [ { service: "$JS.API.CONSUMER.MSG.NEXT.>", response: stream } ] }, IU: { jetstream: enabled users: [ {user: ik, password: bar} ] imports [ { service: { subject: "$JS.API.CONSUMER.MSG.NEXT.*.*", account: JS } }] }, } `, t.TempDir()))) test := func() { s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Connect to JS account and create stream, put some messages into it. nc, js := jsClientConnect(t, s, nats.UserInfo("dlc", "foo")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "PC", Subjects: []string{"foo"}}) require_NoError(t, err) toSend := 100 for i := 0; i < toSend; i++ { _, err := js.Publish("foo", []byte("OK")) require_NoError(t, err) } // Create pull consumer. _, err = js.AddConsumer("PC", &nats.ConsumerConfig{Durable: "PC", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Now access from the importing account. nc2, _ := jsClientConnect(t, s, nats.UserInfo("ik", "bar")) defer nc2.Close() req := &JSApiConsumerGetNextRequest{Batch: 1} jreq, err := json.Marshal(req) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "PC", "PC") sub, err := nc2.SubscribeSync("xx") require_NoError(t, err) wg := sync.WaitGroup{} wg.Add(1) go func() { time.Sleep(5 * time.Millisecond) sub.Unsubscribe() wg.Done() }() for i := 0; i < toSend; i++ { nc2.PublishRequest(rsubj, "xx", jreq) } wg.Wait() } // Need to rerun this test several times to get the race (which then would possible be panic // such as: "fatal error: concurrent map read and map write" for iter := 0; iter < 10; iter++ { test() } } // This tests account export/import replies across a LN connection with account import/export // on both sides of the LN. func TestJetStreamPullConsumerCrossAccountsAndLeafNodes(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` server_name: SJS listen: 127.0.0.1:-1 jetstream: {max_mem_store: 4GB, max_file_store: 1TB, domain: JSD, store_dir: %q } accounts: { JS: { jetstream: enabled users: [ {user: dlc, password: foo} ] exports [ { service: "$JS.API.CONSUMER.MSG.NEXT.>", response: stream } ] }, IU: { users: [ {user: mh, password: bar} ] imports [ { service: { subject: "$JS.API.CONSUMER.MSG.NEXT.*.*", account: JS } }] }, } leaf { listen: "127.0.0.1:-1" } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer s.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` server_name: SLN listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: l, password: p} ] exports [ { service: "$JS.JSD.API.CONSUMER.MSG.NEXT.>", response: stream } ] }, B: { users: [ {user: m, password: p} ] imports [ { service: { subject: "$JS.JSD.API.CONSUMER.MSG.NEXT.*.*", account: A } }] }, } # bind local A to IU account on other side of LN. leaf { remotes [ { url: nats://mh:bar@127.0.0.1:%d; account: A } ] } `, o.LeafNode.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnectedCount(t, s, 1) // Connect to JS account, create stream and consumer and put in some messages. nc, js := jsClientConnect(t, s, nats.UserInfo("dlc", "foo")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "PC", Subjects: []string{"foo"}}) require_NoError(t, err) toSend := 10 for i := 0; i < toSend; i++ { _, err := js.Publish("foo", []byte("OK")) require_NoError(t, err) } // Now create durable pull consumer. _, err = js.AddConsumer("PC", &nats.ConsumerConfig{Durable: "PC", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Now access from the account on the leafnode, so importing on both sides and crossing a leafnode connection. nc2, _ := jsClientConnect(t, s2, nats.UserInfo("m", "p")) defer nc2.Close() req := &JSApiConsumerGetNextRequest{Batch: toSend, Expires: 500 * time.Millisecond} jreq, err := json.Marshal(req) require_NoError(t, err) // Make sure we can get a batch correctly etc. // This requires response stream above in the export definition. sub, err := nc2.SubscribeSync("xx") require_NoError(t, err) rsubj := "$JS.JSD.API.CONSUMER.MSG.NEXT.PC.PC" err = nc2.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) checkSubsPending(t, sub, 10) // Queue up a bunch of requests. for i := 0; i < 10; i++ { err = nc2.PublishRequest(rsubj, "xx", jreq) require_NoError(t, err) } checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { ci, err := js.ConsumerInfo("PC", "PC") require_NoError(t, err) if ci.NumWaiting != 10 { return fmt.Errorf("Expected to see 10 waiting requests, got %d", ci.NumWaiting) } return nil }) // Remove interest. sub.Unsubscribe() // Make sure requests go away eventually after they expire. checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { ci, err := js.ConsumerInfo("PC", "PC") require_NoError(t, err) if ci.NumWaiting != 0 { return fmt.Errorf("Expected to see no waiting requests, got %d", ci.NumWaiting) } return nil }) } // This test is to explicitly test for all combinations of pull consumer behavior. // 1. Long poll, will be used to emulate push. A request is only invalidated when batch is filled, it expires, or we lose interest. // 2. Batch 1, will return no messages or a message. Works today. // 3. Conditional wait, or one shot. This is what the clients do when the do a fetch(). // They expect to wait up to a given time for any messages but will return once they have any to deliver, so parital fills. // 4. Try, which never waits at all ever. func TestJetStreamPullConsumersOneShotBehavior(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", }) require_NoError(t, err) // We will do low level requests by hand for this test as to not depend on any client impl. rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") getNext := func(batch int, expires time.Duration, noWait bool) (numMsgs int, elapsed time.Duration, hdr *nats.Header) { t.Helper() req := &JSApiConsumerGetNextRequest{Batch: batch, Expires: expires, NoWait: noWait} jreq, err := json.Marshal(req) require_NoError(t, err) // Create listener. reply, msgs := nats.NewInbox(), make(chan *nats.Msg, batch) sub, err := nc.ChanSubscribe(reply, msgs) require_NoError(t, err) defer sub.Unsubscribe() // Send request. start := time.Now() err = nc.PublishRequest(rsubj, reply, jreq) require_NoError(t, err) for { select { case m := <-msgs: if len(m.Data) == 0 && m.Header != nil { return numMsgs, time.Since(start), &m.Header } numMsgs++ if numMsgs >= batch { return numMsgs, time.Since(start), nil } case <-time.After(expires + 250*time.Millisecond): t.Fatalf("Did not receive all the msgs in time") } } } expect := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header, lt time.Duration, gt time.Duration) { t.Helper() n, e, h := getNext(batch, expires, noWait) if n != ne { t.Fatalf("Expected %d msgs, got %d", ne, n) } if !reflect.DeepEqual(h, he) { t.Fatalf("Expected %+v hdr, got %+v", he, h) } if lt > 0 && e > lt { t.Fatalf("Expected elapsed of %v to be less than %v", e, lt) } if gt > 0 && e < gt { t.Fatalf("Expected elapsed of %v to be greater than %v", e, gt) } } expectAfter := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header, gt time.Duration) { t.Helper() expect(batch, expires, noWait, ne, he, 0, gt) } expectInstant := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header) { t.Helper() expect(batch, expires, noWait, ne, he, 5*time.Millisecond, 0) } expectOK := func(batch int, expires time.Duration, noWait bool, ne int) { t.Helper() expectInstant(batch, expires, noWait, ne, nil) } noMsgs := &nats.Header{"Status": []string{"404"}, "Description": []string{"No Messages"}} reqTimeout := &nats.Header{"Status": []string{"408"}, "Description": []string{"Request Timeout"}, "Nats-Pending-Bytes": []string{"0"}, "Nats-Pending-Messages": []string{"1"}} // We are empty here, meaning no messages available. // Do not wait, should get noMsgs. expectInstant(1, 0, true, 0, noMsgs) // We should wait here for the full second. expectAfter(1, 250*time.Millisecond, false, 0, reqTimeout, 250*time.Millisecond) // This should also wait since no messages are available. This is the one shot scenario, or wait for at least a message if none are there. expectAfter(1, 500*time.Millisecond, true, 0, reqTimeout, 500*time.Millisecond) // Now let's put some messages into the system. for i := 0; i < 20; i++ { _, err := js.Publish("foo", []byte("HELLO")) require_NoError(t, err) } // Now run same 3 scenarios. expectOK(1, 0, true, 1) expectOK(5, 500*time.Millisecond, false, 5) expectOK(5, 500*time.Millisecond, true, 5) } func TestJetStreamPullConsumersMultipleRequestsExpireOutOfOrder(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", }) require_NoError(t, err) // We will now queue up 4 requests. All should expire but they will do so out of order. // We want to make sure we get them in correct order. rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") sub, err := nc.SubscribeSync("i.*") require_NoError(t, err) defer sub.Unsubscribe() for _, expires := range []time.Duration{200, 100, 25, 75} { reply := fmt.Sprintf("i.%d", expires) req := &JSApiConsumerGetNextRequest{Expires: expires * time.Millisecond} jreq, err := json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(rsubj, reply, jreq) require_NoError(t, err) } start := time.Now() checkSubsPending(t, sub, 4) elapsed := time.Since(start) if elapsed < 200*time.Millisecond || elapsed > 500*time.Millisecond { t.Fatalf("Expected elapsed to be close to %v, but got %v", 200*time.Millisecond, elapsed) } var rs []string for i := 0; i < 4; i++ { m, err := sub.NextMsg(0) require_NoError(t, err) rs = append(rs, m.Subject) } if expected := []string{"i.25", "i.75", "i.100", "i.200"}; !reflect.DeepEqual(rs, expected) { t.Fatalf("Received in wrong order, wanted %+v, got %+v", expected, rs) } } func TestJetStreamConsumerUpdateSurvival(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "X"}) require_NoError(t, err) // First create a consumer with max ack pending. _, err = js.AddConsumer("X", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 1024, }) require_NoError(t, err) // Now do same name but pull. This will update the MaxAcKPending ci, err := js.UpdateConsumer("X", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 22, }) require_NoError(t, err) if ci.Config.MaxAckPending != 22 { t.Fatalf("Expected MaxAckPending to be 22, got %d", ci.Config.MaxAckPending) } // Make sure this survives across a restart. sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() ci, err = js.ConsumerInfo("X", "dlc") require_NoError(t, err) if ci.Config.MaxAckPending != 22 { t.Fatalf("Expected MaxAckPending to be 22, got %d", ci.Config.MaxAckPending) } } func TestJetStreamNakRedeliveryWithNoWait(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("NAK")) require_NoError(t, err) ccReq := &CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ Durable: "dlc", AckPolicy: AckExplicit, MaxDeliver: 3, AckWait: time.Minute, BackOff: []time.Duration{5 * time.Second, 10 * time.Second}, }, } // Do by hand for now until Go client catches up. req, err := json.Marshal(ccReq) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, "TEST", "dlc"), req, time.Second) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected error: %+v", ccResp.Error) } rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") m, err := nc.Request(rsubj, nil, time.Second) require_NoError(t, err) // NAK this message. delay, err := json.Marshal(&ConsumerNakOptions{Delay: 500 * time.Millisecond}) require_NoError(t, err) dnak := []byte(fmt.Sprintf("%s %s", AckNak, delay)) m.Respond(dnak) // This message should come back to us after 500ms. If we do a one-shot request, with NoWait and Expires // this will do the right thing and we get the message. // What we want to test here is a true NoWait request with Expires==0 and eventually seeing the message be redelivered. expires := time.Now().Add(time.Second) for time.Now().Before(expires) { m, err = nc.Request(rsubj, []byte(`{"batch":1, "no_wait": true}`), time.Second) require_NoError(t, err) if len(m.Data) > 0 { // We got our message, so we are good. return } // So we do not spin. time.Sleep(100 * time.Millisecond) } t.Fatalf("Did not get the message in time") } // Test that we properly enforce per subject msg limits when DiscardNew is set. // DiscardNew should only apply to stream limits, subject based limits should always be DiscardOld. func TestJetStreamMaxMsgsPerSubjectWithDiscardNew(t *testing.T) { msc := StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar", "baz", "x"}, Discard: DiscardNew, Storage: MemoryStorage, MaxMsgsPer: 4, MaxMsgs: 10, MaxBytes: 500, } fsc := msc fsc.Storage = FileStorage cases := []struct { name string mconfig *StreamConfig }{ {"MemoryStore", &msc}, {"FileStore", &fsc}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(c.mconfig) require_NoError(t, err) defer mset.delete() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() pubAndCheck := func(subj string, num int, expectedNumMsgs uint64) { t.Helper() for i := 0; i < num; i++ { _, err = js.Publish(subj, []byte("TSLA")) require_NoError(t, err) } si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != expectedNumMsgs { t.Fatalf("Expected %d msgs, got %d", expectedNumMsgs, si.State.Msgs) } } pubExpectErr := func(subj string, sz int) { t.Helper() _, err = js.Publish(subj, bytes.Repeat([]byte("X"), sz)) require_Error(t, err, errors.New("nats: maximum bytes exceeded"), errors.New("nats: maximum messages exceeded")) } pubAndCheck("foo", 1, 1) // We should treat this as DiscardOld and only have 4 msgs after. pubAndCheck("foo", 4, 4) // Same thing here, shoud only have 4 foo and 4 bar for total of 8. pubAndCheck("bar", 8, 8) // We have 8 here, so only 2 left. If we add in a new subject when we have room it will be accepted. pubAndCheck("baz", 2, 10) // Now we are full, but makeup is foo-4 bar-4 baz-2. // We can add to foo and bar since they are at their max and adding new ones there stays the same in terms of total of 10. pubAndCheck("foo", 1, 10) pubAndCheck("bar", 1, 10) // Try to send a large message under an established subject that will exceed the 500 maximum. // Even though we have a bar subject and its at its maximum, the message to be dropped is not big enough, so this should err. pubExpectErr("bar", 300) // Also even though we have room bytes wise, if we introduce a new subject this should fail too on msg limit exceeded. pubExpectErr("x", 2) }) } } func TestJetStreamStreamInfoSubjectsDetails(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() getInfo := func(t *testing.T, filter string) *StreamInfo { t.Helper() // Need to grab StreamInfo by hand for now. req, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), req, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) if si.State.NumSubjects != 3 { t.Fatalf("Expected NumSubjects to be 3, but got %d", si.State.NumSubjects) } return &si } testSubjects := func(t *testing.T, st nats.StorageType) { _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Storage: st, }) require_NoError(t, err) defer js.DeleteStream("TEST") counts, msg := []int{22, 33, 44}, []byte("ok") // Now place msgs, foo-22, bar-33 and baz-44. for i, subj := range []string{"foo", "bar", "baz"} { for n := 0; n < counts[i]; n++ { _, err = js.Publish(subj, msg) require_NoError(t, err) } } // Test all subjects first. expected := map[string]uint64{"foo": 22, "bar": 33, "baz": 44} if si := getInfo(t, nats.AllKeys); !reflect.DeepEqual(si.State.Subjects, expected) { t.Fatalf("Expected subjects of %+v, but got %+v", expected, si.State.Subjects) } if si := getInfo(t, "*"); !reflect.DeepEqual(si.State.Subjects, expected) { t.Fatalf("Expected subjects of %+v, but got %+v", expected, si.State.Subjects) } // Filtered to 1. expected = map[string]uint64{"foo": 22} if si := getInfo(t, "foo"); !reflect.DeepEqual(si.State.Subjects, expected) { t.Fatalf("Expected subjects of %+v, but got %+v", expected, si.State.Subjects) } } t.Run("MemoryStore", func(t *testing.T) { testSubjects(t, nats.MemoryStorage) }) t.Run("FileStore", func(t *testing.T) { testSubjects(t, nats.FileStorage) }) } func TestJetStreamStreamInfoSubjectsDetailsWithDeleteAndPurge(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() getInfo := func(t *testing.T, filter string) *StreamInfo { t.Helper() // Need to grab StreamInfo by hand for now. req, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), req, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) return &si } checkResults := func(t *testing.T, expected map[string]uint64) { t.Helper() si := getInfo(t, nats.AllKeys) if !reflect.DeepEqual(si.State.Subjects, expected) { t.Fatalf("Expected subjects of %+v, but got %+v", expected, si.State.Subjects) } if si.State.NumSubjects != len(expected) { t.Fatalf("Expected NumSubjects to be %d, but got %d", len(expected), si.State.NumSubjects) } } testSubjects := func(t *testing.T, st nats.StorageType) { _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, Storage: st, }) require_NoError(t, err) defer js.DeleteStream("TEST") msg := []byte("ok") js.Publish("foo", msg) // 1 js.Publish("foo", msg) // 2 js.Publish("bar", msg) // 3 js.Publish("baz", msg) // 4 js.Publish("baz", msg) // 5 js.Publish("bar", msg) // 6 js.Publish("bar", msg) // 7 checkResults(t, map[string]uint64{"foo": 2, "bar": 3, "baz": 2}) // Now delete some messages. js.DeleteMsg("TEST", 6) checkResults(t, map[string]uint64{"foo": 2, "bar": 2, "baz": 2}) // Delete and add right back, so no-op js.DeleteMsg("TEST", 5) // baz js.Publish("baz", msg) // 8 checkResults(t, map[string]uint64{"foo": 2, "bar": 2, "baz": 2}) // Now do a purge only of bar. jr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "bar"}) _, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "TEST"), jr, time.Second) require_NoError(t, err) checkResults(t, map[string]uint64{"foo": 2, "baz": 2}) // Now purge everything err = js.PurgeStream("TEST") require_NoError(t, err) si := getInfo(t, nats.AllKeys) if len(si.State.Subjects) != 0 { t.Fatalf("Expected no subjects, but got %+v", si.State.Subjects) } if si.State.NumSubjects != 0 { t.Fatalf("Expected NumSubjects to be 0, but got %d", si.State.NumSubjects) } } t.Run("MemoryStore", func(t *testing.T) { testSubjects(t, nats.MemoryStorage) }) t.Run("FileStore", func(t *testing.T) { testSubjects(t, nats.FileStorage) }) } func TestJetStreamStreamInfoSubjectsDetailsAfterRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() getInfo := func(t *testing.T, filter string) *StreamInfo { t.Helper() // Need to grab StreamInfo by hand for now. req, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), req, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) return &si } _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, }) require_NoError(t, err) defer js.DeleteStream("TEST") msg := []byte("ok") js.Publish("foo", msg) // 1 js.Publish("foo", msg) // 2 js.Publish("bar", msg) // 3 js.Publish("baz", msg) // 4 js.Publish("baz", msg) // 5 si := getInfo(t, nats.AllKeys) if si.State.NumSubjects != 3 { t.Fatalf("Expected 3 subjects, but got %d", si.State.NumSubjects) } // Stop current nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, _ = jsClientConnect(t, s) defer nc.Close() si = getInfo(t, nats.AllKeys) if si.State.NumSubjects != 3 { t.Fatalf("Expected 3 subjects, but got %d", si.State.NumSubjects) } } // Issue #2836 func TestJetStreamInterestRetentionBug(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "c1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) test := func(token string, fseq, msgs uint64) { t.Helper() subj := fmt.Sprintf("foo.%s", token) _, err = js.Publish(subj, nil) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.FirstSeq != fseq { t.Fatalf("Expected first to be %d, got %d", fseq, si.State.FirstSeq) } if si.State.Msgs != msgs { t.Fatalf("Expected msgs to be %d, got %d", msgs, si.State.Msgs) } } test("bar", 1, 1) // Create second filtered consumer. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "c2", FilterSubject: "foo.foo", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) test("bar", 1, 2) } // Under load testing for K3S and the KINE interface we saw some stalls. // These were caused by a dynamic that would not send a second FC item with one // pending, but when we sent the next message and got blocked, if that msg would // exceed the outstanding FC we would become stalled. func TestJetStreamFlowControlStall(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "FC"}) require_NoError(t, err) msg := []byte(strings.Repeat("X", 32768)) _, err = js.Publish("FC", msg) require_NoError(t, err) msg = []byte(strings.Repeat("X", 8*32768)) _, err = js.Publish("FC", msg) require_NoError(t, err) _, err = js.Publish("FC", msg) require_NoError(t, err) sub, err := js.SubscribeSync("FC", nats.OrderedConsumer()) require_NoError(t, err) checkSubsPending(t, sub, 3) } func TestJetStreamConsumerPendingCountWithRedeliveries(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "test", AckPolicy: nats.AckExplicitPolicy, AckWait: 50 * time.Millisecond, MaxDeliver: 1, }) require_NoError(t, err) // Publish 1st message _, err = js.Publish("foo", []byte("msg1")) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "test") require_NoError(t, err) msgs, err := sub.Fetch(1) require_NoError(t, err) for _, m := range msgs { require_Equal(t, string(m.Data), "msg1") // Do not ack the message } // Check consumer info, pending should be 0 ci, err := js.ConsumerInfo("TEST", "test") require_NoError(t, err) if ci.NumPending != 0 { t.Fatalf("Expected consumer info pending count to be 0, got %v", ci.NumPending) } // Wait for more than expiration time.Sleep(100 * time.Millisecond) // Publish 2nd message _, err = js.Publish("foo", []byte("msg2")) require_NoError(t, err) msgs, err = sub.Fetch(1) require_NoError(t, err) for _, m := range msgs { require_Equal(t, string(m.Data), "msg2") // Its deliver count should be 1 meta, err := m.Metadata() require_NoError(t, err) if meta.NumDelivered != 1 { t.Fatalf("Expected message's deliver count to be 1, got %v", meta.NumDelivered) } } // Check consumer info, pending should be 0 ci, err = js.ConsumerInfo("TEST", "test") require_NoError(t, err) if ci.NumPending != 0 { t.Fatalf("Expected consumer info pending count to be 0, got %v", ci.NumPending) } } func TestJetStreamPullConsumerHeartBeats(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "T", Storage: nats.MemoryStorage}) require_NoError(t, err) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) rsubj := fmt.Sprintf(JSApiRequestNextT, "T", "dlc") type tsMsg struct { received time.Time msg *nats.Msg } doReq := func(batch int, hb, expires time.Duration, expected int) []*tsMsg { t.Helper() req := &JSApiConsumerGetNextRequest{Batch: batch, Expires: expires, Heartbeat: hb} jreq, err := json.Marshal(req) require_NoError(t, err) reply := nats.NewInbox() var msgs []*tsMsg var mu sync.Mutex sub, err := nc.Subscribe(reply, func(m *nats.Msg) { mu.Lock() msgs = append(msgs, &tsMsg{time.Now(), m}) mu.Unlock() }) require_NoError(t, err) err = nc.PublishRequest(rsubj, reply, jreq) require_NoError(t, err) checkFor(t, time.Second, 50*time.Millisecond, func() error { mu.Lock() nr := len(msgs) mu.Unlock() if nr >= expected { return nil } return fmt.Errorf("Only have seen %d of %d responses", nr, expected) }) sub.Unsubscribe() return msgs } reqBad := nats.Header{"Status": []string{"400"}, "Description": []string{"Bad Request - heartbeat value too large"}} expectErr := func(msgs []*tsMsg) { t.Helper() if len(msgs) != 1 { t.Fatalf("Expected 1 msg, got %d", len(msgs)) } if !reflect.DeepEqual(msgs[0].msg.Header, reqBad) { t.Fatalf("Expected %+v hdr, got %+v", reqBad, msgs[0].msg.Header) } } // Test errors first. // Setting HB with no expires. expectErr(doReq(1, 100*time.Millisecond, 0, 1)) // If HB larger than 50% of expires.. expectErr(doReq(1, 75*time.Millisecond, 100*time.Millisecond, 1)) expectHBs := func(start time.Time, msgs []*tsMsg, expected int, hbi time.Duration) { t.Helper() if len(msgs) != expected { t.Fatalf("Expected %d but got %d", expected, len(msgs)) } // expected -1 should be all HBs. for i, ts := 0, start; i < expected-1; i++ { tr, m := msgs[i].received, msgs[i].msg if m.Header.Get("Status") != "100" { t.Fatalf("Expected a 100 status code, got %q", m.Header.Get("Status")) } if m.Header.Get("Description") != "Idle Heartbeat" { t.Fatalf("Wrong description, got %q", m.Header.Get("Description")) } ts = ts.Add(hbi) if tr.Before(ts) { t.Fatalf("Received at wrong time: %v vs %v", tr, ts) } } // Last msg should be timeout. lm := msgs[len(msgs)-1].msg if key := lm.Header.Get("Status"); key != "408" { t.Fatalf("Expected 408 Request Timeout, got %s", key) } } // These should work. Test idle first. start, msgs := time.Now(), doReq(1, 50*time.Millisecond, 250*time.Millisecond, 5) expectHBs(start, msgs, 5, 50*time.Millisecond) // Now test that we do not send heartbeats while we receive traffic. go func() { for i := 0; i < 5; i++ { time.Sleep(50 * time.Millisecond) js.Publish("T", nil) } }() msgs = doReq(10, 75*time.Millisecond, 350*time.Millisecond, 6) // The first 5 should be msgs, no HBs. for i := 0; i < 5; i++ { if m := msgs[i].msg; len(m.Header) > 0 { t.Fatalf("Got a potential heartbeat msg when we should not have: %+v", m.Header) } } // Last should be timeout. lm := msgs[len(msgs)-1].msg if key := lm.Header.Get("Status"); key != "408" { t.Fatalf("Expected 408 Request Timeout, got %s", key) } } func TestJetStreamStorageReservedBytes(t *testing.T) { const systemLimit = 1024 opts := DefaultTestOptions opts.Port = -1 opts.JetStream = true opts.JetStreamMaxMemory = systemLimit opts.JetStreamMaxStore = systemLimit opts.StoreDir = t.TempDir() opts.HTTPPort = -1 s := RunServer(&opts) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() getJetStreamVarz := func(hc *http.Client, addr string) (JetStreamVarz, error) { resp, err := hc.Get(addr) if err != nil { return JetStreamVarz{}, err } defer resp.Body.Close() var v Varz if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { return JetStreamVarz{}, err } return v.JetStream, nil } getReserved := func(hc *http.Client, addr string, st nats.StorageType) (uint64, error) { jsv, err := getJetStreamVarz(hc, addr) if err != nil { return 0, err } if st == nats.MemoryStorage { return jsv.Stats.ReservedMemory, nil } return jsv.Stats.ReservedStore, nil } varzAddr := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) hc := &http.Client{Timeout: 5 * time.Second} jsv, err := getJetStreamVarz(hc, varzAddr) require_NoError(t, err) if got, want := systemLimit, int(jsv.Config.MaxMemory); got != want { t.Fatalf("Unexpected max memory: got=%d, want=%d", got, want) } if got, want := systemLimit, int(jsv.Config.MaxStore); got != want { t.Fatalf("Unexpected max store: got=%d, want=%d", got, want) } cases := []struct { name string accountLimit int64 storage nats.StorageType createMaxBytes int64 updateMaxBytes int64 wantUpdateError bool }{ { name: "file reserve 66% of system limit", accountLimit: -1, storage: nats.FileStorage, createMaxBytes: int64(math.Round(float64(systemLimit) * .666)), updateMaxBytes: int64(math.Round(float64(systemLimit)*.666)) + 1, }, { name: "memory reserve 66% of system limit", accountLimit: -1, storage: nats.MemoryStorage, createMaxBytes: int64(math.Round(float64(systemLimit) * .666)), updateMaxBytes: int64(math.Round(float64(systemLimit)*.666)) + 1, }, { name: "file update past system limit", accountLimit: -1, storage: nats.FileStorage, createMaxBytes: systemLimit, updateMaxBytes: systemLimit + 1, wantUpdateError: true, }, { name: "memory update past system limit", accountLimit: -1, storage: nats.MemoryStorage, createMaxBytes: systemLimit, updateMaxBytes: systemLimit + 1, wantUpdateError: true, }, { name: "file update to system limit", accountLimit: -1, storage: nats.FileStorage, createMaxBytes: systemLimit - 1, updateMaxBytes: systemLimit, }, { name: "memory update to system limit", accountLimit: -1, storage: nats.MemoryStorage, createMaxBytes: systemLimit - 1, updateMaxBytes: systemLimit, }, { name: "file reserve 66% of account limit", accountLimit: systemLimit / 2, storage: nats.FileStorage, createMaxBytes: int64(math.Round(float64(systemLimit/2) * .666)), updateMaxBytes: int64(math.Round(float64(systemLimit/2)*.666)) + 1, }, { name: "memory reserve 66% of account limit", accountLimit: systemLimit / 2, storage: nats.MemoryStorage, createMaxBytes: int64(math.Round(float64(systemLimit/2) * .666)), updateMaxBytes: int64(math.Round(float64(systemLimit/2)*.666)) + 1, }, { name: "file update past account limit", accountLimit: systemLimit / 2, storage: nats.FileStorage, createMaxBytes: (systemLimit / 2), updateMaxBytes: (systemLimit / 2) + 1, wantUpdateError: true, }, { name: "memory update past account limit", accountLimit: systemLimit / 2, storage: nats.MemoryStorage, createMaxBytes: (systemLimit / 2), updateMaxBytes: (systemLimit / 2) + 1, wantUpdateError: true, }, { name: "file update to account limit", accountLimit: systemLimit / 2, storage: nats.FileStorage, createMaxBytes: (systemLimit / 2) - 1, updateMaxBytes: (systemLimit / 2), }, { name: "memory update to account limit", accountLimit: systemLimit / 2, storage: nats.MemoryStorage, createMaxBytes: (systemLimit / 2) - 1, updateMaxBytes: (systemLimit / 2), }, } for i := 0; i < len(cases) && !t.Failed(); i++ { c := cases[i] t.Run(c.name, func(st *testing.T) { // Setup limits err = s.GlobalAccount().UpdateJetStreamLimits(map[string]JetStreamAccountLimits{ _EMPTY_: { MaxMemory: c.accountLimit, MaxStore: c.accountLimit, }, }) require_NoError(st, err) // Create initial stream cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: c.storage, MaxBytes: c.createMaxBytes, } _, err = js.AddStream(cfg) require_NoError(st, err) // Update stream MaxBytes cfg.MaxBytes = c.updateMaxBytes info, err := js.UpdateStream(cfg) if c.wantUpdateError && err == nil { got := info.Config.MaxBytes st.Fatalf("Unexpected update success, newMaxBytes=%d; systemLimit=%d; accountLimit=%d", got, systemLimit, c.accountLimit) } else if !c.wantUpdateError && err != nil { st.Fatalf("Unexpected update error: %s", err) } if !c.wantUpdateError && err == nil { // If update was successful, then ensure reserved shows new // amount reserved, err := getReserved(hc, varzAddr, c.storage) require_NoError(st, err) if got, want := reserved, uint64(c.updateMaxBytes); got != want { st.Fatalf("Unexpected reserved: %d, want %d", got, want) } } // Delete stream err = js.DeleteStream("TEST") require_NoError(st, err) // Ensure reserved shows 0 because we've deleted the stream reserved, err := getReserved(hc, varzAddr, c.storage) require_NoError(st, err) if reserved != 0 { st.Fatalf("Unexpected reserved: %d, want 0", reserved) } }) } } func TestJetStreamRestoreBadStream(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() var rreq JSApiStreamRestoreRequest buf, err := os.ReadFile("../test/configs/jetstream/restore_bad_stream/backup.json") require_NoError(t, err) err = json.Unmarshal(buf, &rreq) require_NoError(t, err) data, err := os.Open("../test/configs/jetstream/restore_bad_stream/stream.tar.s2") require_NoError(t, err) defer data.Close() var rresp JSApiStreamRestoreResponse msg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, rreq.Config.Name), buf, 5*time.Second) require_NoError(t, err) json.Unmarshal(msg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Error on restore: %+v", rresp.Error) } var chunk [1024]byte for { n, err := data.Read(chunk[:]) if err == io.EOF { break } require_NoError(t, err) msg, err = nc.Request(rresp.DeliverSubject, chunk[:n], 5*time.Second) require_NoError(t, err) json.Unmarshal(msg.Data, &rresp) if rresp.Error != nil { t.Fatalf("Error on restore: %+v", rresp.Error) } } msg, err = nc.Request(rresp.DeliverSubject, nil, 5*time.Second) require_NoError(t, err) json.Unmarshal(msg.Data, &rresp) if rresp.Error == nil || !strings.Contains(rresp.Error.Description, "unexpected") { t.Fatalf("Expected error about unexpected content, got: %+v", rresp.Error) } dir := filepath.Join(s.JetStreamConfig().StoreDir, globalAccountName) f1 := filepath.Join(dir, "fail1.txt") f2 := filepath.Join(dir, "fail2.txt") for _, f := range []string{f1, f2} { if _, err := os.Stat(f); err == nil { t.Fatalf("Found file %s", f) } } } func TestJetStreamConsumerAckSampling(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", SampleFrequency: "100%", }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "dlc") require_NoError(t, err) _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) msub, err := nc.SubscribeSync("$JS.EVENT.METRIC.>") require_NoError(t, err) for _, m := range fetchMsgs(t, sub, 1, time.Second) { err = m.AckSync() require_NoError(t, err) } m, err := msub.NextMsg(time.Second) require_NoError(t, err) var am JSConsumerAckMetric err = json.Unmarshal(m.Data, &am) require_NoError(t, err) if am.Stream != "TEST" || am.Consumer != "dlc" || am.ConsumerSeq != 1 { t.Fatalf("Not a proper ack metric: %+v", am) } // Do less than 100% _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "alps", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", SampleFrequency: "50%", }) require_NoError(t, err) asub, err := js.PullSubscribe("foo", "alps") require_NoError(t, err) total := 500 for i := 0; i < total; i++ { _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) } mp := 0 for _, m := range fetchMsgs(t, asub, total, time.Second) { err = m.AckSync() require_NoError(t, err) mp++ } nc.Flush() if mp != total { t.Fatalf("Got only %d msgs out of %d", mp, total) } nmsgs, _, err := msub.Pending() require_NoError(t, err) // Should be ~250 if nmsgs < 200 || nmsgs > 300 { t.Fatalf("Expected about 250, got %d", nmsgs) } } func TestJetStreamConsumerAckSamplingSpecifiedUsingUpdateConsumer(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", }) require_NoError(t, err) _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", SampleFrequency: "100%", }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "dlc") require_NoError(t, err) _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) msub, err := nc.SubscribeSync("$JS.EVENT.METRIC.>") require_NoError(t, err) for _, m := range fetchMsgs(t, sub, 1, time.Second) { err = m.AckSync() require_NoError(t, err) } m, err := msub.NextMsg(time.Second) require_NoError(t, err) var am JSConsumerAckMetric err = json.Unmarshal(m.Data, &am) require_NoError(t, err) if am.Stream != "TEST" || am.Consumer != "dlc" || am.ConsumerSeq != 1 { t.Fatalf("Not a proper ack metric: %+v", am) } } func TestJetStreamConsumerMaxDeliverUpdate(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) require_NoError(t, err) maxDeliver := 2 _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "ard", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", MaxDeliver: maxDeliver, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "ard") require_NoError(t, err) checkMaxDeliver := func() { t.Helper() for i := 0; i <= maxDeliver; i++ { msgs, err := sub.Fetch(2, nats.MaxWait(100*time.Millisecond)) if i < maxDeliver { require_NoError(t, err) require_Len(t, 1, len(msgs)) _ = msgs[0].Nak() } else { require_Error(t, err, nats.ErrTimeout) } } } _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) checkMaxDeliver() // update maxDeliver maxDeliver++ _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "ard", AckPolicy: nats.AckExplicitPolicy, FilterSubject: "foo", MaxDeliver: maxDeliver, }) require_NoError(t, err) _, err = js.Publish("foo", []byte("Hello")) require_NoError(t, err) checkMaxDeliver() } func TestJetStreamRemoveExternalSource(t *testing.T) { ho := DefaultTestOptions ho.Port = 4000 //-1 ho.LeafNode.Host = "127.0.0.1" ho.LeafNode.Port = -1 hs := RunServer(&ho) defer hs.Shutdown() lu, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ho.LeafNode.Port)) require_NoError(t, err) lo1 := DefaultTestOptions lo1.Port = 4111 //-1 lo1.ServerName = "a-leaf" lo1.JetStream = true lo1.StoreDir = t.TempDir() lo1.JetStreamDomain = "a-leaf" lo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}} l1 := RunServer(&lo1) defer l1.Shutdown() lo2 := DefaultTestOptions lo2.Port = 2111 //-1 lo2.ServerName = "b-leaf" lo2.JetStream = true lo2.StoreDir = t.TempDir() lo2.JetStreamDomain = "b-leaf" lo2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}} l2 := RunServer(&lo2) defer l2.Shutdown() checkLeafNodeConnected(t, l1) checkLeafNodeConnected(t, l2) checkStreamMsgs := func(js nats.JetStreamContext, stream string, expected uint64) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { si, err := js.StreamInfo(stream) if err != nil { return err } if si.State.Msgs != expected { return fmt.Errorf("Expected %v messages, got %v", expected, si.State.Msgs) } return nil }) } sendToStreamTest := func(js nats.JetStreamContext) { t.Helper() for i := 0; i < 10; i++ { _, err = js.Publish("test", []byte("hello")) require_NoError(t, err) } } nca, jsa := jsClientConnect(t, l1) defer nca.Close() _, err = jsa.AddStream(&nats.StreamConfig{Name: "queue", Subjects: []string{"queue"}}) require_NoError(t, err) _, err = jsa.AddStream(&nats.StreamConfig{Name: "testdel", Subjects: []string{"testdel"}}) require_NoError(t, err) ncb, jsb := jsClientConnect(t, l2) defer ncb.Close() _, err = jsb.AddStream(&nats.StreamConfig{Name: "test", Subjects: []string{"test"}}) require_NoError(t, err) sendToStreamTest(jsb) checkStreamMsgs(jsb, "test", 10) _, err = jsb.AddStream(&nats.StreamConfig{Name: "testdelsrc1", Subjects: []string{"testdelsrc1"}}) require_NoError(t, err) _, err = jsb.AddStream(&nats.StreamConfig{Name: "testdelsrc2", Subjects: []string{"testdelsrc2"}}) require_NoError(t, err) // Add test as source to queue si, err := jsa.UpdateStream(&nats.StreamConfig{ Name: "queue", Subjects: []string{"queue"}, Sources: []*nats.StreamSource{ { Name: "test", External: &nats.ExternalStream{ APIPrefix: "$JS.b-leaf.API", }, }, }, }) require_NoError(t, err) require_True(t, len(si.Config.Sources) == 1) checkStreamMsgs(jsa, "queue", 10) // add more entries to "test" sendToStreamTest(jsb) // verify entries are both in "test" and "queue" checkStreamMsgs(jsb, "test", 20) checkStreamMsgs(jsa, "queue", 20) // Remove source si, err = jsa.UpdateStream(&nats.StreamConfig{ Name: "queue", Subjects: []string{"queue"}, }) require_NoError(t, err) require_True(t, len(si.Config.Sources) == 0) // add more entries to "test" sendToStreamTest(jsb) // verify entries are in "test" checkStreamMsgs(jsb, "test", 30) // But they should not be in "queue". We will wait a bit before checking // to make sure that we are letting enough time for the sourcing to // incorrectly happen if there is a bug. time.Sleep(250 * time.Millisecond) checkStreamMsgs(jsa, "queue", 20) // Test that we delete correctly. First add source to a "testdel" si, err = jsa.UpdateStream(&nats.StreamConfig{ Name: "testdel", Subjects: []string{"testdel"}, Sources: []*nats.StreamSource{ { Name: "testdelsrc1", External: &nats.ExternalStream{ APIPrefix: "$JS.b-leaf.API", }, }, }, }) require_NoError(t, err) require_True(t, len(si.Config.Sources) == 1) // Now add the second one... si, err = jsa.UpdateStream(&nats.StreamConfig{ Name: "testdel", Subjects: []string{"testdel"}, Sources: []*nats.StreamSource{ { Name: "testdelsrc1", External: &nats.ExternalStream{ APIPrefix: "$JS.b-leaf.API", }, }, { Name: "testdelsrc2", External: &nats.ExternalStream{ APIPrefix: "$JS.b-leaf.API", }, }, }, }) require_NoError(t, err) require_True(t, len(si.Config.Sources) == 2) // Now check that the stream testdel has still 2 source consumers... acc, err := l1.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("testdel") require_NoError(t, err) mset.mu.RLock() n := len(mset.sources) mset.mu.RUnlock() if n != 2 { t.Fatalf("Expected 2 sources, got %v", n) } // Restart leaf "a" nca.Close() l1.Shutdown() l1 = RunServer(&lo1) defer l1.Shutdown() // add more entries to "test" sendToStreamTest(jsb) checkStreamMsgs(jsb, "test", 40) nca, jsa = jsClientConnect(t, l1) defer nca.Close() time.Sleep(250 * time.Millisecond) checkStreamMsgs(jsa, "queue", 20) } func TestJetStreamAddStreamWithFilestoreFailure(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Cause failure to create stream with filestore. // In one of ipQueue changes, this could cause a panic, so verify that we get // a failure to create, but no panic. if _, err := s.globalAccount().addStreamWithStore( &StreamConfig{Name: "TEST"}, &FileStoreConfig{BlockSize: 2 * maxBlockSize}); err == nil { t.Fatal("Expected failure, did not get one") } } type checkFastState struct { count int64 StreamStore } func (s *checkFastState) FastState(state *StreamState) { // Keep track only when called from checkPending() if bytes.Contains(debug.Stack(), []byte("checkPending(")) { atomic.AddInt64(&s.count, 1) } s.StreamStore.FastState(state) } func TestJetStreamBackOffCheckPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() // Plug or store to see how many times we invoke FastState, which is done in checkPending mset.mu.Lock() st := &checkFastState{StreamStore: mset.store} mset.store = st mset.mu.Unlock() nc := clientConnectToServer(t, s) defer nc.Close() sendStreamMsg(t, nc, "foo", "Hello World!") sub, _ := nc.SubscribeSync(nats.NewInbox()) defer sub.Unsubscribe() nc.Flush() o, err := mset.addConsumer(&ConsumerConfig{ DeliverSubject: sub.Subject, AckPolicy: AckExplicit, MaxDeliver: 1000, BackOff: []time.Duration{50 * time.Millisecond, 250 * time.Millisecond, time.Second}, }) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer o.delete() // Check the first delivery and the following 2 redeliveries start := time.Now() natsNexMsg(t, sub, time.Second) if dur := time.Since(start); dur >= 50*time.Millisecond { t.Fatalf("Expected first delivery to be fast, took: %v", dur) } start = time.Now() natsNexMsg(t, sub, time.Second) if dur := time.Since(start); dur < 25*time.Millisecond || dur > 75*time.Millisecond { t.Fatalf("Expected first redelivery to be ~50ms, took: %v", dur) } start = time.Now() natsNexMsg(t, sub, time.Second) if dur := time.Since(start); dur < 200*time.Millisecond || dur > 300*time.Millisecond { t.Fatalf("Expected first redelivery to be ~250ms, took: %v", dur) } // There was a bug that would cause checkPending to be invoked based on the // ackWait (which in this case would be the first value of BackOff, which // is 50ms). So we would call checkPending() too many times. time.Sleep(500 * time.Millisecond) // Check now, it should have been invoked twice. if n := atomic.LoadInt64(&st.count); n != 2 { t.Fatalf("Expected checkPending to be invoked 2 times, was %v", n) } } func TestJetStreamCrossAccounts(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream { store_dir = %q } accounts: { A: { users: [ {user: a, password: a} ] jetstream: enabled exports: [ {service: '$JS.API.>' } {service: '$KV.>'} {stream: 'accI.>'} ] }, I: { users: [ {user: i, password: i} ] imports: [ {service: {account: A, subject: '$JS.API.>'}, to: 'fromA.>' } {service: {account: A, subject: '$KV.>'}, to: 'fromA.$KV.>' } {stream: {subject: 'accI.>', account: A}} ] } }`, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() watchNext := func(w nats.KeyWatcher) nats.KeyValueEntry { t.Helper() select { case e := <-w.Updates(): return e case <-time.After(time.Second): t.Fatal("Fail to get the next update") } return nil } nc1, js1 := jsClientConnect(t, s, nats.UserInfo("a", "a")) defer nc1.Close() kv1, err := js1.CreateKeyValue(&nats.KeyValueConfig{Bucket: "Map", History: 10}) if err != nil { t.Fatalf("Error creating kv store: %v", err) } w1, err := kv1.Watch("map") if err != nil { t.Fatalf("Error creating watcher: %v", err) } if e := watchNext(w1); e != nil { t.Fatalf("Expected nil entry, got %+v", e) } nc2, err := nats.Connect(s.ClientURL(), nats.UserInfo("i", "i"), nats.CustomInboxPrefix("accI")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() js2, err := nc2.JetStream(nats.APIPrefix("fromA")) if err != nil { t.Fatalf("Error getting jetstream context: %v", err) } kv2, err := js2.CreateKeyValue(&nats.KeyValueConfig{Bucket: "Map", History: 10}) if err != nil { t.Fatalf("Error creating kv store: %v", err) } w2, err := kv2.Watch("map") if err != nil { t.Fatalf("Error creating watcher: %v", err) } if e := watchNext(w2); e != nil { t.Fatalf("Expected nil entry, got %+v", e) } // Do a Put from kv2 rev, err := kv2.Put("map", []byte("value")) if err != nil { t.Fatalf("Error on put: %v", err) } // Get from kv1 e, err := kv1.Get("map") if err != nil { t.Fatalf("Error on get: %v", err) } if e.Key() != "map" || string(e.Value()) != "value" { t.Fatalf("Unexpected entry: +%v", e) } // Get from kv2 e, err = kv2.Get("map") if err != nil { t.Fatalf("Error on get: %v", err) } if e.Key() != "map" || string(e.Value()) != "value" { t.Fatalf("Unexpected entry: +%v", e) } // Watcher 1 if e := watchNext(w1); e == nil || e.Key() != "map" || string(e.Value()) != "value" { t.Fatalf("Unexpected entry: %+v", e) } // Watcher 2 if e := watchNext(w2); e == nil || e.Key() != "map" || string(e.Value()) != "value" { t.Fatalf("Unexpected entry: %+v", e) } // Try an update form kv2 if _, err := kv2.Update("map", []byte("updated"), rev); err != nil { t.Fatalf("Failed to update: %v", err) } // Get from kv1 e, err = kv1.Get("map") if err != nil { t.Fatalf("Error on get: %v", err) } if e.Key() != "map" || string(e.Value()) != "updated" { t.Fatalf("Unexpected entry: +%v", e) } // Get from kv2 e, err = kv2.Get("map") if err != nil { t.Fatalf("Error on get: %v", err) } if e.Key() != "map" || string(e.Value()) != "updated" { t.Fatalf("Unexpected entry: +%v", e) } // Watcher 1 if e := watchNext(w1); e == nil || e.Key() != "map" || string(e.Value()) != "updated" { t.Fatalf("Unexpected entry: %+v", e) } // Watcher 2 if e := watchNext(w2); e == nil || e.Key() != "map" || string(e.Value()) != "updated" { t.Fatalf("Unexpected entry: %+v", e) } // Purge from kv2 if err := kv2.Purge("map"); err != nil { t.Fatalf("Error on purge: %v", err) } // Check purge ok from w1 if e := watchNext(w1); e == nil || e.Operation() != nats.KeyValuePurge { t.Fatalf("Unexpected entry: %+v", e) } // Check purge ok from w2 if e := watchNext(w2); e == nil || e.Operation() != nats.KeyValuePurge { t.Fatalf("Unexpected entry: %+v", e) } // Delete purge records from kv2 if err := kv2.PurgeDeletes(nats.DeleteMarkersOlderThan(-1)); err != nil { t.Fatalf("Error on purge deletes: %v", err) } // Check all gone from js1 if si, err := js1.StreamInfo("KV_Map"); err != nil || si == nil || si.State.Msgs != 0 { t.Fatalf("Error getting stream info: err=%v si=%+v", err, si) } // Delete key from kv2 if err := kv2.Delete("map"); err != nil { t.Fatalf("Error on delete: %v", err) } // Check key gone from kv1 if e, err := kv1.Get("map"); err != nats.ErrKeyNotFound || e != nil { t.Fatalf("Expected key not found, got err=%v e=%+v", err, e) } } func TestJetStreamInvalidRestoreRequests(t *testing.T) { test := func(t *testing.T, s *Server, replica int) { nc := natsConnect(t, s.ClientURL()) defer nc.Close() // test invalid stream config in restore request require_fail := func(cfg StreamConfig, errDesc string) { t.Helper() rreq := &JSApiStreamRestoreRequest{ Config: cfg, } req, err := json.Marshal(rreq) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, "fail"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var rresp JSApiStreamRestoreResponse json.Unmarshal(rmsg.Data, &rresp) require_True(t, rresp.Error != nil) require_Equal(t, rresp.Error.Description, errDesc) } require_fail(StreamConfig{Name: "fail", MaxBytes: 1024, Storage: FileStorage, Replicas: 6}, "maximum replicas is 5") require_fail(StreamConfig{Name: "fail", MaxBytes: 2 * 1012 * 1024, Storage: FileStorage, Replicas: replica}, "insufficient storage resources available") js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "stream", MaxBytes: 1024, Storage: nats.FileStorage, Replicas: 1}) require_NoError(t, err) require_fail(StreamConfig{Name: "fail", MaxBytes: 1024, Storage: FileStorage}, "maximum number of streams reached") } commonAccSection := ` no_auth_user: u accounts { ONE { users = [ { user: "u", pass: "s3cr3t!" } ] jetstream: { max_store: 1Mb max_streams: 1 } } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` t.Run("clustered", func(t *testing.T) { c := createJetStreamClusterWithTemplate(t, ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] }`+commonAccSection, "clust", 3) defer c.shutdown() s := c.randomServer() test(t, s, 3) }) t.Run("single", func(t *testing.T) { storeDir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} %s`, storeDir, commonAccSection))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() test(t, s, 1) }) } func TestJetStreamLimits(t *testing.T) { test := func(t *testing.T, s *Server) { nc := natsConnect(t, s.ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) si, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) require_NoError(t, err) require_True(t, si.Config.Duplicates == time.Minute) si, err = js.AddStream(&nats.StreamConfig{Name: "bar", Duplicates: 1500 * time.Millisecond}) require_NoError(t, err) require_True(t, si.Config.Duplicates == 1500*time.Millisecond) _, err = js.UpdateStream(&nats.StreamConfig{Name: "bar", Duplicates: 2 * time.Minute}) require_Error(t, err) require_Equal(t, err.Error(), "nats: duplicates window can not be larger then server limit of 1m0s") _, err = js.AddStream(&nats.StreamConfig{Name: "baz", Duplicates: 2 * time.Minute}) require_Error(t, err) require_Equal(t, err.Error(), "nats: duplicates window can not be larger then server limit of 1m0s") ci, err := js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) require_True(t, ci.Config.MaxAckPending == 1000) require_True(t, ci.Config.MaxRequestBatch == 250) _, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy, MaxRequestBatch: 500}) require_Error(t, err) require_Equal(t, err.Error(), "nats: consumer max request batch exceeds server limit of 250") ci, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 500}) require_NoError(t, err) require_True(t, ci.Config.MaxAckPending == 500) require_True(t, ci.Config.MaxRequestBatch == 250) _, err = js.UpdateConsumer("foo", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 2000}) require_Error(t, err) require_Equal(t, err.Error(), "nats: consumer max ack pending exceeds system limit of 1000") _, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 2000}) require_Error(t, err) require_Equal(t, err.Error(), "nats: consumer max ack pending exceeds system limit of 1000") } t.Run("clustered", func(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s', limits: {duplicate_window: "1m", max_request_batch: 250} } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: u accounts { ONE { users = [ { user: "u", pass: "s3cr3t!" } ] jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` limitsTest := func(t *testing.T, tmpl string) { c := createJetStreamClusterWithTemplate(t, tmpl, "clust", 3) defer c.shutdown() s := c.randomServer() test(t, s) } // test with max_ack_pending being defined in operator or account t.Run("operator", func(t *testing.T) { limitsTest(t, strings.Replace(tmpl, "duplicate_window", "max_ack_pending: 1000, duplicate_window", 1)) }) t.Run("account", func(t *testing.T) { limitsTest(t, strings.Replace(tmpl, "jetstream: enabled", "jetstream: {max_ack_pending: 1000}", 1)) }) }) t.Run("single", func(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 jetstream: { max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s', limits: {duplicate_window: "1m", max_request_batch: 250} } no_auth_user: u accounts { ONE { users = [ { user: "u", pass: "s3cr3t!" } ] jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` limitsTest := func(t *testing.T, tmpl string) { storeDir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir))) s, opts := RunServerWithConfig(conf) defer s.Shutdown() require_True(t, opts.JetStreamLimits.Duplicates == time.Minute) test(t, s) } // test with max_ack_pending being defined in operator or account t.Run("operator", func(t *testing.T) { limitsTest(t, strings.Replace(tmpl, "duplicate_window", "max_ack_pending: 1000, duplicate_window", 1)) }) t.Run("account", func(t *testing.T) { limitsTest(t, strings.Replace(tmpl, "jetstream: enabled", "jetstream: {max_ack_pending: 1000}", 1)) }) }) } func TestJetStreamConsumerStreamUpdate(t *testing.T) { test := func(t *testing.T, s *Server, replica int) { nc := natsConnect(t, s.ClientURL()) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "foo", Duplicates: 1 * time.Minute, Replicas: replica}) defer js.DeleteStream("foo") require_NoError(t, err) // Update with no change _, err = js.UpdateStream(&nats.StreamConfig{Name: "foo", Duplicates: 1 * time.Minute, Replicas: replica}) require_NoError(t, err) // Update with change _, err = js.UpdateStream(&nats.StreamConfig{Description: "stream", Name: "foo", Duplicates: 1 * time.Minute, Replicas: replica}) require_NoError(t, err) _, err = js.AddConsumer("foo", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Update with no change _, err = js.UpdateConsumer("foo", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Update with change _, err = js.UpdateConsumer("foo", &nats.ConsumerConfig{Description: "consumer", Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) } t.Run("clustered", func(t *testing.T) { c := createJetStreamClusterWithTemplate(t, ` listen: 127.0.0.1:-1 server_name: %s jetstream: { max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s', } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } no_auth_user: u accounts { ONE { users = [ { user: "u", pass: "s3cr3t!" } ] jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }`, "clust", 3) defer c.shutdown() s := c.randomServer() t.Run("r3", func(t *testing.T) { test(t, s, 3) }) t.Run("r1", func(t *testing.T) { test(t, s, 1) }) }) t.Run("single", func(t *testing.T) { storeDir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}`, storeDir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() test(t, s, 1) }) } func TestJetStreamImportReload(t *testing.T) { storeDir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} accounts: { account_a: { users: [{user: user_a, password: pwd}] exports: [{stream: news.>}] } account_b: { users: [{user: user_b, password: pwd}] jetstream: enabled imports: [{stream: {subject: news.>, account: account_a}}] } }`, storeDir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() ncA := natsConnect(t, s.ClientURL(), nats.UserInfo("user_a", "pwd")) defer ncA.Close() ncB := natsConnect(t, s.ClientURL(), nats.UserInfo("user_b", "pwd")) defer ncB.Close() jsB, err := ncB.JetStream() require_NoError(t, err) _, err = jsB.AddStream(&nats.StreamConfig{Name: "news", Subjects: []string{"news.>"}}) require_NoError(t, err) require_NoError(t, ncA.Publish("news.article", nil)) require_NoError(t, ncA.Flush()) si, err := jsB.StreamInfo("news") require_NoError(t, err) require_True(t, si.State.Msgs == 1) // Remove exports/imports reloadUpdateConfig(t, s, conf, fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} accounts: { account_a: { users: [{user: user_a, password: pwd}] } account_b: { users: [{user: user_b, password: pwd}] jetstream: enabled } }`, storeDir)) require_NoError(t, ncA.Publish("news.article", nil)) require_NoError(t, ncA.Flush()) si, err = jsB.StreamInfo("news") require_NoError(t, err) require_True(t, si.State.Msgs == 1) } func TestJetStreamRecoverSealedAfterServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() _, err = js.UpdateStream(&nats.StreamConfig{Name: "foo", Sealed: true}) require_NoError(t, err) nc.Close() // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("foo") require_NoError(t, err) require_True(t, si.State.Msgs == 100) } func TestJetStreamImportConsumerStreamSubjectRemapSingle(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q} accounts: { JS: { jetstream: enabled users: [ {user: js, password: pwd} ] exports [ # This is streaming to a delivery subject for a push based consumer. { stream: "deliver.*" } { stream: "foo.*" } # This is to ack received messages. This is a service to support sync ack. { service: "$JS.ACK.ORDERS.*.>" } # To support ordered consumers, flow control. { service: "$JS.FC.>" } ] }, IM: { users: [ {user: im, password: pwd} ] imports [ { stream: { account: JS, subject: "deliver.ORDERS" }, to: "d.*" } { stream: { account: JS, subject: "foo.*" }, to: "bar.*" } { service: { account: JS, subject: "$JS.FC.>" }} ] }, } `, t.TempDir()))) test := func(t *testing.T, queue bool) { s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserInfo("js", "pwd")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"foo"}, // The JS subject. Storage: nats.MemoryStorage}, ) require_NoError(t, err) _, err = js.Publish("foo", []byte("OK")) require_NoError(t, err) queueName := "" if queue { queueName = "queue" } _, err = js.AddConsumer("ORDERS", &nats.ConsumerConfig{ DeliverSubject: "deliver.ORDERS", AckPolicy: nats.AckExplicitPolicy, DeliverGroup: queueName, }) require_NoError(t, err) nc2, err := nats.Connect(s.ClientURL(), nats.UserInfo("im", "pwd")) require_NoError(t, err) defer nc2.Close() var sub *nats.Subscription if queue { sub, err = nc2.QueueSubscribeSync("d.ORDERS", queueName) require_NoError(t, err) } else { sub, err = nc2.SubscribeSync("d.ORDERS") require_NoError(t, err) } m, err := sub.NextMsg(time.Second) require_NoError(t, err) if m.Subject != "foo" { t.Fatalf("Subject not mapped correctly across account boundary, expected %q got %q", "foo", m.Subject) } // Now do one that would kick in a transform. _, err = js.AddConsumer("ORDERS", &nats.ConsumerConfig{ DeliverSubject: "foo.ORDERS", AckPolicy: nats.AckExplicitPolicy, DeliverGroup: queueName, }) require_NoError(t, err) if queue { sub, err = nc2.QueueSubscribeSync("bar.ORDERS", queueName) require_NoError(t, err) } else { sub, err = nc2.SubscribeSync("bar.ORDERS") require_NoError(t, err) } m, err = sub.NextMsg(time.Second) require_NoError(t, err) if m.Subject != "foo" { t.Fatalf("Subject not mapped correctly across account boundary, expected %q got %q", "foo", m.Subject) } } t.Run("noqueue", func(t *testing.T) { test(t, false) }) t.Run("queue", func(t *testing.T) { test(t, true) }) } func TestJetStreamWorkQueueSourceRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() sent := 10 _, err := js.AddStream(&nats.StreamConfig{ Name: "FOO", Replicas: 1, Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < sent; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 1, // TODO test will pass when retention commented out Retention: nats.WorkQueuePolicy, Sources: []*nats.StreamSource{{Name: "FOO"}}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "dur", nats.BindStream("TEST")) require_NoError(t, err) time.Sleep(1 * time.Second) ci, err := js.ConsumerInfo("TEST", "dur") require_NoError(t, err) require_True(t, ci.NumPending == uint64(sent)) msgs, err := sub.Fetch(sent) require_NoError(t, err) require_True(t, len(msgs) == sent) for i := 0; i < sent; i++ { err = msgs[i].AckSync() require_NoError(t, err) } ci, err = js.ConsumerInfo("TEST", "dur") require_NoError(t, err) require_True(t, ci.NumPending == 0) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 0) // Restart server nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() time.Sleep(200 * time.Millisecond) s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { hs := s.healthz(nil) if hs.Status == "ok" && hs.Error == _EMPTY_ { return nil } return fmt.Errorf("healthz %s %s", hs.Error, hs.Status) }) nc, js = jsClientConnect(t, s) defer nc.Close() si, err = js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 0 { t.Fatalf("Expected 0 messages on restart, got %d", si.State.Msgs) } ctest, err := js.ConsumerInfo("TEST", "dur") require_NoError(t, err) //TODO (mh) I have experienced in other tests that NumPending has a value of 1 post restart. // seems to go awary in single server setup. It's also unrelated to work queue // but that error seems benign. if ctest.NumPending != 0 { t.Fatalf("Expected pending of 0 but got %d", ctest.NumPending) } sub, err = js.PullSubscribe("foo", "dur", nats.BindStream("TEST")) require_NoError(t, err) _, err = sub.Fetch(1, nats.MaxWait(time.Second)) if err != nats.ErrTimeout { require_NoError(t, err) } } func TestJetStreamWorkQueueSourceNamingRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "C1", Subjects: []string{"foo.*"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "C2", Subjects: []string{"bar.*"}}) require_NoError(t, err) sendCount := 10 for i := 0; i < sendCount; i++ { _, err = js.Publish(fmt.Sprintf("foo.%d", i), nil) require_NoError(t, err) _, err = js.Publish(fmt.Sprintf("bar.%d", i), nil) require_NoError(t, err) } // TODO Test will always pass if pending is 0 pending := 1 // For some yet unknown reason this failure seems to require 2 streams to source from. // This might possibly be timing, as the test sometimes passes streams := 2 totalPending := uint64(streams * pending) totalMsgs := streams * sendCount totalNonPending := streams * (sendCount - pending) // TODO Test will always pass if this is named A (go returns directory names sorted) // A: this stream is recovered BEFORE C1/C2, tbh, I'd expect this to be the case to fail, but it isn't // D: this stream is recovered AFTER C1/C2, which is the case that fails (perhaps it is timing) srcName := "D" _, err = js.AddStream(&nats.StreamConfig{ Name: srcName, Retention: nats.WorkQueuePolicy, Sources: []*nats.StreamSource{{Name: "C1"}, {Name: "C2"}}, }) require_NoError(t, err) // Add a consumer and consume all but totalPending messages _, err = js.AddConsumer(srcName, &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) sub, err := js.PullSubscribe("", "dur", nats.BindStream(srcName)) require_NoError(t, err) checkFor(t, 5*time.Second, time.Millisecond*200, func() error { if ci, err := js.ConsumerInfo(srcName, "dur"); err != nil { return err } else if ci.NumPending != uint64(totalMsgs) { return fmt.Errorf("not enough messages: %d", ci.NumPending) } return nil }) // consume all but messages we want pending msgs, err := sub.Fetch(totalNonPending) require_NoError(t, err) require_True(t, len(msgs) == totalNonPending) for _, m := range msgs { err = m.AckSync() require_NoError(t, err) } ci, err := js.ConsumerInfo(srcName, "dur") require_NoError(t, err) require_True(t, ci.NumPending == totalPending) si, err := js.StreamInfo(srcName) require_NoError(t, err) require_True(t, si.State.Msgs == totalPending) // Restart server nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() time.Sleep(200 * time.Millisecond) s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { hs := s.healthz(nil) if hs.Status == "ok" && hs.Error == _EMPTY_ { return nil } return fmt.Errorf("healthz %s %s", hs.Error, hs.Status) }) nc, js = jsClientConnect(t, s) defer nc.Close() si, err = js.StreamInfo(srcName) require_NoError(t, err) if si.State.Msgs != totalPending { t.Fatalf("Expected 0 messages on restart, got %d", si.State.Msgs) } } func TestJetStreamDisabledHealthz(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("Expected JetStream to be enabled") } s.DisableJetStream() hs := s.healthz(&HealthzOptions{JSEnabledOnly: true}) if hs.Status == "unavailable" && hs.Error == NewJSNotEnabledError().Error() { return } t.Fatalf("Expected healthz to return error if JetStream is disabled, got status: %s", hs.Status) } func TestJetStreamPullTimeout(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "pr", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) const numMessages = 1000 // Send messages in small intervals. go func() { for i := 0; i < numMessages; i++ { time.Sleep(time.Millisecond * 10) sendStreamMsg(t, nc, "TEST", "data") } }() // Prepare manual Pull Request. req := &JSApiConsumerGetNextRequest{Batch: 200, NoWait: false, Expires: time.Millisecond * 100} jreq, _ := json.Marshal(req) subj := fmt.Sprintf(JSApiRequestNextT, "TEST", "pr") reply := "_pr_" var got atomic.Int32 nc.PublishRequest(subj, reply, jreq) // Manually subscribe to inbox subject and send new request only if we get `408 Request Timeout`. sub, _ := nc.Subscribe(reply, func(msg *nats.Msg) { if msg.Header.Get("Status") == "408" && msg.Header.Get("Description") == "Request Timeout" { nc.PublishRequest(subj, reply, jreq) nc.Flush() } else { got.Add(1) msg.Ack() } }) defer sub.Unsubscribe() // Check if we're not stuck. checkFor(t, time.Second*30, time.Second*1, func() error { if got.Load() < int32(numMessages) { return fmt.Errorf("expected %d messages", numMessages) } return nil }) } func TestJetStreamPullMaxBytes(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", }) require_NoError(t, err) // Put in ~2MB, each ~100k msz, dsz := 100_000, 99_950 total, msg := 20, []byte(strings.Repeat("Z", dsz)) for i := 0; i < total; i++ { if _, err := js.Publish("TEST", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "pr", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) req := &JSApiConsumerGetNextRequest{MaxBytes: 100, NoWait: true} jreq, _ := json.Marshal(req) subj := fmt.Sprintf(JSApiRequestNextT, "TEST", "pr") reply := "_pr_" sub, _ := nc.SubscribeSync(reply) defer sub.Unsubscribe() checkHeader := func(m *nats.Msg, expected *nats.Header) { t.Helper() if len(m.Data) != 0 { t.Fatalf("Did not expect data, got %d bytes", len(m.Data)) } expectedStatus, givenStatus := expected.Get("Status"), m.Header.Get("Status") expectedDesc, givenDesc := expected.Get("Description"), m.Header.Get("Description") if expectedStatus != givenStatus || expectedDesc != givenDesc { t.Fatalf("expected %s %s, got %s %s", expectedStatus, expectedDesc, givenStatus, givenDesc) } } // If we ask for less MaxBytes then a single message make sure we get an error. badReq := &nats.Header{"Status": []string{"409"}, "Description": []string{"Message Size Exceeds MaxBytes"}} nc.PublishRequest(subj, reply, jreq) m, err := sub.NextMsg(time.Second) require_NoError(t, err) checkSubsPending(t, sub, 0) checkHeader(m, badReq) // If we request a ton of max bytes make sure batch size overrides. req = &JSApiConsumerGetNextRequest{Batch: 1, MaxBytes: 10_000_000, NoWait: true} jreq, _ = json.Marshal(req) nc.PublishRequest(subj, reply, jreq) // we expect two messages, as the second one should be `Batch Completed` status. checkSubsPending(t, sub, 2) // first one is message from the stream. m, err = sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, len(m.Data) == dsz) require_True(t, len(m.Header) == 0) // second one is the status. m, err = sub.NextMsg(time.Second) require_NoError(t, err) if v := m.Header.Get("Description"); v != "Batch Completed" { t.Fatalf("Expected Batch Completed, got: %s", v) } checkSubsPending(t, sub, 0) // Same but with batch > 1 req = &JSApiConsumerGetNextRequest{Batch: 5, MaxBytes: 10_000_000, NoWait: true} jreq, _ = json.Marshal(req) nc.PublishRequest(subj, reply, jreq) // 6, not 5, as 6th is the status. checkSubsPending(t, sub, 6) for i := 0; i < 5; i++ { m, err = sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, len(m.Data) == dsz) require_True(t, len(m.Header) == 0) } m, err = sub.NextMsg(time.Second) require_NoError(t, err) if v := m.Header.Get("Description"); v != "Batch Completed" { t.Fatalf("Expected Batch Completed, got: %s", v) } checkSubsPending(t, sub, 0) // Now ask for large batch but make sure we are limited by batch size. req = &JSApiConsumerGetNextRequest{Batch: 1_000, MaxBytes: msz * 4, NoWait: true} jreq, _ = json.Marshal(req) nc.PublishRequest(subj, reply, jreq) // Receive 4 messages + the 409 checkSubsPending(t, sub, 5) for i := 0; i < 4; i++ { m, err = sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, len(m.Data) == dsz) require_True(t, len(m.Header) == 0) } m, err = sub.NextMsg(time.Second) require_NoError(t, err) checkHeader(m, badReq) checkSubsPending(t, sub, 0) req = &JSApiConsumerGetNextRequest{Batch: 1_000, MaxBytes: msz, NoWait: true} jreq, _ = json.Marshal(req) nc.PublishRequest(subj, reply, jreq) // Receive 1 message + 409 checkSubsPending(t, sub, 2) m, err = sub.NextMsg(time.Second) require_NoError(t, err) require_True(t, len(m.Data) == dsz) require_True(t, len(m.Header) == 0) m, err = sub.NextMsg(time.Second) require_NoError(t, err) checkHeader(m, badReq) checkSubsPending(t, sub, 0) } func TestJetStreamStreamRepublishCycle(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "RPC", Storage: MemoryStorage, Subjects: []string{"foo.>", "bar.*", "baz"}, } expectFail := func() { t.Helper() req, err := json.Marshal(cfg) require_NoError(t, err) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) var resp JSApiStreamCreateResponse err = json.Unmarshal(rmsg.Data, &resp) require_NoError(t, err) if resp.Type != JSApiStreamCreateResponseType { t.Fatalf("Invalid response type %s expected %s", resp.Type, JSApiStreamCreateResponseType) } if resp.Error == nil { t.Fatalf("Expected error but got none") } if !strings.Contains(resp.Error.Description, "republish destination forms a cycle") { t.Fatalf("Expected cycle error, got %q", resp.Error.Description) } } cfg.RePublish = &RePublish{ Source: "foo.>", Destination: "foo.>", } expectFail() cfg.RePublish = &RePublish{ Source: "bar.bar", Destination: "foo.bar", } expectFail() cfg.RePublish = &RePublish{ Source: "baz", Destination: "bar.bar", } expectFail() } func TestJetStreamStreamRepublishOneTokenMatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "Stream1", Storage: MemoryStorage, Subjects: []string{"one", "four"}, RePublish: &RePublish{ Source: "one", Destination: "uno", HeadersOnly: false, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("uno") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("one", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, toSend) m, err := sub.NextMsg(time.Second) require_NoError(t, err) if !(len(m.Data) > 0) { t.Fatalf("Expected msg data") } } func TestJetStreamStreamRepublishMultiTokenMatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "Stream1", Storage: MemoryStorage, Subjects: []string{"one.>", "four.>"}, RePublish: &RePublish{ Source: "one.two.>", Destination: "uno.dos.>", HeadersOnly: false, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("uno.dos.>") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("one.two.three", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, toSend) m, err := sub.NextMsg(time.Second) require_NoError(t, err) if !(len(m.Data) > 0) { t.Fatalf("Expected msg data") } } func TestJetStreamStreamRepublishAnySubjectMatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "Stream1", Storage: MemoryStorage, Subjects: []string{"one.>", "four.>"}, RePublish: &RePublish{ Destination: "uno.dos.>", HeadersOnly: false, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("uno.dos.>") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("one.two.three", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, toSend) m, err := sub.NextMsg(time.Second) require_NoError(t, err) if !(len(m.Data) > 0) { t.Fatalf("Expected msg data") } } func TestJetStreamStreamRepublishMultiTokenNoMatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "Stream1", Storage: MemoryStorage, Subjects: []string{"one.>", "four.>"}, RePublish: &RePublish{ Source: "one.two.>", Destination: "uno.dos.>", HeadersOnly: true, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("uno.dos.>") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("four.five.six", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, 0) require_NoError(t, err) } func TestJetStreamStreamRepublishOneTokenNoMatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "Stream1", Storage: MemoryStorage, Subjects: []string{"one", "four"}, RePublish: &RePublish{ Source: "one", Destination: "uno", HeadersOnly: true, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("uno") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("four", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, 0) require_NoError(t, err) } func TestJetStreamStreamRepublishHeadersOnly(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "RPC", Storage: MemoryStorage, Subjects: []string{"foo", "bar", "baz"}, RePublish: &RePublish{ Destination: "RP.>", HeadersOnly: true, }, } addStream(t, nc, cfg) sub, err := nc.SubscribeSync("RP.>") require_NoError(t, err) msg, toSend := bytes.Repeat([]byte("Z"), 512), 100 for i := 0; i < toSend; i++ { js.PublishAsync("foo", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkSubsPending(t, sub, toSend) m, err := sub.NextMsg(time.Second) require_NoError(t, err) if len(m.Data) > 0 { t.Fatalf("Expected no msg just headers, but got %d bytes", len(m.Data)) } if sz := m.Header.Get(JSMsgSize); sz != "512" { t.Fatalf("Expected msg size hdr, got %q", sz) } } func TestJetStreamConsumerDeliverNewNotConsumingBeforeRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) inbox := nats.NewInbox() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, Durable: "dur", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, FilterSubject: "foo", }) require_NoError(t, err) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", "msg") } checkCount := func(expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "dur") if err != nil { return err } if n := int(ci.NumPending); n != expected { return fmt.Errorf("Expected %v pending, got %v", expected, n) } return nil }) } checkCount(10) time.Sleep(300 * time.Millisecond) // Check server restart nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() checkCount(10) // Make sure messages can be consumed sub := natsSubSync(t, nc, inbox) for i := 0; i < 10; i++ { msg, err := sub.NextMsg(time.Second) if err != nil { t.Fatalf("i=%v next msg error: %v", i, err) } msg.AckSync() } checkCount(0) } func TestJetStreamConsumerNumPendingWithMaxPerSubjectGreaterThanOne(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() test := func(t *testing.T, st nats.StorageType) { _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"KV.*.*"}, MaxMsgsPerSubject: 2, Storage: st, }) require_NoError(t, err) // If we allow more than one msg per subject, consumer's num pending can be off (bug in store level). // This requires a filtered state, simple states work ok. // Since we now rely on stream's filtered state when asked directly for consumer info in >=2.8.3. js.PublishAsync("KV.plans.foo", []byte("OK")) js.PublishAsync("KV.plans.bar", []byte("OK")) js.PublishAsync("KV.plans.baz", []byte("OK")) // These are required, the consumer needs to filter these out to see the bug. js.PublishAsync("KV.config.foo", []byte("OK")) js.PublishAsync("KV.config.bar", []byte("OK")) js.PublishAsync("KV.config.baz", []byte("OK")) // Double up some now. js.PublishAsync("KV.plans.bar", []byte("OK")) js.PublishAsync("KV.plans.baz", []byte("OK")) ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "d", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverLastPerSubjectPolicy, FilterSubject: "KV.plans.*", }) require_NoError(t, err) err = js.DeleteStream("TEST") require_NoError(t, err) if ci.NumPending != 3 { t.Fatalf("Expected 3 NumPending, but got %d", ci.NumPending) } } t.Run("MemoryStore", func(t *testing.T) { test(t, nats.MemoryStorage) }) t.Run("FileStore", func(t *testing.T) { test(t, nats.FileStorage) }) } func TestJetStreamMsgGetNoAdvisory(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "foo"}) require_NoError(t, err) for i := 0; i < 100; i++ { js.PublishAsync("foo", []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sub, err := nc.SubscribeSync("$JS.EVENT.ADVISORY.>") require_NoError(t, err) _, err = js.GetMsg("foo", 1) require_NoError(t, err) checkSubsPending(t, sub, 0) } func TestJetStreamDirectMsgGet(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "DSMG", Storage: MemoryStorage, Subjects: []string{"foo", "bar", "baz"}, MaxMsgsPer: 1, AllowDirect: true, } addStream(t, nc, cfg) sendStreamMsg(t, nc, "foo", "foo") sendStreamMsg(t, nc, "bar", "bar") sendStreamMsg(t, nc, "baz", "baz") getSubj := fmt.Sprintf(JSDirectMsgGetT, "DSMG") getMsg := func(req *JSApiMsgGetRequest) *nats.Msg { var b []byte var err error if req != nil { b, err = json.Marshal(req) require_NoError(t, err) } m, err := nc.Request(getSubj, b, time.Second) require_NoError(t, err) return m } m := getMsg(&JSApiMsgGetRequest{LastFor: "foo"}) require_True(t, string(m.Data) == "foo") require_True(t, m.Header.Get(JSStream) == "DSMG") require_True(t, m.Header.Get(JSSequence) == "1") require_True(t, m.Header.Get(JSSubject) == "foo") require_True(t, m.Subject != "foo") require_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_) m = getMsg(&JSApiMsgGetRequest{LastFor: "bar"}) require_True(t, string(m.Data) == "bar") require_True(t, m.Header.Get(JSStream) == "DSMG") require_True(t, m.Header.Get(JSSequence) == "2") require_True(t, m.Header.Get(JSSubject) == "bar") require_True(t, m.Subject != "bar") require_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_) m = getMsg(&JSApiMsgGetRequest{LastFor: "baz"}) require_True(t, string(m.Data) == "baz") require_True(t, m.Header.Get(JSStream) == "DSMG") require_True(t, m.Header.Get(JSSequence) == "3") require_True(t, m.Header.Get(JSSubject) == "baz") require_True(t, m.Subject != "baz") require_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_) // Test error conditions // Nil request m = getMsg(nil) require_True(t, len(m.Data) == 0) require_True(t, m.Header.Get("Status") == "408") require_True(t, m.Header.Get("Description") == "Empty Request") // Empty request m = getMsg(&JSApiMsgGetRequest{}) require_True(t, len(m.Data) == 0) require_True(t, m.Header.Get("Status") == "408") require_True(t, m.Header.Get("Description") == "Empty Request") // Both set m = getMsg(&JSApiMsgGetRequest{Seq: 1, LastFor: "foo"}) require_True(t, len(m.Data) == 0) require_True(t, m.Header.Get("Status") == "408") require_True(t, m.Header.Get("Description") == "Bad Request") // Not found m = getMsg(&JSApiMsgGetRequest{LastFor: "foobar"}) require_True(t, len(m.Data) == 0) require_True(t, m.Header.Get("Status") == "404") require_True(t, m.Header.Get("Description") == "Message Not Found") m = getMsg(&JSApiMsgGetRequest{Seq: 22}) require_True(t, len(m.Data) == 0) require_True(t, m.Header.Get("Status") == "404") require_True(t, m.Header.Get("Description") == "Message Not Found") } // This allows support for a get next given a sequence as a starting. // This allows these to be chained together if needed for sparse streams. func TestJetStreamDirectMsgGetNext(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "DSMG", Storage: MemoryStorage, Subjects: []string{"foo", "bar", "baz"}, AllowDirect: true, } addStream(t, nc, cfg) sendStreamMsg(t, nc, "foo", "foo") for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "bar", "bar") } for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "baz", "baz") } sendStreamMsg(t, nc, "foo", "foo") getSubj := fmt.Sprintf(JSDirectMsgGetT, "DSMG") getMsg := func(seq uint64, subj string) *nats.Msg { req := []byte(fmt.Sprintf(`{"seq": %d, "next_by_subj": %q}`, seq, subj)) m, err := nc.Request(getSubj, req, time.Second) require_NoError(t, err) return m } m := getMsg(0, "foo") require_True(t, m.Header.Get(JSSequence) == "1") require_True(t, m.Header.Get(JSSubject) == "foo") m = getMsg(1, "foo") require_True(t, m.Header.Get(JSSequence) == "1") require_True(t, m.Header.Get(JSSubject) == "foo") m = getMsg(2, "foo") require_True(t, m.Header.Get(JSSequence) == "22") require_True(t, m.Header.Get(JSSubject) == "foo") m = getMsg(2, "bar") require_True(t, m.Header.Get(JSSequence) == "2") require_True(t, m.Header.Get(JSSubject) == "bar") m = getMsg(5, "baz") require_True(t, m.Header.Get(JSSequence) == "12") require_True(t, m.Header.Get(JSSubject) == "baz") m = getMsg(14, "baz") require_True(t, m.Header.Get(JSSequence) == "14") require_True(t, m.Header.Get(JSSubject) == "baz") } func TestJetStreamConsumerAndStreamNamesWithPathSeparators(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "usr/bin"}) require_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName) _, err = js.AddStream(&nats.StreamConfig{Name: `Documents\readme.txt`}) require_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName) // Now consumers. _, err = js.AddStream(&nats.StreamConfig{Name: "T"}) require_NoError(t, err) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: "a/b", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName) _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: `a\b`, AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName) } func TestJetStreamConsumerUpdateFilterSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "T", Subjects: []string{"foo", "bar", "baz"}}) require_NoError(t, err) // 10 foo for i := 0; i < 10; i++ { js.PublishAsync("foo", []byte("OK")) } // 20 bar for i := 0; i < 20; i++ { js.PublishAsync("bar", []byte("OK")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sub, err := js.PullSubscribe("foo", "d") require_NoError(t, err) // Consume 5 msgs msgs, err := sub.Fetch(5) require_NoError(t, err) require_True(t, len(msgs) == 5) // Now update to different filter subject. _, err = js.UpdateConsumer("T", &nats.ConsumerConfig{ Durable: "d", FilterSubject: "bar", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) sub, err = js.PullSubscribe("bar", "d") require_NoError(t, err) msgs, err = sub.Fetch(1) require_NoError(t, err) // Make sure meta and pending etc are all correct. m := msgs[0] meta, err := m.Metadata() require_NoError(t, err) if meta.Sequence.Consumer != 6 || meta.Sequence.Stream != 11 { t.Fatalf("Sequence incorrect %+v", meta.Sequence) } if meta.NumDelivered != 1 { t.Fatalf("Expected NumDelivered to be 1, got %d", meta.NumDelivered) } if meta.NumPending != 19 { t.Fatalf("Expected NumPending to be 19, got %d", meta.NumPending) } } // Originally pull consumers were FIFO with respect to the request, not delivery of messages. // We have changed to have the behavior be FIFO but on an individual message basis. // So after a message is delivered, the request, if still outstanding, effectively // goes to the end of the queue of requests pending. func TestJetStreamConsumerPullConsumerFIFO(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "T"}) require_NoError(t, err) // Create pull consumer. _, err = js.AddConsumer("T", &nats.ConsumerConfig{Durable: "d", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // Simulate 10 pull requests each asking for 10 messages. var subs []*nats.Subscription for i := 0; i < 10; i++ { inbox := nats.NewInbox() sub := natsSubSync(t, nc, inbox) subs = append(subs, sub) req := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 60 * time.Second} jreq, err := json.Marshal(req) require_NoError(t, err) err = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, "T", "d"), inbox, jreq) require_NoError(t, err) } // Now send 100 messages. for i := 0; i < 100; i++ { js.PublishAsync("T", []byte("FIFO FTW!")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Wait for messages for index, sub := range subs { checkSubsPending(t, sub, 10) for i := 0; i < 10; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) meta, err := m.Metadata() require_NoError(t, err) // We expect these to be FIFO per message. E.g. sub #1 = [1, 11, 21, 31, ..] if sseq := meta.Sequence.Stream; sseq != uint64(index+1+(10*i)) { t.Fatalf("Expected message #%d for sub #%d to be %d, but got %d", i+1, index+1, index+1+(10*i), sseq) } } } } // Make sure that when we reach an ack limit that we follow one shot semantics. func TestJetStreamConsumerPullConsumerOneShotOnMaxAckLimit(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "T"}) require_NoError(t, err) for i := 0; i < 10; i++ { js.Publish("T", []byte("OK")) } sub, err := js.PullSubscribe("T", "d", nats.MaxAckPending(5)) require_NoError(t, err) start := time.Now() msgs, err := sub.Fetch(10, nats.MaxWait(2*time.Second)) require_NoError(t, err) if elapsed := time.Since(start); elapsed >= 2*time.Second { t.Fatalf("Took too long, not one shot behavior: %v", elapsed) } if len(msgs) != 5 { t.Fatalf("Expected 5 msgs, got %d", len(msgs)) } } /////////////////////////////////////////////////////////////////////////// // Simple JetStream Benchmarks /////////////////////////////////////////////////////////////////////////// func Benchmark__JetStreamPubWithAck(b *testing.B) { s := RunBasicJetStreamServer(b) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "foo"}) if err != nil { b.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, err := nats.Connect(s.ClientURL()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() b.ResetTimer() for i := 0; i < b.N; i++ { nc.Request("foo", []byte("Hello World!"), 50*time.Millisecond) } b.StopTimer() state := mset.state() if int(state.Msgs) != b.N { b.Fatalf("Expected %d messages, got %d", b.N, state.Msgs) } } func Benchmark____JetStreamPubNoAck(b *testing.B) { s := RunBasicJetStreamServer(b) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "foo"}) if err != nil { b.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, err := nats.Connect(s.ClientURL()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() b.ResetTimer() for i := 0; i < b.N; i++ { if err := nc.Publish("foo", []byte("Hello World!")); err != nil { b.Fatalf("Unexpected error: %v", err) } } nc.Flush() b.StopTimer() state := mset.state() if int(state.Msgs) != b.N { b.Fatalf("Expected %d messages, got %d", b.N, state.Msgs) } } func Benchmark_JetStreamPubAsyncAck(b *testing.B) { s := RunBasicJetStreamServer(b) defer s.Shutdown() mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: "foo"}) if err != nil { b.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, err := nats.Connect(s.ClientURL(), nats.NoReconnect()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() // Put ack stream on its own connection. anc, err := nats.Connect(s.ClientURL()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer anc.Close() acks := nats.NewInbox() sub, _ := anc.Subscribe(acks, func(m *nats.Msg) { // Just eat them for this test. }) // set max pending to unlimited. sub.SetPendingLimits(-1, -1) defer sub.Unsubscribe() anc.Flush() runtime.GC() b.ResetTimer() for i := 0; i < b.N; i++ { if err := nc.PublishRequest("foo", acks, []byte("Hello World!")); err != nil { b.Fatalf("[%d] Unexpected error: %v", i, err) } } nc.Flush() b.StopTimer() state := mset.state() if int(state.Msgs) != b.N { b.Fatalf("Expected %d messages, got %d", b.N, state.Msgs) } } func Benchmark____JetStreamSubNoAck(b *testing.B) { if b.N < 10000 { return } s := RunBasicJetStreamServer(b) defer s.Shutdown() mname := "foo" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname}) if err != nil { b.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, err := nats.Connect(s.ClientURL(), nats.NoReconnect()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() // Queue up messages. for i := 0; i < b.N; i++ { nc.Publish(mname, []byte("Hello World!")) } nc.Flush() state := mset.state() if state.Msgs != uint64(b.N) { b.Fatalf("Expected %d messages, got %d", b.N, state.Msgs) } total := int32(b.N) received := int32(0) done := make(chan bool) deliverTo := "DM" oname := "O" nc.Subscribe(deliverTo, func(m *nats.Msg) { // We only are done when we receive all, we could check for gaps too. if atomic.AddInt32(&received, 1) >= total { done <- true } }) nc.Flush() b.ResetTimer() o, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: deliverTo, Durable: oname, AckPolicy: AckNone}) if err != nil { b.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() <-done b.StopTimer() } func benchJetStreamWorkersAndBatch(b *testing.B, numWorkers, batchSize int) { // Avoid running at too low of numbers since that chews up memory and GC. if b.N < numWorkers*batchSize { return } s := RunBasicJetStreamServer(b) defer s.Shutdown() mname := "MSET22" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname}) if err != nil { b.Fatalf("Unexpected error adding stream: %v", err) } defer mset.delete() nc, err := nats.Connect(s.ClientURL(), nats.NoReconnect()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() // Queue up messages. for i := 0; i < b.N; i++ { nc.Publish(mname, []byte("Hello World!")) } nc.Flush() state := mset.state() if state.Msgs != uint64(b.N) { b.Fatalf("Expected %d messages, got %d", b.N, state.Msgs) } // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit}) if err != nil { b.Fatalf("Expected no error with registered interest, got %v", err) } defer o.delete() total := int32(b.N) received := int32(0) start := make(chan bool) done := make(chan bool) batchSizeMsg := []byte(strconv.Itoa(batchSize)) reqNextMsgSubj := o.requestNextMsgSubject() for i := 0; i < numWorkers; i++ { nc, err := nats.Connect(s.ClientURL(), nats.NoReconnect()) if err != nil { b.Fatalf("Failed to create client: %v", err) } defer nc.Close() deliverTo := nats.NewInbox() nc.Subscribe(deliverTo, func(m *nats.Msg) { if atomic.AddInt32(&received, 1) >= total { done <- true } // Ack + Next request. nc.PublishRequest(m.Reply, deliverTo, AckNext) }) nc.Flush() go func() { <-start nc.PublishRequest(reqNextMsgSubj, deliverTo, batchSizeMsg) }() } b.ResetTimer() close(start) <-done b.StopTimer() } func Benchmark___JetStream1x1Worker(b *testing.B) { benchJetStreamWorkersAndBatch(b, 1, 1) } func Benchmark__JetStream1x1kWorker(b *testing.B) { benchJetStreamWorkersAndBatch(b, 1, 1024) } func Benchmark_JetStream10x1kWorker(b *testing.B) { benchJetStreamWorkersAndBatch(b, 10, 1024) } func Benchmark_JetStream4x512Worker(b *testing.B) { benchJetStreamWorkersAndBatch(b, 4, 512) } func TestJetStreamKVMemoryStorePerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "TEST", History: 1, Storage: nats.MemoryStorage}) require_NoError(t, err) start := time.Now() for i := 0; i < 100_000; i++ { _, err := kv.PutString(fmt.Sprintf("foo.%d", i), "HELLO") require_NoError(t, err) } fmt.Printf("Took %v for first run\n", time.Since(start)) start = time.Now() for i := 0; i < 100_000; i++ { _, err := kv.PutString(fmt.Sprintf("foo.%d", i), "HELLO WORLD") require_NoError(t, err) } fmt.Printf("Took %v for second run\n", time.Since(start)) start = time.Now() for i := 0; i < 100_000; i++ { _, err := kv.Get(fmt.Sprintf("foo.%d", i)) require_NoError(t, err) } fmt.Printf("Took %v for get\n", time.Since(start)) } func TestJetStreamKVMemoryStoreDirectGetPerf(t *testing.T) { // Comment out to run, holding place for now. t.SkipNow() s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"foo.*"}, MaxMsgsPer: 1, AllowDirect: true, } addStream(t, nc, cfg) start := time.Now() for i := 0; i < 100_000; i++ { _, err := js.Publish(fmt.Sprintf("foo.%d", i), []byte("HELLO")) require_NoError(t, err) } fmt.Printf("Took %v for put\n", time.Since(start)) getSubj := fmt.Sprintf(JSDirectMsgGetT, "TEST") const tmpl = "{\"last_by_subj\":%q}" start = time.Now() for i := 0; i < 100_000; i++ { req := []byte(fmt.Sprintf(tmpl, fmt.Sprintf("foo.%d", i))) _, err := nc.Request(getSubj, req, time.Second) require_NoError(t, err) } fmt.Printf("Took %v for get\n", time.Since(start)) } func TestJetStreamMultiplePullPerf(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() js.AddStream(&nats.StreamConfig{Name: "mp22", Storage: nats.FileStorage}) defer js.DeleteStream("mp22") n, msg := 1_000_000, []byte("OK") for i := 0; i < n; i++ { js.PublishAsync("mp22", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("mp22") require_NoError(t, err) fmt.Printf("msgs: %d, total_bytes: %v\n", si.State.Msgs, friendlyBytes(int64(si.State.Bytes))) // 10 pull subscribers each asking for 100 msgs. _, err = js.AddConsumer("mp22", &nats.ConsumerConfig{ Durable: "d", MaxAckPending: 8_000, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) startCh := make(chan bool) var wg sync.WaitGroup np, bs := 10, 100 count := 0 for i := 0; i < np; i++ { nc, js := jsClientConnect(t, s) defer nc.Close() sub, err := js.PullSubscribe("mp22", "d") require_NoError(t, err) wg.Add(1) go func(sub *nats.Subscription) { defer wg.Done() <-startCh for i := 0; i < n/(np*bs); i++ { msgs, err := sub.Fetch(bs) if err != nil { t.Logf("Got error on pull: %v", err) return } if len(msgs) != bs { t.Logf("Expected %d msgs, got %d", bs, len(msgs)) return } count += len(msgs) for _, m := range msgs { m.Ack() } } }(sub) } start := time.Now() close(startCh) wg.Wait() tt := time.Since(start) fmt.Printf("Took %v to receive %d msgs [%d]\n", tt, n, count) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) } func TestJetStreamMirrorUpdatesNotSupported(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "SOURCE"}) require_NoError(t, err) cfg := &nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "SOURCE"}, } _, err = js.AddStream(cfg) require_NoError(t, err) cfg.Mirror = nil _, err = js.UpdateStream(cfg) require_Error(t, err, NewJSStreamMirrorNotUpdatableError()) } func TestJetStreamMirrorFirstSeqNotSupported(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() _, err := s.gacc.addStream(&StreamConfig{Name: "SOURCE"}) require_NoError(t, err) cfg := &StreamConfig{ Name: "M", Mirror: &StreamSource{Name: "SOURCE"}, FirstSeq: 123, } _, err = s.gacc.addStream(cfg) require_Error(t, err, NewJSMirrorWithFirstSeqError()) } func TestJetStreamDirectGetBySubject(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} ONLYME = { publish = { allow = "$JS.API.DIRECT.GET.KV.vid.22.>"} } accounts: { A: { jetstream: enabled users: [ { user: admin, password: s3cr3t }, { user: user, password: pwd, permissions: $ONLYME}, ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserInfo("admin", "s3cr3t")) defer nc.Close() // Do by hand for now. cfg := &StreamConfig{ Name: "KV", Storage: MemoryStorage, Subjects: []string{"vid.*.>"}, MaxMsgsPer: 1, AllowDirect: true, } addStream(t, nc, cfg) // Add in mirror as well. cfg = &StreamConfig{ Name: "M", Storage: MemoryStorage, Mirror: &StreamSource{Name: "KV"}, MirrorDirect: true, } addStream(t, nc, cfg) v22 := "vid.22.speed" v33 := "vid.33.speed" _, err := js.Publish(v22, []byte("100")) require_NoError(t, err) _, err = js.Publish(v33, []byte("55")) require_NoError(t, err) // User the restricted user. nc, _ = jsClientConnect(t, s, nats.UserInfo("user", "pwd")) defer nc.Close() errCh := make(chan error, 10) nc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) { select { case errCh <- e: default: } }) getSubj := fmt.Sprintf(JSDirectGetLastBySubjectT, "KV", v22) m, err := nc.Request(getSubj, nil, time.Second) require_NoError(t, err) require_True(t, string(m.Data) == "100") // Now attempt to access vid 33 data.. getSubj = fmt.Sprintf(JSDirectGetLastBySubjectT, "KV", v33) _, err = nc.Request(getSubj, nil, 200*time.Millisecond) require_Error(t, err) // timeout here. select { case e := <-errCh: if !strings.HasPrefix(e.Error(), "nats: permissions violation") { t.Fatalf("Expected a permissions violation but got %v", e) } case <-time.After(time.Second): t.Fatalf("Expected to get a permissions error, got none") } // Now make sure mirrors are doing right thing with new way as well. var sawMirror bool getSubj = fmt.Sprintf(JSDirectGetLastBySubjectT, "KV", v22) for i := 0; i < 100; i++ { m, err := nc.Request(getSubj, nil, time.Second) require_NoError(t, err) if shdr := m.Header.Get(JSStream); shdr == "M" { sawMirror = true break } } if !sawMirror { t.Fatalf("Expected to see the mirror respond at least once") } } func TestJetStreamProperErrorDueToOverlapSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() test := func(t *testing.T, s *Server) { nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, }) require_NoError(t, err) // Now do this by end since we want to check the error returned. sc := &nats.StreamConfig{ Name: "TEST2", Subjects: []string{"foo.>"}, } req, _ := json.Marshal(sc) msg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, sc.Name), req, time.Second) require_NoError(t, err) var scResp JSApiStreamCreateResponse err = json.Unmarshal(msg.Data, &scResp) require_NoError(t, err) if scResp.Error == nil || !IsNatsErr(scResp.Error, JSStreamSubjectOverlapErr) { t.Fatalf("Did not receive correct error: %+v", scResp) } } t.Run("standalone", func(t *testing.T) { test(t, s) }) t.Run("clustered", func(t *testing.T) { test(t, c.randomServer()) }) } func TestJetStreamServerCipherConvert(t *testing.T) { tmpl := ` server_name: S22 listen: 127.0.0.1:-1 jetstream: {key: s3cr3t, store_dir: '%s', cipher: %s} ` storeDir := t.TempDir() // Create a stream and a consumer under one cipher, and restart the server with a new cipher. conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, "AES"))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 0; i < 1000; i++ { msg := []byte(fmt.Sprintf("TOP SECRET DOCUMENT #%d", i+1)) _, err := js.Publish("foo", msg) require_NoError(t, err) } // Make sure consumers convert as well. sub, err := js.PullSubscribe("foo", "dlc") require_NoError(t, err) for _, m := range fetchMsgs(t, sub, 100, 5*time.Second) { m.AckSync() } si, err := js.StreamInfo("TEST") require_NoError(t, err) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Stop current s.Shutdown() conf = createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, "ChaCha"))) s, _ = RunServerWithConfig(conf) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() si2, err := js.StreamInfo("TEST") require_NoError(t, err) if !reflect.DeepEqual(si, si2) { t.Fatalf("Stream infos did not match\n%+v\nvs\n%+v", si, si2) } ci2, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Consumer create times can be slightly off after restore from disk. now := time.Now() ci.Created, ci2.Created = now, now ci.Delivered.Last, ci2.Delivered.Last = nil, nil ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil // Also clusters will be different. ci.Cluster, ci2.Cluster = nil, nil if !reflect.DeepEqual(ci, ci2) { t.Fatalf("Consumer infos did not match\n%+v\nvs\n%+v", ci, ci2) } } func TestJetStreamConsumerDeliverNewMaxRedeliveriesAndServerRestart(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, }) require_NoError(t, err) inbox := nats.NewInbox() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, Durable: "dur", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverNewPolicy, MaxDeliver: 3, AckWait: 250 * time.Millisecond, FilterSubject: "foo.bar", }) require_NoError(t, err) sendStreamMsg(t, nc, "foo.bar", "msg") sub := natsSubSync(t, nc, inbox) for i := 0; i < 3; i++ { natsNexMsg(t, sub, time.Second) } // Now check that there is no more redeliveries if msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout, got msg=%+v err=%v", msg, err) } // Give a chance to things to be persisted time.Sleep(300 * time.Millisecond) // Check server restart nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, _ = jsClientConnect(t, s) defer nc.Close() sub = natsSubSync(t, nc, inbox) // We should not have messages being redelivered. if msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout, got msg=%+v err=%v", msg, err) } } func TestJetStreamConsumerPendingLowerThanStreamFirstSeq(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 100; i++ { sendStreamMsg(t, nc, "foo", "msg") } inbox := nats.NewInbox() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ DeliverSubject: inbox, Durable: "dur", AckPolicy: nats.AckExplicitPolicy, DeliverPolicy: nats.DeliverAllPolicy, }) require_NoError(t, err) sub := natsSubSync(t, nc, inbox) for i := 0; i < 10; i++ { natsNexMsg(t, sub, time.Second) } acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("dur") require_True(t, o != nil) o.stop() mset.store.Compact(1_000_000) nc.Close() sd := s.JetStreamConfig().StoreDir s.Shutdown() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 1_000_000) require_True(t, si.State.LastSeq == 999_999) natsSubSync(t, nc, inbox) checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "dur") if err != nil { return err } if ci.NumAckPending != 0 { return fmt.Errorf("NumAckPending should be 0, got %v", ci.NumAckPending) } if ci.Delivered.Stream != 999_999 { return fmt.Errorf("Delivered.Stream should be 999,999, got %v", ci.Delivered.Stream) } return nil }) } func TestJetStreamAllowDirectAfterUpdate(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "foo", "msg") si, err := js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, AllowDirect: true, }) require_NoError(t, err) require_True(t, si.Config.AllowDirect) _, err = js.GetLastMsg("TEST", "foo", nats.DirectGet(), nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) // Make sure turning off works too. si, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, AllowDirect: false, }) require_NoError(t, err) require_False(t, si.Config.AllowDirect) _, err = js.GetLastMsg("TEST", "foo", nats.DirectGet(), nats.MaxWait(100*time.Millisecond)) require_Error(t, err) } // Bug when stream's consumer config does not force filestore to track per subject information. func TestJetStreamConsumerEOFBugNewFileStore(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.bar.*"}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, }) require_NoError(t, err) dsubj := nats.NewInbox() sub, err := nc.SubscribeSync(dsubj) require_NoError(t, err) nc.Flush() // Filter needs to be a wildcard. Need to bind to the _, err = js.AddConsumer("M", &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: "foo.>"}) require_NoError(t, err) for i := 0; i < 100; i++ { _, err := js.PublishAsync("foo.bar.baz", []byte("OK")) require_NoError(t, err) } for i := 0; i < 100; i++ { m, err := sub.NextMsg(time.Second) require_NoError(t, err) m.Respond(nil) } // Now force an expiration. mset, err := s.GlobalAccount().lookupStream("M") require_NoError(t, err) mset.mu.RLock() store := mset.store.(*fileStore) mset.mu.RUnlock() store.mu.RLock() mb := store.blks[0] store.mu.RUnlock() mb.mu.Lock() mb.fss = nil mb.mu.Unlock() // Now send another message. _, err = js.PublishAsync("foo.bar.baz", []byte("OK")) require_NoError(t, err) // This will fail with the bug. _, err = sub.NextMsg(time.Second) require_NoError(t, err) } func TestJetStreamSubjectBasedFilteredConsumers(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q} accounts: { A: { jetstream: enabled users: [ { user: u, password: p permissions { publish { allow: [ 'ID.>', '$JS.API.INFO', '$JS.API.STREAM.>', '$JS.API.CONSUMER.INFO.>', '$JS.API.CONSUMER.CREATE.TEST.VIN-xxx.ID.foo.>', # Only allow ID.foo. ] deny: [ '$JS.API.CONSUMER.CREATE.*', '$JS.API.CONSUMER.DURABLE.CREATE.*.*'] } } } ] }, } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserInfo("u", "p"), nats.ErrorHandler(noOpErrHandler)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ID.*.*"}, }) require_NoError(t, err) for i := 0; i < 100; i++ { js.Publish(fmt.Sprintf("ID.foo.%d", i*3), nil) js.Publish(fmt.Sprintf("ID.bar.%d", i*3+1), nil) js.Publish(fmt.Sprintf("ID.baz.%d", i*3+2), nil) } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 300) // Trying to create a consumer with non filtered API should fail. js, err = nc.JetStream(nats.MaxWait(200 * time.Millisecond)) require_NoError(t, err) _, err = js.SubscribeSync("ID.foo.*") require_Error(t, err, nats.ErrTimeout, context.DeadlineExceeded) _, err = js.SubscribeSync("ID.foo.*", nats.Durable("dlc")) require_Error(t, err, nats.ErrTimeout, context.DeadlineExceeded) // Direct filtered should work. // Need to do by hand for now. ecSubj := fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "VIN-xxx", "ID.foo.*") crReq := CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverPolicy: DeliverLast, FilterSubject: "ID.foo.*", AckPolicy: AckExplicit, }, } req, err := json.Marshal(crReq) require_NoError(t, err) resp, err := nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) var ccResp JSApiConsumerCreateResponse err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) if ccResp.Error != nil { t.Fatalf("Unexpected error: %v", ccResp.Error) } cfg := ccResp.Config ci := ccResp.ConsumerInfo // Make sure we recognized as an ephemeral (since no durable was set) and that we have an InactiveThreshold. // Make sure we captured preferred ephemeral name. if ci.Name != "VIN-xxx" { t.Fatalf("Did not get correct name, expected %q got %q", "xxx", ci.Name) } if cfg.InactiveThreshold == 0 { t.Fatalf("Expected default inactive threshold to be set, got %v", cfg.InactiveThreshold) } // Make sure we can not use different consumer name since locked above. ecSubj = fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "VIN-zzz", "ID.foo.*") _, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_Error(t, err, nats.ErrTimeout) // Now check that we error when we mismatch filtersubject. crReq = CreateConsumerRequest{ Stream: "TEST", Config: ConsumerConfig{ DeliverPolicy: DeliverLast, FilterSubject: "ID.bar.*", AckPolicy: AckExplicit, }, } req, err = json.Marshal(crReq) require_NoError(t, err) ecSubj = fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "VIN-xxx", "ID.foo.*") resp, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_NoError(t, err) err = json.Unmarshal(resp.Data, &ccResp) require_NoError(t, err) checkNatsError(t, ccResp.Error, JSConsumerCreateFilterSubjectMismatchErr) // Now make sure if we change subject to match that we can not create a filtered consumer on ID.bar.> ecSubj = fmt.Sprintf(JSApiConsumerCreateExT, "TEST", "VIN-xxx", "ID.bar.*") _, err = nc.Request(ecSubj, req, 500*time.Millisecond) require_Error(t, err, nats.ErrTimeout) } func TestJetStreamStreamSubjectsOverlap(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*", "foo.A"}, }) require_Error(t, err) require_True(t, strings.Contains(err.Error(), "overlaps")) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*", "foo.A"}, }) require_Error(t, err) require_True(t, strings.Contains(err.Error(), "overlaps")) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.bar.*", "foo.*.bar"}, }) require_Error(t, err) require_True(t, strings.Contains(err.Error(), "overlaps")) } func TestJetStreamStreamTransformOverlap(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR", Mirror: &nats.StreamSource{Name: "TEST", SubjectTransforms: []nats.SubjectTransformConfig{ { Source: "foo.*.bar", Destination: "baz", }, { Source: "foo.bar.*", Destination: "baz", }, }, }}) require_Error(t, err) require_True(t, strings.Contains(err.Error(), "overlap")) } func TestJetStreamSuppressAllowDirect(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"key.*"}, MaxMsgsPerSubject: 1, AllowDirect: true, }) require_NoError(t, err) require_True(t, si.Config.AllowDirect) si, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"key.*"}, MaxMsgsPerSubject: 1, AllowDirect: false, }) require_NoError(t, err) require_False(t, si.Config.AllowDirect) sendStreamMsg(t, nc, "key.22", "msg") _, err = js.GetLastMsg("TEST", "foo", nats.DirectGet(), nats.MaxWait(100*time.Millisecond)) require_Error(t, err) } func TestJetStreamPullConsumerNoAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ORDERS.*"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) } func TestJetStreamAccountPurge(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, accpub := createKey(t) accClaim := jwt.NewAccountClaims(accpub) accClaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 * 5 accClaim.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 * 5 accJwt := encodeClaim(t, accClaim, accpub) accCreds := newUser(t, accKp) storeDir := t.TempDir() cfg := createConfFile(t, []byte(fmt.Sprintf(` host: 127.0.0.1 port:-1 server_name: S1 operator: %s system_account: %s resolver: { type: full dir: '%s/jwt' } jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s/js'} `, ojwt, syspub, storeDir, storeDir))) defer os.Remove(cfg) s, o := RunServerWithConfig(cfg) updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1) defer s.Shutdown() inspectDirs := func(t *testing.T, accTotal int) error { t.Helper() if accTotal == 0 { files, err := os.ReadDir(filepath.Join(o.StoreDir, "jetstream", accpub)) require_True(t, len(files) == accTotal || err != nil) } else { files, err := os.ReadDir(filepath.Join(o.StoreDir, "jetstream", accpub, "streams")) require_NoError(t, err) require_True(t, len(files) == accTotal) } return nil } createTestData := func() { nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"foo"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST1", &nats.ConsumerConfig{Durable: "DUR1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) } purge := func(t *testing.T) { t.Helper() var resp JSApiAccountPurgeResponse ncsys := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds)) defer ncsys.Close() m, err := ncsys.Request(fmt.Sprintf(JSApiAccountPurgeT, accpub), nil, 5*time.Second) require_NoError(t, err) err = json.Unmarshal(m.Data, &resp) require_NoError(t, err) require_True(t, resp.Initiated) } createTestData() inspectDirs(t, 1) purge(t) inspectDirs(t, 0) createTestData() inspectDirs(t, 1) s.Shutdown() require_NoError(t, os.Remove(storeDir+"/jwt/"+accpub+".jwt")) s, o = RunServerWithConfig(o.ConfigFile) defer s.Shutdown() inspectDirs(t, 1) purge(t) inspectDirs(t, 0) } func TestJetStreamPullConsumerLastPerSubjectRedeliveries(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, }) require_NoError(t, err) for i := 0; i < 20; i++ { sendStreamMsg(t, nc, fmt.Sprintf("foo.%v", i), "msg") } // Create a pull sub with a maxackpending that is <= of the number of // messages in the stream and as much as we are going to Fetch() below. sub, err := js.PullSubscribe(">", "dur", nats.AckExplicit(), nats.BindStream("TEST"), nats.DeliverLastPerSubject(), nats.MaxAckPending(10), nats.MaxRequestBatch(10), nats.AckWait(250*time.Millisecond)) require_NoError(t, err) // Fetch the max number of message we can get, and don't ack them. _, err = sub.Fetch(10, nats.MaxWait(time.Second)) require_NoError(t, err) // Wait for more than redelivery time. time.Sleep(500 * time.Millisecond) // Fetch again, make sure we can get those 10 messages. msgs, err := sub.Fetch(10, nats.MaxWait(time.Second)) require_NoError(t, err) require_True(t, len(msgs) == 10) // Make sure those were the first 10 messages for i, m := range msgs { if m.Subject != fmt.Sprintf("foo.%v", i) { t.Fatalf("Expected message for subject foo.%v, got %v", i, m.Subject) } m.Ack() } } func TestJetStreamPullConsumersTimeoutHeaders(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) nc.Publish("foo.foo", []byte("foo")) nc.Publish("foo.bar", []byte("bar")) nc.Publish("foo.else", []byte("baz")) nc.Flush() // We will do low level requests by hand for this test as to not depend on any client impl. rsubj := fmt.Sprintf(JSApiRequestNextT, "TEST", "dlc") maxBytes := 1024 batch := 50 req := &JSApiConsumerGetNextRequest{Batch: batch, Expires: 100 * time.Millisecond, NoWait: false, MaxBytes: maxBytes} jreq, err := json.Marshal(req) require_NoError(t, err) // Create listener. reply, msgs := nats.NewInbox(), make(chan *nats.Msg, batch) sub, err := nc.ChanSubscribe(reply, msgs) require_NoError(t, err) defer sub.Unsubscribe() // Send request. err = nc.PublishRequest(rsubj, reply, jreq) require_NoError(t, err) bytesReceived := 0 messagesReceived := 0 checkHeaders := func(expectedStatus, expectedDesc string, m *nats.Msg) { t.Helper() if value := m.Header.Get("Status"); value != expectedStatus { t.Fatalf("Expected status %q, got %q", expectedStatus, value) } if value := m.Header.Get("Description"); value != expectedDesc { t.Fatalf("Expected description %q, got %q", expectedDesc, value) } if value := m.Header.Get(JSPullRequestPendingMsgs); value != fmt.Sprint(batch-messagesReceived) { t.Fatalf("Expected %d messages, got %s", batch-messagesReceived, value) } if value := m.Header.Get(JSPullRequestPendingBytes); value != fmt.Sprint(maxBytes-bytesReceived) { t.Fatalf("Expected %d bytes, got %s", maxBytes-bytesReceived, value) } } for done := false; !done; { select { case m := <-msgs: if len(m.Data) == 0 && m.Header != nil { checkHeaders("408", "Request Timeout", m) done = true } else { messagesReceived += 1 bytesReceived += (len(m.Data) + len(m.Header) + len(m.Reply) + len(m.Subject)) } case <-time.After(100 + 250*time.Millisecond): t.Fatalf("Did not receive all the msgs in time") } } // Now resend the request but then shutdown the server and // make sure we have the same info. err = nc.PublishRequest(rsubj, reply, jreq) require_NoError(t, err) natsFlush(t, nc) s.Shutdown() // It is possible that the client did not receive, so let's not fail // on that. But if the 409 indicating the server is shutdown // is received, then it should have the new headers. messagesReceived, bytesReceived = 0, 0 select { case m := <-msgs: checkHeaders("409", "Server Shutdown", m) case <-time.After(500 * time.Millisecond): // we can't fail for that. t.Logf("Subscription did not receive the pull request response on server shutdown") } } // For issue https://github.com/nats-io/nats-server/issues/3612 // Do auto cleanup. func TestJetStreamDanglingMessageAutoCleanup(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub, err := js.PullSubscribe("foo", "dlc", nats.MaxAckPending(10)) require_NoError(t, err) // Send 100 msgs n := 100 for i := 0; i < n; i++ { sendStreamMsg(t, nc, "foo", "msg") } // Grab and ack 10 messages. for _, m := range fetchMsgs(t, sub, 10, time.Second) { m.AckSync() } ci, err := sub.ConsumerInfo() require_NoError(t, err) require_True(t, ci.AckFloor.Stream == 10) // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // We will hand move the ackfloor to simulate dangling message condition. cstore := filepath.Join(sd, "$G", "streams", "TEST", "obs", "dlc", "o.dat") buf, err := os.ReadFile(cstore) require_NoError(t, err) state, err := decodeConsumerState(buf) require_NoError(t, err) // Update from 10 for delivered and ack to 90. state.Delivered.Stream, state.Delivered.Consumer = 90, 90 state.AckFloor.Stream, state.AckFloor.Consumer = 90, 90 err = os.WriteFile(cstore, encodeConsumerState(state), defaultFilePerms) require_NoError(t, err) // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 10 { t.Fatalf("Expected auto-cleanup to have worked but got %d msgs vs 10", si.State.Msgs) } } // Issue https://github.com/nats-io/nats-server/issues/3645 func TestJetStreamMsgIDHeaderCollision(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ORDERS.*"}, }) require_NoError(t, err) m := nats.NewMsg("ORDERS.test") m.Header.Add(JSMsgId, "1") m.Data = []byte("ok") _, err = js.PublishMsg(m) require_NoError(t, err) m.Header = make(nats.Header) m.Header.Add("Orig-Nats-Msg-Id", "1") _, err = js.PublishMsg(m) require_NoError(t, err) m.Header = make(nats.Header) m.Header.Add("Original-Nats-Msg-Id", "1") _, err = js.PublishMsg(m) require_NoError(t, err) m.Header = make(nats.Header) m.Header.Add("Original-Nats-Msg-Id", "1") m.Header.Add("Really-Original-Nats-Msg-Id", "1") _, err = js.PublishMsg(m) require_NoError(t, err) m.Header = make(nats.Header) m.Header.Add("X", "Nats-Msg-Id:1") _, err = js.PublishMsg(m) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 5) } // https://github.com/nats-io/nats-server/issues/3657 func TestJetStreamServerCrashOnPullConsumerDeleteWithInactiveThresholdAfterAck(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "foo", "msg") sub, err := js.PullSubscribe("foo", "dlc", nats.InactiveThreshold(10*time.Second)) require_NoError(t, err) msgs := fetchMsgs(t, sub, 1, time.Second) require_True(t, len(msgs) == 1) msgs[0].Ack() err = js.DeleteConsumer("TEST", "dlc") require_NoError(t, err) // If server crashes this will fail. _, err = js.StreamInfo("TEST") require_NoError(t, err) } func TestJetStreamConsumerMultipleSubjectsLast(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events", "data", "other"}, Name: "name", }) if err != nil { t.Fatalf("error while creating stream") } sendStreamMsg(t, nc, "events", "1") sendStreamMsg(t, nc, "data", "2") sendStreamMsg(t, nc, "other", "3") sendStreamMsg(t, nc, "events", "4") sendStreamMsg(t, nc, "data", "5") sendStreamMsg(t, nc, "data", "6") sendStreamMsg(t, nc, "other", "7") sendStreamMsg(t, nc, "other", "8") // if they're not the same, expect error _, err = mset.addConsumer(&ConsumerConfig{ DeliverPolicy: DeliverLast, AckPolicy: AckExplicit, DeliverSubject: "deliver", FilterSubjects: []string{"events", "data"}, Durable: durable, }) require_NoError(t, err) sub, err := js.SubscribeSync(_EMPTY_, nats.Bind("name", durable)) require_NoError(t, err) msg, err := sub.NextMsg(time.Millisecond * 500) require_NoError(t, err) j, err := strconv.Atoi(string(msg.Data)) require_NoError(t, err) expectedStreamSeq := 6 if j != expectedStreamSeq { t.Fatalf("wrong sequence, expected %v got %v", expectedStreamSeq, j) } require_NoError(t, msg.AckSync()) // check if we don't get more than we wanted msg, err = sub.NextMsg(time.Millisecond * 500) if msg != nil || err == nil { t.Fatalf("should not get more messages") } info, err := js.ConsumerInfo("name", durable) require_NoError(t, err) require_True(t, info.NumAckPending == 0) require_True(t, info.AckFloor.Stream == 8) require_True(t, info.AckFloor.Consumer == 1) require_True(t, info.NumPending == 0) } func TestJetStreamConsumerMultipleSubjectsLastPerSubject(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events.*", "data.>", "other"}, Name: "name", }) if err != nil { t.Fatalf("error while creating stream") } sendStreamMsg(t, nc, "events.1", "bad") sendStreamMsg(t, nc, "events.1", "events.1") sendStreamMsg(t, nc, "data.1", "bad") sendStreamMsg(t, nc, "data.1", "bad") sendStreamMsg(t, nc, "data.1", "bad") sendStreamMsg(t, nc, "data.1", "bad") sendStreamMsg(t, nc, "data.1", "data.1") sendStreamMsg(t, nc, "events.2", "bad") sendStreamMsg(t, nc, "events.2", "bad") // this is last proper sequence, sendStreamMsg(t, nc, "events.2", "events.2") sendStreamMsg(t, nc, "other", "bad") sendStreamMsg(t, nc, "other", "bad") // if they're not the same, expect error _, err = mset.addConsumer(&ConsumerConfig{ DeliverPolicy: DeliverLastPerSubject, AckPolicy: AckExplicit, DeliverSubject: "deliver", FilterSubjects: []string{"events.*", "data.>"}, Durable: durable, }) require_NoError(t, err) sub, err := js.SubscribeSync("", nats.Bind("name", durable)) require_NoError(t, err) checkMessage := func(t *testing.T, subject string, payload string, ack bool) { msg, err := sub.NextMsg(time.Millisecond * 500) require_NoError(t, err) if string(msg.Data) != payload { t.Fatalf("expected %v paylaod, got %v", payload, string(msg.Data)) } if subject != msg.Subject { t.Fatalf("expected %v subject, got %v", subject, msg.Subject) } if ack { msg.AckSync() } } checkMessage(t, "events.1", "events.1", true) checkMessage(t, "data.1", "data.1", true) checkMessage(t, "events.2", "events.2", false) info, err := js.ConsumerInfo("name", durable) require_NoError(t, err) require_True(t, info.AckFloor.Consumer == 2) require_True(t, info.AckFloor.Stream == 9) require_True(t, info.Delivered.Stream == 12) require_True(t, info.Delivered.Consumer == 3) require_NoError(t, err) } func TestJetStreamConsumerMultipleSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, js := jsClientConnect(t, s) defer nc.Close() mset, err := s.GlobalAccount().addStream(&StreamConfig{ Subjects: []string{"events.>", "data.>"}, Name: "name", }) require_NoError(t, err) for i := 0; i < 20; i += 2 { sendStreamMsg(t, nc, "events.created", fmt.Sprintf("created %v", i)) sendStreamMsg(t, nc, "data.processed", fmt.Sprintf("processed %v", i+1)) } _, err = mset.addConsumer(&ConsumerConfig{ Durable: durable, DeliverSubject: "deliver", FilterSubjects: []string{"events.created", "data.processed"}, AckPolicy: AckExplicit, }) require_NoError(t, err) sub, err := js.SubscribeSync("", nats.Bind("name", durable)) require_NoError(t, err) for i := 0; i < 20; i++ { msg, err := sub.NextMsg(time.Millisecond * 500) require_NoError(t, err) require_NoError(t, msg.AckSync()) } info, err := js.ConsumerInfo("name", durable) require_NoError(t, err) require_True(t, info.NumAckPending == 0) require_True(t, info.NumPending == 0) require_True(t, info.AckFloor.Consumer == 20) require_True(t, info.AckFloor.Stream == 20) } func TestJetStreamConsumerMultipleSubjectsWithEmpty(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Subjects: []string{"events.>"}, Name: "name", }) require_NoError(t, err) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "events.created", fmt.Sprintf("%v", i)) } // if they're not the same, expect error _, err = js.AddConsumer("name", &nats.ConsumerConfig{ DeliverSubject: "deliver", FilterSubject: "", Durable: durable, AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) sub, err := js.SubscribeSync("", nats.Bind("name", durable)) require_NoError(t, err) for i := 0; i < 9; i++ { msg, err := sub.NextMsg(time.Millisecond * 500) require_NoError(t, err) j, err := strconv.Atoi(string(msg.Data)) require_NoError(t, err) if j != i { t.Fatalf("wrong sequence, expected %v got %v", i, j) } require_NoError(t, msg.AckSync()) } info, err := js.ConsumerInfo("name", durable) require_NoError(t, err) require_True(t, info.Delivered.Stream == 10) require_True(t, info.Delivered.Consumer == 10) require_True(t, info.AckFloor.Stream == 9) require_True(t, info.AckFloor.Consumer == 9) require_True(t, info.NumAckPending == 1) resp := createConsumer(t, nc, "name", ConsumerConfig{ FilterSubjects: []string{""}, DeliverSubject: "multiple", Durable: "multiple", AckPolicy: AckExplicit, }) require_True(t, resp.Error.ErrCode == 10139) } func SingleFilterConsumerCheck(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events.>"}, Name: "deliver", }) require_NoError(t, err) // if they're not the same, expect error _, err = mset.addConsumer(&ConsumerConfig{ DeliverSubject: "deliver", FilterSubject: "SINGLE", Durable: durable, }) require_Error(t, err) } // createConsumer is a temporary method until nats.go client supports multiple subjects. // it is used where lowe level call on mset is not enough, as we want to test error validation. func createConsumer(t *testing.T, nc *nats.Conn, stream string, config ConsumerConfig) JSApiConsumerCreateResponse { req, err := json.Marshal(&CreateConsumerRequest{Stream: stream, Config: config}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf("$JS.API.CONSUMER.DURABLE.CREATE.%s.%s", stream, config.Durable), req, time.Second*10) require_NoError(t, err) var apiResp JSApiConsumerCreateResponse require_NoError(t, json.Unmarshal(resp.Data, &apiResp)) return apiResp } func TestJetStreamConsumerOverlappingSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() _, err := acc.addStream(&StreamConfig{ Subjects: []string{"events.>"}, Name: "deliver", }) require_NoError(t, err) resp := createConsumer(t, nc, "deliver", ConsumerConfig{ FilterSubjects: []string{"events.one", "events.*"}, Durable: "name", }) if resp.Error.ErrCode != 10138 { t.Fatalf("this should error as we have overlapping subjects, got %+v", resp.Error) } } func TestJetStreamBothFiltersSet(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() _, err := acc.addStream(&StreamConfig{ Subjects: []string{"events.>"}, Name: "deliver", }) require_NoError(t, err) resp := createConsumer(t, nc, "deliver", ConsumerConfig{ FilterSubjects: []string{"events.one", "events.two"}, FilterSubject: "events.three", Durable: "name", }) require_True(t, resp.Error.ErrCode == 10136) } func TestJetStreamMultipleSubjectsPushBasic(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() mset, err := s.GlobalAccount().addStream(&StreamConfig{ Subjects: []string{"events", "data", "other"}, Name: "deliver", }) require_NoError(t, err) _, err = mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data"}, Durable: "name", DeliverSubject: "push", }) require_NoError(t, err) sub, err := nc.SubscribeSync("push") require_NoError(t, err) sendStreamMsg(t, nc, "other", "10") sendStreamMsg(t, nc, "events", "0") sendStreamMsg(t, nc, "data", "1") sendStreamMsg(t, nc, "events", "2") sendStreamMsg(t, nc, "events", "3") sendStreamMsg(t, nc, "other", "10") sendStreamMsg(t, nc, "data", "4") sendStreamMsg(t, nc, "data", "5") for i := 0; i < 6; i++ { msg, err := sub.NextMsg(time.Second * 1) require_NoError(t, err) if fmt.Sprintf("%v", i) != string(msg.Data) { t.Fatalf("bad sequence. Expected %v, got %v", i, string(msg.Data)) } } info, err := js.ConsumerInfo("deliver", "name") require_NoError(t, err) require_True(t, info.AckFloor.Consumer == 6) require_True(t, info.AckFloor.Stream == 8) } func TestJetStreamMultipleSubjectsBasic(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events", "data", "other"}, Name: "deliver", }) require_NoError(t, err) mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data"}, Durable: "name", }) require_NoError(t, err) sendStreamMsg(t, nc, "other", "10") sendStreamMsg(t, nc, "events", "0") sendStreamMsg(t, nc, "data", "1") sendStreamMsg(t, nc, "events", "2") sendStreamMsg(t, nc, "events", "3") sendStreamMsg(t, nc, "other", "10") sendStreamMsg(t, nc, "data", "4") sendStreamMsg(t, nc, "data", "5") consumer, err := js.PullSubscribe("", "name", nats.Bind("deliver", "name")) require_NoError(t, err) msg, err := consumer.Fetch(6) require_NoError(t, err) for i, msg := range msg { if fmt.Sprintf("%v", i) != string(msg.Data) { t.Fatalf("bad sequence. Expected %v, got %v", i, string(msg.Data)) } } _, err = js.ConsumerInfo("deliver", "name") require_NoError(t, err) } func TestJetStreamKVDelete(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "deletion", History: 10, }) require_NoError(t, err) kv.Put("a", nil) kv.Put("a.a", nil) kv.Put("a.b", nil) kv.Put("a.b.c", nil) keys, err := kv.Keys() require_NoError(t, err) require_True(t, len(keys) == 4) info, err := js.AddConsumer("KV_deletion", &nats.ConsumerConfig{ Name: "keys", FilterSubject: "$KV.deletion.a.*", DeliverPolicy: nats.DeliverLastPerSubjectPolicy, DeliverSubject: "keys", MaxDeliver: 1, AckPolicy: nats.AckNonePolicy, MemoryStorage: true, FlowControl: true, Heartbeat: time.Second * 5, }) require_NoError(t, err) require_True(t, info.NumPending == 2) sub, err := js.SubscribeSync("$KV.deletion.a.*", nats.Bind("KV_deletion", "keys")) require_NoError(t, err) _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) msg, err := sub.NextMsg(time.Second * 1) require_True(t, msg == nil) require_Error(t, err) require_NoError(t, kv.Delete("a.a")) require_NoError(t, kv.Delete("a.b")) watcher, err := kv.WatchAll() require_NoError(t, err) updates := watcher.Updates() keys = []string{} for v := range updates { if v == nil { break } if v.Operation() == nats.KeyValueDelete { keys = append(keys, v.Key()) } } require_True(t, len(keys) == 2) } func TestJetStreamDeliverLastPerSubjectWithKV(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", MaxMsgsPerSubject: 5, Subjects: []string{"kv.>"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "kv.a", "bad") sendStreamMsg(t, nc, "kv.a", "bad") sendStreamMsg(t, nc, "kv.a", "bad") sendStreamMsg(t, nc, "kv.a", "a") sendStreamMsg(t, nc, "kv.a.b", "bad") sendStreamMsg(t, nc, "kv.a.b", "bad") sendStreamMsg(t, nc, "kv.a.b", "a.b") sendStreamMsg(t, nc, "kv.a.b.c", "bad") sendStreamMsg(t, nc, "kv.a.b.c", "bad") sendStreamMsg(t, nc, "kv.a.b.c", "bad") sendStreamMsg(t, nc, "kv.a.b.c", "a.b.c") _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "CONSUMER", FilterSubject: "kv.>", DeliverPolicy: nats.DeliverLastPerSubjectPolicy, DeliverSubject: "deliver", MaxDeliver: 1, AckPolicy: nats.AckNonePolicy, MemoryStorage: true, FlowControl: true, Heartbeat: time.Second * 5, }) require_NoError(t, err) sub, err := js.SubscribeSync("kv.>", nats.Bind("TEST", "CONSUMER")) require_NoError(t, err) for i := 1; i <= 3; i++ { _, err := sub.NextMsg(time.Second * 1) require_NoError(t, err) } msg, err := sub.NextMsg(time.Second * 1) if err == nil || msg != nil { t.Fatalf("should not get any more messages") } } func TestJetStreamConsumerMultipleSubjectsAck(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events", "data", "other"}, Name: "deliver", }) require_NoError(t, err) _, err = mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data"}, Durable: "name", AckPolicy: AckExplicit, Replicas: 1, }) require_NoError(t, err) sendStreamMsg(t, nc, "events", "1") sendStreamMsg(t, nc, "data", "2") sendStreamMsg(t, nc, "data", "3") sendStreamMsg(t, nc, "data", "4") sendStreamMsg(t, nc, "events", "5") sendStreamMsg(t, nc, "data", "6") sendStreamMsg(t, nc, "data", "7") consumer, err := js.PullSubscribe("", "name", nats.Bind("deliver", "name")) require_NoError(t, err) msg, err := consumer.Fetch(3) require_NoError(t, err) require_True(t, len(msg) == 3) require_NoError(t, msg[0].AckSync()) require_NoError(t, msg[1].AckSync()) info, err := js.ConsumerInfo("deliver", "name") require_NoError(t, err) if info.AckFloor.Consumer != 2 { t.Fatalf("bad consumer sequence. expected %v, got %v", 2, info.AckFloor.Consumer) } if info.AckFloor.Stream != 2 { t.Fatalf("bad stream sequence. expected %v, got %v", 2, info.AckFloor.Stream) } if info.NumPending != 4 { t.Fatalf("bad num pending. Expected %v, got %v", 2, info.NumPending) } } func TestJetStreamConsumerMultipleSubjectAndNewAPI(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() _, err := acc.addStream(&StreamConfig{ Subjects: []string{"data", "events"}, Name: "deliver", }) if err != nil { t.Fatalf("error while creating stream") } req, err := json.Marshal(&CreateConsumerRequest{Stream: "deliver", Config: ConsumerConfig{ FilterSubjects: []string{"events", "data"}, Name: "name", Durable: "name", }}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf("$JS.API.CONSUMER.CREATE.%s.%s.%s", "deliver", "name", "data.>"), req, time.Second*10) var apiResp JSApiConsumerCreateResponse json.Unmarshal(resp.Data, &apiResp) require_NoError(t, err) if apiResp.Error.ErrCode != 10137 { t.Fatal("this should error as multiple subject filters is incompatible with new API and didn't") } } func TestJetStreamConsumerMultipleSubjectsWithAddedMessages(t *testing.T) { s := RunBasicJetStreamServer(t) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() durable := "durable" nc, js := jsClientConnect(t, s) defer nc.Close() acc := s.GlobalAccount() mset, err := acc.addStream(&StreamConfig{ Subjects: []string{"events.>"}, Name: "deliver", }) require_NoError(t, err) // if they're not the same, expect error _, err = mset.addConsumer(&ConsumerConfig{ DeliverSubject: "deliver", FilterSubjects: []string{"events.created", "events.processed"}, Durable: durable, AckPolicy: AckExplicit, }) require_NoError(t, err) sendStreamMsg(t, nc, "events.created", "0") sendStreamMsg(t, nc, "events.created", "1") sendStreamMsg(t, nc, "events.created", "2") sendStreamMsg(t, nc, "events.created", "3") sendStreamMsg(t, nc, "events.other", "BAD") sendStreamMsg(t, nc, "events.processed", "4") sendStreamMsg(t, nc, "events.processed", "5") sendStreamMsg(t, nc, "events.processed", "6") sendStreamMsg(t, nc, "events.other", "BAD") sendStreamMsg(t, nc, "events.processed", "7") sendStreamMsg(t, nc, "events.processed", "8") sub, err := js.SubscribeSync("", nats.Bind("deliver", durable)) if err != nil { t.Fatalf("error while subscribing to Consumer: %v", err) } for i := 0; i < 10; i++ { if i == 5 { sendStreamMsg(t, nc, "events.created", "9") } if i == 9 { sendStreamMsg(t, nc, "events.other", "BAD") sendStreamMsg(t, nc, "events.created", "11") } if i == 7 { sendStreamMsg(t, nc, "events.processed", "10") } msg, err := sub.NextMsg(time.Second * 1) require_NoError(t, err) j, err := strconv.Atoi(string(msg.Data)) require_NoError(t, err) if j != i { t.Fatalf("wrong sequence, expected %v got %v", i, j) } if err := msg.AckSync(); err != nil { t.Fatalf("error while acking the message :%v", err) } } info, err := js.ConsumerInfo("deliver", durable) require_NoError(t, err) require_True(t, info.Delivered.Consumer == 12) require_True(t, info.Delivered.Stream == 15) require_True(t, info.AckFloor.Stream == 12) require_True(t, info.AckFloor.Consumer == 10) } func TestJetStreamConsumerThreeFilters(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() mset, err := s.GlobalAccount().addStream(&StreamConfig{ Name: "TEST", Subjects: []string{"events", "data", "other", "ignored"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "ignored", "100") sendStreamMsg(t, nc, "events", "0") sendStreamMsg(t, nc, "events", "1") sendStreamMsg(t, nc, "data", "2") sendStreamMsg(t, nc, "ignored", "100") sendStreamMsg(t, nc, "data", "3") sendStreamMsg(t, nc, "other", "4") sendStreamMsg(t, nc, "data", "5") sendStreamMsg(t, nc, "other", "6") sendStreamMsg(t, nc, "data", "7") sendStreamMsg(t, nc, "ignored", "100") mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data", "other"}, Durable: "multi", AckPolicy: AckExplicit, }) consumer, err := js.PullSubscribe("", "multi", nats.Bind("TEST", "multi")) require_NoError(t, err) msgs, err := consumer.Fetch(6) require_NoError(t, err) for i, msg := range msgs { require_Equal(t, string(msg.Data), fmt.Sprintf("%d", i)) require_NoError(t, msg.AckSync()) } info, err := js.ConsumerInfo("TEST", "multi") require_NoError(t, err) require_True(t, info.Delivered.Stream == 8) require_True(t, info.Delivered.Consumer == 6) require_True(t, info.NumPending == 2) require_True(t, info.NumAckPending == 0) require_True(t, info.AckFloor.Consumer == 6) require_True(t, info.AckFloor.Stream == 8) } func TestJetStreamConsumerUpdateFilterSubjects(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() mset, err := s.GlobalAccount().addStream(&StreamConfig{ Name: "TEST", Subjects: []string{"events", "data", "other"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "other", "100") sendStreamMsg(t, nc, "events", "0") sendStreamMsg(t, nc, "events", "1") sendStreamMsg(t, nc, "data", "2") sendStreamMsg(t, nc, "data", "3") sendStreamMsg(t, nc, "other", "4") sendStreamMsg(t, nc, "data", "5") _, err = mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data"}, Durable: "multi", AckPolicy: AckExplicit, }) require_NoError(t, err) consumer, err := js.PullSubscribe("", "multi", nats.Bind("TEST", "multi")) require_NoError(t, err) msgs, err := consumer.Fetch(3) require_NoError(t, err) for i, msg := range msgs { require_Equal(t, string(msg.Data), fmt.Sprintf("%d", i)) require_NoError(t, msg.AckSync()) } _, err = mset.addConsumer(&ConsumerConfig{ FilterSubjects: []string{"events", "data", "other"}, Durable: "multi", AckPolicy: AckExplicit, }) require_NoError(t, err) updatedConsumer, err := js.PullSubscribe("", "multi", nats.Bind("TEST", "multi")) require_NoError(t, err) msgs, err = updatedConsumer.Fetch(3) require_NoError(t, err) for i, msg := range msgs { require_Equal(t, string(msg.Data), fmt.Sprintf("%d", i+3)) require_NoError(t, msg.AckSync()) } } func TestJetStreamStreamUpdateSubjectsOverlapOthers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"TEST"}, }) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"TEST", "foo.a"}, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"TEST2"}, }) require_NoError(t, err) // we expect an error updating stream TEST2 with subject that overlaps that used by TEST // foo.a fails too, but foo.* also double-check for sophisticated overlap match _, err = js.UpdateStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"TEST2", "foo.*"}, }) require_Error(t, err) require_Contains(t, err.Error(), "overlap") } func TestJetStreamMetaDataFailOnKernelFault(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", "OK") } sd := s.JetStreamConfig().StoreDir sdir := filepath.Join(sd, "$G", "streams", "TEST") s.Shutdown() // Emulate if the kernel did not flush out to disk the meta information. // so we will zero out both meta.inf and meta.sum. err = os.WriteFile(filepath.Join(sdir, JetStreamMetaFile), nil, defaultFilePerms) require_NoError(t, err) err = os.WriteFile(filepath.Join(sdir, JetStreamMetaFileSum), nil, defaultFilePerms) require_NoError(t, err) // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() // The stream will have not been recovered. So err is normal. _, err = js.StreamInfo("TEST") require_Error(t, err) // Make sure we are signaled here from healthz hs := s.healthz(nil) const expected = "JetStream stream '$G > TEST' could not be recovered" if hs.Status != "unavailable" || hs.Error == _EMPTY_ { t.Fatalf("Expected healthz to return an error") } else if hs.Error != expected { t.Fatalf("Expected healthz error %q got %q", expected, hs.Error) } // If we add it back, this should recover the msgs. _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) // Make sure we recovered. si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 10) // Now if we restart the server, meta should be correct, // and the stream should be restored. s.Shutdown() s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() // Make sure we recovered the stream correctly after re-adding. si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 10) } func TestJetstreamConsumerSingleTokenSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() filterSubject := "foo" _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{filterSubject}, }) require_NoError(t, err) req, err := json.Marshal(&CreateConsumerRequest{Stream: "TEST", Config: ConsumerConfig{ FilterSubject: filterSubject, Name: "name", }}) if err != nil { t.Fatalf("failed to marshal consumer create request: %v", err) } resp, err := nc.Request(fmt.Sprintf("$JS.API.CONSUMER.CREATE.%s.%s.%s", "TEST", "name", "not_filter_subject"), req, time.Second*10) var apiResp ApiResponse json.Unmarshal(resp.Data, &apiResp) if err != nil { t.Fatalf("failed to unmarshal response: %v", err) } if apiResp.Error == nil { t.Fatalf("expected error, got nil") } if apiResp.Error.ErrCode != 10131 { t.Fatalf("expected error 10131, got %v", apiResp.Error) } } // https://github.com/nats-io/nats-server/issues/3734 func TestJetStreamMsgBlkFailOnKernelFault(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxBytes: 10 * 1024 * 1024, // 10MB }) require_NoError(t, err) msgSize := 1024 * 1024 // 1MB msg := make([]byte, msgSize) crand.Read(msg) for i := 0; i < 20; i++ { _, err := js.Publish("foo", msg) require_NoError(t, err) } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Bytes < uint64(si.Config.MaxBytes)) // Now emulate a kernel fault that fails to write the last blk properly. mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() fs := mset.store.(*fileStore) fs.mu.RLock() require_True(t, len(fs.blks) > 2) // Here we do not grab the last one, which we handle correctly. We grab an interior one near the end. lmbf := fs.blks[len(fs.blks)-2].mfn fs.mu.RUnlock() mset.mu.RUnlock() sd := s.JetStreamConfig().StoreDir s.Shutdown() // Remove block. require_NoError(t, os.Remove(lmbf)) s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() _, err = js.GetMsg("TEST", 17) require_Error(t, err, nats.ErrMsgNotFound) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.NumDeleted == 3) // Test detailed version as well. si, err = js.StreamInfo("TEST", &nats.StreamInfoRequest{DeletedDetails: true}) require_NoError(t, err) require_True(t, si.State.NumDeleted == 3) if !reflect.DeepEqual(si.State.Deleted, []uint64{16, 17, 18}) { t.Fatalf("Expected deleted of %+v, got %+v", []uint64{16, 17, 18}, si.State.Deleted) } for i := 0; i < 20; i++ { _, err := js.Publish("foo", msg) require_NoError(t, err) } si, err = js.StreamInfo("TEST") require_NoError(t, err) if si.State.Bytes > uint64(si.Config.MaxBytes) { t.Fatalf("MaxBytes not enforced with empty interior msg blk, max %v, bytes %v", friendlyBytes(si.Config.MaxBytes), friendlyBytes(int64(si.State.Bytes))) } } func TestJetStreamPullConsumerBatchCompleted(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) msgSize := 128 msg := make([]byte, msgSize) crand.Read(msg) for i := 0; i < 10; i++ { _, err := js.Publish("foo", msg) require_NoError(t, err) } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dur", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) req := JSApiConsumerGetNextRequest{Batch: 0, MaxBytes: 1024, Expires: 250 * time.Millisecond} reqb, _ := json.Marshal(req) sub := natsSubSync(t, nc, nats.NewInbox()) err = nc.PublishRequest("$JS.API.CONSUMER.MSG.NEXT.TEST.dur", sub.Subject, reqb) require_NoError(t, err) // Expect first message to arrive normally. _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) // Second message should be info that batch is complete, but there were pending bytes. pullMsg, err := sub.NextMsg(time.Second * 1) require_NoError(t, err) if v := pullMsg.Header.Get("Status"); v != "409" { t.Fatalf("Expected 409, got: %s", v) } if v := pullMsg.Header.Get("Description"); v != "Batch Completed" { t.Fatalf("Expected Batch Completed, got: %s", v) } } func TestJetStreamConsumerAndStreamMetadata(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() metadata := map[string]string{"key": "value", "_nats_created_version": "2.9.11"} acc := s.GlobalAccount() // Check stream's first. mset, err := acc.addStream(&StreamConfig{Name: "foo", Metadata: metadata}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } if cfg := mset.config(); !reflect.DeepEqual(metadata, cfg.Metadata) { t.Fatalf("Expected a metadata of %q, got %q", metadata, cfg.Metadata) } // Now consumer o, err := mset.addConsumer(&ConsumerConfig{ Metadata: metadata, DeliverSubject: "to", AckPolicy: AckNone}) if err != nil { t.Fatalf("Unexpected error adding consumer: %v", err) } if cfg := o.config(); !reflect.DeepEqual(metadata, cfg.Metadata) { t.Fatalf("Expected a metadata of %q, got %q", metadata, cfg.Metadata) } // Test max. data := make([]byte, JSMaxMetadataLen/100) crand.Read(data) bigValue := base64.StdEncoding.EncodeToString(data) bigMetadata := make(map[string]string, 101) for i := 0; i < 101; i++ { bigMetadata[fmt.Sprintf("key%d", i)] = bigValue } _, err = acc.addStream(&StreamConfig{Name: "bar", Metadata: bigMetadata}) if err == nil || !strings.Contains(err.Error(), "stream metadata exceeds") { t.Fatalf("Expected an error but got none") } _, err = mset.addConsumer(&ConsumerConfig{ Metadata: bigMetadata, DeliverSubject: "to", AckPolicy: AckNone}) if err == nil || !strings.Contains(err.Error(), "consumer metadata exceeds") { t.Fatalf("Expected an error but got none") } } func TestJetStreamConsumerPurge(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"test.>"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "test.1", "hello") sendStreamMsg(t, nc, "test.2", "hello") sub, err := js.PullSubscribe("test.>", "consumer") require_NoError(t, err) // Purge one of the subjects. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: "test.2"}) require_NoError(t, err) info, err := js.ConsumerInfo("TEST", "consumer") require_NoError(t, err) require_True(t, info.NumPending == 1) // Expect to get message from not purged subject. _, err = sub.Fetch(1) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "OTHER", Subjects: []string{"other.>"}, }) require_NoError(t, err) // Publish two items to two subjects. sendStreamMsg(t, nc, "other.1", "hello") sendStreamMsg(t, nc, "other.2", "hello") sub, err = js.PullSubscribe("other.>", "other_consumer") require_NoError(t, err) // Purge whole stream. err = js.PurgeStream("OTHER", &nats.StreamPurgeRequest{}) require_NoError(t, err) info, err = js.ConsumerInfo("OTHER", "other_consumer") require_NoError(t, err) require_True(t, info.NumPending == 0) // This time expect error, as we purged whole stream, _, err = sub.Fetch(1) require_Error(t, err) } func TestJetStreamConsumerFilterUpdate(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>", "bar.>"}, }) require_NoError(t, err) for i := 0; i < 3; i++ { sendStreamMsg(t, nc, "foo.data", "OK") } sub, err := nc.SubscribeSync("deliver") require_NoError(t, err) js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "consumer", DeliverSubject: "deliver", FilterSubject: "foo.data", }) _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "consumer", DeliverSubject: "deliver", FilterSubject: "foo.>", }) require_NoError(t, err) sendStreamMsg(t, nc, "foo.other", "data") // This will timeout if filters were not properly updated. _, err = sub.NextMsg(time.Second * 1) require_NoError(t, err) mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkNumFilter := func(expected int) { t.Helper() mset.mu.RLock() nf := mset.numFilter mset.mu.RUnlock() if nf != expected { t.Fatalf("Expected stream's numFilter to be %d, got %d", expected, nf) } } checkNumFilter(1) // Update consumer once again, now not having any filters _, err = js.UpdateConsumer("TEST", &nats.ConsumerConfig{ Durable: "consumer", DeliverSubject: "deliver", FilterSubject: _EMPTY_, }) require_NoError(t, err) // and expect that numFilter reports correctly. checkNumFilter(0) } func TestJetStreamPurgeExAndAccounting(t *testing.T) { cases := []struct { name string cfg *nats.StreamConfig }{ {name: "MemoryStore", cfg: &nats.StreamConfig{ Name: "TEST", Storage: nats.MemoryStorage, Subjects: []string{"*"}, }}, {name: "FileStore", cfg: &nats.StreamConfig{ Name: "TEST", Storage: nats.FileStorage, Subjects: []string{"*"}, }}, } for _, c := range cases { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(c.cfg) require_NoError(t, err) msg := []byte("accounting") for i := 0; i < 100; i++ { _, err = js.Publish("foo", msg) require_NoError(t, err) _, err = js.Publish("bar", msg) require_NoError(t, err) } info, err := js.AccountInfo() require_NoError(t, err) err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Subject: "foo"}) require_NoError(t, err) ninfo, err := js.AccountInfo() require_NoError(t, err) // Make sure we did the proper accounting. if c.cfg.Storage == nats.MemoryStorage { if ninfo.Memory != info.Memory/2 { t.Fatalf("Accounting information incorrect for Memory: %d vs %d", ninfo.Memory, info.Memory/2) } } else { if ninfo.Store != info.Store/2 { t.Fatalf("Accounting information incorrect for FileStore: %d vs %d", ninfo.Store, info.Store/2) } } } } func TestJetStreamRollup(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() const STREAM = "S" const SUBJ = "S.*" js.AddStream(&nats.StreamConfig{ Name: STREAM, Subjects: []string{SUBJ}, AllowRollup: true, }) for i := 1; i <= 10; i++ { sendStreamMsg(t, nc, "S.A", fmt.Sprintf("%v", i)) sendStreamMsg(t, nc, "S.B", fmt.Sprintf("%v", i)) } sinfo, err := js.StreamInfo(STREAM) require_NoError(t, err) require_True(t, sinfo.State.Msgs == 20) cinfo, err := js.AddConsumer(STREAM, &nats.ConsumerConfig{ Durable: "DUR-A", FilterSubject: "S.A", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) require_True(t, cinfo.NumPending == 10) m := nats.NewMsg("S.A") m.Header.Set(JSMsgRollup, JSMsgRollupSubject) _, err = js.PublishMsg(m) require_NoError(t, err) cinfo, err = js.ConsumerInfo("S", "DUR-A") require_NoError(t, err) require_True(t, cinfo.NumPending == 1) sinfo, err = js.StreamInfo(STREAM) require_NoError(t, err) require_True(t, sinfo.State.Msgs == 11) cinfo, err = js.AddConsumer(STREAM, &nats.ConsumerConfig{ Durable: "DUR-B", FilterSubject: "S.B", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) require_True(t, cinfo.NumPending == 10) } func TestJetStreamPartialPurgeWithAckPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) nmsgs := 100 for i := 0; i < nmsgs; i++ { sendStreamMsg(t, nc, "foo", "OK") } sub, err := js.PullSubscribe("foo", "dlc", nats.AckWait(time.Second)) require_NoError(t, err) // Queue up all for ack pending. _, err = sub.Fetch(nmsgs) require_NoError(t, err) keep := nmsgs / 2 require_NoError(t, js.PurgeStream("TEST", &nats.StreamPurgeRequest{Keep: uint64(keep)})) // Should be able to be redelivered now. time.Sleep(2 * time.Second) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure we calculated correctly. require_True(t, ci.AckFloor.Consumer == uint64(keep)) require_True(t, ci.AckFloor.Stream == uint64(keep)) require_True(t, ci.NumAckPending == keep) require_True(t, ci.NumPending == 0) for i := 0; i < nmsgs; i++ { sendStreamMsg(t, nc, "foo", "OK") } ci, err = js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure we calculated correctly. // Top 3 will be same. require_True(t, ci.AckFloor.Consumer == uint64(keep)) require_True(t, ci.AckFloor.Stream == uint64(keep)) require_True(t, ci.NumAckPending == keep) require_True(t, ci.NumPending == uint64(nmsgs)) require_True(t, ci.NumRedelivered == 0) msgs, err := sub.Fetch(keep) require_NoError(t, err) require_True(t, len(msgs) == keep) ci, err = js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure we calculated correctly. require_True(t, ci.Delivered.Consumer == uint64(nmsgs+keep)) require_True(t, ci.Delivered.Stream == uint64(nmsgs)) require_True(t, ci.AckFloor.Consumer == uint64(keep)) require_True(t, ci.AckFloor.Stream == uint64(keep)) require_True(t, ci.NumAckPending == keep) require_True(t, ci.NumPending == uint64(nmsgs)) require_True(t, ci.NumRedelivered == keep) // Ack all. for _, m := range msgs { m.Ack() } nc.Flush() ci, err = js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Same for Delivered require_True(t, ci.Delivered.Consumer == uint64(nmsgs+keep)) require_True(t, ci.Delivered.Stream == uint64(nmsgs)) require_True(t, ci.AckFloor.Consumer == uint64(nmsgs+keep)) require_True(t, ci.AckFloor.Stream == uint64(nmsgs)) require_True(t, ci.NumAckPending == 0) require_True(t, ci.NumPending == uint64(nmsgs)) require_True(t, ci.NumRedelivered == 0) msgs, err = sub.Fetch(nmsgs) require_NoError(t, err) require_True(t, len(msgs) == nmsgs) // Ack all again for _, m := range msgs { m.Ack() } nc.Flush() ci, err = js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) // Make sure we calculated correctly. require_True(t, ci.Delivered.Consumer == uint64(nmsgs*2+keep)) require_True(t, ci.Delivered.Stream == uint64(nmsgs*2)) require_True(t, ci.AckFloor.Consumer == uint64(nmsgs*2+keep)) require_True(t, ci.AckFloor.Stream == uint64(nmsgs*2)) require_True(t, ci.NumAckPending == 0) require_True(t, ci.NumPending == 0) require_True(t, ci.NumRedelivered == 0) } func TestJetStreamPurgeWithRedeliveredPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) nmsgs := 100 for i := 0; i < nmsgs; i++ { sendStreamMsg(t, nc, "foo", "OK") } sub, err := js.PullSubscribe("foo", "dlc", nats.AckWait(time.Second)) require_NoError(t, err) // Queue up all for ack pending. msgs, err := sub.Fetch(nmsgs) require_NoError(t, err) require_True(t, len(msgs) == nmsgs) // Should be able to be redelivered now. time.Sleep(2 * time.Second) // Queue up all for ack pending again. msgs, err = sub.Fetch(nmsgs) require_NoError(t, err) require_True(t, len(msgs) == nmsgs) require_NoError(t, js.PurgeStream("TEST")) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) require_True(t, ci.Delivered.Consumer == uint64(2*nmsgs)) require_True(t, ci.Delivered.Stream == uint64(nmsgs)) require_True(t, ci.AckFloor.Consumer == uint64(2*nmsgs)) require_True(t, ci.AckFloor.Stream == uint64(nmsgs)) require_True(t, ci.NumAckPending == 0) require_True(t, ci.NumPending == 0) require_True(t, ci.NumRedelivered == 0) } func TestJetStreamConsumerAckFloorWithExpired(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, MaxAge: 2 * time.Second, }) require_NoError(t, err) nmsgs := 100 for i := 0; i < nmsgs; i++ { sendStreamMsg(t, nc, "foo", "OK") } sub, err := js.PullSubscribe("foo", "dlc", nats.AckWait(time.Second)) require_NoError(t, err) // Queue up all for ack pending. msgs, err := sub.Fetch(nmsgs) require_NoError(t, err) require_True(t, len(msgs) == nmsgs) // Let all messages expire. time.Sleep(3 * time.Second) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 0) ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) require_True(t, ci.Delivered.Consumer == uint64(nmsgs)) require_True(t, ci.Delivered.Stream == uint64(nmsgs)) require_True(t, ci.AckFloor.Consumer == uint64(nmsgs)) require_True(t, ci.AckFloor.Stream == uint64(nmsgs)) require_True(t, ci.NumAckPending == 0) require_True(t, ci.NumPending == 0) require_True(t, ci.NumRedelivered == 0) } func TestJetStreamConsumerIsFiltered(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() tests := []struct { name string streamSubjects []string filters []string isFiltered bool }{ { name: "single_subject", streamSubjects: []string{"one"}, filters: []string{"one"}, isFiltered: false, }, { name: "single_subject_filtered", streamSubjects: []string{"one.>"}, filters: []string{"one.filter"}, isFiltered: true, }, { name: "multi_subject_non_filtered", streamSubjects: []string{"multi", "foo", "bar.>"}, filters: []string{"multi", "bar.>", "foo"}, isFiltered: false, }, { name: "multi_subject_filtered_wc", streamSubjects: []string{"events", "data"}, filters: []string{"data"}, isFiltered: true, }, { name: "multi_subject_filtered", streamSubjects: []string{"machines", "floors"}, filters: []string{"machines"}, isFiltered: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { mset, err := acc.addStream(&StreamConfig{ Name: test.name, Subjects: test.streamSubjects, }) require_NoError(t, err) o, err := mset.addConsumer(&ConsumerConfig{ FilterSubjects: test.filters, Durable: test.name, }) require_NoError(t, err) require_True(t, o.isFiltered() == test.isFiltered) }) } } func TestJetStreamConsumerWithFormattingSymbol(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "Test%123", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "foo", "OK") } _, err = js.AddConsumer("Test%123", &nats.ConsumerConfig{ Durable: "Test%123", FilterSubject: "foo", DeliverSubject: "bar", }) require_NoError(t, err) sub, err := js.SubscribeSync("foo", nats.Bind("Test%123", "Test%123")) require_NoError(t, err) _, err = sub.NextMsg(time.Second * 5) require_NoError(t, err) } func TestJetStreamStreamUpdateWithExternalSource(t *testing.T) { ho := DefaultTestOptions ho.Port = -1 ho.LeafNode.Host = "127.0.0.1" ho.LeafNode.Port = -1 ho.JetStream = true ho.JetStreamDomain = "hub" ho.StoreDir = t.TempDir() hs := RunServer(&ho) defer hs.Shutdown() lu, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ho.LeafNode.Port)) require_NoError(t, err) lo1 := DefaultTestOptions lo1.Port = -1 lo1.ServerName = "a-leaf" lo1.JetStream = true lo1.StoreDir = t.TempDir() lo1.JetStreamDomain = "a-leaf" lo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}} l1 := RunServer(&lo1) defer l1.Shutdown() checkLeafNodeConnected(t, l1) // Test sources with `External` provided ncl, jsl := jsClientConnect(t, l1) defer ncl.Close() // Hub stream. _, err = jsl.AddStream(&nats.StreamConfig{Name: "stream", Subjects: []string{"leaf"}}) require_NoError(t, err) nch, jsh := jsClientConnect(t, hs) defer nch.Close() // Leaf stream. // Both streams uses the same name, as we're testing if overlap does not check against itself // if `External` stream has the same name. _, err = jsh.AddStream(&nats.StreamConfig{ Name: "stream", Subjects: []string{"hub"}, }) require_NoError(t, err) // Add `Sources`. // This should not validate subjects overlap against itself. _, err = jsh.UpdateStream(&nats.StreamConfig{ Name: "stream", Subjects: []string{"hub"}, Sources: []*nats.StreamSource{ { Name: "stream", FilterSubject: "leaf", External: &nats.ExternalStream{ APIPrefix: "$JS.a-leaf.API", }, }, }, }) require_NoError(t, err) // Specifying not existing FilterSubject should also be fine, as we do not validate `External` stream. _, err = jsh.UpdateStream(&nats.StreamConfig{ Name: "stream", Subjects: []string{"hub"}, Sources: []*nats.StreamSource{ { Name: "stream", FilterSubject: "foo", External: &nats.ExternalStream{ APIPrefix: "$JS.a-leaf.API", }, }, }, }) require_NoError(t, err) } func TestJetStreamKVHistoryRegression(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() for _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} { t.Run(storage.String(), func(t *testing.T) { js.DeleteKeyValue("TEST") kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", History: 4, Storage: storage, }) require_NoError(t, err) r1, err := kv.Create("foo", []byte("a")) require_NoError(t, err) _, err = kv.Update("foo", []byte("ab"), r1) require_NoError(t, err) err = kv.Delete("foo") require_NoError(t, err) _, err = kv.Create("foo", []byte("abc")) require_NoError(t, err) err = kv.Delete("foo") require_NoError(t, err) history, err := kv.History("foo") require_NoError(t, err) require_True(t, len(history) == 4) _, err = kv.Update("foo", []byte("abcd"), history[len(history)-1].Revision()) require_NoError(t, err) err = kv.Purge("foo") require_NoError(t, err) _, err = kv.Create("foo", []byte("abcde")) require_NoError(t, err) err = kv.Purge("foo") require_NoError(t, err) history, err = kv.History("foo") require_NoError(t, err) require_True(t, len(history) == 1) }) } } func TestJetStreamSnapshotRestoreStallAndHealthz(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"orders.*"}, }) require_NoError(t, err) for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "orders.created", "new order") } hs := s.healthz(nil) if hs.Status != "ok" || hs.Error != _EMPTY_ { t.Fatalf("Expected health to be ok, got %+v", hs) } // Simulate the staging directory for restores. This is normally cleaned up // but since its at the root of the storage directory make sure healthz is not affected. snapDir := filepath.Join(s.getJetStream().config.StoreDir, snapStagingDir) require_NoError(t, os.MkdirAll(snapDir, defaultDirPerms)) // Make sure healthz ok. hs = s.healthz(nil) if hs.Status != "ok" || hs.Error != _EMPTY_ { t.Fatalf("Expected health to be ok, got %+v", hs) } } // https://github.com/nats-io/nats-server/pull/4163 func TestJetStreamMaxBytesIgnored(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, MaxBytes: 10 * 1024 * 1024, }) require_NoError(t, err) msg := bytes.Repeat([]byte("A"), 1024*1024) for i := 0; i < 10; i++ { _, err := js.Publish("x", msg) require_NoError(t, err) } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == 9) // Stop current sd := s.JetStreamConfig().StoreDir s.Shutdown() // We will truncate blk file. mdir := filepath.Join(sd, "$G", "streams", "TEST", "msgs") // Truncate blk err = os.WriteFile(filepath.Join(mdir, "1.blk"), nil, defaultFilePerms) require_NoError(t, err) // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() for i := 0; i < 10; i++ { _, err := js.Publish("x", msg) require_NoError(t, err) } si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Bytes <= 10*1024*1024) } func TestJetStreamLastSequenceBySubjectConcurrent(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc0, js0 := jsClientConnect(t, c.randomServer()) defer nc0.Close() nc1, js1 := jsClientConnect(t, c.randomServer()) defer nc1.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: st, Replicas: 3, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do manually for now. m, err := nc0.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) require_NoError(t, err) si, err := js0.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v, respmsg: %q", err, string(m.Data)) } if si == nil || si.Config.Name != "KV" { t.Fatalf("StreamInfo is not correct %+v", si) } pub := func(js nats.JetStreamContext, subj, data, seq string) { t.Helper() m := nats.NewMsg(subj) m.Data = []byte(data) m.Header.Set(JSExpectedLastSubjSeq, seq) js.PublishMsg(m) } ready := make(chan struct{}) wg := &sync.WaitGroup{} wg.Add(2) go func() { <-ready pub(js0, "kv.foo", "0-0", "0") pub(js0, "kv.foo", "0-1", "1") pub(js0, "kv.foo", "0-2", "2") wg.Done() }() go func() { <-ready pub(js1, "kv.foo", "1-0", "0") pub(js1, "kv.foo", "1-1", "1") pub(js1, "kv.foo", "1-2", "2") wg.Done() }() time.Sleep(50 * time.Millisecond) close(ready) wg.Wait() // Read the messages. sub, err := js0.PullSubscribe(_EMPTY_, _EMPTY_, nats.BindStream("KV")) require_NoError(t, err) msgs, err := sub.Fetch(10) require_NoError(t, err) if len(msgs) != 3 { t.Errorf("Expected 3 messages, got %d", len(msgs)) } for i, m := range msgs { if m.Header.Get(JSExpectedLastSubjSeq) != fmt.Sprint(i) { t.Errorf("Expected %d for last sequence, got %q", i, m.Header.Get(JSExpectedLastSubjSeq)) } } }) } } func TestJetStreamServerReencryption(t *testing.T) { storeDir := t.TempDir() for i, algo := range []struct { from string to string }{ {"aes", "aes"}, {"aes", "chacha"}, {"chacha", "chacha"}, {"chacha", "aes"}, } { t.Run(fmt.Sprintf("%s_to_%s", algo.from, algo.to), func(t *testing.T) { streamName := fmt.Sprintf("TEST_%d", i) subjectName := fmt.Sprintf("foo_%d", i) expected := 30 checkStream := func(js nats.JetStreamContext) { si, err := js.StreamInfo(streamName) if err != nil { t.Fatal(err) } if si.State.Msgs != uint64(expected) { t.Fatalf("Should be %d messages but got %d messages", expected, si.State.Msgs) } sub, err := js.PullSubscribe(subjectName, "") if err != nil { t.Fatalf("Unexpected error: %v", err) } c := 0 for _, m := range fetchMsgs(t, sub, expected, 5*time.Second) { m.AckSync() c++ } if c != expected { t.Fatalf("Should have read back %d messages but got %d messages", expected, c) } } // First off, we start up using the original encryption key and algorithm. // We'll create a stream and populate it with some messages. t.Run("setup", func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` server_name: S22 listen: 127.0.0.1:-1 jetstream: { key: %q, cipher: %s, store_dir: %q } `, "firstencryptionkey", algo.from, storeDir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: streamName, Subjects: []string{subjectName}, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 0; i < expected; i++ { if _, err := js.Publish(subjectName, []byte("ENCRYPTED PAYLOAD!!")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } checkStream(js) }) // Next up, we will restart the server, this time with both the new key // and algorithm and also the old key. At startup, the server will detect // the change in encryption key and/or algorithm and re-encrypt the stream. t.Run("reencrypt", func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` server_name: S22 listen: 127.0.0.1:-1 jetstream: { key: %q, cipher: %s, prev_key: %q, store_dir: %q } `, "secondencryptionkey", algo.to, "firstencryptionkey", storeDir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() checkStream(js) }) // Finally, we'll restart the server using only the new key and algorithm. // At this point everything should have been re-encrypted, so we should still // be able to access the stream. t.Run("restart", func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` server_name: S22 listen: 127.0.0.1:-1 jetstream: { key: %q, cipher: %s, store_dir: %q } `, "secondencryptionkey", algo.to, storeDir))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() checkStream(js) }) }) } } func TestJetStreamLimitsToInterestPolicy(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.leader()) defer nc.Close() // This is the index of the consumer that we'll create as R1 // instead of R3, just to prove that it blocks the stream // update from happening properly. singleReplica := 3 streamCfg := nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.LimitsPolicy, Storage: nats.MemoryStorage, Replicas: 3, } stream, err := js.AddStream(&streamCfg) require_NoError(t, err) for i := 0; i < 10; i++ { replicas := streamCfg.Replicas if i == singleReplica { // Make one of the consumers R1 so that we can check // that the switch to interest-based retention is also // turning it into an R3 consumer. replicas = 1 } cname := fmt.Sprintf("test_%d", i) _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: cname, Durable: cname, AckPolicy: nats.AckAllPolicy, Replicas: replicas, }) require_NoError(t, err) } for i := 0; i < 20; i++ { _, err := js.Publish("foo", []byte{1, 2, 3, 4, 5}) require_NoError(t, err) } // Pull 10 or more messages from the stream. We will never pull // less than 10, which guarantees that the lowest ack floor of // all consumers should be 10. for i := 0; i < 10; i++ { cname := fmt.Sprintf("test_%d", i) count := 10 + i // At least 10 messages sub, err := js.PullSubscribe("foo", cname) require_NoError(t, err) msgs, err := sub.Fetch(count) require_NoError(t, err) require_Equal(t, len(msgs), count) require_NoError(t, msgs[len(msgs)-1].AckSync()) // At this point the ack floor should match the count of // messages we received. info, err := js.ConsumerInfo("TEST", cname) require_NoError(t, err) require_Equal(t, info.AckFloor.Consumer, uint64(count)) } // Try updating to interest-based. This should fail because // we have a consumer that is R1 on an R3 stream. streamCfg = stream.Config streamCfg.Retention = nats.InterestPolicy _, err = js.UpdateStream(&streamCfg) require_Error(t, err) // Now we'll make the R1 consumer an R3. cname := fmt.Sprintf("test_%d", singleReplica) cinfo, err := js.ConsumerInfo("TEST", cname) require_NoError(t, err) cinfo.Config.Replicas = streamCfg.Replicas _, _ = js.UpdateConsumer("TEST", &cinfo.Config) // TODO(nat): The jsConsumerCreateRequest update doesn't always // respond when there are no errors updating a consumer, so this // nearly always returns a timeout, despite actually doing what // it should. We'll make sure the replicas were updated by doing // another consumer info just to be sure. // require_NoError(t, err) c.waitOnAllCurrent() c.waitOnConsumerLeader(globalAccountName, "TEST", cname) cinfo, err = js.ConsumerInfo("TEST", cname) require_NoError(t, err) require_Equal(t, cinfo.Config.Replicas, streamCfg.Replicas) require_Equal(t, len(cinfo.Cluster.Replicas), streamCfg.Replicas-1) // This time it should succeed. _, err = js.UpdateStream(&streamCfg) require_NoError(t, err) // We need to wait for all nodes to have applied the new stream // configuration. c.waitOnAllCurrent() // Now we should only have 10 messages left in the stream, as // each consumer has acked at least the first 10 messages. info, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, info.State.FirstSeq, 11) require_Equal(t, info.State.Msgs, 10) } func TestJetStreamLimitsToInterestPolicyWhileAcking(t *testing.T) { for _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.leader()) defer nc.Close() streamCfg := nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.LimitsPolicy, Storage: st, Replicas: 3, } stream, err := js.AddStream(&streamCfg) require_NoError(t, err) wg := sync.WaitGroup{} ctx, cancel := context.WithCancel(context.Background()) payload := []byte(strings.Repeat("A", 128)) wg.Add(1) go func() { defer wg.Done() for range time.NewTicker(10 * time.Millisecond).C { select { case <-ctx.Done(): return default: } js.Publish("foo", payload) } }() for i := 0; i < 5; i++ { cname := fmt.Sprintf("test_%d", i) sub, err := js.PullSubscribe("foo", cname) require_NoError(t, err) wg.Add(1) go func() { defer wg.Done() for range time.NewTicker(10 * time.Millisecond).C { select { case <-ctx.Done(): return default: } msgs, err := sub.Fetch(1) if err != nil && errors.Is(err, nats.ErrTimeout) { t.Logf("ERROR: %v", err) } for _, msg := range msgs { msg.Ack() } } }() } // Leave running for a few secs then do the change on a different connection. time.Sleep(5 * time.Second) nc2, js2 := jsClientConnect(t, c.leader()) defer nc2.Close() // Try updating to interest-based and changing replicas too. streamCfg = stream.Config streamCfg.Retention = nats.InterestPolicy _, err = js2.UpdateStream(&streamCfg) require_NoError(t, err) // We need to wait for all nodes to have applied the new stream // configuration. c.waitOnAllCurrent() var retention nats.RetentionPolicy checkFor(t, 15*time.Second, 500*time.Millisecond, func() error { info, err := js2.StreamInfo("TEST", nats.MaxWait(500*time.Millisecond)) if err != nil { return err } retention = info.Config.Retention return nil }) require_Equal(t, retention, nats.InterestPolicy) // Cancel and wait for goroutines underneath. cancel() wg.Wait() }) } } func TestJetStreamUsageSyncDeadlock(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, }) require_NoError(t, err) sendStreamMsg(t, nc, "foo", "hello") // Now purposely mess up the usage that will force a sync. // Without the fix this will deadlock. jsa := s.getJetStream().lookupAccount(s.GlobalAccount()) jsa.usageMu.Lock() st, ok := jsa.usage[_EMPTY_] require_True(t, ok) st.local.store = -1000 jsa.usageMu.Unlock() sendStreamMsg(t, nc, "foo", "hello") } // https://github.com/nats-io/nats.go/issues/1382 // https://github.com/nats-io/nats-server/issues/4445 func TestJetStreamChangeMaxMessagesPerSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"one.>"}, MaxMsgsPerSubject: 5, }) require_NoError(t, err) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "one.data", "data") } expectMsgs := func(num int32) error { t.Helper() var msgs atomic.Int32 sub, err := js.Subscribe("one.>", func(msg *nats.Msg) { msgs.Add(1) msg.Ack() }) require_NoError(t, err) defer sub.Unsubscribe() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if nm := msgs.Load(); nm != num { return fmt.Errorf("expected to get %v messages, got %v instead", num, nm) } return nil }) return nil } require_NoError(t, expectMsgs(5)) js.UpdateStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"one.>"}, MaxMsgsPerSubject: 3, }) info, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, info.Config.MaxMsgsPerSubject == 3) require_True(t, info.State.Msgs == 3) require_NoError(t, expectMsgs(3)) for i := 0; i < 10; i++ { sendStreamMsg(t, nc, "one.data", "data") } require_NoError(t, expectMsgs(3)) } func TestJetStreamConsumerDefaultsFromStream(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() acc := s.GlobalAccount() if _, err := acc.addStream(&StreamConfig{ Name: "test", Subjects: []string{"test.*"}, ConsumerLimits: StreamConsumerLimits{ MaxAckPending: 15, InactiveThreshold: time.Second, }, }); err != nil { t.Fatalf("Failed to add stream: %v", err) } nc := clientConnectToServer(t, s) defer nc.Close() js, err := nc.JetStream() if err != nil { t.Fatalf("Failed to connect to JetStream: %v", err) } t.Run("InheritDefaultsFromStream", func(t *testing.T) { ci, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "InheritDefaultsFromStream", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) switch { case ci.Config.InactiveThreshold != time.Second: t.Fatalf("InactiveThreshold should be 1s, got %s", ci.Config.InactiveThreshold) case ci.Config.MaxAckPending != 15: t.Fatalf("MaxAckPending should be 15, got %d", ci.Config.MaxAckPending) } }) t.Run("CreateConsumerErrorOnExceedMaxAckPending", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "CreateConsumerErrorOnExceedMaxAckPending", MaxAckPending: 30, }) switch e := err.(type) { case *nats.APIError: if ErrorIdentifier(e.ErrorCode) != JSConsumerMaxPendingAckExcessErrF { t.Fatalf("invalid error code, got %d, wanted %d", e.ErrorCode, JSConsumerMaxPendingAckExcessErrF) } default: t.Fatalf("should have returned API error, got %T", e) } }) t.Run("CreateConsumerErrorOnExceedInactiveThreshold", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "CreateConsumerErrorOnExceedInactiveThreshold", InactiveThreshold: time.Second * 2, }) switch e := err.(type) { case *nats.APIError: if ErrorIdentifier(e.ErrorCode) != JSConsumerInactiveThresholdExcess { t.Fatalf("invalid error code, got %d, wanted %d", e.ErrorCode, JSConsumerInactiveThresholdExcess) } default: t.Fatalf("should have returned API error, got %T", e) } }) t.Run("UpdateStreamErrorOnViolateConsumerMaxAckPending", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "UpdateStreamErrorOnViolateConsumerMaxAckPending", MaxAckPending: 15, }) require_NoError(t, err) stream, err := acc.lookupStream("test") require_NoError(t, err) err = stream.update(&StreamConfig{ Name: "test", Subjects: []string{"test.*"}, ConsumerLimits: StreamConsumerLimits{ MaxAckPending: 10, }, }) if err == nil { t.Fatalf("stream update should have errored but didn't") } }) t.Run("UpdateStreamErrorOnViolateConsumerInactiveThreshold", func(t *testing.T) { _, err := js.AddConsumer("test", &nats.ConsumerConfig{ Name: "UpdateStreamErrorOnViolateConsumerInactiveThreshold", InactiveThreshold: time.Second, }) require_NoError(t, err) stream, err := acc.lookupStream("test") require_NoError(t, err) err = stream.update(&StreamConfig{ Name: "test", Subjects: []string{"test.*"}, ConsumerLimits: StreamConsumerLimits{ InactiveThreshold: time.Second / 2, }, }) if err == nil { t.Fatalf("stream update should have errored but didn't") } }) } func TestJetStreamSyncInterval(t *testing.T) { sd := t.TempDir() tmpl := ` listen: 127.0.0.1:-1 jetstream: { store_dir: %q %s }` for _, test := range []struct { name string sync string expected time.Duration always bool }{ {"Default", _EMPTY_, defaultSyncInterval, false}, {"10s", "sync_interval: 10s", time.Duration(10 * time.Second), false}, {"Always", "sync_interval: always", defaultSyncInterval, true}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, sd, test.sync))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() opts := s.getOpts() require_True(t, opts.SyncInterval == test.expected) nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"test.>"}, }) require_NoError(t, err) mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) fs := mset.store.(*fileStore) fs.mu.RLock() fsSync := fs.fcfg.SyncInterval syncAlways := fs.fcfg.SyncAlways fs.mu.RUnlock() require_True(t, fsSync == test.expected) require_True(t, syncAlways == test.always) }) } } func TestJetStreamFilteredSubjectUsesNewConsumerCreateSubject(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() extEndpoint := make(chan *nats.Msg, 1) normalEndpoint := make(chan *nats.Msg, 1) _, err := nc.ChanSubscribe(JSApiConsumerCreateEx, extEndpoint) require_NoError(t, err) _, err = nc.ChanSubscribe(JSApiConsumerCreate, normalEndpoint) require_NoError(t, err) testStreamSource := func(name string, shouldBeExtended bool, ss StreamSource) { t.Run(name, func(t *testing.T) { req := StreamConfig{ Name: name, Storage: MemoryStorage, Subjects: []string{fmt.Sprintf("foo.%s", name)}, Sources: []*StreamSource{&ss}, } reqJson, err := json.Marshal(req) require_NoError(t, err) _, err = nc.Request(fmt.Sprintf(JSApiStreamCreateT, name), reqJson, time.Second) require_NoError(t, err) select { case <-time.After(time.Second * 5): t.Fatalf("Timed out waiting for receive consumer create") case <-extEndpoint: if !shouldBeExtended { t.Fatalf("Expected normal consumer create, got extended") } case <-normalEndpoint: if shouldBeExtended { t.Fatalf("Expected extended consumer create, got normal") } } }) } testStreamSource("OneFilterSubject", true, StreamSource{ Name: "source", FilterSubject: "bar.>", }) testStreamSource("OneTransform", true, StreamSource{ Name: "source", SubjectTransforms: []SubjectTransformConfig{ { Source: "bar.one.>", Destination: "bar.two.>", }, }, }) testStreamSource("TwoTransforms", false, StreamSource{ Name: "source", SubjectTransforms: []SubjectTransformConfig{ { Source: "bar.one.>", Destination: "bar.two.>", }, { Source: "baz.one.>", Destination: "baz.two.>", }, }, }) } // Make sure when we downgrade history to a smaller number that the account info // is properly updated and all keys are still accessible. // There was a bug calculating next first that was not taking into account the dbit slots. func TestJetStreamKVReductionInHistory(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() startHistory := 4 kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "TEST", History: uint8(startHistory)}) require_NoError(t, err) numKeys, msg := 1000, bytes.Repeat([]byte("ABC"), 330) // ~1000bytes for { key := fmt.Sprintf("%X", rand.Intn(numKeys)+1) _, err = kv.Put(key, msg) require_NoError(t, err) status, err := kv.Status() require_NoError(t, err) if status.Values() >= uint64(startHistory*numKeys) { break } } info, err := js.AccountInfo() require_NoError(t, err) checkAllKeys := func() { t.Helper() // Make sure we can retrieve all of the keys. keys, err := kv.Keys() require_NoError(t, err) require_Equal(t, len(keys), numKeys) for _, key := range keys { _, err := kv.Get(key) require_NoError(t, err) } } // Quick sanity check. checkAllKeys() si, err := js.StreamInfo("KV_TEST") require_NoError(t, err) // Adjust down to history of 1. cfg := si.Config cfg.MaxMsgsPerSubject = 1 _, err = js.UpdateStream(&cfg) require_NoError(t, err) // Make sure the accounting was updated. ninfo, err := js.AccountInfo() require_NoError(t, err) require_True(t, info.Store > ninfo.Store) // Make sure all keys still accessible. checkAllKeys() } // Server issue 4685 func TestJetStreamConsumerPendingForKV(t *testing.T) { for _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} { t.Run(st.String(), func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"test.>"}, Storage: st, MaxMsgsPerSubject: 10, Discard: nats.DiscardNew, }) require_NoError(t, err) _, err = js.Publish("test.1", []byte("x")) require_NoError(t, err) var msg *nats.Msg // this is the detail that triggers the off by one, remove this code and all tests pass msg = nats.NewMsg("test.1") msg.Data = []byte("y") msg.Header.Add(nats.ExpectedLastSeqHdr, "1") _, err = js.PublishMsg(msg) require_NoError(t, err) _, err = js.Publish("test.2", []byte("x")) require_NoError(t, err) _, err = js.Publish("test.3", []byte("x")) require_NoError(t, err) _, err = js.Publish("test.4", []byte("x")) require_NoError(t, err) _, err = js.Publish("test.5", []byte("x")) require_NoError(t, err) nfo, err := js.StreamInfo("TEST", &nats.StreamInfoRequest{SubjectsFilter: ">"}) require_NoError(t, err) require_Equal(t, len(nfo.State.Subjects), 5) sub, err := js.SubscribeSync("test.>", nats.DeliverLastPerSubject()) require_NoError(t, err) msg, err = sub.NextMsg(time.Second) require_NoError(t, err) meta, err := msg.Metadata() require_NoError(t, err) require_Equal(t, meta.NumPending, 4) }) } } func TestJetStreamConsumerNakThenAckFloorMove(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) for i := 0; i < 11; i++ { js.Publish("foo", []byte("OK")) } sub, err := js.PullSubscribe("foo", "dlc", nats.AckWait(100*time.Millisecond)) require_NoError(t, err) msgs, err := sub.Fetch(11) require_NoError(t, err) // Nak first one msgs[0].Nak() // Ack 2-10 for i := 1; i < 10; i++ { msgs[i].AckSync() } // Hold onto last. lastMsg := msgs[10] ci, err := sub.ConsumerInfo() require_NoError(t, err) require_Equal(t, ci.AckFloor.Consumer, 0) require_Equal(t, ci.AckFloor.Stream, 0) require_Equal(t, ci.NumAckPending, 2) // Grab first messsage again and ack this time. msgs, err = sub.Fetch(1) require_NoError(t, err) msgs[0].AckSync() ci, err = sub.ConsumerInfo() require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 12) require_Equal(t, ci.Delivered.Stream, 11) require_Equal(t, ci.AckFloor.Consumer, 10) require_Equal(t, ci.AckFloor.Stream, 10) require_Equal(t, ci.NumAckPending, 1) // Make sure when we ack last one we collapse the AckFloor.Consumer // with the higher delivered due to re-deliveries. lastMsg.AckSync() ci, err = sub.ConsumerInfo() require_NoError(t, err) require_Equal(t, ci.Delivered.Consumer, 12) require_Equal(t, ci.Delivered.Stream, 11) require_Equal(t, ci.AckFloor.Consumer, 12) require_Equal(t, ci.AckFloor.Stream, 11) require_Equal(t, ci.NumAckPending, 0) } func TestJetStreamSubjectFilteredPurgeClearsPendingAcks(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, }) require_NoError(t, err) for i := 0; i < 5; i++ { js.Publish("foo", []byte("OK")) js.Publish("bar", []byte("OK")) } // Note that there are no subject filters here, this is deliberate // as previously the purge with filter code was checking for them. // We want to prove that unfiltered consumers also get purged. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: "my_consumer", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 10, }) require_NoError(t, err) require_Equal(t, ci.NumPending, 10) require_Equal(t, ci.NumAckPending, 0) sub, err := js.PullSubscribe(">", "", nats.Bind("TEST", "my_consumer")) require_NoError(t, err) msgs, err := sub.Fetch(10) require_NoError(t, err) require_Len(t, len(msgs), 10) ci, err = js.ConsumerInfo("TEST", "my_consumer") require_NoError(t, err) require_Equal(t, ci.NumPending, 0) require_Equal(t, ci.NumAckPending, 10) require_NoError(t, js.PurgeStream("TEST", &nats.StreamPurgeRequest{ Subject: "foo", })) ci, err = js.ConsumerInfo("TEST", "my_consumer") require_NoError(t, err) require_Equal(t, ci.NumPending, 0) require_Equal(t, ci.NumAckPending, 5) for i := 0; i < 5; i++ { js.Publish("foo", []byte("OK")) } msgs, err = sub.Fetch(5) require_NoError(t, err) require_Len(t, len(msgs), 5) ci, err = js.ConsumerInfo("TEST", "my_consumer") require_NoError(t, err) require_Equal(t, ci.NumPending, 0) require_Equal(t, ci.NumAckPending, 10) } // https://github.com/nats-io/nats-server/issues/4878 func TestJetStreamInterestStreamConsumerFilterEdit(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "INTEREST", Retention: nats.InterestPolicy, Subjects: []string{"interest.>"}, }) require_NoError(t, err) _, err = js.AddConsumer("INTEREST", &nats.ConsumerConfig{ Durable: "C0", FilterSubject: "interest.>", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err = js.Publish(fmt.Sprintf("interest.%d", i), []byte(strconv.Itoa(i))) require_NoError(t, err) } // we check we got 10 messages nfo, err := js.StreamInfo("INTEREST") require_NoError(t, err) if nfo.State.Msgs != 10 { t.Fatalf("expected 10 messages got %d", nfo.State.Msgs) } // now we lower the consumer interest from all subjects to 1, // then check the stream state and check if interest behavior still works _, err = js.UpdateConsumer("INTEREST", &nats.ConsumerConfig{ Durable: "C0", FilterSubject: "interest.1", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // we should now have only one message left nfo, err = js.StreamInfo("INTEREST") require_NoError(t, err) if nfo.State.Msgs != 1 { t.Fatalf("expected 1 message got %d", nfo.State.Msgs) } } // https://github.com/nats-io/nats-server/issues/5383 func TestJetStreamInterestStreamWithFilterSubjectsConsumer(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "INTEREST", Retention: nats.InterestPolicy, Subjects: []string{"interest.>"}, }) require_NoError(t, err) _, err = js.AddConsumer("INTEREST", &nats.ConsumerConfig{ Durable: "C", FilterSubjects: []string{"interest.a", "interest.b"}, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for _, sub := range []string{"interest.a", "interest.b", "interest.c"} { _, err = js.Publish(sub, nil) require_NoError(t, err) } // we check we got 2 messages, interest.c doesn't have interest nfo, err := js.StreamInfo("INTEREST") require_NoError(t, err) if nfo.State.Msgs != 2 { t.Fatalf("expected 2 messages got %d", nfo.State.Msgs) } } func TestJetStreamAckAllWithLargeFirstSequenceAndNoAckFloor(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, }) require_NoError(t, err) // Set first sequence to something very big here. This shows the issue with AckAll the // first time it is called and existing ack floor is 0. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 10_000_000_000}) require_NoError(t, err) // Now add in 100 msgs for i := 0; i < 100; i++ { js.Publish("foo.bar", []byte("hello")) } ss, err := js.PullSubscribe("foo.*", "C1", nats.AckAll()) require_NoError(t, err) msgs, err := ss.Fetch(10, nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) require_Equal(t, len(msgs), 10) start := time.Now() msgs[9].AckSync() if elapsed := time.Since(start); elapsed > 250*time.Millisecond { t.Fatalf("AckSync took too long %v", elapsed) } // Make sure next fetch works right away with low timeout. msgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) require_Equal(t, len(msgs), 10) _, err = js.StreamInfo("TEST", nats.MaxWait(250*time.Millisecond)) require_NoError(t, err) // Now make sure that if we ack in the middle, meaning we still have ack pending, // that we do the right thing as well. ss, err = js.PullSubscribe("foo.*", "C2", nats.AckAll()) require_NoError(t, err) msgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) require_Equal(t, len(msgs), 10) start = time.Now() msgs[5].AckSync() if elapsed := time.Since(start); elapsed > 250*time.Millisecond { t.Fatalf("AckSync took too long %v", elapsed) } // Make sure next fetch works right away with low timeout. msgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) require_Equal(t, len(msgs), 10) _, err = js.StreamInfo("TEST", nats.MaxWait(250*time.Millisecond)) require_NoError(t, err) } func TestJetStreamAckAllWithLargeFirstSequenceAndNoAckFloorWithInterestPolicy(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) // Set first sequence to something very big here. This shows the issue with AckAll the // first time it is called and existing ack floor is 0. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 10_000_000_000}) require_NoError(t, err) ss, err := js.PullSubscribe("foo.*", "C1", nats.AckAll()) require_NoError(t, err) // Now add in 100 msgs for i := 0; i < 100; i++ { js.Publish("foo.bar", []byte("hello")) } msgs, err := ss.Fetch(10, nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) require_Equal(t, len(msgs), 10) start := time.Now() msgs[5].AckSync() if elapsed := time.Since(start); elapsed > 250*time.Millisecond { t.Fatalf("AckSync took too long %v", elapsed) } // We are testing for run away loops acking messages in the stream that are not there. _, err = js.StreamInfo("TEST", nats.MaxWait(100*time.Millisecond)) require_NoError(t, err) } // Allow streams with $JS or $SYS prefixes for audit purposes but require no pub ack be set. func TestJetStreamAuditStreams(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() jsOverlap := errors.New("subjects that overlap with jetstream api require no-ack to be true") sysOverlap := errors.New("subjects that overlap with system api require no-ack to be true") _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"$JS.>"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap)) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"$JS.API.>"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap)) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"$JSC.>"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap)) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"$SYS.>"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(sysOverlap)) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{">"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(errors.New("capturing all subjects requires no-ack to be true"))) // These should all be ok if no pub ack. _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST1", Subjects: []string{"$JS.>"}, NoAck: true, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST2", Subjects: []string{"$JSC.>"}, NoAck: true, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST3", Subjects: []string{"$SYS.>"}, NoAck: true, }) require_NoError(t, err) // Since prior behavior did allow $JS.EVENT to be captured without no-ack, these might break // on a server upgrade so make sure they still work ok without --no-ack. // To avoid overlap error. err = js.DeleteStream("TEST1") require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST4", Subjects: []string{"$JS.EVENT.>"}, }) require_NoError(t, err) // Also allow $SYS.ACCOUNT to be captured without no-ack, these also might break // on a server upgrade so make sure they still work ok without --no-ack. // To avoid overlap error. err = js.DeleteStream("TEST3") require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST5", Subjects: []string{"$SYS.ACCOUNT.>"}, }) require_NoError(t, err) // We will test handling of ">" on a cluster here. // Specific test for capturing everything which will require both no-ack and replicas of 1. c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{">"}, }) require_Error(t, err, NewJSStreamInvalidConfigError(errors.New("capturing all subjects requires no-ack to be true"))) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{">"}, Replicas: 3, NoAck: true, }) require_Error(t, err, NewJSStreamInvalidConfigError(errors.New("capturing all subjects requires replicas of 1"))) // Ths should work ok. _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{">"}, Replicas: 1, NoAck: true, }) require_NoError(t, err) } // https://github.com/nats-io/nats-server/issues/5570 func TestJetStreamBadSubjectMappingStream(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "test"}) require_NoError(t, err) _, err = js.UpdateStream(&nats.StreamConfig{ Name: "test", Sources: []*nats.StreamSource{ { Name: "mapping", SubjectTransforms: []nats.SubjectTransformConfig{ { Source: "events.*", Destination: "events.{{wildcard(1)}}{{split(3,1)}}", }, }, }, }, }) require_Error(t, err, NewJSStreamUpdateError(errors.New("unable to get subject transform for source: invalid mapping destination: too many arguments passed to the function in {{wildcard(1)}}{{split(3,1)}}"))) } func TestJetStreamConsumerInfoNumPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "LIMITS", Subjects: []string{"js.in.limits"}, MaxMsgs: 100, }) require_NoError(t, err) _, err = js.AddConsumer("LIMITS", &nats.ConsumerConfig{ Name: "PULL", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) for i := 0; i < 1000; i++ { js.Publish("js.in.limits", []byte("x")) } ci, err := js.ConsumerInfo("LIMITS", "PULL") require_NoError(t, err) require_Equal(t, ci.NumPending, 100) // Now restart the server. sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() nc, js = jsClientConnect(t, s) defer nc.Close() ci, err = js.ConsumerInfo("LIMITS", "PULL") require_NoError(t, err) require_Equal(t, ci.NumPending, 100) } func TestJetStreamInterestStreamWithDuplicateMessages(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "INTEREST", Subjects: []string{"interest"}, Replicas: 1, Retention: nats.InterestPolicy, } _, err := js.AddStream(cfg) require_NoError(t, err) // Publishing the first time should give a sequence, even when there's no interest. pa, err := js.Publish("interest", nil, nats.MsgId("dedupe")) require_NoError(t, err) require_Equal(t, pa.Sequence, 1) require_Equal(t, pa.Duplicate, false) // Publishing a duplicate with no interest should return the same sequence as above. pa, err = js.Publish("interest", nil, nats.MsgId("dedupe")) require_NoError(t, err) require_Equal(t, pa.Sequence, 1) require_Equal(t, pa.Duplicate, true) } func TestJetStreamSourcingClipStartSeq(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORIGIN", Subjects: []string{"test"}, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err := js.Publish("test", nil) require_NoError(t, err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "SOURCING", Sources: []*nats.StreamSource{ { Name: "ORIGIN", OptStartSeq: 20, }, }, }) require_NoError(t, err) // Wait for sourcing consumer to be created. time.Sleep(time.Second) mset, err := s.GlobalAccount().lookupStream("ORIGIN") require_NoError(t, err) require_True(t, mset != nil) require_Len(t, len(mset.consumers), 1) for _, o := range mset.consumers { // Should have been clipped back to below 20 as only // 10 messages in the origin stream. require_Equal(t, o.sseq, 11) } } func TestJetStreamMirroringClipStartSeq(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORIGIN", Subjects: []string{"test"}, }) require_NoError(t, err) for i := 0; i < 10; i++ { _, err := js.Publish("test", nil) require_NoError(t, err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRRORING", Mirror: &nats.StreamSource{ Name: "ORIGIN", OptStartSeq: 20, }, }) require_NoError(t, err) // Wait for mirroring consumer to be created. time.Sleep(time.Second) mset, err := s.GlobalAccount().lookupStream("ORIGIN") require_NoError(t, err) require_True(t, mset != nil) require_Len(t, len(mset.consumers), 1) for _, o := range mset.consumers { // Should have been clipped back to below 20 as only // 10 messages in the origin stream. require_Equal(t, o.sseq, 11) } } func TestJetStreamMemoryPurgeClearsSubjectsState(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Storage: nats.MemoryStorage, }) require_NoError(t, err) pa, err := js.Publish("foo", nil) require_NoError(t, err) require_Equal(t, pa.Sequence, 1) // When requesting stream info, we expect one subject foo with one entry. si, err := js.StreamInfo("TEST", &nats.StreamInfoRequest{SubjectsFilter: ">"}) require_NoError(t, err) require_Len(t, len(si.State.Subjects), 1) require_Equal(t, si.State.Subjects["foo"], 1) // After purging, moving the sequence up, the subjects state should be cleared. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 100}) require_NoError(t, err) si, err = js.StreamInfo("TEST", &nats.StreamInfoRequest{SubjectsFilter: ">"}) require_NoError(t, err) require_Len(t, len(si.State.Subjects), 0) } func TestJetStreamWouldExceedLimits(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() js := s.getJetStream() require_NotNil(t, js) // Storing exactly up to the limit should work. require_False(t, js.wouldExceedLimits(MemoryStorage, int(js.config.MaxMemory))) require_False(t, js.wouldExceedLimits(FileStorage, int(js.config.MaxStore))) // Storing one more than the max should exceed limits. require_True(t, js.wouldExceedLimits(MemoryStorage, int(js.config.MaxMemory)+1)) require_True(t, js.wouldExceedLimits(FileStorage, int(js.config.MaxStore)+1)) } func TestJetStreamConsumerDontDecrementPendingCountOnSkippedMsg(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}}) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "CONSUMER"}) require_NoError(t, err) acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") requireExpected := func(expected int64) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { o.mu.RLock() npc := o.npc o.mu.RUnlock() if npc != expected { return fmt.Errorf("expected npc=%d, got %d", expected, npc) } return nil }) } // Should initially report no messages available. requireExpected(0) // A new message is available, should report in pending. _, err = js.Publish("foo", nil) require_NoError(t, err) requireExpected(1) // Pending count should decrease when the message is deleted. err = js.DeleteMsg("TEST", 1) require_NoError(t, err) requireExpected(0) // Make more messages available, should report in pending. _, err = js.Publish("foo", nil) require_NoError(t, err) _, err = js.Publish("foo", nil) require_NoError(t, err) requireExpected(2) // Simulate getNextMsg being called and the starting sequence to skip over a deleted message. // Also simulate one pending message. o.mu.Lock() o.sseq = 100 o.npc-- o.pending = make(map[uint64]*Pending) o.pending[2] = &Pending{} o.mu.Unlock() // Since this message is pending we should not decrement pending count as we've done so already. o.decStreamPending(2, "foo") requireExpected(1) // This is the deleted message that was skipped, we've hit the race condition and are not able to // fix it at this point. If we decrement then we could have decremented it twice if the message // was removed as a result of an Ack with Interest or WorkQueue retention, instead of due to contention. o.decStreamPending(3, "foo") requireExpected(1) } func TestJetStreamConsumerPendingCountAfterMsgAckAboveFloor(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) for i := 0; i < 2; i++ { _, err = js.Publish("foo", nil) require_NoError(t, err) } sub, err := js.PullSubscribe("foo", "CONSUMER") require_NoError(t, err) acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("CONSUMER") requireExpected := func(expected int64) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { o.mu.RLock() npc := o.npc o.mu.RUnlock() if npc != expected { return fmt.Errorf("expected npc=%d, got %d", expected, npc) } return nil }) } // Expect 2 messages pending. requireExpected(2) // Fetch 2 messages and ack the last. msgs, err := sub.Fetch(2) require_NoError(t, err) require_Len(t, len(msgs), 2) msg := msgs[1] err = msg.AckSync() require_NoError(t, err) // We've fetched 2 message so should report 0 pending. requireExpected(0) } // https://github.com/nats-io/nats-server/issues/6538 func TestJetStreamInterestMaxDeliveryReached(t *testing.T) { maxWait := 250 * time.Millisecond for _, useNak := range []bool{true, false} { for _, test := range []struct { title string action func(s *Server, sub *nats.Subscription) }{ { title: "fetch", action: func(s *Server, sub *nats.Subscription) { time.Sleep(time.Second) // max deliver 1 so this will fail _, err := sub.Fetch(1, nats.MaxWait(maxWait)) require_Error(t, err) }, }, { title: "expire pending", action: func(s *Server, sub *nats.Subscription) { acc, err := s.lookupAccount(globalAccountName) require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("consumer") require_NotNil(t, o) o.mu.Lock() o.forceExpirePending() o.mu.Unlock() }, }, } { title := fmt.Sprintf("nak/%s", test.title) if !useNak { title = fmt.Sprintf("no-%s", title) } t.Run(title, func(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Storage: nats.FileStorage, Subjects: []string{"test"}, Replicas: 1, Retention: nats.InterestPolicy, }) require_NoError(t, err) sub, err := js.PullSubscribe("test", "consumer", nats.AckWait(time.Second), nats.MaxDeliver(1)) require_NoError(t, err) _, err = nc.Request("test", []byte("hello"), maxWait) require_NoError(t, err) nfo, err := js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, nfo.State.Msgs, uint64(1)) msg, err := sub.Fetch(1, nats.MaxWait(maxWait)) require_NoError(t, err) require_Len(t, 1, len(msg)) if useNak { require_NoError(t, msg[0].Nak()) } cnfo, err := js.ConsumerInfo("TEST", "consumer") require_NoError(t, err) require_Equal(t, cnfo.NumAckPending, 1) test.action(s, sub) // max deliver 1 so this will fail _, err = sub.Fetch(1, nats.MaxWait(maxWait)) require_Error(t, err) cnfo, err = js.ConsumerInfo("TEST", "consumer") require_NoError(t, err) require_Equal(t, cnfo.NumAckPending, 0) nfo, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, nfo.State.Msgs, uint64(1)) sub2, err := js.PullSubscribe("test", "consumer2") require_NoError(t, err) msg, err = sub2.Fetch(1) require_NoError(t, err) require_Len(t, 1, len(msg)) require_NoError(t, msg[0].AckSync()) nfo, err = js.StreamInfo("TEST") require_NoError(t, err) require_Equal(t, nfo.State.Msgs, uint64(1)) }) } } } nats-server-2.10.27/server/jwt.go000066400000000000000000000161071477524627100166210ustar00rootroot00000000000000// Copyright 2018-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "errors" "fmt" "net" "os" "strings" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" ) // All JWTs once encoded start with this const jwtPrefix = "eyJ" // ReadOperatorJWT will read a jwt file for an operator claim. This can be a decorated file. func ReadOperatorJWT(jwtfile string) (*jwt.OperatorClaims, error) { _, claim, err := readOperatorJWT(jwtfile) return claim, err } func readOperatorJWT(jwtfile string) (string, *jwt.OperatorClaims, error) { contents, err := os.ReadFile(jwtfile) if err != nil { // Check to see if the JWT has been inlined. if !strings.HasPrefix(jwtfile, jwtPrefix) { return "", nil, err } // We may have an inline jwt here. contents = []byte(jwtfile) } defer wipeSlice(contents) theJWT, err := jwt.ParseDecoratedJWT(contents) if err != nil { return "", nil, err } opc, err := jwt.DecodeOperatorClaims(theJWT) if err != nil { return "", nil, err } return theJWT, opc, nil } // Just wipe slice with 'x', for clearing contents of nkey seed file. func wipeSlice(buf []byte) { for i := range buf { buf[i] = 'x' } } // validateTrustedOperators will check that we do not have conflicts with // assigned trusted keys and trusted operators. If operators are defined we // will expand the trusted keys in options. func validateTrustedOperators(o *Options) error { if len(o.TrustedOperators) == 0 { return nil } if o.AccountResolver == nil { return fmt.Errorf("operators require an account resolver to be configured") } if len(o.Accounts) > 0 { return fmt.Errorf("operators do not allow Accounts to be configured directly") } if len(o.Users) > 0 || len(o.Nkeys) > 0 { return fmt.Errorf("operators do not allow users to be configured directly") } if len(o.TrustedOperators) > 0 && len(o.TrustedKeys) > 0 { return fmt.Errorf("conflicting options for 'TrustedKeys' and 'TrustedOperators'") } if o.SystemAccount != _EMPTY_ { foundSys := false foundNonEmpty := false for _, op := range o.TrustedOperators { if op.SystemAccount != _EMPTY_ { foundNonEmpty = true } if op.SystemAccount == o.SystemAccount { foundSys = true break } } if foundNonEmpty && !foundSys { return fmt.Errorf("system_account in config and operator JWT must be identical") } } else if o.TrustedOperators[0].SystemAccount == _EMPTY_ { // In case the system account is neither defined in config nor in the first operator. // If it would be needed due to the nats account resolver, raise an error. switch o.AccountResolver.(type) { case *DirAccResolver, *CacheDirAccResolver: return fmt.Errorf("using nats based account resolver - the system account needs to be specified in configuration or the operator jwt") } } ver := strings.Split(strings.Split(strings.Split(VERSION, "-")[0], ".RC")[0], ".beta")[0] srvMajor, srvMinor, srvUpdate, _ := jwt.ParseServerVersion(ver) for _, opc := range o.TrustedOperators { if major, minor, update, err := jwt.ParseServerVersion(opc.AssertServerVersion); err != nil { return fmt.Errorf("operator %s expects version %s got error instead: %s", opc.Subject, opc.AssertServerVersion, err) } else if major > srvMajor { return fmt.Errorf("operator %s expected major version %d > server major version %d", opc.Subject, major, srvMajor) } else if srvMajor > major { } else if minor > srvMinor { return fmt.Errorf("operator %s expected minor version %d > server minor version %d", opc.Subject, minor, srvMinor) } else if srvMinor > minor { } else if update > srvUpdate { return fmt.Errorf("operator %s expected update version %d > server update version %d", opc.Subject, update, srvUpdate) } } // If we have operators, fill in the trusted keys. // FIXME(dlc) - We had TrustedKeys before TrustedOperators. The jwt.OperatorClaims // has a DidSign(). Use that longer term. For now we can expand in place. for _, opc := range o.TrustedOperators { if o.TrustedKeys == nil { o.TrustedKeys = make([]string, 0, 4) } if !opc.StrictSigningKeyUsage { o.TrustedKeys = append(o.TrustedKeys, opc.Subject) } o.TrustedKeys = append(o.TrustedKeys, opc.SigningKeys...) } for _, key := range o.TrustedKeys { if !nkeys.IsValidPublicOperatorKey(key) { return fmt.Errorf("trusted Keys %q are required to be a valid public operator nkey", key) } } if len(o.resolverPinnedAccounts) > 0 { for key := range o.resolverPinnedAccounts { if !nkeys.IsValidPublicAccountKey(key) { return fmt.Errorf("pinned account key %q is not a valid public account nkey", key) } } // ensure the system account (belonging to the operator can always connect) if o.SystemAccount != _EMPTY_ { o.resolverPinnedAccounts[o.SystemAccount] = struct{}{} } } // If we have an auth callout defined make sure we are not in operator mode. if o.AuthCallout != nil { return errors.New("operators do not allow authorization callouts to be configured directly") } return nil } func validateSrc(claims *jwt.UserClaims, host string) bool { if claims == nil { return false } else if len(claims.Src) == 0 { return true } else if host == "" { return false } ip := net.ParseIP(host) if ip == nil { return false } for _, cidr := range claims.Src { if _, net, err := net.ParseCIDR(cidr); err != nil { return false // should not happen as this jwt is invalid } else if net.Contains(ip) { return true } } return false } func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) { if claims == nil { return false, time.Duration(0) } else if len(claims.Times) == 0 { return true, time.Duration(0) } now := time.Now() loc := time.Local if claims.Locale != "" { var err error if loc, err = time.LoadLocation(claims.Locale); err != nil { return false, time.Duration(0) // parsing not expected to fail at this point } now = now.In(loc) } for _, timeRange := range claims.Times { y, m, d := now.Date() m = m - 1 d = d - 1 start, err := time.ParseInLocation("15:04:05", timeRange.Start, loc) if err != nil { return false, time.Duration(0) // parsing not expected to fail at this point } end, err := time.ParseInLocation("15:04:05", timeRange.End, loc) if err != nil { return false, time.Duration(0) // parsing not expected to fail at this point } if start.After(end) { start = start.AddDate(y, int(m), d) d++ // the intent is to be the next day } else { start = start.AddDate(y, int(m), d) } if start.Before(now) { end = end.AddDate(y, int(m), d) if end.After(now) { return true, end.Sub(now) } } } return false, time.Duration(0) } nats-server-2.10.27/server/jwt_test.go000066400000000000000000006445041477524627100176700ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) var ( // This matches ./configs/nkeys_jwts/test.seed oSeed = []byte("SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU") // This matches ./configs/nkeys/op.jwt ojwt = "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw" oKp nkeys.KeyPair ) func init() { var err error oKp, err = nkeys.FromSeed(oSeed) if err != nil { panic(fmt.Sprintf("Parsing oSeed failed with: %v", err)) } } func chanRecv(t *testing.T, recvChan <-chan struct{}, limit time.Duration) { t.Helper() select { case <-recvChan: case <-time.After(limit): t.Fatal("Should have received from channel") } } func opTrustBasicSetup() *Server { kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() opts := defaultServerOptions opts.TrustedKeys = []string{pub} s, c, _, _ := rawSetup(opts) c.close() return s } func buildMemAccResolver(s *Server) { mr := &MemAccResolver{} s.SetAccountResolver(mr) } func addAccountToMemResolver(s *Server, pub, jwtclaim string) { s.AccountResolver().Store(pub, jwtclaim) } func createClient(t *testing.T, s *Server, akp nkeys.KeyPair) (*testAsyncClient, *bufio.Reader, string) { return createClientWithIssuer(t, s, akp, "") } func createClientWithIssuer(t *testing.T, s *Server, akp nkeys.KeyPair, optIssuerAccount string) (*testAsyncClient, *bufio.Reader, string) { t.Helper() nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc := jwt.NewUserClaims(pub) if optIssuerAccount != "" { nuc.IssuerAccount = optIssuerAccount } ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } c, cr, l := newClientForServer(s) // Sign Nonce var info nonceInfo json.Unmarshal([]byte(l[5:]), &info) sigraw, _ := nkp.Sign([]byte(info.Nonce)) sig := base64.RawURLEncoding.EncodeToString(sigraw) cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"sig\":\"%s\"}\r\nPING\r\n", ujwt, sig) return c, cr, cs } func setupJWTTestWithClaims(t *testing.T, nac *jwt.AccountClaims, nuc *jwt.UserClaims, expected string) (*Server, nkeys.KeyPair, *testAsyncClient, *bufio.Reader) { t.Helper() akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() if nac == nil { nac = jwt.NewAccountClaims(apub) } else { nac.Subject = apub } ajwt, err := nac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() if nuc == nil { nuc = jwt.NewUserClaims(pub) } else { nuc.Subject = pub } jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } s := opTrustBasicSetup() buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, l := newClientForServer(s) // Sign Nonce var info nonceInfo json.Unmarshal([]byte(l[5:]), &info) sigraw, _ := nkp.Sign([]byte(info.Nonce)) sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK/-ERR to us. cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt, sig) wg := sync.WaitGroup{} wg.Add(1) go func() { c.parse([]byte(cs)) wg.Done() }() l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, expected) { t.Fatalf("Expected %q, got %q", expected, l) } wg.Wait() return s, akp, c, cr } func setupJWTTestWitAccountClaims(t *testing.T, nac *jwt.AccountClaims, expected string) (*Server, nkeys.KeyPair, *testAsyncClient, *bufio.Reader) { t.Helper() return setupJWTTestWithClaims(t, nac, nil, expected) } // This is used in test to create account claims and pass it // to setupJWTTestWitAccountClaims. func newJWTTestAccountClaims() *jwt.AccountClaims { // We call NewAccountClaims() because it sets some defaults. // However, this call needs a subject, but the real subject will // be set in setupJWTTestWitAccountClaims(). Use some temporary one // here. return jwt.NewAccountClaims("temp") } func setupJWTTestWithUserClaims(t *testing.T, nuc *jwt.UserClaims, expected string) (*Server, *testAsyncClient, *bufio.Reader) { t.Helper() s, _, c, cr := setupJWTTestWithClaims(t, nil, nuc, expected) return s, c, cr } // This is used in test to create user claims and pass it // to setupJWTTestWithUserClaims. func newJWTTestUserClaims() *jwt.UserClaims { // As of now, tests could simply do &jwt.UserClaims{}, but in // case some defaults are later added, we call NewUserClaims(). // However, this call needs a subject, but the real subject will // be set in setupJWTTestWithUserClaims(). Use some temporary one // here. return jwt.NewUserClaims("temp") } func TestJWTUser(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() // Check to make sure we would have an authTimer if !s.info.AuthRequired { t.Fatalf("Expect the server to require auth") } c, cr, _ := newClientForServer(s) defer c.close() // Don't send jwt field, should fail. c.parseAsync("CONNECT {\"verbose\":true,\"pedantic\":true}\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } okp, _ := nkeys.FromSeed(oSeed) // Create an account that will be expired. akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } c, cr, cs := createClient(t, s, akp) defer c.close() // PING needed to flush the +OK/-ERR to us. // This should fail too since no account resolver is defined. c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } // Ok now let's walk through and make sure all is good. // We will set the account resolver by hand to a memory resolver. buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, cs = createClient(t, s, akp) defer c.close() c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG, got %q", l) } } func TestJWTUserBadTrusted(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() // Check to make sure we would have an authTimer if !s.info.AuthRequired { t.Fatalf("Expect the server to require auth") } // Now place bad trusted key s.mu.Lock() s.trustedKeys = []string{"bad"} s.mu.Unlock() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create an account that will be expired. akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, apub, ajwt) c, cr, cs := createClient(t, s, akp) defer c.close() c.parseAsync(cs) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } } // Test that if a user tries to connect with an expired user JWT we do the right thing. func TestJWTUserExpired(t *testing.T) { nuc := newJWTTestUserClaims() nuc.IssuedAt = time.Now().Add(-10 * time.Second).Unix() nuc.Expires = time.Now().Add(-2 * time.Second).Unix() s, c, _ := setupJWTTestWithUserClaims(t, nuc, "-ERR ") c.close() s.Shutdown() } func TestJWTUserExpiresAfterConnect(t *testing.T) { nuc := newJWTTestUserClaims() nuc.IssuedAt = time.Now().Unix() nuc.Expires = time.Now().Add(time.Second).Unix() s, c, cr := setupJWTTestWithUserClaims(t, nuc, "+OK") defer s.Shutdown() defer c.close() l, err := cr.ReadString('\n') if err != nil { t.Fatalf("Received %v", err) } if !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG") } // Now we should expire after 1 second or so. time.Sleep(1250 * time.Millisecond) l, err = cr.ReadString('\n') if err != nil { t.Fatalf("Received %v", err) } if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "Expired") { t.Fatalf("Expected 'Expired' to be in the error") } } func TestJWTUserPermissionClaims(t *testing.T) { nuc := newJWTTestUserClaims() nuc.Permissions.Pub.Allow.Add("foo") nuc.Permissions.Pub.Allow.Add("bar") nuc.Permissions.Pub.Deny.Add("baz") nuc.Permissions.Sub.Allow.Add("foo") nuc.Permissions.Sub.Allow.Add("bar") nuc.Permissions.Sub.Deny.Add("baz") s, c, _ := setupJWTTestWithUserClaims(t, nuc, "+OK") defer s.Shutdown() defer c.close() // Now check client to make sure permissions transferred. c.mu.Lock() defer c.mu.Unlock() if c.perms == nil { t.Fatalf("Expected client permissions to be set") } if lpa := c.perms.pub.allow.Count(); lpa != 2 { t.Fatalf("Expected 2 publish allow subjects, got %d", lpa) } if lpd := c.perms.pub.deny.Count(); lpd != 1 { t.Fatalf("Expected 1 publish deny subjects, got %d", lpd) } if lsa := c.perms.sub.allow.Count(); lsa != 2 { t.Fatalf("Expected 2 subscribe allow subjects, got %d", lsa) } if lsd := c.perms.sub.deny.Count(); lsd != 1 { t.Fatalf("Expected 1 subscribe deny subjects, got %d", lsd) } } func TestJWTUserResponsePermissionClaims(t *testing.T) { nuc := newJWTTestUserClaims() nuc.Permissions.Resp = &jwt.ResponsePermission{ MaxMsgs: 22, Expires: 100 * time.Millisecond, } s, c, _ := setupJWTTestWithUserClaims(t, nuc, "+OK") defer s.Shutdown() defer c.close() // Now check client to make sure permissions transferred. c.mu.Lock() defer c.mu.Unlock() if c.perms == nil { t.Fatalf("Expected client permissions to be set") } if c.perms.pub.allow == nil { t.Fatalf("Expected client perms for pub allow to be non-nil") } if lpa := c.perms.pub.allow.Count(); lpa != 0 { t.Fatalf("Expected 0 publish allow subjects, got %d", lpa) } if c.perms.resp == nil { t.Fatalf("Expected client perms for response permissions to be non-nil") } if c.perms.resp.MaxMsgs != nuc.Permissions.Resp.MaxMsgs { t.Fatalf("Expected client perms for response permissions MaxMsgs to be same as jwt: %d vs %d", c.perms.resp.MaxMsgs, nuc.Permissions.Resp.MaxMsgs) } if c.perms.resp.Expires != nuc.Permissions.Resp.Expires { t.Fatalf("Expected client perms for response permissions Expires to be same as jwt: %v vs %v", c.perms.resp.Expires, nuc.Permissions.Resp.Expires) } } func TestJWTUserResponsePermissionClaimsDefaultValues(t *testing.T) { nuc := newJWTTestUserClaims() nuc.Permissions.Resp = &jwt.ResponsePermission{} s, c, _ := setupJWTTestWithUserClaims(t, nuc, "+OK") defer s.Shutdown() defer c.close() // Now check client to make sure permissions transferred // and defaults are set. c.mu.Lock() defer c.mu.Unlock() if c.perms == nil { t.Fatalf("Expected client permissions to be set") } if c.perms.pub.allow == nil { t.Fatalf("Expected client perms for pub allow to be non-nil") } if lpa := c.perms.pub.allow.Count(); lpa != 0 { t.Fatalf("Expected 0 publish allow subjects, got %d", lpa) } if c.perms.resp == nil { t.Fatalf("Expected client perms for response permissions to be non-nil") } if c.perms.resp.MaxMsgs != DEFAULT_ALLOW_RESPONSE_MAX_MSGS { t.Fatalf("Expected client perms for response permissions MaxMsgs to be default %v, got %v", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, c.perms.resp.MaxMsgs) } if c.perms.resp.Expires != DEFAULT_ALLOW_RESPONSE_EXPIRATION { t.Fatalf("Expected client perms for response permissions Expires to be default %v, got %v", DEFAULT_ALLOW_RESPONSE_EXPIRATION, c.perms.resp.Expires) } } func TestJWTUserResponsePermissionClaimsNegativeValues(t *testing.T) { nuc := newJWTTestUserClaims() nuc.Permissions.Resp = &jwt.ResponsePermission{ MaxMsgs: -1, Expires: -1 * time.Second, } s, c, _ := setupJWTTestWithUserClaims(t, nuc, "+OK") defer s.Shutdown() defer c.close() // Now check client to make sure permissions transferred // and negative values are transferred. c.mu.Lock() defer c.mu.Unlock() if c.perms == nil { t.Fatalf("Expected client permissions to be set") } if c.perms.pub.allow == nil { t.Fatalf("Expected client perms for pub allow to be non-nil") } if lpa := c.perms.pub.allow.Count(); lpa != 0 { t.Fatalf("Expected 0 publish allow subjects, got %d", lpa) } if c.perms.resp == nil { t.Fatalf("Expected client perms for response permissions to be non-nil") } if c.perms.resp.MaxMsgs != -1 { t.Fatalf("Expected client perms for response permissions MaxMsgs to be %v, got %v", -1, c.perms.resp.MaxMsgs) } if c.perms.resp.Expires != -1*time.Second { t.Fatalf("Expected client perms for response permissions Expires to be %v, got %v", -1*time.Second, c.perms.resp.Expires) } } func TestJWTAccountExpired(t *testing.T) { nac := newJWTTestAccountClaims() nac.IssuedAt = time.Now().Add(-10 * time.Second).Unix() nac.Expires = time.Now().Add(-2 * time.Second).Unix() s, _, c, _ := setupJWTTestWitAccountClaims(t, nac, "-ERR ") defer s.Shutdown() defer c.close() } func TestJWTAccountExpiresAfterConnect(t *testing.T) { nac := newJWTTestAccountClaims() now := time.Now() nac.IssuedAt = now.Add(-10 * time.Second).Unix() nac.Expires = now.Round(time.Second).Add(time.Second).Unix() s, akp, c, cr := setupJWTTestWitAccountClaims(t, nac, "+OK") defer s.Shutdown() defer c.close() apub, _ := akp.PublicKey() acc, err := s.LookupAccount(apub) if acc == nil || err != nil { t.Fatalf("Expected to retrieve the account") } if l, _ := cr.ReadString('\n'); !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected PONG, got %q", l) } // Wait for the account to be expired. checkFor(t, 3*time.Second, 100*time.Millisecond, func() error { if acc.IsExpired() { return nil } return fmt.Errorf("Account not expired yet") }) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error, got %q", l) } if !strings.Contains(l, "Expired") { t.Fatalf("Expected 'Expired' to be in the error") } // Now make sure that accounts that have expired return an error. c, cr, cs := createClient(t, s, akp) defer c.close() c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } } func TestJWTAccountRenew(t *testing.T) { nac := newJWTTestAccountClaims() // Create an account that has expired. nac.IssuedAt = time.Now().Add(-10 * time.Second).Unix() nac.Expires = time.Now().Add(-2 * time.Second).Unix() // Expect an error s, akp, c, _ := setupJWTTestWitAccountClaims(t, nac, "-ERR ") defer s.Shutdown() defer c.close() okp, _ := nkeys.FromSeed(oSeed) apub, _ := akp.PublicKey() // Now update with new expiration nac.IssuedAt = time.Now().Unix() nac.Expires = time.Now().Add(5 * time.Second).Unix() ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Update the account addAccountToMemResolver(s, apub, ajwt) acc, _ := s.LookupAccount(apub) if acc == nil { t.Fatalf("Expected to retrieve the account") } s.UpdateAccountClaims(acc, nac) // Now make sure we can connect. c, cr, cs := createClient(t, s, akp) defer c.close() c.parseAsync(cs) if l, _ := cr.ReadString('\n'); !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG, got: %q", l) } } func TestJWTAccountRenewFromResolver(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) nac.IssuedAt = time.Now().Add(-10 * time.Second).Unix() nac.Expires = time.Now().Add(time.Second).Unix() ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, apub, ajwt) // Force it to be loaded by the server and start the expiration timer. acc, _ := s.LookupAccount(apub) if acc == nil { t.Fatalf("Could not retrieve account for %q", apub) } // Create a new user c, cr, cs := createClient(t, s, akp) defer c.close() // Wait for expiration. time.Sleep(1250 * time.Millisecond) c.parseAsync(cs) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } // Now update with new expiration nac.IssuedAt = time.Now().Unix() nac.Expires = time.Now().Add(5 * time.Second).Unix() ajwt, err = nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Update the account addAccountToMemResolver(s, apub, ajwt) // Make sure the too quick update suppression does not bite us. acc.mu.Lock() acc.updated = time.Now().UTC().Add(-1 * time.Hour) acc.mu.Unlock() // Do not update the account directly. The resolver should // happen automatically. // Now make sure we can connect. c, cr, cs = createClient(t, s, akp) defer c.close() c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG, got: %q", l) } } func TestJWTAccountBasicImportExport(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) // Now create Exports. streamExport := &jwt.Export{Subject: "foo", Type: jwt.Stream} streamExport2 := &jwt.Export{Subject: "private", Type: jwt.Stream, TokenReq: true} serviceExport := &jwt.Export{Subject: "req.echo", Type: jwt.Service, TokenReq: true} serviceExport2 := &jwt.Export{Subject: "req.add", Type: jwt.Service, TokenReq: true} fooAC.Exports.Add(streamExport, streamExport2, serviceExport, serviceExport2) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) acc, _ := s.LookupAccount(fooPub) if acc == nil { t.Fatalf("Expected to retrieve the account") } // Check to make sure exports transferred over. if les := len(acc.exports.streams); les != 2 { t.Fatalf("Expected exports streams len of 2, got %d", les) } if les := len(acc.exports.services); les != 2 { t.Fatalf("Expected exports services len of 2, got %d", les) } _, ok := acc.exports.streams["foo"] if !ok { t.Fatalf("Expected to map a stream export") } se, ok := acc.exports.services["req.echo"] if !ok || se == nil { t.Fatalf("Expected to map a service export") } if !se.tokenReq { t.Fatalf("Expected the service export to require tokens") } barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) streamImport := &jwt.Import{Account: fooPub, Subject: "foo", To: "import.foo", Type: jwt.Stream} serviceImport := &jwt.Import{Account: fooPub, Subject: "req.echo", Type: jwt.Service} barAC.Imports.Add(streamImport, serviceImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) acc, _ = s.LookupAccount(barPub) if acc == nil { t.Fatalf("Expected to retrieve the account") } if les := len(acc.imports.streams); les != 1 { t.Fatalf("Expected imports streams len of 1, got %d", les) } // Our service import should have failed without a token. if les := len(acc.imports.services); les != 0 { t.Fatalf("Expected imports services len of 0, got %d", les) } // Now add in a bad activation token. barAC = jwt.NewAccountClaims(barPub) serviceImport = &jwt.Import{Account: fooPub, Subject: "req.echo", Token: "not a token", Type: jwt.Service} barAC.Imports.Add(serviceImport) barJWT, err = barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) s.UpdateAccountClaims(acc, barAC) // Our service import should have failed with a bad token. if les := len(acc.imports.services); les != 0 { t.Fatalf("Expected imports services len of 0, got %d", les) } // Now make a correct one. barAC = jwt.NewAccountClaims(barPub) serviceImport = &jwt.Import{Account: fooPub, Subject: "req.echo", Type: jwt.Service} activation := jwt.NewActivationClaims(barPub) activation.ImportSubject = "req.echo" activation.ImportType = jwt.Service actJWT, err := activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } serviceImport.Token = actJWT barAC.Imports.Add(serviceImport) barJWT, err = barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } vr := jwt.ValidationResults{} barAC.Validate(&vr) if vr.IsBlocking(true) { t.Fatalf("Error generating account JWT: %v", vr) } addAccountToMemResolver(s, barPub, barJWT) s.UpdateAccountClaims(acc, barAC) // Our service import should have succeeded. if les := len(acc.imports.services); les != 1 { t.Fatalf("Expected imports services len of 1, got %d", les) } // Now streams barAC = jwt.NewAccountClaims(barPub) streamImport = &jwt.Import{Account: fooPub, Subject: "private", To: "import.private", Type: jwt.Stream} barAC.Imports.Add(streamImport) barJWT, err = barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) s.UpdateAccountClaims(acc, barAC) // Our stream import should have not succeeded. if les := len(acc.imports.streams); les != 0 { t.Fatalf("Expected imports services len of 0, got %d", les) } // Now add in activation. barAC = jwt.NewAccountClaims(barPub) streamImport = &jwt.Import{Account: fooPub, Subject: "private", To: "import.private", Type: jwt.Stream} activation = jwt.NewActivationClaims(barPub) activation.ImportSubject = "private" activation.ImportType = jwt.Stream actJWT, err = activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } streamImport.Token = actJWT barAC.Imports.Add(streamImport) barJWT, err = barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) s.UpdateAccountClaims(acc, barAC) // Our stream import should have not succeeded. if les := len(acc.imports.streams); les != 1 { t.Fatalf("Expected imports services len of 1, got %d", les) } } func TestJWTAccountExportWithResponseType(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) // Now create Exports. serviceStreamExport := &jwt.Export{Subject: "test.stream", Type: jwt.Service, ResponseType: jwt.ResponseTypeStream, TokenReq: false} serviceChunkExport := &jwt.Export{Subject: "test.chunk", Type: jwt.Service, ResponseType: jwt.ResponseTypeChunked, TokenReq: false} serviceSingletonExport := &jwt.Export{Subject: "test.single", Type: jwt.Service, ResponseType: jwt.ResponseTypeSingleton, TokenReq: true} serviceDefExport := &jwt.Export{Subject: "test.def", Type: jwt.Service, TokenReq: true} serviceOldExport := &jwt.Export{Subject: "test.old", Type: jwt.Service, TokenReq: false} fooAC.Exports.Add(serviceStreamExport, serviceSingletonExport, serviceChunkExport, serviceDefExport, serviceOldExport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) fooAcc, _ := s.LookupAccount(fooPub) if fooAcc == nil { t.Fatalf("Expected to retrieve the account") } services := fooAcc.exports.services if len(services) != 5 { t.Fatalf("Expected 4 services") } se, ok := services["test.stream"] if !ok || se == nil { t.Fatalf("Expected to map a service export") } if se.tokenReq { t.Fatalf("Expected the service export to not require tokens") } if se.respType != Streamed { t.Fatalf("Expected the service export to respond with a stream") } se, ok = services["test.chunk"] if !ok || se == nil { t.Fatalf("Expected to map a service export") } if se.tokenReq { t.Fatalf("Expected the service export to not require tokens") } if se.respType != Chunked { t.Fatalf("Expected the service export to respond with a stream") } se, ok = services["test.def"] if !ok || se == nil { t.Fatalf("Expected to map a service export") } if !se.tokenReq { t.Fatalf("Expected the service export to not require tokens") } if se.respType != Singleton { t.Fatalf("Expected the service export to respond with a stream") } se, ok = services["test.single"] if !ok || se == nil { t.Fatalf("Expected to map a service export") } if !se.tokenReq { t.Fatalf("Expected the service export to not require tokens") } if se.respType != Singleton { t.Fatalf("Expected the service export to respond with a stream") } se, ok = services["test.old"] if !ok || se == nil || len(se.approved) > 0 { t.Fatalf("Service with a singleton response and no tokens should not be nil and have no approvals") } } func expectPong(t *testing.T, cr *bufio.Reader) { t.Helper() l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG, got %q", l) } } func expectMsg(t *testing.T, cr *bufio.Reader, sub, payload string) { t.Helper() l, _ := cr.ReadString('\n') expected := "MSG " + sub if !strings.HasPrefix(l, expected) { t.Fatalf("Expected %q, got %q", expected, l) } l, _ = cr.ReadString('\n') if l != payload+"\r\n" { t.Fatalf("Expected %q, got %q", payload, l) } expectPong(t, cr) } func TestJWTAccountImportExportUpdates(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) streamExport := &jwt.Export{Subject: "foo", Type: jwt.Stream} fooAC.Exports.Add(streamExport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) streamImport := &jwt.Import{Account: fooPub, Subject: "foo", To: "import", Type: jwt.Stream} barAC.Imports.Add(streamImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) // Create a client. c, cr, cs := createClient(t, s, barKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) c.parseAsync("SUB import.foo 1\r\nPING\r\n") expectPong(t, cr) checkShadow := func(expected int) { t.Helper() c.mu.Lock() defer c.mu.Unlock() sub := c.subs["1"] if ls := len(sub.shadow); ls != expected { t.Fatalf("Expected shadows to be %d, got %d", expected, ls) } } // We created a SUB on foo which should create a shadow subscription. checkShadow(1) // Now update bar and remove the import which should make the shadow go away. barAC = jwt.NewAccountClaims(barPub) barJWT, _ = barAC.Encode(okp) addAccountToMemResolver(s, barPub, barJWT) acc, _ := s.LookupAccount(barPub) s.UpdateAccountClaims(acc, barAC) checkShadow(0) // Now add it back and make sure the shadow comes back. streamImport = &jwt.Import{Account: string(fooPub), Subject: "foo", To: "import", Type: jwt.Stream} barAC.Imports.Add(streamImport) barJWT, _ = barAC.Encode(okp) addAccountToMemResolver(s, barPub, barJWT) s.UpdateAccountClaims(acc, barAC) checkShadow(1) // Now change export and make sure it goes away as well. So no exports anymore. fooAC = jwt.NewAccountClaims(fooPub) fooJWT, _ = fooAC.Encode(okp) addAccountToMemResolver(s, fooPub, fooJWT) acc, _ = s.LookupAccount(fooPub) s.UpdateAccountClaims(acc, fooAC) checkShadow(0) // Now add it in but with permission required. streamExport = &jwt.Export{Subject: "foo", Type: jwt.Stream, TokenReq: true} fooAC.Exports.Add(streamExport) fooJWT, _ = fooAC.Encode(okp) addAccountToMemResolver(s, fooPub, fooJWT) s.UpdateAccountClaims(acc, fooAC) checkShadow(0) // Now put it back as normal. fooAC = jwt.NewAccountClaims(fooPub) streamExport = &jwt.Export{Subject: "foo", Type: jwt.Stream} fooAC.Exports.Add(streamExport) fooJWT, _ = fooAC.Encode(okp) addAccountToMemResolver(s, fooPub, fooJWT) s.UpdateAccountClaims(acc, fooAC) checkShadow(1) } func TestJWTAccountImportActivationExpires(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) streamExport := &jwt.Export{Subject: "foo", Type: jwt.Stream, TokenReq: true} fooAC.Exports.Add(streamExport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) acc, _ := s.LookupAccount(fooPub) if acc == nil { t.Fatalf("Expected to retrieve the account") } barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) streamImport := &jwt.Import{Account: fooPub, Subject: "foo", To: "import.", Type: jwt.Stream} activation := jwt.NewActivationClaims(barPub) activation.ImportSubject = "foo" activation.ImportType = jwt.Stream now := time.Now() activation.IssuedAt = now.Add(-10 * time.Second).Unix() // These are second resolution. So round up before adding a second. activation.Expires = now.Round(time.Second).Add(time.Second).Unix() actJWT, err := activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } streamImport.Token = actJWT barAC.Imports.Add(streamImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) if acc, _ := s.LookupAccount(barPub); acc == nil { t.Fatalf("Expected to retrieve the account") } // Create a client. c, cr, cs := createClient(t, s, barKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) c.parseAsync("SUB import.foo 1\r\nPING\r\n") expectPong(t, cr) checkShadow := func(t *testing.T, expected int) { t.Helper() checkFor(t, 3*time.Second, 15*time.Millisecond, func() error { c.mu.Lock() defer c.mu.Unlock() sub := c.subs["1"] if ls := len(sub.shadow); ls != expected { return fmt.Errorf("Expected shadows to be %d, got %d", expected, ls) } return nil }) } // We created a SUB on foo which should create a shadow subscription. checkShadow(t, 1) time.Sleep(1250 * time.Millisecond) // Should have expired and been removed. checkShadow(t, 0) } func TestJWTAccountLimitsSubs(t *testing.T) { fooAC := newJWTTestAccountClaims() fooAC.Limits.Subs = 10 s, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, "+OK") defer s.Shutdown() defer c.close() okp, _ := nkeys.FromSeed(oSeed) fooPub, _ := fooKP.PublicKey() // Create a client. c, cr, cs := createClient(t, s, fooKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) // Check to make sure we have the limit set. // Account first fooAcc, _ := s.LookupAccount(fooPub) fooAcc.mu.RLock() if fooAcc.msubs != 10 { fooAcc.mu.RUnlock() t.Fatalf("Expected account to have msubs of 10, got %d", fooAcc.msubs) } fooAcc.mu.RUnlock() // Now test that the client has limits too. c.mu.Lock() if c.msubs != 10 { c.mu.Unlock() t.Fatalf("Expected client msubs to be 10, got %d", c.msubs) } c.mu.Unlock() // Now make sure its enforced. /// These should all work ok. for i := 0; i < 10; i++ { c.parseAsync(fmt.Sprintf("SUB foo %d\r\nPING\r\n", i)) expectPong(t, cr) } // This one should fail. c.parseAsync("SUB foo 22\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR") { t.Fatalf("Expected an ERR, got: %v", l) } if !strings.Contains(l, "maximum subscriptions exceeded") { t.Fatalf("Expected an ERR for max subscriptions exceeded, got: %v", l) } // Now update the claims and expect if max is lower to be disconnected. fooAC.Limits.Subs = 5 fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) s.UpdateAccountClaims(fooAcc, fooAC) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR") { t.Fatalf("Expected an ERR, got: %v", l) } if !strings.Contains(l, "maximum subscriptions exceeded") { t.Fatalf("Expected an ERR for max subscriptions exceeded, got: %v", l) } } func TestJWTAccountLimitsSubsButServerOverrides(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) // override with server setting of 2. opts := s.getOpts() opts.MaxSubs = 2 okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) fooAC.Limits.Subs = 10 fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) fooAcc, _ := s.LookupAccount(fooPub) fooAcc.mu.RLock() if fooAcc.msubs != 10 { fooAcc.mu.RUnlock() t.Fatalf("Expected account to have msubs of 10, got %d", fooAcc.msubs) } fooAcc.mu.RUnlock() // Create a client. c, cr, cs := createClient(t, s, fooKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) c.parseAsync("SUB foo 1\r\nSUB bar 2\r\nSUB baz 3\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "maximum subscriptions exceeded") { t.Fatalf("Expected an ERR for max subscriptions exceeded, got: %v", l) } // Read last PONG so does not hold up test. cr.ReadString('\n') } func TestJWTAccountLimitsMaxPayload(t *testing.T) { fooAC := newJWTTestAccountClaims() fooAC.Limits.Payload = 8 s, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, "+OK") defer s.Shutdown() defer c.close() fooPub, _ := fooKP.PublicKey() // Create a client. c, cr, cs := createClient(t, s, fooKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) // Check to make sure we have the limit set. // Account first fooAcc, _ := s.LookupAccount(fooPub) fooAcc.mu.RLock() if fooAcc.mpay != 8 { fooAcc.mu.RUnlock() t.Fatalf("Expected account to have mpay of 8, got %d", fooAcc.mpay) } fooAcc.mu.RUnlock() // Now test that the client has limits too. c.mu.Lock() if c.mpay != 8 { c.mu.Unlock() t.Fatalf("Expected client to have mpay of 10, got %d", c.mpay) } c.mu.Unlock() c.parseAsync("PUB foo 4\r\nXXXX\r\nPING\r\n") expectPong(t, cr) c.parseAsync("PUB foo 10\r\nXXXXXXXXXX\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "Maximum Payload") { t.Fatalf("Expected an ERR for max payload violation, got: %v", l) } } func TestJWTAccountLimitsMaxPayloadButServerOverrides(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) // override with server setting of 4. opts := s.getOpts() opts.MaxPayload = 4 okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) fooAC.Limits.Payload = 8 fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) // Create a client. c, cr, cs := createClient(t, s, fooKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) c.parseAsync("PUB foo 6\r\nXXXXXX\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "Maximum Payload") { t.Fatalf("Expected an ERR for max payload violation, got: %v", l) } } func TestJWTAccountLimitsMaxConns(t *testing.T) { fooAC := newJWTTestAccountClaims() fooAC.Limits.Conn = 8 s, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, "+OK") defer s.Shutdown() defer c.close() newClient := func(expPre string) *testAsyncClient { t.Helper() // Create a client. c, cr, cs := createClient(t, s, fooKP) c.parseAsync(cs) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, expPre) { t.Fatalf("Expected a response starting with %q, got %q", expPre, l) } return c } // A connection is created in setupJWTTestWitAccountClaims(), so limit // to 7 here (8 total). for i := 0; i < 7; i++ { c := newClient("PONG") defer c.close() } // Now this one should fail. c = newClient("-ERR ") c.close() } // This will test that we can switch from a public export to a private // one and back with export claims to make sure the claim update mechanism // is working properly. func TestJWTAccountServiceImportAuthSwitch(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) serviceExport := &jwt.Export{Subject: "ngs.usage.*", Type: jwt.Service} fooAC.Exports.Add(serviceExport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) serviceImport := &jwt.Import{Account: fooPub, Subject: "ngs.usage", To: "ngs.usage.DEREK", Type: jwt.Service} barAC.Imports.Add(serviceImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) // Create a client that will send the request ca, cra, csa := createClient(t, s, barKP) defer ca.close() ca.parseAsync(csa) expectPong(t, cra) // Create the client that will respond to the requests. cb, crb, csb := createClient(t, s, fooKP) defer cb.close() cb.parseAsync(csb) expectPong(t, crb) // Create Subscriber. cb.parseAsync("SUB ngs.usage.* 1\r\nPING\r\n") expectPong(t, crb) // Send Request ca.parseAsync("PUB ngs.usage 2\r\nhi\r\nPING\r\n") expectPong(t, cra) // We should receive the request mapped into our account. PING needed to flush. cb.parseAsync("PING\r\n") expectMsg(t, crb, "ngs.usage.DEREK", "hi") // Now update to make the export private. fooACPrivate := jwt.NewAccountClaims(fooPub) serviceExport = &jwt.Export{Subject: "ngs.usage.*", Type: jwt.Service, TokenReq: true} fooACPrivate.Exports.Add(serviceExport) fooJWTPrivate, err := fooACPrivate.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWTPrivate) acc, _ := s.LookupAccount(fooPub) s.UpdateAccountClaims(acc, fooACPrivate) // Send Another Request ca.parseAsync("PUB ngs.usage 2\r\nhi\r\nPING\r\n") expectPong(t, cra) // We should not receive the request this time. cb.parseAsync("PING\r\n") expectPong(t, crb) // Now put it back again to public and make sure it works again. addAccountToMemResolver(s, fooPub, fooJWT) s.UpdateAccountClaims(acc, fooAC) // Send Request ca.parseAsync("PUB ngs.usage 2\r\nhi\r\nPING\r\n") expectPong(t, cra) // We should receive the request mapped into our account. PING needed to flush. cb.parseAsync("PING\r\n") expectMsg(t, crb, "ngs.usage.DEREK", "hi") } func TestJWTAccountServiceImportExpires(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) serviceExport := &jwt.Export{Subject: "foo", Type: jwt.Service} fooAC.Exports.Add(serviceExport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) serviceImport := &jwt.Import{Account: fooPub, Subject: "foo", Type: jwt.Service} barAC.Imports.Add(serviceImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) // Create a client that will send the request ca, cra, csa := createClient(t, s, barKP) defer ca.close() ca.parseAsync(csa) expectPong(t, cra) // Create the client that will respond to the requests. cb, crb, csb := createClient(t, s, fooKP) defer cb.close() cb.parseAsync(csb) expectPong(t, crb) // Create Subscriber. cb.parseAsync("SUB foo 1\r\nPING\r\n") expectPong(t, crb) // Send Request ca.parseAsync("PUB foo 2\r\nhi\r\nPING\r\n") expectPong(t, cra) // We should receive the request. PING needed to flush. cb.parseAsync("PING\r\n") expectMsg(t, crb, "foo", "hi") // Now update the exported service to require auth. fooAC = jwt.NewAccountClaims(fooPub) serviceExport = &jwt.Export{Subject: "foo", Type: jwt.Service, TokenReq: true} fooAC.Exports.Add(serviceExport) fooJWT, err = fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) acc, _ := s.LookupAccount(fooPub) s.UpdateAccountClaims(acc, fooAC) // Send Another Request ca.parseAsync("PUB foo 2\r\nhi\r\nPING\r\n") expectPong(t, cra) // We should not receive the request this time. cb.parseAsync("PING\r\n") expectPong(t, crb) // Now get an activation token such that it will work, but will expire. barAC = jwt.NewAccountClaims(barPub) serviceImport = &jwt.Import{Account: fooPub, Subject: "foo", Type: jwt.Service} now := time.Now() activation := jwt.NewActivationClaims(barPub) activation.ImportSubject = "foo" activation.ImportType = jwt.Service activation.IssuedAt = now.Add(-10 * time.Second).Unix() activation.Expires = now.Add(time.Second).Round(time.Second).Unix() actJWT, err := activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } serviceImport.Token = actJWT barAC.Imports.Add(serviceImport) barJWT, err = barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) acc, _ = s.LookupAccount(barPub) s.UpdateAccountClaims(acc, barAC) // Now it should work again. // Send Another Request ca.parseAsync("PUB foo 3\r\nhi2\r\nPING\r\n") expectPong(t, cra) // We should receive the request. PING needed to flush. cb.parseAsync("PING\r\n") expectMsg(t, crb, "foo", "hi2") // Now wait for it to expire, then retry. waitTime := time.Duration(activation.Expires-time.Now().Unix()) * time.Second time.Sleep(waitTime + 250*time.Millisecond) // Send Another Request ca.parseAsync("PUB foo 3\r\nhi3\r\nPING\r\n") expectPong(t, cra) // We should NOT receive the request. PING needed to flush. cb.parseAsync("PING\r\n") expectPong(t, crb) } func TestJWTAccountURLResolver(t *testing.T) { for _, test := range []struct { name string useTLS bool }{ {"plain", false}, {"tls", true}, } { t.Run(test.name, func(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(kp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(ajwt)) }) var ts *httptest.Server if test.useTLS { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConfig, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } ts = httptest.NewUnstartedServer(hf) ts.TLS = tlsConfig ts.StartTLS() } else { ts = httptest.NewServer(hf) } defer ts.Close() confTemplate := ` operator: %s listen: 127.0.0.1:-1 resolver: URL("%s/ngs/v1/accounts/jwt/") resolver_tls { cert_file: "../test/configs/certs/client-cert.pem" key_file: "../test/configs/certs/client-key.pem" ca_file: "../test/configs/certs/ca.pem" } ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL))) s, opts := RunServerWithConfig(conf) pub, _ := kp.PublicKey() opts.TrustedKeys = []string{pub} defer s.Shutdown() acc, _ := s.LookupAccount(apub) if acc == nil { t.Fatalf("Expected to receive an account") } if acc.Name != apub { t.Fatalf("Account name did not match claim key") } }) } } func TestJWTAccountURLResolverTimeout(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(kp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } basePath := "/ngs/v1/accounts/jwt/" ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == basePath { w.Write([]byte("ok")) return } // Purposely be slow on account lookup. time.Sleep(200 * time.Millisecond) w.Write([]byte(ajwt)) })) defer ts.Close() confTemplate := ` listen: 127.0.0.1:-1 resolver: URL("%s%s") ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ts.URL, basePath))) s, opts := RunServerWithConfig(conf) pub, _ := kp.PublicKey() opts.TrustedKeys = []string{pub} defer s.Shutdown() // Lower default timeout to speed-up test s.AccountResolver().(*URLAccResolver).c.Timeout = 50 * time.Millisecond acc, _ := s.LookupAccount(apub) if acc != nil { t.Fatalf("Expected to not receive an account due to timeout") } } func TestJWTAccountURLResolverNoFetchOnReload(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(kp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(ajwt)) })) defer ts.Close() confTemplate := ` operator: %s listen: 127.0.0.1:-1 resolver: URL("%s/ngs/v1/accounts/jwt/") ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() acc, _ := s.LookupAccount(apub) if acc == nil { t.Fatalf("Expected to receive an account") } // Reload would produce a DATA race during the DeepEqual check for the account resolver, // so close the current one and we will create a new one that keeps track of fetch calls. ts.Close() fetch := int32(0) ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&fetch, 1) w.Write([]byte(ajwt)) })) defer ts.Close() changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL))) if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } if atomic.LoadInt32(&fetch) != 0 { t.Fatalf("Fetch invoked during reload") } // Now stop the resolver and make sure that on startup, we report URL resolver failure s.Shutdown() s = nil ts.Close() opts := LoadConfig(conf) if s, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), "could not fetch") { if s != nil { s.Shutdown() } t.Fatalf("Expected error regarding account resolver, got %v", err) } } func TestJWTAccountURLResolverFetchFailureInServer1(t *testing.T) { const subj = "test" const crossAccSubj = "test" // Create Exporting Account expkp, _ := nkeys.CreateAccount() exppub, _ := expkp.PublicKey() expac := jwt.NewAccountClaims(exppub) expac.Exports.Add(&jwt.Export{ Subject: crossAccSubj, Type: jwt.Stream, }) expjwt, err := expac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create importing Account impkp, _ := nkeys.CreateAccount() imppub, _ := impkp.PublicKey() impac := jwt.NewAccountClaims(imppub) impac.Imports.Add(&jwt.Import{ Account: exppub, Subject: crossAccSubj, Type: jwt.Stream, }) impjwt, err := impac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Simulate an account server that drops the first request to exppub chanImpA := make(chan struct{}, 10) defer close(chanImpA) chanExpS := make(chan struct{}, 10) defer close(chanExpS) chanExpF := make(chan struct{}, 1) defer close(chanExpF) failureCnt := int32(0) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/A/" { // Server startup w.Write(nil) chanImpA <- struct{}{} } else if r.URL.Path == "/A/"+imppub { w.Write([]byte(impjwt)) chanImpA <- struct{}{} } else if r.URL.Path == "/A/"+exppub { if atomic.AddInt32(&failureCnt, 1) <= 1 { // skip the write to simulate the failure chanExpF <- struct{}{} } else { w.Write([]byte(expjwt)) chanExpS <- struct{}{} } } else { t.Fatal("not expected") } })) defer ts.Close() // Create server confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") `, ojwt, ts.URL))) sA := RunServer(LoadConfig(confA)) defer sA.Shutdown() // server observed one fetch on startup chanRecv(t, chanImpA, 10*time.Second) // Create first client ncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp)) defer ncA.Close() // create a test subscription subA, err := ncA.SubscribeSync(subj) if err != nil { t.Fatalf("Expected no error during subscribe: %v", err) } defer subA.Unsubscribe() // Connect of client triggered a fetch of both accounts // the fetch for the imported account will fail chanRecv(t, chanImpA, 10*time.Second) chanRecv(t, chanExpF, 10*time.Second) // create second client for user exporting ncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp)) defer ncB.Close() chanRecv(t, chanExpS, 10*time.Second) // Connect of client triggered another fetch, this time passing checkSubInterest(t, sA, imppub, subj, 10*time.Second) checkSubInterest(t, sA, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue } func TestJWTAccountURLResolverFetchFailurePushReorder(t *testing.T) { const subj = "test" const crossAccSubj = "test" // Create System Account syskp, _ := nkeys.CreateAccount() syspub, _ := syskp.PublicKey() sysAc := jwt.NewAccountClaims(syspub) sysjwt, err := sysAc.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create Exporting Account expkp, _ := nkeys.CreateAccount() exppub, _ := expkp.PublicKey() expac := jwt.NewAccountClaims(exppub) expjwt1, err := expac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } expac.Exports.Add(&jwt.Export{ Subject: crossAccSubj, Type: jwt.Stream, }) expjwt2, err := expac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create importing Account impkp, _ := nkeys.CreateAccount() imppub, _ := impkp.PublicKey() impac := jwt.NewAccountClaims(imppub) impac.Imports.Add(&jwt.Import{ Account: exppub, Subject: crossAccSubj, Type: jwt.Stream, }) impjwt, err := impac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Simulate an account server that does not serve the updated jwt for exppub chanImpA := make(chan struct{}, 10) defer close(chanImpA) chanExpS := make(chan struct{}, 10) defer close(chanExpS) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/A/" { // Server startup w.Write(nil) chanImpA <- struct{}{} } else if r.URL.Path == "/A/"+imppub { w.Write([]byte(impjwt)) chanImpA <- struct{}{} } else if r.URL.Path == "/A/"+exppub { // respond with jwt that does not have the export // this simulates an ordering issue w.Write([]byte(expjwt1)) chanExpS <- struct{}{} } else if r.URL.Path == "/A/"+syspub { w.Write([]byte(sysjwt)) } else { t.Fatal("not expected") } })) defer ts.Close() confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") system_account: %s `, ojwt, ts.URL, syspub))) sA := RunServer(LoadConfig(confA)) defer sA.Shutdown() // server observed one fetch on startup chanRecv(t, chanImpA, 10*time.Second) // Create first client ncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp)) defer ncA.Close() // create a test subscription subA, err := ncA.SubscribeSync(subj) if err != nil { t.Fatalf("Expected no error during subscribe: %v", err) } defer subA.Unsubscribe() // Connect of client triggered a fetch of both accounts // the fetch for the imported account will fail chanRecv(t, chanImpA, 10*time.Second) chanRecv(t, chanExpS, 10*time.Second) // create second client for user exporting ncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp)) defer ncB.Close() // update expjwt2, this will correct the import issue sysc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, syskp)) defer sysc.Close() natsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjNew, exppub), []byte(expjwt2)) sysc.Flush() // updating expjwt should cause this to pass checkSubInterest(t, sA, imppub, subj, 10*time.Second) checkSubInterest(t, sA, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue } type captureDebugLogger struct { DummyLogger dbgCh chan string } func (l *captureDebugLogger) Debugf(format string, v ...any) { select { case l.dbgCh <- fmt.Sprintf(format, v...): default: } } func TestJWTAccountURLResolverPermanentFetchFailure(t *testing.T) { const crossAccSubj = "test" expkp, _ := nkeys.CreateAccount() exppub, _ := expkp.PublicKey() impkp, _ := nkeys.CreateAccount() imppub, _ := impkp.PublicKey() // Create System Account syskp, _ := nkeys.CreateAccount() syspub, _ := syskp.PublicKey() sysAc := jwt.NewAccountClaims(syspub) sysjwt, err := sysAc.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create 2 Accounts. Each importing from the other, but NO matching export expac := jwt.NewAccountClaims(exppub) expac.Imports.Add(&jwt.Import{ Account: imppub, Subject: crossAccSubj, Type: jwt.Stream, }) expjwt, err := expac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create importing Account impac := jwt.NewAccountClaims(imppub) impac.Imports.Add(&jwt.Import{ Account: exppub, Subject: crossAccSubj, Type: jwt.Stream, }) impjwt, err := impac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Simulate an account server that does not serve the updated jwt for exppub ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/A/" { // Server startup w.Write(nil) } else if r.URL.Path == "/A/"+imppub { w.Write([]byte(impjwt)) } else if r.URL.Path == "/A/"+exppub { w.Write([]byte(expjwt)) } else if r.URL.Path == "/A/"+syspub { w.Write([]byte(sysjwt)) } else { t.Fatal("not expected") } })) defer ts.Close() confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") system_account: %s `, ojwt, ts.URL, syspub))) o := LoadConfig(confA) sA := RunServer(o) defer sA.Shutdown() l := &captureDebugLogger{dbgCh: make(chan string, 100)} // has enough space to not block sA.SetLogger(l, true, false) // Create clients ncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp)) defer ncA.Close() ncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp)) defer ncB.Close() sysc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, syskp)) defer sysc.Close() // push accounts natsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjNew, imppub), []byte(impjwt)) natsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjOld, exppub), []byte(expjwt)) sysc.Flush() importErrCnt := 0 tmr := time.NewTimer(500 * time.Millisecond) defer tmr.Stop() for { select { case line := <-l.dbgCh: if strings.HasPrefix(line, "Error adding stream import to account") { importErrCnt++ } case <-tmr.C: // connecting and updating, each cause 3 traces (2 + 1 on iteration) + 1 xtra fetch if importErrCnt != 7 { t.Fatalf("Expected 7 debug traces, got %d", importErrCnt) } return } } } func TestJWTAccountURLResolverFetchFailureInCluster(t *testing.T) { assertChanLen := func(x int, chans ...chan struct{}) { t.Helper() for _, c := range chans { if len(c) != x { t.Fatalf("length of channel is not %d", x) } } } const subj = ">" const crossAccSubj = "test" // Create Exporting Account expkp, _ := nkeys.CreateAccount() exppub, _ := expkp.PublicKey() expac := jwt.NewAccountClaims(exppub) expac.Exports.Add(&jwt.Export{ Subject: crossAccSubj, Type: jwt.Stream, }) expjwt, err := expac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create importing Account impkp, _ := nkeys.CreateAccount() imppub, _ := impkp.PublicKey() impac := jwt.NewAccountClaims(imppub) impac.Imports.Add(&jwt.Import{ Account: exppub, Subject: crossAccSubj, Type: jwt.Stream, }) impac.Exports.Add(&jwt.Export{ Subject: "srvc", Type: jwt.Service, }) impjwt, err := impac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create User nkp, _ := nkeys.CreateUser() uSeed, _ := nkp.Seed() upub, _ := nkp.PublicKey() nuc := newJWTTestUserClaims() nuc.Subject = upub uJwt, err := nuc.Encode(impkp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } creds := genCredsFile(t, uJwt, uSeed) // Simulate an account server that drops the first request to /B/acc chanImpA := make(chan struct{}, 4) defer close(chanImpA) chanImpB := make(chan struct{}, 4) defer close(chanImpB) chanExpA := make(chan struct{}, 4) defer close(chanExpA) chanExpB := make(chan struct{}, 4) defer close(chanExpB) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/A/" { // Server A startup w.Write(nil) chanImpA <- struct{}{} } else if r.URL.Path == "/B/" { // Server B startup w.Write(nil) chanImpB <- struct{}{} } else if r.URL.Path == "/A/"+imppub { // First Client connecting to Server A w.Write([]byte(impjwt)) chanImpA <- struct{}{} } else if r.URL.Path == "/B/"+imppub { // Second Client connecting to Server B w.Write([]byte(impjwt)) chanImpB <- struct{}{} } else if r.URL.Path == "/A/"+exppub { // First Client connecting to Server A w.Write([]byte(expjwt)) chanExpA <- struct{}{} } else if r.URL.Path == "/B/"+exppub { // Second Client connecting to Server B w.Write([]byte(expjwt)) chanExpB <- struct{}{} } else { t.Fatal("not expected") } })) defer ts.Close() // Create seed server A confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") cluster { name: clust no_advertise: true listen: 127.0.0.1:-1 } `, ojwt, ts.URL))) sA := RunServer(LoadConfig(confA)) defer sA.Shutdown() // Create Server B (using no_advertise to prevent failover) confB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/B/") cluster { name: clust no_advertise: true listen: 127.0.0.1:-1 routes [ nats-route://127.0.0.1:%d ] } `, ojwt, ts.URL, sA.opts.Cluster.Port))) sB := RunServer(LoadConfig(confB)) defer sB.Shutdown() // startup cluster checkClusterFormed(t, sA, sB) // Both server observed one fetch on startup chanRecv(t, chanImpA, 10*time.Second) chanRecv(t, chanImpB, 10*time.Second) assertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB) // Create first client, directly connects to A urlA := fmt.Sprintf("nats://%s:%d", sA.opts.Host, sA.opts.Port) ncA, err := nats.Connect(urlA, nats.UserCredentials(creds), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { if err != nil { t.Fatal("error not expected in this test", err) } }), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { t.Fatal("error not expected in this test", err) }), ) if err != nil { t.Fatalf("Expected to connect, got %v %s", err, urlA) } defer ncA.Close() // create a test subscription subA, err := ncA.SubscribeSync(subj) if err != nil { t.Fatalf("Expected no error during subscribe: %v", err) } defer subA.Unsubscribe() // Connect of client triggered a fetch by Server A chanRecv(t, chanImpA, 10*time.Second) chanRecv(t, chanExpA, 10*time.Second) assertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB) //time.Sleep(10 * time.Second) // create second client, directly connect to B urlB := fmt.Sprintf("nats://%s:%d", sB.opts.Host, sB.opts.Port) ncB, err := nats.Connect(urlB, nats.UserCredentials(creds), nats.NoReconnect()) if err != nil { t.Fatalf("Expected to connect, got %v %s", err, urlB) } defer ncB.Close() // Connect of client triggered a fetch by Server B chanRecv(t, chanImpB, 10*time.Second) chanRecv(t, chanExpB, 10*time.Second) assertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB) checkClusterFormed(t, sA, sB) // the route subscription was lost due to the failed fetch // Now we test if some recover mechanism is in play checkSubInterest(t, sB, imppub, subj, 10*time.Second) // Will fail as a result of this issue checkSubInterest(t, sB, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue if err := ncB.Publish(subj, []byte("msg")); err != nil { t.Fatalf("Expected to publish %v", err) } // expect the message from B to flow to A if m, err := subA.NextMsg(10 * time.Second); err != nil { t.Fatalf("Expected to receive a message %v", err) } else if string(m.Data) != "msg" { t.Fatalf("Expected to receive 'msg', got: %s", string(m.Data)) } assertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB) } func TestJWTAccountURLResolverReturnDifferentOperator(t *testing.T) { // Create a valid chain of op/acc/usr using a different operator // This is so we can test if the server rejects this chain. // Create Operator op, _ := nkeys.CreateOperator() // Create Account, this account is the one returned by the resolver akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(op) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create User nkp, _ := nkeys.CreateUser() uSeed, _ := nkp.Seed() upub, _ := nkp.PublicKey() nuc := newJWTTestUserClaims() nuc.Subject = upub uJwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } creds := genCredsFile(t, uJwt, uSeed) // Simulate an account server that was hijacked/mis configured ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(ajwt)) })) defer ts.Close() // Create Server confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") `, ojwt, ts.URL))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() // Create first client, directly connects to A urlA := fmt.Sprintf("nats://%s:%d", sA.opts.Host, sA.opts.Port) if _, err := nats.Connect(urlA, nats.UserCredentials(creds), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { if err != nil { t.Fatal("error not expected in this test", err) } }), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { t.Fatal("error not expected in this test", err) }), ); err == nil { t.Fatal("Expected connect to fail") } // Test if the server has the account in memory. (shouldn't) if v, ok := sA.accounts.Load(apub); ok { t.Fatalf("Expected account to NOT be in memory: %v", v.(*Account)) } } func TestJWTUserSigningKey(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() // Check to make sure we would have an authTimer if !s.info.AuthRequired { t.Fatalf("Expect the server to require auth") } c, cr, _ := newClientForServer(s) defer c.close() // Don't send jwt field, should fail. c.parseAsync("CONNECT {\"verbose\":true,\"pedantic\":true}\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } okp, _ := nkeys.FromSeed(oSeed) // Create an account akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() // Create a signing key for the account askp, _ := nkeys.CreateAccount() aspub, _ := askp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create a client with the account signing key c, cr, cs := createClientWithIssuer(t, s, askp, apub) defer c.close() // PING needed to flush the +OK/-ERR to us. // This should fail too since no account resolver is defined. c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } // Ok now let's walk through and make sure all is good. // We will set the account resolver by hand to a memory resolver. buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) // Create a client with a signing key c, cr, cs = createClientWithIssuer(t, s, askp, apub) defer c.close() // should fail because the signing key is not known c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error: %v", l) } // add a signing key nac.SigningKeys.Add(aspub) // update the memory resolver acc, _ := s.LookupAccount(apub) s.UpdateAccountClaims(acc, nac) // Create a client with a signing key c, cr, cs = createClientWithIssuer(t, s, askp, apub) defer c.close() // expect this to work c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "PONG") { t.Fatalf("Expected a PONG, got %q", l) } isClosed := func() bool { c.mu.Lock() defer c.mu.Unlock() return c.isClosed() } if isClosed() { t.Fatal("expected client to be alive") } // remove the signing key should bounce client nac.SigningKeys = nil acc, _ = s.LookupAccount(apub) s.UpdateAccountClaims(acc, nac) if !isClosed() { t.Fatal("expected client to be gone") } } func TestJWTAccountImportSignerRemoved(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Exporter keys srvKP, _ := nkeys.CreateAccount() srvPK, _ := srvKP.PublicKey() srvSignerKP, _ := nkeys.CreateAccount() srvSignerPK, _ := srvSignerKP.PublicKey() // Importer keys clientKP, _ := nkeys.CreateAccount() clientPK, _ := clientKP.PublicKey() createSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) { ac := jwt.NewAccountClaims(srvPK) ac.SigningKeys.Add(signingKeys...) ac.Exports.Add(&jwt.Export{Subject: "foo", Type: jwt.Service, TokenReq: true}) ac.Exports.Add(&jwt.Export{Subject: "bar", Type: jwt.Stream, TokenReq: true}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating exporter JWT: %v", err) } return token, ac } createImportToken := func(sub string, kind jwt.ExportType) string { actC := jwt.NewActivationClaims(clientPK) actC.IssuerAccount = srvPK actC.ImportType = kind actC.ImportSubject = jwt.Subject(sub) token, err := actC.Encode(srvSignerKP) if err != nil { t.Fatal(err) } return token } createClientJwt := func() string { ac := jwt.NewAccountClaims(clientPK) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "foo", Type: jwt.Service, Token: createImportToken("foo", jwt.Service)}) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "bar", Type: jwt.Stream, Token: createImportToken("bar", jwt.Stream)}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating importer JWT: %v", err) } return token } srvJWT, _ := createSrvJwt(srvSignerPK) addAccountToMemResolver(s, srvPK, srvJWT) clientJWT := createClientJwt() addAccountToMemResolver(s, clientPK, clientJWT) // Create a client that will send the request client, clientReader, clientCS := createClient(t, s, clientKP) defer client.close() client.parseAsync(clientCS) expectPong(t, clientReader) checkShadow := func(expected int) { t.Helper() client.mu.Lock() defer client.mu.Unlock() sub := client.subs["1"] count := 0 if sub != nil { count = len(sub.shadow) } if count != expected { t.Fatalf("Expected shadows to be %d, got %d", expected, count) } } checkShadow(0) // Create the client that will respond to the requests. srv, srvReader, srvCS := createClient(t, s, srvKP) defer srv.close() srv.parseAsync(srvCS) expectPong(t, srvReader) // Create Subscriber. srv.parseAsync("SUB foo 1\r\nPING\r\n") expectPong(t, srvReader) // Send Request client.parseAsync("PUB foo 2\r\nhi\r\nPING\r\n") expectPong(t, clientReader) // We should receive the request. PING needed to flush. srv.parseAsync("PING\r\n") expectMsg(t, srvReader, "foo", "hi") client.parseAsync("SUB bar 1\r\nPING\r\n") expectPong(t, clientReader) checkShadow(1) srv.parseAsync("PUB bar 2\r\nhi\r\nPING\r\n") expectPong(t, srvReader) // We should receive from stream. PING needed to flush. client.parseAsync("PING\r\n") expectMsg(t, clientReader, "bar", "hi") // Now update the exported service no signer srvJWT, srvAC := createSrvJwt() addAccountToMemResolver(s, srvPK, srvJWT) acc, _ := s.LookupAccount(srvPK) s.UpdateAccountClaims(acc, srvAC) // Send Another Request client.parseAsync("PUB foo 2\r\nhi\r\nPING\r\n") expectPong(t, clientReader) // We should not receive the request this time. srv.parseAsync("PING\r\n") expectPong(t, srvReader) // Publish on the stream srv.parseAsync("PUB bar 2\r\nhi\r\nPING\r\n") expectPong(t, srvReader) // We should not receive from the stream this time client.parseAsync("PING\r\n") expectPong(t, clientReader) checkShadow(0) } func TestJWTAccountImportSignerDeadlock(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Exporter keys srvKP, _ := nkeys.CreateAccount() srvPK, _ := srvKP.PublicKey() srvSignerKP, _ := nkeys.CreateAccount() srvSignerPK, _ := srvSignerKP.PublicKey() // Importer keys clientKP, _ := nkeys.CreateAccount() clientPK, _ := clientKP.PublicKey() createSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) { ac := jwt.NewAccountClaims(srvPK) ac.SigningKeys.Add(signingKeys...) ac.Exports.Add(&jwt.Export{Subject: "foo", Type: jwt.Service, TokenReq: true}) ac.Exports.Add(&jwt.Export{Subject: "bar", Type: jwt.Stream, TokenReq: true}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating exporter JWT: %v", err) } return token, ac } createImportToken := func(sub string, kind jwt.ExportType) string { actC := jwt.NewActivationClaims(clientPK) actC.IssuerAccount = srvPK actC.ImportType = kind actC.ImportSubject = jwt.Subject(sub) token, err := actC.Encode(srvSignerKP) if err != nil { t.Fatal(err) } return token } createClientJwt := func() string { ac := jwt.NewAccountClaims(clientPK) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "foo", Type: jwt.Service, Token: createImportToken("foo", jwt.Service)}) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "bar", Type: jwt.Stream, Token: createImportToken("bar", jwt.Stream)}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating importer JWT: %v", err) } return token } srvJWT, _ := createSrvJwt(srvSignerPK) addAccountToMemResolver(s, srvPK, srvJWT) clientJWT := createClientJwt() addAccountToMemResolver(s, clientPK, clientJWT) acc, _ := s.LookupAccount(srvPK) // Have a go routine that constantly gets/releases the acc's write lock. // There was a bug that could cause AddServiceImportWithClaim to deadlock. ch := make(chan bool, 1) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { select { case <-ch: return default: acc.mu.Lock() acc.mu.Unlock() time.Sleep(time.Millisecond) } } }() // Create a client that will send the request client, clientReader, clientCS := createClient(t, s, clientKP) defer client.close() client.parseAsync(clientCS) expectPong(t, clientReader) close(ch) wg.Wait() } func TestJWTAccountImportWrongIssuerAccount(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) l := &captureErrorLogger{errCh: make(chan string, 2)} s.SetLogger(l, false, false) okp, _ := nkeys.FromSeed(oSeed) // Exporter keys srvKP, _ := nkeys.CreateAccount() srvPK, _ := srvKP.PublicKey() srvSignerKP, _ := nkeys.CreateAccount() srvSignerPK, _ := srvSignerKP.PublicKey() // Importer keys clientKP, _ := nkeys.CreateAccount() clientPK, _ := clientKP.PublicKey() createSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) { ac := jwt.NewAccountClaims(srvPK) ac.SigningKeys.Add(signingKeys...) ac.Exports.Add(&jwt.Export{Subject: "foo", Type: jwt.Service, TokenReq: true}) ac.Exports.Add(&jwt.Export{Subject: "bar", Type: jwt.Stream, TokenReq: true}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating exporter JWT: %v", err) } return token, ac } createImportToken := func(sub string, kind jwt.ExportType) string { actC := jwt.NewActivationClaims(clientPK) // Reference ourselves, which is wrong. actC.IssuerAccount = clientPK actC.ImportType = kind actC.ImportSubject = jwt.Subject(sub) token, err := actC.Encode(srvSignerKP) if err != nil { t.Fatal(err) } return token } createClientJwt := func() string { ac := jwt.NewAccountClaims(clientPK) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "foo", Type: jwt.Service, Token: createImportToken("foo", jwt.Service)}) ac.Imports.Add(&jwt.Import{Account: srvPK, Subject: "bar", Type: jwt.Stream, Token: createImportToken("bar", jwt.Stream)}) token, err := ac.Encode(okp) if err != nil { t.Fatalf("Error generating importer JWT: %v", err) } return token } srvJWT, _ := createSrvJwt(srvSignerPK) addAccountToMemResolver(s, srvPK, srvJWT) clientJWT := createClientJwt() addAccountToMemResolver(s, clientPK, clientJWT) // Create a client that will send the request client, clientReader, clientCS := createClient(t, s, clientKP) defer client.close() client.parseAsync(clientCS) if l, _, err := clientReader.ReadLine(); err != nil { t.Fatalf("Expected no Error, got: %v", err) } else if !strings.Contains(string(l), "-ERR 'Authorization Violation'") { t.Fatalf("Expected Error, got: %v", l) } } func TestJWTUserRevokedOnAccountUpdate(t *testing.T) { nac := newJWTTestAccountClaims() s, akp, c, cr := setupJWTTestWitAccountClaims(t, nac, "+OK") defer s.Shutdown() defer c.close() expectPong(t, cr) okp, _ := nkeys.FromSeed(oSeed) apub, _ := akp.PublicKey() c.mu.Lock() pub := c.user.Nkey c.mu.Unlock() // Now revoke the user. nac.Revoke(pub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Update the account on the server. addAccountToMemResolver(s, apub, ajwt) acc, err := s.LookupAccount(apub) if err != nil { t.Fatalf("Error looking up the account: %v", err) } // This is simulating a system update for the account claims. go s.updateAccountWithClaimJWT(acc, ajwt) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "Revoked") { t.Fatalf("Expected 'Revoked' to be in the error") } } func TestJWTUserRevoked(t *testing.T) { okp, _ := nkeys.FromSeed(oSeed) // Create a new user that we will make sure has been revoked. nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc := jwt.NewUserClaims(pub) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) // Revoke the user right away. nac.Revoke(pub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Sign for the user. jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, l := newClientForServer(s) defer c.close() // Sign Nonce var info nonceInfo json.Unmarshal([]byte(l[5:]), &info) sigraw, _ := nkp.Sign([]byte(info.Nonce)) sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK/-ERR to us. cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"sig\":\"%s\"}\r\nPING\r\n", jwt, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } if !strings.Contains(l, "Authorization") { t.Fatalf("Expected 'Revoked' to be in the error") } } // Test that an account update that revokes an import authorization cancels the import. func TestJWTImportTokenRevokedAfter(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) // Now create Exports. export := &jwt.Export{Subject: "foo.private", Type: jwt.Stream, TokenReq: true} fooAC.Exports.Add(export) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) simport := &jwt.Import{Account: fooPub, Subject: "foo.private", Type: jwt.Stream} activation := jwt.NewActivationClaims(barPub) activation.ImportSubject = "foo.private" activation.ImportType = jwt.Stream actJWT, err := activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } simport.Token = actJWT barAC.Imports.Add(simport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) // Now revoke the export. decoded, _ := jwt.DecodeActivationClaims(actJWT) export.Revoke(decoded.Subject) fooJWT, err = fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) fooAcc, _ := s.LookupAccount(fooPub) if fooAcc == nil { t.Fatalf("Expected to retrieve the account") } // Now lookup bar account and make sure it was revoked. acc, _ := s.LookupAccount(barPub) if acc == nil { t.Fatalf("Expected to retrieve the account") } if les := len(acc.imports.streams); les != 0 { t.Fatalf("Expected imports streams len of 0, got %d", les) } } // Test that an account update that revokes an import authorization cancels the import. func TestJWTImportTokenRevokedBefore(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) // Now create Exports. export := &jwt.Export{Subject: "foo.private", Type: jwt.Stream, TokenReq: true} fooAC.Exports.Add(export) // Import account barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) simport := &jwt.Import{Account: fooPub, Subject: "foo.private", Type: jwt.Stream} activation := jwt.NewActivationClaims(barPub) activation.ImportSubject = "foo.private" activation.ImportType = jwt.Stream actJWT, err := activation.Encode(fooKP) if err != nil { t.Fatalf("Error generating activation token: %v", err) } simport.Token = actJWT barAC.Imports.Add(simport) // Now revoke the export. decoded, _ := jwt.DecodeActivationClaims(actJWT) export.Revoke(decoded.Subject) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) fooAcc, _ := s.LookupAccount(fooPub) if fooAcc == nil { t.Fatalf("Expected to retrieve the account") } // Now lookup bar account and make sure it was revoked. acc, _ := s.LookupAccount(barPub) if acc == nil { t.Fatalf("Expected to retrieve the account") } if les := len(acc.imports.streams); les != 0 { t.Fatalf("Expected imports streams len of 0, got %d", les) } } func TestJWTCircularAccountServiceImport(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) barKP, _ := nkeys.CreateAccount() barPub, _ := barKP.PublicKey() barAC := jwt.NewAccountClaims(barPub) // Create service export/import for account foo serviceExport := &jwt.Export{Subject: "foo", Type: jwt.Service, TokenReq: true} serviceImport := &jwt.Import{Account: barPub, Subject: "bar", Type: jwt.Service} fooAC.Exports.Add(serviceExport) fooAC.Imports.Add(serviceImport) fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) // Create service export/import for account bar serviceExport = &jwt.Export{Subject: "bar", Type: jwt.Service, TokenReq: true} serviceImport = &jwt.Import{Account: fooPub, Subject: "foo", Type: jwt.Service} barAC.Exports.Add(serviceExport) barAC.Imports.Add(serviceImport) barJWT, err := barAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, barPub, barJWT) c, cr, cs := createClient(t, s, fooKP) defer c.close() c.parseAsync(cs) expectPong(t, cr) c.parseAsync("SUB foo 1\r\nPING\r\n") expectPong(t, cr) } // This test ensures that connected clients are properly evicted // (no deadlock) if the max conns of an account has been lowered // and the account is being updated (following expiration during // a lookup). func TestJWTAccountLimitsMaxConnsAfterExpired(t *testing.T) { s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) okp, _ := nkeys.FromSeed(oSeed) // Create accounts and imports/exports. fooKP, _ := nkeys.CreateAccount() fooPub, _ := fooKP.PublicKey() fooAC := jwt.NewAccountClaims(fooPub) fooAC.Limits.Conn = 10 fooJWT, err := fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) newClient := func(expPre string) *testAsyncClient { t.Helper() // Create a client. c, cr, cs := createClient(t, s, fooKP) c.parseAsync(cs) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, expPre) { t.Fatalf("Expected a response starting with %q, got %q", expPre, l) } go func() { for { if _, _, err := cr.ReadLine(); err != nil { return } } }() return c } for i := 0; i < 4; i++ { c := newClient("PONG") defer c.close() } // We will simulate that the account has expired. When // a new client will connect, the server will do a lookup // and find the account expired, which then will cause // a fetch and a rebuild of the account. Since max conns // is now lower, some clients should have been removed. acc, _ := s.LookupAccount(fooPub) acc.mu.Lock() acc.expired.Store(true) acc.updated = time.Now().UTC().Add(-2 * time.Second) // work around updating to quickly acc.mu.Unlock() // Now update with new expiration and max connections lowered to 2 fooAC.Limits.Conn = 2 fooJWT, err = fooAC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } addAccountToMemResolver(s, fooPub, fooJWT) // Cause the lookup that will detect that account was expired // and rebuild it, and kick clients out. c := newClient("-ERR ") defer c.close() acc, _ = s.LookupAccount(fooPub) checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { acc.mu.RLock() numClients := len(acc.clients) acc.mu.RUnlock() if numClients != 2 { return fmt.Errorf("Should have 2 clients, got %v", numClients) } return nil }) } func TestJWTBearerToken(t *testing.T) { okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc := newJWTTestUserClaims() nuc.Subject = pub // Set bearer token. nuc.BearerToken = true jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, _ := newClientForServer(s) defer c.close() // Skip nonce signature... // PING needed to flush the +OK/-ERR to us. cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt) wg := sync.WaitGroup{} wg.Add(1) go func() { c.parse([]byte(cs)) wg.Done() }() l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected +OK, got %s", l) } wg.Wait() } func TestJWTBearerWithIssuerSameAsAccountToken(t *testing.T) { okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc := newJWTTestUserClaims() // we are setting the issuer account here to trigger verification // of the issuer - the account has no signing keys, but the issuer // account is set to the public key of the account which should be OK. nuc.IssuerAccount = apub nuc.Subject = pub // Set bearer token. nuc.BearerToken = true jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, _ := newClientForServer(s) defer c.close() // Skip nonce signature... // PING needed to flush the +OK/-ERR to us. cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt) wg := sync.WaitGroup{} wg.Add(1) go func() { c.parse([]byte(cs)) wg.Done() }() l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected +OK, got %s", l) } wg.Wait() } func TestJWTBearerWithBadIssuerToken(t *testing.T) { okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc := newJWTTestUserClaims() bakp, _ := nkeys.CreateAccount() bapub, _ := bakp.PublicKey() nuc.IssuerAccount = bapub nuc.Subject = pub // Set bearer token. nuc.BearerToken = true jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } s := opTrustBasicSetup() defer s.Shutdown() buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c, cr, _ := newClientForServer(s) defer c.close() // Skip nonce signature... // PING needed to flush the +OK/-ERR to us. cs := fmt.Sprintf("CONNECT {\"jwt\":%q,\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt) wg := sync.WaitGroup{} wg.Add(1) go func() { c.parse([]byte(cs)) wg.Done() }() l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR") { t.Fatalf("Expected -ERR, got %s", l) } wg.Wait() } func TestJWTExpiredUserCredentialsRenewal(t *testing.T) { createTmpFile := func(t *testing.T, content []byte) string { t.Helper() conf := createTempFile(t, _EMPTY_) fName := conf.Name() conf.Close() if err := os.WriteFile(fName, content, 0666); err != nil { t.Fatalf("Error writing conf file: %v", err) } return fName } waitTime := func(ch chan bool, timeout time.Duration) error { select { case <-ch: return nil case <-time.After(timeout): } return errors.New("timeout") } okp, _ := nkeys.FromSeed(oSeed) akp, err := nkeys.CreateAccount() if err != nil { t.Fatalf("Error generating account") } aPub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(aPub) aJwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } kp, _ := nkeys.FromSeed(oSeed) oPub, _ := kp.PublicKey() opts := defaultServerOptions opts.TrustedKeys = []string{oPub} s := RunServer(&opts) if s == nil { t.Fatal("Server did not start") } defer s.Shutdown() buildMemAccResolver(s) addAccountToMemResolver(s, aPub, aJwt) nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() uSeed, _ := nkp.Seed() nuc := newJWTTestUserClaims() nuc.Subject = pub nuc.Expires = time.Now().Add(time.Second).Unix() uJwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } creds, err := jwt.FormatUserConfig(uJwt, uSeed) if err != nil { t.Fatalf("Error encoding credentials: %v", err) } chainedFile := createTmpFile(t, creds) rch := make(chan bool) url := fmt.Sprintf("nats://%s:%d", s.opts.Host, s.opts.Port) nc, err := nats.Connect(url, nats.UserCredentials(chainedFile), nats.ReconnectWait(25*time.Millisecond), nats.ReconnectJitter(0, 0), nats.MaxReconnects(2), nats.ErrorHandler(noOpErrHandler), nats.ReconnectHandler(func(nc *nats.Conn) { rch <- true }), ) if err != nil { t.Fatalf("Expected to connect, got %v %s", err, url) } defer nc.Close() // Place new credentials underneath. nuc.Expires = time.Now().Add(30 * time.Second).Unix() uJwt, err = nuc.Encode(akp) if err != nil { t.Fatalf("Error encoding user jwt: %v", err) } creds, err = jwt.FormatUserConfig(uJwt, uSeed) if err != nil { t.Fatalf("Error encoding credentials: %v", err) } if err := os.WriteFile(chainedFile, creds, 0666); err != nil { t.Fatalf("Error writing conf file: %v", err) } // Make sure we get disconnected and reconnected first. if err := waitTime(rch, 2*time.Second); err != nil { t.Fatal("Should have reconnected.") } // We should not have been closed. if nc.IsClosed() { t.Fatal("Got disconnected when we should have reconnected.") } // Check that we clear the lastErr that can cause the disconnect. // Our reconnect CB will happen before the clear. So check after a bit. time.Sleep(50 * time.Millisecond) if nc.LastError() != nil { t.Fatalf("Expected lastErr to be cleared, got %q", nc.LastError()) } } func updateJwt(t *testing.T, url string, creds string, jwt string, respCnt int) int { t.Helper() require_NextMsg := func(sub *nats.Subscription) bool { t.Helper() msg := natsNexMsg(t, sub, time.Second) content := make(map[string]any) json.Unmarshal(msg.Data, &content) if _, ok := content["data"]; ok { return true } return false } c := natsConnect(t, url, nats.UserCredentials(creds), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { if err != nil { t.Fatal("error not expected in this test", err) } }), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { t.Fatal("error not expected in this test", err) }), ) defer c.Close() resp := c.NewRespInbox() sub := natsSubSync(t, c, resp) err := sub.AutoUnsubscribe(respCnt) require_NoError(t, err) require_NoError(t, c.PublishRequest(accClaimsReqSubj, resp, []byte(jwt))) passCnt := 0 for i := 0; i < respCnt; i++ { if require_NextMsg(sub) { passCnt++ } } return passCnt } func require_JWTAbsent(t *testing.T, dir string, pub string) { t.Helper() _, err := os.Stat(filepath.Join(dir, pub+".jwt")) require_Error(t, err) require_True(t, os.IsNotExist(err)) } func require_JWTPresent(t *testing.T, dir string, pub string) { t.Helper() _, err := os.Stat(filepath.Join(dir, pub+".jwt")) require_NoError(t, err) } func require_JWTEqual(t *testing.T, dir string, pub string, jwt string) { t.Helper() content, err := os.ReadFile(filepath.Join(dir, pub+".jwt")) require_NoError(t, err) require_Equal(t, string(content), jwt) } func createTempFile(t testing.TB, prefix string) *os.File { t.Helper() tempDir := t.TempDir() f, err := os.CreateTemp(tempDir, prefix) require_NoError(t, err) return f } func removeDir(t testing.TB, dir string) { t.Helper() if err := os.RemoveAll(dir); err != nil { t.Fatal(err) } } func removeFile(t testing.TB, p string) { t.Helper() if err := os.Remove(p); err != nil { t.Fatal(err) } } func writeJWT(t *testing.T, dir string, pub string, jwt string) { t.Helper() err := os.WriteFile(filepath.Join(dir, pub+".jwt"), []byte(jwt), 0644) require_NoError(t, err) } func TestJWTAccountNATSResolverFetch(t *testing.T) { origEventsHBInterval := eventsHBInterval eventsHBInterval = 50 * time.Millisecond // speed up eventing defer func() { eventsHBInterval = origEventsHBInterval }() require_NoLocalOrRemoteConnections := func(account string, srvs ...*Server) { t.Helper() for _, srv := range srvs { if acc, ok := srv.accounts.Load(account); ok { checkAccClientsCount(t, acc.(*Account), 0) } } } // After each connection check, require_XConnection and connect assures that // listed server have no connections for the account used require_1Connection := func(url, creds, acc string, srvs ...*Server) { t.Helper() func() { t.Helper() c := natsConnect(t, url, nats.UserCredentials(creds)) defer c.Close() if _, err := nats.Connect(url, nats.UserCredentials(creds)); err == nil { t.Fatal("Second connection was supposed to fail due to limits") } else if !strings.Contains(err.Error(), ErrTooManyAccountConnections.Error()) { t.Fatal("Second connection was supposed to fail with too many conns") } }() require_NoLocalOrRemoteConnections(acc, srvs...) } require_2Connection := func(url, creds, acc string, srvs ...*Server) { t.Helper() func() { t.Helper() c1 := natsConnect(t, url, nats.UserCredentials(creds)) defer c1.Close() c2 := natsConnect(t, url, nats.UserCredentials(creds)) defer c2.Close() if _, err := nats.Connect(url, nats.UserCredentials(creds)); err == nil { t.Fatal("Third connection was supposed to fail due to limits") } else if !strings.Contains(err.Error(), ErrTooManyAccountConnections.Error()) { t.Fatal("Third connection was supposed to fail with too many conns") } }() require_NoLocalOrRemoteConnections(acc, srvs...) } connect := func(url string, credsfile string, acc string, srvs ...*Server) { t.Helper() nc := natsConnect(t, url, nats.UserCredentials(credsfile), nats.Timeout(5*time.Second)) nc.Close() require_NoLocalOrRemoteConnections(acc, srvs...) } createAccountAndUser := func(limit bool, done chan struct{}, pubKey, jwt1, jwt2, creds *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) if limit { claim.Limits.Conn = 1 } var err error *jwt1, err = claim.Encode(oKp) require_NoError(t, err) // need to assure that create time differs (resolution is sec) time.Sleep(time.Millisecond * 1100) // create updated claim allowing more connections if limit { claim.Limits.Conn = 2 } *jwt2, err = claim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt, err := uclaim.Encode(kp) require_NoError(t, err) *creds = genCredsFile(t, ujwt, seed) done <- struct{}{} } // Create Accounts and corresponding user creds. Do so concurrently to speed up the test doneChan := make(chan struct{}, 5) defer close(doneChan) var syspub, sysjwt, dummy1, sysCreds string go createAccountAndUser(false, doneChan, &syspub, &sysjwt, &dummy1, &sysCreds) var apub, ajwt1, ajwt2, aCreds string go createAccountAndUser(true, doneChan, &apub, &ajwt1, &ajwt2, &aCreds) var bpub, bjwt1, bjwt2, bCreds string go createAccountAndUser(true, doneChan, &bpub, &bjwt1, &bjwt2, &bCreds) var cpub, cjwt1, cjwt2, cCreds string go createAccountAndUser(true, doneChan, &cpub, &cjwt1, &cjwt2, &cCreds) var dpub, djwt1, dummy2, dCreds string // extra user used later in the test in order to test limits go createAccountAndUser(true, doneChan, &dpub, &djwt1, &dummy2, &dCreds) for i := 0; i < cap(doneChan); i++ { <-doneChan } // Create one directory for each server dirA := t.TempDir() dirB := t.TempDir() dirC := t.TempDir() // simulate a restart of the server by storing files in them // Server A/B will completely sync, so after startup each server // will contain the union off all stored/configured jwt // Server C will send out lookup requests for jwt it does not store itself writeJWT(t, dirA, apub, ajwt1) writeJWT(t, dirB, bpub, bjwt1) writeJWT(t, dirC, cpub, cjwt1) // Create seed server A (using no_advertise to prevent fail over) confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-A operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" limit: 4 } resolver_preload: { %s: %s } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true } `, ojwt, syspub, dirA, cpub, cjwt1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() // during startup resolver_preload causes the directory to contain data require_JWTPresent(t, dirA, cpub) // Create Server B (using no_advertise to prevent fail over) confB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-B operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" limit: 4 } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } `, ojwt, syspub, dirB, sA.opts.Cluster.Port))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() // Create Server C (using no_advertise to prevent fail over) fmtC := ` listen: 127.0.0.1:-1 server_name: srv-C operator: %s system_account: %s resolver: { type: cache dir: '%s' ttl: "%dms" limit: 4 } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } ` confClongTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, 10000, sA.opts.Cluster.Port))) confCshortTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, 1000, sA.opts.Cluster.Port))) sC, _ := RunServerWithConfig(confClongTTL) // use long ttl to assure it is not kicking defer sC.Shutdown() // startup cluster checkClusterFormed(t, sA, sB, sC) time.Sleep(500 * time.Millisecond) // wait for the protocol to converge // Check all accounts require_JWTPresent(t, dirA, apub) // was already present on startup require_JWTPresent(t, dirB, apub) // was copied from server A require_JWTAbsent(t, dirC, apub) require_JWTPresent(t, dirA, bpub) // was copied from server B require_JWTPresent(t, dirB, bpub) // was already present on startup require_JWTAbsent(t, dirC, bpub) require_JWTPresent(t, dirA, cpub) // was present in preload require_JWTPresent(t, dirB, cpub) // was copied from server A require_JWTPresent(t, dirC, cpub) // was already present on startup // This is to test that connecting to it still works require_JWTAbsent(t, dirA, syspub) require_JWTAbsent(t, dirB, syspub) require_JWTAbsent(t, dirC, syspub) // system account client can connect to every server connect(sA.ClientURL(), sysCreds, "") connect(sB.ClientURL(), sysCreds, "") connect(sC.ClientURL(), sysCreds, "") checkClusterFormed(t, sA, sB, sC) // upload system account and require a response from each server passCnt := updateJwt(t, sA.ClientURL(), sysCreds, sysjwt, 3) require_True(t, passCnt == 3) require_JWTPresent(t, dirA, syspub) // was just received require_JWTPresent(t, dirB, syspub) // was just received require_JWTPresent(t, dirC, syspub) // was just received // Only files missing are in C, which is only caching connect(sC.ClientURL(), aCreds, apub, sA, sB, sC) connect(sC.ClientURL(), bCreds, bpub, sA, sB, sC) require_JWTPresent(t, dirC, apub) // was looked up form A or B require_JWTPresent(t, dirC, bpub) // was looked up from A or B // Check limits and update jwt B connecting to server A for port, v := range map[string]struct{ pub, jwt, creds string }{ sB.ClientURL(): {bpub, bjwt2, bCreds}, sC.ClientURL(): {cpub, cjwt2, cCreds}, } { require_1Connection(sA.ClientURL(), v.creds, v.pub, sA, sB, sC) require_1Connection(sB.ClientURL(), v.creds, v.pub, sA, sB, sC) require_1Connection(sC.ClientURL(), v.creds, v.pub, sA, sB, sC) passCnt := updateJwt(t, port, sysCreds, v.jwt, 3) require_True(t, passCnt == 3) require_2Connection(sA.ClientURL(), v.creds, v.pub, sA, sB, sC) require_2Connection(sB.ClientURL(), v.creds, v.pub, sA, sB, sC) require_2Connection(sC.ClientURL(), v.creds, v.pub, sA, sB, sC) require_JWTEqual(t, dirA, v.pub, v.jwt) require_JWTEqual(t, dirB, v.pub, v.jwt) require_JWTEqual(t, dirC, v.pub, v.jwt) } // Simulates A having missed an update // shutting B down as it has it will directly connect to A and connect right away sB.Shutdown() writeJWT(t, dirB, apub, ajwt2) // this will be copied to server A sB, _ = RunServerWithConfig(confB) defer sB.Shutdown() checkClusterFormed(t, sA, sB, sC) time.Sleep(500 * time.Millisecond) // wait for the protocol to converge // Restart server C. this is a workaround to force C to do a lookup in the absence of account cleanup sC.Shutdown() sC, _ = RunServerWithConfig(confClongTTL) //TODO remove this once we clean up accounts defer sC.Shutdown() require_JWTEqual(t, dirA, apub, ajwt2) // was copied from server B require_JWTEqual(t, dirB, apub, ajwt2) // was restarted with this require_JWTEqual(t, dirC, apub, ajwt1) // still contains old cached value require_2Connection(sA.ClientURL(), aCreds, apub, sA, sB, sC) require_2Connection(sB.ClientURL(), aCreds, apub, sA, sB, sC) require_1Connection(sC.ClientURL(), aCreds, apub, sA, sB, sC) // Restart server C. this is a workaround to force C to do a lookup in the absence of account cleanup sC.Shutdown() sC, _ = RunServerWithConfig(confCshortTTL) //TODO remove this once we clean up accounts defer sC.Shutdown() require_JWTEqual(t, dirC, apub, ajwt1) // still contains old cached value checkClusterFormed(t, sA, sB, sC) // Force next connect to do a lookup exceeds ttl fname := filepath.Join(dirC, apub+".jwt") checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { _, err := os.Stat(fname) if os.IsNotExist(err) { return nil } return fmt.Errorf("File not removed in time") }) connect(sC.ClientURL(), aCreds, apub, sA, sB, sC) // When lookup happens require_JWTEqual(t, dirC, apub, ajwt2) // was looked up form A or B require_2Connection(sC.ClientURL(), aCreds, apub, sA, sB, sC) // Test exceeding limit. For the exclusive directory resolver, limit is a stop gap measure. // It is not expected to be hit. When hit the administrator is supposed to take action. passCnt = updateJwt(t, sA.ClientURL(), sysCreds, djwt1, 3) require_True(t, passCnt == 1) // Only Server C updated for _, srv := range []*Server{sA, sB, sC} { if a, ok := srv.accounts.Load(syspub); ok { acc := a.(*Account) checkFor(t, time.Second, 20*time.Millisecond, func() error { acc.mu.Lock() defer acc.mu.Unlock() if acc.ctmr != nil { return fmt.Errorf("Timer still exists") } return nil }) } } } func TestJWTAccountNATSResolverCrossClusterFetch(t *testing.T) { connect := func(url string, credsfile string) { t.Helper() nc := natsConnect(t, url, nats.UserCredentials(credsfile)) nc.Close() } createAccountAndUser := func(done chan struct{}, pubKey, jwt1, jwt2, creds *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) var err error *jwt1, err = claim.Encode(oKp) require_NoError(t, err) // need to assure that create time differs (resolution is sec) time.Sleep(time.Millisecond * 1100) // create updated claim claim.Tags.Add("tag") *jwt2, err = claim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt, err := uclaim.Encode(kp) require_NoError(t, err) *creds = genCredsFile(t, ujwt, seed) done <- struct{}{} } // Create Accounts and corresponding user creds. Do so concurrently to speed up the test doneChan := make(chan struct{}, 3) defer close(doneChan) var syspub, sysjwt, dummy1, sysCreds string go createAccountAndUser(doneChan, &syspub, &sysjwt, &dummy1, &sysCreds) var apub, ajwt1, ajwt2, aCreds string go createAccountAndUser(doneChan, &apub, &ajwt1, &ajwt2, &aCreds) var bpub, bjwt1, bjwt2, bCreds string go createAccountAndUser(doneChan, &bpub, &bjwt1, &bjwt2, &bCreds) for i := 0; i < cap(doneChan); i++ { <-doneChan } // Create one directory for each server dirAA := t.TempDir() dirAB := t.TempDir() dirBA := t.TempDir() dirBB := t.TempDir() // simulate a restart of the server by storing files in them // Server AA & AB will completely sync // Server BA & BB will completely sync // Be aware that no syncing will occur between cluster writeJWT(t, dirAA, apub, ajwt1) writeJWT(t, dirBA, bpub, bjwt1) // Create seed server A (using no_advertise to prevent fail over) confAA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-A-A operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" } gateway: { name: "clust-A" listen: 127.0.0.1:-1 } cluster { name: clust-A listen: 127.0.0.1:-1 no_advertise: true } `, ojwt, syspub, dirAA))) sAA, _ := RunServerWithConfig(confAA) defer sAA.Shutdown() // Create Server B (using no_advertise to prevent fail over) confAB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-A-B operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" } gateway: { name: "clust-A" listen: 127.0.0.1:-1 } cluster { name: clust-A listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } `, ojwt, syspub, dirAB, sAA.opts.Cluster.Port))) sAB, _ := RunServerWithConfig(confAB) defer sAB.Shutdown() // Create Server C (using no_advertise to prevent fail over) confBA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-B-A operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" } gateway: { name: "clust-B" listen: 127.0.0.1:-1 gateways: [ {name: "clust-A", url: "nats://127.0.0.1:%d"}, ] } cluster { name: clust-B listen: 127.0.0.1:-1 no_advertise: true } `, ojwt, syspub, dirBA, sAA.opts.Gateway.Port))) sBA, _ := RunServerWithConfig(confBA) defer sBA.Shutdown() // Create Server BA (using no_advertise to prevent fail over) confBB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-B-B operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" } cluster { name: clust-B listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } gateway: { name: "clust-B" listen: 127.0.0.1:-1 gateways: [ {name: "clust-A", url: "nats://127.0.0.1:%d"}, ] } `, ojwt, syspub, dirBB, sBA.opts.Cluster.Port, sAA.opts.Cluster.Port))) sBB, _ := RunServerWithConfig(confBB) defer sBB.Shutdown() // Assert topology checkClusterFormed(t, sAA, sAB) checkClusterFormed(t, sBA, sBB) waitForOutboundGateways(t, sAA, 1, 5*time.Second) waitForOutboundGateways(t, sAB, 1, 5*time.Second) waitForOutboundGateways(t, sBA, 1, 5*time.Second) waitForOutboundGateways(t, sBB, 1, 5*time.Second) time.Sleep(500 * time.Millisecond) // wait for the protocol to converge updateJwt(t, sAA.ClientURL(), sysCreds, sysjwt, 4) // update system account jwt on all server require_JWTEqual(t, dirAA, syspub, sysjwt) // assure this update made it to every server require_JWTEqual(t, dirAB, syspub, sysjwt) // assure this update made it to every server require_JWTEqual(t, dirBA, syspub, sysjwt) // assure this update made it to every server require_JWTEqual(t, dirBB, syspub, sysjwt) // assure this update made it to every server require_JWTAbsent(t, dirAA, bpub) // assure that jwt are not synced across cluster require_JWTAbsent(t, dirAB, bpub) // assure that jwt are not synced across cluster require_JWTAbsent(t, dirBA, apub) // assure that jwt are not synced across cluster require_JWTAbsent(t, dirBB, apub) // assure that jwt are not synced across cluster connect(sAA.ClientURL(), aCreds) // connect to cluster where jwt was initially stored connect(sAB.ClientURL(), aCreds) // connect to cluster where jwt was initially stored connect(sBA.ClientURL(), bCreds) // connect to cluster where jwt was initially stored connect(sBB.ClientURL(), bCreds) // connect to cluster where jwt was initially stored time.Sleep(500 * time.Millisecond) // wait for the protocol to (NOT) converge require_JWTAbsent(t, dirAA, bpub) // assure that jwt are still not synced across cluster require_JWTAbsent(t, dirAB, bpub) // assure that jwt are still not synced across cluster require_JWTAbsent(t, dirBA, apub) // assure that jwt are still not synced across cluster require_JWTAbsent(t, dirBB, apub) // assure that jwt are still not synced across cluster // We have verified that account B does not exist in cluster A, neither does account A in cluster B // Despite that clients from account B can connect to server A, same for account A in cluster B connect(sAA.ClientURL(), bCreds) // connect to cluster where jwt was not initially stored connect(sAB.ClientURL(), bCreds) // connect to cluster where jwt was not initially stored connect(sBA.ClientURL(), aCreds) // connect to cluster where jwt was not initially stored connect(sBB.ClientURL(), aCreds) // connect to cluster where jwt was not initially stored require_JWTEqual(t, dirAA, bpub, bjwt1) // assure that now jwt used in connect is stored require_JWTEqual(t, dirAB, bpub, bjwt1) // assure that now jwt used in connect is stored require_JWTEqual(t, dirBA, apub, ajwt1) // assure that now jwt used in connect is stored require_JWTEqual(t, dirBB, apub, ajwt1) // assure that now jwt used in connect is stored updateJwt(t, sAA.ClientURL(), sysCreds, bjwt2, 4) // update bjwt, expect updates from everywhere updateJwt(t, sBA.ClientURL(), sysCreds, ajwt2, 4) // update ajwt, expect updates from everywhere require_JWTEqual(t, dirAA, bpub, bjwt2) // assure that jwt got updated accordingly require_JWTEqual(t, dirAB, bpub, bjwt2) // assure that jwt got updated accordingly require_JWTEqual(t, dirBA, apub, ajwt2) // assure that jwt got updated accordingly require_JWTEqual(t, dirBB, apub, ajwt2) // assure that jwt got updated accordingly } func newTimeRange(start time.Time, dur time.Duration) jwt.TimeRange { return jwt.TimeRange{Start: start.Format("15:04:05"), End: start.Add(dur).Format("15:04:05")} } func createUserWithLimit(t *testing.T, accKp nkeys.KeyPair, expiration time.Time, limits func(*jwt.UserPermissionLimits)) string { t.Helper() ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub if limits != nil { limits(&uclaim.UserPermissionLimits) } if !expiration.IsZero() { uclaim.Expires = expiration.Unix() } vr := jwt.ValidationResults{} uclaim.Validate(&vr) require_Len(t, len(vr.Errors()), 0) ujwt, err := uclaim.Encode(accKp) require_NoError(t, err) return genCredsFile(t, ujwt, seed) } func TestJWTUserLimits(t *testing.T) { // helper for time inAnHour := time.Now().Add(time.Hour) inTwoHours := time.Now().Add(2 * time.Hour) doNotExpire := time.Now().AddDate(1, 0, 0) // create account kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() claim := jwt.NewAccountClaims(aPub) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: MEM resolver_preload: { %s: %s } `, ojwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() for _, v := range []struct { pass bool f func(*jwt.UserPermissionLimits) }{ {true, nil}, {false, func(j *jwt.UserPermissionLimits) { j.Src.Set("8.8.8.8/8") }}, {true, func(j *jwt.UserPermissionLimits) { j.Src.Set("8.8.8.8/0") }}, {true, func(j *jwt.UserPermissionLimits) { j.Src.Set("127.0.0.1/8") }}, {true, func(j *jwt.UserPermissionLimits) { j.Src.Set("8.8.8.8/8,127.0.0.1/8") }}, {false, func(j *jwt.UserPermissionLimits) { j.Src.Set("8.8.8.8/8,9.9.9.9/8") }}, {true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), time.Hour)) }}, {false, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now().Add(time.Hour), time.Hour)) }}, {true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(inAnHour, time.Hour), newTimeRange(time.Now(), time.Hour)) }}, // last one is within range {false, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(inAnHour, time.Hour), newTimeRange(inTwoHours, time.Hour)) }}, // out of range {false, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(inAnHour, 3*time.Hour), newTimeRange(inTwoHours, 2*time.Hour)) }}, // overlapping [a[]b] out of range*/ {false, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(inAnHour, 3*time.Hour), newTimeRange(inTwoHours, time.Hour)) }}, // overlapping [a[b]] out of range // next day tests where end < begin {true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), 25*time.Hour)) }}, {true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), -time.Hour)) }}, } { t.Run("", func(t *testing.T) { creds := createUserWithLimit(t, kp, doNotExpire, v.f) if c, err := nats.Connect(sA.ClientURL(), nats.UserCredentials(creds)); err == nil { c.Close() if !v.pass { t.Fatalf("Expected failure got none") } } else if v.pass { t.Fatalf("Expected success got %v", err) } else if !strings.Contains(err.Error(), "Authorization Violation") { t.Fatalf("Expected error other than %v", err) } }) } } func TestJWTTimeExpiration(t *testing.T) { validFor := 1500 * time.Millisecond validRange := 500 * time.Millisecond doNotExpire := time.Now().AddDate(1, 0, 0) // create account kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() claim := jwt.NewAccountClaims(aPub) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: MEM resolver_preload: { %s: %s } `, ojwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() for _, l := range []string{"", "Europe/Berlin", "America/New_York"} { t.Run("simple expiration "+l, func(t *testing.T) { start := time.Now() creds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { if l == _EMPTY_ { j.Times = []jwt.TimeRange{newTimeRange(start, validFor)} } else { loc, err := time.LoadLocation(l) require_NoError(t, err) j.Times = []jwt.TimeRange{newTimeRange(start.In(loc), validFor)} j.Locale = l } }) disconnectChan := make(chan struct{}) defer close(disconnectChan) errChan := make(chan struct{}) defer close(errChan) c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds), nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { if err != io.EOF { return } disconnectChan <- struct{}{} }), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { if err != nats.ErrAuthExpired { return } now := time.Now() stop := start.Add(validFor) // assure event happens within a second of stop if stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) { errChan <- struct{}{} } })) defer c.Close() chanRecv(t, errChan, 10*time.Second) chanRecv(t, disconnectChan, 10*time.Second) require_True(t, c.IsReconnecting()) require_False(t, c.IsConnected()) }) } t.Run("double expiration", func(t *testing.T) { start1 := time.Now() start2 := start1.Add(2 * validFor) creds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { j.Times = []jwt.TimeRange{newTimeRange(start1, validFor), newTimeRange(start2, validFor)} }) errChan := make(chan struct{}) defer close(errChan) reConnectChan := make(chan struct{}) defer close(reConnectChan) c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds), nats.ReconnectHandler(func(conn *nats.Conn) { reConnectChan <- struct{}{} }), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { if err != nats.ErrAuthExpired { return } now := time.Now() stop := start1.Add(validFor) // assure event happens within a second of stop if stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) { errChan <- struct{}{} return } stop = start2.Add(validFor) // assure event happens within a second of stop if stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) { errChan <- struct{}{} } })) defer c.Close() chanRecv(t, errChan, 10*time.Second) chanRecv(t, reConnectChan, 10*time.Second) require_False(t, c.IsReconnecting()) require_True(t, c.IsConnected()) chanRecv(t, errChan, 10*time.Second) }) t.Run("lower jwt expiration overwrites time", func(t *testing.T) { start := time.Now() creds := createUserWithLimit(t, kp, start.Add(validFor), func(j *jwt.UserPermissionLimits) { j.Times = []jwt.TimeRange{newTimeRange(start, 2*validFor)} }) disconnectChan := make(chan struct{}) defer close(disconnectChan) errChan := make(chan struct{}) defer close(errChan) c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds), nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { if err != io.EOF { return } disconnectChan <- struct{}{} }), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { if err != nats.ErrAuthExpired { return } now := time.Now() stop := start.Add(validFor) // assure event happens within a second of stop if stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) { errChan <- struct{}{} } })) defer c.Close() chanRecv(t, errChan, 10*time.Second) chanRecv(t, disconnectChan, 10*time.Second) require_True(t, c.IsReconnecting()) require_False(t, c.IsConnected()) }) } func NewJwtAccountClaim(name string) (nkeys.KeyPair, string, *jwt.AccountClaims) { sysKp, _ := nkeys.CreateAccount() sysPub, _ := sysKp.PublicKey() claim := jwt.NewAccountClaims(sysPub) claim.Name = name return sysKp, sysPub, claim } func TestJWTSysImportForDifferentAccount(t *testing.T) { _, sysPub, sysClaim := NewJwtAccountClaim("SYS") sysClaim.Exports.Add(&jwt.Export{ Type: jwt.Service, Subject: "$SYS.REQ.ACCOUNT.*.INFO", }) sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // create account aKp, aPub, claim := NewJwtAccountClaim("A") claim.Imports.Add(&jwt.Import{ Type: jwt.Service, Subject: "$SYS.REQ.ACCOUNT.*.INFO", LocalSubject: "COMMON.ADVISORY.SYS.REQ.ACCOUNT.*.INFO", Account: sysPub, }) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() nc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp)) defer nc.Close() // user for account a requests for a different account, the system account m, err := nc.Request(fmt.Sprintf("COMMON.ADVISORY.SYS.REQ.ACCOUNT.%s.INFO", sysPub), nil, time.Second) require_NoError(t, err) resp := &ServerAPIResponse{} require_NoError(t, json.Unmarshal(m.Data, resp)) require_True(t, resp.Error == nil) } func TestJWTSysImportFromNothing(t *testing.T) { _, sysPub, sysClaim := NewJwtAccountClaim("SYS") sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // create account aKp, aPub, claim := NewJwtAccountClaim("A") claim.Imports.Add(&jwt.Import{ Type: jwt.Service, // fails as it's not for own account, but system account Subject: jwt.Subject(fmt.Sprintf("$SYS.REQ.ACCOUNT.%s.CONNZ", sysPub)), LocalSubject: "fail1", Account: sysPub, }) claim.Imports.Add(&jwt.Import{ Type: jwt.Service, // fails as it's not for own account but all accounts Subject: "$SYS.REQ.ACCOUNT.*.CONNZ", LocalSubject: "fail2.*", Account: sysPub, }) claim.Imports.Add(&jwt.Import{ Type: jwt.Service, Subject: jwt.Subject(fmt.Sprintf("$SYS.REQ.ACCOUNT.%s.CONNZ", aPub)), LocalSubject: "pass", Account: sysPub, }) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() nc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp)) defer nc.Close() // user for account a requests for a different account, the system account _, err = nc.Request("pass", nil, time.Second) require_NoError(t, err) // default import _, err = nc.Request("$SYS.REQ.ACCOUNT.PING.CONNZ", nil, time.Second) require_NoError(t, err) _, err = nc.Request("fail1", nil, time.Second) require_Error(t, err) require_Contains(t, err.Error(), "no responders") // fails even for own account, as the import itself is bad _, err = nc.Request("fail2."+aPub, nil, time.Second) require_Error(t, err) require_Contains(t, err.Error(), "no responders") } func TestJWTSysImportOverwritePublic(t *testing.T) { _, sysPub, sysClaim := NewJwtAccountClaim("SYS") // this changes the export permissions to allow for requests for every account sysClaim.Exports.Add(&jwt.Export{ Type: jwt.Service, Subject: "$SYS.REQ.ACCOUNT.*.>", }) sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // create account aKp, aPub, claim := NewJwtAccountClaim("A") claim.Imports.Add(&jwt.Import{ Type: jwt.Service, Subject: jwt.Subject(fmt.Sprintf("$SYS.REQ.ACCOUNT.%s.CONNZ", sysPub)), LocalSubject: "pass1", Account: sysPub, }) claim.Imports.Add(&jwt.Import{ Type: jwt.Service, Subject: jwt.Subject(fmt.Sprintf("$SYS.REQ.ACCOUNT.%s.CONNZ", aPub)), LocalSubject: "pass2", Account: sysPub, }) claim.Imports.Add(&jwt.Import{ Type: jwt.Service, Subject: "$SYS.REQ.ACCOUNT.*.CONNZ", LocalSubject: "pass3.*", Account: sysPub, }) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() nc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp)) defer nc.Close() // user for account a requests for a different account, the system account _, err = nc.Request("pass1", nil, time.Second) require_NoError(t, err) _, err = nc.Request("pass2", nil, time.Second) require_NoError(t, err) _, err = nc.Request("pass3."+sysPub, nil, time.Second) require_NoError(t, err) _, err = nc.Request("pass3."+aPub, nil, time.Second) require_NoError(t, err) _, err = nc.Request("pass3.PING", nil, time.Second) require_NoError(t, err) } func TestJWTSysImportOverwriteToken(t *testing.T) { _, sysPub, sysClaim := NewJwtAccountClaim("SYS") // this changes the export permissions in a way that the internal imports can't satisfy sysClaim.Exports.Add(&jwt.Export{ Type: jwt.Service, Subject: "$SYS.REQ.>", TokenReq: true, }) sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) // create account aKp, aPub, claim := NewJwtAccountClaim("A") aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() nc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp)) defer nc.Close() // make sure the internal import still got added _, err = nc.Request("$SYS.REQ.ACCOUNT.PING.CONNZ", nil, time.Second) require_NoError(t, err) } func TestJWTLimits(t *testing.T) { doNotExpire := time.Now().AddDate(1, 0, 0) // create account kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() claim := jwt.NewAccountClaims(aPub) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: MEM resolver_preload: { %s: %s } `, ojwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() errChan := make(chan struct{}) defer close(errChan) t.Run("subs", func(t *testing.T) { creds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { j.Subs = 1 }) c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { if e := conn.LastError(); e != nil && strings.Contains(e.Error(), "maximum subscriptions exceeded") { errChan <- struct{}{} } }), ) defer c.Close() if _, err := c.Subscribe("foo", func(msg *nats.Msg) {}); err != nil { t.Fatalf("couldn't subscribe: %v", err) } if _, err = c.Subscribe("bar", func(msg *nats.Msg) {}); err != nil { t.Fatalf("expected error got: %v", err) } chanRecv(t, errChan, time.Second) }) t.Run("payload", func(t *testing.T) { creds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { j.Payload = 5 }) c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds)) defer c.Close() if err := c.Flush(); err != nil { t.Fatalf("flush failed %v", err) } if err := c.Publish("foo", []byte("world")); err != nil { t.Fatalf("couldn't publish: %v", err) } if err := c.Publish("foo", []byte("worldX")); err != nats.ErrMaxPayload { t.Fatalf("couldn't publish: %v", err) } }) } func TestJwtTemplates(t *testing.T) { kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() ukp, _ := nkeys.CreateUser() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Name = "myname" uclaim.Subject = upub uclaim.SetScoped(true) uclaim.IssuerAccount = aPub uclaim.Tags.Add("foo:foo1") uclaim.Tags.Add("foo:foo2") uclaim.Tags.Add("bar:bar1") uclaim.Tags.Add("bar:bar2") uclaim.Tags.Add("bar:bar3") lim := jwt.UserPermissionLimits{} lim.Pub.Allow.Add("{{tag(foo)}}.none.{{tag(bar)}}") lim.Pub.Deny.Add("{{tag(foo)}}.{{account-tag(acc)}}") lim.Sub.Allow.Add("{{tag(NOT_THERE)}}") // expect to not emit this lim.Sub.Deny.Add("foo.{{name()}}.{{subject()}}.{{account-name()}}.{{account-subject()}}.bar") acc := &Account{nameTag: "accname", tags: []string{"acc:acc1", "acc:acc2"}} resLim, err := processUserPermissionsTemplate(lim, uclaim, acc) require_NoError(t, err) test := func(expectedSubjects []string, res jwt.StringList) { t.Helper() require_True(t, len(res) == len(expectedSubjects)) for _, expetedSubj := range expectedSubjects { require_True(t, res.Contains(expetedSubj)) } } test(resLim.Pub.Allow, []string{"foo1.none.bar1", "foo1.none.bar2", "foo1.none.bar3", "foo2.none.bar1", "foo2.none.bar2", "foo2.none.bar3"}) test(resLim.Pub.Deny, []string{"foo1.acc1", "foo1.acc2", "foo2.acc1", "foo2.acc2"}) require_True(t, len(resLim.Sub.Allow) == 0) require_True(t, len(resLim.Sub.Deny) == 2) require_Contains(t, resLim.Sub.Deny[0], fmt.Sprintf("foo.myname.%s.accname.%s.bar", upub, aPub)) // added in to compensate for sub allow not resolving require_Contains(t, resLim.Sub.Deny[1], ">") lim.Pub.Deny.Add("{{tag(NOT_THERE)}}") _, err = processUserPermissionsTemplate(lim, uclaim, acc) require_Error(t, err) require_Contains(t, err.Error(), "generated invalid subject") } func TestJwtTemplateGoodTagAfterBadTag(t *testing.T) { kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() ukp, _ := nkeys.CreateUser() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Name = "myname" uclaim.Subject = upub uclaim.SetScoped(true) uclaim.IssuerAccount = aPub uclaim.Tags.Add("foo:foo1") lim := jwt.UserPermissionLimits{} lim.Pub.Deny.Add("{{tag(NOT_THERE)}}.{{tag(foo)}}") acc := &Account{nameTag: "accname", tags: []string{"acc:acc1", "acc:acc2"}} _, err := processUserPermissionsTemplate(lim, uclaim, acc) require_Error(t, err) require_Contains(t, err.Error(), "generated invalid subject") } func TestJWTLimitsTemplate(t *testing.T) { kp, _ := nkeys.CreateAccount() aPub, _ := kp.PublicKey() claim := jwt.NewAccountClaims(aPub) aSignScopedKp, aSignScopedPub := createKey(t) signer := jwt.NewUserScope() signer.Key = aSignScopedPub signer.Template.Pub.Deny.Add("denied") signer.Template.Pub.Allow.Add("foo.{{name()}}") signer.Template.Sub.Allow.Add("foo.{{name()}}") claim.SigningKeys.AddScopedSigner(signer) aJwt, err := claim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: MEM resolver_preload: { %s: %s } `, ojwt, aPub, aJwt))) sA, _ := RunServerWithConfig(conf) defer sA.Shutdown() errChan := make(chan struct{}) defer close(errChan) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Name = "myname" uclaim.Subject = upub uclaim.SetScoped(true) uclaim.IssuerAccount = aPub ujwt, err := uclaim.Encode(aSignScopedKp) require_NoError(t, err) creds := genCredsFile(t, ujwt, seed) t.Run("pass", func(t *testing.T) { c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds)) defer c.Close() sub, err := c.SubscribeSync("foo.myname") require_NoError(t, err) require_NoError(t, c.Flush()) require_NoError(t, c.Publish("foo.myname", nil)) _, err = sub.NextMsg(time.Second) require_NoError(t, err) }) t.Run("fail", func(t *testing.T) { c := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if strings.Contains(err.Error(), `Permissions Violation for Publish to "foo.othername"`) { errChan <- struct{}{} } })) defer c.Close() require_NoError(t, c.Publish("foo.othername", nil)) select { case <-errChan: case <-time.After(time.Second * 2): require_True(t, false) } }) } func TestJWTNoOperatorMode(t *testing.T) { for _, login := range []bool{true, false} { t.Run("", func(t *testing.T) { opts := DefaultOptions() if login { opts.Users = append(opts.Users, &User{Username: "u", Password: "pwd"}) } sA := RunServer(opts) defer sA.Shutdown() kp, _ := nkeys.CreateAccount() creds := createUserWithLimit(t, kp, time.Now().Add(time.Hour), nil) url := sA.ClientURL() if login { url = fmt.Sprintf("nats://u:pwd@%s:%d", sA.opts.Host, sA.opts.Port) } c := natsConnect(t, url, nats.UserCredentials(creds)) defer c.Close() sA.mu.Lock() defer sA.mu.Unlock() if len(sA.clients) != 1 { t.Fatalf("Expected exactly one client") } for _, v := range sA.clients { if v.opts.JWT != "" { t.Fatalf("Expected no jwt %v", v.opts.JWT) } } }) } } func TestJWTUserRevocation(t *testing.T) { test := func(all bool) { createAccountAndUser := func(done chan struct{}, pubKey, jwt1, jwt2, creds1, creds2 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) var err error *jwt1, err = claim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) // create updated claim need to assure that issue time differs if all { claim.Revoke(jwt.All) // revokes all jwt from now on } else { claim.Revoke(upub) // revokes this jwt from now on } time.Sleep(time.Millisecond * 1100) *jwt2, err = claim.Encode(oKp) require_NoError(t, err) ujwt2, err := uclaim.Encode(kp) require_NoError(t, err) *creds2 = genCredsFile(t, ujwt2, seed) done <- struct{}{} } // Create Accounts and corresponding revoked and non revoked user creds. Do so concurrently to speed up the test doneChan := make(chan struct{}, 2) defer close(doneChan) var syspub, sysjwt, dummy1, sysCreds, dummyCreds string go createAccountAndUser(doneChan, &syspub, &sysjwt, &dummy1, &sysCreds, &dummyCreds) var apub, ajwt1, ajwt2, aCreds1, aCreds2 string go createAccountAndUser(doneChan, &apub, &ajwt1, &ajwt2, &aCreds1, &aCreds2) for i := 0; i < cap(doneChan); i++ { <-doneChan } dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, syspub, dirSrv))) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() updateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt updateJwt(t, srv.ClientURL(), sysCreds, ajwt1, 1) // set account jwt without revocation ncSys := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds), nats.Name("conn name")) defer ncSys.Close() ncChan := make(chan *nats.Msg, 10) defer close(ncChan) sub, _ := ncSys.ChanSubscribe(fmt.Sprintf(disconnectEventSubj, apub), ncChan) // observe disconnect message defer sub.Unsubscribe() // use credentials that will be revoked ans assure that the connection will be disconnected nc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds1), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil && strings.Contains(err.Error(), "authentication revoked") { doneChan <- struct{}{} } }), ) defer nc.Close() // update account jwt to contain revocation if updateJwt(t, srv.ClientURL(), sysCreds, ajwt2, 1) != 1 { t.Fatalf("Expected jwt update to pass") } // assure that nc got disconnected due to the revocation select { case <-doneChan: case <-time.After(time.Second): t.Fatalf("Expected connection to have failed") } m := <-ncChan require_Len(t, strings.Count(string(m.Data), apub), 2) require_True(t, strings.Contains(string(m.Data), `"jwt":"eyJ0`)) // try again with old credentials. Expected to fail if nc1, err := nats.Connect(srv.ClientURL(), nats.UserCredentials(aCreds1)); err == nil { nc1.Close() t.Fatalf("Expected revoked credentials to fail") } // Assure new creds pass nc2 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds2)) defer nc2.Close() } t.Run("specific-key", func(t *testing.T) { test(false) }) t.Run("all-key", func(t *testing.T) { test(true) }) } func TestJWTActivationRevocation(t *testing.T) { test := func(all bool) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpClaim.Name = "Export" aExpClaim.Exports.Add(&jwt.Export{ Subject: "foo", Type: jwt.Stream, TokenReq: true, }) aExp1Jwt := encodeClaim(t, aExpClaim, aExpPub) aExpCreds := newUser(t, aExpKp) aImpKp, aImpPub := createKey(t) ac := &jwt.ActivationClaims{} ac.Subject = aImpPub ac.ImportSubject = "foo" ac.ImportType = jwt.Stream token, err := ac.Encode(aExpKp) require_NoError(t, err) revPubKey := aImpPub if all { revPubKey = jwt.All } aExpClaim.Exports[0].RevokeAt(revPubKey, time.Now()) aExp2Jwt := encodeClaim(t, aExpClaim, aExpPub) aExpClaim.Exports[0].ClearRevocation(revPubKey) aExp3Jwt := encodeClaim(t, aExpClaim, aExpPub) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "foo", Type: jwt.Stream, Account: aExpPub, Token: token, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, syspub, dirSrv))) t.Run("token-expired-on-connect", func(t *testing.T) { srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() defer removeDir(t, dirSrv) // clean jwt directory updateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt updateJwt(t, srv.ClientURL(), sysCreds, aExp2Jwt, 1) // set account jwt without revocation updateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1) ncExp1 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds)) defer ncExp1.Close() ncImp := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds)) defer ncImp.Close() sub, err := ncImp.SubscribeSync("foo") require_NoError(t, err) require_NoError(t, ncImp.Flush()) require_NoError(t, ncExp1.Publish("foo", []byte("1"))) _, err = sub.NextMsg(time.Second) require_Error(t, err) require_Equal(t, err.Error(), "nats: timeout") }) t.Run("token-expired-on-update", func(t *testing.T) { srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() defer removeDir(t, dirSrv) // clean jwt directory updateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt updateJwt(t, srv.ClientURL(), sysCreds, aExp1Jwt, 1) // set account jwt without revocation updateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1) ncExp1 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds)) defer ncExp1.Close() ncImp := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds)) defer ncImp.Close() sub, err := ncImp.SubscribeSync("foo") require_NoError(t, err) require_NoError(t, ncImp.Flush()) require_NoError(t, ncExp1.Publish("foo", []byte("1"))) m1, err := sub.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, string(m1.Data), "1") updateJwt(t, srv.ClientURL(), sysCreds, aExp2Jwt, 1) // set account jwt with revocation require_NoError(t, ncExp1.Publish("foo", []byte("2"))) _, err = sub.NextMsg(time.Second) require_Error(t, err) require_Equal(t, err.Error(), "nats: timeout") updateJwt(t, srv.ClientURL(), sysCreds, aExp3Jwt, 1) // set account with revocation cleared require_NoError(t, ncExp1.Publish("foo", []byte("3"))) m2, err := sub.NextMsg(time.Second) require_NoError(t, err) require_Equal(t, string(m2.Data), "3") }) } t.Run("specific-key", func(t *testing.T) { test(false) }) t.Run("all-key", func(t *testing.T) { test(true) }) } func TestJWTAccountFetchTimeout(t *testing.T) { createAccountAndUser := func(pubKey, jwt1, creds1 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) var err error *jwt1, err = claim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) } for _, cfg := range []string{ `type: full`, `type: cache`, } { t.Run("", func(t *testing.T) { var syspub, sysjwt, sysCreds string createAccountAndUser(&syspub, &sysjwt, &sysCreds) var apub, ajwt1, aCreds1 string createAccountAndUser(&apub, &ajwt1, &aCreds1) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { %s timeout: "100ms" dir: '%s' } `, ojwt, syspub, cfg, dirSrv))) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() updateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt start := time.Now() nc, err := nats.Connect(srv.ClientURL(), nats.UserCredentials(aCreds1)) if err == nil { t.Fatal("expected an error, got none") } else if !strings.Contains(err.Error(), "Authorization Violation") { t.Fatalf("expected an authorization violation, got: %v", err) } if time.Since(start) > 300*time.Millisecond { t.Fatal("expected timeout earlier") } defer nc.Close() }) } } func TestJWTAccountOps(t *testing.T) { op, _ := nkeys.CreateOperator() opPk, _ := op.PublicKey() sk, _ := nkeys.CreateOperator() skPk, _ := sk.PublicKey() opClaim := jwt.NewOperatorClaims(opPk) opClaim.SigningKeys.Add(skPk) opJwt, err := opClaim.Encode(op) require_NoError(t, err) createAccountAndUser := func(pubKey, jwt1, creds1 *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) var err error *jwt1, err = claim.Encode(sk) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt1, err := uclaim.Encode(kp) require_NoError(t, err) *creds1 = genCredsFile(t, ujwt1, seed) } generateRequest := func(accs []string, kp nkeys.KeyPair) []byte { t.Helper() opk, _ := kp.PublicKey() c := jwt.NewGenericClaims(opk) c.Data["accounts"] = accs cJwt, err := c.Encode(kp) if err != nil { t.Fatalf("Expected no error %v", err) } return []byte(cJwt) } for _, cfg := range []string{ `type: full allow_delete: true`, `type: cache`, } { t.Run("", func(t *testing.T) { var syspub, sysjwt, sysCreds string createAccountAndUser(&syspub, &sysjwt, &sysCreds) var apub, ajwt1, aCreds1 string createAccountAndUser(&apub, &ajwt1, &aCreds1) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { %s dir: '%s' } `, opJwt, syspub, cfg, dirSrv))) disconnectErrChan := make(chan struct{}, 1) defer close(disconnectErrChan) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() updateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt // push jwt (for full resolver) updateJwt(t, srv.ClientURL(), sysCreds, ajwt1, 1) // set jwt nc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds)) defer nc.Close() // simulate nas resolver in case of a lookup request (cache) nc.Subscribe(fmt.Sprintf(accLookupReqSubj, apub), func(msg *nats.Msg) { msg.Respond([]byte(ajwt1)) }) // connect so there is a reason to cache the request and so disconnect can be observed ncA := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds1), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil && strings.Contains(err.Error(), "account authentication expired") { disconnectErrChan <- struct{}{} } })) defer ncA.Close() resp, err := nc.Request(accListReqSubj, nil, time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), apub)) require_True(t, strings.Contains(string(resp.Data), syspub)) // delete nothing resp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{}, op), time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 0 accounts"`)) // issue delete, twice to also delete a non existing account // also switch which key used to sign the request for i := 0; i < 2; i++ { resp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second) require_NoError(t, err) require_True(t, strings.Contains(string(resp.Data), `"message":"deleted 1 accounts"`)) resp, err = nc.Request(accListReqSubj, nil, time.Second) require_False(t, strings.Contains(string(resp.Data), apub)) require_True(t, strings.Contains(string(resp.Data), syspub)) require_NoError(t, err) if i > 0 { continue } select { case <-disconnectErrChan: case <-time.After(time.Second): t.Fatal("Callback not executed") } } }) } } func createKey(t *testing.T) (nkeys.KeyPair, string) { t.Helper() kp, _ := nkeys.CreateAccount() syspub, _ := kp.PublicKey() return kp, syspub } func encodeClaim(t *testing.T, claim *jwt.AccountClaims, _ string) string { t.Helper() theJWT, err := claim.Encode(oKp) require_NoError(t, err) return theJWT } // returns user creds func newUserEx(t *testing.T, accKp nkeys.KeyPair, scoped bool, issuerAccount string) string { ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub uclaim.SetScoped(scoped) if issuerAccount != _EMPTY_ { uclaim.IssuerAccount = issuerAccount } ujwt, err := uclaim.Encode(accKp) require_NoError(t, err) return genCredsFile(t, ujwt, seed) } // returns user creds func newUser(t *testing.T, accKp nkeys.KeyPair) string { return newUserEx(t, accKp, false, "") } func TestJWTHeader(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) test := func(share bool) { aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpClaim.Exports.Add(&jwt.Export{ Name: "test", Subject: "srvc", Type: jwt.Service, TokenReq: false, Latency: &jwt.ServiceLatency{ Sampling: jwt.Headers, Results: "res", }, }) aExpJwt := encodeClaim(t, aExpClaim, aExpPub) aExpCreds := newUser(t, aExpKp) aImpKp, aImpPub := createKey(t) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Imports.Add(&jwt.Import{ Name: "test", Subject: "srvc", Account: aExpPub, Type: jwt.Service, Share: share, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, syspub, dirSrv))) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() updateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt updateJwt(t, srv.ClientURL(), sysCreds, aExpJwt, 1) updateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1) expNc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds)) defer expNc.Close() resChan := make(chan *nats.Msg, 1) expNc.ChanSubscribe("res", resChan) sub, err := expNc.Subscribe("srvc", func(msg *nats.Msg) { msg.Respond(nil) }) require_NoError(t, err) defer sub.Unsubscribe() impNc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds)) defer impNc.Close() // send request w/o header _, err = impNc.Request("srvc", []byte("msg1"), time.Second) require_NoError(t, err) require_True(t, len(resChan) == 0) _, err = impNc.RequestMsg(&nats.Msg{ Subject: "srvc", Data: []byte("msg2"), Header: nats.Header{ "X-B3-Sampled": []string{"1"}, "Share": []string{"Me"}}}, time.Second) require_NoError(t, err) select { case <-time.After(time.Second): t.Fatalf("should have received a response") case m := <-resChan: obj := map[string]any{} err = json.Unmarshal(m.Data, &obj) require_NoError(t, err) // test if shared is honored reqInfo := obj["requestor"].(map[string]any) // fields always set require_True(t, reqInfo["acc"] != nil) require_True(t, reqInfo["rtt"] != nil) // fields only set when shared _, ok1 := reqInfo["lang"] _, ok2 := reqInfo["ver"] _, ok3 := reqInfo["host"] _, ok4 := reqInfo["start"] if !share { ok1 = !ok1 ok2 = !ok2 ok3 = !ok3 ok4 = !ok4 } require_True(t, ok1) require_True(t, ok2) require_True(t, ok3) require_True(t, ok4) } require_True(t, len(resChan) == 0) } test(true) test(false) } func TestJWTAccountImportsWithWildcardSupport(t *testing.T) { test := func(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds string, jsEnabled bool, exSubExpect, exPub, imReq, imSubExpect string) { t.Helper() var jsSetting string if jsEnabled { jsSetting = "jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb}" } _, aSysPub := createKey(t) aSysClaim := jwt.NewAccountClaims(aSysPub) aSysJwt := encodeClaim(t, aSysClaim, aSysPub) cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" %s : "%s" } system_account: %s %s `, ojwt, aExpPub, aExpJwt, aImpPub, aImpJwt, aSysPub, aSysJwt, aSysPub, jsSetting))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncExp := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.UserCredentials(aExpCreds)) defer ncExp.Close() ncImp := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.UserCredentials(aImpCreds)) defer ncImp.Close() // Create subscriber for the service endpoint in foo. _, err := ncExp.Subscribe(exSubExpect, func(m *nats.Msg) { m.Respond([]byte("yes!")) }) require_NoError(t, err) ncExp.Flush() // Now test service import. if resp, err := ncImp.Request(imReq, []byte("yes?"), time.Second); err != nil { t.Fatalf("Expected a response to request %s got: %v", imReq, err) } else if string(resp.Data) != "yes!" { t.Fatalf("Expected a response of %q, got %q", "yes!", resp.Data) } subBar, err := ncImp.SubscribeSync(imSubExpect) require_NoError(t, err) ncImp.Flush() ncExp.Publish(exPub, []byte("event!")) if m, err := subBar.NextMsg(time.Second); err != nil { t.Fatalf("Expected a stream message got %v", err) } else if string(m.Data) != "event!" { t.Fatalf("Expected a response of %q, got %q", "event!", m.Data) } } createExporter := func() (string, string, string) { t.Helper() aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpClaim.Name = "Export" aExpClaim.Exports.Add(&jwt.Export{ Subject: "$request.*.$in.*.>", Type: jwt.Service, }, &jwt.Export{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, }) aExpJwt := encodeClaim(t, aExpClaim, aExpPub) aExpCreds := newUser(t, aExpKp) return aExpPub, aExpJwt, aExpCreds } t.Run("To", func(t *testing.T) { aExpPub, aExpJwt, aExpCreds := createExporter() aImpKp, aImpPub := createKey(t) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "my.request.*.*.>", Type: jwt.Service, To: "$request.*.$in.*.>", // services have local and remote switched between Subject and To Account: aExpPub, }, &jwt.Import{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, To: "prefix", Account: aExpPub, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) test(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, false, "$request.1.$in.2.bar", "$events.1.$in.2.bar", "my.request.1.2.bar", "prefix.$events.1.$in.2.bar") }) t.Run("LocalSubject-No-Reorder", func(t *testing.T) { aExpPub, aExpJwt, aExpCreds := createExporter() aImpKp, aImpPub := createKey(t) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "$request.*.$in.*.>", Type: jwt.Service, LocalSubject: "my.request.*.*.>", Account: aExpPub, }, &jwt.Import{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, LocalSubject: "my.events.*.*.>", Account: aExpPub, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) test(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, false, "$request.1.$in.2.bar", "$events.1.$in.2.bar", "my.request.1.2.bar", "my.events.1.2.bar") }) t.Run("LocalSubject-Reorder", func(t *testing.T) { for _, jsEnabled := range []bool{false, true} { t.Run(fmt.Sprintf("%t", jsEnabled), func(t *testing.T) { aExpPub, aExpJwt, aExpCreds := createExporter() aImpKp, aImpPub := createKey(t) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "$request.*.$in.*.>", Type: jwt.Service, LocalSubject: "my.request.$2.$1.>", Account: aExpPub, }, &jwt.Import{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, LocalSubject: "my.events.$2.$1.>", Account: aExpPub, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) test(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, jsEnabled, "$request.2.$in.1.bar", "$events.1.$in.2.bar", "my.request.1.2.bar", "my.events.2.1.bar") }) } }) } func TestJWTAccountTokenImportMisuse(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpClaim.Name = "Export" aExpClaim.Exports.Add(&jwt.Export{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, TokenReq: true, }, &jwt.Export{ Subject: "foo", Type: jwt.Stream, TokenReq: true, }) aExpJwt := encodeClaim(t, aExpClaim, aExpPub) createImportingAccountClaim := func(aImpKp nkeys.KeyPair, aExpPub string, ac *jwt.ActivationClaims) (string, string) { t.Helper() token, err := ac.Encode(aExpKp) require_NoError(t, err) aImpPub, err := aImpKp.PublicKey() require_NoError(t, err) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "$events.*.$in.*.>", Type: jwt.Stream, Account: aExpPub, Token: token, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) return aImpJwt, aImpCreds } testConnect := func(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds string) { t.Helper() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/A/" { // Server startup w.Write(nil) } else if r.URL.Path == "/A/"+aExpPub { w.Write([]byte(aExpJwt)) } else if r.URL.Path == "/A/"+aImpPub { w.Write([]byte(aImpJwt)) } else { t.Fatal("not expected") } })) defer ts.Close() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: URL("%s/A/") `, ojwt, ts.URL))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncImp, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.UserCredentials(aImpCreds)) require_Error(t, err) // misuse needs to result in an error defer ncImp.Close() } testNatsResolver := func(aImpJwt string) { t.Helper() dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, syspub, dirSrv))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() require_True(t, updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) == 1) require_True(t, updateJwt(t, s.ClientURL(), sysCreds, aExpJwt, 1) == 1) require_True(t, updateJwt(t, s.ClientURL(), sysCreds, aImpJwt, 1) == 0) // assure this did not succeed } t.Run("wrong-account", func(t *testing.T) { aImpKp, aImpPub := createKey(t) ac := &jwt.ActivationClaims{} _, ac.Subject = createKey(t) // on purpose issue this token for another account ac.ImportSubject = "$events.*.$in.*.>" ac.ImportType = jwt.Stream aImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac) testConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds) testNatsResolver(aImpJwt) }) t.Run("different-subject", func(t *testing.T) { aImpKp, aImpPub := createKey(t) ac := &jwt.ActivationClaims{} ac.Subject = aImpPub ac.ImportSubject = "foo" // on purpose use a subject from another export ac.ImportType = jwt.Stream aImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac) testConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds) testNatsResolver(aImpJwt) }) t.Run("non-existing-subject", func(t *testing.T) { aImpKp, aImpPub := createKey(t) ac := &jwt.ActivationClaims{} ac.Subject = aImpPub ac.ImportSubject = "does-not-exist-or-from-different-export" // on purpose use a non exported subject ac.ImportType = jwt.Stream aImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac) testConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds) testNatsResolver(aImpJwt) }) } func TestJWTResponseThreshold(t *testing.T) { respThresh := 20 * time.Millisecond aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpClaim.Name = "Export" aExpClaim.Exports.Add(&jwt.Export{ Subject: "srvc", Type: jwt.Service, ResponseThreshold: respThresh, }) aExpJwt := encodeClaim(t, aExpClaim, aExpPub) aExpCreds := newUser(t, aExpKp) aImpKp, aImpPub := createKey(t) aImpClaim := jwt.NewAccountClaims(aImpPub) aImpClaim.Name = "Import" aImpClaim.Imports.Add(&jwt.Import{ Subject: "srvc", Type: jwt.Service, Account: aExpPub, }) aImpJwt := encodeClaim(t, aImpClaim, aImpPub) aImpCreds := newUser(t, aImpKp) cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" } `, ojwt, aExpPub, aExpJwt, aImpPub, aImpJwt))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() ncExp := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.UserCredentials(aExpCreds)) defer ncExp.Close() ncImp := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.UserCredentials(aImpCreds)) defer ncImp.Close() delayChan := make(chan time.Duration, 1) // Create subscriber for the service endpoint in foo. _, err := ncExp.Subscribe("srvc", func(m *nats.Msg) { time.Sleep(<-delayChan) m.Respond([]byte("yes!")) }) require_NoError(t, err) ncExp.Flush() t.Run("No-Timeout", func(t *testing.T) { delayChan <- respThresh / 2 if resp, err := ncImp.Request("srvc", []byte("yes?"), 4*respThresh); err != nil { t.Fatalf("Expected a response to request srvc got: %v", err) } else if string(resp.Data) != "yes!" { t.Fatalf("Expected a response of %q, got %q", "yes!", resp.Data) } }) t.Run("Timeout", func(t *testing.T) { delayChan <- 2 * respThresh if _, err := ncImp.Request("srvc", []byte("yes?"), 4*respThresh); err == nil || err != nats.ErrTimeout { t.Fatalf("Expected a timeout") } }) } func TestJWTJetStreamTiers(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, accPub := createKey(t) accClaim := jwt.NewAccountClaims(accPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1100, MemoryStorage: 0, Consumer: 2, Streams: 2} accJwt1 := encodeClaim(t, accClaim, accPub) accCreds := newUser(t, accKp) start := time.Now() storeDir := t.TempDir() dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: s1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } operator: %s system_account: %s resolver: { type: full dir: '%s' } `, storeDir, ojwt, syspub, dirSrv))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) updateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1) nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) // Test tiers up to stream limits _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-2", Replicas: 1, Subjects: []string{"testR1-2"}}) require_NoError(t, err) // Test exceeding tiered stream limit _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum number of streams reached") // Test tiers up to consumer limits _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) // test exceeding tiered consumer limits _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur4", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur5", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") // test tiered storage limit msg := [512]byte{} _, err = js.Publish("testR1-1", msg[:]) require_NoError(t, err) _, err = js.Publish("testR1-2", msg[:]) require_NoError(t, err) ainfo, err := js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 1100) // test exceeding tiered storage limit _, err = js.Publish("testR1-1", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") // Check that storage has not increased after the rejected publish. ainfo, err = js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 1100) time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: 1650, MemoryStorage: 0, Consumer: 1, Streams: 3} accJwt2 := encodeClaim(t, accClaim, accPub) updateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1) // test same sequence as before, add stream, fail add stream, add consumer, fail add consumer, publish, fail publish _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-4", Replicas: 1, Subjects: []string{"testR1-4"}}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum number of streams reached") _, err = js.AddConsumer("testR1-3", &nats.ConsumerConfig{Durable: "dur6", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) _, err = js.AddConsumer("testR1-3", &nats.ConsumerConfig{Durable: "dur7", AckPolicy: nats.AckExplicitPolicy}) require_Error(t, err) require_Equal(t, err.Error(), "nats: maximum consumers limit reached") // At this point it will be exactly at the DiskStorage limit so it should not fail. _, err = js.Publish("testR1-3", msg[:]) require_NoError(t, err) ainfo, err = js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 1650) _, err = js.Publish("testR1-3", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") // Should remain at the same usage. ainfo, err = js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 1650) } func TestJWTJetStreamMaxAckPending(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, accPub := createKey(t) accClaim := jwt.NewAccountClaims(accPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit, Streams: jwt.NoLimit, MaxAckPending: int64(1000), } accJwt1 := encodeClaim(t, accClaim, accPub) accCreds := newUser(t, accKp) start := time.Now() storeDir := t.TempDir() dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: s1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } operator: %s system_account: %s resolver: { type: full dir: '%s' } `, storeDir, ojwt, syspub, dirSrv))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) updateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1) nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1}) require_NoError(t, err) _, err = js.AddConsumer("foo", &nats.ConsumerConfig{ Durable: "dur1", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000}) require_Error(t, err) require_Equal(t, err.Error(), "nats: consumer max ack pending exceeds system limit of 1000") ci, err := js.AddConsumer("foo", &nats.ConsumerConfig{ Durable: "dur2", AckPolicy: nats.AckAllPolicy, MaxAckPending: 500}) require_NoError(t, err) require_True(t, ci.Config.MaxAckPending == 500) _, err = js.UpdateConsumer("foo", &nats.ConsumerConfig{ Durable: "dur2", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000}) require_Error(t, err) require_Equal(t, err.Error(), "nats: consumer max ack pending exceeds system limit of 1000") time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit, Streams: jwt.NoLimit, MaxAckPending: int64(2000)} accJwt2 := encodeClaim(t, accClaim, accPub) updateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1) ci, err = js.UpdateConsumer("foo", &nats.ConsumerConfig{ Durable: "dur2", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000}) require_NoError(t, err) require_True(t, ci.Config.MaxAckPending == 2000) } func TestJWTJetStreamMaxStreamBytes(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) accKp, accPub := createKey(t) accClaim := jwt.NewAccountClaims(accPub) accClaim.Name = "acc" accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit, Streams: jwt.NoLimit, DiskMaxStreamBytes: 1024, MaxBytesRequired: false, } accJwt1 := encodeClaim(t, accClaim, accPub) accCreds := newUser(t, accKp) start := time.Now() storeDir := t.TempDir() dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: s1 jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } operator: %s system_account: %s resolver: { type: full dir: '%s' } `, storeDir, ojwt, syspub, dirSrv))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) updateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1) nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds)) defer nc.Close() js, err := nc.JetStream() require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, MaxBytes: 2048}) require_Error(t, err) require_Equal(t, err.Error(), "nats: stream max bytes exceeds account limit max stream bytes") _, err = js.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, MaxBytes: 1024}) require_NoError(t, err) msg := [900]byte{} _, err = js.AddStream(&nats.StreamConfig{Name: "baz", Replicas: 1}) require_NoError(t, err) _, err = js.Publish("baz", msg[:]) require_NoError(t, err) _, err = js.Publish("baz", msg[:]) // exceeds max stream bytes require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ DiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit, Streams: jwt.NoLimit, DiskMaxStreamBytes: 2048, MaxBytesRequired: true} accJwt2 := encodeClaim(t, accClaim, accPub) updateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1) _, err = js.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, MaxBytes: 3000}) require_Error(t, err) require_Equal(t, err.Error(), "nats: stream max bytes exceeds account limit max stream bytes") _, err = js.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, MaxBytes: 2048}) require_NoError(t, err) ainfo, err := js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 933) // This should be exactly at the limit of the account. _, err = js.Publish("baz", []byte(strings.Repeat("A", 1082))) require_NoError(t, err) ainfo, err = js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 2048) // Exceed max stream bytes limit. _, err = js.Publish("baz", []byte("1")) require_Error(t, err) require_Equal(t, err.Error(), "nats: resource limits exceeded for account") // Confirm no changes after rejected publish. ainfo, err = js.AccountInfo() require_NoError(t, err) require_Equal(t, ainfo.Tiers["R1"].Store, 2048) // test disabling max bytes required _, err = js.UpdateStream(&nats.StreamConfig{Name: "bar", Replicas: 1}) require_Error(t, err) require_Equal(t, err.Error(), "nats: account requires a stream config to have max bytes set") } func TestJWTQueuePermissions(t *testing.T) { aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) aExpJwt := encodeClaim(t, aExpClaim, aExpPub) newUser := func(t *testing.T, permType string) string { ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub switch permType { case "allow": uclaim.Permissions.Sub.Allow.Add("foo.> *.dev") case "deny": uclaim.Permissions.Sub.Deny.Add("foo.> *.dev") } ujwt, err := uclaim.Encode(aExpKp) require_NoError(t, err) return genCredsFile(t, ujwt, seed) } confFileName := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s resolver = MEMORY resolver_preload = { %s : %s }`, ojwt, aExpPub, aExpJwt))) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } opts.NoLog, opts.NoSigs = true, true errChan := make(chan error, 1) defer close(errChan) s := RunServer(opts) defer s.Shutdown() for _, test := range []struct { permType string queue string errExpected bool }{ {"allow", "queue.dev", false}, {"allow", "", true}, {"allow", "bad", true}, {"deny", "", false}, {"deny", "queue.dev", true}, } { t.Run(test.permType+test.queue, func(t *testing.T) { usrCreds := newUser(t, test.permType) nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", opts.Port), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { errChan <- err }), nats.UserCredentials(usrCreds)) if err != nil { t.Fatalf("No error expected: %v", err) } defer nc.Close() if test.queue == "" { if _, err := nc.Subscribe("foo.bar", func(msg *nats.Msg) {}); err != nil { t.Fatalf("no error expected: %v", err) } } else { if _, err := nc.QueueSubscribe("foo.bar", test.queue, func(msg *nats.Msg) {}); err != nil { t.Fatalf("no error expected: %v", err) } } nc.Flush() select { case err := <-errChan: if !test.errExpected { t.Fatalf("Expected no error, got %v", err) } if !strings.Contains(err.Error(), `Permissions Violation for Subscription to "foo.bar"`) { t.Fatalf("error %v", err) } case <-time.After(150 * time.Millisecond): if test.errExpected { t.Fatal("Expected an error") } } }) } } func TestJWScopedSigningKeys(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) _, aExpPub := createKey(t) accClaim := jwt.NewAccountClaims(aExpPub) accClaim.Name = "acc" aSignNonScopedKp, aSignNonScopedPub := createKey(t) accClaim.SigningKeys.Add(aSignNonScopedPub) aSignScopedKp, aSignScopedPub := createKey(t) signer := jwt.NewUserScope() signer.Key = aSignScopedPub signer.Template.Pub.Deny.Add("denied") signer.Template.Payload = 5 signer.Template.AllowedConnectionTypes.Add(jwt.ConnectionTypeStandard) accClaim.SigningKeys.AddScopedSigner(signer) accJwt := encodeClaim(t, accClaim, aExpPub) aNonScopedCreds := newUserEx(t, aSignNonScopedKp, false, aExpPub) aBadScopedCreds := newUserEx(t, aSignScopedKp, false, aExpPub) aScopedCreds := newUserEx(t, aSignScopedKp, true, aExpPub) dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } websocket { listen: 127.0.0.1:-1 no_tls: true } `, ojwt, syspub, dirSrv))) s, opts := RunServerWithConfig(cf) defer s.Shutdown() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) wsUrl := fmt.Sprintf("ws://%s:%d", opts.Websocket.Host, opts.Websocket.Port) errChan := make(chan error, 1) defer close(errChan) awaitError := func(expected bool) { t.Helper() select { case err := <-errChan: if !expected { t.Fatalf("Expected no error, got %v", err) } case <-time.After(150 * time.Millisecond): if expected { t.Fatal("Expected an error") } } } errHdlr := nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { errChan <- err }) if updateJwt(t, url, sysCreds, sysJwt, 1) != 1 { t.Error("Expected update to pass") } else if updateJwt(t, url, sysCreds, accJwt, 1) != 1 { t.Error("Expected update to pass") } t.Run("bad-scoped-signing-key", func(t *testing.T) { _, err := nats.Connect(url, nats.UserCredentials(aBadScopedCreds)) require_Error(t, err) }) t.Run("regular-signing-key", func(t *testing.T) { nc := natsConnect(t, url, nats.UserCredentials(aNonScopedCreds), errHdlr) defer nc.Close() nc.Flush() err := nc.Publish("denied", nil) require_NoError(t, err) nc.Flush() awaitError(false) }) t.Run("scoped-signing-key-client-side", func(t *testing.T) { nc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr) defer nc.Close() nc.Flush() err := nc.Publish("too-long", []byte("way.too.long.for.payload.limit")) require_Error(t, err) require_True(t, strings.Contains(err.Error(), ErrMaxPayload.Error())) }) t.Run("scoped-signing-key-server-side", func(t *testing.T) { nc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr) defer nc.Close() nc.Flush() err := nc.Publish("denied", nil) require_NoError(t, err) nc.Flush() awaitError(true) }) t.Run("scoped-signing-key-allowed-conn-types", func(t *testing.T) { _, err := nats.Connect(wsUrl, nats.UserCredentials(aScopedCreds)) require_Error(t, err) }) t.Run("scoped-signing-key-reload", func(t *testing.T) { reconChan := make(chan struct{}, 1) defer close(reconChan) msgChan := make(chan *nats.Msg, 2) defer close(msgChan) nc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr, nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { if err != nil { errChan <- err } }), nats.ReconnectHandler(func(conn *nats.Conn) { reconChan <- struct{}{} }), nats.ReconnectWait(100*time.Millisecond), ) defer nc.Close() _, err := nc.ChanSubscribe("denied", msgChan) require_NoError(t, err) nc.Flush() err = nc.Publish("denied", nil) require_NoError(t, err) awaitError(true) require_Len(t, len(msgChan), 0) // Alter scoped permissions and update signer.Template.Payload = -1 signer.Template.Pub.Deny.Remove("denied") signer.Template.AllowedConnectionTypes.Add(jwt.ConnectionTypeWebsocket) accClaim.SigningKeys.AddScopedSigner(signer) accUpdatedJwt := encodeClaim(t, accClaim, aExpPub) if updateJwt(t, url, sysCreds, accUpdatedJwt, 1) != 1 { t.Error("Expected update to pass") } // disconnect triggered by update awaitError(true) <-reconChan nc.Flush() err = nc.Publish("denied", []byte("way.too.long.for.old.payload.limit")) require_NoError(t, err) awaitError(false) msg := <-msgChan require_Equal(t, string(msg.Data), "way.too.long.for.old.payload.limit") require_Len(t, len(msgChan), 0) nc.Close() nc, err = nats.Connect(wsUrl, nats.UserCredentials(aScopedCreds)) require_NoError(t, err) nc.Flush() defer nc.Close() }) require_Len(t, len(errChan), 0) } func TestJWTStrictSigningKeys(t *testing.T) { newAccount := func(opKp nkeys.KeyPair) (nkeys.KeyPair, nkeys.KeyPair, string, *jwt.AccountClaims, string) { accId, err := nkeys.CreateAccount() require_NoError(t, err) accIdPub, err := accId.PublicKey() require_NoError(t, err) accSig, err := nkeys.CreateAccount() require_NoError(t, err) accSigPub, err := accSig.PublicKey() require_NoError(t, err) aClaim := jwt.NewAccountClaims(accIdPub) aClaim.SigningKeys.Add(accSigPub) theJwt, err := aClaim.Encode(opKp) require_NoError(t, err) return accId, accSig, accIdPub, aClaim, theJwt } opId, err := nkeys.CreateOperator() require_NoError(t, err) opIdPub, err := opId.PublicKey() require_NoError(t, err) opSig, err := nkeys.CreateOperator() require_NoError(t, err) opSigPub, err := opSig.PublicKey() require_NoError(t, err) aBadBadKp, aBadGoodKp, aBadPub, _, aBadJwt := newAccount(opId) aGoodBadKp, aGoodGoodKp, aGoodPub, _, aGoodJwt := newAccount(opSig) _, aSysKp, aSysPub, _, aSysJwt := newAccount(opSig) oClaim := jwt.NewOperatorClaims(opIdPub) oClaim.StrictSigningKeyUsage = true oClaim.SigningKeys.Add(opSigPub) oClaim.SystemAccount = aSysPub oJwt, err := oClaim.Encode(opId) require_NoError(t, err) uBadBadCreds := newUserEx(t, aBadBadKp, false, aBadPub) uBadGoodCreds := newUserEx(t, aBadGoodKp, false, aBadPub) uGoodBadCreds := newUserEx(t, aGoodBadKp, false, aGoodPub) uGoodGoodCreds := newUserEx(t, aGoodGoodKp, false, aGoodPub) uSysCreds := newUserEx(t, aSysKp, false, aSysPub) connectTest := func(url string) { for _, test := range []struct { creds string fail bool }{ {uBadBadCreds, true}, {uBadGoodCreds, true}, {uGoodBadCreds, true}, {uGoodGoodCreds, false}, } { nc, err := nats.Connect(url, nats.UserCredentials(test.creds)) nc.Close() if test.fail { require_Error(t, err) } else { require_NoError(t, err) } } } t.Run("resolver", func(t *testing.T) { dirSrv := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s resolver: { type: full dir: '%s' } resolver_preload = { %s : "%s" } `, oJwt, dirSrv, aSysPub, aSysJwt))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() url := s.ClientURL() if updateJwt(t, url, uSysCreds, aBadJwt, 1) != 0 { t.Fatal("Expected negative response") } if updateJwt(t, url, uSysCreds, aGoodJwt, 1) != 1 { t.Fatal("Expected positive response") } connectTest(url) }) t.Run("mem-resolver", func(t *testing.T) { cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s resolver: MEMORY resolver_preload = { %s : "%s" %s : "%s" %s : "%s" } `, oJwt, aSysPub, aSysJwt, aBadPub, aBadJwt, aGoodPub, aGoodJwt))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() connectTest(s.ClientURL()) }) } func TestJWTAccountProtectedImport(t *testing.T) { srvFmt := ` port: -1 operator = %s resolver: MEMORY resolver_preload = { %s : "%s" %s : "%s" } ` setupAccounts := func(pass bool) (nkeys.KeyPair, string, string, string, nkeys.KeyPair, string, string, string, string) { // Create accounts and imports/exports. exportKP, _ := nkeys.CreateAccount() exportPub, _ := exportKP.PublicKey() exportAC := jwt.NewAccountClaims(exportPub) exportAC.Exports.Add(&jwt.Export{Subject: "service.*", Type: jwt.Service, AccountTokenPosition: 2}) exportAC.Exports.Add(&jwt.Export{Subject: "stream.*", Type: jwt.Stream, AccountTokenPosition: 2}) exportJWT, err := exportAC.Encode(oKp) require_NoError(t, err) // create alternative exporter jwt without account token pos set exportAC.Exports = jwt.Exports{} exportAC.Exports.Add(&jwt.Export{Subject: "service.*", Type: jwt.Service}) exportAC.Exports.Add(&jwt.Export{Subject: "stream.*", Type: jwt.Stream}) exportJWTNoPos, err := exportAC.Encode(oKp) require_NoError(t, err) importKP, _ := nkeys.CreateAccount() importPub, _ := importKP.PublicKey() importAc := jwt.NewAccountClaims(importPub) srvcSub, strmSub := "service.foo", "stream.foo" if pass { srvcSub = fmt.Sprintf("service.%s", importPub) strmSub = fmt.Sprintf("stream.%s", importPub) } importAc.Imports.Add(&jwt.Import{Account: exportPub, Subject: jwt.Subject(srvcSub), Type: jwt.Service}) importAc.Imports.Add(&jwt.Import{Account: exportPub, Subject: jwt.Subject(strmSub), Type: jwt.Stream}) importJWT, err := importAc.Encode(oKp) require_NoError(t, err) return exportKP, exportPub, exportJWT, exportJWTNoPos, importKP, importPub, importJWT, srvcSub, strmSub } t.Run("pass", func(t *testing.T) { exportKp, exportPub, exportJWT, _, importKp, importPub, importJWT, srvcSub, strmSub := setupAccounts(true) cf := createConfFile(t, []byte(fmt.Sprintf(srvFmt, ojwt, exportPub, exportJWT, importPub, importJWT))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() ncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp)) defer ncExp.Close() ncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp)) defer ncImp.Close() t.Run("service", func(t *testing.T) { sub, err := ncExp.Subscribe("service.*", func(msg *nats.Msg) { msg.Respond([]byte("world")) }) defer sub.Unsubscribe() require_NoError(t, err) ncExp.Flush() msg, err := ncImp.Request(srvcSub, []byte("hello"), time.Second) require_NoError(t, err) require_Equal(t, string(msg.Data), "world") }) t.Run("stream", func(t *testing.T) { msgChan := make(chan *nats.Msg, 4) defer close(msgChan) sub, err := ncImp.ChanSubscribe(strmSub, msgChan) defer sub.Unsubscribe() require_NoError(t, err) ncImp.Flush() err = ncExp.Publish("stream.foo", []byte("hello")) require_NoError(t, err) err = ncExp.Publish(strmSub, []byte("hello")) require_NoError(t, err) msg := <-msgChan require_Equal(t, string(msg.Data), "hello") require_True(t, len(msgChan) == 0) }) }) t.Run("fail", func(t *testing.T) { exportKp, exportPub, exportJWT, _, importKp, importPub, importJWT, srvcSub, strmSub := setupAccounts(false) cf := createConfFile(t, []byte(fmt.Sprintf(srvFmt, ojwt, exportPub, exportJWT, importPub, importJWT))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() ncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp)) defer ncExp.Close() ncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp)) defer ncImp.Close() t.Run("service", func(t *testing.T) { sub, err := ncExp.Subscribe("service.*", func(msg *nats.Msg) { msg.Respond([]byte("world")) }) defer sub.Unsubscribe() require_NoError(t, err) ncExp.Flush() _, err = ncImp.Request(srvcSub, []byte("hello"), time.Second) require_Error(t, err) require_Contains(t, err.Error(), "no responders available for request") }) t.Run("stream", func(t *testing.T) { msgChan := make(chan *nats.Msg, 4) defer close(msgChan) _, err := ncImp.ChanSubscribe(strmSub, msgChan) require_NoError(t, err) ncImp.Flush() err = ncExp.Publish("stream.foo", []byte("hello")) require_NoError(t, err) err = ncExp.Publish(strmSub, []byte("hello")) require_NoError(t, err) select { case <-msgChan: t.Fatal("did not expect a message") case <-time.After(250 * time.Millisecond): } require_True(t, len(msgChan) == 0) }) }) t.Run("reload-off-2-on", func(t *testing.T) { exportKp, exportPub, exportJWTOn, exportJWTOff, importKp, _, importJWT, srvcSub, strmSub := setupAccounts(false) dirSrv := t.TempDir() // set up system account. Relying bootstrapping system account to not create JWT sysAcc, err := nkeys.CreateAccount() require_NoError(t, err) sysPub, err := sysAcc.PublicKey() require_NoError(t, err) sysUsrCreds := newUserEx(t, sysAcc, false, sysPub) cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 operator = %s system_account = %s resolver: { type: full dir: '%s' }`, ojwt, sysPub, dirSrv))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() updateJwt(t, s.ClientURL(), sysUsrCreds, importJWT, 1) updateJwt(t, s.ClientURL(), sysUsrCreds, exportJWTOff, 1) ncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp)) defer ncExp.Close() ncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp)) defer ncImp.Close() msgChan := make(chan *nats.Msg, 4) defer close(msgChan) // ensure service passes subSrvc, err := ncExp.Subscribe("service.*", func(msg *nats.Msg) { msg.Respond([]byte("world")) }) defer subSrvc.Unsubscribe() require_NoError(t, err) ncExp.Flush() respMst, err := ncImp.Request(srvcSub, []byte("hello"), time.Second) require_NoError(t, err) require_Equal(t, string(respMst.Data), "world") // ensure stream passes subStrm, err := ncImp.ChanSubscribe(strmSub, msgChan) defer subStrm.Unsubscribe() require_NoError(t, err) ncImp.Flush() err = ncExp.Publish(strmSub, []byte("hello")) require_NoError(t, err) msg := <-msgChan require_Equal(t, string(msg.Data), "hello") require_True(t, len(msgChan) == 0) updateJwt(t, s.ClientURL(), sysUsrCreds, exportJWTOn, 1) // ensure service fails _, err = ncImp.Request(srvcSub, []byte("hello"), time.Second) require_Error(t, err, nats.ErrNoResponders) s.AccountResolver().Store(exportPub, exportJWTOn) // ensure stream fails err = ncExp.Publish(strmSub, []byte("hello")) require_NoError(t, err) select { case <-msgChan: t.Fatal("did not expect a message") case <-time.After(250 * time.Millisecond): } require_True(t, len(msgChan) == 0) }) } // Headers are ignored in claims update, but passing them should not cause error. func TestJWTClaimsUpdateWithHeaders(t *testing.T) { skp, spub := createKey(t) newUser(t, skp) sclaim := jwt.NewAccountClaims(spub) encodeClaim(t, sclaim, spub) akp, apub := createKey(t) newUser(t, akp) claim := jwt.NewAccountClaims(apub) jwtClaim := encodeClaim(t, claim, apub) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, spub, dirSrv))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() type zapi struct { Server *ServerInfo Data *Connz Error *ApiError } sc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, skp)) defer sc.Close() // Pass claims update with headers. msg := &nats.Msg{ Subject: "$SYS.REQ.CLAIMS.UPDATE", Data: []byte(jwtClaim), Header: map[string][]string{"key": {"value"}}, } resp, err := sc.RequestMsg(msg, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cz zapi if err := json.Unmarshal(resp.Data, &cz); err != nil { t.Fatalf("Unexpected error: %v", err) } if cz.Error != nil { t.Fatalf("Unexpected error: %+v", cz.Error) } } func TestJWTMappings(t *testing.T) { sysKp, syspub := createKey(t) sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) sysCreds := newUser(t, sysKp) // create two jwt, one with and one without mapping aKp, aPub := createKey(t) aClaim := jwt.NewAccountClaims(aPub) aJwtNoM := encodeClaim(t, aClaim, aPub) aClaim.AddMapping("foo1", jwt.WeightedMapping{Subject: "bar1"}) aJwtMap1 := encodeClaim(t, aClaim, aPub) aClaim.Mappings = map[jwt.Subject][]jwt.WeightedMapping{} aClaim.AddMapping("foo2", jwt.WeightedMapping{Subject: "bar2"}) aJwtMap2 := encodeClaim(t, aClaim, aPub) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, syspub, dirSrv))) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() updateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt test := func(pub, sub string, fail bool) { t.Helper() nc := natsConnect(t, srv.ClientURL(), createUserCreds(t, srv, aKp)) defer nc.Close() s, err := nc.SubscribeSync(sub) require_NoError(t, err) nc.Flush() err = nc.Publish(pub, nil) require_NoError(t, err) _, err = s.NextMsg(500 * time.Millisecond) switch { case fail && err == nil: t.Fatal("expected error, got none") case !fail && err != nil: t.Fatalf("expected no error, got %v", err) } } // turn mappings on require_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtMap1, 1)) test("foo1", "bar1", false) // alter mappings require_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtMap2, 1)) test("foo1", "bar1", true) test("foo2", "bar2", false) // turn mappings off require_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtNoM, 1)) test("foo2", "bar2", true) } func TestJWTOperatorPinnedAccounts(t *testing.T) { kps, pubs, jwts := [4]nkeys.KeyPair{}, [4]string{}, [4]string{} for i := 0; i < 4; i++ { kps[i], pubs[i] = createKey(t) jwts[i] = encodeClaim(t, jwt.NewAccountClaims(pubs[i]), pubs[i]) } // create system account user credentials, index 0 is handled as system account newUser(t, kps[0]) cfgCommon := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s:%s %s:%s %s:%s %s:%s }`, ojwt, pubs[0], pubs[0], jwts[0], pubs[1], jwts[1], pubs[2], jwts[2], pubs[3], jwts[3]) cfgFmt := cfgCommon + ` resolver_pinned_accounts: [%s, %s] ` conf := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, pubs[1], pubs[2]))) srv, _ := RunServerWithConfig(conf) defer srv.Shutdown() connectPass := func(keys ...nkeys.KeyPair) { for _, kp := range keys { nc, err := nats.Connect(srv.ClientURL(), createUserCreds(t, srv, kp)) require_NoError(t, err) defer nc.Close() } } var pinnedFail uint64 connectFail := func(key nkeys.KeyPair) { _, err := nats.Connect(srv.ClientURL(), createUserCreds(t, srv, key)) require_Error(t, err) require_Contains(t, err.Error(), "Authorization Violation") v, err := srv.Varz(&VarzOptions{}) require_NoError(t, err) require_True(t, pinnedFail+1 == v.PinnedAccountFail) pinnedFail = v.PinnedAccountFail } connectPass(kps[0], kps[1], kps[2]) // make sure user from accounts listed and system account (index 0) work connectFail(kps[3]) // make sure the other user does not work // reload and test again reloadUpdateConfig(t, srv, conf, fmt.Sprintf(cfgFmt, pubs[2], pubs[3])) connectPass(kps[0], kps[2], kps[3]) // make sure user from accounts listed and system account (index 0) work connectFail(kps[1]) // make sure the other user does not work // completely disable and test again reloadUpdateConfig(t, srv, conf, cfgCommon) connectPass(kps[0], kps[1], kps[2], kps[3]) // make sure every account and system account (index 0) can connect // re-enable and test again reloadUpdateConfig(t, srv, conf, fmt.Sprintf(cfgFmt, pubs[2], pubs[3])) connectPass(kps[0], kps[2], kps[3]) // make sure user from accounts listed and system account (index 0) work connectFail(kps[1]) // make sure the other user does not work } func TestJWTNoSystemAccountButNatsResolver(t *testing.T) { dirSrv := t.TempDir() for _, resType := range []string{"full", "cache"} { t.Run(resType, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s resolver: { type: %s dir: '%s' }`, ojwt, resType, dirSrv))) opts := LoadConfig(conf) s, err := NewServer(opts) // Since the server cannot be stopped, since it did not start, // let's manually close the account resolver to avoid leaking go routines. opts.AccountResolver.Close() s.Shutdown() require_Error(t, err) require_Contains(t, err.Error(), "the system account needs to be specified in configuration or the operator jwt") }) } } func TestJWTAccountConnzAccessAfterClaimUpdate(t *testing.T) { skp, spub := createKey(t) newUser(t, skp) sclaim := jwt.NewAccountClaims(spub) sclaim.AddMapping("foo.bar", jwt.WeightedMapping{Subject: "foo.baz"}) sjwt := encodeClaim(t, sclaim, spub) // create two jwt, one with and one without mapping akp, apub := createKey(t) newUser(t, akp) claim := jwt.NewAccountClaims(apub) jwt1 := encodeClaim(t, claim, apub) claim.AddMapping("foo.bar", jwt.WeightedMapping{Subject: "foo.baz"}) jwt2 := encodeClaim(t, claim, apub) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: { type: full dir: '%s' } `, ojwt, spub, dirSrv))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() type zapi struct { Server *ServerInfo Data *Connz Error *ApiError } updateJWT := func(jwt string) { t.Helper() sc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, skp)) defer sc.Close() resp, err := sc.Request("$SYS.REQ.CLAIMS.UPDATE", []byte(jwt), time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cz zapi if err := json.Unmarshal(resp.Data, &cz); err != nil { t.Fatalf("Unexpected error: %v", err) } if cz.Error != nil { t.Fatalf("Unexpected error: %+v", cz.Error) } } updateJWT(jwt1) nc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, akp)) defer nc.Close() doRequest := func() { t.Helper() resp, err := nc.Request("$SYS.REQ.SERVER.PING.CONNZ", nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cz zapi if err := json.Unmarshal(resp.Data, &cz); err != nil { t.Fatalf("Unexpected error: %v", err) } if cz.Error != nil { t.Fatalf("Unexpected error: %+v", cz.Error) } } doRequest() updateJWT(jwt2) // If we accidentally wipe the system import this will fail with no responders. doRequest() // Now test updating system account. updateJWT(sjwt) // If export was wiped this would fail with timeout. doRequest() } func TestAccountWeightedMappingInSuperCluster(t *testing.T) { skp, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "SYS" sysCreds := newUser(t, skp) akp, apub := createKey(t) aUsr := createUserCreds(t, nil, akp) claim := jwt.NewAccountClaims(apub) aJwtMap := encodeClaim(t, claim, apub) // We are using the createJetStreamSuperClusterWithTemplateAndModHook() // helper, but this test is not about JetStream... tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 3, func(serverName, clusterName, storeDir, conf string) string { dirSrv := t.TempDir() return fmt.Sprintf(`%s operator: %s system_account: %s resolver: { type: full dir: '%s' } `, conf, ojwt, spub, dirSrv) }, nil) defer sc.shutdown() // Update from C2 require_Len(t, 1, updateJwt(t, sc.clusterForName("C2").randomServer().ClientURL(), sysCreds, aJwtMap, 1)) // We will connect our services in the C3 cluster. nc1 := natsConnect(t, sc.clusterForName("C3").randomServer().ClientURL(), aUsr) defer nc1.Close() nc2 := natsConnect(t, sc.clusterForName("C3").randomServer().ClientURL(), aUsr) defer nc2.Close() natsSub(t, nc1, "foo", func(m *nats.Msg) { m.Respond([]byte("foo")) }) natsSub(t, nc1, "bar.v1", func(m *nats.Msg) { m.Respond([]byte("v1")) }) natsSub(t, nc2, "bar.v2", func(m *nats.Msg) { m.Respond([]byte("v2")) }) natsFlush(t, nc1) natsFlush(t, nc2) // Now we will update the account to add weighted subject mapping claim.Mappings = map[jwt.Subject][]jwt.WeightedMapping{} // Start with foo->bar.v2 at 40%, the server will auto-add foo->foo at 60%. wm := []jwt.WeightedMapping{{Subject: "bar.v2", Weight: 40}} claim.AddMapping("foo", wm...) aJwtMap = encodeClaim(t, claim, apub) // We will update from C2 require_Len(t, 1, updateJwt(t, sc.clusterForName("C2").randomServer().ClientURL(), sysCreds, aJwtMap, 1)) time.Sleep(time.Second) // And we will publish from C1 nc := natsConnect(t, sc.clusterForName("C1").randomServer().ClientURL(), aUsr) defer nc.Close() var foo, v1, v2 int pubAndCount := func() { for i := 0; i < 1000; i++ { msg, err := nc.Request("foo", []byte("req"), 500*time.Millisecond) if err != nil { continue } switch string(msg.Data) { case "foo": foo++ case "v1": v1++ case "v2": v2++ } } } pubAndCount() if foo < 550 || foo > 650 { t.Fatalf("Expected foo to receive 60%%, got %v/1000", foo) } if v1 != 0 { t.Fatalf("Expected v1 to receive no message, got %v/1000", v1) } if v2 < 350 || v2 > 450 { t.Fatalf("Expected v2 to receive 40%%, got %v/1000", v2) } // Now send a new update with foo-> bar.v2(40) and bar.v1(60). // The auto-add of "foo" should no longer be used by the server. wm = []jwt.WeightedMapping{ {Subject: "bar.v2", Weight: 40}, {Subject: "bar.v1", Weight: 60}, } claim.AddMapping("foo", wm...) aJwtMap = encodeClaim(t, claim, apub) // We will update from C2 require_Len(t, 1, updateJwt(t, sc.clusterForName("C2").randomServer().ClientURL(), sysCreds, aJwtMap, 1)) time.Sleep(time.Second) foo, v1, v2 = 0, 0, 0 pubAndCount() if foo != 0 { t.Fatalf("Expected foo to receive no message, got %v/1000", foo) } if v1 < 550 || v1 > 650 { t.Fatalf("Expected v1 to receive 60%%, got %v/1000", v1) } if v2 < 350 || v2 > 450 { t.Fatalf("Expected v2 to receive 40%%, got %v/1000", v2) } } func TestServerOperatorModeNoAuthRequired(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) akp, apub := createKey(t) accClaim := jwt.NewAccountClaims(apub) accClaim.Name = "TEST" accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() nuc := jwt.NewUserClaims(upub) ujwt, err := nuc.Encode(akp) require_NoError(t, err) creds := genCredsFile(t, ujwt, seed) dirSrv := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-A operator: %s system_account: %s resolver: { type: full dir: '%s' interval: "200ms" limit: 4 } resolver_preload: { %s: %s %s: %s } `, ojwt, spub, dirSrv, spub, sysJwt, apub, accJwt))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(creds)) defer nc.Close() require_True(t, nc.AuthRequired()) } func TestServerOperatorModeUserInfoExpiration(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "$SYS" sysJwt, err := sysClaim.Encode(oKp) require_NoError(t, err) akp, apub := createKey(t) accClaim := jwt.NewAccountClaims(apub) accClaim.Name = "TEST" accJwt, err := accClaim.Encode(oKp) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } `, ojwt, spub, apub, accJwt, spub, sysJwt))) defer removeFile(t, conf) s, _ := RunServerWithConfig(conf) defer s.Shutdown() expires := time.Now().Add(time.Minute) creds := createUserWithLimit(t, akp, expires, func(j *jwt.UserPermissionLimits) {}) nc := natsConnect(t, s.ClientURL(), nats.UserCredentials(creds)) defer nc.Close() resp, err := nc.Request("$SYS.REQ.USER.INFO", nil, time.Second) require_NoError(t, err) now := time.Now() response := ServerAPIResponse{Data: &UserInfo{}} err = json.Unmarshal(resp.Data, &response) require_NoError(t, err) userInfo := response.Data.(*UserInfo) require_True(t, userInfo.Expires != 0) // We need to round the expiration time to the second because the server // will truncate the expiration time to the second. expiresDurRounded := expires.Sub(now).Truncate(time.Second) // Checking range to avoid flaky tests where the expiration time is // off by a couple of seconds. require_True(t, expiresDurRounded >= userInfo.Expires-2*time.Second && expiresDurRounded <= userInfo.Expires+2*time.Second) } func TestJWTAccountNATSResolverWrongCreds(t *testing.T) { require_NoLocalOrRemoteConnections := func(account string, srvs ...*Server) { t.Helper() for _, srv := range srvs { if acc, ok := srv.accounts.Load(account); ok { checkAccClientsCount(t, acc.(*Account), 0) } } } connect := func(url string, credsfile string, acc string, srvs ...*Server) { t.Helper() nc := natsConnect(t, url, nats.UserCredentials(credsfile), nats.Timeout(5*time.Second)) nc.Close() require_NoLocalOrRemoteConnections(acc, srvs...) } createAccountAndUser := func(pubKey, jwt1, jwt2, creds *string) { t.Helper() kp, _ := nkeys.CreateAccount() *pubKey, _ = kp.PublicKey() claim := jwt.NewAccountClaims(*pubKey) var err error *jwt1, err = claim.Encode(oKp) require_NoError(t, err) *jwt2, err = claim.Encode(oKp) require_NoError(t, err) ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt, err := uclaim.Encode(kp) require_NoError(t, err) *creds = genCredsFile(t, ujwt, seed) } // Create Accounts and corresponding user creds. var syspub, sysjwt, dummy1, sysCreds string createAccountAndUser(&syspub, &sysjwt, &dummy1, &sysCreds) var apub, ajwt1, ajwt2, aCreds string createAccountAndUser(&apub, &ajwt1, &ajwt2, &aCreds) var bpub, bjwt1, bjwt2, bCreds string createAccountAndUser(&bpub, &bjwt1, &bjwt2, &bCreds) // The one that is going to be missing. var cpub, cjwt1, cjwt2, cCreds string createAccountAndUser(&cpub, &cjwt1, &cjwt2, &cCreds) // Create one directory for each server dirA := t.TempDir() dirB := t.TempDir() dirC := t.TempDir() // Store accounts on servers A and B, then let C sync on its own. writeJWT(t, dirA, apub, ajwt1) writeJWT(t, dirB, bpub, bjwt1) ///////////////////////////////////////// // // // Server A: has creds from client A // // // ///////////////////////////////////////// confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-A operator: %s system_account: %s debug: true resolver: { type: full dir: '%s' allow_delete: true timeout: "1.5s" interval: "200ms" } resolver_preload: { %s: %s } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true } `, ojwt, syspub, dirA, apub, ajwt1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() require_JWTPresent(t, dirA, apub) ///////////////////////////////////////// // // // Server B: has creds from client B // // // ///////////////////////////////////////// confB := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: srv-B operator: %s system_account: %s resolver: { type: full dir: '%s' allow_delete: true timeout: "1.5s" interval: "200ms" } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } `, ojwt, syspub, dirB, sA.opts.Cluster.Port))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() ///////////////////////////////////////// // // // Server C: has no creds // // // ///////////////////////////////////////// fmtC := ` listen: 127.0.0.1:-1 server_name: srv-C operator: %s system_account: %s resolver: { type: full dir: '%s' allow_delete: true timeout: "1.5s" interval: "200ms" } cluster { name: clust listen: 127.0.0.1:-1 no_advertise: true routes [ nats-route://127.0.0.1:%d ] } ` confClongTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, sA.opts.Cluster.Port))) sC, _ := RunServerWithConfig(confClongTTL) // use long ttl to assure it is not kicking defer sC.Shutdown() // startup cluster checkClusterFormed(t, sA, sB, sC) time.Sleep(1 * time.Second) // wait for the protocol to converge // // Check all accounts require_JWTPresent(t, dirA, apub) // was already present on startup require_JWTPresent(t, dirB, apub) // was copied from server A require_JWTPresent(t, dirA, bpub) // was copied from server B require_JWTPresent(t, dirB, bpub) // was already present on startup // There should be no state about the missing account. require_JWTAbsent(t, dirA, cpub) require_JWTAbsent(t, dirB, cpub) require_JWTAbsent(t, dirC, cpub) // system account client can connect to every server connect(sA.ClientURL(), sysCreds, "") connect(sB.ClientURL(), sysCreds, "") connect(sC.ClientURL(), sysCreds, "") // A and B clients can connect to any server. connect(sA.ClientURL(), aCreds, "") connect(sB.ClientURL(), aCreds, "") connect(sC.ClientURL(), aCreds, "") connect(sA.ClientURL(), bCreds, "") connect(sB.ClientURL(), bCreds, "") connect(sC.ClientURL(), bCreds, "") // Check that trying to connect with bad credentials should not hang until the fetch timeout // and instead return a faster response when an account is not found. _, err := nats.Connect(sC.ClientURL(), nats.UserCredentials(cCreds), nats.Timeout(500*time.Second)) if err != nil && !errors.Is(err, nats.ErrAuthorization) { t.Fatalf("Expected auth error: %v", err) } } // Issue 5480: https://github.com/nats-io/nats-server/issues/5480 func TestJWTImportsOnServerRestartAndClientsReconnect(t *testing.T) { type namedCreds struct { name string creds nats.Option } preload := make(map[string]string) users := make(map[string]*namedCreds) // sys account _, sysAcc, sysAccClaim := NewJwtAccountClaim("sys") sysAccJWT, err := sysAccClaim.Encode(oKp) require_NoError(t, err) preload[sysAcc] = sysAccJWT // main account, other accounts will import from this. mainAccKP, mainAcc, mainAccClaim := NewJwtAccountClaim("main") mainAccClaim.Exports.Add(&jwt.Export{ Type: jwt.Stream, Subject: "city.>", }) // main account user mainUserClaim := jwt.NewUserClaims("publisher") mainUserClaim.Permissions = jwt.Permissions{ Pub: jwt.Permission{ Allow: []string{"city.>"}, }, } mainCreds := createUserCredsEx(t, mainUserClaim, mainAccKP) // The main account will be importing from all other accounts. maxAccounts := 100 for i := 0; i < maxAccounts; i++ { name := fmt.Sprintf("secondary-%d", i) accKP, acc, accClaim := NewJwtAccountClaim(name) accClaim.Exports.Add(&jwt.Export{ Type: jwt.Stream, Subject: "internal.*", }) accClaim.Imports.Add(&jwt.Import{ Type: jwt.Stream, Subject: jwt.Subject(fmt.Sprintf("city.%d-1.*", i)), Account: mainAcc, }) // main account imports from the secondary accounts mainAccClaim.Imports.Add(&jwt.Import{ Type: jwt.Stream, Subject: jwt.Subject(fmt.Sprintf("internal.%d", i)), Account: acc, }) accJWT, err := accClaim.Encode(oKp) require_NoError(t, err) preload[acc] = accJWT userClaim := jwt.NewUserClaims("subscriber") userClaim.Permissions = jwt.Permissions{ Sub: jwt.Permission{ Allow: []string{"city.>", "internal.*"}, }, Pub: jwt.Permission{ Allow: []string{"internal.*"}, }, } userCreds := createUserCredsEx(t, userClaim, accKP) users[acc] = &namedCreds{name, userCreds} } mainAccJWT, err := mainAccClaim.Encode(oKp) require_NoError(t, err) preload[mainAcc] = mainAccJWT // Start the server with the preload. resolverPreload, err := json.Marshal(preload) require_NoError(t, err) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:4747 http: 127.0.0.1:8222 operator: %s system_account: %s resolver: MEM resolver_preload: %s `, ojwt, sysAcc, string(resolverPreload)))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Have a connection ready for each one of the accounts. type namedSub struct { name string sub *nats.Subscription } subs := make(map[string]*namedSub) for acc, user := range users { nc := natsConnect(t, s.ClientURL(), user.creds, // Make the clients attempt to reconnect too fast, // changing this to be above ~200ms mitigates the issue. nats.ReconnectWait(15*time.Millisecond), nats.Name(user.name), nats.MaxReconnects(-1), ) defer nc.Close() sub, err := nc.SubscribeSync("city.>") require_NoError(t, err) subs[acc] = &namedSub{user.name, sub} } nc := natsConnect(t, s.ClientURL(), mainCreds, nats.ReconnectWait(15*time.Millisecond), nats.MaxReconnects(-1)) defer nc.Close() send := func(t *testing.T) { t.Helper() for i := 0; i < maxAccounts; i++ { nc.Publish(fmt.Sprintf("city.%d-1.A4BDB048-69DC-4F10-916C-2B998249DC11", i), []byte(fmt.Sprintf("test:%d", i))) } nc.Flush() } ctx, done := context.WithCancel(context.Background()) defer done() go func() { for range time.NewTicker(200 * time.Millisecond).C { select { case <-ctx.Done(): return default: } send(t) } }() receive := func(t *testing.T) { t.Helper() received := 0 for _, nsub := range subs { // Drain first any pending messages. pendingMsgs, _, _ := nsub.sub.Pending() for i, _ := 0, 0; i < pendingMsgs; i++ { nsub.sub.NextMsg(500 * time.Millisecond) } _, err = nsub.sub.NextMsg(500 * time.Millisecond) if err != nil { t.Logf("WRN: Failed to receive message on account %q: %v", nsub.name, err) } else { received++ } } if received < (maxAccounts / 2) { t.Fatalf("Too many missed messages after restart. Received %d", received) } } receive(t) time.Sleep(1 * time.Second) restart := func(t *testing.T) *Server { t.Helper() s.Shutdown() s.WaitForShutdown() s, _ = RunServerWithConfig(conf) hctx, hcancel := context.WithTimeout(context.Background(), 5*time.Second) defer hcancel() for range time.NewTicker(2 * time.Second).C { select { case <-hctx.Done(): t.Logf("WRN: Timed out waiting for healthz from %s", s) default: } status := s.healthz(nil) if status.StatusCode == 200 { return s } } return nil } // Takes a few restarts for issue to show up. for i := 0; i < 5; i++ { s := restart(t) defer s.Shutdown() time.Sleep(2 * time.Second) receive(t) } } nats-server-2.10.27/server/leafnode.go000066400000000000000000003001161477524627100175660ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "crypto/tls" "encoding/base64" "encoding/json" "fmt" "math/rand" "net" "net/http" "net/url" "os" "path" "reflect" "regexp" "runtime" "strconv" "strings" "sync" "sync/atomic" "time" "unicode" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) const ( // Warning when user configures leafnode TLS insecure leafnodeTLSInsecureWarning = "TLS certificate chain and hostname of solicited leafnodes will not be verified. DO NOT USE IN PRODUCTION!" // When a loop is detected, delay the reconnect of solicited connection. leafNodeReconnectDelayAfterLoopDetected = 30 * time.Second // When a server receives a message causing a permission violation, the // connection is closed and it won't attempt to reconnect for that long. leafNodeReconnectAfterPermViolation = 30 * time.Second // When we have the same cluster name as the hub. leafNodeReconnectDelayAfterClusterNameSame = 30 * time.Second // Prefix for loop detection subject leafNodeLoopDetectionSubjectPrefix = "$LDS." // Path added to URL to indicate to WS server that the connection is a // LEAF connection as opposed to a CLIENT. leafNodeWSPath = "/leafnode" // This is the time the server will wait, when receiving a CONNECT, // before closing the connection if the required minimum version is not met. leafNodeWaitBeforeClose = 5 * time.Second ) type leaf struct { // We have any auth stuff here for solicited connections. remote *leafNodeCfg // isSpoke tells us what role we are playing. // Used when we receive a connection but otherside tells us they are a hub. isSpoke bool // remoteCluster is when we are a hub but the spoke leafnode is part of a cluster. remoteCluster string // remoteServer holds onto the remove server's name or ID. remoteServer string // domain name of remote server remoteDomain string // account name of remote server remoteAccName string // Used to suppress sub and unsub interest. Same as routes but our audience // here is tied to this leaf node. This will hold all subscriptions except this // leaf nodes. This represents all the interest we want to send to the other side. smap map[string]int32 // This map will contain all the subscriptions that have been added to the smap // during initLeafNodeSmapAndSendSubs. It is short lived and is there to avoid // race between processing of a sub where sub is added to account sublist but // updateSmap has not be called on that "thread", while in the LN readloop, // when processing CONNECT, initLeafNodeSmapAndSendSubs is invoked and add // this subscription to smap. When processing of the sub then calls updateSmap, // we would add it a second time in the smap causing later unsub to suppress the LS-. tsub map[*subscription]struct{} tsubt *time.Timer // Selected compression mode, which may be different from the server configured mode. compression string // This is for GW map replies. gwSub *subscription } // Used for remote (solicited) leafnodes. type leafNodeCfg struct { sync.RWMutex *RemoteLeafOpts urls []*url.URL curURL *url.URL tlsName string username string password string perms *Permissions connDelay time.Duration // Delay before a connect, could be used while detecting loop condition, etc.. } // Check to see if this is a solicited leafnode. We do special processing for solicited. func (c *client) isSolicitedLeafNode() bool { return c.kind == LEAF && c.leaf.remote != nil } // Returns true if this is a solicited leafnode and is not configured to be treated as a hub or a receiving // connection leafnode where the otherside has declared itself to be the hub. func (c *client) isSpokeLeafNode() bool { return c.kind == LEAF && c.leaf.isSpoke } func (c *client) isHubLeafNode() bool { return c.kind == LEAF && !c.leaf.isSpoke } // This will spin up go routines to solicit the remote leaf node connections. func (s *Server) solicitLeafNodeRemotes(remotes []*RemoteLeafOpts) { sysAccName := _EMPTY_ sAcc := s.SystemAccount() if sAcc != nil { sysAccName = sAcc.Name } addRemote := func(r *RemoteLeafOpts, isSysAccRemote bool) *leafNodeCfg { s.mu.Lock() remote := newLeafNodeCfg(r) creds := remote.Credentials accName := remote.LocalAccount s.leafRemoteCfgs = append(s.leafRemoteCfgs, remote) // Print notice if if isSysAccRemote { if len(remote.DenyExports) > 0 { s.Noticef("Remote for System Account uses restricted export permissions") } if len(remote.DenyImports) > 0 { s.Noticef("Remote for System Account uses restricted import permissions") } } s.mu.Unlock() if creds != _EMPTY_ { contents, err := os.ReadFile(creds) defer wipeSlice(contents) if err != nil { s.Errorf("Error reading LeafNode Remote Credentials file %q: %v", creds, err) } else if items := credsRe.FindAllSubmatch(contents, -1); len(items) < 2 { s.Errorf("LeafNode Remote Credentials file %q malformed", creds) } else if _, err := nkeys.FromSeed(items[1][1]); err != nil { s.Errorf("LeafNode Remote Credentials file %q has malformed seed", creds) } else if uc, err := jwt.DecodeUserClaims(string(items[0][1])); err != nil { s.Errorf("LeafNode Remote Credentials file %q has malformed user jwt", creds) } else if isSysAccRemote { if !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil { s.Noticef("LeafNode Remote for System Account uses credentials file %q with restricted permissions", creds) } } else { if !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil { s.Noticef("LeafNode Remote for Account %s uses credentials file %q with restricted permissions", accName, creds) } } } return remote } for _, r := range remotes { remote := addRemote(r, r.LocalAccount == sysAccName) s.startGoRoutine(func() { s.connectToRemoteLeafNode(remote, true) }) } } func (s *Server) remoteLeafNodeStillValid(remote *leafNodeCfg) bool { for _, ri := range s.getOpts().LeafNode.Remotes { // FIXME(dlc) - What about auth changes? if reflect.DeepEqual(ri.URLs, remote.URLs) { return true } } return false } // Ensure that leafnode is properly configured. func validateLeafNode(o *Options) error { if err := validateLeafNodeAuthOptions(o); err != nil { return err } // Users can bind to any local account, if its empty we will assume the $G account. for _, r := range o.LeafNode.Remotes { if r.LocalAccount == _EMPTY_ { r.LocalAccount = globalAccountName } } // In local config mode, check that leafnode configuration refers to accounts that exist. if len(o.TrustedOperators) == 0 { accNames := map[string]struct{}{} for _, a := range o.Accounts { accNames[a.Name] = struct{}{} } // global account is always created accNames[DEFAULT_GLOBAL_ACCOUNT] = struct{}{} // in the context of leaf nodes, empty account means global account accNames[_EMPTY_] = struct{}{} // system account either exists or, if not disabled, will be created if o.SystemAccount == _EMPTY_ && !o.NoSystemAccount { accNames[DEFAULT_SYSTEM_ACCOUNT] = struct{}{} } checkAccountExists := func(accName string, cfgType string) error { if _, ok := accNames[accName]; !ok { return fmt.Errorf("cannot find local account %q specified in leafnode %s", accName, cfgType) } return nil } if err := checkAccountExists(o.LeafNode.Account, "authorization"); err != nil { return err } for _, lu := range o.LeafNode.Users { if lu.Account == nil { // means global account continue } if err := checkAccountExists(lu.Account.Name, "authorization"); err != nil { return err } } for _, r := range o.LeafNode.Remotes { if err := checkAccountExists(r.LocalAccount, "remote"); err != nil { return err } } } else { if len(o.LeafNode.Users) != 0 { return fmt.Errorf("operator mode does not allow specifying users in leafnode config") } for _, r := range o.LeafNode.Remotes { if !nkeys.IsValidPublicAccountKey(r.LocalAccount) { return fmt.Errorf( "operator mode requires account nkeys in remotes. " + "Please add an `account` key to each remote in your `leafnodes` section, to assign it to an account. " + "Each account value should be a 56 character public key, starting with the letter 'A'") } } if o.LeafNode.Port != 0 && o.LeafNode.Account != "" && !nkeys.IsValidPublicAccountKey(o.LeafNode.Account) { return fmt.Errorf("operator mode and non account nkeys are incompatible") } } // Validate compression settings if o.LeafNode.Compression.Mode != _EMPTY_ { if err := validateAndNormalizeCompressionOption(&o.LeafNode.Compression, CompressionS2Auto); err != nil { return err } } // If a remote has a websocket scheme, all need to have it. for _, rcfg := range o.LeafNode.Remotes { if len(rcfg.URLs) >= 2 { firstIsWS, ok := isWSURL(rcfg.URLs[0]), true for i := 1; i < len(rcfg.URLs); i++ { u := rcfg.URLs[i] if isWS := isWSURL(u); isWS && !firstIsWS || !isWS && firstIsWS { ok = false break } } if !ok { return fmt.Errorf("remote leaf node configuration cannot have a mix of websocket and non-websocket urls: %q", redactURLList(rcfg.URLs)) } } // Validate compression settings if rcfg.Compression.Mode != _EMPTY_ { if err := validateAndNormalizeCompressionOption(&rcfg.Compression, CompressionS2Auto); err != nil { return err } } } if o.LeafNode.Port == 0 { return nil } // If MinVersion is defined, check that it is valid. if mv := o.LeafNode.MinVersion; mv != _EMPTY_ { if err := checkLeafMinVersionConfig(mv); err != nil { return err } } // The checks below will be done only when detecting that we are configured // with gateways. So if an option validation needs to be done regardless, // it MUST be done before this point! if o.Gateway.Name == _EMPTY_ && o.Gateway.Port == 0 { return nil } // If we are here we have both leaf nodes and gateways defined, make sure there // is a system account defined. if o.SystemAccount == _EMPTY_ { return fmt.Errorf("leaf nodes and gateways (both being defined) require a system account to also be configured") } if err := validatePinnedCerts(o.LeafNode.TLSPinnedCerts); err != nil { return fmt.Errorf("leafnode: %v", err) } return nil } func checkLeafMinVersionConfig(mv string) error { if ok, err := versionAtLeastCheckError(mv, 2, 8, 0); !ok || err != nil { if err != nil { return fmt.Errorf("invalid leafnode's minimum version: %v", err) } else { return fmt.Errorf("the minimum version should be at least 2.8.0") } } return nil } // Used to validate user names in LeafNode configuration. // - rejects mix of single and multiple users. // - rejects duplicate user names. func validateLeafNodeAuthOptions(o *Options) error { if len(o.LeafNode.Users) == 0 { return nil } if o.LeafNode.Username != _EMPTY_ { return fmt.Errorf("can not have a single user/pass and a users array") } if o.LeafNode.Nkey != _EMPTY_ { return fmt.Errorf("can not have a single nkey and a users array") } users := map[string]struct{}{} for _, u := range o.LeafNode.Users { if _, exists := users[u.Username]; exists { return fmt.Errorf("duplicate user %q detected in leafnode authorization", u.Username) } users[u.Username] = struct{}{} } return nil } // Update remote LeafNode TLS configurations after a config reload. func (s *Server) updateRemoteLeafNodesTLSConfig(opts *Options) { max := len(opts.LeafNode.Remotes) if max == 0 { return } s.mu.RLock() defer s.mu.RUnlock() // Changes in the list of remote leaf nodes is not supported. // However, make sure that we don't go over the arrays. if len(s.leafRemoteCfgs) < max { max = len(s.leafRemoteCfgs) } for i := 0; i < max; i++ { ro := opts.LeafNode.Remotes[i] cfg := s.leafRemoteCfgs[i] if ro.TLSConfig != nil { cfg.Lock() cfg.TLSConfig = ro.TLSConfig.Clone() cfg.TLSHandshakeFirst = ro.TLSHandshakeFirst cfg.Unlock() } } } func (s *Server) reConnectToRemoteLeafNode(remote *leafNodeCfg) { delay := s.getOpts().LeafNode.ReconnectInterval select { case <-time.After(delay): case <-s.quitCh: s.grWG.Done() return } s.connectToRemoteLeafNode(remote, false) } // Creates a leafNodeCfg object that wraps the RemoteLeafOpts. func newLeafNodeCfg(remote *RemoteLeafOpts) *leafNodeCfg { cfg := &leafNodeCfg{ RemoteLeafOpts: remote, urls: make([]*url.URL, 0, len(remote.URLs)), } if len(remote.DenyExports) > 0 || len(remote.DenyImports) > 0 { perms := &Permissions{} if len(remote.DenyExports) > 0 { perms.Publish = &SubjectPermission{Deny: remote.DenyExports} } if len(remote.DenyImports) > 0 { perms.Subscribe = &SubjectPermission{Deny: remote.DenyImports} } cfg.perms = perms } // Start with the one that is configured. We will add to this // array when receiving async leafnode INFOs. cfg.urls = append(cfg.urls, cfg.URLs...) // If allowed to randomize, do it on our copy of URLs if !remote.NoRandomize { rand.Shuffle(len(cfg.urls), func(i, j int) { cfg.urls[i], cfg.urls[j] = cfg.urls[j], cfg.urls[i] }) } // If we are TLS make sure we save off a proper servername if possible. // Do same for user/password since we may need them to connect to // a bare URL that we get from INFO protocol. for _, u := range cfg.urls { cfg.saveTLSHostname(u) cfg.saveUserPassword(u) // If the url(s) have the "wss://" scheme, and we don't have a TLS // config, mark that we should be using TLS anyway. if !cfg.TLS && isWSSURL(u) { cfg.TLS = true } } return cfg } // Will pick an URL from the list of available URLs. func (cfg *leafNodeCfg) pickNextURL() *url.URL { cfg.Lock() defer cfg.Unlock() // If the current URL is the first in the list and we have more than // one URL, then move that one to end of the list. if cfg.curURL != nil && len(cfg.urls) > 1 && urlsAreEqual(cfg.curURL, cfg.urls[0]) { first := cfg.urls[0] copy(cfg.urls, cfg.urls[1:]) cfg.urls[len(cfg.urls)-1] = first } cfg.curURL = cfg.urls[0] return cfg.curURL } // Returns the current URL func (cfg *leafNodeCfg) getCurrentURL() *url.URL { cfg.RLock() defer cfg.RUnlock() return cfg.curURL } // Returns how long the server should wait before attempting // to solicit a remote leafnode connection. func (cfg *leafNodeCfg) getConnectDelay() time.Duration { cfg.RLock() delay := cfg.connDelay cfg.RUnlock() return delay } // Sets the connect delay. func (cfg *leafNodeCfg) setConnectDelay(delay time.Duration) { cfg.Lock() cfg.connDelay = delay cfg.Unlock() } // Ensure that non-exported options (used in tests) have // been properly set. func (s *Server) setLeafNodeNonExportedOptions() { opts := s.getOpts() s.leafNodeOpts.dialTimeout = opts.LeafNode.dialTimeout if s.leafNodeOpts.dialTimeout == 0 { // Use same timeouts as routes for now. s.leafNodeOpts.dialTimeout = DEFAULT_ROUTE_DIAL } s.leafNodeOpts.resolver = opts.LeafNode.resolver if s.leafNodeOpts.resolver == nil { s.leafNodeOpts.resolver = net.DefaultResolver } } const sharedSysAccDelay = 250 * time.Millisecond func (s *Server) connectToRemoteLeafNode(remote *leafNodeCfg, firstConnect bool) { defer s.grWG.Done() if remote == nil || len(remote.URLs) == 0 { s.Debugf("Empty remote leafnode definition, nothing to connect") return } opts := s.getOpts() reconnectDelay := opts.LeafNode.ReconnectInterval s.mu.Lock() dialTimeout := s.leafNodeOpts.dialTimeout resolver := s.leafNodeOpts.resolver var isSysAcc bool if s.eventsEnabled() { isSysAcc = remote.LocalAccount == s.sys.account.Name } s.mu.Unlock() // If we are sharing a system account and we are not standalone delay to gather some info prior. if firstConnect && isSysAcc && !s.standAloneMode() { s.Debugf("Will delay first leafnode connect to shared system account due to clustering") remote.setConnectDelay(sharedSysAccDelay) } if connDelay := remote.getConnectDelay(); connDelay > 0 { select { case <-time.After(connDelay): case <-s.quitCh: return } remote.setConnectDelay(0) } var conn net.Conn const connErrFmt = "Error trying to connect as leafnode to remote server %q (attempt %v): %v" attempts := 0 for s.isRunning() && s.remoteLeafNodeStillValid(remote) { rURL := remote.pickNextURL() url, err := s.getRandomIP(resolver, rURL.Host, nil) if err == nil { var ipStr string if url != rURL.Host { ipStr = fmt.Sprintf(" (%s)", url) } // Some test may want to disable remotes from connecting if s.isLeafConnectDisabled() { s.Debugf("Will not attempt to connect to remote server on %q%s, leafnodes currently disabled", rURL.Host, ipStr) err = ErrLeafNodeDisabled } else { s.Debugf("Trying to connect as leafnode to remote server on %q%s", rURL.Host, ipStr) conn, err = natsDialTimeout("tcp", url, dialTimeout) } } if err != nil { jitter := time.Duration(rand.Int63n(int64(reconnectDelay))) delay := reconnectDelay + jitter attempts++ if s.shouldReportConnectErr(firstConnect, attempts) { s.Errorf(connErrFmt, rURL.Host, attempts, err) } else { s.Debugf(connErrFmt, rURL.Host, attempts, err) } select { case <-s.quitCh: return case <-time.After(delay): // Check if we should migrate any JetStream assets while this remote is down. s.checkJetStreamMigrate(remote) continue } } if !s.remoteLeafNodeStillValid(remote) { conn.Close() return } // We have a connection here to a remote server. // Go ahead and create our leaf node and return. s.createLeafNode(conn, rURL, remote, nil) // Clear any observer states if we had them. s.clearObserverState(remote) return } } // This will clear any observer state such that stream or consumer assets on this server can become leaders again. func (s *Server) clearObserverState(remote *leafNodeCfg) { s.mu.RLock() accName := remote.LocalAccount s.mu.RUnlock() acc, err := s.LookupAccount(accName) if err != nil { s.Warnf("Error looking up account [%s] checking for JetStream clear observer state on a leafnode", accName) return } acc.jscmMu.Lock() defer acc.jscmMu.Unlock() // Walk all streams looking for any clustered stream, skip otherwise. for _, mset := range acc.streams() { node := mset.raftNode() if node == nil { // Not R>1 continue } // Check consumers for _, o := range mset.getConsumers() { if n := o.raftNode(); n != nil { // Ensure we can become a leader again. n.SetObserver(false) } } // Ensure we can not become a leader again. node.SetObserver(false) } } // Check to see if we should migrate any assets from this account. func (s *Server) checkJetStreamMigrate(remote *leafNodeCfg) { s.mu.RLock() accName, shouldMigrate := remote.LocalAccount, remote.JetStreamClusterMigrate s.mu.RUnlock() if !shouldMigrate { return } acc, err := s.LookupAccount(accName) if err != nil { s.Warnf("Error looking up account [%s] checking for JetStream migration on a leafnode", accName) return } acc.jscmMu.Lock() defer acc.jscmMu.Unlock() // Walk all streams looking for any clustered stream, skip otherwise. // If we are the leader force stepdown. for _, mset := range acc.streams() { node := mset.raftNode() if node == nil { // Not R>1 continue } // Collect any consumers for _, o := range mset.getConsumers() { if n := o.raftNode(); n != nil { if n.Leader() { n.StepDown() } // Ensure we can not become a leader while in this state. n.SetObserver(true) } } // Stepdown if this stream was leader. if node.Leader() { node.StepDown() } // Ensure we can not become a leader while in this state. node.SetObserver(true) } } // Helper for checking. func (s *Server) isLeafConnectDisabled() bool { s.mu.RLock() defer s.mu.RUnlock() return s.leafDisableConnect } // Save off the tlsName for when we use TLS and mix hostnames and IPs. IPs usually // come from the server we connect to. // // We used to save the name only if there was a TLSConfig or scheme equal to "tls". // However, this was causing failures for users that did not set the scheme (and // their remote connections did not have a tls{} block). // We now save the host name regardless in case the remote returns an INFO indicating // that TLS is required. func (cfg *leafNodeCfg) saveTLSHostname(u *url.URL) { if cfg.tlsName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil { cfg.tlsName = u.Hostname() } } // Save off the username/password for when we connect using a bare URL // that we get from the INFO protocol. func (cfg *leafNodeCfg) saveUserPassword(u *url.URL) { if cfg.username == _EMPTY_ && u.User != nil { cfg.username = u.User.Username() cfg.password, _ = u.User.Password() } } // This starts the leafnode accept loop in a go routine, unless it // is detected that the server has already been shutdown. func (s *Server) startLeafNodeAcceptLoop() { // Snapshot server options. opts := s.getOpts() port := opts.LeafNode.Port if port == -1 { port = 0 } if s.isShuttingDown() { return } s.mu.Lock() hp := net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(port)) l, e := natsListen("tcp", hp) s.leafNodeListenerErr = e if e != nil { s.mu.Unlock() s.Fatalf("Error listening on leafnode port: %d - %v", opts.LeafNode.Port, e) return } s.Noticef("Listening for leafnode connections on %s", net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) tlsRequired := opts.LeafNode.TLSConfig != nil tlsVerify := tlsRequired && opts.LeafNode.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert // Do not set compression in this Info object, it would possibly cause // issues when sending asynchronous INFO to the remote. info := Info{ ID: s.info.ID, Name: s.info.Name, Version: s.info.Version, GitCommit: gitCommit, GoVersion: runtime.Version(), AuthRequired: true, TLSRequired: tlsRequired, TLSVerify: tlsVerify, MaxPayload: s.info.MaxPayload, // TODO(dlc) - Allow override? Headers: s.supportsHeaders(), JetStream: opts.JetStream, Domain: opts.JetStreamDomain, Proto: 1, // Fixed for now. InfoOnConnect: true, } // If we have selected a random port... if port == 0 { // Write resolved port back to options. opts.LeafNode.Port = l.Addr().(*net.TCPAddr).Port } s.leafNodeInfo = info // Possibly override Host/Port and set IP based on Cluster.Advertise if err := s.setLeafNodeInfoHostPortAndIP(); err != nil { s.Fatalf("Error setting leafnode INFO with LeafNode.Advertise value of %s, err=%v", opts.LeafNode.Advertise, err) l.Close() s.mu.Unlock() return } s.leafURLsMap[s.leafNodeInfo.IP]++ s.generateLeafNodeInfoJSON() // Setup state that can enable shutdown s.leafNodeListener = l // As of now, a server that does not have remotes configured would // never solicit a connection, so we should not have to warn if // InsecureSkipVerify is set in main LeafNodes config (since // this TLS setting matters only when soliciting a connection). // Still, warn if insecure is set in any of LeafNode block. // We need to check remotes, even if tls is not required on accept. warn := tlsRequired && opts.LeafNode.TLSConfig.InsecureSkipVerify if !warn { for _, r := range opts.LeafNode.Remotes { if r.TLSConfig != nil && r.TLSConfig.InsecureSkipVerify { warn = true break } } } if warn { s.Warnf(leafnodeTLSInsecureWarning) } go s.acceptConnections(l, "Leafnode", func(conn net.Conn) { s.createLeafNode(conn, nil, nil, nil) }, nil) s.mu.Unlock() } // RegEx to match a creds file with user JWT and Seed. var credsRe = regexp.MustCompile(`\s*(?:(?:[-]{3,}.*[-]{3,}\r?\n)([\w\-.=]+)(?:\r?\n[-]{3,}.*[-]{3,}(\r?\n|\z)))`) // clusterName is provided as argument to avoid lock ordering issues with the locked client c // Lock should be held entering here. func (c *client) sendLeafConnect(clusterName string, headers bool) error { // We support basic user/pass and operator based user JWT with signatures. cinfo := leafConnectInfo{ Version: VERSION, ID: c.srv.info.ID, Domain: c.srv.info.Domain, Name: c.srv.info.Name, Hub: c.leaf.remote.Hub, Cluster: clusterName, Headers: headers, JetStream: c.acc.jetStreamConfigured(), DenyPub: c.leaf.remote.DenyImports, Compression: c.leaf.compression, RemoteAccount: c.acc.GetName(), } // If a signature callback is specified, this takes precedence over anything else. if cb := c.leaf.remote.SignatureCB; cb != nil { nonce := c.nonce c.mu.Unlock() jwt, sigraw, err := cb(nonce) c.mu.Lock() if err == nil && c.isClosed() { err = ErrConnectionClosed } if err != nil { c.Errorf("Error signing the nonce: %v", err) return err } sig := base64.RawURLEncoding.EncodeToString(sigraw) cinfo.JWT, cinfo.Sig = jwt, sig } else if creds := c.leaf.remote.Credentials; creds != _EMPTY_ { // Check for credentials first, that will take precedence.. c.Debugf("Authenticating with credentials file %q", c.leaf.remote.Credentials) contents, err := os.ReadFile(creds) if err != nil { c.Errorf("%v", err) return err } defer wipeSlice(contents) items := credsRe.FindAllSubmatch(contents, -1) if len(items) < 2 { c.Errorf("Credentials file malformed") return err } // First result should be the user JWT. // We copy here so that the file containing the seed will be wiped appropriately. raw := items[0][1] tmp := make([]byte, len(raw)) copy(tmp, raw) // Seed is second item. kp, err := nkeys.FromSeed(items[1][1]) if err != nil { c.Errorf("Credentials file has malformed seed") return err } // Wipe our key on exit. defer kp.Wipe() sigraw, _ := kp.Sign(c.nonce) sig := base64.RawURLEncoding.EncodeToString(sigraw) cinfo.JWT = bytesToString(tmp) cinfo.Sig = sig } else if nkey := c.leaf.remote.Nkey; nkey != _EMPTY_ { kp, err := nkeys.FromSeed([]byte(nkey)) if err != nil { c.Errorf("Remote nkey has malformed seed") return err } // Wipe our key on exit. defer kp.Wipe() sigraw, _ := kp.Sign(c.nonce) sig := base64.RawURLEncoding.EncodeToString(sigraw) pkey, _ := kp.PublicKey() cinfo.Nkey = pkey cinfo.Sig = sig } // In addition, and this is to allow auth callout, set user/password or // token if applicable. if userInfo := c.leaf.remote.curURL.User; userInfo != nil { // For backward compatibility, if only username is provided, set both // Token and User, not just Token. cinfo.User = userInfo.Username() var ok bool cinfo.Pass, ok = userInfo.Password() if !ok { cinfo.Token = cinfo.User } } else if c.leaf.remote.username != _EMPTY_ { cinfo.User = c.leaf.remote.username cinfo.Pass = c.leaf.remote.password } b, err := json.Marshal(cinfo) if err != nil { c.Errorf("Error marshaling CONNECT to remote leafnode: %v\n", err) return err } // Although this call is made before the writeLoop is created, // we don't really need to send in place. The protocol will be // sent out by the writeLoop. c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) return nil } // Makes a deep copy of the LeafNode Info structure. // The server lock is held on entry. func (s *Server) copyLeafNodeInfo() *Info { clone := s.leafNodeInfo // Copy the array of urls. if len(s.leafNodeInfo.LeafNodeURLs) > 0 { clone.LeafNodeURLs = append([]string(nil), s.leafNodeInfo.LeafNodeURLs...) } return &clone } // Adds a LeafNode URL that we get when a route connects to the Info structure. // Regenerates the JSON byte array so that it can be sent to LeafNode connections. // Returns a boolean indicating if the URL was added or not. // Server lock is held on entry func (s *Server) addLeafNodeURL(urlStr string) bool { if s.leafURLsMap.addUrl(urlStr) { s.generateLeafNodeInfoJSON() return true } return false } // Removes a LeafNode URL of the route that is disconnecting from the Info structure. // Regenerates the JSON byte array so that it can be sent to LeafNode connections. // Returns a boolean indicating if the URL was removed or not. // Server lock is held on entry. func (s *Server) removeLeafNodeURL(urlStr string) bool { // Don't need to do this if we are removing the route connection because // we are shuting down... if s.isShuttingDown() { return false } if s.leafURLsMap.removeUrl(urlStr) { s.generateLeafNodeInfoJSON() return true } return false } // Server lock is held on entry func (s *Server) generateLeafNodeInfoJSON() { s.leafNodeInfo.Cluster = s.cachedClusterName() s.leafNodeInfo.LeafNodeURLs = s.leafURLsMap.getAsStringSlice() s.leafNodeInfo.WSConnectURLs = s.websocket.connectURLsMap.getAsStringSlice() s.leafNodeInfoJSON = generateInfoJSON(&s.leafNodeInfo) } // Sends an async INFO protocol so that the connected servers can update // their list of LeafNode urls. func (s *Server) sendAsyncLeafNodeInfo() { for _, c := range s.leafs { c.mu.Lock() c.enqueueProto(s.leafNodeInfoJSON) c.mu.Unlock() } } // Called when an inbound leafnode connection is accepted or we create one for a solicited leafnode. func (s *Server) createLeafNode(conn net.Conn, rURL *url.URL, remote *leafNodeCfg, ws *websocket) *client { // Snapshot server options. opts := s.getOpts() maxPay := int32(opts.MaxPayload) maxSubs := int32(opts.MaxSubs) // For system, maxSubs of 0 means unlimited, so re-adjust here. if maxSubs == 0 { maxSubs = -1 } now := time.Now().UTC() c := &client{srv: s, nc: conn, kind: LEAF, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now} // Do not update the smap here, we need to do it in initLeafNodeSmapAndSendSubs c.leaf = &leaf{} // For accepted LN connections, ws will be != nil if it was accepted // through the Websocket port. c.ws = ws // For remote, check if the scheme starts with "ws", if so, we will initiate // a remote Leaf Node connection as a websocket connection. if remote != nil && rURL != nil && isWSURL(rURL) { remote.RLock() c.ws = &websocket{compress: remote.Websocket.Compression, maskwrite: !remote.Websocket.NoMasking} remote.RUnlock() } // Determines if we are soliciting the connection or not. var solicited bool var acc *Account var remoteSuffix string if remote != nil { // For now, if lookup fails, we will constantly try // to recreate this LN connection. lacc := remote.LocalAccount var err error acc, err = s.LookupAccount(lacc) if err != nil { // An account not existing is something that can happen with nats/http account resolver and the account // has not yet been pushed, or the request failed for other reasons. // remote needs to be set or retry won't happen c.leaf.remote = remote c.closeConnection(MissingAccount) s.Errorf("Unable to lookup account %s for solicited leafnode connection: %v", lacc, err) return nil } remoteSuffix = fmt.Sprintf(" for account: %s", acc.traceLabel()) } c.mu.Lock() c.initClient() c.Noticef("Leafnode connection created%s %s", remoteSuffix, c.opts.Name) var tlsFirst bool var infoTimeout time.Duration if remote != nil { solicited = true remote.Lock() c.leaf.remote = remote c.setPermissions(remote.perms) if !c.leaf.remote.Hub { c.leaf.isSpoke = true } tlsFirst = remote.TLSHandshakeFirst infoTimeout = remote.FirstInfoTimeout remote.Unlock() c.acc = acc } else { c.flags.set(expectConnect) if ws != nil { c.Debugf("Leafnode compression=%v", c.ws.compress) } } c.mu.Unlock() var nonce [nonceLen]byte var info *Info // Grab this before the client lock below. if !solicited { // Grab server variables s.mu.Lock() info = s.copyLeafNodeInfo() // For tests that want to simulate old servers, do not set the compression // on the INFO protocol if configured with CompressionNotSupported. if cm := opts.LeafNode.Compression.Mode; cm != CompressionNotSupported { info.Compression = cm } s.generateNonce(nonce[:]) s.mu.Unlock() } // Grab lock c.mu.Lock() var preBuf []byte if solicited { // For websocket connection, we need to send an HTTP request, // and get the response before starting the readLoop to get // the INFO, etc.. if c.isWebsocket() { var err error var closeReason ClosedState preBuf, closeReason, err = c.leafNodeSolicitWSConnection(opts, rURL, remote) if err != nil { c.Errorf("Error soliciting websocket connection: %v", err) c.mu.Unlock() if closeReason != 0 { c.closeConnection(closeReason) } return nil } } else { // If configured to do TLS handshake first if tlsFirst { if _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil { c.mu.Unlock() return nil } } // We need to wait for the info, but not for too long. c.nc.SetReadDeadline(time.Now().Add(infoTimeout)) } // We will process the INFO from the readloop and finish by // sending the CONNECT and finish registration later. } else { // Send our info to the other side. // Remember the nonce we sent here for signatures, etc. c.nonce = make([]byte, nonceLen) copy(c.nonce, nonce[:]) info.Nonce = bytesToString(c.nonce) info.CID = c.cid proto := generateInfoJSON(info) if !opts.LeafNode.TLSHandshakeFirst { // We have to send from this go routine because we may // have to block for TLS handshake before we start our // writeLoop go routine. The other side needs to receive // this before it can initiate the TLS handshake.. c.sendProtoNow(proto) // The above call could have marked the connection as closed (due to TCP error). if c.isClosed() { c.mu.Unlock() c.closeConnection(WriteError) return nil } } // Check to see if we need to spin up TLS. if !c.isWebsocket() && info.TLSRequired { // Perform server-side TLS handshake. if err := c.doTLSServerHandshake(tlsHandshakeLeaf, opts.LeafNode.TLSConfig, opts.LeafNode.TLSTimeout, opts.LeafNode.TLSPinnedCerts); err != nil { c.mu.Unlock() return nil } } // If the user wants the TLS handshake to occur first, now that it is // done, send the INFO protocol. if opts.LeafNode.TLSHandshakeFirst { c.sendProtoNow(proto) if c.isClosed() { c.mu.Unlock() c.closeConnection(WriteError) return nil } } // Leaf nodes will always require a CONNECT to let us know // when we are properly bound to an account. // // If compression is configured, we can't set the authTimer here because // it would cause the parser to fail any incoming protocol that is not a // CONNECT (and we need to exchange INFO protocols for compression // negotiation). So instead, use the ping timer until we are done with // negotiation and can set the auth timer. timeout := secondsToDuration(opts.LeafNode.AuthTimeout) if needsCompression(opts.LeafNode.Compression.Mode) { c.ping.tmr = time.AfterFunc(timeout, func() { c.authTimeout() }) } else { c.setAuthTimer(timeout) } } // Keep track in case server is shutdown before we can successfully register. if !s.addToTempClients(c.cid, c) { c.mu.Unlock() c.setNoReconnect() c.closeConnection(ServerShutdown) return nil } // Spin up the read loop. s.startGoRoutine(func() { c.readLoop(preBuf) }) // We will spin the write loop for solicited connections only // when processing the INFO and after switching to TLS if needed. if !solicited { s.startGoRoutine(func() { c.writeLoop() }) } c.mu.Unlock() return c } // Will perform the client-side TLS handshake if needed. Assumes that this // is called by the solicit side (remote will be non nil). Returns `true` // if TLS is required, `false` otherwise. // Lock held on entry. func (c *client) leafClientHandshakeIfNeeded(remote *leafNodeCfg, opts *Options) (bool, error) { // Check if TLS is required and gather TLS config variables. tlsRequired, tlsConfig, tlsName, tlsTimeout := c.leafNodeGetTLSConfigForSolicit(remote) if !tlsRequired { return false, nil } // If TLS required, peform handshake. // Get the URL that was used to connect to the remote server. rURL := remote.getCurrentURL() // Perform the client-side TLS handshake. if resetTLSName, err := c.doTLSClientHandshake(tlsHandshakeLeaf, rURL, tlsConfig, tlsName, tlsTimeout, opts.LeafNode.TLSPinnedCerts); err != nil { // Check if we need to reset the remote's TLS name. if resetTLSName { remote.Lock() remote.tlsName = _EMPTY_ remote.Unlock() } return false, err } return true, nil } func (c *client) processLeafnodeInfo(info *Info) { c.mu.Lock() if c.leaf == nil || c.isClosed() { c.mu.Unlock() return } s := c.srv opts := s.getOpts() remote := c.leaf.remote didSolicit := remote != nil firstINFO := !c.flags.isSet(infoReceived) // In case of websocket, the TLS handshake has been already done. // So check only for non websocket connections and for configurations // where the TLS Handshake was not done first. if didSolicit && !c.flags.isSet(handshakeComplete) && !c.isWebsocket() && !remote.TLSHandshakeFirst { // If the server requires TLS, we need to set this in the remote // otherwise if there is no TLS configuration block for the remote, // the solicit side will not attempt to perform the TLS handshake. if firstINFO && info.TLSRequired { remote.TLS = true } if _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil { c.mu.Unlock() return } } // Check for compression, unless already done. if firstINFO && !c.flags.isSet(compressionNegotiated) { // Prevent from getting back here. c.flags.set(compressionNegotiated) var co *CompressionOpts if !didSolicit { co = &opts.LeafNode.Compression } else { co = &remote.Compression } if needsCompression(co.Mode) { // Release client lock since following function will need server lock. c.mu.Unlock() compress, err := s.negotiateLeafCompression(c, didSolicit, info.Compression, co) if err != nil { c.sendErrAndErr(err.Error()) c.closeConnection(ProtocolViolation) return } if compress { // Done for now, will get back another INFO protocol... return } // No compression because one side does not want/can't, so proceed. c.mu.Lock() // Check that the connection did not close if the lock was released. if c.isClosed() { c.mu.Unlock() return } } else { // Coming from an old server, the Compression field would be the empty // string. For servers that are configured with CompressionNotSupported, // this makes them behave as old servers. if info.Compression == _EMPTY_ || co.Mode == CompressionNotSupported { c.leaf.compression = CompressionNotSupported } else { c.leaf.compression = CompressionOff } } // Accepting side does not normally process an INFO protocol during // initial connection handshake. So we keep it consistent by returning // if we are not soliciting. if !didSolicit { // If we had created the ping timer instead of the auth timer, we will // clear the ping timer and set the auth timer now that the compression // negotiation is done. if info.Compression != _EMPTY_ && c.ping.tmr != nil { clearTimer(&c.ping.tmr) c.setAuthTimer(secondsToDuration(opts.LeafNode.AuthTimeout)) } c.mu.Unlock() return } // Fall through and process the INFO protocol as usual. } // Note: For now, only the initial INFO has a nonce. We // will probably do auto key rotation at some point. if firstINFO { // Mark that the INFO protocol has been received. c.flags.set(infoReceived) // Prevent connecting to non leafnode port. Need to do this only for // the first INFO, not for async INFO updates... // // Content of INFO sent by the server when accepting a tcp connection. // ------------------------------------------------------------------- // Listen Port Of | CID | ClientConnectURLs | LeafNodeURLs | Gateway | // ------------------------------------------------------------------- // CLIENT | X* | X** | | | // ROUTE | | X** | X*** | | // GATEWAY | | | | X | // LEAFNODE | X | | X | | // ------------------------------------------------------------------- // * Not on older servers. // ** Not if "no advertise" is enabled. // *** Not if leafnode's "no advertise" is enabled. // // As seen from above, a solicited LeafNode connection should receive // from the remote server an INFO with CID and LeafNodeURLs. Anything // else should be considered an attempt to connect to a wrong port. if didSolicit && (info.CID == 0 || info.LeafNodeURLs == nil) { c.mu.Unlock() c.Errorf(ErrConnectedToWrongPort.Error()) c.closeConnection(WrongPort) return } // Capture a nonce here. c.nonce = []byte(info.Nonce) if info.TLSRequired && didSolicit { remote.TLS = true } supportsHeaders := c.srv.supportsHeaders() c.headers = supportsHeaders && info.Headers // Remember the remote server. // Pre 2.2.0 servers are not sending their server name. // In that case, use info.ID, which, for those servers, matches // the content of the field `Name` in the leafnode CONNECT protocol. if info.Name == _EMPTY_ { c.leaf.remoteServer = info.ID } else { c.leaf.remoteServer = info.Name } c.leaf.remoteDomain = info.Domain c.leaf.remoteCluster = info.Cluster } // For both initial INFO and async INFO protocols, Possibly // update our list of remote leafnode URLs we can connect to. if didSolicit && (len(info.LeafNodeURLs) > 0 || len(info.WSConnectURLs) > 0) { // Consider the incoming array as the most up-to-date // representation of the remote cluster's list of URLs. c.updateLeafNodeURLs(info) } // Check to see if we have permissions updates here. if info.Import != nil || info.Export != nil { perms := &Permissions{ Publish: info.Export, Subscribe: info.Import, } // Check if we have local deny clauses that we need to merge. if remote := c.leaf.remote; remote != nil { if len(remote.DenyExports) > 0 { if perms.Publish == nil { perms.Publish = &SubjectPermission{} } perms.Publish.Deny = append(perms.Publish.Deny, remote.DenyExports...) } if len(remote.DenyImports) > 0 { if perms.Subscribe == nil { perms.Subscribe = &SubjectPermission{} } perms.Subscribe.Deny = append(perms.Subscribe.Deny, remote.DenyImports...) } } c.setPermissions(perms) } var resumeConnect bool // If this is a remote connection and this is the first INFO protocol, // then we need to finish the connect process by sending CONNECT, etc.. if firstINFO && didSolicit { // Clear deadline that was set in createLeafNode while waiting for the INFO. c.nc.SetDeadline(time.Time{}) resumeConnect = true } else if !firstINFO && didSolicit { c.leaf.remoteAccName = info.RemoteAccount } // Check if we have the remote account information and if so make sure it's stored. if info.RemoteAccount != _EMPTY_ { s.leafRemoteAccounts.Store(c.acc.Name, info.RemoteAccount) } c.mu.Unlock() finishConnect := info.ConnectInfo if resumeConnect && s != nil { s.leafNodeResumeConnectProcess(c) if !info.InfoOnConnect { finishConnect = true } } if finishConnect { s.leafNodeFinishConnectProcess(c) } } func (s *Server) negotiateLeafCompression(c *client, didSolicit bool, infoCompression string, co *CompressionOpts) (bool, error) { // Negotiate the appropriate compression mode (or no compression) cm, err := selectCompressionMode(co.Mode, infoCompression) if err != nil { return false, err } c.mu.Lock() // For "auto" mode, set the initial compression mode based on RTT if cm == CompressionS2Auto { if c.rttStart.IsZero() { c.rtt = computeRTT(c.start) } cm = selectS2AutoModeBasedOnRTT(c.rtt, co.RTTThresholds) } // Keep track of the negotiated compression mode. c.leaf.compression = cm cid := c.cid var nonce string if !didSolicit { nonce = bytesToString(c.nonce) } c.mu.Unlock() if !needsCompression(cm) { return false, nil } // If we end-up doing compression... // Generate an INFO with the chosen compression mode. s.mu.Lock() info := s.copyLeafNodeInfo() info.Compression, info.CID, info.Nonce = compressionModeForInfoProtocol(co, cm), cid, nonce infoProto := generateInfoJSON(info) s.mu.Unlock() // If we solicited, then send this INFO protocol BEFORE switching // to compression writer. However, if we did not, we send it after. c.mu.Lock() if didSolicit { c.enqueueProto(infoProto) // Make sure it is completely flushed (the pending bytes goes to // 0) before proceeding. for c.out.pb > 0 && !c.isClosed() { c.flushOutbound() } } // This is to notify the readLoop that it should switch to a // (de)compression reader. c.in.flags.set(switchToCompression) // Create the compress writer before queueing the INFO protocol for // a route that did not solicit. It will make sure that that proto // is sent with compression on. c.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...) if !didSolicit { c.enqueueProto(infoProto) } c.mu.Unlock() return true, nil } // When getting a leaf node INFO protocol, use the provided // array of urls to update the list of possible endpoints. func (c *client) updateLeafNodeURLs(info *Info) { cfg := c.leaf.remote cfg.Lock() defer cfg.Unlock() // We have ensured that if a remote has a WS scheme, then all are. // So check if first is WS, then add WS URLs, otherwise, add non WS ones. if len(cfg.URLs) > 0 && isWSURL(cfg.URLs[0]) { // It does not really matter if we use "ws://" or "wss://" here since // we will have already marked that the remote should use TLS anyway. // But use proper scheme for log statements, etc... proto := wsSchemePrefix if cfg.TLS { proto = wsSchemePrefixTLS } c.doUpdateLNURLs(cfg, proto, info.WSConnectURLs) return } c.doUpdateLNURLs(cfg, "nats-leaf", info.LeafNodeURLs) } func (c *client) doUpdateLNURLs(cfg *leafNodeCfg, scheme string, URLs []string) { cfg.urls = make([]*url.URL, 0, 1+len(URLs)) // Add the ones we receive in the protocol for _, surl := range URLs { url, err := url.Parse(fmt.Sprintf("%s://%s", scheme, surl)) if err != nil { // As per below, the URLs we receive should not have contained URL info, so this should be safe to log. c.Errorf("Error parsing url %q: %v", surl, err) continue } // Do not add if it's the same as what we already have configured. var dup bool for _, u := range cfg.URLs { // URLs that we receive never have user info, but the // ones that were configured may have. Simply compare // host and port to decide if they are equal or not. if url.Host == u.Host && url.Port() == u.Port() { dup = true break } } if !dup { cfg.urls = append(cfg.urls, url) cfg.saveTLSHostname(url) } } // Add the configured one cfg.urls = append(cfg.urls, cfg.URLs...) } // Similar to setInfoHostPortAndGenerateJSON, but for leafNodeInfo. func (s *Server) setLeafNodeInfoHostPortAndIP() error { opts := s.getOpts() if opts.LeafNode.Advertise != _EMPTY_ { advHost, advPort, err := parseHostPort(opts.LeafNode.Advertise, opts.LeafNode.Port) if err != nil { return err } s.leafNodeInfo.Host = advHost s.leafNodeInfo.Port = advPort } else { s.leafNodeInfo.Host = opts.LeafNode.Host s.leafNodeInfo.Port = opts.LeafNode.Port // If the host is "0.0.0.0" or "::" we need to resolve to a public IP. // This will return at most 1 IP. hostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(s.leafNodeInfo.Host, false) if err != nil { return err } if hostIsIPAny { if len(ips) == 0 { s.Errorf("Could not find any non-local IP for leafnode's listen specification %q", s.leafNodeInfo.Host) } else { // Take the first from the list... s.leafNodeInfo.Host = ips[0] } } } // Use just host:port for the IP s.leafNodeInfo.IP = net.JoinHostPort(s.leafNodeInfo.Host, strconv.Itoa(s.leafNodeInfo.Port)) if opts.LeafNode.Advertise != _EMPTY_ { s.Noticef("Advertise address for leafnode is set to %s", s.leafNodeInfo.IP) } return nil } // Add the connection to the map of leaf nodes. // If `checkForDup` is true (invoked when a leafnode is accepted), then we check // if a connection already exists for the same server name and account. // That can happen when the remote is attempting to reconnect while the accepting // side did not detect the connection as broken yet. // But it can also happen when there is a misconfiguration and the remote is // creating two (or more) connections that bind to the same account on the accept // side. // When a duplicate is found, the new connection is accepted and the old is closed // (this solves the stale connection situation). An error is returned to help the // remote detect the misconfiguration when the duplicate is the result of that // misconfiguration. func (s *Server) addLeafNodeConnection(c *client, srvName, clusterName string, checkForDup bool) { var accName string c.mu.Lock() cid := c.cid acc := c.acc if acc != nil { accName = acc.Name } myRemoteDomain := c.leaf.remoteDomain mySrvName := c.leaf.remoteServer remoteAccName := c.leaf.remoteAccName myClustName := c.leaf.remoteCluster solicited := c.leaf.remote != nil c.mu.Unlock() var old *client s.mu.Lock() // We check for empty because in some test we may send empty CONNECT{} if checkForDup && srvName != _EMPTY_ { for _, ol := range s.leafs { ol.mu.Lock() // We care here only about non solicited Leafnode. This function // is more about replacing stale connections than detecting loops. // We have code for the loop detection elsewhere, which also delays // attempt to reconnect. if !ol.isSolicitedLeafNode() && ol.leaf.remoteServer == srvName && ol.leaf.remoteCluster == clusterName && ol.acc.Name == accName && remoteAccName != _EMPTY_ && ol.leaf.remoteAccName == remoteAccName { old = ol } ol.mu.Unlock() if old != nil { break } } } // Store new connection in the map s.leafs[cid] = c s.mu.Unlock() s.removeFromTempClients(cid) // If applicable, evict the old one. if old != nil { old.sendErrAndErr(DuplicateRemoteLeafnodeConnection.String()) old.closeConnection(DuplicateRemoteLeafnodeConnection) c.Warnf("Replacing connection from same server") } srvDecorated := func() string { if myClustName == _EMPTY_ { return mySrvName } return fmt.Sprintf("%s/%s", mySrvName, myClustName) } opts := s.getOpts() sysAcc := s.SystemAccount() js := s.getJetStream() var meta *raft if js != nil { if mg := js.getMetaGroup(); mg != nil { meta = mg.(*raft) } } blockMappingOutgoing := false // Deny (non domain) JetStream API traffic unless system account is shared // and domain names are identical and extending is not disabled // Check if backwards compatibility has been enabled and needs to be acted on forceSysAccDeny := false if len(opts.JsAccDefaultDomain) > 0 { if acc == sysAcc { for _, d := range opts.JsAccDefaultDomain { if d == _EMPTY_ { // Extending JetStream via leaf node is mutually exclusive with a domain mapping to the empty/default domain. // As soon as one mapping to "" is found, disable the ability to extend JS via a leaf node. c.Noticef("Not extending remote JetStream domain %q due to presence of empty default domain", myRemoteDomain) forceSysAccDeny = true break } } } else if domain, ok := opts.JsAccDefaultDomain[accName]; ok && domain == _EMPTY_ { // for backwards compatibility with old setups that do not have a domain name set c.Debugf("Skipping deny %q for account %q due to default domain", jsAllAPI, accName) return } } // If the server has JS disabled, it may still be part of a JetStream that could be extended. // This is either signaled by js being disabled and a domain set, // or in cases where no domain name exists, an extension hint is set. // However, this is only relevant in mixed setups. // // If the system account connects but default domains are present, JetStream can't be extended. if opts.JetStreamDomain != myRemoteDomain || (!opts.JetStream && (opts.JetStreamDomain == _EMPTY_ && opts.JetStreamExtHint != jsWillExtend)) || sysAcc == nil || acc == nil || forceSysAccDeny { // If domain names mismatch always deny. This applies to system accounts as well as non system accounts. // Not having a system account, account or JetStream disabled is considered a mismatch as well. if acc != nil && acc == sysAcc { c.Noticef("System account connected from %s", srvDecorated()) c.Noticef("JetStream not extended, domains differ") c.mergeDenyPermissionsLocked(both, denyAllJs) // When a remote with a system account is present in a server, unless otherwise disabled, the server will be // started in observer mode. Now that it is clear that this not used, turn the observer mode off. if solicited && meta != nil && meta.IsObserver() { meta.setObserver(false, extNotExtended) c.Debugf("Turning JetStream metadata controller Observer Mode off") // Take note that the domain was not extended to avoid this state from startup. writePeerState(js.config.StoreDir, meta.currentPeerState()) // Meta controller can't be leader yet. // Yet it is possible that due to observer mode every server already stopped campaigning. // Therefore this server needs to be kicked into campaigning gear explicitly. meta.Campaign() } } else { c.Noticef("JetStream using domains: local %q, remote %q", opts.JetStreamDomain, myRemoteDomain) c.mergeDenyPermissionsLocked(both, denyAllClientJs) } blockMappingOutgoing = true } else if acc == sysAcc { // system account and same domain s.sys.client.Noticef("Extending JetStream domain %q as System Account connected from server %s", myRemoteDomain, srvDecorated()) // In an extension use case, pin leadership to server remotes connect to. // Therefore, server with a remote that are not already in observer mode, need to be put into it. if solicited && meta != nil && !meta.IsObserver() { meta.setObserver(true, extExtended) c.Debugf("Turning JetStream metadata controller Observer Mode on - System Account Connected") // Take note that the domain was not extended to avoid this state next startup. writePeerState(js.config.StoreDir, meta.currentPeerState()) // If this server is the leader already, step down so a new leader can be elected (that is not an observer) meta.StepDown() } } else { // This deny is needed in all cases (system account shared or not) // If the system account is shared, jsAllAPI traffic will go through the system account. // So in order to prevent duplicate delivery (from system and actual account) suppress it on the account. // If the system account is NOT shared, jsAllAPI traffic has no business c.Debugf("Adding deny %+v for account %q", denyAllClientJs, accName) c.mergeDenyPermissionsLocked(both, denyAllClientJs) } // If we have a specified JetStream domain we will want to add a mapping to // allow access cross domain for each non-system account. if opts.JetStreamDomain != _EMPTY_ && opts.JetStream && acc != nil && acc != sysAcc { for src, dest := range generateJSMappingTable(opts.JetStreamDomain) { if err := acc.AddMapping(src, dest); err != nil { c.Debugf("Error adding JetStream domain mapping: %s", err.Error()) } else { c.Debugf("Adding JetStream Domain Mapping %q -> %s to account %q", src, dest, accName) } } if blockMappingOutgoing { src := fmt.Sprintf(jsDomainAPI, opts.JetStreamDomain) // make sure that messages intended for this domain, do not leave the cluster via this leaf node connection // This is a guard against a miss-config with two identical domain names and will only cover some forms // of this issue, not all of them. // This guards against a hub and a spoke having the same domain name. // But not two spokes having the same one and the request coming from the hub. c.mergeDenyPermissionsLocked(pub, []string{src}) c.Debugf("Adding deny %q for outgoing messages to account %q", src, accName) } } } func (s *Server) removeLeafNodeConnection(c *client) { c.mu.Lock() cid := c.cid if c.leaf != nil { if c.leaf.tsubt != nil { c.leaf.tsubt.Stop() c.leaf.tsubt = nil } if c.leaf.gwSub != nil { s.gwLeafSubs.Remove(c.leaf.gwSub) // We need to set this to nil for GC to release the connection c.leaf.gwSub = nil } } c.mu.Unlock() s.mu.Lock() delete(s.leafs, cid) s.mu.Unlock() s.removeFromTempClients(cid) } // Connect information for solicited leafnodes. type leafConnectInfo struct { Version string `json:"version,omitempty"` Nkey string `json:"nkey,omitempty"` JWT string `json:"jwt,omitempty"` Sig string `json:"sig,omitempty"` User string `json:"user,omitempty"` Pass string `json:"pass,omitempty"` Token string `json:"auth_token,omitempty"` ID string `json:"server_id,omitempty"` Domain string `json:"domain,omitempty"` Name string `json:"name,omitempty"` Hub bool `json:"is_hub,omitempty"` Cluster string `json:"cluster,omitempty"` Headers bool `json:"headers,omitempty"` JetStream bool `json:"jetstream,omitempty"` DenyPub []string `json:"deny_pub,omitempty"` // There was an existing field called: // >> Comp bool `json:"compression,omitempty"` // that has never been used. With support for compression, we now need // a field that is a string. So we use a different json tag: Compression string `json:"compress_mode,omitempty"` // Just used to detect wrong connection attempts. Gateway string `json:"gateway,omitempty"` // Tells the accept side which account the remote is binding to. RemoteAccount string `json:"remote_account,omitempty"` } // processLeafNodeConnect will process the inbound connect args. // Once we are here we are bound to an account, so can send any interest that // we would have to the other side. func (c *client) processLeafNodeConnect(s *Server, arg []byte, lang string) error { // Way to detect clients that incorrectly connect to the route listen // port. Client provided "lang" in the CONNECT protocol while LEAFNODEs don't. if lang != _EMPTY_ { c.sendErrAndErr(ErrClientConnectedToLeafNodePort.Error()) c.closeConnection(WrongPort) return ErrClientConnectedToLeafNodePort } // Unmarshal as a leaf node connect protocol proto := &leafConnectInfo{} if err := json.Unmarshal(arg, proto); err != nil { return err } // Reject a cluster that contains spaces or line breaks. if proto.Cluster != _EMPTY_ && strings.ContainsFunc(proto.Cluster, unicode.IsSpace) { c.sendErrAndErr(ErrClusterNameHasSpaces.Error()) c.closeConnection(ProtocolViolation) return ErrClusterNameHasSpaces } // Check for cluster name collisions. if cn := s.cachedClusterName(); cn != _EMPTY_ && proto.Cluster != _EMPTY_ && proto.Cluster == cn { c.sendErrAndErr(ErrLeafNodeHasSameClusterName.Error()) c.closeConnection(ClusterNamesIdentical) return ErrLeafNodeHasSameClusterName } // Reject if this has Gateway which means that it would be from a gateway // connection that incorrectly connects to the leafnode port. if proto.Gateway != _EMPTY_ { errTxt := fmt.Sprintf("Rejecting connection from gateway %q on the leafnode port", proto.Gateway) c.Errorf(errTxt) c.sendErr(errTxt) c.closeConnection(WrongGateway) return ErrWrongGateway } if mv := s.getOpts().LeafNode.MinVersion; mv != _EMPTY_ { major, minor, update, _ := versionComponents(mv) if !versionAtLeast(proto.Version, major, minor, update) { // We are going to send back an INFO because otherwise recent // versions of the remote server would simply break the connection // after 2 seconds if not receiving it. Instead, we want the // other side to just "stall" until we finish waiting for the holding // period and close the connection below. s.sendPermsAndAccountInfo(c) c.sendErrAndErr(fmt.Sprintf("connection rejected since minimum version required is %q", mv)) select { case <-c.srv.quitCh: case <-time.After(leafNodeWaitBeforeClose): } c.closeConnection(MinimumVersionRequired) return ErrMinimumVersionRequired } } // Check if this server supports headers. supportHeaders := c.srv.supportsHeaders() c.mu.Lock() // Leaf Nodes do not do echo or verbose or pedantic. c.opts.Verbose = false c.opts.Echo = false c.opts.Pedantic = false // This inbound connection will be marked as supporting headers if this server // support headers and the remote has sent in the CONNECT protocol that it does // support headers too. c.headers = supportHeaders && proto.Headers // If the compression level is still not set, set it based on what has been // given to us in the CONNECT protocol. if c.leaf.compression == _EMPTY_ { // But if proto.Compression is _EMPTY_, set it to CompressionNotSupported if proto.Compression == _EMPTY_ { c.leaf.compression = CompressionNotSupported } else { c.leaf.compression = proto.Compression } } // Remember the remote server. c.leaf.remoteServer = proto.Name // Remember the remote account name c.leaf.remoteAccName = proto.RemoteAccount // If the other side has declared itself a hub, so we will take on the spoke role. if proto.Hub { c.leaf.isSpoke = true } // The soliciting side is part of a cluster. if proto.Cluster != _EMPTY_ { c.leaf.remoteCluster = proto.Cluster } c.leaf.remoteDomain = proto.Domain // When a leaf solicits a connection to a hub, the perms that it will use on the soliciting leafnode's // behalf are correct for them, but inside the hub need to be reversed since data is flowing in the opposite direction. if !c.isSolicitedLeafNode() && c.perms != nil { sp, pp := c.perms.sub, c.perms.pub c.perms.sub, c.perms.pub = pp, sp if c.opts.Import != nil { c.darray = c.opts.Import.Deny } else { c.darray = nil } } // Set the Ping timer c.setFirstPingTimer() // If we received pub deny permissions from the other end, merge with existing ones. c.mergeDenyPermissions(pub, proto.DenyPub) c.mu.Unlock() // Register the cluster, even if empty, as long as we are acting as a hub. if !proto.Hub { c.acc.registerLeafNodeCluster(proto.Cluster) } // Add in the leafnode here since we passed through auth at this point. s.addLeafNodeConnection(c, proto.Name, proto.Cluster, true) // If we have permissions bound to this leafnode we need to send then back to the // origin server for local enforcement. s.sendPermsAndAccountInfo(c) // Create and initialize the smap since we know our bound account now. // This will send all registered subs too. s.initLeafNodeSmapAndSendSubs(c) // Announce the account connect event for a leaf node. // This will no-op as needed. s.sendLeafNodeConnect(c.acc) return nil } // Returns the remote cluster name. This is set only once so does not require a lock. func (c *client) remoteCluster() string { if c.leaf == nil { return _EMPTY_ } return c.leaf.remoteCluster } // Sends back an info block to the soliciting leafnode to let it know about // its permission settings for local enforcement. func (s *Server) sendPermsAndAccountInfo(c *client) { // Copy info := s.copyLeafNodeInfo() c.mu.Lock() info.CID = c.cid info.Import = c.opts.Import info.Export = c.opts.Export info.RemoteAccount = c.acc.Name info.ConnectInfo = true c.enqueueProto(generateInfoJSON(info)) c.mu.Unlock() } // Snapshot the current subscriptions from the sublist into our smap which // we will keep updated from now on. // Also send the registered subscriptions. func (s *Server) initLeafNodeSmapAndSendSubs(c *client) { acc := c.acc if acc == nil { c.Debugf("Leafnode does not have an account bound") return } // Collect all account subs here. _subs := [1024]*subscription{} subs := _subs[:0] ims := []string{} // Hold the client lock otherwise there can be a race and miss some subs. c.mu.Lock() defer c.mu.Unlock() acc.mu.RLock() accName := acc.Name accNTag := acc.nameTag // To make printing look better when no friendly name present. if accNTag != _EMPTY_ { accNTag = "/" + accNTag } // If we are solicited we only send interest for local clients. if c.isSpokeLeafNode() { acc.sl.localSubs(&subs, true) } else { acc.sl.All(&subs) } // Check if we have an existing service import reply. siReply := copyBytes(acc.siReply) // Since leaf nodes only send on interest, if the bound // account has import services we need to send those over. for isubj := range acc.imports.services { if c.isSpokeLeafNode() && !c.canSubscribe(isubj) { c.Debugf("Not permitted to import service %q on behalf of %s%s", isubj, accName, accNTag) continue } ims = append(ims, isubj) } // Likewise for mappings. for _, m := range acc.mappings { if c.isSpokeLeafNode() && !c.canSubscribe(m.src) { c.Debugf("Not permitted to import mapping %q on behalf of %s%s", m.src, accName, accNTag) continue } ims = append(ims, m.src) } // Create a unique subject that will be used for loop detection. lds := acc.lds acc.mu.RUnlock() // Check if we have to create the LDS. if lds == _EMPTY_ { lds = leafNodeLoopDetectionSubjectPrefix + nuid.Next() acc.mu.Lock() acc.lds = lds acc.mu.Unlock() } // Now check for gateway interest. Leafnodes will put this into // the proper mode to propagate, but they are not held in the account. gwsa := [16]*client{} gws := gwsa[:0] s.getOutboundGatewayConnections(&gws) for _, cgw := range gws { cgw.mu.Lock() gw := cgw.gw cgw.mu.Unlock() if gw != nil { if ei, _ := gw.outsim.Load(accName); ei != nil { if e := ei.(*outsie); e != nil && e.sl != nil { e.sl.All(&subs) } } } } applyGlobalRouting := s.gateway.enabled if c.isSpokeLeafNode() { // Add a fake subscription for this solicited leafnode connection // so that we can send back directly for mapped GW replies. // We need to keep track of this subscription so it can be removed // when the connection is closed so that the GC can release it. c.leaf.gwSub = &subscription{client: c, subject: []byte(gwReplyPrefix + ">")} c.srv.gwLeafSubs.Insert(c.leaf.gwSub) } // Now walk the results and add them to our smap rc := c.leaf.remoteCluster c.leaf.smap = make(map[string]int32) for _, sub := range subs { // Check perms regardless of role. if c.perms != nil && !c.canSubscribe(string(sub.subject)) { c.Debugf("Not permitted to subscribe to %q on behalf of %s%s", sub.subject, accName, accNTag) continue } // We ignore ourselves here. // Also don't add the subscription if it has a origin cluster and the // cluster name matches the one of the client we are sending to. if c != sub.client && (sub.origin == nil || (bytesToString(sub.origin) != rc)) { count := int32(1) if len(sub.queue) > 0 && sub.qw > 0 { count = sub.qw } c.leaf.smap[keyFromSub(sub)] += count if c.leaf.tsub == nil { c.leaf.tsub = make(map[*subscription]struct{}) } c.leaf.tsub[sub] = struct{}{} } } // FIXME(dlc) - We need to update appropriately on an account claims update. for _, isubj := range ims { c.leaf.smap[isubj]++ } // If we have gateways enabled we need to make sure the other side sends us responses // that have been augmented from the original subscription. // TODO(dlc) - Should we lock this down more? if applyGlobalRouting { c.leaf.smap[oldGWReplyPrefix+"*.>"]++ c.leaf.smap[gwReplyPrefix+">"]++ } // Detect loops by subscribing to a specific subject and checking // if this sub is coming back to us. c.leaf.smap[lds]++ // Check if we need to add an existing siReply to our map. // This will be a prefix so add on the wildcard. if siReply != nil { wcsub := append(siReply, '>') c.leaf.smap[string(wcsub)]++ } // Queue all protocols. There is no max pending limit for LN connection, // so we don't need chunking. The writes will happen from the writeLoop. var b bytes.Buffer for key, n := range c.leaf.smap { c.writeLeafSub(&b, key, n) } if b.Len() > 0 { c.enqueueProto(b.Bytes()) } if c.leaf.tsub != nil { // Clear the tsub map after 5 seconds. c.leaf.tsubt = time.AfterFunc(5*time.Second, func() { c.mu.Lock() if c.leaf != nil { c.leaf.tsub = nil c.leaf.tsubt = nil } c.mu.Unlock() }) } } // updateInterestForAccountOnGateway called from gateway code when processing RS+ and RS-. func (s *Server) updateInterestForAccountOnGateway(accName string, sub *subscription, delta int32) { acc, err := s.LookupAccount(accName) if acc == nil || err != nil { s.Debugf("No or bad account for %q, failed to update interest from gateway", accName) return } acc.updateLeafNodes(sub, delta) } // updateLeafNodes will make sure to update the account smap for the subscription. // Will also forward to all leaf nodes as needed. func (acc *Account) updateLeafNodes(sub *subscription, delta int32) { if acc == nil || sub == nil { return } // We will do checks for no leafnodes and same cluster here inline and under the // general account read lock. // If we feel we need to update the leafnodes we will do that out of line to avoid // blocking routes or GWs. acc.mu.RLock() // First check if we even have leafnodes here. if acc.nleafs == 0 { acc.mu.RUnlock() return } // Is this a loop detection subject. isLDS := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix)) // Capture the cluster even if its empty. var cluster string if sub.origin != nil { cluster = bytesToString(sub.origin) } // If we have an isolated cluster we can return early, as long as it is not a loop detection subject. // Empty clusters will return false for the check. if !isLDS && acc.isLeafNodeClusterIsolated(cluster) { acc.mu.RUnlock() return } // We can release the general account lock. acc.mu.RUnlock() // We can hold the list lock here to avoid having to copy a large slice. acc.lmu.RLock() defer acc.lmu.RUnlock() // Do this once. subject := string(sub.subject) // Walk the connected leafnodes. for _, ln := range acc.lleafs { if ln == sub.client { continue } // Check to make sure this sub does not have an origin cluster that matches the leafnode. ln.mu.Lock() // If skipped, make sure that we still let go the "$LDS." subscription that allows // the detection of loops as long as different cluster. clusterDifferent := cluster != ln.remoteCluster() if (isLDS && clusterDifferent) || ((cluster == _EMPTY_ || clusterDifferent) && (delta <= 0 || ln.canSubscribe(subject))) { ln.updateSmap(sub, delta, isLDS) } ln.mu.Unlock() } } // This will make an update to our internal smap and determine if we should send out // an interest update to the remote side. // Lock should be held. func (c *client) updateSmap(sub *subscription, delta int32, isLDS bool) { if c.leaf.smap == nil { return } // If we are solicited make sure this is a local client or a non-solicited leaf node skind := sub.client.kind updateClient := skind == CLIENT || skind == SYSTEM || skind == JETSTREAM || skind == ACCOUNT if !isLDS && c.isSpokeLeafNode() && !(updateClient || (skind == LEAF && !sub.client.isSpokeLeafNode())) { return } // For additions, check if that sub has just been processed during initLeafNodeSmapAndSendSubs if delta > 0 && c.leaf.tsub != nil { if _, present := c.leaf.tsub[sub]; present { delete(c.leaf.tsub, sub) if len(c.leaf.tsub) == 0 { c.leaf.tsub = nil c.leaf.tsubt.Stop() c.leaf.tsubt = nil } return } } key := keyFromSub(sub) n, ok := c.leaf.smap[key] if delta < 0 && !ok { return } // We will update if its a queue, if count is zero (or negative), or we were 0 and are N > 0. update := sub.queue != nil || (n <= 0 && n+delta > 0) || (n > 0 && n+delta <= 0) n += delta if n > 0 { c.leaf.smap[key] = n } else { delete(c.leaf.smap, key) } if update { c.sendLeafNodeSubUpdate(key, n) } } // Used to force add subjects to the subject map. func (c *client) forceAddToSmap(subj string) { c.mu.Lock() defer c.mu.Unlock() if c.leaf.smap == nil { return } n := c.leaf.smap[subj] if n != 0 { return } // Place into the map since it was not there. c.leaf.smap[subj] = 1 c.sendLeafNodeSubUpdate(subj, 1) } // Used to force remove a subject from the subject map. func (c *client) forceRemoveFromSmap(subj string) { c.mu.Lock() defer c.mu.Unlock() if c.leaf.smap == nil { return } n := c.leaf.smap[subj] if n == 0 { return } n-- if n == 0 { // Remove is now zero delete(c.leaf.smap, subj) c.sendLeafNodeSubUpdate(subj, 0) } else { c.leaf.smap[subj] = n } } // Send the subscription interest change to the other side. // Lock should be held. func (c *client) sendLeafNodeSubUpdate(key string, n int32) { // If we are a spoke, we need to check if we are allowed to send this subscription over to the hub. if c.isSpokeLeafNode() { checkPerms := true if len(key) > 0 && (key[0] == '$' || key[0] == '_') { if strings.HasPrefix(key, leafNodeLoopDetectionSubjectPrefix) || strings.HasPrefix(key, oldGWReplyPrefix) || strings.HasPrefix(key, gwReplyPrefix) { checkPerms = false } } if checkPerms { var subject string if sep := strings.IndexByte(key, ' '); sep != -1 { subject = key[:sep] } else { subject = key } if !c.canSubscribe(subject) { return } } } // If we are here we can send over to the other side. _b := [64]byte{} b := bytes.NewBuffer(_b[:0]) c.writeLeafSub(b, key, n) c.enqueueProto(b.Bytes()) } // Helper function to build the key. func keyFromSub(sub *subscription) string { var sb strings.Builder sb.Grow(len(sub.subject) + len(sub.queue) + 1) sb.Write(sub.subject) if sub.queue != nil { // Just make the key subject spc group, e.g. 'foo bar' sb.WriteByte(' ') sb.Write(sub.queue) } return sb.String() } const ( keyRoutedSub = "R" keyRoutedSubByte = 'R' keyRoutedLeafSub = "L" keyRoutedLeafSubByte = 'L' ) // Helper function to build the key that prevents collisions between normal // routed subscriptions and routed subscriptions on behalf of a leafnode. // Keys will look like this: // "R foo" -> plain routed sub on "foo" // "R foo bar" -> queue routed sub on "foo", queue "bar" // "L foo bar" -> plain routed leaf sub on "foo", leaf "bar" // "L foo bar baz" -> queue routed sub on "foo", queue "bar", leaf "baz" func keyFromSubWithOrigin(sub *subscription) string { var sb strings.Builder sb.Grow(2 + len(sub.origin) + 1 + len(sub.subject) + 1 + len(sub.queue)) leaf := len(sub.origin) > 0 if leaf { sb.WriteByte(keyRoutedLeafSubByte) } else { sb.WriteByte(keyRoutedSubByte) } sb.WriteByte(' ') sb.Write(sub.subject) if sub.queue != nil { sb.WriteByte(' ') sb.Write(sub.queue) } if leaf { sb.WriteByte(' ') sb.Write(sub.origin) } return sb.String() } // Lock should be held. func (c *client) writeLeafSub(w *bytes.Buffer, key string, n int32) { if key == _EMPTY_ { return } if n > 0 { w.WriteString("LS+ " + key) // Check for queue semantics, if found write n. if strings.Contains(key, " ") { w.WriteString(" ") var b [12]byte var i = len(b) for l := n; l > 0; l /= 10 { i-- b[i] = digits[l%10] } w.Write(b[i:]) if c.trace { arg := fmt.Sprintf("%s %d", key, n) c.traceOutOp("LS+", []byte(arg)) } } else if c.trace { c.traceOutOp("LS+", []byte(key)) } } else { w.WriteString("LS- " + key) if c.trace { c.traceOutOp("LS-", []byte(key)) } } w.WriteString(CR_LF) } // processLeafSub will process an inbound sub request for the remote leaf node. func (c *client) processLeafSub(argo []byte) (err error) { // Indicate activity. c.in.subs++ srv := c.srv if srv == nil { return nil } // Copy so we do not reference a potentially large buffer arg := make([]byte, len(argo)) copy(arg, argo) args := splitArg(arg) sub := &subscription{client: c} delta := int32(1) switch len(args) { case 1: sub.queue = nil case 3: sub.queue = args[1] sub.qw = int32(parseSize(args[2])) // TODO: (ik) We should have a non empty queue name and a queue // weight >= 1. For 2.11, we may want to return an error if that // is not the case, but for now just overwrite `delta` if queue // weight is greater than 1 (it is possible after a reconnect/ // server restart to receive a queue weight > 1 for a new sub). if sub.qw > 1 { delta = sub.qw } default: return fmt.Errorf("processLeafSub Parse Error: '%s'", arg) } sub.subject = args[0] c.mu.Lock() if c.isClosed() { c.mu.Unlock() return nil } acc := c.acc // Check if we have a loop. ldsPrefix := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix)) if ldsPrefix && bytesToString(sub.subject) == acc.getLDSubject() { c.mu.Unlock() c.handleLeafNodeLoop(true) return nil } // Check permissions if applicable. (but exclude the $LDS, $GR and _GR_) checkPerms := true if sub.subject[0] == '$' || sub.subject[0] == '_' { if ldsPrefix || bytes.HasPrefix(sub.subject, []byte(oldGWReplyPrefix)) || bytes.HasPrefix(sub.subject, []byte(gwReplyPrefix)) { checkPerms = false } } // If we are a hub check that we can publish to this subject. if checkPerms { subj := string(sub.subject) if subjectIsLiteral(subj) && !c.pubAllowedFullCheck(subj, true, true) { c.mu.Unlock() c.leafSubPermViolation(sub.subject) c.Debugf(fmt.Sprintf("Permissions Violation for Subscription to %q", sub.subject)) return nil } } // Check if we have a maximum on the number of subscriptions. if c.subsAtLimit() { c.mu.Unlock() c.maxSubsExceeded() return nil } // If we have an origin cluster associated mark that in the sub. if rc := c.remoteCluster(); rc != _EMPTY_ { sub.origin = []byte(rc) } // Like Routes, we store local subs by account and subject and optionally queue name. // If we have a queue it will have a trailing weight which we do not want. if sub.queue != nil { sub.sid = arg[:len(arg)-len(args[2])-1] } else { sub.sid = arg } key := bytesToString(sub.sid) osub := c.subs[key] if osub == nil { c.subs[key] = sub // Now place into the account sl. if err := acc.sl.Insert(sub); err != nil { delete(c.subs, key) c.mu.Unlock() c.Errorf("Could not insert subscription: %v", err) c.sendErr("Invalid Subscription") return nil } } else if sub.queue != nil { // For a queue we need to update the weight. delta = sub.qw - atomic.LoadInt32(&osub.qw) atomic.StoreInt32(&osub.qw, sub.qw) acc.sl.UpdateRemoteQSub(osub) } spoke := c.isSpokeLeafNode() c.mu.Unlock() // Only add in shadow subs if a new sub or qsub. if osub == nil { if err := c.addShadowSubscriptions(acc, sub, true); err != nil { c.Errorf(err.Error()) } } // If we are not solicited, treat leaf node subscriptions similar to a // client subscription, meaning we forward them to routes, gateways and // other leaf nodes as needed. if !spoke { // If we are routing add to the route map for the associated account. srv.updateRouteSubscriptionMap(acc, sub, delta) if srv.gateway.enabled { srv.gatewayUpdateSubInterest(acc.Name, sub, delta) } } // Now check on leafnode updates for other leaf nodes. We understand solicited // and non-solicited state in this call so we will do the right thing. acc.updateLeafNodes(sub, delta) return nil } // If the leafnode is a solicited, set the connect delay based on default // or private option (for tests). Sends the error to the other side, log and // close the connection. func (c *client) handleLeafNodeLoop(sendErr bool) { accName, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterLoopDetected) errTxt := fmt.Sprintf("Loop detected for leafnode account=%q. Delaying attempt to reconnect for %v", accName, delay) if sendErr { c.sendErr(errTxt) } c.Errorf(errTxt) // If we are here with "sendErr" false, it means that this is the server // that received the error. The other side will have closed the connection, // but does not hurt to close here too. c.closeConnection(ProtocolViolation) } // processLeafUnsub will process an inbound unsub request for the remote leaf node. func (c *client) processLeafUnsub(arg []byte) error { // Indicate any activity, so pub and sub or unsubs. c.in.subs++ acc := c.acc srv := c.srv c.mu.Lock() if c.isClosed() { c.mu.Unlock() return nil } spoke := c.isSpokeLeafNode() // We store local subs by account and subject and optionally queue name. // LS- will have the arg exactly as the key. sub, ok := c.subs[string(arg)] if !ok { // If not found, don't try to update routes/gws/leaf nodes. c.mu.Unlock() return nil } delta := int32(1) if len(sub.queue) > 0 { delta = sub.qw } c.mu.Unlock() c.unsubscribe(acc, sub, true, true) if !spoke { // If we are routing subtract from the route map for the associated account. srv.updateRouteSubscriptionMap(acc, sub, -delta) // Gateways if srv.gateway.enabled { srv.gatewayUpdateSubInterest(acc.Name, sub, -delta) } } // Now check on leafnode updates for other leaf nodes. acc.updateLeafNodes(sub, -delta) return nil } func (c *client) processLeafHeaderMsgArgs(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_MSG_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 0, 1, 2: return fmt.Errorf("processLeafHeaderMsgArgs Parse Error: '%s'", args) case 3: c.pa.reply = nil c.pa.queues = nil c.pa.hdb = args[1] c.pa.hdr = parseSize(args[1]) c.pa.szb = args[2] c.pa.size = parseSize(args[2]) case 4: c.pa.reply = args[1] c.pa.queues = nil c.pa.hdb = args[2] c.pa.hdr = parseSize(args[2]) c.pa.szb = args[3] c.pa.size = parseSize(args[3]) default: // args[1] is our reply indicator. Should be + or | normally. if len(args[1]) != 1 { return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) } switch args[1][0] { case '+': c.pa.reply = args[2] case '|': c.pa.reply = nil default: return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) } // Grab header size. c.pa.hdb = args[len(args)-2] c.pa.hdr = parseSize(c.pa.hdb) // Grab size. c.pa.szb = args[len(args)-1] c.pa.size = parseSize(c.pa.szb) // Grab queue names. if c.pa.reply != nil { c.pa.queues = args[3 : len(args)-2] } else { c.pa.queues = args[2 : len(args)-2] } } if c.pa.hdr < 0 { return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Header Size: '%s'", arg) } if c.pa.size < 0 { return fmt.Errorf("processLeafHeaderMsgArgs Bad or Missing Size: '%s'", args) } if c.pa.hdr > c.pa.size { return fmt.Errorf("processLeafHeaderMsgArgs Header Size larger then TotalSize: '%s'", arg) } // Common ones processed after check for arg length c.pa.subject = args[0] return nil } func (c *client) processLeafMsgArgs(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_MSG_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 0, 1: return fmt.Errorf("processLeafMsgArgs Parse Error: '%s'", args) case 2: c.pa.reply = nil c.pa.queues = nil c.pa.szb = args[1] c.pa.size = parseSize(args[1]) case 3: c.pa.reply = args[1] c.pa.queues = nil c.pa.szb = args[2] c.pa.size = parseSize(args[2]) default: // args[1] is our reply indicator. Should be + or | normally. if len(args[1]) != 1 { return fmt.Errorf("processLeafMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) } switch args[1][0] { case '+': c.pa.reply = args[2] case '|': c.pa.reply = nil default: return fmt.Errorf("processLeafMsgArgs Bad or Missing Reply Indicator: '%s'", args[1]) } // Grab size. c.pa.szb = args[len(args)-1] c.pa.size = parseSize(c.pa.szb) // Grab queue names. if c.pa.reply != nil { c.pa.queues = args[3 : len(args)-1] } else { c.pa.queues = args[2 : len(args)-1] } } if c.pa.size < 0 { return fmt.Errorf("processLeafMsgArgs Bad or Missing Size: '%s'", args) } // Common ones processed after check for arg length c.pa.subject = args[0] return nil } // processInboundLeafMsg is called to process an inbound msg from a leaf node. func (c *client) processInboundLeafMsg(msg []byte) { // Update statistics // The msg includes the CR_LF, so pull back out for accounting. c.in.msgs++ c.in.bytes += int32(len(msg) - LEN_CR_LF) srv, acc, subject := c.srv, c.acc, string(c.pa.subject) // Mostly under testing scenarios. if srv == nil || acc == nil { return } // Match the subscriptions. We will use our own L1 map if // it's still valid, avoiding contention on the shared sublist. var r *SublistResult var ok bool genid := atomic.LoadUint64(&c.acc.sl.genid) if genid == c.in.genid && c.in.results != nil { r, ok = c.in.results[subject] } else { // Reset our L1 completely. c.in.results = make(map[string]*SublistResult) c.in.genid = genid } // Go back to the sublist data structure. if !ok { r = c.acc.sl.Match(subject) // Prune the results cache. Keeps us from unbounded growth. Random delete. if len(c.in.results) >= maxResultCacheSize { n := 0 for subj := range c.in.results { delete(c.in.results, subj) if n++; n > pruneSize { break } } } // Then add the new cache entry. c.in.results[subject] = r } // Collect queue names if needed. var qnames [][]byte // Check for no interest, short circuit if so. // This is the fanout scale. if len(r.psubs)+len(r.qsubs) > 0 { flag := pmrNoFlag // If we have queue subs in this cluster, then if we run in gateway // mode and the remote gateways have queue subs, then we need to // collect the queue groups this message was sent to so that we // exclude them when sending to gateways. if len(r.qsubs) > 0 && c.srv.gateway.enabled && atomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 { flag |= pmrCollectQueueNames } // If this is a mapped subject that means the mapped interest // is what got us here, but this might not have a queue designation // If that is the case, make sure we ignore to process local queue subscribers. if len(c.pa.mapped) > 0 && len(c.pa.queues) == 0 { flag |= pmrIgnoreEmptyQueueFilter } _, qnames = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flag) } // Now deal with gateways if c.srv.gateway.enabled { c.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, qnames, true) } } // Handles a subscription permission violation. // See leafPermViolation() for details. func (c *client) leafSubPermViolation(subj []byte) { c.leafPermViolation(false, subj) } // Common function to process publish or subscribe leafnode permission violation. // Sends the permission violation error to the remote, logs it and closes the connection. // If this is from a server soliciting, the reconnection will be delayed. func (c *client) leafPermViolation(pub bool, subj []byte) { if c.isSpokeLeafNode() { // For spokes these are no-ops since the hub server told us our permissions. // We just need to not send these over to the other side since we will get cutoff. return } // FIXME(dlc) ? c.setLeafConnectDelayIfSoliciting(leafNodeReconnectAfterPermViolation) var action string if pub { c.sendErr(fmt.Sprintf("Permissions Violation for Publish to %q", subj)) action = "Publish" } else { c.sendErr(fmt.Sprintf("Permissions Violation for Subscription to %q", subj)) action = "Subscription" } c.Errorf("%s Violation on %q - Check other side configuration", action, subj) // TODO: add a new close reason that is more appropriate? c.closeConnection(ProtocolViolation) } // Invoked from generic processErr() for LEAF connections. func (c *client) leafProcessErr(errStr string) { // Check if we got a cluster name collision. if strings.Contains(errStr, ErrLeafNodeHasSameClusterName.Error()) { _, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterClusterNameSame) c.Errorf("Leafnode connection dropped with same cluster name error. Delaying attempt to reconnect for %v", delay) return } // We will look for Loop detected error coming from the other side. // If we solicit, set the connect delay. if !strings.Contains(errStr, "Loop detected") { return } c.handleLeafNodeLoop(false) } // If this leaf connection solicits, sets the connect delay to the given value, // or the one from the server option's LeafNode.connDelay if one is set (for tests). // Returns the connection's account name and delay. func (c *client) setLeafConnectDelayIfSoliciting(delay time.Duration) (string, time.Duration) { c.mu.Lock() if c.isSolicitedLeafNode() { if s := c.srv; s != nil { if srvdelay := s.getOpts().LeafNode.connDelay; srvdelay != 0 { delay = srvdelay } } c.leaf.remote.setConnectDelay(delay) } accName := c.acc.Name c.mu.Unlock() return accName, delay } // For the given remote Leafnode configuration, this function returns // if TLS is required, and if so, will return a clone of the TLS Config // (since some fields will be changed during handshake), the TLS server // name that is remembered, and the TLS timeout. func (c *client) leafNodeGetTLSConfigForSolicit(remote *leafNodeCfg) (bool, *tls.Config, string, float64) { var ( tlsConfig *tls.Config tlsName string tlsTimeout float64 ) remote.RLock() defer remote.RUnlock() tlsRequired := remote.TLS || remote.TLSConfig != nil if tlsRequired { if remote.TLSConfig != nil { tlsConfig = remote.TLSConfig.Clone() } else { tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} } tlsName = remote.tlsName tlsTimeout = remote.TLSTimeout if tlsTimeout == 0 { tlsTimeout = float64(TLS_TIMEOUT / time.Second) } } return tlsRequired, tlsConfig, tlsName, tlsTimeout } // Initiates the LeafNode Websocket connection by: // - doing the TLS handshake if needed // - sending the HTTP request // - waiting for the HTTP response // // Since some bufio reader is used to consume the HTTP response, this function // returns the slice of buffered bytes (if any) so that the readLoop that will // be started after that consume those first before reading from the socket. // The boolean // // Lock held on entry. func (c *client) leafNodeSolicitWSConnection(opts *Options, rURL *url.URL, remote *leafNodeCfg) ([]byte, ClosedState, error) { remote.RLock() compress := remote.Websocket.Compression // By default the server will mask outbound frames, but it can be disabled with this option. noMasking := remote.Websocket.NoMasking infoTimeout := remote.FirstInfoTimeout remote.RUnlock() // Will do the client-side TLS handshake if needed. tlsRequired, err := c.leafClientHandshakeIfNeeded(remote, opts) if err != nil { // 0 will indicate that the connection was already closed return nil, 0, err } // For http request, we need the passed URL to contain either http or https scheme. scheme := "http" if tlsRequired { scheme = "https" } // We will use the `/leafnode` path to tell the accepting WS server that it should // create a LEAF connection, not a CLIENT. // In case we use the user's URL path in the future, make sure we append the user's // path to our `/leafnode` path. lpath := leafNodeWSPath if curPath := rURL.EscapedPath(); curPath != _EMPTY_ { if curPath[0] == '/' { curPath = curPath[1:] } lpath = path.Join(curPath, lpath) } else { lpath = lpath[1:] } ustr := fmt.Sprintf("%s://%s/%s", scheme, rURL.Host, lpath) u, _ := url.Parse(ustr) req := &http.Request{ Method: "GET", URL: u, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: u.Host, } wsKey, err := wsMakeChallengeKey() if err != nil { return nil, WriteError, err } req.Header["Upgrade"] = []string{"websocket"} req.Header["Connection"] = []string{"Upgrade"} req.Header["Sec-WebSocket-Key"] = []string{wsKey} req.Header["Sec-WebSocket-Version"] = []string{"13"} if compress { req.Header.Add("Sec-WebSocket-Extensions", wsPMCReqHeaderValue) } if noMasking { req.Header.Add(wsNoMaskingHeader, wsNoMaskingValue) } c.nc.SetDeadline(time.Now().Add(infoTimeout)) if err := req.Write(c.nc); err != nil { return nil, WriteError, err } var resp *http.Response br := bufio.NewReaderSize(c.nc, MAX_CONTROL_LINE_SIZE) resp, err = http.ReadResponse(br, req) if err == nil && (resp.StatusCode != 101 || !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || resp.Header.Get("Sec-Websocket-Accept") != wsAcceptKey(wsKey)) { err = fmt.Errorf("invalid websocket connection") } // Check compression extension... if err == nil && c.ws.compress { // Check that not only permessage-deflate extension is present, but that // we also have server and client no context take over. srvCompress, noCtxTakeover := wsPMCExtensionSupport(resp.Header, false) // If server does not support compression, then simply disable it in our side. if !srvCompress { c.ws.compress = false } else if !noCtxTakeover { err = fmt.Errorf("compression negotiation error") } } // Same for no masking... if err == nil && noMasking { // Check if server accepts no masking if resp.Header.Get(wsNoMaskingHeader) != wsNoMaskingValue { // Nope, need to mask our writes as any client would do. c.ws.maskwrite = true } } if resp != nil { resp.Body.Close() } if err != nil { return nil, ReadError, err } c.Debugf("Leafnode compression=%v masking=%v", c.ws.compress, c.ws.maskwrite) var preBuf []byte // We have to slurp whatever is in the bufio reader and pass that to the readloop. if n := br.Buffered(); n != 0 { preBuf, _ = br.Peek(n) } return preBuf, 0, nil } const connectProcessTimeout = 2 * time.Second // This is invoked for remote LEAF remote connections after processing the INFO // protocol. func (s *Server) leafNodeResumeConnectProcess(c *client) { clusterName := s.ClusterName() c.mu.Lock() if c.isClosed() { c.mu.Unlock() return } if err := c.sendLeafConnect(clusterName, c.headers); err != nil { c.mu.Unlock() c.closeConnection(WriteError) return } // Spin up the write loop. s.startGoRoutine(func() { c.writeLoop() }) // timeout leafNodeFinishConnectProcess c.ping.tmr = time.AfterFunc(connectProcessTimeout, func() { c.mu.Lock() // check if leafNodeFinishConnectProcess was called and prevent later leafNodeFinishConnectProcess if !c.flags.setIfNotSet(connectProcessFinished) { c.mu.Unlock() return } clearTimer(&c.ping.tmr) closed := c.isClosed() c.mu.Unlock() if !closed { c.sendErrAndDebug("Stale Leaf Node Connection - Closing") c.closeConnection(StaleConnection) } }) c.mu.Unlock() c.Debugf("Remote leafnode connect msg sent") } // This is invoked for remote LEAF connections after processing the INFO // protocol and leafNodeResumeConnectProcess. // This will send LS+ the CONNECT protocol and register the leaf node. func (s *Server) leafNodeFinishConnectProcess(c *client) { c.mu.Lock() if !c.flags.setIfNotSet(connectProcessFinished) { c.mu.Unlock() return } if c.isClosed() { c.mu.Unlock() s.removeLeafNodeConnection(c) return } remote := c.leaf.remote // Check if we will need to send the system connect event. remote.RLock() sendSysConnectEvent := remote.Hub remote.RUnlock() // Capture account before releasing lock acc := c.acc // cancel connectProcessTimeout clearTimer(&c.ping.tmr) c.mu.Unlock() // Make sure we register with the account here. if err := c.registerWithAccount(acc); err != nil { if err == ErrTooManyAccountConnections { c.maxAccountConnExceeded() return } else if err == ErrLeafNodeLoop { c.handleLeafNodeLoop(true) return } c.Errorf("Registering leaf with account %s resulted in error: %v", acc.Name, err) c.closeConnection(ProtocolViolation) return } s.addLeafNodeConnection(c, _EMPTY_, _EMPTY_, false) s.initLeafNodeSmapAndSendSubs(c) if sendSysConnectEvent { s.sendLeafNodeConnect(acc) } // The above functions are not atomically under the client // lock doing those operations. It is possible - since we // have started the read/write loops - that the connection // is closed before or in between. This would leave the // closed LN connection possible registered with the account // and/or the server's leafs map. So check if connection // is closed, and if so, manually cleanup. c.mu.Lock() closed := c.isClosed() if !closed { c.setFirstPingTimer() } c.mu.Unlock() if closed { s.removeLeafNodeConnection(c) if prev := acc.removeClient(c); prev == 1 { s.decActiveAccounts() } } } nats-server-2.10.27/server/leafnode_test.go000066400000000000000000010070111477524627100206240ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "context" "crypto/tls" "fmt" "math/rand" "net" "net/http" "net/http/httptest" "net/url" "os" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nkeys" "github.com/klauspost/compress/s2" jwt "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nats-server/v2/internal/fastrand" "github.com/nats-io/nats-server/v2/internal/testhelper" ) type captureLeafNodeRandomIPLogger struct { DummyLogger ch chan struct{} ips [3]int } func (c *captureLeafNodeRandomIPLogger) Debugf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "hostname_to_resolve") { ippos := strings.Index(msg, "127.0.0.") if ippos != -1 { n := int(msg[ippos+8] - '1') c.ips[n]++ for _, v := range c.ips { if v < 2 { return } } // All IPs got at least some hit, we are done. c.ch <- struct{}{} } } } func TestLeafNodeRandomIP(t *testing.T) { u, err := url.Parse("nats://hostname_to_resolve:1234") if err != nil { t.Fatalf("Error parsing: %v", err) } resolver := &myDummyDNSResolver{ips: []string{"127.0.0.1", "127.0.0.2", "127.0.0.3"}} o := DefaultOptions() o.Host = "127.0.0.1" o.Port = -1 o.LeafNode.Port = 0 o.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} o.LeafNode.ReconnectInterval = 50 * time.Millisecond o.LeafNode.resolver = resolver o.LeafNode.dialTimeout = 15 * time.Millisecond s := RunServer(o) defer s.Shutdown() l := &captureLeafNodeRandomIPLogger{ch: make(chan struct{})} s.SetLogger(l, true, true) select { case <-l.ch: case <-time.After(3 * time.Second): t.Fatalf("Does not seem to have used random IPs") } } func TestLeafNodeRandomRemotes(t *testing.T) { // 16! possible permutations. orderedURLs := make([]*url.URL, 0, 16) for i := 0; i < cap(orderedURLs); i++ { orderedURLs = append(orderedURLs, &url.URL{ Scheme: "nats-leaf", Host: fmt.Sprintf("host%d:7422", i), }) } o := DefaultOptions() o.LeafNode.Remotes = []*RemoteLeafOpts{ {NoRandomize: true}, {NoRandomize: false}, } o.LeafNode.Remotes[0].URLs = make([]*url.URL, cap(orderedURLs)) copy(o.LeafNode.Remotes[0].URLs, orderedURLs) o.LeafNode.Remotes[1].URLs = make([]*url.URL, cap(orderedURLs)) copy(o.LeafNode.Remotes[1].URLs, orderedURLs) s := RunServer(o) defer s.Shutdown() s.mu.Lock() r1 := s.leafRemoteCfgs[0] r2 := s.leafRemoteCfgs[1] s.mu.Unlock() r1.RLock() gotOrdered := r1.urls r1.RUnlock() if got, want := len(gotOrdered), len(orderedURLs); got != want { t.Fatalf("Unexpected rem0 len URLs, got %d, want %d", got, want) } // These should be IN order. for i := range orderedURLs { if got, want := gotOrdered[i].String(), orderedURLs[i].String(); got != want { t.Fatalf("Unexpected ordered url, got %s, want %s", got, want) } } r2.RLock() gotRandom := r2.urls r2.RUnlock() if got, want := len(gotRandom), len(orderedURLs); got != want { t.Fatalf("Unexpected rem1 len URLs, got %d, want %d", got, want) } // These should be OUT of order. var random bool for i := range orderedURLs { if gotRandom[i].String() != orderedURLs[i].String() { random = true break } } if !random { t.Fatal("Expected urls to be random") } } type testLoopbackResolver struct{} func (r *testLoopbackResolver) LookupHost(ctx context.Context, host string) ([]string, error) { return []string{"127.0.0.1"}, nil } func TestLeafNodeTLSWithCerts(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 leaf { listen: "127.0.0.1:-1" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/server.pem" key_file: "../test/configs/certs/tlsauth/server-key.pem" timeout: 2 } } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() u, err := url.Parse(fmt.Sprintf("nats://localhost:%d", o1.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/client.pem" key_file: "../test/configs/certs/tlsauth/client-key.pem" timeout: 2 } } ] } `, u.String()))) o2, err := ProcessConfigFile(conf2) if err != nil { t.Fatalf("Error processing config file: %v", err) } o2.NoLog, o2.NoSigs = true, true o2.LeafNode.resolver = &testLoopbackResolver{} s2 := RunServer(o2) defer s2.Shutdown() checkFor(t, 3*time.Second, 10*time.Millisecond, func() error { if nln := s1.NumLeafNodes(); nln != 1 { return fmt.Errorf("Number of leaf nodes is %d", nln) } return nil }) } func TestLeafNodeTLSRemoteWithNoCerts(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 leaf { listen: "127.0.0.1:-1" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/server.pem" key_file: "../test/configs/certs/tlsauth/server-key.pem" timeout: 2 } } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() u, err := url.Parse(fmt.Sprintf("nats://localhost:%d", o1.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" timeout: 5 } } ] } `, u.String()))) o2, err := ProcessConfigFile(conf2) if err != nil { t.Fatalf("Error processing config file: %v", err) } if len(o2.LeafNode.Remotes) == 0 { t.Fatal("Expected at least a single leaf remote") } var ( got float64 = o2.LeafNode.Remotes[0].TLSTimeout expected float64 = 5 ) if got != expected { t.Fatalf("Expected %v, got: %v", expected, got) } o2.NoLog, o2.NoSigs = true, true o2.LeafNode.resolver = &testLoopbackResolver{} s2 := RunServer(o2) defer s2.Shutdown() checkFor(t, 3*time.Second, 10*time.Millisecond, func() error { if nln := s1.NumLeafNodes(); nln != 1 { return fmt.Errorf("Number of leaf nodes is %d", nln) } return nil }) // Here we only process options without starting the server // and without a root CA for the remote. conf3 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { timeout: 10 } } ] } `, u.String()))) o3, err := ProcessConfigFile(conf3) if err != nil { t.Fatalf("Error processing config file: %v", err) } if len(o3.LeafNode.Remotes) == 0 { t.Fatal("Expected at least a single leaf remote") } got = o3.LeafNode.Remotes[0].TLSTimeout expected = 10 if got != expected { t.Fatalf("Expected %v, got: %v", expected, got) } // Here we only process options without starting the server // and check the default for leafnode remotes. conf4 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" } } ] } `, u.String()))) o4, err := ProcessConfigFile(conf4) if err != nil { t.Fatalf("Error processing config file: %v", err) } if len(o4.LeafNode.Remotes) == 0 { t.Fatal("Expected at least a single leaf remote") } got = o4.LeafNode.Remotes[0].TLSTimeout expected = float64(DEFAULT_LEAF_TLS_TIMEOUT) / float64(time.Second) if int(got) != int(expected) { t.Fatalf("Expected %v, got: %v", expected, got) } } type captureErrorLogger struct { DummyLogger errCh chan string } func (l *captureErrorLogger) Errorf(format string, v ...any) { select { case l.errCh <- fmt.Sprintf(format, v...): default: } } func TestLeafNodeAccountNotFound(t *testing.T) { ob := DefaultOptions() ob.LeafNode.Host = "127.0.0.1" ob.LeafNode.Port = -1 sb := RunServer(ob) defer sb.Shutdown() u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ob.LeafNode.Port)) oa := DefaultOptions() oa.Cluster.Name = "xyz" oa.LeafNode.ReconnectInterval = 10 * time.Millisecond oa.LeafNode.Remotes = []*RemoteLeafOpts{ { LocalAccount: "foo", URLs: []*url.URL{u}, }, } // Expected to fail if _, err := NewServer(oa); err == nil || !strings.Contains(err.Error(), "local account") { t.Fatalf("Expected server to fail with error about no local account, got %v", err) } oa.Accounts = []*Account{NewAccount("foo")} sa := RunServer(oa) defer sa.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 1)} sa.SetLogger(l, false, false) checkLeafNodeConnected(t, sa) // Now simulate account is removed with config reload, or it expires. sa.accounts.Delete("foo") // Restart B (with same Port) sb.Shutdown() sb = RunServer(ob) defer sb.Shutdown() // Wait for report of error select { case e := <-l.errCh: if !strings.Contains(e, "Unable to lookup account") { t.Fatalf("Expected error about no local account, got %s", e) } case <-time.After(2 * time.Second): t.Fatalf("Did not get the error") } // TODO below test is bogus. Instead add the account, do a reload, and make sure the connection works. // For now, sa would try to recreate the connection for ever. // Check that lid is increasing... checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if lid := atomic.LoadUint64(&sa.gcid); lid < 3 { return fmt.Errorf("Seems like connection was not retried, lid currently only at %d", lid) } return nil }) } // This test ensures that we can connect using proper user/password // to a LN URL that was discovered through the INFO protocol. // We also check that the password doesn't leak to debug/trace logs. func TestLeafNodeBasicAuthFailover(t *testing.T) { // Something a little longer than "pwd" to prevent false positives amongst many log lines; // don't make it complex enough to be subject to %-escaping, we want a simple needle search. fatalPassword := "pwdfatal" content := ` listen: "127.0.0.1:-1" cluster { name: "abc" listen: "127.0.0.1:-1" %s } leafnodes { listen: "127.0.0.1:-1" authorization { user: foo password: %s timeout: 1 } } ` conf := createConfFile(t, []byte(fmt.Sprintf(content, "", fatalPassword))) sb1, ob1 := RunServerWithConfig(conf) defer sb1.Shutdown() conf = createConfFile(t, []byte(fmt.Sprintf(content, fmt.Sprintf("routes: [nats://127.0.0.1:%d]", ob1.Cluster.Port), fatalPassword))) sb2, _ := RunServerWithConfig(conf) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) content = ` port: -1 accounts { foo {} } leafnodes { listen: "127.0.0.1:-1" remotes [ { account: "foo" url: "nats://foo:%s@127.0.0.1:%d" } ] } ` conf = createConfFile(t, []byte(fmt.Sprintf(content, fatalPassword, ob1.LeafNode.Port))) sa, _ := RunServerWithConfig(conf) defer sa.Shutdown() l := testhelper.NewDummyLogger(100) sa.SetLogger(l, true, true) // we want debug & trace logs, to check for passwords in them checkLeafNodeConnected(t, sa) // Shutdown sb1, sa should reconnect to sb2 sb1.Shutdown() // Wait a bit to make sure there was a disconnect and attempt to reconnect. time.Sleep(250 * time.Millisecond) // Should be able to reconnect checkLeafNodeConnected(t, sa) // Look at all our logs for the password; at time of writing it doesn't appear // but we want to safe-guard against it. l.CheckForProhibited(t, "fatal password", fatalPassword) } func TestLeafNodeRTT(t *testing.T) { ob := DefaultOptions() ob.PingInterval = 15 * time.Millisecond ob.LeafNode.Host = "127.0.0.1" ob.LeafNode.Port = -1 sb := RunServer(ob) defer sb.Shutdown() lnBURL, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ob.LeafNode.Port)) oa := DefaultOptions() oa.Cluster.Name = "xyz" oa.PingInterval = 15 * time.Millisecond oa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lnBURL}}} sa := RunServer(oa) defer sa.Shutdown() checkLeafNodeConnected(t, sa) checkRTT := func(t *testing.T, s *Server) time.Duration { t.Helper() var ln *client s.mu.Lock() for _, l := range s.leafs { ln = l break } s.mu.Unlock() var rtt time.Duration checkFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error { ln.mu.Lock() rtt = ln.rtt ln.mu.Unlock() if rtt == 0 { return fmt.Errorf("RTT not tracked") } return nil }) return rtt } prevA := checkRTT(t, sa) prevB := checkRTT(t, sb) // Wait to see if RTT is updated checkUpdated := func(t *testing.T, s *Server, prev time.Duration) { attempts := 0 timeout := time.Now().Add(2 * firstPingInterval) for time.Now().Before(timeout) { if rtt := checkRTT(t, s); rtt != prev { return } attempts++ if attempts == 5 { s.mu.Lock() for _, ln := range s.leafs { ln.mu.Lock() ln.rtt = 0 ln.mu.Unlock() break } s.mu.Unlock() } time.Sleep(15 * time.Millisecond) } t.Fatalf("RTT probably not updated") } checkUpdated(t, sa, prevA) checkUpdated(t, sb, prevB) sa.Shutdown() sb.Shutdown() // Now check that initial RTT is computed prior to first PingInterval // Get new options to avoid possible race changing the ping interval. ob = DefaultOptions() ob.Cluster.Name = "xyz" ob.PingInterval = time.Minute ob.LeafNode.Host = "127.0.0.1" ob.LeafNode.Port = -1 sb = RunServer(ob) defer sb.Shutdown() lnBURL, _ = url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ob.LeafNode.Port)) oa = DefaultOptions() oa.PingInterval = time.Minute oa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lnBURL}}} sa = RunServer(oa) defer sa.Shutdown() checkLeafNodeConnected(t, sa) checkRTT(t, sa) checkRTT(t, sb) } func TestLeafNodeValidateAuthOptions(t *testing.T) { opts := DefaultOptions() opts.LeafNode.Username = "user1" opts.LeafNode.Password = "pwd" opts.LeafNode.Users = []*User{{Username: "user", Password: "pwd"}} if _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), "can not have a single user/pass and a users array") { t.Fatalf("Expected error about mixing single/multi users, got %v", err) } // Check duplicate user names opts.LeafNode.Username = _EMPTY_ opts.LeafNode.Password = _EMPTY_ opts.LeafNode.Users = append(opts.LeafNode.Users, &User{Username: "user", Password: "pwd"}) if _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), "duplicate user") { t.Fatalf("Expected error about duplicate user, got %v", err) } } func TestLeafNodeBasicAuthSingleton(t *testing.T) { opts := DefaultOptions() opts.LeafNode.Port = -1 opts.LeafNode.Account = "unknown" if s, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), "cannot find") { if s != nil { s.Shutdown() } t.Fatalf("Expected error about account not found, got %v", err) } template := ` port: -1 accounts: { ACC1: { users = [{user: "user1", password: "user1"}] } ACC2: { users = [{user: "user2", password: "user2"}] } } leafnodes: { port: -1 authorization { %s account: "ACC1" } } ` for iter, test := range []struct { name string userSpec string lnURLCreds string shouldFail bool }{ {"no user creds required and no user so binds to ACC1", "", "", false}, {"no user creds required and pick user2 associated to ACC2", "", "user2:user2@", false}, {"no user creds required and unknown user should fail", "", "unknown:user@", true}, {"user creds required so binds to ACC1", "user: \"ln\"\npass: \"pwd\"", "ln:pwd@", false}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(template, test.userSpec))) s1, o1 := RunServerWithConfig(conf) defer s1.Shutdown() // Create a sub on "foo" for account ACC1 (user user1), which is the one // bound to the accepted LN connection. ncACC1 := natsConnect(t, fmt.Sprintf("nats://user1:user1@%s:%d", o1.Host, o1.Port)) defer ncACC1.Close() sub1 := natsSubSync(t, ncACC1, "foo") natsFlush(t, ncACC1) // Create a sub on "foo" for account ACC2 (user user2). This one should // not receive any message. ncACC2 := natsConnect(t, fmt.Sprintf("nats://user2:user2@%s:%d", o1.Host, o1.Port)) defer ncACC2.Close() sub2 := natsSubSync(t, ncACC2, "foo") natsFlush(t, ncACC2) conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes: { remotes = [ { url: "nats-leaf://%s%s:%d" } ] } `, test.lnURLCreds, o1.LeafNode.Host, o1.LeafNode.Port))) s2, _ := RunServerWithConfig(conf) defer s2.Shutdown() if test.shouldFail { // Wait a bit and ensure that there is no leaf node connection time.Sleep(100 * time.Millisecond) checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := s1.NumLeafNodes(); n != 0 { return fmt.Errorf("Expected no leafnode connection, got %v", n) } return nil }) return } checkLeafNodeConnected(t, s2) nc := natsConnect(t, s2.ClientURL()) defer nc.Close() natsPub(t, nc, "foo", []byte("hello")) // If url contains known user, even when there is no credentials // required, the connection will be bound to the user's account. if iter == 1 { // Should not receive on "ACC1", but should on "ACC2" if _, err := sub1.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout error, got %v", err) } natsNexMsg(t, sub2, time.Second) } else { // Should receive on "ACC1"... natsNexMsg(t, sub1, time.Second) // but not received on "ACC2" since leafnode bound to account "ACC1". if _, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout error, got %v", err) } } }) } } func TestLeafNodeBasicAuthMultiple(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts: { S1ACC1: { users = [{user: "user1", password: "user1"}] } S1ACC2: { users = [{user: "user2", password: "user2"}] } } leafnodes: { port: -1 authorization { users = [ {user: "ln1", password: "ln1", account: "S1ACC1"} {user: "ln2", password: "ln2", account: "S1ACC2"} {user: "ln3", password: "ln3"} ] } } `)) s1, o1 := RunServerWithConfig(conf) defer s1.Shutdown() // Make sure that we reject a LN connection if user does not match conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes: { remotes = [{url: "nats-leaf://wron:user@%s:%d"}] } `, o1.LeafNode.Host, o1.LeafNode.Port))) s2, _ := RunServerWithConfig(conf) defer s2.Shutdown() // Give a chance for s2 to attempt to connect and make sure that s1 // did not register a LN connection. time.Sleep(100 * time.Millisecond) if n := s1.NumLeafNodes(); n != 0 { t.Fatalf("Expected no leafnode connection, got %v", n) } s2.Shutdown() ncACC1 := natsConnect(t, fmt.Sprintf("nats://user1:user1@%s:%d", o1.Host, o1.Port)) defer ncACC1.Close() sub1 := natsSubSync(t, ncACC1, "foo") natsFlush(t, ncACC1) ncACC2 := natsConnect(t, fmt.Sprintf("nats://user2:user2@%s:%d", o1.Host, o1.Port)) defer ncACC2.Close() sub2 := natsSubSync(t, ncACC2, "foo") natsFlush(t, ncACC2) // We will start s2 with 2 LN connections that should bind local account S2ACC1 // to account S1ACC1 and S2ACC2 to account S1ACC2 on s1. conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 accounts { S2ACC1 { users = [{user: "user1", password: "user1"}] } S2ACC2 { users = [{user: "user2", password: "user2"}] } } leafnodes: { remotes = [ { url: "nats-leaf://ln1:ln1@%s:%d" account: "S2ACC1" } { url: "nats-leaf://ln2:ln2@%s:%d" account: "S2ACC2" } ] } `, o1.LeafNode.Host, o1.LeafNode.Port, o1.LeafNode.Host, o1.LeafNode.Port))) s2, o2 := RunServerWithConfig(conf) defer s2.Shutdown() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if nln := s2.NumLeafNodes(); nln != 2 { return fmt.Errorf("Expected 2 connected leafnodes for server %q, got %d", s2.ID(), nln) } return nil }) // Create a user connection on s2 that binds to S2ACC1 (use user1). nc1 := natsConnect(t, fmt.Sprintf("nats://user1:user1@%s:%d", o2.Host, o2.Port)) defer nc1.Close() // Create an user connection on s2 that binds to S2ACC2 (use user2). nc2 := natsConnect(t, fmt.Sprintf("nats://user2:user2@%s:%d", o2.Host, o2.Port)) defer nc2.Close() // Now if a message is published from nc1, sub1 should receive it since // their account are bound together. natsPub(t, nc1, "foo", []byte("hello")) natsNexMsg(t, sub1, time.Second) // But sub2 should not receive it since different account. if _, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout error, got %v", err) } // Now use nc2 (S2ACC2) to publish natsPub(t, nc2, "foo", []byte("hello")) // Expect sub2 to receive and sub1 not to. natsNexMsg(t, sub2, time.Second) if _, err := sub1.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout error, got %v", err) } // Now check that we don't panic if no account is specified for // a given user. conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes: { remotes = [ { url: "nats-leaf://ln3:ln3@%s:%d" } ] } `, o1.LeafNode.Host, o1.LeafNode.Port))) s3, _ := RunServerWithConfig(conf) defer s3.Shutdown() } type loopDetectedLogger struct { DummyLogger ch chan string } func (l *loopDetectedLogger) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "Loop") { select { case l.ch <- msg: default: } } } func TestLeafNodeLoop(t *testing.T) { test := func(t *testing.T, cluster bool) { // This test requires that we set the port to known value because // we want A point to B and B to A. oa := DefaultOptions() oa.ServerName = "A" if !cluster { oa.Cluster.Port = 0 oa.Cluster.Name = _EMPTY_ } oa.LeafNode.ReconnectInterval = 10 * time.Millisecond oa.LeafNode.Port = 1234 ub, _ := url.Parse("nats://127.0.0.1:5678") oa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ub}}} oa.LeafNode.connDelay = 50 * time.Millisecond sa := RunServer(oa) defer sa.Shutdown() la := &loopDetectedLogger{ch: make(chan string, 1)} sa.SetLogger(la, false, false) ob := DefaultOptions() ob.ServerName = "B" if !cluster { ob.Cluster.Port = 0 ob.Cluster.Name = _EMPTY_ } else { ob.Cluster.Name = "xyz" } ob.LeafNode.ReconnectInterval = 10 * time.Millisecond ob.LeafNode.Port = 5678 ua, _ := url.Parse("nats://127.0.0.1:1234") ob.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}} ob.LeafNode.connDelay = 50 * time.Millisecond sb := RunServer(ob) defer sb.Shutdown() lb := &loopDetectedLogger{ch: make(chan string, 1)} sb.SetLogger(lb, false, false) select { case <-la.ch: // OK! case <-lb.ch: // OK! case <-time.After(5 * time.Second): t.Fatalf("Did not get any error regarding loop") } sb.Shutdown() ob.Port = -1 ob.Cluster.Port = -1 ob.LeafNode.Remotes = nil sb = RunServer(ob) defer sb.Shutdown() checkLeafNodeConnected(t, sa) } t.Run("standalone", func(t *testing.T) { test(t, false) }) t.Run("cluster", func(t *testing.T) { test(t, true) }) } func TestLeafNodeLoopFromDAG(t *testing.T) { // We want B & C to point to A, A itself does not point to any other server. // We need to cancel clustering since now this will suppress on its own. oa := DefaultOptions() oa.ServerName = "A" oa.LeafNode.connDelay = 50 * time.Millisecond oa.LeafNode.ReconnectInterval = 10 * time.Millisecond oa.LeafNode.Port = -1 oa.Cluster = ClusterOpts{} sa := RunServer(oa) defer sa.Shutdown() ua, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", oa.LeafNode.Port)) // B will point to A ob := DefaultOptions() ob.ServerName = "B" ob.LeafNode.connDelay = 50 * time.Millisecond ob.LeafNode.ReconnectInterval = 10 * time.Millisecond ob.LeafNode.Port = -1 ob.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}} ob.Cluster = ClusterOpts{} sb := RunServer(ob) defer sb.Shutdown() ub, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ob.LeafNode.Port)) checkLeafNodeConnected(t, sa) checkLeafNodeConnected(t, sb) // C will point to A and B oc := DefaultOptions() oc.ServerName = "C" oc.LeafNode.connDelay = 50 * time.Millisecond oc.LeafNode.ReconnectInterval = 10 * time.Millisecond oc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}, {URLs: []*url.URL{ub}}} oc.LeafNode.connDelay = 100 * time.Millisecond // Allow logger to be attached before connecting. oc.Cluster = ClusterOpts{} sc := RunServer(oc) lc := &loopDetectedLogger{ch: make(chan string, 1)} sc.SetLogger(lc, false, false) // We should get an error. select { case <-lc.ch: // OK case <-time.After(2 * time.Second): t.Fatalf("Did not get any error regarding loop") } // C should not be connected to anything. checkLeafNodeConnectedCount(t, sc, 0) // A and B are connected to each other. checkLeafNodeConnectedCount(t, sa, 1) checkLeafNodeConnectedCount(t, sb, 1) // Shutdown C and restart without the loop. sc.Shutdown() oc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ub}}} sc = RunServer(oc) defer sc.Shutdown() checkLeafNodeConnectedCount(t, sa, 1) checkLeafNodeConnectedCount(t, sb, 2) checkLeafNodeConnectedCount(t, sc, 1) } func TestLeafNodeCloseTLSConnection(t *testing.T) { opts := DefaultOptions() opts.DisableShortFirstPing = true opts.LeafNode.Host = "127.0.0.1" opts.LeafNode.Port = -1 opts.LeafNode.TLSTimeout = 100 tc := &TLSConfigOpts{ CertFile: "./configs/certs/server.pem", KeyFile: "./configs/certs/key.pem", Insecure: true, } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } opts.LeafNode.TLSConfig = tlsConf opts.NoLog = true opts.NoSigs = true s := RunServer(opts) defer s.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.LeafNode.Host, opts.LeafNode.Port) conn, err := net.DialTimeout("tcp", endpoint, 2*time.Second) if err != nil { t.Fatalf("Unexpected error on dial: %v", err) } defer conn.Close() br := bufio.NewReaderSize(conn, 100) if _, err := br.ReadString('\n'); err != nil { t.Fatalf("Unexpected error reading INFO: %v", err) } tlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true}) defer tlsConn.Close() if err := tlsConn.Handshake(); err != nil { t.Fatalf("Unexpected error during handshake: %v", err) } connectOp := []byte("CONNECT {\"name\":\"leaf\",\"verbose\":false,\"pedantic\":false}\r\n") if _, err := tlsConn.Write(connectOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } if _, err := tlsConn.Write([]byte("PING\r\n")); err != nil { t.Fatalf("Unexpected error writing PING: %v", err) } checkLeafNodeConnected(t, s) // Get leaf connection var leaf *client s.mu.Lock() for _, l := range s.leafs { leaf = l break } s.mu.Unlock() // Fill the buffer. We want to timeout on write so that nc.Close() // would block due to a write that cannot complete. buf := make([]byte, 64*1024) done := false for !done { leaf.nc.SetWriteDeadline(time.Now().Add(time.Second)) if _, err := leaf.nc.Write(buf); err != nil { done = true } leaf.nc.SetWriteDeadline(time.Time{}) } ch := make(chan bool) go func() { select { case <-ch: return case <-time.After(3 * time.Second): fmt.Println("!!!! closeConnection is blocked, test will hang !!!") return } }() // Close the route leaf.closeConnection(SlowConsumerWriteDeadline) ch <- true } func TestLeafNodeTLSSaveName(t *testing.T) { opts := DefaultOptions() opts.LeafNode.Host = "127.0.0.1" opts.LeafNode.Port = -1 tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-noip.pem", KeyFile: "../test/configs/certs/server-key-noip.pem", Insecure: true, } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } opts.LeafNode.TLSConfig = tlsConf s := RunServer(opts) defer s.Shutdown() lo := DefaultOptions() u, _ := url.Parse(fmt.Sprintf("nats://localhost:%d", opts.LeafNode.Port)) lo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} lo.LeafNode.ReconnectInterval = 15 * time.Millisecond ln := RunServer(lo) defer ln.Shutdown() // We know connection will fail, but it should not fail because of error such as: // "cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs" // This would mean that we are not saving the hostname to use during the TLS handshake. le := &captureErrorLogger{errCh: make(chan string, 100)} ln.SetLogger(le, false, false) tm := time.NewTimer(time.Second) var done bool for !done { select { case err := <-le.errCh: if strings.Contains(err, "doesn't contain any IP SANs") { t.Fatalf("Got this error: %q", err) } case <-tm.C: done = true } } } func TestLeafNodeRemoteWrongPort(t *testing.T) { for _, test1 := range []struct { name string clusterAdvertise bool leafnodeAdvertise bool }{ {"advertise_on", false, false}, {"cluster_no_advertise", true, false}, {"leafnode_no_advertise", false, true}, } { t.Run(test1.name, func(t *testing.T) { oa := DefaultOptions() // Make sure we have all ports (client, route, gateway) and we will try // to create a leafnode to connection to each and make sure we get the error. oa.Cluster.NoAdvertise = test1.clusterAdvertise oa.Cluster.Name = "A" oa.Cluster.Host = "127.0.0.1" oa.Cluster.Port = -1 oa.Gateway.Host = "127.0.0.1" oa.Gateway.Port = -1 oa.Gateway.Name = "A" oa.LeafNode.Host = "127.0.0.1" oa.LeafNode.Port = -1 oa.LeafNode.NoAdvertise = test1.leafnodeAdvertise oa.Accounts = []*Account{NewAccount("sys")} oa.SystemAccount = "sys" sa := RunServer(oa) defer sa.Shutdown() ob := DefaultOptions() ob.Cluster.NoAdvertise = test1.clusterAdvertise ob.Cluster.Name = "A" ob.Cluster.Host = "127.0.0.1" ob.Cluster.Port = -1 ob.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa.Cluster.Host, oa.Cluster.Port)) ob.Gateway.Host = "127.0.0.1" ob.Gateway.Port = -1 ob.Gateway.Name = "A" ob.LeafNode.Host = "127.0.0.1" ob.LeafNode.Port = -1 ob.LeafNode.NoAdvertise = test1.leafnodeAdvertise ob.Accounts = []*Account{NewAccount("sys")} ob.SystemAccount = "sys" sb := RunServer(ob) defer sb.Shutdown() checkClusterFormed(t, sa, sb) for _, test := range []struct { name string port int }{ {"client", oa.Port}, {"cluster", oa.Cluster.Port}, {"gateway", oa.Gateway.Port}, } { t.Run(test.name, func(t *testing.T) { oc := DefaultOptions() // Server with the wrong config against non leafnode port. leafURL, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", test.port)) oc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{leafURL}}} oc.LeafNode.ReconnectInterval = 5 * time.Millisecond sc := RunServer(oc) defer sc.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} sc.SetLogger(l, true, true) select { case e := <-l.errCh: if strings.Contains(e, ErrConnectedToWrongPort.Error()) { return } case <-time.After(2 * time.Second): t.Fatalf("Did not get any error about connecting to wrong port for %q - %q", test1.name, test.name) } }) } }) } } func TestLeafNodeRemoteIsHub(t *testing.T) { oa := testDefaultOptionsForGateway("A") oa.Accounts = []*Account{NewAccount("sys")} oa.SystemAccount = "sys" sa := RunServer(oa) defer sa.Shutdown() lno := DefaultOptions() lno.LeafNode.Host = "127.0.0.1" lno.LeafNode.Port = -1 ln := RunServer(lno) defer ln.Shutdown() ob1 := testGatewayOptionsFromToWithServers(t, "B", "A", sa) ob1.Accounts = []*Account{NewAccount("sys")} ob1.SystemAccount = "sys" ob1.Cluster.Host = "127.0.0.1" ob1.Cluster.Port = -1 ob1.LeafNode.Host = "127.0.0.1" ob1.LeafNode.Port = -1 u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", lno.LeafNode.Port)) ob1.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, Hub: true, }, } sb1 := RunServer(ob1) defer sb1.Shutdown() waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForInboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) checkLeafNodeConnected(t, sb1) // For now, due to issue 977, let's restart the leafnode so that the // leafnode connect is propagated in the super-cluster. ln.Shutdown() ln = RunServer(lno) defer ln.Shutdown() checkLeafNodeConnected(t, sb1) // Connect another server in cluster B ob2 := testGatewayOptionsFromToWithServers(t, "B", "A", sa) ob2.Accounts = []*Account{NewAccount("sys")} ob2.SystemAccount = "sys" ob2.Cluster.Host = "127.0.0.1" ob2.Cluster.Port = -1 ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) sb2 := RunServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sb2, 1, 2*time.Second) expectedSubs := ln.NumSubscriptions() + 2 // Create sub on "foo" connected to sa ncA := natsConnect(t, sa.ClientURL()) defer ncA.Close() subFoo := natsSubSync(t, ncA, "foo") // Create sub on "bar" connected to sb2 ncB2 := natsConnect(t, sb2.ClientURL()) defer ncB2.Close() subBar := natsSubSync(t, ncB2, "bar") // Make sure subscriptions have propagated to the leafnode. checkFor(t, time.Second, 10*time.Millisecond, func() error { if subs := ln.NumSubscriptions(); subs < expectedSubs { return fmt.Errorf("Number of subs is %d", subs) } return nil }) // Create pub connection on leafnode ncLN := natsConnect(t, ln.ClientURL()) defer ncLN.Close() // Publish on foo and make sure it is received. natsPub(t, ncLN, "foo", []byte("msg")) natsNexMsg(t, subFoo, time.Second) // Publish on foo and make sure it is received. natsPub(t, ncLN, "bar", []byte("msg")) natsNexMsg(t, subBar, time.Second) } func TestLeafNodePermissions(t *testing.T) { lo1 := DefaultOptions() lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 ln1 := RunServer(lo1) defer ln1.Shutdown() errLog := &captureErrorLogger{errCh: make(chan string, 1)} ln1.SetLogger(errLog, false, false) u, _ := url.Parse(fmt.Sprintf("nats://%s:%d", lo1.LeafNode.Host, lo1.LeafNode.Port)) lo2 := DefaultOptions() lo2.Cluster.Name = "xyz" lo2.LeafNode.ReconnectInterval = 5 * time.Millisecond lo2.LeafNode.connDelay = 100 * time.Millisecond lo2.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, DenyExports: []string{"export.*", "export"}, DenyImports: []string{"import.*", "import"}, }, } ln2 := RunServer(lo2) defer ln2.Shutdown() checkLeafNodeConnected(t, ln1) // Create clients on ln1 and ln2 nc1, err := nats.Connect(ln1.ClientURL()) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc1.Close() nc2, err := nats.Connect(ln2.ClientURL()) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc2.Close() checkSubs := func(acc *Account, expected int) { t.Helper() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := acc.TotalSubs(); n != expected { return fmt.Errorf("Expected %d subs, got %v", expected, n) } return nil }) } // Create a sub on ">" on LN1 subAll := natsSubSync(t, nc1, ">") // this should be registered in LN2 (there is 1 sub for LN1 $LDS subject) + SYS IMPORTS checkSubs(ln2.globalAccount(), 12) // Check deny export clause from messages published from LN2 for _, test := range []struct { name string subject string received bool }{ {"do not send on export.bat", "export.bat", false}, {"do not send on export", "export", false}, {"send on foo", "foo", true}, {"send on export.this.one", "export.this.one", true}, } { t.Run(test.name, func(t *testing.T) { nc2.Publish(test.subject, []byte("msg")) if test.received { natsNexMsg(t, subAll, time.Second) } else { if _, err := subAll.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Should not have received message on %q", test.subject) } } }) } subAll.Unsubscribe() // Goes down by 1. checkSubs(ln2.globalAccount(), 11) // We used to make sure we would not do subscriptions however that // was incorrect. We need to check publishes, not the subscriptions. // For instance if we can publish across a leafnode to foo, and the // other side has a subsxcription for '*' the message should cross // the leafnode. The old way would not allow this. // Now check deny import clause. // As of now, we don't suppress forwarding of subscriptions on LN2 that // match the deny import clause to be forwarded to LN1. However, messages // should still not be able to come back to LN2. for _, test := range []struct { name string subSubject string pubSubject string ok bool }{ {"reject import on import.*", "import.*", "import.bad", false}, {"reject import on import", "import", "import", false}, {"accepts import on foo", "foo", "foo", true}, {"accepts import on import.this.one.ok", "import.*.>", "import.this.one.ok", true}, } { t.Run(test.name, func(t *testing.T) { sub := natsSubSync(t, nc2, test.subSubject) checkSubs(ln2.globalAccount(), 12) if !test.ok { nc1.Publish(test.pubSubject, []byte("msg")) if _, err := sub.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Did not expect to get the message") } } else { checkSubs(ln1.globalAccount(), 11) nc1.Publish(test.pubSubject, []byte("msg")) natsNexMsg(t, sub, time.Second) } sub.Unsubscribe() checkSubs(ln1.globalAccount(), 10) }) } } func TestLeafNodePermissionsConcurrentAccess(t *testing.T) { lo1 := DefaultOptions() lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 ln1 := RunServer(lo1) defer ln1.Shutdown() nc1 := natsConnect(t, ln1.ClientURL()) defer nc1.Close() natsSub(t, nc1, "_INBOX.>", func(_ *nats.Msg) {}) natsFlush(t, nc1) ch := make(chan struct{}, 1) wg := sync.WaitGroup{} wg.Add(2) publish := func(nc *nats.Conn) { defer wg.Done() for { select { case <-ch: return default: nc.Publish(nats.NewInbox(), []byte("hello")) } } } go publish(nc1) u, _ := url.Parse(fmt.Sprintf("nats://%s:%d", lo1.LeafNode.Host, lo1.LeafNode.Port)) lo2 := DefaultOptions() lo2.Cluster.Name = "xyz" lo2.LeafNode.ReconnectInterval = 5 * time.Millisecond lo2.LeafNode.connDelay = 500 * time.Millisecond lo2.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, DenyExports: []string{"foo"}, DenyImports: []string{"bar"}, }, } ln2 := RunServer(lo2) defer ln2.Shutdown() nc2 := natsConnect(t, ln2.ClientURL()) defer nc2.Close() natsSub(t, nc2, "_INBOX.>", func(_ *nats.Msg) {}) natsFlush(t, nc2) go publish(nc2) checkLeafNodeConnected(t, ln1) checkLeafNodeConnected(t, ln2) time.Sleep(50 * time.Millisecond) close(ch) wg.Wait() } func TestLeafNodePubAllowedPruning(t *testing.T) { c := &client{} c.setPermissions(&Permissions{Publish: &SubjectPermission{Allow: []string{"foo"}}}) gr := 100 wg := sync.WaitGroup{} wg.Add(gr) for i := 0; i < gr; i++ { go func() { defer wg.Done() for i := 0; i < 100; i++ { c.pubAllowed(nats.NewInbox()) } }() } wg.Wait() if n := int(atomic.LoadInt32(&c.perms.pcsz)); n > maxPermCacheSize { t.Fatalf("Expected size to be less than %v, got %v", maxPermCacheSize, n) } if n := atomic.LoadInt32(&c.perms.prun); n != 0 { t.Fatalf("c.perms.prun should be 0, was %v", n) } } func TestLeafNodeExportPermissionsNotForSpecialSubs(t *testing.T) { lo1 := DefaultOptions() lo1.Accounts = []*Account{NewAccount("SYS")} lo1.SystemAccount = "SYS" lo1.Cluster.Name = "A" lo1.Gateway.Name = "A" lo1.Gateway.Port = -1 lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 ln1 := RunServer(lo1) defer ln1.Shutdown() u, _ := url.Parse(fmt.Sprintf("nats://%s:%d", lo1.LeafNode.Host, lo1.LeafNode.Port)) lo2 := DefaultOptions() lo2.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, DenyExports: []string{">"}, }, } ln2 := RunServer(lo2) defer ln2.Shutdown() checkLeafNodeConnected(t, ln1) // The deny is totally restrictive, but make sure that we still accept the $LDS, $GR and _GR_ go from LN1. checkFor(t, time.Second, 15*time.Millisecond, func() error { // We should have registered the 3 subs from the accepting leafnode. if n := ln2.globalAccount().TotalSubs(); n != 9 { return fmt.Errorf("Expected %d subs, got %v", 9, n) } return nil }) } // Make sure that if the node that detects the loop (and sends the error and // close the connection) is the accept side, the remote node (the one that solicits) // properly use the reconnect delay. func TestLeafNodeLoopDetectedOnAcceptSide(t *testing.T) { bo := DefaultOptions() bo.LeafNode.Host = "127.0.0.1" bo.LeafNode.Port = -1 b := RunServer(bo) defer b.Shutdown() l := &loopDetectedLogger{ch: make(chan string, 1)} b.SetLogger(l, false, false) u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", bo.LeafNode.Port)) ao := testDefaultOptionsForGateway("A") ao.Accounts = []*Account{NewAccount("SYS")} ao.SystemAccount = "SYS" ao.LeafNode.ReconnectInterval = 5 * time.Millisecond ao.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, Hub: true, }, } a := RunServer(ao) defer a.Shutdown() co := testGatewayOptionsFromToWithServers(t, "C", "A", a) co.Accounts = []*Account{NewAccount("SYS")} co.SystemAccount = "SYS" co.LeafNode.ReconnectInterval = 5 * time.Millisecond co.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, Hub: true, }, } c := RunServer(co) defer c.Shutdown() for i := 0; i < 2; i++ { select { case <-l.ch: // OK case <-time.After(200 * time.Millisecond): // We are likely to detect from each A and C servers, // but consider a failure if we did not receive any. if i == 0 { t.Fatalf("Should have detected loop") } } } // The reconnect attempt is set to 5ms, but the default loop delay // is 30 seconds, so we should not get any new error for that long. // Check if we are getting more errors.. select { case e := <-l.ch: t.Fatalf("Should not have gotten another error, got %q", e) case <-time.After(50 * time.Millisecond): // OK! } } func TestLeafNodeHubWithGateways(t *testing.T) { ao := DefaultOptions() ao.ServerName = "A" ao.LeafNode.Host = "127.0.0.1" ao.LeafNode.Port = -1 a := RunServer(ao) defer a.Shutdown() ua, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ao.LeafNode.Port)) bo := testDefaultOptionsForGateway("B") bo.ServerName = "B" bo.Accounts = []*Account{NewAccount("SYS")} bo.SystemAccount = "SYS" bo.LeafNode.ReconnectInterval = 5 * time.Millisecond bo.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{ua}, Hub: true, }, } b := RunServer(bo) defer b.Shutdown() do := DefaultOptions() do.ServerName = "D" do.LeafNode.Host = "127.0.0.1" do.LeafNode.Port = -1 d := RunServer(do) defer d.Shutdown() ud, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", do.LeafNode.Port)) co := testGatewayOptionsFromToWithServers(t, "C", "B", b) co.ServerName = "C" co.Accounts = []*Account{NewAccount("SYS")} co.SystemAccount = "SYS" co.LeafNode.ReconnectInterval = 5 * time.Millisecond co.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{ud}, Hub: true, }, } c := RunServer(co) defer c.Shutdown() waitForInboundGateways(t, b, 1, 2*time.Second) waitForInboundGateways(t, c, 1, 2*time.Second) checkLeafNodeConnected(t, a) checkLeafNodeConnected(t, d) // Create a responder on D ncD := natsConnect(t, d.ClientURL()) defer ncD.Close() ncD.Subscribe("service", func(m *nats.Msg) { m.Respond([]byte("reply")) }) ncD.Flush() checkFor(t, time.Second, 15*time.Millisecond, func() error { acc := a.globalAccount() if r := acc.sl.Match("service"); r != nil && len(r.psubs) == 1 { return nil } return fmt.Errorf("subscription still not registered") }) // Create requestor on A and send the request, expect a reply. ncA := natsConnect(t, a.ClientURL()) defer ncA.Close() if msg, err := ncA.Request("service", []byte("request"), time.Second); err != nil { t.Fatalf("Failed to get reply: %v", err) } else if string(msg.Data) != "reply" { t.Fatalf("Unexpected reply: %q", msg.Data) } } func TestLeafNodeTmpClients(t *testing.T) { ao := DefaultOptions() ao.LeafNode.Host = "127.0.0.1" ao.LeafNode.Port = -1 a := RunServer(ao) defer a.Shutdown() c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", ao.LeafNode.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer c.Close() // Read info br := bufio.NewReader(c) br.ReadLine() checkTmp := func(expected int) { t.Helper() checkFor(t, time.Second, 15*time.Millisecond, func() error { a.grMu.Lock() l := len(a.grTmpClients) a.grMu.Unlock() if l != expected { return fmt.Errorf("Expected tmp map to have %v entries, got %v", expected, l) } return nil }) } checkTmp(1) // Close client and wait check that it is removed. c.Close() checkTmp(0) // Check with normal leafnode connection that once connected, // the tmp map is also emptied. bo := DefaultOptions() bo.Cluster.Name = "xyz" bo.LeafNode.ReconnectInterval = 5 * time.Millisecond u, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", ao.LeafNode.Port)) if err != nil { t.Fatalf("Error creating url: %v", err) } bo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} b := RunServer(bo) defer b.Shutdown() checkLeafNodeConnected(t, b) checkTmp(0) } func TestLeafNodeTLSVerifyAndMap(t *testing.T) { accName := "MyAccount" acc := NewAccount(accName) certUserName := "CN=example.com,OU=NATS.io" users := []*User{{Username: certUserName, Account: acc}} for _, test := range []struct { name string leafUsers bool provideCert bool }{ {"no users override, provides cert", false, true}, {"no users override, does not provide cert", false, false}, {"users override, provides cert", true, true}, {"users override, does not provide cert", true, false}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() o.Accounts = []*Account{acc} o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 if test.leafUsers { o.LeafNode.Users = users } else { o.Users = users } tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/server.pem", KeyFile: "../test/configs/certs/tlsauth/server-key.pem", CaFile: "../test/configs/certs/tlsauth/ca.pem", Verify: true, } tlsc, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating tls config: %v", err) } o.LeafNode.TLSConfig = tlsc o.LeafNode.TLSMap = true s := RunServer(o) defer s.Shutdown() slo := DefaultOptions() slo.Cluster.Name = "xyz" sltlsc := &tls.Config{} if test.provideCert { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/client.pem", KeyFile: "../test/configs/certs/tlsauth/client-key.pem", } var err error sltlsc, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } } sltlsc.InsecureSkipVerify = true u, _ := url.Parse(fmt.Sprintf("nats://%s:%d", o.LeafNode.Host, o.LeafNode.Port)) slo.LeafNode.Remotes = []*RemoteLeafOpts{ { TLSConfig: sltlsc, URLs: []*url.URL{u}, }, } sl := RunServer(slo) defer sl.Shutdown() if !test.provideCert { // Wait a bit and make sure we are not connecting time.Sleep(100 * time.Millisecond) checkLeafNodeConnectedCount(t, s, 0) return } checkLeafNodeConnected(t, s) var uname string var accname string s.mu.Lock() for _, c := range s.leafs { c.mu.Lock() uname = c.opts.Username if c.acc != nil { accname = c.acc.GetName() } c.mu.Unlock() } s.mu.Unlock() if uname != certUserName { t.Fatalf("Expected username %q, got %q", certUserName, uname) } if accname != accName { t.Fatalf("Expected account %q, got %v", accName, accname) } }) } } type chanLogger struct { DummyLogger triggerChan chan string } func (l *chanLogger) Warnf(format string, v ...any) { l.triggerChan <- fmt.Sprintf(format, v...) } func (l *chanLogger) Errorf(format string, v ...any) { l.triggerChan <- fmt.Sprintf(format, v...) } const ( testLeafNodeTLSVerifyAndMapSrvA = ` listen: 127.0.0.1:-1 leaf { listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 verify_and_map: true } authorization { users [{ user: "%s" }] } } ` testLeafNodeTLSVerifyAndMapSrvB = ` listen: -1 leaf { remotes [ { url: "tls://user-provided-in-url@localhost:%d" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" } } ] }` ) func TestLeafNodeTLSVerifyAndMapCfgPass(t *testing.T) { l := &chanLogger{triggerChan: make(chan string, 100)} defer close(l.triggerChan) confA := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvA, "localhost"))) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() srvA.SetLogger(l, true, true) confB := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvB, optsA.LeafNode.Port))) ob := LoadConfig(confB) ob.LeafNode.ReconnectInterval = 50 * time.Millisecond srvB := RunServer(ob) defer srvB.Shutdown() // Now make sure that the leaf node connection is up and the correct account was picked checkFor(t, 10*time.Second, 10*time.Millisecond, func() error { for _, srv := range []*Server{srvA, srvB} { if nln := srv.NumLeafNodes(); nln != 1 { return fmt.Errorf("Number of leaf nodes is %d", nln) } if leafz, err := srv.Leafz(nil); err != nil { if len(leafz.Leafs) != 1 { return fmt.Errorf("Number of leaf nodes returned by LEAFZ is not one: %d", len(leafz.Leafs)) } else if leafz.Leafs[0].Account != DEFAULT_GLOBAL_ACCOUNT { return fmt.Errorf("Account used is not $G: %s", leafz.Leafs[0].Account) } } } return nil }) // Make sure that the user name in the url was ignored and a warning printed for { select { case w := <-l.triggerChan: if w == `User "user-provided-in-url" found in connect proto, but user required from cert` { return } case <-time.After(2 * time.Second): t.Fatal("Did not get expected warning") } } } func TestLeafNodeTLSVerifyAndMapCfgFail(t *testing.T) { l := &chanLogger{triggerChan: make(chan string, 100)} defer close(l.triggerChan) // use certificate with SAN localhost, but configure the server to not accept it // instead provide a name matching the user (to be matched by failed confA := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvA, "user-provided-in-url"))) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() srvA.SetLogger(l, true, true) confB := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvB, optsA.LeafNode.Port))) ob := LoadConfig(confB) ob.LeafNode.ReconnectInterval = 50 * time.Millisecond srvB := RunServer(ob) defer srvB.Shutdown() // Now make sure that the leaf node connection is down checkFor(t, 10*time.Second, 10*time.Millisecond, func() error { for _, srv := range []*Server{srvA, srvB} { if nln := srv.NumLeafNodes(); nln != 0 { return fmt.Errorf("Number of leaf nodes is %d", nln) } } return nil }) // Make sure that the connection was closed for the right reason for { select { case w := <-l.triggerChan: if strings.Contains(w, ErrAuthentication.Error()) { return } case <-time.After(2 * time.Second): t.Fatal("Did not get expected warning") } } } func TestLeafNodeOriginClusterInfo(t *testing.T) { hopts := DefaultOptions() hopts.ServerName = "hub" hopts.LeafNode.Port = -1 hub := RunServer(hopts) defer hub.Shutdown() conf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "nats://127.0.0.1:%d" } ] } `, hopts.LeafNode.Port))) opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoLog, opts.NoSigs = true, true s := RunServer(opts) defer s.Shutdown() checkLeafNodeConnected(t, s) // Check the info on the leadnode client in the hub. grabLeaf := func() *client { var l *client hub.mu.Lock() for _, l = range hub.leafs { break } hub.mu.Unlock() return l } l := grabLeaf() if rc := l.remoteCluster(); rc != "" { t.Fatalf("Expected an empty remote cluster, got %q", rc) } s.Shutdown() // Now make our leafnode part of a cluster. conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "nats://127.0.0.1:%d" } ] } cluster { name: "xyz" listen: "127.0.0.1:-1" } `, hopts.LeafNode.Port))) opts, err = ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoLog, opts.NoSigs = true, true s = RunServer(opts) defer s.Shutdown() checkLeafNodeConnected(t, s) l = grabLeaf() if rc := l.remoteCluster(); rc != "xyz" { t.Fatalf("Expected a remote cluster name of \"xyz\", got %q", rc) } pcid := l.cid // Now make sure that if we update our cluster name, simulating the settling // of dynamic cluster names between competing servers. s.setClusterName("xyz") // Make sure we disconnect and reconnect. checkLeafNodeConnectedCount(t, s, 0) checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, hub) l = grabLeaf() if rc := l.remoteCluster(); rc != "xyz" { t.Fatalf("Expected a remote cluster name of \"xyz\", got %q", rc) } // Make sure we reconnected and have a new CID. if l.cid == pcid { t.Fatalf("Expected a different id, got the same") } } type proxyAcceptDetectFailureLate struct { sync.Mutex wg sync.WaitGroup acceptPort int l net.Listener srvs []net.Conn leaf net.Conn startChan chan struct{} } func (p *proxyAcceptDetectFailureLate) run(t *testing.T) int { return p.runEx(t, false) } func (p *proxyAcceptDetectFailureLate) runEx(t *testing.T, needStart bool) int { l, err := natsListen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error on listen: %v", err) } p.Lock() var startChan chan struct{} if needStart { startChan = make(chan struct{}) p.startChan = startChan } p.l = l p.Unlock() port := l.Addr().(*net.TCPAddr).Port p.wg.Add(1) go func() { defer p.wg.Done() defer l.Close() defer func() { p.Lock() for _, c := range p.srvs { c.Close() } p.Unlock() }() if startChan != nil { <-startChan } for { c, err := l.Accept() if err != nil { return } srv, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", p.acceptPort)) if err != nil { return } p.Lock() p.leaf = c p.srvs = append(p.srvs, srv) p.Unlock() transfer := func(c1, c2 net.Conn) { var buf [1024]byte for { n, err := c1.Read(buf[:]) if err != nil { return } if _, err := c2.Write(buf[:n]); err != nil { return } } } go transfer(srv, c) go transfer(c, srv) } }() return port } func (p *proxyAcceptDetectFailureLate) start() { p.Lock() if p.startChan != nil { close(p.startChan) p.startChan = nil } p.Unlock() } func (p *proxyAcceptDetectFailureLate) close() { p.Lock() if p.startChan != nil { close(p.startChan) p.startChan = nil } p.l.Close() p.Unlock() p.wg.Wait() } type oldConnReplacedLogger struct { DummyLogger errCh chan string warnCh chan string } func (l *oldConnReplacedLogger) Errorf(format string, v ...any) { select { case l.errCh <- fmt.Sprintf(format, v...): default: } } func (l *oldConnReplacedLogger) Warnf(format string, v ...any) { select { case l.warnCh <- fmt.Sprintf(format, v...): default: } } // This test will simulate that the accept side does not detect the connection // has been closed early enough. The soliciting side will attempt to reconnect // and we should not be getting the "loop detected" error. func TestLeafNodeLoopDetectedDueToReconnect(t *testing.T) { o := DefaultOptions() o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 s := RunServer(o) defer s.Shutdown() l := &oldConnReplacedLogger{errCh: make(chan string, 10), warnCh: make(chan string, 10)} s.SetLogger(l, false, false) p := &proxyAcceptDetectFailureLate{acceptPort: o.LeafNode.Port} defer p.close() port := p.run(t) aurl, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } ol := DefaultOptions() ol.Cluster.Name = "cde" ol.LeafNode.ReconnectInterval = 50 * time.Millisecond ol.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{aurl}}} sl := RunServer(ol) defer sl.Shutdown() checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, sl) // Cause disconnect client side... p.Lock() p.leaf.Close() p.Unlock() // Make sure we did not get the loop detected error select { case e := <-l.errCh: if strings.Contains(e, "Loop detected") { t.Fatalf("Loop detected: %v", e) } case <-time.After(250 * time.Millisecond): // We are ok } // Now make sure we got the warning select { case w := <-l.warnCh: if !strings.Contains(w, "Replacing connection from same server") { t.Fatalf("Unexpected warning: %v", w) } case <-time.After(time.Second): t.Fatal("Did not get expected warning") } checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, sl) } func TestLeafNodeTwoRemotesBindToSameHubAccount(t *testing.T) { opts := DefaultOptions() opts.LeafNode.Host = "127.0.0.1" opts.LeafNode.Port = -1 s := RunServer(opts) defer s.Shutdown() for _, test := range []struct { name string account string fail bool }{ {"different local accounts", "b", false}, {"same local accounts", "a", true}, } { t.Run(test.name, func(t *testing.T) { conf := ` listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1 } accounts { a { users [ {user: a, password: a} ]} b { users [ {user: b, password: b} ]} } leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:%[1]d account: a } { url: nats-leaf://127.0.0.1:%[1]d account: %s } ] } ` lconf := createConfFile(t, []byte(fmt.Sprintf(conf, opts.LeafNode.Port, test.account))) lopts, err := ProcessConfigFile(lconf) if err != nil { t.Fatalf("Error loading config file: %v", err) } lopts.NoLog = false ln, err := NewServer(lopts) if err != nil { t.Fatalf("Error creating server: %v", err) } defer ln.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} ln.SetLogger(l, false, false) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() ln.Start() }() select { case err := <-l.errCh: if test.fail && !strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) { t.Fatalf("Did not get expected duplicate connection error: %v", err) } else if !test.fail && strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) { t.Fatalf("Incorrectly detected a duplicate connection: %v", err) } case <-time.After(250 * time.Millisecond): if test.fail { t.Fatal("Did not get expected error") } } ln.Shutdown() wg.Wait() }) } } func TestLeafNodeNoDuplicateWithinCluster(t *testing.T) { // This set the cluster name to "abc" oSrv1 := DefaultOptions() oSrv1.ServerName = "srv1" oSrv1.LeafNode.Host = "127.0.0.1" oSrv1.LeafNode.Port = -1 srv1 := RunServer(oSrv1) defer srv1.Shutdown() u, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", oSrv1.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } oLeaf1 := DefaultOptions() oLeaf1.ServerName = "leaf1" oLeaf1.Cluster.Name = "xyz" oLeaf1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} leaf1 := RunServer(oLeaf1) defer leaf1.Shutdown() leaf1ClusterURL := fmt.Sprintf("nats://127.0.0.1:%d", oLeaf1.Cluster.Port) oLeaf2 := DefaultOptions() oLeaf2.ServerName = "leaf2" oLeaf2.Cluster.Name = "xyz" oLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} oLeaf2.Routes = RoutesFromStr(leaf1ClusterURL) leaf2 := RunServer(oLeaf2) defer leaf2.Shutdown() checkClusterFormed(t, leaf1, leaf2) checkLeafNodeConnectedCount(t, srv1, 2) checkLeafNodeConnected(t, leaf1) checkLeafNodeConnected(t, leaf2) ncSrv1 := natsConnect(t, srv1.ClientURL()) defer ncSrv1.Close() natsQueueSub(t, ncSrv1, "foo", "queue", func(m *nats.Msg) { m.Data = []byte("from srv1") m.RespondMsg(m) }) ncLeaf1 := natsConnect(t, leaf1.ClientURL()) defer ncLeaf1.Close() natsQueueSub(t, ncLeaf1, "foo", "queue", func(m *nats.Msg) { m.Data = []byte("from leaf1") m.RespondMsg(m) }) ncLeaf2 := natsConnect(t, leaf2.ClientURL()) defer ncLeaf2.Close() // Check that "foo" interest is available everywhere. for _, s := range []*Server{srv1, leaf1, leaf2} { gacc := s.GlobalAccount() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := gacc.Interest("foo"); n != 2 { return fmt.Errorf("Expected interest for %q to be 2, got %v", "foo", n) } return nil }) } // Send requests (from leaf2). For this test to make sure that // there is no duplicate, we want to make sure that we check for // multiple replies and that the reply subject subscription has // been propagated everywhere. sub := natsSubSync(t, ncLeaf2, "reply_subj") natsFlush(t, ncLeaf2) // Here we have a single sub on "reply_subj" so using checkSubInterest is ok. checkSubInterest(t, srv1, globalAccountName, "reply_subj", time.Second) checkSubInterest(t, leaf1, globalAccountName, "reply_subj", time.Second) checkSubInterest(t, leaf2, globalAccountName, "reply_subj", time.Second) for i := 0; i < 100; i++ { // Now send the request reqID := fmt.Sprintf("req.%d", i) msg := nats.NewMsg("foo") msg.Data = []byte("req") msg.Header.Set("ReqId", reqID) msg.Reply = sub.Subject if err := ncLeaf2.PublishMsg(msg); err != nil { t.Fatalf("Error on publish: %v", err) } // Check that we get the reply replyMsg := natsNexMsg(t, sub, time.Second) // But make sure no duplicate. We do so by checking that the reply's // header ReqId matches our current reqID. if respReqID := replyMsg.Header.Get("ReqId"); respReqID != reqID { t.Fatalf("Current request is %q, got duplicate with %q", reqID, respReqID) } // We also should have preferred the queue sub that is in the leaf cluster. if string(replyMsg.Data) != "from leaf1" { t.Fatalf("Expected reply from leaf1, got %q", replyMsg.Data) } } } func TestLeafNodeLMsgSplit(t *testing.T) { // This set the cluster name to "abc" oSrv1 := DefaultOptions() oSrv1.LeafNode.Host = "127.0.0.1" oSrv1.LeafNode.Port = -1 srv1 := RunServer(oSrv1) defer srv1.Shutdown() oSrv2 := DefaultOptions() oSrv2.LeafNode.Host = "127.0.0.1" oSrv2.LeafNode.Port = -1 oSrv2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", oSrv1.Cluster.Port)) srv2 := RunServer(oSrv2) defer srv2.Shutdown() checkClusterFormed(t, srv1, srv2) u1, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", oSrv1.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } u2, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", oSrv2.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } oLeaf1 := DefaultOptions() oLeaf1.Cluster.Name = "xyz" oLeaf1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u1, u2}}} leaf1 := RunServer(oLeaf1) defer leaf1.Shutdown() oLeaf2 := DefaultOptions() oLeaf2.Cluster.Name = "xyz" oLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u1, u2}}} oLeaf2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", oLeaf1.Cluster.Port)) leaf2 := RunServer(oLeaf2) defer leaf2.Shutdown() checkClusterFormed(t, leaf1, leaf2) checkLeafNodeConnected(t, leaf1) checkLeafNodeConnected(t, leaf2) ncSrv2 := natsConnect(t, srv2.ClientURL()) defer ncSrv2.Close() natsQueueSub(t, ncSrv2, "foo", "queue", func(m *nats.Msg) { m.Respond([]byte("from srv2")) }) // Check that "foo" interest is available everywhere. checkSubInterest(t, srv1, globalAccountName, "foo", time.Second) checkSubInterest(t, srv2, globalAccountName, "foo", time.Second) checkSubInterest(t, leaf1, globalAccountName, "foo", time.Second) checkSubInterest(t, leaf2, globalAccountName, "foo", time.Second) // Not required, but have a request payload that is more than 100 bytes reqPayload := make([]byte, 150) for i := 0; i < len(reqPayload); i++ { reqPayload[i] = byte((i % 26)) + 'A' } // Send repeated requests (from scratch) from leaf-2: sendReq := func() { t.Helper() ncLeaf2 := natsConnect(t, leaf2.ClientURL()) defer ncLeaf2.Close() if _, err := ncLeaf2.Request("foo", reqPayload, time.Second); err != nil { t.Fatalf("Did not receive reply: %v", err) } } for i := 0; i < 100; i++ { sendReq() } } type parseRouteLSUnsubLogger struct { DummyLogger gotTrace chan struct{} gotErr chan error } func (l *parseRouteLSUnsubLogger) Errorf(format string, v ...any) { err := fmt.Errorf(format, v...) select { case l.gotErr <- err: default: } } func (l *parseRouteLSUnsubLogger) Tracef(format string, v ...any) { trace := fmt.Sprintf(format, v...) if strings.Contains(trace, "LS- xyz $G foo bar") { l.gotTrace <- struct{}{} } } func TestLeafNodeRouteParseLSUnsub(t *testing.T) { // This set the cluster name to "abc" oSrv1 := DefaultOptions() oSrv1.LeafNode.Host = "127.0.0.1" oSrv1.LeafNode.Port = -1 srv1 := RunServer(oSrv1) defer srv1.Shutdown() l := &parseRouteLSUnsubLogger{gotTrace: make(chan struct{}, 1), gotErr: make(chan error, 1)} srv1.SetLogger(l, true, true) oSrv2 := DefaultOptions() oSrv2.LeafNode.Host = "127.0.0.1" oSrv2.LeafNode.Port = -1 oSrv2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", oSrv1.Cluster.Port)) srv2 := RunServer(oSrv2) defer srv2.Shutdown() checkClusterFormed(t, srv1, srv2) u2, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", oSrv2.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } oLeaf2 := DefaultOptions() oLeaf2.Cluster.Name = "xyz" oLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u2}}} leaf2 := RunServer(oLeaf2) defer leaf2.Shutdown() checkLeafNodeConnected(t, srv2) checkLeafNodeConnected(t, leaf2) ncLeaf2 := natsConnect(t, leaf2.ClientURL()) defer ncLeaf2.Close() sub := natsQueueSubSync(t, ncLeaf2, "foo", "bar") // The issue was with the unsubscribe of this queue subscription natsUnsub(t, sub) // We should get the trace select { case <-l.gotTrace: // OK! case <-time.After(100 * time.Millisecond): t.Fatalf("Did not get LS- trace") } // And no error... select { case e := <-l.gotErr: t.Fatalf("There was an error on server 1: %q", e.Error()) case <-time.After(100 * time.Millisecond): // OK! } } func TestLeafNodeOperatorBadCfg(t *testing.T) { sysAcc, err := nkeys.CreateAccount() require_NoError(t, err) sysAccPk, err := sysAcc.PublicKey() require_NoError(t, err) tmpDir := t.TempDir() configTmpl := ` port: -1 operator: %s system_account: %s resolver: { type: cache dir: '%s' } leafnodes: { %s } ` cases := []struct { name string errorText string cfg string }{ { name: "Operator with Leafnode", errorText: "operator mode does not allow specifying users in leafnode config", cfg: ` port: -1 authorization { users = [{user: "u", password: "p"}] }`, }, { name: "Operator with NKey", errorText: "operator mode and non account nkeys are incompatible", cfg: ` port: -1 authorization { account: notankey }`, }, { name: "Operator remote account NKeys", errorText: "operator mode requires account nkeys in remotes. " + "Please add an `account` key to each remote in your `leafnodes` section, to assign it to an account. " + "Each account value should be a 56 character public key, starting with the letter 'A'", cfg: `remotes: [{url: u}]`, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(configTmpl, ojwt, sysAccPk, tmpDir, c.cfg))) opts := LoadConfig(conf) s, err := NewServer(opts) if err == nil { s.Shutdown() t.Fatal("Expected an error") } // Since the server cannot be stopped, since it did not start, // let's manually close the account resolver to avoid leaking go routines. opts.AccountResolver.Close() if err.Error() != c.errorText { t.Fatalf("Expected error %q but got %q", c.errorText, err) } }) } } func TestLeafNodeTLSConfigReload(t *testing.T) { template := ` listen: 127.0.0.1:-1 leaf { listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" %s timeout: 2 verify: true } } ` confA := createConfFile(t, []byte(fmt.Sprintf(template, ""))) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() lg := &captureErrorLogger{errCh: make(chan string, 10)} srvA.SetLogger(lg, false, false) confB := createConfFile(t, []byte(fmt.Sprintf(` listen: -1 leaf { remotes [ { url: "tls://127.0.0.1:%d" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" } } ] } `, optsA.LeafNode.Port))) optsB, err := ProcessConfigFile(confB) if err != nil { t.Fatalf("Error processing config file: %v", err) } optsB.LeafNode.ReconnectInterval = 50 * time.Millisecond optsB.NoLog, optsB.NoSigs = true, true srvB := RunServer(optsB) defer srvB.Shutdown() // Wait for the error select { case err := <-lg.errCh: // Since Go 1.18, we had to regenerate certs to not have to use GODEBUG="x509sha1=1" // But on macOS, with our test CA certs, no SCTs included, it will fail // for the reason "x509: “localhost†certificate is not standards compliant" // instead of "unknown authority". if !strings.Contains(err, "unknown") && !strings.Contains(err, "compliant") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(2 * time.Second): t.Fatalf("Did not get TLS error") } // Add the CA to srvA reloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file: "../test/configs/certs/ca.pem"`)) // Now make sure that srvB can create a LN connection. checkFor(t, 3*time.Second, 10*time.Millisecond, func() error { if nln := srvB.NumLeafNodes(); nln != 1 { return fmt.Errorf("Number of leaf nodes is %d", nln) } return nil }) } func TestLeafNodeTLSConfigReloadForRemote(t *testing.T) { confA := createConfFile(t, []byte(` listen: 127.0.0.1:-1 leaf { listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 verify: true } } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() lg := &captureErrorLogger{errCh: make(chan string, 10)} srvA.SetLogger(lg, false, false) template := ` listen: -1 leaf { remotes [ { url: "tls://127.0.0.1:%d" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" %s } } ] } ` confB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.LeafNode.Port, ""))) srvB, _ := RunServerWithConfig(confB) defer srvB.Shutdown() // Wait for the error select { case err := <-lg.errCh: if !strings.Contains(err, "bad certificate") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(2 * time.Second): t.Fatalf("Did not get TLS error") } // Add the CA to srvB reloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.LeafNode.Port, `ca_file: "../test/configs/certs/ca.pem"`)) // Now make sure that srvB can create a LN connection. checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if nln := srvB.NumLeafNodes(); nln != 1 { return fmt.Errorf("Number of leaf nodes is %d", nln) } return nil }) } func testDefaultLeafNodeWSOptions() *Options { o := DefaultOptions() o.Websocket.Host = "127.0.0.1" o.Websocket.Port = -1 o.Websocket.NoTLS = true o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 return o } func testDefaultRemoteLeafNodeWSOptions(t *testing.T, o *Options, tls bool) *Options { // Use some path in the URL.. we don't use that, but internally // the server will prefix the path with /leafnode so that the // WS webserver knows that it needs to create a LEAF connection. u, _ := url.Parse(fmt.Sprintf("ws://127.0.0.1:%d/some/path", o.Websocket.Port)) lo := DefaultOptions() lo.Cluster.Name = "LN" remote := &RemoteLeafOpts{URLs: []*url.URL{u}} if tls { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } // GenTLSConfig sets the CA in ClientCAs, but since here we act // as a client, set RootCAs... tlsConf.RootCAs = tlsConf.ClientCAs remote.TLSConfig = tlsConf } lo.LeafNode.Remotes = []*RemoteLeafOpts{remote} return lo } func TestLeafNodeWSMixURLs(t *testing.T) { for _, test := range []struct { name string urls []string }{ {"mix 1", []string{"nats://127.0.0.1:1234", "ws://127.0.0.1:5678", "wss://127.0.0.1:9012"}}, {"mix 2", []string{"ws://127.0.0.1:1234", "nats://127.0.0.1:5678", "wss://127.0.0.1:9012"}}, {"mix 3", []string{"wss://127.0.0.1:1234", "ws://127.0.0.1:5678", "nats://127.0.0.1:9012"}}, {"mix 4", []string{"ws://127.0.0.1:1234", "nats://127.0.0.1:9012"}}, {"mix 5", []string{"nats://127.0.0.1:1234", "ws://127.0.0.1:9012"}}, {"mix 6", []string{"wss://127.0.0.1:1234", "nats://127.0.0.1:9012"}}, {"mix 7", []string{"nats://127.0.0.1:1234", "wss://127.0.0.1:9012"}}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() remote := &RemoteLeafOpts{} urls := make([]*url.URL, 0, 3) for _, ustr := range test.urls { u, err := url.Parse(ustr) if err != nil { t.Fatalf("Error parsing url: %v", err) } urls = append(urls, u) } remote.URLs = urls o.LeafNode.Remotes = []*RemoteLeafOpts{remote} s, err := NewServer(o) if err == nil || !strings.Contains(err.Error(), "mix") { if s != nil { s.Shutdown() } t.Fatalf("Unexpected error: %v", err) } }) } } type testConnTrackSize struct { sync.Mutex net.Conn sz int } func (c *testConnTrackSize) Write(p []byte) (int, error) { c.Lock() defer c.Unlock() n, err := c.Conn.Write(p) c.sz += n return n, err } func TestLeafNodeWSBasic(t *testing.T) { for _, test := range []struct { name string masking bool tls bool acceptCompression bool remoteCompression bool }{ {"masking plain no compression", true, false, false, false}, {"masking plain compression", true, false, true, true}, {"masking plain compression disagree", true, false, false, true}, {"masking plain compression disagree 2", true, false, true, false}, {"masking tls no compression", true, true, false, false}, {"masking tls compression", true, true, true, true}, {"masking tls compression disagree", true, true, false, true}, {"masking tls compression disagree 2", true, true, true, false}, {"no masking plain no compression", false, false, false, false}, {"no masking plain compression", false, false, true, true}, {"no masking plain compression disagree", false, false, false, true}, {"no masking plain compression disagree 2", false, false, true, false}, {"no masking tls no compression", false, true, false, false}, {"no masking tls compression", false, true, true, true}, {"no masking tls compression disagree", false, true, false, true}, {"no masking tls compression disagree 2", false, true, true, false}, } { t.Run(test.name, func(t *testing.T) { o := testDefaultLeafNodeWSOptions() o.Websocket.NoTLS = !test.tls if test.tls { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } o.Websocket.TLSConfig = tlsConf } o.Websocket.Compression = test.acceptCompression s := RunServer(o) defer s.Shutdown() lo := testDefaultRemoteLeafNodeWSOptions(t, o, test.tls) lo.LeafNode.Remotes[0].Websocket.Compression = test.remoteCompression lo.LeafNode.Remotes[0].Websocket.NoMasking = !test.masking ln := RunServer(lo) defer ln.Shutdown() checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) var trackSizeConn *testConnTrackSize if !test.tls { var cln *client ln.mu.Lock() for _, l := range ln.leafs { cln = l break } ln.mu.Unlock() cln.mu.Lock() trackSizeConn = &testConnTrackSize{Conn: cln.nc} cln.nc = trackSizeConn cln.mu.Unlock() } nc1 := natsConnect(t, s.ClientURL()) defer nc1.Close() sub1 := natsSubSync(t, nc1, "foo") natsFlush(t, nc1) checkSubInterest(t, ln, globalAccountName, "foo", time.Second) nc2 := natsConnect(t, ln.ClientURL()) defer nc2.Close() msg1Payload := make([]byte, 2048) for i := 0; i < len(msg1Payload); i++ { msg1Payload[i] = 'A' } natsPub(t, nc2, "foo", msg1Payload) msg := natsNexMsg(t, sub1, time.Second) if !bytes.Equal(msg.Data, msg1Payload) { t.Fatalf("Invalid message: %q", msg.Data) } sub2 := natsSubSync(t, nc2, "bar") natsFlush(t, nc2) checkSubInterest(t, s, globalAccountName, "bar", time.Second) msg2Payload := make([]byte, 2048) for i := 0; i < len(msg2Payload); i++ { msg2Payload[i] = 'B' } natsPub(t, nc1, "bar", msg2Payload) msg = natsNexMsg(t, sub2, time.Second) if !bytes.Equal(msg.Data, msg2Payload) { t.Fatalf("Invalid message: %q", msg.Data) } if !test.tls { trackSizeConn.Lock() size := trackSizeConn.sz trackSizeConn.Unlock() if test.acceptCompression && test.remoteCompression { if size >= 1024 { t.Fatalf("Seems that there was no compression: size=%v", size) } } else if size < 2048 { t.Fatalf("Seems compression was on while it should not: size=%v", size) } } }) } } func TestLeafNodeWSRemoteCompressAndMaskingOptions(t *testing.T) { for _, test := range []struct { name string compress bool compStr string noMasking bool noMaskStr string }{ {"compression masking", true, "true", false, "false"}, {"compression no masking", true, "true", true, "true"}, {"no compression masking", false, "false", false, "false"}, {"no compression no masking", false, "false", true, "true"}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { remotes [ {url: "ws://127.0.0.1:1234", ws_compression: %s, ws_no_masking: %s} ] } `, test.compStr, test.noMaskStr))) o, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error loading conf: %v", err) } if nr := len(o.LeafNode.Remotes); nr != 1 { t.Fatalf("Expected 1 remote, got %v", nr) } r := o.LeafNode.Remotes[0] if cur := r.Websocket.Compression; cur != test.compress { t.Fatalf("Expected compress to be %v, got %v", test.compress, cur) } if cur := r.Websocket.NoMasking; cur != test.noMasking { t.Fatalf("Expected ws_masking to be %v, got %v", test.noMasking, cur) } }) } } func TestLeafNodeWSNoMaskingRejected(t *testing.T) { wsTestRejectNoMasking = true defer func() { wsTestRejectNoMasking = false }() o := testDefaultLeafNodeWSOptions() s := RunServer(o) defer s.Shutdown() lo := testDefaultRemoteLeafNodeWSOptions(t, o, false) lo.LeafNode.Remotes[0].Websocket.NoMasking = true ln := RunServer(lo) defer ln.Shutdown() checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) var cln *client ln.mu.Lock() for _, l := range ln.leafs { cln = l break } ln.mu.Unlock() cln.mu.Lock() maskWrite := cln.ws.maskwrite cln.mu.Unlock() if !maskWrite { t.Fatal("Leafnode remote connection should mask writes, it does not") } } func TestLeafNodeWSSubPath(t *testing.T) { o := testDefaultLeafNodeWSOptions() s := RunServer(o) defer s.Shutdown() lo := testDefaultRemoteLeafNodeWSOptions(t, o, false) ln := RunServer(lo) defer ln.Shutdown() // Confirm that it can connect using the subpath. checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) // Add another leafnode that tries to connect to the subpath // but intercept the attempt for the test. o2 := testDefaultLeafNodeWSOptions() lo2 := testDefaultRemoteLeafNodeWSOptions(t, o2, false) attempts := make(chan string, 2) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { attempts <- r.URL.String() })) defer ts.Close() u, _ := url.Parse(fmt.Sprintf("%v/some/path", ts.URL)) u.Scheme = "ws" lo2.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{u}, }, } ln2 := RunServer(lo2) defer ln2.Shutdown() expected := "/some/path/leafnode" select { case got := <-attempts: if got != expected { t.Fatalf("Expected: %v, got: %v", expected, got) } case <-time.After(2 * time.Second): t.Fatal("Timed out waiting for leaf ws connect attempt") } } func TestLeafNodeWSFailedConnection(t *testing.T) { o := testDefaultLeafNodeWSOptions() s := RunServer(o) defer s.Shutdown() lo := testDefaultRemoteLeafNodeWSOptions(t, o, true) lo.LeafNode.ReconnectInterval = 100 * time.Millisecond ln := RunServer(lo) defer ln.Shutdown() el := &captureErrorLogger{errCh: make(chan string, 100)} ln.SetLogger(el, false, false) select { case err := <-el.errCh: if !strings.Contains(err, "handshake error") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(time.Second): t.Fatal("No error reported!") } ln.Shutdown() s.Shutdown() lst, err := natsListen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error starting listener: %v", err) } defer lst.Close() wg := sync.WaitGroup{} wg.Add(2) go func() { defer wg.Done() for i := 0; i < 10; i++ { c, err := lst.Accept() if err != nil { return } time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) if rand.Intn(2) == 1 { c.Write([]byte("something\r\n")) } c.Close() } }() time.Sleep(100 * time.Millisecond) port := lst.Addr().(*net.TCPAddr).Port u, _ := url.Parse(fmt.Sprintf("ws://127.0.0.1:%d", port)) lo = DefaultOptions() lo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} lo.LeafNode.ReconnectInterval = 10 * time.Millisecond ln, _ = NewServer(lo) el = &captureErrorLogger{errCh: make(chan string, 100)} ln.SetLogger(el, false, false) go func() { ln.Start() wg.Done() }() timeout := time.NewTimer(time.Second) for i := 0; i < 10; i++ { select { case err := <-el.errCh: if !strings.Contains(err, "Error soliciting") { t.Fatalf("Unexpected error: %v", err) } case <-timeout.C: t.Fatal("No error reported!") } } ln.Shutdown() lst.Close() wg.Wait() } func TestLeafNodeWSAuth(t *testing.T) { template := ` port: -1 authorization { users [ {user: "user", pass: "puser", connection_types: ["%s"]} {user: "leaf", pass: "pleaf", connection_types: ["%s"%s]} ] } websocket { port: -1 no_tls: true } leafnodes { port: -1 } ` s, o, conf := runReloadServerWithContent(t, []byte(fmt.Sprintf(template, jwt.ConnectionTypeStandard, jwt.ConnectionTypeLeafnode, ""))) defer s.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(l, false, false) lo := testDefaultRemoteLeafNodeWSOptions(t, o, false) u, _ := url.Parse(fmt.Sprintf("ws://leaf:pleaf@127.0.0.1:%d", o.Websocket.Port)) remote := &RemoteLeafOpts{URLs: []*url.URL{u}} lo.LeafNode.Remotes = []*RemoteLeafOpts{remote} lo.LeafNode.ReconnectInterval = 50 * time.Millisecond ln := RunServer(lo) defer ln.Shutdown() var lasterr string tm := time.NewTimer(2 * time.Second) for done := false; !done; { select { case lasterr = <-l.errCh: if strings.Contains(lasterr, "authentication") { done = true } case <-tm.C: t.Fatalf("Expected auth error, got %v", lasterr) } } ws := fmt.Sprintf(`, "%s"`, jwt.ConnectionTypeLeafnodeWS) reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, jwt.ConnectionTypeStandard, jwt.ConnectionTypeLeafnode, ws)) checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) nc1 := natsConnect(t, fmt.Sprintf("nats://user:puser@127.0.0.1:%d", o.Port)) defer nc1.Close() sub := natsSubSync(t, nc1, "foo") natsFlush(t, nc1) checkSubInterest(t, ln, globalAccountName, "foo", time.Second) nc2 := natsConnect(t, ln.ClientURL()) defer nc2.Close() natsPub(t, nc2, "foo", []byte("msg1")) msg := natsNexMsg(t, sub, time.Second) if md := string(msg.Data); md != "msg1" { t.Fatalf("Invalid message: %q", md) } } func TestLeafNodeWSGossip(t *testing.T) { o1 := testDefaultLeafNodeWSOptions() s1 := RunServer(o1) defer s1.Shutdown() // Now connect from a server that knows only about s1 lo := testDefaultRemoteLeafNodeWSOptions(t, o1, false) lo.LeafNode.ReconnectInterval = 15 * time.Millisecond ln := RunServer(lo) defer ln.Shutdown() checkLeafNodeConnected(t, s1) checkLeafNodeConnected(t, ln) // Now add a routed server to s1 o2 := testDefaultLeafNodeWSOptions() o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() // Wait for cluster to form checkClusterFormed(t, s1, s2) // Now shutdown s1 and check that ln is able to reconnect to s2. s1.Shutdown() checkLeafNodeConnected(t, s2) checkLeafNodeConnected(t, ln) // Make sure that the reconnection was as a WS connection, not simply to // the regular LN port. var s2lc *client s2.mu.Lock() for _, l := range s2.leafs { s2lc = l break } s2.mu.Unlock() s2lc.mu.Lock() isWS := s2lc.isWebsocket() s2lc.mu.Unlock() if !isWS { t.Fatal("Leafnode connection is not websocket!") } } // This test was showing an issue if one set maxBufSize to very small value, // such as maxBufSize = 10. With such small value, we would get a corruption // in that LMSG would arrive with missing bytes. We are now always making // a copy when dealing with messages that are bigger than maxBufSize. func TestLeafNodeWSNoBufferCorruption(t *testing.T) { o := testDefaultLeafNodeWSOptions() s := RunServer(o) defer s.Shutdown() lo1 := testDefaultRemoteLeafNodeWSOptions(t, o, false) lo1.LeafNode.ReconnectInterval = 15 * time.Millisecond ln1 := RunServer(lo1) defer ln1.Shutdown() lo2 := DefaultOptions() lo2.Cluster.Name = "LN" lo2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", lo1.Cluster.Port)) ln2 := RunServer(lo2) defer ln2.Shutdown() checkClusterFormed(t, ln1, ln2) checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln1) nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub := natsSubSync(t, nc, "foo") nc1 := natsConnect(t, ln1.ClientURL()) defer nc1.Close() nc2 := natsConnect(t, ln2.ClientURL()) defer nc2.Close() sub2 := natsSubSync(t, nc2, "foo") checkSubInterest(t, s, globalAccountName, "foo", time.Second) checkSubInterest(t, ln2, globalAccountName, "foo", time.Second) checkSubInterest(t, ln1, globalAccountName, "foo", time.Second) payload := make([]byte, 100*1024) for i := 0; i < len(payload); i++ { payload[i] = 'A' } natsPub(t, nc1, "foo", payload) checkMsgRcv := func(sub *nats.Subscription) { msg := natsNexMsg(t, sub, time.Second) if !bytes.Equal(msg.Data, payload) { t.Fatalf("Invalid message content: %q", msg.Data) } } checkMsgRcv(sub2) checkMsgRcv(sub) } func TestLeafNodeWSRemoteNoTLSBlockWithWSSProto(t *testing.T) { o := testDefaultLeafNodeWSOptions() o.Websocket.NoTLS = false tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } o.Websocket.TLSConfig = tlsConf s := RunServer(o) defer s.Shutdown() // The test will make sure that if the protocol is "wss://", a TLS handshake must // be initiated, regardless of the presence of a TLS config block in config file // or here directly. // A bug was causing the absence of TLS config block to initiate a non TLS connection // even if "wss://" proto was specified, which would lead to "invalid websocket connection" // errors in the log. // With the fix, the connection will fail because the remote will fail to verify // the root CA, but at least, we will make sure that this is not an "invalid websocket connection" u, _ := url.Parse(fmt.Sprintf("wss://127.0.0.1:%d/some/path", o.Websocket.Port)) lo := DefaultOptions() lo.Cluster.Name = "LN" remote := &RemoteLeafOpts{URLs: []*url.URL{u}} lo.LeafNode.Remotes = []*RemoteLeafOpts{remote} lo.LeafNode.ReconnectInterval = 100 * time.Millisecond ln := RunServer(lo) defer ln.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} ln.SetLogger(l, false, false) select { case e := <-l.errCh: if strings.Contains(e, "invalid websocket connection") { t.Fatalf("The remote did not try to create a TLS connection: %v", e) } // OK! return case <-time.After(2 * time.Second): t.Fatal("Connection should fail") } } func TestLeafNodeWSNoAuthUser(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts { A { users [ {user: a, password: a} ]} B { users [ {user: b, password: b} ]} } websocket { port: -1 no_tls: true no_auth_user: a } leafnodes { port: -1 } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() nc1 := natsConnect(t, fmt.Sprintf("nats://a:a@127.0.0.1:%d", o.Port)) defer nc1.Close() lconf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 accounts { A { users [ {user: a, password: a} ]} B { users [ {user: b, password: b} ]} } leafnodes { remotes [ { url: "ws://127.0.0.1:%d" account: A } ] } `, o.Websocket.Port))) ln, lo := RunServerWithConfig(lconf) defer ln.Shutdown() checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) nc2 := natsConnect(t, fmt.Sprintf("nats://a:a@127.0.0.1:%d", lo.Port)) defer nc2.Close() sub := natsSubSync(t, nc2, "foo") natsFlush(t, nc2) checkSubInterest(t, s, "A", "foo", time.Second) natsPub(t, nc1, "foo", []byte("msg1")) msg := natsNexMsg(t, sub, time.Second) if md := string(msg.Data); md != "msg1" { t.Fatalf("Invalid message: %q", md) } } func TestLeafNodeStreamImport(t *testing.T) { o1 := DefaultOptions() o1.LeafNode.Port = -1 accA := NewAccount("A") o1.Accounts = []*Account{accA} o1.Users = []*User{{Username: "a", Password: "a", Account: accA}} o1.LeafNode.Account = "A" o1.NoAuthUser = "a" s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.LeafNode.Port = -1 o2.Cluster.Name = "xyz" accB := NewAccount("B") if err := accB.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } accC := NewAccount("C") if err := accC.AddStreamImport(accB, ">", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } o2.Accounts = []*Account{accB, accC} o2.Users = []*User{{Username: "b", Password: "b", Account: accB}, {Username: "c", Password: "c", Account: accC}} o2.NoAuthUser = "b" u, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", o1.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } o2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}, LocalAccount: "C"}} s2 := RunServer(o2) defer s2.Shutdown() nc1 := natsConnect(t, s1.ClientURL()) defer nc1.Close() sub := natsSubSync(t, nc1, "a") checkSubInterest(t, s2, "C", "a", time.Second) nc2 := natsConnect(t, s2.ClientURL()) defer nc2.Close() natsPub(t, nc2, "a", []byte("hello?")) natsNexMsg(t, sub, time.Second) } func TestLeafNodeRouteSubWithOrigin(t *testing.T) { lo1 := DefaultOptions() lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 lo1.Cluster.Name = "local" lo1.Cluster.Host = "127.0.0.1" lo1.Cluster.Port = -1 l1 := RunServer(lo1) defer l1.Shutdown() lo2 := DefaultOptions() lo2.LeafNode.Host = "127.0.0.1" lo2.LeafNode.Port = -1 lo2.Cluster.Name = "local" lo2.Cluster.Host = "127.0.0.1" lo2.Cluster.Port = -1 lo2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", lo1.Cluster.Port)) l2 := RunServer(lo2) defer l2.Shutdown() checkClusterFormed(t, l1, l2) u1, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", lo1.LeafNode.Port)) urls := []*url.URL{u1} ro1 := DefaultOptions() ro1.Cluster.Name = "remote" ro1.Cluster.Host = "127.0.0.1" ro1.Cluster.Port = -1 ro1.LeafNode.ReconnectInterval = 50 * time.Millisecond ro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}} r1 := RunServer(ro1) defer r1.Shutdown() checkLeafNodeConnected(t, r1) nc := natsConnect(t, r1.ClientURL(), nats.NoReconnect()) defer nc.Close() natsSubSync(t, nc, "foo") natsQueueSubSync(t, nc, "bar", "baz") checkSubInterest(t, l2, globalAccountName, "foo", time.Second) checkSubInterest(t, l2, globalAccountName, "bar", time.Second) // Now shutdown the leafnode and check that any subscription for $G on l2 are gone. r1.Shutdown() checkFor(t, time.Second, 15*time.Millisecond, func() error { acc := l2.GlobalAccount() if n := acc.TotalSubs(); n != 5 { return fmt.Errorf("Account %q should have 5 subs, got %v", acc.GetName(), n) } return nil }) } func TestLeafNodeLoopDetectionWithMultipleClusters(t *testing.T) { lo1 := DefaultOptions() lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 lo1.Cluster.Name = "local" lo1.Cluster.Host = "127.0.0.1" lo1.Cluster.Port = -1 l1 := RunServer(lo1) defer l1.Shutdown() lo2 := DefaultOptions() lo2.LeafNode.Host = "127.0.0.1" lo2.LeafNode.Port = -1 lo2.Cluster.Name = "local" lo2.Cluster.Host = "127.0.0.1" lo2.Cluster.Port = -1 lo2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", lo1.Cluster.Port)) l2 := RunServer(lo2) defer l2.Shutdown() checkClusterFormed(t, l1, l2) ro1 := DefaultOptions() ro1.Cluster.Name = "remote" ro1.Cluster.Host = "127.0.0.1" ro1.Cluster.Port = -1 ro1.LeafNode.ReconnectInterval = 50 * time.Millisecond ro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ {Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", lo1.LeafNode.Port)}, {Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", lo2.LeafNode.Port)}, }}} r1 := RunServer(ro1) defer r1.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 100)} r1.SetLogger(l, false, false) ro2 := DefaultOptions() ro2.Cluster.Name = "remote" ro2.Cluster.Host = "127.0.0.1" ro2.Cluster.Port = -1 ro2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ro1.Cluster.Port)) ro2.LeafNode.ReconnectInterval = 50 * time.Millisecond ro2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ {Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", lo1.LeafNode.Port)}, {Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", lo2.LeafNode.Port)}, }}} r2 := RunServer(ro2) defer r2.Shutdown() checkClusterFormed(t, r1, r2) checkLeafNodeConnected(t, r1) checkLeafNodeConnected(t, r2) l1.Shutdown() // Now wait for r1 and r2 to reconnect, they should not have a problem of loop detection. checkLeafNodeConnected(t, r1) checkLeafNodeConnected(t, r2) // Wait and make sure we don't have a loop error timeout := time.NewTimer(500 * time.Millisecond) for { select { case err := <-l.errCh: if strings.Contains(err, "Loop detected") { t.Fatal(err) } case <-timeout.C: // OK, we are done. return } } } func TestLeafNodeUnsubOnRouteDisconnect(t *testing.T) { lo1 := DefaultOptions() lo1.LeafNode.Host = "127.0.0.1" lo1.LeafNode.Port = -1 lo1.Cluster.Name = "local" lo1.Cluster.Host = "127.0.0.1" lo1.Cluster.Port = -1 l1 := RunServer(lo1) defer l1.Shutdown() lo2 := DefaultOptions() lo2.LeafNode.Host = "127.0.0.1" lo2.LeafNode.Port = -1 lo2.Cluster.Name = "local" lo2.Cluster.Host = "127.0.0.1" lo2.Cluster.Port = -1 lo2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", lo1.Cluster.Port)) l2 := RunServer(lo2) defer l2.Shutdown() checkClusterFormed(t, l1, l2) u1, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", lo1.LeafNode.Port)) u2, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", lo2.LeafNode.Port)) urls := []*url.URL{u1, u2} ro1 := DefaultOptions() // DefaultOptions sets a cluster name, so make sure they are different. // Also, we don't have r1 and r2 clustered in this test, so set port to 0. ro1.Cluster.Name = _EMPTY_ ro1.Cluster.Port = 0 ro1.LeafNode.ReconnectInterval = 50 * time.Millisecond ro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}} r1 := RunServer(ro1) defer r1.Shutdown() ro2 := DefaultOptions() ro1.Cluster.Name = _EMPTY_ ro2.Cluster.Port = 0 ro2.LeafNode.ReconnectInterval = 50 * time.Millisecond // Have this one point only to l2 ro2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u2}}} r2 := RunServer(ro2) defer r2.Shutdown() checkLeafNodeConnected(t, r1) checkLeafNodeConnected(t, r2) // Create a subscription on r1. nc := natsConnect(t, r1.ClientURL()) defer nc.Close() sub := natsSubSync(t, nc, "foo") natsFlush(t, nc) checkSubInterest(t, l2, globalAccountName, "foo", time.Second) checkSubInterest(t, r2, globalAccountName, "foo", time.Second) nc2 := natsConnect(t, r2.ClientURL()) defer nc2.Close() natsPub(t, nc, "foo", []byte("msg1")) // Check message received natsNexMsg(t, sub, time.Second) // Now shutdown l1, l2 should update subscription interest to r2. // When r1 reconnects to l2, subscription should be updated too. l1.Shutdown() // Wait a bit (so that the check of interest is not OK just because // the route would not have been yet detected as broken), and check // interest still present on r2, l2. time.Sleep(100 * time.Millisecond) checkSubInterest(t, l2, globalAccountName, "foo", time.Second) checkSubInterest(t, r2, globalAccountName, "foo", time.Second) // Check again that message received ok natsPub(t, nc, "foo", []byte("msg2")) natsNexMsg(t, sub, time.Second) // Now close client. Interest should disappear on r2. Due to a bug, // it was not. nc.Close() checkFor(t, time.Second, 15*time.Millisecond, func() error { acc := r2.GlobalAccount() if n := acc.Interest("foo"); n != 0 { return fmt.Errorf("Still interest on subject: %v", n) } return nil }) } func TestLeafNodeNoPingBeforeConnect(t *testing.T) { o := DefaultOptions() o.LeafNode.Port = -1 o.LeafNode.AuthTimeout = 0.5 // For this test we need to disable compression, because we do use // the ping timer instead of the auth timer before the negotiation // is complete. o.LeafNode.Compression.Mode = CompressionOff s := RunServer(o) defer s.Shutdown() addr := fmt.Sprintf("127.0.0.1:%d", o.LeafNode.Port) c, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error on dial: %v", err) } defer c.Close() // Read the info br := bufio.NewReader(c) c.SetReadDeadline(time.Now().Add(time.Second)) l, _, err := br.ReadLine() if err != nil { t.Fatalf("Error on read: %v", err) } if !strings.HasPrefix(string(l), "INFO") { t.Fatalf("Wrong proto: %q", l) } var leaf *client checkFor(t, time.Second, 15*time.Millisecond, func() error { s.grMu.Lock() for _, l := range s.grTmpClients { leaf = l break } s.grMu.Unlock() if leaf == nil { return fmt.Errorf("No leaf connection found") } return nil }) // Make sure that ping timer is not set leaf.mu.Lock() ptmrSet := leaf.ping.tmr != nil leaf.mu.Unlock() if ptmrSet { t.Fatal("Ping timer was set before CONNECT was processed") } // Send CONNECT if _, err := c.Write([]byte("CONNECT {}\r\n")); err != nil { t.Fatalf("Error writing connect: %v", err) } // Check that we correctly set the timer now checkFor(t, time.Second, 15*time.Millisecond, func() error { leaf.mu.Lock() ptmrSet := leaf.ping.tmr != nil leaf.mu.Unlock() if !ptmrSet { return fmt.Errorf("Timer still not set") } return nil }) // Reduce the first ping.. leaf.mu.Lock() leaf.ping.tmr.Reset(15 * time.Millisecond) leaf.mu.Unlock() // Now consume that PING (we may get LS+, etc..) for { c.SetReadDeadline(time.Now().Add(time.Second)) l, _, err = br.ReadLine() if err != nil { t.Fatalf("Error on read: %v", err) } if strings.HasPrefix(string(l), "PING") { checkLeafNodeConnected(t, s) return } } } func TestLeafNodeNoMsgLoop(t *testing.T) { hubConf := ` listen: "127.0.0.1:-1" accounts { FOO { users [ {username: leaf, password: pass} {username: user, password: pass} ] } } cluster { name: "hub" listen: "127.0.0.1:-1" %s } leafnodes { listen: "127.0.0.1:-1" authorization { account: FOO } } ` configS1 := createConfFile(t, []byte(fmt.Sprintf(hubConf, ""))) s1, o1 := RunServerWithConfig(configS1) defer s1.Shutdown() configS2S3 := createConfFile(t, []byte(fmt.Sprintf(hubConf, fmt.Sprintf(`routes: ["nats://127.0.0.1:%d"]`, o1.Cluster.Port)))) s2, o2 := RunServerWithConfig(configS2S3) defer s2.Shutdown() s3, _ := RunServerWithConfig(configS2S3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) contentLN := ` listen: "127.0.0.1:%d" accounts { FOO { users [ {username: leaf, password: pass} {username: user, password: pass} ] } } leafnodes { remotes = [ { url: "nats://leaf:pass@127.0.0.1:%d" account: FOO } ] } ` lnconf := createConfFile(t, []byte(fmt.Sprintf(contentLN, -1, o1.LeafNode.Port))) sl1, slo1 := RunServerWithConfig(lnconf) defer sl1.Shutdown() sl2, slo2 := RunServerWithConfig(lnconf) defer sl2.Shutdown() checkLeafNodeConnected(t, sl1) checkLeafNodeConnected(t, sl2) // Create users on each leafnode nc1, err := nats.Connect(fmt.Sprintf("nats://user:pass@127.0.0.1:%d", slo1.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() rch := make(chan struct{}, 1) nc2, err := nats.Connect( fmt.Sprintf("nats://user:pass@127.0.0.1:%d", slo2.Port), nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(func(_ *nats.Conn) { rch <- struct{}{} }), ) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Create queue subs on sl2 nc2.QueueSubscribe("foo", "bar", func(_ *nats.Msg) {}) nc2.QueueSubscribe("foo", "bar", func(_ *nats.Msg) {}) nc2.Flush() // Wait for interest to propagate to sl1 checkSubInterest(t, sl1, "FOO", "foo", 250*time.Millisecond) // Create sub on sl1 ch := make(chan *nats.Msg, 10) nc1.Subscribe("foo", func(m *nats.Msg) { select { case ch <- m: default: } }) nc1.Flush() checkSubInterest(t, sl2, "FOO", "foo", 250*time.Millisecond) // Produce from sl1 nc1.Publish("foo", []byte("msg1")) // Check message is received by plain sub select { case <-ch: case <-time.After(time.Second): t.Fatalf("Did not receive message") } // Restart leaf node, this time make sure we connect to 2nd server. sl2.Shutdown() // Use config file but this time reuse the client port and set the 2nd server for // the remote leaf node port. lnconf = createConfFile(t, []byte(fmt.Sprintf(contentLN, slo2.Port, o2.LeafNode.Port))) sl2, _ = RunServerWithConfig(lnconf) defer sl2.Shutdown() checkLeafNodeConnected(t, sl2) // Wait for client to reconnect select { case <-rch: case <-time.After(time.Second): t.Fatalf("Did not reconnect") } // Produce a new messages for i := 0; i < 10; i++ { nc1.Publish("foo", []byte(fmt.Sprintf("msg%d", 2+i))) // Check sub receives 1 message select { case <-ch: case <-time.After(time.Second): t.Fatalf("Did not receive message") } // Check that there is no more... select { case m := <-ch: t.Fatalf("Loop: received second message %s", m.Data) case <-time.After(50 * time.Millisecond): // OK } } } func TestLeafNodeInterestPropagationDaisychain(t *testing.T) { aTmpl := ` port: %d leafnodes { port: %d } ` confA := createConfFile(t, []byte(fmt.Sprintf(aTmpl, -1, -1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() aPort := sA.opts.Port aLeafPort := sA.opts.LeafNode.Port confB := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { port: -1 remotes = [{ url:"nats://127.0.0.1:%d" }] }`, aLeafPort))) sB, _ := RunServerWithConfig(confB) defer sB.Shutdown() confC := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { port: -1 remotes = [{url:"nats://127.0.0.1:%d"}] }`, sB.opts.LeafNode.Port))) sC, _ := RunServerWithConfig(confC) defer sC.Shutdown() checkLeafNodeConnectedCount(t, sC, 1) checkLeafNodeConnectedCount(t, sB, 2) checkLeafNodeConnectedCount(t, sA, 1) ncC := natsConnect(t, sC.ClientURL()) defer ncC.Close() _, err := ncC.SubscribeSync("foo") require_NoError(t, err) require_NoError(t, ncC.Flush()) checkSubInterest(t, sC, "$G", "foo", time.Second) checkSubInterest(t, sB, "$G", "foo", time.Second) checkSubInterest(t, sA, "$G", "foo", time.Second) ncA := natsConnect(t, sA.ClientURL()) defer ncA.Close() sA.Shutdown() sA.WaitForShutdown() confAA := createConfFile(t, []byte(fmt.Sprintf(aTmpl, aPort, aLeafPort))) sAA, _ := RunServerWithConfig(confAA) defer sAA.Shutdown() checkLeafNodeConnectedCount(t, sAA, 1) checkLeafNodeConnectedCount(t, sB, 2) checkLeafNodeConnectedCount(t, sC, 1) checkSubInterest(t, sC, "$G", "foo", time.Second) checkSubInterest(t, sB, "$G", "foo", time.Second) checkSubInterest(t, sAA, "$G", "foo", time.Second) // failure issue 2448 } func TestLeafNodeQueueGroupDistribution(t *testing.T) { hc := createClusterWithName(t, "HUB", 3) defer hc.shutdown() // Now have a cluster of leafnodes with each one connecting to corresponding HUB(n) node. c1 := ` server_name: LEAF1 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1 } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, hc.opts[0].LeafNode.Port))) ln1, lopts1 := RunServerWithConfig(lconf1) defer ln1.Shutdown() c2 := ` server_name: LEAF2 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, hc.opts[1].LeafNode.Port))) ln2, _ := RunServerWithConfig(lconf2) defer ln2.Shutdown() c3 := ` server_name: LEAF3 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, hc.opts[2].LeafNode.Port))) ln3, _ := RunServerWithConfig(lconf3) defer ln3.Shutdown() // Check leaf cluster is formed and all connected to the HUB. lnServers := []*Server{ln1, ln2, ln3} checkClusterFormed(t, lnServers...) for _, s := range lnServers { checkLeafNodeConnected(t, s) } // Check each node in the hub has 1 connection from the leaf cluster. for i := 0; i < 3; i++ { checkLeafNodeConnectedCount(t, hc.servers[i], 1) } // Create a client and qsub on LEAF1 and LEAF2. nc1 := natsConnect(t, ln1.ClientURL()) defer nc1.Close() var qsub1Count atomic.Int32 natsQueueSub(t, nc1, "foo", "queue1", func(_ *nats.Msg) { qsub1Count.Add(1) }) natsFlush(t, nc1) nc2 := natsConnect(t, ln2.ClientURL()) defer nc2.Close() var qsub2Count atomic.Int32 natsQueueSub(t, nc2, "foo", "queue1", func(_ *nats.Msg) { qsub2Count.Add(1) }) natsFlush(t, nc2) // Make sure that the propagation interest is done before sending. for _, s := range hc.servers { gacc := s.GlobalAccount() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := gacc.Interest("foo"); n != 2 { return fmt.Errorf("Expected interest for %q to be 2, got %v", "foo", n) } return nil }) } sendAndCheck := func(idx int) { t.Helper() nchub := natsConnect(t, hc.servers[idx].ClientURL()) defer nchub.Close() total := 1000 for i := 0; i < total; i++ { natsPub(t, nchub, "foo", []byte("from hub")) } checkFor(t, time.Second, 15*time.Millisecond, func() error { if trecv := int(qsub1Count.Load() + qsub2Count.Load()); trecv != total { return fmt.Errorf("Expected %v messages, got %v", total, trecv) } return nil }) // Now that we have made sure that all messages were received, // check that qsub1 and qsub2 are getting at least some. if n := int(qsub1Count.Load()); n <= total/10 { t.Fatalf("Expected qsub1 to get some messages, but got %v", n) } if n := int(qsub2Count.Load()); n <= total/10 { t.Fatalf("Expected qsub2 to get some messages, but got %v", n) } // Reset the counters. qsub1Count.Store(0) qsub2Count.Store(0) } // Send from HUB1 sendAndCheck(0) // Send from HUB2 sendAndCheck(1) // Send from HUB3 sendAndCheck(2) } func TestLeafNodeQueueGroupDistributionVariant(t *testing.T) { hc := createClusterWithName(t, "HUB", 3) defer hc.shutdown() // Now have a cluster of leafnodes with LEAF1 and LEAF2 connecting to HUB1. c1 := ` server_name: LEAF1 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1 } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, hc.opts[0].LeafNode.Port))) ln1, lopts1 := RunServerWithConfig(lconf1) defer ln1.Shutdown() c2 := ` server_name: LEAF2 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, hc.opts[0].LeafNode.Port))) ln2, _ := RunServerWithConfig(lconf2) defer ln2.Shutdown() // And LEAF3 to HUB3 c3 := ` server_name: LEAF3 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, hc.opts[2].LeafNode.Port))) ln3, _ := RunServerWithConfig(lconf3) defer ln3.Shutdown() // Check leaf cluster is formed and all connected to the HUB. lnServers := []*Server{ln1, ln2, ln3} checkClusterFormed(t, lnServers...) for _, s := range lnServers { checkLeafNodeConnected(t, s) } // Check that HUB1 has 2 leaf connections, HUB2 has 0 and HUB3 has 1. checkLeafNodeConnectedCount(t, hc.servers[0], 2) checkLeafNodeConnectedCount(t, hc.servers[1], 0) checkLeafNodeConnectedCount(t, hc.servers[2], 1) // Create a client and qsub on LEAF1 and LEAF2. nc1 := natsConnect(t, ln1.ClientURL()) defer nc1.Close() var qsub1Count atomic.Int32 natsQueueSub(t, nc1, "foo", "queue1", func(_ *nats.Msg) { qsub1Count.Add(1) }) natsFlush(t, nc1) nc2 := natsConnect(t, ln2.ClientURL()) defer nc2.Close() var qsub2Count atomic.Int32 natsQueueSub(t, nc2, "foo", "queue1", func(_ *nats.Msg) { qsub2Count.Add(1) }) natsFlush(t, nc2) // Make sure that the propagation interest is done before sending. for i, s := range hc.servers { gacc := s.GlobalAccount() var ei int switch i { case 0: ei = 2 default: ei = 1 } checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := gacc.Interest("foo"); n != ei { return fmt.Errorf("Expected interest for %q to be %d, got %v", "foo", ei, n) } return nil }) } sendAndCheck := func(idx int) { t.Helper() nchub := natsConnect(t, hc.servers[idx].ClientURL()) defer nchub.Close() total := 1000 for i := 0; i < total; i++ { natsPub(t, nchub, "foo", []byte("from hub")) } checkFor(t, time.Second, 15*time.Millisecond, func() error { if trecv := int(qsub1Count.Load() + qsub2Count.Load()); trecv != total { return fmt.Errorf("Expected %v messages, got %v", total, trecv) } return nil }) // Now that we have made sure that all messages were received, // check that qsub1 and qsub2 are getting at least some. if n := int(qsub1Count.Load()); n <= total/10 { t.Fatalf("Expected qsub1 to get some messages, but got %v (qsub2=%v)", n, qsub2Count.Load()) } if n := int(qsub2Count.Load()); n <= total/10 { t.Fatalf("Expected qsub2 to get some messages, but got %v (qsub1=%v)", n, qsub1Count.Load()) } // Reset the counters. qsub1Count.Store(0) qsub2Count.Store(0) } // Send from HUB1 sendAndCheck(0) // Send from HUB2 sendAndCheck(1) // Send from HUB3 sendAndCheck(2) } func TestLeafNodeQueueGroupDistributionWithDaisyChainAndGateway(t *testing.T) { SetGatewaysSolicitDelay(0) defer ResetGatewaysSolicitDelay() // We create a sort of a ladder of servers with connections that look like this: // // D1 <--- route ---> D2 // | | // GW GW // | | // C1 <--- route ---> C2 // | | // Leaf Leaf // | | // B1 <--- route ---> B2 // | | // Leaf Leaf // | | // A1 <--- route ---> A2 // // We will then place queue subscriptions (different sub-tests) on A1, A2 // B1, B2, D1 and D2. accs := ` accounts { SYS: {users: [{user:sys, password: pwd}]} USER: {users: [{user:user, password: pwd}]} } system_account: SYS ` dConf := ` %s server_name: %s port: -1 cluster { name: "D" port: -1 %s } gateway { name: "D" port: -1 } ` d1Conf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, "GW1", _EMPTY_))) d1, d1Opts := RunServerWithConfig(d1Conf) defer d1.Shutdown() d2Conf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, "GW2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", d1Opts.Cluster.Port)))) d2, d2Opts := RunServerWithConfig(d2Conf) defer d2.Shutdown() checkClusterFormed(t, d1, d2) leafCConf := ` %s server_name: %s port: -1 cluster { name: C port: -1 %s } leafnodes { port: -1 } gateway { name: C port: -1 gateways [ { name: D url: "nats://127.0.0.1:%d" } ] } ` c1Conf := createConfFile(t, []byte(fmt.Sprintf(leafCConf, accs, "C1", _EMPTY_, d1Opts.Gateway.Port))) c1, c1Opts := RunServerWithConfig(c1Conf) defer c1.Shutdown() waitForOutboundGateways(t, c1, 1, time.Second) waitForInboundGateways(t, d1, 1, time.Second) c2Conf := createConfFile(t, []byte(fmt.Sprintf(leafCConf, accs, "C2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", c1Opts.Cluster.Port), d2Opts.Gateway.Port))) c2, c2Opts := RunServerWithConfig(c2Conf) defer c2.Shutdown() waitForOutboundGateways(t, c2, 1, time.Second) waitForInboundGateways(t, d2, 1, time.Second) checkClusterFormed(t, c1, c2) leafABConf := ` %s server_name: %s port: -1 cluster { name: %s port: -1 %s } leafnodes { port: -1 remotes [ { url: "nats://user:pwd@127.0.0.1:%d" account: USER } ] } ` b1Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, "B1", "B", _EMPTY_, c1Opts.LeafNode.Port))) b1, b1Opts := RunServerWithConfig(b1Conf) defer b1.Shutdown() checkLeafNodeConnected(t, b1) checkLeafNodeConnected(t, c1) b2Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, "B2", "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", b1Opts.Cluster.Port), c2Opts.LeafNode.Port))) b2, b2Opts := RunServerWithConfig(b2Conf) defer b2.Shutdown() checkLeafNodeConnected(t, b2) checkLeafNodeConnected(t, c2) checkClusterFormed(t, b1, b2) a1Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, "A1", "A", _EMPTY_, b1Opts.LeafNode.Port))) a1, a1Opts := RunServerWithConfig(a1Conf) defer a1.Shutdown() checkLeafNodeConnectedCount(t, b1, 2) checkLeafNodeConnected(t, a1) a2Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, "A2", "A", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", a1Opts.Cluster.Port), b2Opts.LeafNode.Port))) a2, _ := RunServerWithConfig(a2Conf) defer a2.Shutdown() checkLeafNodeConnectedCount(t, b2, 2) checkLeafNodeConnected(t, a2) checkClusterFormed(t, a1, a2) // Create our client connections to all servers where we may need to have // queue subscriptions. ncD1 := natsConnect(t, d1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncD1.Close() ncD2 := natsConnect(t, d2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncD2.Close() ncB1 := natsConnect(t, b1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncB1.Close() ncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncB2.Close() ncA1 := natsConnect(t, a1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncA1.Close() ncA2 := natsConnect(t, a2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncA2.Close() // Helper to check that the interest is propagated to all servers checkInterest := func(t *testing.T, subj string) { t.Helper() for _, s := range []*Server{a1, a2, b1, b2, c1, c2, d1, d2} { acc, err := s.LookupAccount("USER") require_NoError(t, err) checkFor(t, time.Second, 10*time.Millisecond, func() error { if acc.Interest(subj) != 0 { return nil } return fmt.Errorf("Still no interest on %q in server %q", subj, s) }) } } // Helper to send messages on given subject. We are always sending // from cluster B in this test, but we pick randomly between B1 and B2. total := 1000 send := func(t *testing.T, subj string) { for i := 0; i < total; i++ { var nc *nats.Conn if fastrand.Uint32n(2) == 0 { nc = ncB1 } else { nc = ncB2 } natsPub(t, nc, subj, []byte(fmt.Sprintf("msg_%d", i+1))) } } const queue = "queue" for i, test := range []struct { name string a1 bool a2 bool b1 bool b2 bool d1 bool d2 bool }{ // Cases with QSubs in A, B and D {"A1 __ B1 __ D1 __", true, false, true, false, true, false}, {"A1 __ B1 __ __ D2", true, false, true, false, false, true}, {"A1 __ B1 __ D1 D2", true, false, true, false, true, true}, {"A1 __ __ B2 D1 __", true, false, false, true, true, false}, {"A1 __ __ B2 __ D2", true, false, false, true, false, true}, {"A1 __ __ B2 D1 D2", true, false, false, true, true, true}, {"A1 __ B1 B2 D1 __", true, false, true, true, true, false}, {"A1 __ B1 B2 __ D2", true, false, true, true, false, true}, {"A1 __ B1 B2 D1 D2", true, false, true, true, true, true}, {"__ A2 B1 __ D1 __", false, true, true, false, true, false}, {"__ A2 B1 __ __ D2", false, true, true, false, false, true}, {"__ A2 B1 __ D1 D2", false, true, true, false, true, true}, {"__ A2 __ B2 D1 __", false, true, false, true, true, false}, {"__ A2 __ B2 __ D2", false, true, false, true, false, true}, {"__ A2 __ B2 D1 D2", false, true, false, true, true, true}, {"__ A2 B1 B2 D1 __", false, true, true, true, true, false}, {"__ A2 B1 B2 __ D2", false, true, true, true, false, true}, {"__ A2 B1 B2 D1 D2", false, true, true, true, true, true}, {"A1 A2 B1 __ D1 __", true, true, true, false, true, false}, {"A1 A2 B1 __ __ D2", true, true, true, false, false, true}, {"A1 A2 B1 __ D1 D2", true, true, true, false, true, true}, {"A1 A2 __ B2 D1 __", true, true, false, true, true, false}, {"A1 A2 __ B2 __ D2", true, true, false, true, false, true}, {"A1 A2 __ B2 D1 D2", true, true, false, true, true, true}, {"A1 A2 B1 B2 D1 __", true, true, true, true, true, false}, {"A1 A2 B1 B2 __ D2", true, true, true, true, false, true}, {"A1 A2 B1 B2 D1 D2", true, true, true, true, true, true}, // Now without any QSub in B cluster (so just A and D) {"A1 __ __ __ D1 __", true, false, false, false, true, false}, {"A1 __ __ __ __ D2", true, false, false, false, false, true}, {"A1 __ __ __ D1 D2", true, false, false, false, true, true}, {"__ A2 __ __ D1 __", false, true, false, false, true, false}, {"__ A2 __ __ __ D2", false, true, false, false, false, true}, {"__ A2 __ __ D1 D2", false, true, false, false, true, true}, {"A1 A2 __ __ D1 __", true, true, false, false, true, false}, {"A1 A2 __ __ __ D2", true, true, false, false, false, true}, {"A1 A2 __ __ D1 D2", true, true, false, false, true, true}, } { t.Run(test.name, func(t *testing.T) { subj := fmt.Sprintf("foo.%d", i+1) var aCount, bCount, dCount atomic.Int32 if test.a1 { qsA1 := natsQueueSub(t, ncA1, subj, queue, func(_ *nats.Msg) { aCount.Add(1) }) defer qsA1.Unsubscribe() } if test.a2 { qsA2 := natsQueueSub(t, ncA2, subj, queue, func(_ *nats.Msg) { aCount.Add(1) }) defer qsA2.Unsubscribe() } if test.b1 { qsB1 := natsQueueSub(t, ncB1, subj, queue, func(_ *nats.Msg) { bCount.Add(1) }) defer qsB1.Unsubscribe() } if test.b2 { qsB2 := natsQueueSub(t, ncB2, subj, queue, func(_ *nats.Msg) { bCount.Add(1) }) defer qsB2.Unsubscribe() } if test.d1 { qsD1 := natsQueueSub(t, ncD1, subj, queue, func(_ *nats.Msg) { dCount.Add(1) }) defer qsD1.Unsubscribe() } if test.d2 { qsD2 := natsQueueSub(t, ncD2, subj, queue, func(_ *nats.Msg) { dCount.Add(1) }) defer qsD2.Unsubscribe() } checkInterest(t, subj) // Now send messages send(t, subj) // Check that appropriate queue subs receive all messages. checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { n := aCount.Load() + bCount.Load() + dCount.Load() if n == int32(total) { return nil } return fmt.Errorf("Got only %v/%v messages (a=%v b=%v d=%v)", n, total, aCount.Load(), bCount.Load(), dCount.Load()) }) // When there is (are) qsub(s) on b, then only B should // get the messages. Otherwise, it should be between A and D if test.b1 || test.b2 { require_Equal(t, aCount.Load(), 0) require_Equal(t, dCount.Load(), 0) } else { require_Equal(t, bCount.Load(), 0) // We should have receive some on A and D require_True(t, aCount.Load() > 0) require_True(t, dCount.Load() > 0) } }) } } func TestLeafNodeAndGatewaysSingleMsgPerQueueGroup(t *testing.T) { SetGatewaysSolicitDelay(0) defer ResetGatewaysSolicitDelay() accs := ` accounts { SYS: {users: [{user:sys, password: pwd}]} USER: {users: [{user:user, password: pwd}]} } system_account: SYS ` gwUSConfTmpl := ` %s server_name: GW_US listen: "127.0.0.1:-1" gateway { name: US listen: "127.0.0.1:-1" } leafnodes { listen: "127.0.0.1:-1" } ` gwUSConf := createConfFile(t, []byte(fmt.Sprintf(gwUSConfTmpl, accs))) gwUS, gwUSOpts := RunServerWithConfig(gwUSConf) defer gwUS.Shutdown() gwEUConfTmpl := ` %s server_name: GW_EU listen: "127.0.0.1:-1" gateway { name: EU listen: "127.0.0.1:-1" gateways [ { name: US url: "nats://127.0.0.1:%d" } ] } leafnodes { listen: "127.0.0.1:-1" } ` gwEUConf := createConfFile(t, []byte(fmt.Sprintf(gwEUConfTmpl, accs, gwUSOpts.Gateway.Port))) gwEU, gwEUOpts := RunServerWithConfig(gwEUConf) defer gwEU.Shutdown() waitForOutboundGateways(t, gwUS, 1, time.Second) waitForOutboundGateways(t, gwEU, 1, time.Second) waitForInboundGateways(t, gwUS, 1, time.Second) waitForInboundGateways(t, gwEU, 1, time.Second) leafConfTmpl := ` %s server_name: %s listen: "127.0.0.1:-1" leafnodes { remotes [ { url: "nats://user:pwd@127.0.0.1:%d" account: USER } ] } ` leafUSConf := createConfFile(t, []byte(fmt.Sprintf(leafConfTmpl, accs, "LEAF_US", gwUSOpts.LeafNode.Port))) leafUS, _ := RunServerWithConfig(leafUSConf) defer leafUS.Shutdown() checkLeafNodeConnected(t, leafUS) leafEUConf := createConfFile(t, []byte(fmt.Sprintf(leafConfTmpl, accs, "LEAF_EU", gwEUOpts.LeafNode.Port))) leafEU, _ := RunServerWithConfig(leafEUConf) defer leafEU.Shutdown() checkLeafNodeConnected(t, leafEU) // Order is important! (see rest of test to understand why) var usLeafQ, usLeafPS, usGWQ, usGWPS, euGWQ, euGWPS, euLeafQ, euLeafPS, euLeafQBaz atomic.Int32 counters := []*atomic.Int32{&usLeafQ, &usLeafPS, &usGWQ, &usGWPS, &euGWQ, &euGWPS, &euLeafQ, &euLeafPS, &euLeafQBaz} counterNames := []string{"usLeafQ", "usLeafPS", "usGWQ", "usGWPS", "euGWQ", "euGWPS", "euLeafQ", "euLeafPS", "euLeafQBaz"} if len(counters) != len(counterNames) { panic("Fix test!") } resetCounters := func() { for _, a := range counters { a.Store(0) } } // This test will always produce from the US leaf. ncProd := natsConnect(t, leafUS.ClientURL(), nats.UserInfo("user", "pwd")) defer ncProd.Close() total := int32(1) check := func(t *testing.T, expected []int32) { time.Sleep(50 * time.Millisecond) resetCounters() for i := 0; i < int(total); i++ { natsPub(t, ncProd, "foo.1", []byte("hello")) } checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for i := 0; i < len(expected); i++ { if n := counters[i].Load(); n != expected[i] { return fmt.Errorf("Expected counter %q to be %v, got %v", counterNames[i], expected[i], n) } } return nil }) } // "usLeafQ", "usLeafPS", "usGWQ", "usGWPS", "euGWQ", "euGWPS", "euLeafQ", "euLeafPS", "euLeafQBaz" for _, test := range []struct { subs []int expected []int32 }{ // We will always have the qsub on leaf EU, and have some permutations // of queue and plain subs on other server(s) and check we get the // expected distribution. // Simple test firs, qsubs on leaf US and leaf EU, all messages stay in leaf US. { []int{1, 0, 0, 0, 0, 0, 1, 0, 0}, []int32{total, 0, 0, 0, 0, 0, 0, 0, 0}, }, // Move the queue sub from leaf US to GW US. { []int{0, 0, 1, 0, 0, 0, 1, 0, 0}, []int32{0, 0, total, 0, 0, 0, 0, 0, 0}, }, // Now move it to GW EU. { []int{0, 0, 0, 0, 1, 0, 1, 0, 0}, []int32{0, 0, 0, 0, total, 0, 0, 0, 0}, }, // More combinations... { []int{1, 1, 0, 0, 0, 0, 1, 0, 0}, []int32{total, total, 0, 0, 0, 0, 0, 0, 0}, }, { []int{0, 1, 1, 0, 0, 0, 1, 0, 0}, []int32{0, total, total, 0, 0, 0, 0, 0, 0}, }, { []int{0, 1, 1, 1, 0, 0, 1, 0, 0}, []int32{0, total, total, total, 0, 0, 0, 0, 0}, }, { []int{0, 1, 0, 1, 1, 0, 1, 0, 0}, []int32{0, total, 0, total, total, 0, 0, 0, 0}, }, { []int{0, 1, 0, 1, 1, 1, 1, 0, 0}, []int32{0, total, 0, total, total, total, 0, 0, 0}, }, // If we have the qsub in leaf US, does not matter if we have // qsubs in GW US and EU, only leaf US should receive the messages, // but plain sub in GW servers should get them too. { []int{1, 1, 1, 0, 0, 0, 1, 0, 0}, []int32{total, total, 0, 0, 0, 0, 0, 0, 0}, }, { []int{1, 1, 1, 1, 0, 0, 1, 0, 0}, []int32{total, total, 0, total, 0, 0, 0, 0, 0}, }, { []int{1, 1, 1, 1, 1, 0, 1, 0, 0}, []int32{total, total, 0, total, 0, 0, 0, 0, 0}, }, { []int{1, 1, 1, 1, 1, 1, 1, 0, 0}, []int32{total, total, 0, total, 0, total, 0, 0, 0}, }, // Now back to a qsub on leaf US and leaf EU, but introduce plain sub // interest in leaf EU { []int{1, 0, 0, 0, 0, 0, 1, 1, 0}, []int32{total, 0, 0, 0, 0, 0, 0, total, 0}, }, // And add a different queue group in leaf EU and it should get the messages too. { []int{1, 0, 0, 0, 0, 0, 1, 1, 1}, []int32{total, 0, 0, 0, 0, 0, 0, total, total}, }, // Keep plain and baz queue sub interests in leaf EU and add more combinations. { []int{1, 1, 0, 0, 0, 0, 1, 1, 1}, []int32{total, total, 0, 0, 0, 0, 0, total, total}, }, { []int{1, 1, 1, 0, 0, 0, 1, 1, 1}, []int32{total, total, 0, 0, 0, 0, 0, total, total}, }, { []int{1, 1, 1, 1, 0, 0, 1, 1, 1}, []int32{total, total, 0, total, 0, 0, 0, total, total}, }, { []int{1, 1, 1, 1, 1, 0, 1, 1, 1}, []int32{total, total, 0, total, 0, 0, 0, total, total}, }, { []int{1, 1, 1, 1, 1, 1, 1, 1, 1}, []int32{total, total, 0, total, 0, total, 0, total, total}, }, } { t.Run(_EMPTY_, func(t *testing.T) { if len(test.subs) != len(counters) || len(test.expected) != len(counters) { panic("Fix test") } ncUS := natsConnect(t, leafUS.ClientURL(), nats.UserInfo("user", "pwd")) defer ncUS.Close() ncGWUS := natsConnect(t, gwUS.ClientURL(), nats.UserInfo("user", "pwd")) defer ncGWUS.Close() ncGWEU := natsConnect(t, gwEU.ClientURL(), nats.UserInfo("user", "pwd")) defer ncGWEU.Close() ncEU := natsConnect(t, leafEU.ClientURL(), nats.UserInfo("user", "pwd")) defer ncEU.Close() if test.subs[0] > 0 { natsQueueSub(t, ncUS, "foo.*", "bar", func(_ *nats.Msg) { usLeafQ.Add(1) }) } if test.subs[1] > 0 { natsSub(t, ncUS, "foo.>", func(_ *nats.Msg) { usLeafPS.Add(1) }) } natsFlush(t, ncUS) if test.subs[2] > 0 { natsQueueSub(t, ncGWUS, "foo.*", "bar", func(_ *nats.Msg) { usGWQ.Add(1) }) } if test.subs[3] > 0 { natsSub(t, ncGWUS, "foo.>", func(_ *nats.Msg) { usGWPS.Add(1) }) } natsFlush(t, ncGWUS) if test.subs[4] > 0 { natsQueueSub(t, ncGWEU, "foo.*", "bar", func(_ *nats.Msg) { euGWQ.Add(1) }) } if test.subs[5] > 0 { natsSub(t, ncGWEU, "foo.>", func(_ *nats.Msg) { euGWPS.Add(1) }) } natsFlush(t, ncGWEU) if test.subs[6] > 0 { natsQueueSub(t, ncEU, "foo.*", "bar", func(_ *nats.Msg) { euLeafQ.Add(1) }) } if test.subs[7] > 0 { natsSub(t, ncEU, "foo.>", func(_ *nats.Msg) { euLeafPS.Add(1) }) } if test.subs[8] > 0 { // Create on different group, called "baz" natsQueueSub(t, ncEU, "foo.*", "baz", func(_ *nats.Msg) { euLeafQBaz.Add(1) }) } natsFlush(t, ncEU) // Check that we have what we expect. check(t, test.expected) }) } } func TestLeafNodeQueueGroupWeightCorrectOnConnectionCloseInSuperCluster(t *testing.T) { SetGatewaysSolicitDelay(0) defer ResetGatewaysSolicitDelay() // // D // | // Leaf // | // v // C // ^ ^ // / \ // GW GW // / \ // v \ // B1 <--- route ---> B2 <----*----------* // ^ <---* | | // | | Leaf Leaf // Leaf *-- Leaf ---* | | // | | | | // A1 <--- route ---> A2 OTHER1 OTHER2 // accs := ` accounts { SYS: {users: [{user:sys, password: pwd}]} USER: {users: [{user:user, password: pwd}]} } system_account: SYS ` bConf := ` %s server_name: %s listen: "127.0.0.1:-1" cluster { name: "B" listen: "127.0.0.1:-1" no_advertise: true %s } gateway { name: "B" listen: "127.0.0.1:-1" } leafnodes { listen: "127.0.0.1:-1" } ` sb1Conf := createConfFile(t, []byte(fmt.Sprintf(bConf, accs, "B1", _EMPTY_))) sb1, sb1o := RunServerWithConfig(sb1Conf) defer sb1.Shutdown() sb2Conf := createConfFile(t, []byte(fmt.Sprintf(bConf, accs, "B2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", sb1o.Cluster.Port)))) sb2, sb2o := RunServerWithConfig(sb2Conf) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) cConf := ` %s server_name: C listen: "127.0.0.1:-1" cluster { name: "C" listen: "127.0.0.1:-1" } gateway { name: "C" listen: "127.0.0.1:-1" gateways [ { name: B url: "nats://127.0.0.1:%d" } ] } leafnodes { listen: "127.0.0.1:-1" } ` scConf := createConfFile(t, []byte(fmt.Sprintf(cConf, accs, sb1o.Gateway.Port))) sc, sco := RunServerWithConfig(scConf) defer sc.Shutdown() waitForOutboundGateways(t, sc, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sc, 2, 2*time.Second) waitForInboundGateways(t, sb1, 1, 2*time.Second) dConf := ` %s server_name: D listen: "127.0.0.1:-1" cluster { name: "D" listen: "127.0.0.1:-1" } leafnodes { remotes [ { url: "nats://user:pwd@127.0.0.1:%d" account: USER } ] } ` sdConf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, sco.LeafNode.Port))) sd, _ := RunServerWithConfig(sdConf) defer sd.Shutdown() checkLeafNodeConnected(t, sc) checkLeafNodeConnected(t, sd) aConf := ` %s server_name: %s listen: "127.0.0.1:-1" cluster { name: A listen: "127.0.0.1:-1" no_advertise: true %s } leafnodes { remotes [ { url: "nats://user:pwd@127.0.0.1:%d" account: USER } ] } ` a1Conf := createConfFile(t, []byte(fmt.Sprintf(aConf, accs, "A1", _EMPTY_, sb1o.LeafNode.Port))) sa1, sa1o := RunServerWithConfig(a1Conf) defer sa1.Shutdown() checkLeafNodeConnected(t, sa1) checkLeafNodeConnected(t, sb1) a2Conf := createConfFile(t, []byte(fmt.Sprintf(aConf, accs, "A2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", sa1o.Cluster.Port), sb1o.LeafNode.Port))) sa2, _ := RunServerWithConfig(a2Conf) defer sa2.Shutdown() checkClusterFormed(t, sa1, sa2) checkLeafNodeConnected(t, sa2) checkLeafNodeConnectedCount(t, sb1, 2) otherLeafsConf := ` %s server_name: %s listen: "127.0.0.1:-1" leafnodes { remotes [ { url: "nats://user:pwd@127.0.0.1:%d" account: USER } ] } ` o1Conf := createConfFile(t, []byte(fmt.Sprintf(otherLeafsConf, accs, "OTHERLEAF1", sb2o.LeafNode.Port))) so1, _ := RunServerWithConfig(o1Conf) defer so1.Shutdown() checkLeafNodeConnected(t, so1) checkLeafNodeConnectedCount(t, sb2, 1) o2Conf := createConfFile(t, []byte(fmt.Sprintf(otherLeafsConf, accs, "OTHERLEAF2", sb2o.LeafNode.Port))) so2, _ := RunServerWithConfig(o2Conf) defer so2.Shutdown() checkLeafNodeConnected(t, so2) checkLeafNodeConnectedCount(t, sb2, 2) // Helper to check that the interest is propagated to all servers checkInterest := func(t *testing.T, expected []int, expectedGW int32) { t.Helper() subj := "foo" for i, s := range []*Server{sa1, sa2, so1, so2, sb1, sb2, sc, sd} { if s == sc || !s.isRunning() { continue } acc, err := s.LookupAccount("USER") require_NoError(t, err) checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { n := acc.Interest(subj) if n == expected[i] { return nil } return fmt.Errorf("Expected interest count for server %q to be %v, got %v", s, expected[i], n) }) } // For server C, need to check in gateway's account. checkForRegisteredQSubInterest(t, sc, "B", "USER", "foo", expected[6], time.Second) // For server B1 and B2, check that we have the proper counts in the map. for _, s := range []*Server{sb1, sb2} { if !s.isRunning() { continue } checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { s.gateway.pasi.Lock() accMap := s.gateway.pasi.m st := accMap["USER"] var n int32 entry, ok := st["foo bar"] if ok { n = entry.n } s.gateway.pasi.Unlock() if n == expectedGW { return nil } return fmt.Errorf("Expected GW interest count for server %q to be %v, got %v", s, expectedGW, n) }) } } ncA1 := natsConnect(t, sa1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncA1.Close() for i := 0; i < 3; i++ { natsQueueSubSync(t, ncA1, "foo", "bar") } natsFlush(t, ncA1) // With 3 queue subs on A1, we should have for servers (in order checked in checkInterest) // for A1: 3 locals, for all others, 1 for the remote sub from A1. // B1 and B2 GW map will be 3 (1 for each sub) checkInterest(t, []int{3, 1, 1, 1, 1, 1, 1, 1}, 3) ncA2 := natsConnect(t, sa2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncA2.Close() ncA2qsub1 := natsQueueSubSync(t, ncA2, "foo", "bar") ncA2qsub2 := natsQueueSubSync(t, ncA2, "foo", "bar") natsFlush(t, ncA2) // A1 will have 1 more for remote sub, same for A2 (2 locals + 1 remote). // B1 will have 2 interest (1 per leaf connection) // B1 and B2 GW map goes to 5. checkInterest(t, []int{4, 3, 1, 1, 2, 1, 1, 1}, 5) ncOther1 := natsConnect(t, so1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncOther1.Close() natsQueueSubSync(t, ncOther1, "foo", "bar") natsQueueSubSync(t, ncOther1, "foo", "bar") natsFlush(t, ncOther1) // A1, A2 will have one more because of routed interest // O1 will have 3 (2 locals + 1 for remote interest) // O2 has still 1 for remote interest // B1 has 1 more because of new leaf interest and B2 because of routed interest. // B1 and B2 GW map goes to 7. checkInterest(t, []int{5, 4, 3, 1, 3, 2, 1, 1}, 7) ncOther2 := natsConnect(t, so2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncOther2.Close() natsQueueSubSync(t, ncOther2, "foo", "bar") natsFlush(t, ncOther2) // O2 1 more for local interest // B2 1 more for the new leaf interest // B1 and B2 GW map goes to 8. checkInterest(t, []int{5, 4, 3, 2, 3, 3, 1, 1}, 8) // Stop the server so1. so1.Shutdown() so1.WaitForShutdown() checkLeafNodeConnectedCount(t, sb2, 1) // Now check interest still valid, but wait a little bit to make sure that // even with the bug where we would send an RS- through the gateway, there // would be enough time for it to propagate before we check for interest. time.Sleep(250 * time.Millisecond) // O1 is stopped, so expect 0 // B2 has 1 less because leaf connection went away. // B1 and B2 GW map goes down to 6. checkInterest(t, []int{5, 4, 0, 2, 3, 2, 1, 1}, 6) // Store server sa1. sa1.Shutdown() sa1.WaitForShutdown() checkLeafNodeConnectedCount(t, sb1, 1) time.Sleep(250 * time.Millisecond) // A1 and O1 are gone, so 0 // A2 has 1 less due to loss of routed interest // B1 has 1 less because 1 leaf connection went away. // B1 and B2 GW map goes down to 3. checkInterest(t, []int{0, 3, 0, 2, 2, 2, 1, 1}, 3) // Now remove the queue subs from A2 ncA2qsub1.Unsubscribe() natsFlush(t, ncA2) // A2 has 1 less checkInterest(t, []int{0, 2, 0, 2, 2, 2, 1, 1}, 2) ncA2qsub2.Unsubscribe() natsFlush(t, ncA2) // A2 has 1 (no more locals but still interest for O2). // O2 has 1 (no more for remote interest, only local). // B1, B2 has 1 less since no interest from any of its leaf connections. checkInterest(t, []int{0, 1, 0, 1, 1, 1, 1, 1}, 1) // Removing (closing connection) of the sub on O2 will remove // interest globally. ncOther2.Close() checkInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0) // Resubscribe now, and again, interest should be propagated. natsQueueSubSync(t, ncA2, "foo", "bar") natsFlush(t, ncA2) checkInterest(t, []int{0, 1, 0, 1, 1, 1, 1, 1}, 1) natsQueueSubSync(t, ncA2, "foo", "bar") natsFlush(t, ncA2) checkInterest(t, []int{0, 2, 0, 1, 1, 1, 1, 1}, 2) // Close the client connection that has the 2 queue subs. ncA2.Close() checkInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0) // Now we will test when a route is lost on a server that has gateway enabled // that we update counts properly. ncB2 := natsConnect(t, sb2.ClientURL(), nats.UserInfo("user", "pwd")) defer ncB2.Close() natsQueueSubSync(t, ncB2, "foo", "bar") natsQueueSubSync(t, ncB2, "foo", "bar") natsQueueSubSync(t, ncB2, "foo", "bar") natsFlush(t, ncB2) checkInterest(t, []int{0, 1, 0, 1, 1, 3, 1, 1}, 3) ncB1 := natsConnect(t, sb1.ClientURL(), nats.UserInfo("user", "pwd")) defer ncB1.Close() natsQueueSubSync(t, ncB1, "foo", "bar") natsQueueSubSync(t, ncB1, "foo", "bar") checkInterest(t, []int{0, 1, 0, 1, 3, 4, 1, 1}, 5) // Now shutdown B2 sb2.Shutdown() sa1.WaitForShutdown() time.Sleep(250 * time.Millisecond) checkInterest(t, []int{0, 1, 0, 0, 2, 0, 1, 1}, 2) ncB1.Close() checkInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0) } func TestLeafNodeQueueInterestAndWeightCorrectAfterServerRestartOrConnectionClose(t *testing.T) { // Note that this is not what a normal configuration should be. Users should // configure each leafnode to have the URLs of both B1 and B2 so that when // a server fails, the leaf can reconnect to the other running server. But // we force it to be this way to demonstrate what the issue was and see that // it is now fixed. // // B1 <--- route ---> B2 // | | // Leaf Leaf // | | // A1 <--- route ---> A2 // for _, test := range []struct { name string pinnedAccount string }{ {"without pinned account", _EMPTY_}, {"with pinned account", "accounts: [\"A\"]"}, } { t.Run(test.name, func(t *testing.T) { leafBConf := ` accounts { A { users: [{user:a, password: pwd}] } } server_name: %s listen: "127.0.0.1:-1" cluster { name: HUB listen: "127.0.0.1:-1" %s %s } leafnodes { listen: "127.0.0.1:-1" no_advertise: true } ` b1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B1", _EMPTY_, test.pinnedAccount))) b1, b1Opts := RunServerWithConfig(b1Conf) defer b1.Shutdown() b2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", b1Opts.Cluster.Port), test.pinnedAccount))) b2, b2Opts := RunServerWithConfig(b2Conf) defer b2.Shutdown() checkClusterFormed(t, b1, b2) leafAConf := ` accounts { A { users: [{user:a, password: pwd}] } } server_name: %s listen: "127.0.0.1:-1" cluster { name: LEAF listen: "127.0.0.1:-1" %s %s } leafnodes { listen: "127.0.0.1:-1" remotes: [ { url: "nats://a:pwd@127.0.0.1:%d" account: A } ] no_advertise: true } ` a1Conf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, "A1", _EMPTY_, test.pinnedAccount, b1Opts.LeafNode.Port))) a1, a1Opts := RunServerWithConfig(a1Conf) defer a1.Shutdown() checkLeafNodeConnected(t, b1) checkLeafNodeConnected(t, a1) a2Conf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, "A2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", a1Opts.Cluster.Port), test.pinnedAccount, b2Opts.LeafNode.Port))) a2, _ := RunServerWithConfig(a2Conf) defer a2.Shutdown() checkLeafNodeConnected(t, b2) checkLeafNodeConnected(t, a2) checkClusterFormed(t, a1, a2) // Create a client on A2 and 3 queue subs. ncA2 := natsConnect(t, a2.ClientURL(), nats.UserInfo("a", "pwd")) defer ncA2.Close() var qsubs []*nats.Subscription for i := 0; i < 3; i++ { qsubs = append(qsubs, natsQueueSub(t, ncA2, "foo", "queue", func(_ *nats.Msg) {})) } natsFlush(t, ncA2) subj := "foo" checkInterest := func(expected bool) { t.Helper() for _, s := range []*Server{a1, a2, b1, b2} { acc, err := s.LookupAccount("A") require_NoError(t, err) checkFor(t, time.Second, 100*time.Millisecond, func() error { i := acc.Interest(subj) if expected && i == 0 { return fmt.Errorf("Still no interest on %q in server %q", subj, s) } else if !expected && i > 0 { return fmt.Errorf("Still interest on %q in server %q", subj, s) } return nil }) } } checkInterest(true) // Check that Leafz from A1 (which connects to B1) has the expected sub // interest on "foo". checkLeafA1 := func(expected bool) { t.Helper() // We will wait a bit before checking Leafz since with the bug, it would // take a bit of time after the action to reproduce the issue for the // LS+ to be sent to the wrong cluster, or the interest to not be removed. time.Sleep(100 * time.Millisecond) // Now check Leafz leafsz, err := a1.Leafz(&LeafzOptions{Subscriptions: true}) require_NoError(t, err) require_Equal(t, leafsz.NumLeafs, 1) require_True(t, leafsz.Leafs[0] != nil) lz := leafsz.Leafs[0] require_Equal(t, lz.Name, "B1") require_Equal(t, lz.NumSubs, uint32(len(lz.Subs))) var ok bool for _, sub := range lz.Subs { if sub == "foo" { if expected { ok = true break } t.Fatalf("Did not expect to have the %q subscription", sub) } } if expected && !ok { t.Fatalf("Expected to have the %q subscription", "foo") } } checkLeafA1(false) // Now restart server "B1". We need to create a conf file with the ports // that it used. restartBConf := createConfFile(t, []byte(fmt.Sprintf(` accounts { A { users: [{user:a, password: pwd}] } } server_name: B1 listen: "127.0.0.1:%d" cluster { name: HUB listen: "127.0.0.1:%d" %s } leafnodes { listen: "127.0.0.1:%d" no_advertise: true } `, b1Opts.Port, b1Opts.Cluster.Port, test.pinnedAccount, b1Opts.LeafNode.Port))) b1.Shutdown() b1, _ = RunServerWithConfig(restartBConf) defer b1.Shutdown() checkLeafNodeConnected(t, b1) checkLeafNodeConnected(t, a1) // Stop one of the queue sub. qsubs[0].Unsubscribe() natsFlush(t, ncA2) // Check that "foo" does not show up in the subscription list // for the leaf from A1 to B1. checkLeafA1(false) // Now stop the other 2 and check again. qsubs[1].Unsubscribe() qsubs[2].Unsubscribe() natsFlush(t, ncA2) checkInterest(false) checkLeafA1(false) // Now recreate 3 queue subs. for i := 0; i < 3; i++ { natsQueueSub(t, ncA2, "foo", "queue", func(_ *nats.Msg) {}) } // Check interest is present in all servers checkInterest(true) // But A1's leaf to B1 should still not have a sub interest for "foo". checkLeafA1(false) // Now stop the client connection instead of removing queue sub // one at a time. This will ensure that we properly handle an LS- // on B2 with an interest with a queue weight more than 1 still // present at the time of processing. ncA2.Close() checkInterest(false) checkLeafA1(false) // We will now test that if the queue subs are created on B2, // we have proper interest on A1, but when we close the connection, // the interest disappears. ncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo("a", "pwd")) defer ncB2.Close() for i := 0; i < 3; i++ { natsQueueSub(t, ncB2, "foo", "queue", func(_ *nats.Msg) {}) } checkInterest(true) checkLeafA1(true) // Close the connection, so all queue subs should be removed at once. ncB2.Close() checkInterest(false) checkLeafA1(false) }) } } func TestLeafNodeQueueWeightCorrectOnRestart(t *testing.T) { leafBConf := ` server_name: %s listen: "127.0.0.1:-1" cluster { name: HUB listen: "127.0.0.1:-1" %s } leafnodes { listen: "127.0.0.1:-1" no_advertise: true } ` b1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B1", _EMPTY_))) b1, b1Opts := RunServerWithConfig(b1Conf) defer b1.Shutdown() b2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", b1Opts.Cluster.Port)))) b2, b2Opts := RunServerWithConfig(b2Conf) defer b2.Shutdown() checkClusterFormed(t, b1, b2) leafAConf := ` server_name: LEAF listen: "127.0.0.1:-1" leafnodes { remotes: [{url: "nats://127.0.0.1:%d"}] reconnect: "50ms" } ` aConf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, b2Opts.LeafNode.Port))) a, _ := RunServerWithConfig(aConf) defer a.Shutdown() checkLeafNodeConnected(t, b2) checkLeafNodeConnected(t, a) nc := natsConnect(t, a.ClientURL()) defer nc.Close() for i := 0; i < 2; i++ { natsQueueSubSync(t, nc, "foo", "queue") } natsFlush(t, nc) checkQueueWeight := func() { for _, s := range []*Server{b1, b2} { gacc := s.GlobalAccount() gacc.mu.RLock() sl := gacc.sl gacc.mu.RUnlock() checkFor(t, time.Second, 10*time.Millisecond, func() error { // For remote queue interest, Match() will expand to queue weight. // So we should have 1 group and 2 queue subs present. res := sl.Match("foo") for _, qsubs := range res.qsubs { for _, sub := range qsubs { if string(sub.subject) == "foo" && string(sub.queue) == "queue" && atomic.LoadInt32(&sub.qw) == 2 { return nil } } } return fmt.Errorf("Server %q does not have expected queue interest with expected weight", s) }) } } checkQueueWeight() // Now restart server "B2". We need to create a conf file with the ports // that it used. restartBConf := createConfFile(t, []byte(fmt.Sprintf(` server_name: B2 listen: "127.0.0.1:%d" cluster { name: HUB listen: "127.0.0.1:%d" %s } leafnodes { listen: "127.0.0.1:%d" no_advertise: true } `, b2Opts.Port, b2Opts.Cluster.Port, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", b1Opts.Cluster.Port), b2Opts.LeafNode.Port))) b2.Shutdown() b2, _ = RunServerWithConfig(restartBConf) defer b2.Shutdown() checkLeafNodeConnected(t, b2) checkLeafNodeConnected(t, a) checkQueueWeight() } func TestLeafNodeRoutedSubKeyDifferentBetweenLeafSubAndRoutedSub(t *testing.T) { for _, test := range []struct { name string pinnedAccount string lnocu bool }{ {"without pinned account", _EMPTY_, true}, {"with pinned account", "accounts: [\"XYZ\"]", true}, {"old server without pinned account", _EMPTY_, false}, {"old server with pinned account", "accounts: [\"XYZ\"]", false}, } { t.Run(test.name, func(t *testing.T) { leafBConf := ` accounts: {XYZ {users:[{user:a, password:pwd}]}} server_name: %s listen: "127.0.0.1:-1" cluster { name: HUB listen: "127.0.0.1:-1" %s %s } leafnodes { listen: "127.0.0.1:-1" no_advertise: true } ` b1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B1", _EMPTY_, test.pinnedAccount))) b1, b1Opts := RunServerWithConfig(b1Conf) defer b1.Shutdown() b2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, "B2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", b1Opts.Cluster.Port), test.pinnedAccount))) b2, b2Opts := RunServerWithConfig(b2Conf) defer b2.Shutdown() checkClusterFormed(t, b1, b2) // To make route connections behave like if the server was connected // to an older server, change the routes' lnocu field. if !test.lnocu { for _, s := range []*Server{b1, b2} { s.mu.RLock() s.forEachRoute(func(r *client) { r.mu.Lock() r.route.lnocu = false r.mu.Unlock() }) s.mu.RUnlock() } } // This leaf will have a cluster name that matches an account name. // The idea is to make sure that hub servers are not using incorrect // keys to differentiate a routed queue interest on subject "A" with // queue name "foo" for account "A" in their cluster: "RS+ A A foo" // with a leafnode plain subscription, which since there is an origin // would be: "LS+ A A foo", that is, origin is "A", account is "A" // and subject is "foo". leafAConf := ` accounts: {XYZ {users:[{user:a, password:pwd}]}} server_name: LEAF listen: "127.0.0.1:-1" cluster { name: XYZ listen: "127.0.0.1:-1" } leafnodes { remotes: [ { url: "nats://a:pwd@127.0.0.1:%d" account: "XYZ" } ] } ` aConf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, b2Opts.LeafNode.Port))) a, _ := RunServerWithConfig(aConf) defer a.Shutdown() checkLeafNodeConnected(t, b2) checkLeafNodeConnected(t, a) ncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo("a", "pwd")) defer ncB2.Close() // Create a plain sub on "foo" natsSubSync(t, ncB2, "foo") // And a queue sub on "XYZ" with queue name "foo" natsQueueSubSync(t, ncB2, "XYZ", "foo") natsFlush(t, ncB2) ncA := natsConnect(t, a.ClientURL(), nats.UserInfo("a", "pwd")) defer ncA.Close() // From the leafnode, create a plain sub on "foo" natsSubSync(t, ncA, "foo") // And a queue sub on "XYZ" with queue name "foo" natsQueueSubSync(t, ncA, "XYZ", "foo") natsFlush(t, ncA) // Check the acc.rm on B2 b2Acc, err := b2.LookupAccount("XYZ") require_NoError(t, err) rsubKey := keyFromSubWithOrigin(&subscription{subject: []byte("foo")}) rqsubKey := keyFromSubWithOrigin(&subscription{subject: []byte("XYZ"), queue: []byte("foo")}) rlsubKey := keyFromSubWithOrigin(&subscription{origin: []byte("XYZ"), subject: []byte("foo")}) rlqsubKey := keyFromSubWithOrigin(&subscription{origin: []byte("XYZ"), subject: []byte("XYZ"), queue: []byte("foo")}) // Ensure all keys are different require_True(t, rsubKey != rqsubKey && rqsubKey != rlsubKey && rlsubKey != rlqsubKey) checkFor(t, time.Second, 10*time.Millisecond, func() error { b2Acc.mu.RLock() defer b2Acc.mu.RUnlock() for _, key := range []string{rsubKey, rqsubKey, rlsubKey, rlqsubKey} { v, ok := b2Acc.rm[key] if !ok { return fmt.Errorf("Did not find key %q for sub: %+v", key, sub) } if v != 1 { return fmt.Errorf("Key %q v=%v for sub: %+v", key, v, sub) } } return nil }) // Now check that on B1, we have 2 distinct subs for the route. b1Acc, err := b1.LookupAccount("XYZ") require_NoError(t, err) var route *client if test.pinnedAccount == _EMPTY_ { b1Acc.mu.RLock() rIdx := b1Acc.routePoolIdx b1Acc.mu.RUnlock() b1.mu.RLock() b1.forEachRouteIdx(rIdx, func(r *client) bool { route = r return false }) b1.mu.RUnlock() } else { b1.mu.RLock() remotes := b1.accRoutes["XYZ"] for _, r := range remotes { route = r break } b1.mu.RUnlock() } checkFor(t, time.Second, 10*time.Millisecond, func() error { // Check that route.subs has 4 entries for the subs we // created in this test. var entries []string route.mu.Lock() for key := range route.subs { if strings.Contains(key, "foo") { entries = append(entries, key) } } route.mu.Unlock() // With new servers, we expect 4 entries, but with older servers, // we have collisions and have only 2. var expected int if test.lnocu { expected = 4 } else { expected = 2 } if len(entries) != expected { return fmt.Errorf("Expected %d entries with %q, got this: %q", expected, "foo", entries) } return nil }) // Close the connections and expect all gone. ncB2.Close() ncA.Close() checkFor(t, time.Second, 10*time.Millisecond, func() error { b2Acc.mu.RLock() defer b2Acc.mu.RUnlock() for _, key := range []string{rsubKey, rqsubKey, rlsubKey, rlqsubKey} { if _, ok := b2Acc.rm[key]; ok { return fmt.Errorf("Key %q still present", key) } } return nil }) checkFor(t, time.Second, 10*time.Millisecond, func() error { var entries []string route.mu.Lock() for key := range route.subs { if strings.Contains(key, "foo") { entries = append(entries, key) } } route.mu.Unlock() if len(entries) != 0 { return fmt.Errorf("Still routed subscriptions on %q: %q", "foo", entries) } return nil }) }) } } func TestLeafNodeQueueGroupWithLateLNJoin(t *testing.T) { /* Topology: A cluster of leafnodes LN2 and LN3, connect to a cluster C1, C2. sub(foo) sub(foo) \ / C1 <-> C2 ^ ^ | | LN2 <-> LN3 / \ sub(foo) sub(foo) Once the above is set, start LN1 that connects to C1. sub(foo) sub(foo) \ / LN1 -> C1 <-> C2 ^ ^ | | LN2 <-> LN3 / \ sub(foo) sub(foo) Remove subs to LN3, C2 and C1. LN1 -> C1 <-> C2 ^ ^ | | LN2 <-> LN3 / sub(foo) Publish from LN1 and verify message is received by sub on LN2. pub(foo) \ LN1 -> C1 <-> C2 ^ ^ | | LN2 <-> LN3 / sub(foo) */ co1 := DefaultOptions() co1.LeafNode.Host = "127.0.0.1" co1.LeafNode.Port = -1 co1.Cluster.Name = "ngs" co1.Cluster.Host = "127.0.0.1" co1.Cluster.Port = -1 c1 := RunServer(co1) defer c1.Shutdown() co2 := DefaultOptions() co2.LeafNode.Host = "127.0.0.1" co2.LeafNode.Port = -1 co2.Cluster.Name = "ngs" co2.Cluster.Host = "127.0.0.1" co2.Cluster.Port = -1 co2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", co1.Cluster.Port)) c2 := RunServer(co2) defer c2.Shutdown() checkClusterFormed(t, c1, c2) lo2 := DefaultOptions() lo2.Cluster.Name = "local" lo2.Cluster.Host = "127.0.0.1" lo2.Cluster.Port = -1 lo2.LeafNode.ReconnectInterval = 50 * time.Millisecond lo2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", co1.LeafNode.Port)}}}} ln2 := RunServer(lo2) defer ln2.Shutdown() lo3 := DefaultOptions() lo3.Cluster.Name = "local" lo3.Cluster.Host = "127.0.0.1" lo3.Cluster.Port = -1 lo3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", lo2.Cluster.Port)) lo3.LeafNode.ReconnectInterval = 50 * time.Millisecond lo3.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", co2.LeafNode.Port)}}}} ln3 := RunServer(lo3) defer ln3.Shutdown() checkClusterFormed(t, ln2, ln3) checkLeafNodeConnected(t, ln2) checkLeafNodeConnected(t, ln3) cln2 := natsConnect(t, ln2.ClientURL()) defer cln2.Close() sln2 := natsQueueSubSync(t, cln2, "foo", "qgroup") natsFlush(t, cln2) cln3 := natsConnect(t, ln3.ClientURL()) defer cln3.Close() sln3 := natsQueueSubSync(t, cln3, "foo", "qgroup") natsFlush(t, cln3) cc1 := natsConnect(t, c1.ClientURL()) defer cc1.Close() sc1 := natsQueueSubSync(t, cc1, "foo", "qgroup") natsFlush(t, cc1) cc2 := natsConnect(t, c2.ClientURL()) defer cc2.Close() sc2 := natsQueueSubSync(t, cc2, "foo", "qgroup") natsFlush(t, cc2) checkSubInterest(t, c1, globalAccountName, "foo", time.Second) checkSubInterest(t, c2, globalAccountName, "foo", time.Second) checkSubInterest(t, ln2, globalAccountName, "foo", time.Second) checkSubInterest(t, ln3, globalAccountName, "foo", time.Second) lo1 := DefaultOptions() lo1.LeafNode.ReconnectInterval = 50 * time.Millisecond lo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", co1.LeafNode.Port)}}}} ln1 := RunServer(lo1) defer ln1.Shutdown() checkLeafNodeConnected(t, ln1) checkSubInterest(t, ln1, globalAccountName, "foo", time.Second) sln3.Unsubscribe() natsFlush(t, cln3) sc2.Unsubscribe() natsFlush(t, cc2) sc1.Unsubscribe() natsFlush(t, cc1) cln1 := natsConnect(t, ln1.ClientURL()) defer cln1.Close() natsPub(t, cln1, "foo", []byte("hello")) natsNexMsg(t, sln2, time.Second) } func TestLeafNodeJetStreamDomainMapCrossTalk(t *testing.T) { accs := ` accounts :{ A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, SYS:{ users:[ {user:s1,password:s1}]}, } system_account: SYS ` sd1 := t.TempDir() confA := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 %s jetstream: { domain: da, store_dir: '%s', max_mem: 50Mb, max_file: 50Mb } leafnodes: { listen: 127.0.0.1:-1 no_advertise: true authorization: { timeout: 0.5 } } `, accs, sd1))) sA, _ := RunServerWithConfig(confA) defer sA.Shutdown() sd2 := t.TempDir() confL := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 %s jetstream: { domain: dl, store_dir: '%s', max_mem: 50Mb, max_file: 50Mb } leafnodes:{ no_advertise: true remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] } `, accs, sd2, sA.opts.LeafNode.Port, sA.opts.LeafNode.Port))) sL, _ := RunServerWithConfig(confL) defer sL.Shutdown() ncA := natsConnect(t, sA.ClientURL(), nats.UserInfo("a1", "a1")) defer ncA.Close() ncL := natsConnect(t, sL.ClientURL(), nats.UserInfo("a1", "a1")) defer ncL.Close() test := func(jsA, jsL nats.JetStreamContext) { kvA, err := jsA.CreateKeyValue(&nats.KeyValueConfig{Bucket: "bucket"}) require_NoError(t, err) kvL, err := jsL.CreateKeyValue(&nats.KeyValueConfig{Bucket: "bucket"}) require_NoError(t, err) _, err = kvA.Put("A", nil) require_NoError(t, err) _, err = kvL.Put("L", nil) require_NoError(t, err) // check for unwanted cross talk _, err = kvA.Get("A") require_NoError(t, err) _, err = kvA.Get("l") require_Error(t, err) require_True(t, err == nats.ErrKeyNotFound) _, err = kvL.Get("A") require_Error(t, err) require_True(t, err == nats.ErrKeyNotFound) _, err = kvL.Get("L") require_NoError(t, err) err = jsA.DeleteKeyValue("bucket") require_NoError(t, err) err = jsL.DeleteKeyValue("bucket") require_NoError(t, err) } jsA, err := ncA.JetStream() require_NoError(t, err) jsL, err := ncL.JetStream() require_NoError(t, err) test(jsA, jsL) jsAL, err := ncA.JetStream(nats.Domain("dl")) require_NoError(t, err) jsLA, err := ncL.JetStream(nats.Domain("da")) require_NoError(t, err) test(jsAL, jsLA) jsAA, err := ncA.JetStream(nats.Domain("da")) require_NoError(t, err) jsLL, err := ncL.JetStream(nats.Domain("dl")) require_NoError(t, err) test(jsAA, jsLL) } type checkLeafMinVersionLogger struct { DummyLogger errCh chan string connCh chan string } func (l *checkLeafMinVersionLogger) Errorf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "minimum version") { select { case l.errCh <- msg: default: } } } func (l *checkLeafMinVersionLogger) Noticef(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "Leafnode connection created") { select { case l.connCh <- msg: default: } } } func TestLeafNodeMinVersion(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 leafnodes { port: -1 min_version: 2.8.0 } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() rconf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { remotes [ {url: "nats://127.0.0.1:%d" } ] } `, o.LeafNode.Port))) ln, _ := RunServerWithConfig(rconf) defer ln.Shutdown() checkLeafNodeConnected(t, s) checkLeafNodeConnected(t, ln) ln.Shutdown() s.Shutdown() // Now makes sure we validate options, not just config file. for _, test := range []struct { name string version string err string }{ {"invalid version", "abc", "semver"}, {"version too low", "2.7.9", "the minimum version should be at least 2.8.0"}, } { t.Run(test.name, func(t *testing.T) { o.Port = -1 o.LeafNode.Port = -1 o.LeafNode.MinVersion = test.version if s, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) { if s != nil { s.Shutdown() } t.Fatalf("Expected error to contain %q, got %v", test.err, err) } }) } // Ok, so now to verify that a server rejects a leafnode connection // we will set the min_version above our current VERSION. So first // decompose the version: major, minor, _, err := versionComponents(VERSION) if err != nil { t.Fatalf("The current server version %q is not valid: %v", VERSION, err) } // Let's make our minimum server an minor version above mv := fmt.Sprintf("%d.%d.0", major, minor+1) conf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { port: -1 min_version: "%s" } `, mv))) s, o = RunServerWithConfig(conf) defer s.Shutdown() l := &checkLeafMinVersionLogger{errCh: make(chan string, 1), connCh: make(chan string, 1)} s.SetLogger(l, false, false) rconf = createConfFile(t, []byte(fmt.Sprintf(` port: -1 leafnodes { remotes [ {url: "nats://127.0.0.1:%d" } ] } `, o.LeafNode.Port))) lo := LoadConfig(rconf) lo.LeafNode.ReconnectInterval = 50 * time.Millisecond ln = RunServer(lo) defer ln.Shutdown() select { case <-l.connCh: case <-time.After(time.Second): t.Fatal("Remote did not try to connect") } select { case <-l.errCh: case <-time.After(time.Second): t.Fatal("Did not get the minimum version required error") } // Since we have a very small reconnect interval, if the connection was // closed "right away", then we should have had a reconnect attempt with // another failure. This should not be the case because the server will // wait 5s before closing the connection. select { case <-l.connCh: t.Fatal("Should not have tried to reconnect") case <-time.After(250 * time.Millisecond): // OK } } func TestLeafNodeStreamAndShadowSubs(t *testing.T) { hubConf := createConfFile(t, []byte(` port: -1 leafnodes { port: -1 authorization: { user: leaf password: leaf account: B } } accounts: { A: { users = [{user: usrA, password: usrA}] exports: [{stream: foo.*.>}] } B: { imports: [{stream: {account: A, subject: foo.*.>}}] } } `)) hub, hubo := RunServerWithConfig(hubConf) defer hub.Shutdown() leafConfContet := fmt.Sprintf(` port: -1 leafnodes { remotes = [ { url: "nats-leaf://leaf:leaf@127.0.0.1:%d" account: B } ] } accounts: { B: { exports: [{stream: foo.*.>}] } C: { users: [{user: usrC, password: usrC}] imports: [{stream: {account: B, subject: foo.bar.>}}] } } `, hubo.LeafNode.Port) leafConf := createConfFile(t, []byte(leafConfContet)) leafo := LoadConfig(leafConf) leafo.LeafNode.ReconnectInterval = 50 * time.Millisecond leaf := RunServer(leafo) defer leaf.Shutdown() checkLeafNodeConnected(t, hub) checkLeafNodeConnected(t, leaf) subPubAndCheck := func() { t.Helper() ncl, err := nats.Connect(leaf.ClientURL(), nats.UserInfo("usrC", "usrC")) if err != nil { t.Fatalf("Error connecting: %v", err) } defer ncl.Close() // This will send an LS+ to the "hub" server. sub, err := ncl.SubscribeSync("foo.*.baz") if err != nil { t.Fatalf("Error subscribing: %v", err) } ncl.Flush() ncm, err := nats.Connect(hub.ClientURL(), nats.UserInfo("usrA", "usrA")) if err != nil { t.Fatalf("Error connecting: %v", err) } defer ncm.Close() // Try a few times in case subject interest has not propagated yet for i := 0; i < 5; i++ { ncm.Publish("foo.bar.baz", []byte("msg")) if _, err := sub.NextMsg(time.Second); err == nil { // OK, done! return } } t.Fatal("Message was not received") } subPubAndCheck() // Now cause a restart of the accepting side so that the leaf connection // is recreated. hub.Shutdown() hub = RunServer(hubo) defer hub.Shutdown() checkLeafNodeConnected(t, hub) checkLeafNodeConnected(t, leaf) subPubAndCheck() // Issue a config reload even though we make no modification. There was // a defect that caused the interest propagation to break. // Set the ReconnectInterval to the default value so that reload does not complain. leaf.getOpts().LeafNode.ReconnectInterval = DEFAULT_LEAF_NODE_RECONNECT reloadUpdateConfig(t, leaf, leafConf, leafConfContet) // Check again subPubAndCheck() } func TestLeafNodeAuthConfigReload(t *testing.T) { template := ` listen: 127.0.0.1:-1 accounts { test: {} } leaf { listen: "127.0.0.1:7422" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" } authorization { # These are only fields allowed atm. users = [ { user: test, password: "s3cret1", account: "test" } ] } } ` conf := createConfFile(t, []byte(template)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() lg := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(lg, false, false) // Reload here should work ok. reloadUpdateConfig(t, s, conf, template) } func TestLeafNodeSignatureCB(t *testing.T) { content := ` port: -1 server_name: OP operator = "../test/configs/nkeys/op.jwt" resolver = MEMORY listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() _, akp := createAccount(s) kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } lopts := &DefaultTestOptions u, err := url.Parse(fmt.Sprintf("nats://%s:%d", opts.LeafNode.Host, opts.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } remote := &RemoteLeafOpts{URLs: []*url.URL{u}} remote.SignatureCB = func(nonce []byte) (string, []byte, error) { return "", nil, fmt.Errorf("on purpose") } lopts.LeafNode.Remotes = []*RemoteLeafOpts{remote} lopts.LeafNode.ReconnectInterval = 100 * time.Millisecond sl := RunServer(lopts) defer sl.Shutdown() slog := &captureErrorLogger{errCh: make(chan string, 10)} sl.SetLogger(slog, false, false) // Now check that the leafnode got the error that the callback returned. select { case err := <-slog.errCh: if !strings.Contains(err, "on purpose") { t.Fatalf("Expected error from cb, got %v", err) } case <-time.After(time.Second): t.Fatal("Did not get expected error") } sl.Shutdown() // Now check what happens if the connection is closed while in the callback. blockCh := make(chan struct{}) remote.SignatureCB = func(nonce []byte) (string, []byte, error) { <-blockCh sig, err := kp.Sign(nonce) return ujwt, sig, err } sl = RunServer(lopts) defer sl.Shutdown() // Recreate the logger so that we are sure not to have possible previous errors slog = &captureErrorLogger{errCh: make(chan string, 10)} sl.SetLogger(slog, false, false) // Get the leaf connection from the temp clients map and close it. checkFor(t, time.Second, 15*time.Millisecond, func() error { var c *client sl.grMu.Lock() for _, cli := range sl.grTmpClients { c = cli } sl.grMu.Unlock() if c == nil { return fmt.Errorf("Client still not found in temp map") } c.closeConnection(ClientClosed) return nil }) // Release the callback, and check we get the appropriate error. close(blockCh) select { case err := <-slog.errCh: if !strings.Contains(err, ErrConnectionClosed.Error()) { t.Fatalf("Expected error that connection was closed, got %v", err) } case <-time.After(time.Second): t.Fatal("Did not get expected error") } sl.Shutdown() // Change to a good CB and now it should work remote.SignatureCB = func(nonce []byte) (string, []byte, error) { sig, err := kp.Sign(nonce) return ujwt, sig, err } sl = RunServer(lopts) defer sl.Shutdown() checkLeafNodeConnected(t, sl) } type testLeafTraceLogger struct { DummyLogger ch chan string } func (l *testLeafTraceLogger) Tracef(format string, v ...any) { msg := fmt.Sprintf(format, v...) // We will sub to 'baz' and to 'bar', so filter on 'ba' prefix. if strings.Contains(msg, "[LS+ ba") { select { case l.ch <- msg: default: } } } // Make sure permissioned denied subs do not make it to the leafnode even if existing. func TestLeafNodePermsSuppressSubs(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 authorization { PERMS = { publish = "foo" subscribe = ["_INBOX.>"] } users = [ {user: "user", password: "pass"} {user: "ln", password: "pass" , permissions: $PERMS } ] } no_auth_user: user leafnodes { listen: 127.0.0.1:7422 } `)) lconf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 leafnodes { remotes = [ { url: "nats://ln:pass@127.0.0.1" } ] } trace = true `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Connect client to the hub. nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() // This should not be seen on leafnode side since we only allow pub to "foo" _, err = nc.SubscribeSync("baz") require_NoError(t, err) ln, _ := RunServerWithConfig(lconf) defer ln.Shutdown() // Setup logger to capture trace events. l := &testLeafTraceLogger{ch: make(chan string, 10)} ln.SetLogger(l, true, true) checkLeafNodeConnected(t, ln) // Need to have ot reconnect to trigger since logger attaches too late. ln.mu.Lock() for _, c := range ln.leafs { c.mu.Lock() c.nc.Close() c.mu.Unlock() } ln.mu.Unlock() checkLeafNodeConnectedCount(t, ln, 0) checkLeafNodeConnectedCount(t, ln, 1) select { case msg := <-l.ch: t.Fatalf("Unexpected LS+ seen on leafnode: %s", msg) case <-time.After(50 * time.Millisecond): // OK } // Now double check that new subs also do not propagate. // This behavior was working already. _, err = nc.SubscribeSync("bar") require_NoError(t, err) select { case msg := <-l.ch: t.Fatalf("Unexpected LS+ seen on leafnode: %s", msg) case <-time.After(50 * time.Millisecond): // OK } } func TestLeafNodeDuplicateMsg(t *testing.T) { // This involves 2 clusters with leafnodes to each other with a different // account, and those accounts import/export a subject that caused // duplicate messages. This test requires static ports since we need to // have A->B and B->A. a1Conf := createConfFile(t, []byte(` cluster : { name : A port : -1 } leafnodes : { port : 14333 remotes : [{ account : A urls : [nats://leafa:pwd@127.0.0.1:24333] }] } port : -1 server_name : A_1 accounts:{ A:{ users:[ {user: leafa, password: pwd}, {user: usera, password: usera, permissions: { publish:{ allow:["iot.b.topic"] } subscribe:{ allow:["iot.a.topic"] } }} ] imports:[ {stream:{account:"B", subject:"iot.a.topic"}} ] }, B:{ users:[ {user: leafb, password: pwd}, ] exports:[ {stream: "iot.a.topic", accounts: ["A"]} ] } } `)) a1, oa1 := RunServerWithConfig(a1Conf) defer a1.Shutdown() a2Conf := createConfFile(t, []byte(fmt.Sprintf(` cluster : { name : A port : -1 routes : [nats://127.0.0.1:%d] } leafnodes : { port : 14334 remotes : [{ account : A urls : [nats://leafa:pwd@127.0.0.1:24334] }] } port : -1 server_name : A_2 accounts:{ A:{ users:[ {user: leafa, password: pwd}, {user: usera, password: usera, permissions: { publish:{ allow:["iot.b.topic"] } subscribe:{ allow:["iot.a.topic"] } }} ] imports:[ {stream:{account:"B", subject:"iot.a.topic"}} ] }, B:{ users:[ {user: leafb, password: pwd}, ] exports:[ {stream: "iot.a.topic", accounts: ["A"]} ] } }`, oa1.Cluster.Port))) a2, _ := RunServerWithConfig(a2Conf) defer a2.Shutdown() checkClusterFormed(t, a1, a2) b1Conf := createConfFile(t, []byte(` cluster : { name : B port : -1 } leafnodes : { port : 24333 remotes : [{ account : B urls : [nats://leafb:pwd@127.0.0.1:14333] }] } port : -1 server_name : B_1 accounts:{ A:{ users:[ {user: leafa, password: pwd}, ] exports:[ {stream: "iot.b.topic", accounts: ["B"]} ] }, B:{ users:[ {user: leafb, password: pwd}, {user: userb, password: userb, permissions: { publish:{ allow:["iot.a.topic"] }, subscribe:{ allow:["iot.b.topic"] } }} ] imports:[ {stream:{account:"A", subject:"iot.b.topic"}} ] } }`)) b1, ob1 := RunServerWithConfig(b1Conf) defer b1.Shutdown() b2Conf := createConfFile(t, []byte(fmt.Sprintf(` cluster : { name : B port : -1 routes : [nats://127.0.0.1:%d] } leafnodes : { port : 24334 remotes : [{ account : B urls : [nats://leafb:pwd@127.0.0.1:14334] }] } port : -1 server_name : B_2 accounts:{ A:{ users:[ {user: leafa, password: pwd}, ] exports:[ {stream: "iot.b.topic", accounts: ["B"]} ] }, B:{ users:[ {user: leafb, password: pwd}, {user: userb, password: userb, permissions: { publish:{ allow:["iot.a.topic"] }, subscribe:{ allow:["iot.b.topic"] } }} ] imports:[ {stream:{account:"A", subject:"iot.b.topic"}} ] } }`, ob1.Cluster.Port))) b2, _ := RunServerWithConfig(b2Conf) defer b2.Shutdown() checkClusterFormed(t, b1, b2) checkLeafNodeConnectedCount(t, a1, 2) checkLeafNodeConnectedCount(t, a2, 2) checkLeafNodeConnectedCount(t, b1, 2) checkLeafNodeConnectedCount(t, b2, 2) check := func(t *testing.T, subSrv *Server, pubSrv *Server) { sc := natsConnect(t, subSrv.ClientURL(), nats.UserInfo("userb", "userb")) defer sc.Close() subject := "iot.b.topic" sub := natsSubSync(t, sc, subject) // Wait for this to be available in A cluster checkSubInterest(t, a1, "A", subject, time.Second) checkSubInterest(t, a2, "A", subject, time.Second) pb := natsConnect(t, pubSrv.ClientURL(), nats.UserInfo("usera", "usera")) defer pb.Close() natsPub(t, pb, subject, []byte("msg")) natsNexMsg(t, sub, time.Second) // Should be only 1 if msg, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Received duplicate on %q: %s", msg.Subject, msg.Data) } } t.Run("sub_b1_pub_a1", func(t *testing.T) { check(t, b1, a1) }) t.Run("sub_b1_pub_a2", func(t *testing.T) { check(t, b1, a2) }) t.Run("sub_b2_pub_a1", func(t *testing.T) { check(t, b2, a1) }) t.Run("sub_b2_pub_a2", func(t *testing.T) { check(t, b2, a2) }) } func TestLeafNodeTLSHandshakeFirstVerifyNoInfoSent(t *testing.T) { confHub := createConfFile(t, []byte(` port : -1 leafnodes : { port : -1 tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 handshake_first: true } } `)) s1, o1 := RunServerWithConfig(confHub) defer s1.Shutdown() c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", o1.LeafNode.Port), 2*time.Second) require_NoError(t, err) defer c.Close() buf := make([]byte, 1024) // We will wait for up to 500ms to see if the server is sending (incorrectly) // the INFO. c.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) n, err := c.Read(buf) c.SetReadDeadline(time.Time{}) // If we did not get an error, this is an issue... if err == nil { t.Fatalf("Should not have received anything, got n=%v buf=%s", n, buf[:n]) } // We expect a timeout error if ne, ok := err.(net.Error); !ok || !ne.Timeout() { t.Fatalf("Expected a timeout error, got %v", err) } } func TestLeafNodeTLSHandshakeFirst(t *testing.T) { tmpl1 := ` port : -1 leafnodes : { port : -1 tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 handshake_first: %s } } ` confHub := createConfFile(t, []byte(fmt.Sprintf(tmpl1, "true"))) s1, o1 := RunServerWithConfig(confHub) defer s1.Shutdown() tmpl2 := ` port: -1 leafnodes : { port : -1 remotes : [ { urls : [tls://127.0.0.1:%d] tls { cert_file: "../test/configs/certs/client-cert.pem" key_file: "../test/configs/certs/client-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 first: %s } } ] } ` confSpoke := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, "true"))) s2, _ := RunServerWithConfig(confSpoke) defer s2.Shutdown() checkLeafNodeConnected(t, s2) s2.Shutdown() // Now check that there will be a failure if the remote does not ask for // handshake first since the hub is configured that way. // Set a logger on s1 to capture errors l := &captureErrorLogger{errCh: make(chan string, 10)} s1.SetLogger(l, false, false) confSpoke = createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, "false"))) s2, _ = RunServerWithConfig(confSpoke) defer s2.Shutdown() select { case err := <-l.errCh: if !strings.Contains(err, "handshake error") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(2 * time.Second): t.Fatal("Did not get TLS handshake failure") } // Check configuration reload for this remote reloadUpdateConfig(t, s2, confSpoke, fmt.Sprintf(tmpl2, o1.LeafNode.Port, "true")) checkLeafNodeConnected(t, s2) s2.Shutdown() // Drain the logger error channel for done := false; !done; { select { case <-l.errCh: default: done = true } } // Now change the config on the hub reloadUpdateConfig(t, s1, confHub, fmt.Sprintf(tmpl1, "false")) // Restart s2 s2, _ = RunServerWithConfig(confSpoke) defer s2.Shutdown() select { case err := <-l.errCh: if !strings.Contains(err, "handshake error") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(2 * time.Second): t.Fatal("Did not get TLS handshake failure") } // Reload again with "true" reloadUpdateConfig(t, s1, confHub, fmt.Sprintf(tmpl1, "true")) checkLeafNodeConnected(t, s2) } func TestLeafNodeTLSHandshakeEvenForRemoteWithNoTLSBlock(t *testing.T) { confHub := createConfFile(t, []byte(` port : -1 leafnodes : { port : -1 tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" ca_file: "../test/configs/certs/ca.pem" timeout: 2 } } `)) s1, o1 := RunServerWithConfig(confHub) defer s1.Shutdown() tmpl2 := ` port: -1 leafnodes : { port : -1 remotes : [ { urls : [tls://127.0.0.1:%d] } ] } ` confSpoke := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port))) s2, _ := RunServerWithConfig(confSpoke) defer s2.Shutdown() l := &captureDebugLogger{dbgCh: make(chan string, 100)} s2.SetLogger(l, true, false) tm := time.NewTimer(2 * time.Second) defer tm.Stop() for { select { case l := <-l.dbgCh: if strings.Contains(l, "Starting TLS") { // OK! return } case <-tm.C: t.Fatalf("Did not perform a TLS handshake") } } } func TestLeafNodeCompressionOptions(t *testing.T) { org := testDefaultLeafNodeCompression testDefaultLeafNodeCompression = _EMPTY_ defer func() { testDefaultLeafNodeCompression = org }() tmpl := ` port: -1 leafnodes { port: -1 compression: %s } ` for _, test := range []struct { name string mode string rttVals []int expected string rtts []time.Duration }{ {"boolean enabled", "true", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string enabled", "enabled", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string EnaBled", "EnaBled", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string on", "on", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string ON", "ON", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string fast", "fast", nil, CompressionS2Fast, nil}, {"string Fast", "Fast", nil, CompressionS2Fast, nil}, {"string s2_fast", "s2_fast", nil, CompressionS2Fast, nil}, {"string s2_Fast", "s2_Fast", nil, CompressionS2Fast, nil}, {"boolean disabled", "false", nil, CompressionOff, nil}, {"string disabled", "disabled", nil, CompressionOff, nil}, {"string DisableD", "DisableD", nil, CompressionOff, nil}, {"string off", "off", nil, CompressionOff, nil}, {"string OFF", "OFF", nil, CompressionOff, nil}, {"better", "better", nil, CompressionS2Better, nil}, {"Better", "Better", nil, CompressionS2Better, nil}, {"s2_better", "s2_better", nil, CompressionS2Better, nil}, {"S2_BETTER", "S2_BETTER", nil, CompressionS2Better, nil}, {"best", "best", nil, CompressionS2Best, nil}, {"BEST", "BEST", nil, CompressionS2Best, nil}, {"s2_best", "s2_best", nil, CompressionS2Best, nil}, {"S2_BEST", "S2_BEST", nil, CompressionS2Best, nil}, {"auto no rtts", "auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"s2_auto no rtts", "s2_auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"auto", "{mode: auto, rtt_thresholds: [%s]}", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}}, {"Auto", "{Mode: Auto, thresholds: [%s]}", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}}, {"s2_auto", "{mode: s2_auto, thresholds: [%s]}", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}}, {"s2_AUTO", "{mode: s2_AUTO, thresholds: [%s]}", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}}, {"s2_auto:-10,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,15", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}}, {"s2_auto:0,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,0,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,0", "{mode: s2_auto, rtt_thresholds: [%s]}", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}}, } { t.Run(test.name, func(t *testing.T) { var val string if len(test.rttVals) > 0 { var rtts string for i, v := range test.rttVals { if i > 0 { rtts += ", " } rtts += fmt.Sprintf("%dms", v) } val = fmt.Sprintf(test.mode, rtts) } else { val = test.mode } conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val))) s, o := RunServerWithConfig(conf) defer s.Shutdown() if cm := o.LeafNode.Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, o.LeafNode.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, o.LeafNode.Compression.RTTThresholds) } s.Shutdown() o.LeafNode.Port = -1 o.LeafNode.Compression.Mode = test.mode if len(test.rttVals) > 0 { o.LeafNode.Compression.Mode = CompressionS2Auto o.LeafNode.Compression.RTTThresholds = o.LeafNode.Compression.RTTThresholds[:0] for _, v := range test.rttVals { o.LeafNode.Compression.RTTThresholds = append(o.LeafNode.Compression.RTTThresholds, time.Duration(v)*time.Millisecond) } } s = RunServer(o) defer s.Shutdown() if cm := o.LeafNode.Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, o.LeafNode.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, o.LeafNode.Compression.RTTThresholds) } }) } // Same, but with remotes tmpl = ` port: -1 leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:1234" compression: %s } ] } ` for _, test := range []struct { name string mode string rttVals []int expected string rtts []time.Duration }{ {"boolean enabled", "true", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string enabled", "enabled", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string EnaBled", "EnaBled", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string on", "on", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string ON", "ON", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"string fast", "fast", nil, CompressionS2Fast, nil}, {"string Fast", "Fast", nil, CompressionS2Fast, nil}, {"string s2_fast", "s2_fast", nil, CompressionS2Fast, nil}, {"string s2_Fast", "s2_Fast", nil, CompressionS2Fast, nil}, {"boolean disabled", "false", nil, CompressionOff, nil}, {"string disabled", "disabled", nil, CompressionOff, nil}, {"string DisableD", "DisableD", nil, CompressionOff, nil}, {"string off", "off", nil, CompressionOff, nil}, {"string OFF", "OFF", nil, CompressionOff, nil}, {"better", "better", nil, CompressionS2Better, nil}, {"Better", "Better", nil, CompressionS2Better, nil}, {"s2_better", "s2_better", nil, CompressionS2Better, nil}, {"S2_BETTER", "S2_BETTER", nil, CompressionS2Better, nil}, {"best", "best", nil, CompressionS2Best, nil}, {"BEST", "BEST", nil, CompressionS2Best, nil}, {"s2_best", "s2_best", nil, CompressionS2Best, nil}, {"S2_BEST", "S2_BEST", nil, CompressionS2Best, nil}, {"auto no rtts", "auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"s2_auto no rtts", "s2_auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"auto", "{mode: auto, rtt_thresholds: [%s]}", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}}, {"Auto", "{Mode: Auto, thresholds: [%s]}", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}}, {"s2_auto", "{mode: s2_auto, thresholds: [%s]}", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}}, {"s2_AUTO", "{mode: s2_AUTO, thresholds: [%s]}", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}}, {"s2_auto:-10,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,15", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}}, {"s2_auto:0,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,0,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,0", "{mode: s2_auto, rtt_thresholds: [%s]}", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}}, } { t.Run("remote leaf "+test.name, func(t *testing.T) { var val string if len(test.rttVals) > 0 { var rtts string for i, v := range test.rttVals { if i > 0 { rtts += ", " } rtts += fmt.Sprintf("%dms", v) } val = fmt.Sprintf(test.mode, rtts) } else { val = test.mode } conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val))) s, o := RunServerWithConfig(conf) defer s.Shutdown() r := o.LeafNode.Remotes[0] if cm := r.Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, r.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, r.Compression.RTTThresholds) } s.Shutdown() o.LeafNode.Port = -1 o.LeafNode.Remotes[0].Compression.Mode = test.mode if len(test.rttVals) > 0 { o.LeafNode.Remotes[0].Compression.Mode = CompressionS2Auto o.LeafNode.Remotes[0].Compression.RTTThresholds = o.LeafNode.Remotes[0].Compression.RTTThresholds[:0] for _, v := range test.rttVals { o.LeafNode.Remotes[0].Compression.RTTThresholds = append(o.LeafNode.Remotes[0].Compression.RTTThresholds, time.Duration(v)*time.Millisecond) } } s = RunServer(o) defer s.Shutdown() if cm := o.LeafNode.Remotes[0].Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, o.LeafNode.Remotes[0].Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, o.LeafNode.Remotes[0].Compression.RTTThresholds) } }) } // Test that with no compression specified, we default to "s2_auto" conf := createConfFile(t, []byte(` port: -1 leafnodes { port: -1 } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() if o.LeafNode.Compression.Mode != CompressionS2Auto { t.Fatalf("Expected compression value to be %q, got %q", CompressionAccept, o.LeafNode.Compression.Mode) } if !reflect.DeepEqual(defaultCompressionS2AutoRTTThresholds, o.LeafNode.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", defaultCompressionS2AutoRTTThresholds, o.LeafNode.Compression.RTTThresholds) } // Same for remotes conf = createConfFile(t, []byte(` port: -1 leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:1234" } ] } `)) s, o = RunServerWithConfig(conf) defer s.Shutdown() if cm := o.LeafNode.Remotes[0].Compression.Mode; cm != CompressionS2Auto { t.Fatalf("Expected compression value to be %q, got %q", CompressionAccept, cm) } if !reflect.DeepEqual(defaultCompressionS2AutoRTTThresholds, o.LeafNode.Remotes[0].Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", defaultCompressionS2AutoRTTThresholds, o.LeafNode.Remotes[0].Compression.RTTThresholds) } for _, test := range []struct { name string mode string rtts []time.Duration err string }{ {"unsupported mode", "gzip", nil, "unsupported"}, {"not ascending order", "s2_auto", []time.Duration{ 5 * time.Millisecond, 10 * time.Millisecond, 2 * time.Millisecond, }, "ascending"}, {"too many thresholds", "s2_auto", []time.Duration{ 5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 60 * time.Millisecond, }, "more than 4"}, {"all 0", "s2_auto", []time.Duration{0, 0, 0, 0}, "at least one"}, {"single 0", "s2_auto", []time.Duration{0}, "at least one"}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() o.LeafNode.Port = -1 o.LeafNode.Compression = CompressionOpts{test.mode, test.rtts} if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Unexpected error: %v", err) } // Same with remotes o.LeafNode.Compression = CompressionOpts{} o.LeafNode.Remotes = []*RemoteLeafOpts{{Compression: CompressionOpts{test.mode, test.rtts}}} if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Unexpected error: %v", err) } }) } } func TestLeafNodeCompression(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 server_name: "Hub" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } C { users: [{user: c, password: pwd}] } } leafnodes { port: -1 compression: s2_fast } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() port := o1.LeafNode.Port conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "Spoke" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } C { users: [{user: c, password: pwd}] } } leafnodes { remotes [ { url: "nats://a:pwd@127.0.0.1:%d", account: "A", compression: s2_better } { url: "nats://b:pwd@127.0.0.1:%d", account: "B", compression: s2_best } { url: "nats://c:pwd@127.0.0.1:%d", account: "C", compression: off } ] } `, port, port, port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnectedCount(t, s1, 3) checkLeafNodeConnectedCount(t, s2, 3) s1.mu.RLock() for _, l := range s1.leafs { l.mu.Lock() l.nc = &testConnSentBytes{Conn: l.nc} l.mu.Unlock() } s1.mu.RUnlock() var payloads [][]byte totalPayloadSize := 0 count := 26 for i := 0; i < count; i++ { n := rand.Intn(2048) + 1 p := make([]byte, n) for j := 0; j < n; j++ { p[j] = byte(i) + 'A' } totalPayloadSize += len(p) payloads = append(payloads, p) } check := func(acc, user, subj string) { t.Helper() nc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, "pwd")) defer nc2.Close() sub := natsSubSync(t, nc2, subj) natsFlush(t, nc2) checkSubInterest(t, s1, acc, subj, time.Second) nc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, "pwd")) defer nc1.Close() for i := 0; i < count; i++ { natsPub(t, nc1, subj, payloads[i]) } for i := 0; i < count; i++ { m := natsNexMsg(t, sub, time.Second) if !bytes.Equal(m.Data, payloads[i]) { t.Fatalf("Expected payload %q - got %q", payloads[i], m.Data) } } // Also check that the leafnode stats shows that compression likely occurred var out int s1.mu.RLock() for _, l := range s1.leafs { l.mu.Lock() if l.acc.Name == acc && l.nc != nil { nc := l.nc.(*testConnSentBytes) nc.Lock() out = nc.sent nc.sent = 0 nc.Unlock() } l.mu.Unlock() } s1.mu.RUnlock() // Except for account "C", where compression should be off, // "out" should at least be smaller than totalPayloadSize, use 20%. if acc == "C" { if int(out) < totalPayloadSize { t.Fatalf("Expected s1's sent bytes to be at least payload size (%v), got %v", totalPayloadSize, out) } } else { limit := totalPayloadSize * 80 / 100 if int(out) > limit { t.Fatalf("Expected s1's sent bytes to be less than %v, got %v (total payload was %v)", limit, out, totalPayloadSize) } } } check("A", "a", "foo") check("B", "b", "bar") check("C", "c", "baz") // Check compression settings. S1 should always be s2_fast, except for account "C" // since "C" wanted compression "off" l, err := s1.Leafz(nil) require_NoError(t, err) for _, r := range l.Leafs { switch r.Account { case "C": if r.Compression != CompressionOff { t.Fatalf("Expected compression of remote for C account to be %q, got %q", CompressionOff, r.Compression) } default: if r.Compression != CompressionS2Fast { t.Fatalf("Expected compression of remote for %s account to be %q, got %q", r.Account, CompressionS2Fast, r.Compression) } } } l, err = s2.Leafz(nil) require_NoError(t, err) for _, r := range l.Leafs { switch r.Account { case "A": if r.Compression != CompressionS2Better { t.Fatalf("Expected compression for A account to be %q, got %q", CompressionS2Better, r.Compression) } case "B": if r.Compression != CompressionS2Best { t.Fatalf("Expected compression for B account to be %q, got %q", CompressionS2Best, r.Compression) } case "C": if r.Compression != CompressionOff { t.Fatalf("Expected compression for C account to be %q, got %q", CompressionOff, r.Compression) } } } } func BenchmarkLeafNodeCompression(b *testing.B) { conf1 := createConfFile(b, []byte(` port: -1 server_name: "Hub" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } C { users: [{user: c, password: pwd}] } D { users: [{user: d, password: pwd}] } } leafnodes { port: -1 } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() port := o1.LeafNode.Port conf2 := createConfFile(b, []byte(fmt.Sprintf(` port: -1 server_name: "Spoke" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } C { users: [{user: c, password: pwd}] } D { users: [{user: d, password: pwd}] } } leafnodes { remotes [ { url: "nats://a:pwd@127.0.0.1:%d", account: "A", compression: s2_better } { url: "nats://b:pwd@127.0.0.1:%d", account: "B", compression: s2_best } { url: "nats://c:pwd@127.0.0.1:%d", account: "C", compression: s2_fast } { url: "nats://d:pwd@127.0.0.1:%d", account: "D", compression: off } ] } `, port, port, port, port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnectedCount(b, s1, 4) checkLeafNodeConnectedCount(b, s2, 4) l, err := s2.Leafz(nil) require_NoError(b, err) for _, r := range l.Leafs { switch { case r.Account == "A" && r.Compression == CompressionS2Better: case r.Account == "B" && r.Compression == CompressionS2Best: case r.Account == "C" && r.Compression == CompressionS2Fast: case r.Account == "D" && r.Compression == CompressionOff: default: b.Fatalf("Account %q had incorrect compression mode %q on leaf connection", r.Account, r.Compression) } } msg := make([]byte, 1024) for _, p := range []struct { algo string user string }{ {"Better", "a"}, {"Best", "b"}, {"Fast", "c"}, {"Off", "d"}, } { nc1 := natsConnect(b, s1.ClientURL(), nats.UserInfo(p.user, "pwd")) nc2 := natsConnect(b, s2.ClientURL(), nats.UserInfo(p.user, "pwd")) sub, err := nc1.SubscribeSync("foo") require_NoError(b, err) time.Sleep(time.Second) b.Run(p.algo, func(b *testing.B) { start := time.Now() for i := 0; i < b.N; i++ { err = nc2.Publish("foo", msg) require_NoError(b, err) _, err = sub.NextMsg(time.Second) require_NoError(b, err) } b.ReportMetric(float64(len(msg)*b.N)/1024/1024, "MB") b.ReportMetric(float64(len(msg)*b.N)/1024/1024/float64(time.Since(start).Seconds()), "MB/sec") }) nc1.Close() nc2.Close() } } func TestLeafNodeCompressionMatrixModes(t *testing.T) { for _, test := range []struct { name string s1 string s2 string s1Expected string s2Expected string }{ {"off off", "off", "off", CompressionOff, CompressionOff}, {"off accept", "off", "accept", CompressionOff, CompressionOff}, {"off on", "off", "on", CompressionOff, CompressionOff}, {"off better", "off", "better", CompressionOff, CompressionOff}, {"off best", "off", "best", CompressionOff, CompressionOff}, {"accept off", "accept", "off", CompressionOff, CompressionOff}, {"accept accept", "accept", "accept", CompressionOff, CompressionOff}, // Note: "on", means s2_auto, which will mean uncompressed since RTT is low. {"accept on", "accept", "on", CompressionS2Fast, CompressionS2Uncompressed}, {"accept better", "accept", "better", CompressionS2Better, CompressionS2Better}, {"accept best", "accept", "best", CompressionS2Best, CompressionS2Best}, {"on off", "on", "off", CompressionOff, CompressionOff}, {"on accept", "on", "accept", CompressionS2Uncompressed, CompressionS2Fast}, {"on on", "on", "on", CompressionS2Uncompressed, CompressionS2Uncompressed}, {"on better", "on", "better", CompressionS2Uncompressed, CompressionS2Better}, {"on best", "on", "best", CompressionS2Uncompressed, CompressionS2Best}, {"better off", "better", "off", CompressionOff, CompressionOff}, {"better accept", "better", "accept", CompressionS2Better, CompressionS2Better}, {"better on", "better", "on", CompressionS2Better, CompressionS2Uncompressed}, {"better better", "better", "better", CompressionS2Better, CompressionS2Better}, {"better best", "better", "best", CompressionS2Better, CompressionS2Best}, {"best off", "best", "off", CompressionOff, CompressionOff}, {"best accept", "best", "accept", CompressionS2Best, CompressionS2Best}, {"best on", "best", "on", CompressionS2Best, CompressionS2Uncompressed}, {"best better", "best", "better", CompressionS2Best, CompressionS2Better}, {"best best", "best", "best", CompressionS2Best, CompressionS2Best}, } { t.Run(test.name, func(t *testing.T) { conf1 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "A" leafnodes { port: -1 compression: %s } `, test.s1))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" leafnodes { remotes: [ {url: "nats://127.0.0.1:%d", compression: %s} ] } `, o1.LeafNode.Port, test.s2))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) nc1 := natsConnect(t, s1.ClientURL()) defer nc1.Close() nc2 := natsConnect(t, s2.ClientURL()) defer nc2.Close() payload := make([]byte, 128) check := func(ncp, ncs *nats.Conn, subj string, s *Server) { t.Helper() sub := natsSubSync(t, ncs, subj) checkSubInterest(t, s, globalAccountName, subj, time.Second) natsPub(t, ncp, subj, payload) natsNexMsg(t, sub, time.Second) for _, srv := range []*Server{s1, s2} { lz, err := srv.Leafz(nil) require_NoError(t, err) var expected string if srv == s1 { expected = test.s1Expected } else { expected = test.s2Expected } if cm := lz.Leafs[0].Compression; cm != expected { t.Fatalf("Server %s - expected compression %q, got %q", srv, expected, cm) } } } check(nc1, nc2, "foo", s1) check(nc2, nc1, "bar", s2) }) } } func TestLeafNodeCompressionWithOlderServer(t *testing.T) { tmpl1 := ` port: -1 server_name: "A" leafnodes { port: -1 compression: "%s" } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, CompressionS2Fast))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() tmpl2 := ` port: -1 server_name: "B" leafnodes { remotes [ {url: "nats://127.0.0.1:%d", compression: "%s"} ] } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, CompressionNotSupported))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) getLeafCompMode := func(s *Server) string { var cm string s.mu.RLock() defer s.mu.RUnlock() for _, l := range s1.leafs { l.mu.Lock() cm = l.leaf.compression l.mu.Unlock() return cm } return _EMPTY_ } for _, s := range []*Server{s1, s2} { if cm := getLeafCompMode(s); cm != CompressionNotSupported { t.Fatalf("Expected compression not supported, got %q", cm) } } s2.Shutdown() s1.Shutdown() conf1 = createConfFile(t, []byte(fmt.Sprintf(tmpl1, CompressionNotSupported))) s1, o1 = RunServerWithConfig(conf1) defer s1.Shutdown() conf2 = createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, CompressionS2Fast))) s2, _ = RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) for _, s := range []*Server{s1, s2} { if cm := getLeafCompMode(s); cm != CompressionNotSupported { t.Fatalf("Expected compression not supported, got %q", cm) } } } func TestLeafNodeCompressionAuto(t *testing.T) { for _, test := range []struct { name string s1Ping string s1Compression string s2Ping string s2Compression string checkS1 bool }{ {"remote side", "10s", CompressionS2Fast, "100ms", "{mode: s2_auto, rtt_thresholds: [10ms, 20ms, 30ms]}", false}, {"accept side", "100ms", "{mode: s2_auto, rtt_thresholds: [10ms, 20ms, 30ms]}", "10s", CompressionS2Fast, true}, } { t.Run(test.name, func(t *testing.T) { conf1 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "A" ping_interval: "%s" leafnodes { port: -1 compression: %s } `, test.s1Ping, test.s1Compression))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() // Start with 0ms RTT np := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf("nats://127.0.0.1:%d", o1.LeafNode.Port), true) conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" ping_interval: "%s" leafnodes { remotes [ {url: %s, compression %s} ] } `, test.s2Ping, np.routeURL(), test.s2Compression))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() defer np.stop() checkLeafNodeConnected(t, s2) checkComp := func(expected string) { t.Helper() var s *Server if test.checkS1 { s = s1 } else { s = s2 } checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { s.mu.RLock() defer s.mu.RUnlock() for _, l := range s.leafs { l.mu.Lock() cm := l.leaf.compression l.mu.Unlock() if cm != expected { return fmt.Errorf("Leaf %v compression mode expected to be %q, got %q", l, expected, cm) } } return nil }) } checkComp(CompressionS2Uncompressed) // Change the proxy RTT and we should get compression "fast" np.updateRTT(15 * time.Millisecond) checkComp(CompressionS2Fast) // Now 25ms, and get "better" np.updateRTT(25 * time.Millisecond) checkComp(CompressionS2Better) // Above 35 and we should get "best" np.updateRTT(35 * time.Millisecond) checkComp(CompressionS2Best) // Down to 1ms and again should get "uncompressed" np.updateRTT(1 * time.Millisecond) checkComp(CompressionS2Uncompressed) }) } // Make sure that if compression is off on one side, the update of RTT does // not trigger a compression change. conf1 := createConfFile(t, []byte(` port: -1 server_name: "A" leafnodes { port: -1 compression: off } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() // Start with 0ms RTT np := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf("nats://127.0.0.1:%d", o1.LeafNode.Port), true) conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" ping_interval: "50ms" leafnodes { remotes [ {url: %s, compression s2_auto} ] } `, np.routeURL()))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() defer np.stop() checkLeafNodeConnected(t, s2) // Even with a bug of updating compression level while it should have been // off, the check done below would almost always pass because after // reconnecting, there could be a chance to get at first compression set // to "off". So we will double check that the leaf node CID did not change // at the end of the test. getCID := func() uint64 { s2.mu.RLock() defer s2.mu.RUnlock() for _, l := range s2.leafs { l.mu.Lock() cid := l.cid l.mu.Unlock() return cid } return 0 } oldCID := getCID() checkCompOff := func() { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { s2.mu.RLock() defer s2.mu.RUnlock() if len(s2.leafs) != 1 { return fmt.Errorf("Leaf not currently connected") } for _, l := range s2.leafs { l.mu.Lock() cm := l.leaf.compression l.mu.Unlock() if cm != CompressionOff { return fmt.Errorf("Leaf %v compression mode expected to be %q, got %q", l, CompressionOff, cm) } } return nil }) } checkCompOff() // Now change RTT and again, make sure that it is still off np.updateRTT(20 * time.Millisecond) time.Sleep(100 * time.Millisecond) checkCompOff() if cid := getCID(); cid != oldCID { t.Fatalf("Leafnode has reconnected, cid was %v, now %v", oldCID, cid) } } func TestLeafNodeCompressionWithWSCompression(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 server_name: "A" websocket { port: -1 no_tls: true compression: true } leafnodes { port: -1 compression: s2_fast } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" leafnodes { remotes [ { url: "ws://127.0.0.1:%d" ws_compression: true compression: s2_fast } ] } `, o1.Websocket.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) nc1 := natsConnect(t, s1.ClientURL()) defer nc1.Close() sub := natsSubSync(t, nc1, "foo") checkSubInterest(t, s2, globalAccountName, "foo", time.Second) nc2 := natsConnect(t, s2.ClientURL()) defer nc2.Close() payload := make([]byte, 1024) for i := 0; i < len(payload); i++ { payload[i] = 'A' } natsPub(t, nc2, "foo", payload) msg := natsNexMsg(t, sub, time.Second) require_True(t, len(msg.Data) == 1024) for i := 0; i < len(msg.Data); i++ { if msg.Data[i] != 'A' { t.Fatalf("Invalid msg: %s", msg.Data) } } } func TestLeafNodeCompressionWithWSGetNeedsData(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 server_name: "A" websocket { port: -1 no_tls: true } leafnodes { port: -1 compression: s2_fast } `)) srv1, o1 := RunServerWithConfig(conf1) defer srv1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" leafnodes { remotes [ { url: "ws://127.0.0.1:%d" ws_no_masking: true compression: s2_fast } ] } `, o1.Websocket.Port))) srv2, _ := RunServerWithConfig(conf2) defer srv2.Shutdown() checkLeafNodeConnected(t, srv2) nc1 := natsConnect(t, srv1.ClientURL()) defer nc1.Close() sub := natsSubSync(t, nc1, "foo") checkSubInterest(t, srv2, globalAccountName, "foo", time.Second) // We want to have the payload more than 126 bytes so that the websocket // code need to read 2 bytes for the length. See below. payload := "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ" sentBytes := []byte("LMSG foo 156\r\n" + payload + "\r\n") h, _ := wsCreateFrameHeader(false, false, wsBinaryMessage, len(sentBytes)) combined := &bytes.Buffer{} combined.Write(h) combined.Write(sentBytes) toSend := combined.Bytes() // We will make a compressed block that cuts the websocket header that // makes the reader want to read bytes directly from the connection. // We want to make sure that we are not going to get compressed data // without going through the (de)compress library. So for that, compress // the first 3 bytes. b := &bytes.Buffer{} w := s2.NewWriter(b) w.Write(toSend[:3]) w.Close() var nc net.Conn srv2.mu.RLock() for _, l := range srv2.leafs { l.mu.Lock() nc = l.nc l.mu.Unlock() } srv2.mu.RUnlock() nc.Write(b.Bytes()) // Pause to make sure other side just gets a partial of the whole WS frame. time.Sleep(100 * time.Millisecond) b.Reset() w.Reset(b) w.Write(toSend[3:]) w.Close() nc.Write(b.Bytes()) msg := natsNexMsg(t, sub, time.Second) require_True(t, len(msg.Data) == 156) require_Equal(t, string(msg.Data), payload) } func TestLeafNodeCompressionAuthTimeout(t *testing.T) { hconf := createConfFile(t, []byte(` port: -1 server_name: "hub" leafnodes { port: -1 authorization { timeout: 0.75 } } `)) sh, oh := RunServerWithConfig(hconf) defer sh.Shutdown() sconfTmpl := ` port: -1 server_name: "%s" cluster { port: -1 name: "spoke" %s } leafnodes { port: -1 remotes [ { url: "nats://127.0.0.1:%d" } ] } ` s1conf := createConfFile(t, []byte(fmt.Sprintf(sconfTmpl, "SP1", _EMPTY_, oh.LeafNode.Port))) s1, o1 := RunServerWithConfig(s1conf) defer s1.Shutdown() s2conf := createConfFile(t, []byte(fmt.Sprintf(sconfTmpl, "SP2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), oh.LeafNode.Port))) s2, _ := RunServerWithConfig(s2conf) defer s2.Shutdown() checkClusterFormed(t, s1, s2) checkLeafNodeConnected(t, s1) checkLeafNodeConnected(t, s2) getCID := func(s *Server) uint64 { s.mu.RLock() defer s.mu.RUnlock() var cid uint64 for _, l := range s.leafs { l.mu.Lock() cid = l.cid l.mu.Unlock() } return cid } leaf1 := getCID(s1) leaf2 := getCID(s2) // Wait for more than auth timeout time.Sleep(time.Second) checkLeafNodeConnected(t, s1) checkLeafNodeConnected(t, s2) if l1 := getCID(s1); l1 != leaf1 { t.Fatalf("Leaf connection first connection had CID %v, now %v", leaf1, l1) } if l2 := getCID(s2); l2 != leaf2 { t.Fatalf("Leaf connection first connection had CID %v, now %v", leaf2, l2) } } func TestLeafNodeWithWeightedDQRequestsToSuperClusterWithSeparateAccounts(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 2) defer sc.shutdown() // Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, ONE and TWO. var lnTmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var leafFrag = ` leaf { listen: 127.0.0.1:-1 remotes [ { urls: [ %s ] } { urls: [ %s ] } ] }` // We want to have two leaf node connections that join to the same local account on the leafnode servers, // but connect to different accounts in different clusters. c1 := sc.clusters[0] // Will connect to account ONE c2 := sc.clusters[1] // Will connect to account TWO genLeafTmpl := func(tmpl string) string { t.Helper() var ln1, ln2 []string for _, s := range c1.servers { if s.ClusterName() != c1.name { continue } ln := s.getOpts().LeafNode ln1 = append(ln1, fmt.Sprintf("nats://one:p@%s:%d", ln.Host, ln.Port)) } for _, s := range c2.servers { if s.ClusterName() != c2.name { continue } ln := s.getOpts().LeafNode ln2 = append(ln2, fmt.Sprintf("nats://two:p@%s:%d", ln.Host, ln.Port)) } return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(leafFrag, strings.Join(ln1, ", "), strings.Join(ln2, ", ")), 1) } tmpl := strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "SA"), 1) tmpl = genLeafTmpl(tmpl) ln := createJetStreamCluster(t, tmpl, "SA", "SA-", 3, 22280, false) ln.waitOnClusterReady() defer ln.shutdown() for _, s := range ln.servers { checkLeafNodeConnectedCount(t, s, 2) } // Now connect DQ subscribers to each cluster and they separate accounts, and make sure we get the right behavior, balanced between // them when requests originate from the leaf cluster. // Create 5 clients for each cluster / account var c1c, c2c []*nats.Conn for i := 0; i < 5; i++ { nc1, _ := jsClientConnect(t, c1.randomServer(), nats.UserInfo("one", "p")) defer nc1.Close() c1c = append(c1c, nc1) nc2, _ := jsClientConnect(t, c2.randomServer(), nats.UserInfo("two", "p")) defer nc2.Close() c2c = append(c2c, nc2) } createSubs := func(num int, conns []*nats.Conn) (subs []*nats.Subscription) { for i := 0; i < num; i++ { nc := conns[rand.Intn(len(conns))] sub, err := nc.QueueSubscribeSync("REQUEST", "MC") require_NoError(t, err) subs = append(subs, sub) nc.Flush() } // Let subs propagate. time.Sleep(100 * time.Millisecond) return subs } closeSubs := func(subs []*nats.Subscription) { for _, sub := range subs { sub.Unsubscribe() } } // Simple test first. subs1 := createSubs(1, c1c) defer closeSubs(subs1) subs2 := createSubs(1, c2c) defer closeSubs(subs2) sendRequests := func(num int) { t.Helper() // Now connect to the leaf cluster and send some requests. nc, _ := jsClientConnect(t, ln.randomServer()) defer nc.Close() for i := 0; i < num; i++ { require_NoError(t, nc.Publish("REQUEST", []byte("HELP"))) } nc.Flush() } pending := func(subs []*nats.Subscription) (total int) { t.Helper() for _, sub := range subs { n, _, err := sub.Pending() require_NoError(t, err) total += n } return total } num := 1000 checkAllReceived := func() error { total := pending(subs1) + pending(subs2) if total == num { return nil } return fmt.Errorf("Not all received: %d vs %d", total, num) } checkBalanced := func(total, pc1, pc2 int) { t.Helper() tf := float64(total) e1 := tf * (float64(pc1) / 100.00) e2 := tf * (float64(pc2) / 100.00) delta := tf / 10 p1 := float64(pending(subs1)) if p1 < e1-delta || p1 > e1+delta { t.Fatalf("Value out of range for subs1, expected %v got %v", e1, p1) } p2 := float64(pending(subs2)) if p2 < e2-delta || p2 > e2+delta { t.Fatalf("Value out of range for subs2, expected %v got %v", e2, p2) } } // Now connect to the leaf cluster and send some requests. // Simple 50/50 sendRequests(num) checkFor(t, time.Second, 200*time.Millisecond, checkAllReceived) checkBalanced(num, 50, 50) closeSubs(subs1) closeSubs(subs2) // Now test unbalanced. 10/90 subs1 = createSubs(1, c1c) defer closeSubs(subs1) subs2 = createSubs(9, c2c) defer closeSubs(subs2) sendRequests(num) checkFor(t, time.Second, 200*time.Millisecond, checkAllReceived) checkBalanced(num, 10, 90) // Now test draining the subs as we are sending from an initial balanced situation simulating a draining of a cluster. closeSubs(subs1) closeSubs(subs2) subs1, subs2 = nil, nil // These subs slightly different. var r1, r2 atomic.Uint64 for i := 0; i < 20; i++ { nc := c1c[rand.Intn(len(c1c))] sub, err := nc.QueueSubscribe("REQUEST", "MC", func(m *nats.Msg) { r1.Add(1) }) require_NoError(t, err) subs1 = append(subs1, sub) nc.Flush() nc = c2c[rand.Intn(len(c2c))] sub, err = nc.QueueSubscribe("REQUEST", "MC", func(m *nats.Msg) { r2.Add(1) }) require_NoError(t, err) subs2 = append(subs2, sub) nc.Flush() } defer closeSubs(subs1) defer closeSubs(subs2) nc, _ := jsClientConnect(t, ln.randomServer()) defer nc.Close() for i, dindex := 0, 1; i < num; i++ { require_NoError(t, nc.Publish("REQUEST", []byte("HELP"))) // Check if we have more to simulate draining. // Will drain within first ~100 requests using 20% rand test below. // Will leave 1 behind. if dindex < len(subs1)-1 && rand.Intn(6) > 4 { sub := subs1[dindex] dindex++ sub.Drain() } } nc.Flush() checkFor(t, time.Second, 200*time.Millisecond, func() error { total := int(r1.Load() + r2.Load()) if total == num { return nil } return fmt.Errorf("Not all received: %d vs %d", total, num) }) require_True(t, r2.Load() > r1.Load()) } func TestLeafNodeWithWeightedDQRequestsToSuperClusterWithStreamImportAccounts(t *testing.T) { var tmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { EFG { users = [ { user: "efg", pass: "p" } ] jetstream: enabled imports [ { stream: { account: STL, subject: "REQUEST"} } { stream: { account: KSC, subject: "REQUEST"} } ] exports [ { stream: "RESPONSE" } ] } STL { users = [ { user: "stl", pass: "p" } ] exports [ { stream: "REQUEST" } ] imports [ { stream: { account: EFG, subject: "RESPONSE"} } ] } KSC { users = [ { user: "ksc", pass: "p" } ] exports [ { stream: "REQUEST" } ] imports [ { stream: { account: EFG, subject: "RESPONSE"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` sc := createJetStreamSuperClusterWithTemplate(t, tmpl, 5, 2) defer sc.shutdown() // Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, STL and KSC. var lnTmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var leafFrag = ` leaf { listen: 127.0.0.1:-1 remotes [ { urls: [ %s ] } { urls: [ %s ] } { urls: [ %s ] ; deny_export: [REQUEST, RESPONSE], deny_import: RESPONSE } ] }` // We want to have two leaf node connections that join to the same local account on the leafnode servers, // but connect to different accounts in different clusters. c1 := sc.clusters[0] // Will connect to account KSC c2 := sc.clusters[1] // Will connect to account STL genLeafTmpl := func(tmpl string) string { t.Helper() var ln1, ln2, ln3 []string for _, s := range c1.servers { if s.ClusterName() != c1.name { continue } ln := s.getOpts().LeafNode ln1 = append(ln1, fmt.Sprintf("nats://ksc:p@%s:%d", ln.Host, ln.Port)) } for _, s := range c2.servers { if s.ClusterName() != c2.name { continue } ln := s.getOpts().LeafNode ln2 = append(ln2, fmt.Sprintf("nats://stl:p@%s:%d", ln.Host, ln.Port)) ln3 = append(ln3, fmt.Sprintf("nats://efg:p@%s:%d", ln.Host, ln.Port)) } return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(leafFrag, strings.Join(ln1, ", "), strings.Join(ln2, ", "), strings.Join(ln3, ", ")), 1) } tmpl = strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "SA"), 1) tmpl = genLeafTmpl(tmpl) ln := createJetStreamCluster(t, tmpl, "SA", "SA-", 3, 22280, false) ln.waitOnClusterReady() defer ln.shutdown() for _, s := range ln.servers { checkLeafNodeConnectedCount(t, s, 3) } // Now connect DQ subscribers to each cluster but to the global account. // Create 5 clients for each cluster / account var c1c, c2c []*nats.Conn for i := 0; i < 5; i++ { nc1, _ := jsClientConnect(t, c1.randomServer(), nats.UserInfo("efg", "p")) defer nc1.Close() c1c = append(c1c, nc1) nc2, _ := jsClientConnect(t, c2.randomServer(), nats.UserInfo("efg", "p")) defer nc2.Close() c2c = append(c2c, nc2) } createSubs := func(num int, conns []*nats.Conn) (subs []*nats.Subscription) { for i := 0; i < num; i++ { nc := conns[rand.Intn(len(conns))] sub, err := nc.QueueSubscribeSync("REQUEST", "MC") require_NoError(t, err) subs = append(subs, sub) nc.Flush() } // Let subs propagate. time.Sleep(100 * time.Millisecond) return subs } closeSubs := func(subs []*nats.Subscription) { for _, sub := range subs { sub.Unsubscribe() } } // Simple test first. subs1 := createSubs(1, c1c) defer closeSubs(subs1) subs2 := createSubs(1, c2c) defer closeSubs(subs2) sendRequests := func(num int) { t.Helper() // Now connect to the leaf cluster and send some requests. nc, _ := jsClientConnect(t, ln.randomServer()) defer nc.Close() for i := 0; i < num; i++ { require_NoError(t, nc.Publish("REQUEST", []byte("HELP"))) } nc.Flush() } pending := func(subs []*nats.Subscription) (total int) { t.Helper() for _, sub := range subs { n, _, err := sub.Pending() require_NoError(t, err) total += n } return total } num := 1000 checkAllReceived := func() error { total := pending(subs1) + pending(subs2) if total == num { return nil } return fmt.Errorf("Not all received: %d vs %d", total, num) } checkBalanced := func(total, pc1, pc2 int) { t.Helper() tf := float64(total) e1 := tf * (float64(pc1) / 100.00) e2 := tf * (float64(pc2) / 100.00) delta := tf / 10 p1 := float64(pending(subs1)) if p1 < e1-delta || p1 > e1+delta { t.Fatalf("Value out of range for subs1, expected %v got %v", e1, p1) } p2 := float64(pending(subs2)) if p2 < e2-delta || p2 > e2+delta { t.Fatalf("Value out of range for subs2, expected %v got %v", e2, p2) } } // Now connect to the leaf cluster and send some requests. // Simple 50/50 sendRequests(num) checkFor(t, time.Second, 200*time.Millisecond, checkAllReceived) checkBalanced(num, 50, 50) closeSubs(subs1) closeSubs(subs2) // Now test unbalanced. 10/90 subs1 = createSubs(1, c1c) defer closeSubs(subs1) subs2 = createSubs(9, c2c) defer closeSubs(subs2) sendRequests(num) checkFor(t, time.Second, 200*time.Millisecond, checkAllReceived) checkBalanced(num, 10, 90) closeSubs(subs1) closeSubs(subs2) // Now test unbalanced. 80/20 subs1 = createSubs(80, c1c) defer closeSubs(subs1) subs2 = createSubs(20, c2c) defer closeSubs(subs2) sendRequests(num) checkFor(t, time.Second, 200*time.Millisecond, checkAllReceived) checkBalanced(num, 80, 20) // Now test draining the subs as we are sending from an initial balanced situation simulating a draining of a cluster. closeSubs(subs1) closeSubs(subs2) subs1, subs2 = nil, nil // These subs slightly different. var r1, r2 atomic.Uint64 for i := 0; i < 20; i++ { nc := c1c[rand.Intn(len(c1c))] sub, err := nc.QueueSubscribe("REQUEST", "MC", func(m *nats.Msg) { r1.Add(1) }) require_NoError(t, err) subs1 = append(subs1, sub) nc.Flush() nc = c2c[rand.Intn(len(c2c))] sub, err = nc.QueueSubscribe("REQUEST", "MC", func(m *nats.Msg) { r2.Add(1) }) require_NoError(t, err) subs2 = append(subs2, sub) nc.Flush() } // Let's them propagate time.Sleep(100 * time.Millisecond) defer closeSubs(subs1) defer closeSubs(subs2) nc, _ := jsClientConnect(t, ln.randomServer()) defer nc.Close() for i, dindex := 0, 1; i < num; i++ { require_NoError(t, nc.Publish("REQUEST", []byte("HELP"))) // Check if we have more to simulate draining. // Will drain within first ~100 requests using 20% rand test below. // Will leave 1 behind. if dindex < len(subs1)-1 && rand.Intn(6) > 4 { sub := subs1[dindex] dindex++ sub.Drain() } } nc.Flush() checkFor(t, time.Second, 200*time.Millisecond, func() error { total := int(r1.Load() + r2.Load()) if total == num { return nil } return fmt.Errorf("Not all received: %d vs %d", total, num) }) require_True(t, r2.Load() > r1.Load()) // Now check opposite flow for responses. // Create 10 subscribers. var rsubs []*nats.Subscription for i := 0; i < 10; i++ { nc, _ := jsClientConnect(t, ln.randomServer()) defer nc.Close() sub, err := nc.QueueSubscribeSync("RESPONSE", "SA") require_NoError(t, err) nc.Flush() rsubs = append(rsubs, sub) } // Let's them propagate time.Sleep(100 * time.Millisecond) nc, _ = jsClientConnect(t, ln.randomServer()) defer nc.Close() _, err := nc.SubscribeSync("RESPONSE") require_NoError(t, err) nc.Flush() // Now connect and send responses from EFG in cloud. nc, _ = jsClientConnect(t, sc.randomServer(), nats.UserInfo("efg", "p")) defer nc.Close() for i := 0; i < 100; i++ { require_NoError(t, nc.Publish("RESPONSE", []byte("OK"))) } nc.Flush() checkAllRespReceived := func() error { p := pending(rsubs) if p == 100 { return nil } return fmt.Errorf("Not all responses received: %d vs %d", p, 100) } checkFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived) } func TestLeafNodeWithWeightedDQResponsesWithStreamImportAccountsWithUnsub(t *testing.T) { var tmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { EFG { users = [ { user: "efg", pass: "p" } ] jetstream: enabled exports [ { stream: "RESPONSE" } ] } STL { users = [ { user: "stl", pass: "p" } ] imports [ { stream: { account: EFG, subject: "RESPONSE"} } ] } KSC { users = [ { user: "ksc", pass: "p" } ] imports [ { stream: { account: EFG, subject: "RESPONSE"} } ] } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } }` c := createJetStreamClusterWithTemplate(t, tmpl, "US-CENTRAL", 3) defer c.shutdown() // Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, STL and KSC. var lnTmpl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] }} ` var leafFrag = ` leaf { listen: 127.0.0.1:-1 remotes [ { urls: [ %s ] } ] }` genLeafTmpl := func(tmpl string) string { t.Helper() var ln []string for _, s := range c.servers { lno := s.getOpts().LeafNode ln = append(ln, fmt.Sprintf("nats://ksc:p@%s:%d", lno.Host, lno.Port)) } return strings.Replace(tmpl, "{{leaf}}", fmt.Sprintf(leafFrag, strings.Join(ln, ", ")), 1) } tmpl = strings.Replace(lnTmpl, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, "SA"), 1) tmpl = genLeafTmpl(tmpl) ln := createJetStreamCluster(t, tmpl, "SA", "SA-", 3, 22280, false) ln.waitOnClusterReady() defer ln.shutdown() for _, s := range ln.servers { checkLeafNodeConnectedCount(t, s, 1) } // Create 10 subscribers. var rsubs []*nats.Subscription closeSubs := func(subs []*nats.Subscription) { for _, sub := range subs { sub.Unsubscribe() } } checkAllRespReceived := func() error { t.Helper() var total int for _, sub := range rsubs { n, _, err := sub.Pending() require_NoError(t, err) total += n } if total == 100 { return nil } return fmt.Errorf("Not all responses received: %d vs %d", total, 100) } s := ln.randomServer() for i := 0; i < 4; i++ { nc, _ := jsClientConnect(t, s) defer nc.Close() sub, err := nc.QueueSubscribeSync("RESPONSE", "SA") require_NoError(t, err) nc.Flush() rsubs = append(rsubs, sub) } // Now connect and send responses from EFG in cloud. nc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("efg", "p")) defer nc.Close() for i := 0; i < 100; i++ { require_NoError(t, nc.Publish("RESPONSE", []byte("OK"))) } nc.Flush() // Make sure all received. checkFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived) checkAccountInterest := func(s *Server, accName string) *SublistResult { t.Helper() acc, err := s.LookupAccount(accName) require_NoError(t, err) acc.mu.RLock() r := acc.sl.Match("RESPONSE") acc.mu.RUnlock() return r } checkInterest := func() error { t.Helper() for _, s := range c.servers { if r := checkAccountInterest(s, "KSC"); len(r.psubs)+len(r.qsubs) > 0 { return fmt.Errorf("Subs still present for %q: %+v", "KSC", r) } if r := checkAccountInterest(s, "EFG"); len(r.psubs)+len(r.qsubs) > 0 { return fmt.Errorf("Subs still present for %q: %+v", "EFG", r) } } return nil } // Now unsub them and create new ones on a different server. closeSubs(rsubs) rsubs = rsubs[:0] // Also restart the server that we had all the rsubs on. s.Shutdown() s.WaitForShutdown() s = ln.restartServer(s) ln.waitOnClusterReady() ln.waitOnServerCurrent(s) checkFor(t, time.Second, 200*time.Millisecond, checkInterest) for i := 0; i < 4; i++ { nc, _ := jsClientConnect(t, s) defer nc.Close() sub, err := nc.QueueSubscribeSync("RESPONSE", "SA") require_NoError(t, err) nc.Flush() rsubs = append(rsubs, sub) } for i := 0; i < 100; i++ { require_NoError(t, nc.Publish("RESPONSE", []byte("OK"))) } nc.Flush() // Make sure all received. checkFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived) closeSubs(rsubs) checkFor(t, time.Second, 200*time.Millisecond, checkInterest) } func TestLeafNodeTwoRemotesToSameHubAccount(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 server_name: "hub" accounts { HA { users: [{user: ha, password: pwd}] } } leafnodes { port: -1 } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "spoke" accounts { A { users: [{user: A, password: pwd}] } B { users: [{user: B, password: pwd}] } C { users: [{user: C, password: pwd}] } } leafnodes { remotes [ { url: "nats://ha:pwd@127.0.0.1:%d" local: "A" } { url: "nats://ha:pwd@127.0.0.1:%d" local: "C" } ] } `, o1.LeafNode.Port, o1.LeafNode.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} s2.SetLogger(l, false, false) checkLeafNodeConnectedCount(t, s2, 2) // Make sure we don't get duplicate leafnode connection errors deadline := time.NewTimer(1500 * time.Millisecond) for done := false; !done; { select { case err := <-l.errCh: if strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) { t.Fatalf("Got error: %v", err) } case <-deadline.C: done = true } } nca := natsConnect(t, s2.ClientURL(), nats.UserInfo("A", "pwd")) defer nca.Close() suba := natsSubSync(t, nca, "A") ncb := natsConnect(t, s2.ClientURL(), nats.UserInfo("B", "pwd")) defer ncb.Close() subb := natsSubSync(t, ncb, "B") ncc := natsConnect(t, s2.ClientURL(), nats.UserInfo("C", "pwd")) defer ncc.Close() subc := natsSubSync(t, ncc, "C") subs := map[string]*nats.Subscription{"A": suba, "B": subb, "C": subc} for _, subj := range []string{"A", "C"} { checkSubInterest(t, s1, "HA", subj, time.Second) } nc := natsConnect(t, s1.ClientURL(), nats.UserInfo("ha", "pwd")) defer nc.Close() for _, subj := range []string{"A", "B", "C"} { natsPub(t, nc, subj, []byte("hello")) } for _, subj := range []string{"A", "B", "C"} { var expected bool if subj != "B" { expected = true } sub := subs[subj] if expected { natsNexMsg(t, sub, time.Second) } else { if _, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected timeout error, got %v", err) } } } } func TestLeafNodeTwoRemotesToSameHubAccountWithClusters(t *testing.T) { hubTmpl := ` port: -1 server_name: "%s" accounts { HA { users: [{user: HA, password: pwd}] } } cluster { name: "hub" port: -1 %s } leafnodes { port: -1 } ` confH1 := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, "H1", _EMPTY_))) sh1, oh1 := RunServerWithConfig(confH1) defer sh1.Shutdown() confH2 := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, "H2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", oh1.Cluster.Port)))) sh2, oh2 := RunServerWithConfig(confH2) defer sh2.Shutdown() checkClusterFormed(t, sh1, sh2) spokeTmpl := ` port: -1 server_name: "%s" accounts { A { users: [{user: A, password: pwd}] } B { users: [{user: B, password: pwd}] } } cluster { name: "spoke" port: -1 %s } leafnodes { remotes [ { url: "nats://HA:pwd@127.0.0.1:%d" local: "A" } { url: "nats://HA:pwd@127.0.0.1:%d" local: "B" } ] } ` for _, test := range []struct { name string sp2Leafport int }{ {"connect to different hub servers", oh2.LeafNode.Port}, {"connect to same hub server", oh1.LeafNode.Port}, } { t.Run(test.name, func(t *testing.T) { confSP1 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, "SP1", _EMPTY_, oh1.LeafNode.Port, oh1.LeafNode.Port))) sp1, osp1 := RunServerWithConfig(confSP1) defer sp1.Shutdown() confSP2 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, "SP2", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", osp1.Cluster.Port), test.sp2Leafport, test.sp2Leafport))) sp2, _ := RunServerWithConfig(confSP2) defer sp2.Shutdown() checkClusterFormed(t, sp1, sp2) checkLeafNodeConnectedCount(t, sp1, 2) checkLeafNodeConnectedCount(t, sp2, 2) var conns []*nats.Conn createConn := func(s *Server, user string) { t.Helper() nc := natsConnect(t, s.ClientURL(), nats.UserInfo(user, "pwd")) conns = append(conns, nc) } createConn(sh1, "HA") createConn(sh2, "HA") createConn(sp1, "A") createConn(sp2, "A") createConn(sp1, "B") createConn(sp2, "B") for _, nc := range conns { defer nc.Close() } check := func(subConn *nats.Conn, subj string, checkA, checkB bool) { t.Helper() sub := natsSubSync(t, subConn, subj) defer sub.Unsubscribe() checkSubInterest(t, sh1, "HA", subj, time.Second) checkSubInterest(t, sh2, "HA", subj, time.Second) if checkA { checkSubInterest(t, sp1, "A", subj, time.Second) checkSubInterest(t, sp2, "A", subj, time.Second) } if checkB { checkSubInterest(t, sp1, "B", subj, time.Second) checkSubInterest(t, sp2, "B", subj, time.Second) } for i, ncp := range conns { // Don't publish from account "A" connections if we are // dealing with account "B", and vice-versa. if !checkA && i >= 2 && i <= 3 { continue } if !checkB && i >= 4 { continue } natsPub(t, ncp, subj, []byte("hello")) natsNexMsg(t, sub, time.Second) // Make sure we don't get a duplicate if msg, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected message or error: msg=%v - err=%v", msg, err) } } } check(conns[0], "HA.1", true, true) check(conns[1], "HA.2", true, true) check(conns[2], "SPA.1", true, false) check(conns[3], "SPA.2", true, false) check(conns[4], "SPB.1", false, true) check(conns[5], "SPB.2", false, true) }) } } func TestLeafNodeSameLocalAccountToMultipleHubs(t *testing.T) { hub1Conf := createConfFile(t, []byte(` port: -1 server_name: hub1 accounts { hub1 { users: [{user: hub1, password: pwd}] } } leafnodes { port: -1 } `)) sh1, oh1 := RunServerWithConfig(hub1Conf) defer sh1.Shutdown() hub2Conf := createConfFile(t, []byte(` port: -1 server_name: hub2 accounts { hub2 { users: [{user: hub2, password: pwd}] } } leafnodes { port: -1 } `)) sh2, oh2 := RunServerWithConfig(hub2Conf) defer sh2.Shutdown() lconf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: leaf accounts { A { users: [{user: A, password: pwd}] } B { users: [{user: B, password: pwd}] } C { users: [{user: C, password: pwd}] } } leafnodes { port: -1 remotes [ { url: nats://hub1:pwd@127.0.0.1:%[1]d local: "A" } { url: nats://hub1:pwd@127.0.0.1:%[1]d local: "C" } { url: nats://hub2:pwd@127.0.0.1:%[2]d local: "A" } { url: nats://hub2:pwd@127.0.0.1:%[2]d local: "B" } ] } `, oh1.LeafNode.Port, oh2.LeafNode.Port))) s, _ := RunServerWithConfig(lconf) defer s.Shutdown() // The leafnode to hub1 should have 2 connections (A and C) // while the one to hub2 should have 2 connections (A and B) checkLeafNodeConnectedCount(t, sh1, 2) checkLeafNodeConnectedCount(t, sh2, 2) checkLeafNodeConnectedCount(t, s, 4) nca := natsConnect(t, s.ClientURL(), nats.UserInfo("A", "pwd")) defer nca.Close() suba := natsSubSync(t, nca, "A") ncb := natsConnect(t, s.ClientURL(), nats.UserInfo("B", "pwd")) defer ncb.Close() subb := natsSubSync(t, ncb, "B") ncc := natsConnect(t, s.ClientURL(), nats.UserInfo("C", "pwd")) defer ncc.Close() subc := natsSubSync(t, ncc, "C") checkSubInterest(t, sh1, "hub1", "A", time.Second) checkSubNoInterest(t, sh1, "hub1", "B", time.Second) checkSubInterest(t, sh1, "hub1", "C", time.Second) checkSubInterest(t, sh2, "hub2", "A", time.Second) checkSubInterest(t, sh2, "hub2", "B", time.Second) checkSubNoInterest(t, sh2, "hub2", "C", time.Second) nch1 := natsConnect(t, sh1.ClientURL(), nats.UserInfo("hub1", "pwd")) defer nch1.Close() nch2 := natsConnect(t, sh2.ClientURL(), nats.UserInfo("hub2", "pwd")) defer nch2.Close() checkNoMsg := func(sub *nats.Subscription) { t.Helper() if msg, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Unexpected message: %s", msg.Data) } } checkSub := func(sub *nats.Subscription, subj, payload string) { t.Helper() msg := natsNexMsg(t, sub, time.Second) require_Equal(t, subj, msg.Subject) require_Equal(t, payload, string(msg.Data)) // Make sure we don't get duplicates checkNoMsg(sub) } natsPub(t, nch1, "A", []byte("msgA1")) checkSub(suba, "A", "msgA1") natsPub(t, nch1, "B", []byte("msgB1")) checkNoMsg(subb) natsPub(t, nch1, "C", []byte("msgC1")) checkSub(subc, "C", "msgC1") natsPub(t, nch2, "A", []byte("msgA2")) checkSub(suba, "A", "msgA2") natsPub(t, nch2, "B", []byte("msgB2")) checkSub(subb, "B", "msgB2") natsPub(t, nch2, "C", []byte("msgC2")) checkNoMsg(subc) } func TestLeafNodeSlowConsumer(t *testing.T) { ao := DefaultOptions() ao.LeafNode.Host = "127.0.0.1" ao.LeafNode.Port = -1 ao.WriteDeadline = 1 * time.Millisecond a := RunServer(ao) defer a.Shutdown() c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", ao.LeafNode.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } time.Sleep(5 * time.Millisecond) a.mu.Lock() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { a.grMu.Lock() defer a.grMu.Unlock() for _, cli := range a.grTmpClients { cli.out.wdl = time.Nanosecond return nil } return nil }) a.mu.Unlock() // Only leafnode slow consumers that made it past connect are tracked // in the slow consumers counter. if _, err := c.Write([]byte("CONNECT {}\r\n")); err != nil { t.Fatalf("Error writing connect: %v", err) } // Read info br := bufio.NewReader(c) br.ReadLine() for i := 0; i < 10; i++ { if _, err := c.Write([]byte("PING\r\n")); err != nil { t.Fatalf("Unexpected error writing PING: %v", err) } } defer c.Close() timeout := time.Now().Add(time.Second) var ( got uint64 expected uint64 = 1 ) for time.Now().Before(timeout) { got = a.NumSlowConsumersLeafs() if got == expected { return } time.Sleep(1 * time.Millisecond) } t.Fatalf("Timed out waiting for slow consumer leafnodes, got: %v, expected: %v", got, expected) } // https://github.com/nats-io/nats-server/issues/4367 func TestLeafNodeDQMultiAccountExportImport(t *testing.T) { bConf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: cluster-b-0 accounts { $SYS: { users: [ { user: admin, password: pwd } ] }, AGG: { exports: [ { service: "PING.>" } ] users: [ { user: agg, password: agg } ] } } leaf { listen: 127.0.0.1:-1 } `)) sb, ob := RunServerWithConfig(bConf) defer sb.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { store_dir: '%s' } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS: { users: [ { user: admin, password: pwd } ] }, A: { mappings: { "A.>" : ">" } exports: [ { service: A.> } ] users: [ { user: a, password: a } ] }, AGG: { imports: [ { service: { subject: A.>, account: A } } ] users: [ { user: agg, password: agg } ] }, } leaf { remotes: [ { urls: [ nats-leaf://agg:agg@127.0.0.1:{LEAF_PORT} ] account: AGG } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", ob.LeafNode.Port), 1) c := createJetStreamCluster(t, tmpl, "cluster-a", "cluster-a-", 3, 22110, false) defer c.shutdown() // Make sure all servers are connected via leafnode to the hub, the b server. for _, s := range c.servers { checkLeafNodeConnectedCount(t, s, 1) } // Connect to a server in the cluster and create a DQ listener. nc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("a", "a")) defer nc.Close() var got atomic.Int32 natsQueueSub(t, nc, "PING", "Q", func(m *nats.Msg) { got.Add(1) m.Respond([]byte("REPLY")) }) // Now connect to B and send the request. ncb, _ := jsClientConnect(t, sb, nats.UserInfo("agg", "agg")) defer ncb.Close() _, err := ncb.Request("A.PING", []byte("REQUEST"), time.Second) require_NoError(t, err) require_Equal(t, got.Load(), 1) } // https://github.com/nats-io/nats-server/issues/4934 func TestLeafNodeServerReloadSubjectMappings(t *testing.T) { stmpl := ` listen: 127.0.0.1:-1 server_name: test-server mappings = { "source1": "target" } leaf { listen: 127.0.0.1:-1 } ` conf := createConfFile(t, []byte(stmpl)) s, o := RunServerWithConfig(conf) defer s.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: test-leaf leaf { remotes: [ { urls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ] } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", o.LeafNode.Port), 1) lConf := createConfFile(t, []byte(tmpl)) l, _ := RunServerWithConfig(lConf) defer l.Shutdown() checkLeafNodeConnected(t, l) // Create our subscriber. nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub := natsSubSync(t, nc, "target") natsFlush(t, nc) // Create our publisher. ncl := natsConnect(t, l.ClientURL()) defer ncl.Close() // Publish our message. ncl.Publish("source1", []byte("OK")) // Make sure we receive it. checkSubsPending(t, sub, 1) // Now change mapping. reloadUpdateConfig(t, s, conf, strings.Replace(stmpl, "source1", "source2", 1)) // Also make sure we do not have subscription interest for source1 on leaf anymore. checkSubInterest(t, l, globalAccountName, "source2", 2*time.Second) // Publish our new message. ncl.Publish("source2", []byte("OK")) // Make sure we receive it. checkSubsPending(t, sub, 2) // Also make sure we do not have subscription interest for source1 on leaf anymore. checkSubNoInterest(t, l, globalAccountName, "source1", 2*time.Second) } // https://github.com/nats-io/nats-server/issues/5099 func TestLeafNodeServerReloadSubjectMappingsWithSameSubject(t *testing.T) { stmpl := ` listen: 127.0.0.1:-1 server_name: test-server mappings = { "source": "target1" } leaf { listen: 127.0.0.1:-1 } ` conf := createConfFile(t, []byte(stmpl)) s, o := RunServerWithConfig(conf) defer s.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: test-leaf leaf { remotes: [ { urls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ] } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", o.LeafNode.Port), 1) lConf := createConfFile(t, []byte(tmpl)) l, _ := RunServerWithConfig(lConf) defer l.Shutdown() checkLeafNodeConnected(t, l) // Create our subscriber. nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub1 := natsSubSync(t, nc, "target1") sub2 := natsSubSync(t, nc, "target2") natsFlush(t, nc) // Create our publisher. ncl := natsConnect(t, l.ClientURL()) defer ncl.Close() // Publish our message. ncl.Publish("source", []byte("OK")) // Make sure we receive it. checkSubsPending(t, sub1, 1) // Make sure the other does not. checkSubsPending(t, sub2, 0) // Now change mapping, but only the "to" subject, keeping same "from" reloadUpdateConfig(t, s, conf, strings.Replace(stmpl, "target1", "target2", 1)) checkLeafNodeConnected(t, l) // Publish our new message. ncl.Publish("source", []byte("OK")) // Make sure we receive it. checkSubsPending(t, sub2, 1) // Make sure the other does not. checkSubsPending(t, sub1, 1) } func TestLeafNodeNkeyAuth(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: test-server leaf { listen: 127.0.0.1:-1 authorization: { nkey: UCSTG5CRF5GEJERAFKUUYRODGABTBVWY2NPE4GGKRQVQOH74PIAKTVKO } } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: test-leaf leaf { remotes: [ { url: nats-leaf://127.0.0.1:{LEAF_PORT} seed: SUACJN3OSKWWPQXME4JUNFJ3PARXPO657GGNWNU7PK7G3AUQQYHLW26XH4 } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", o.LeafNode.Port), 1) lConf := createConfFile(t, []byte(tmpl)) l, _ := RunServerWithConfig(lConf) defer l.Shutdown() checkLeafNodeConnected(t, l) } func TestLeafNodeAccountNkeysAuth(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: test-server leaf { listen: 127.0.0.1:-1 } accounts { A { users [ {nkey: UCSTG5CRF5GEJERAFKUUYRODGABTBVWY2NPE4GGKRQVQOH74PIAKTVKO } ] } } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: test-leaf leaf { remotes: [ { url: nats-leaf://127.0.0.1:{LEAF_PORT} seed: SUACJN3OSKWWPQXME4JUNFJ3PARXPO657GGNWNU7PK7G3AUQQYHLW26XH4 } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", o.LeafNode.Port), 1) lConf := createConfFile(t, []byte(tmpl)) l, _ := RunServerWithConfig(lConf) defer l.Shutdown() checkLeafNodeConnected(t, l) } // https://github.com/nats-io/nats-server/issues/5117 func TestLeafNodeLoopDetectionOnActualLoop(t *testing.T) { // Setup: B --[leaf]--> A C --[leaf]--> A C --[leaf] --> B accConf := ` accounts: { APP: { users: [ { user:u, password: u, permissions: { publish = "u.>", subscribe = "u.>" }} ] } $SYS: { users = [ {user: "s", password: "s"} ] } }` confA := createConfFile(t, []byte(fmt.Sprintf(` server_name: a1 port: -1 cluster: { name: A } leafnodes { port: 17422 } %s`, accConf))) confB := createConfFile(t, []byte(fmt.Sprintf(` server_name: b1 port: -1 cluster: { name: B } leafnodes { port: 17432 remotes [ { urls: ["nats-leaf://u:u@localhost:17422"], account: "APP" } ] reconnect: "2s" } %s`, accConf))) confC := createConfFile(t, []byte(fmt.Sprintf(` server_name: c1 port: -1 cluster: { name: C } leafnodes { port: 17442 remotes [ { urls: ["nats-leaf://u:u@localhost:17422"], account: "APP" } # This one creates the loop { urls: ["nats-leaf://u:u@localhost:17432"], account: "APP" } ] reconnect: "0.5s" } %s`, accConf))) // Start order will be B -> C -> A // We will force C to connect to A first before B using different reconnect intervals. // If B connects first we detect loops fine. If C connects first we do not. srvB, _ := RunServerWithConfig(confB) defer srvB.Shutdown() lb := &loopDetectedLogger{ch: make(chan string, 1)} srvB.SetLogger(lb, false, false) srvC, _ := RunServerWithConfig(confC) defer srvC.Shutdown() lc := &loopDetectedLogger{ch: make(chan string, 1)} srvC.SetLogger(lc, false, false) // C should connect to B checkLeafNodeConnectedCount(t, srvC, 1) srvA, _ := RunServerWithConfig(confA) defer srvA.Shutdown() la := &loopDetectedLogger{ch: make(chan string, 1)} srvA.SetLogger(la, false, false) select { case <-la.ch: case <-lb.ch: case <-lc.ch: case <-time.After(5 * time.Second): t.Fatalf("Did not get any error regarding loop") } } // https://github.com/nats-io/nats-server/issues/5473 func TestLeafNodeDupeDeliveryQueueSubAndPlainSub(t *testing.T) { clusterCommonConf := ` accounts: { tenant: { users: [ { user:t, password: t } ] exports: [{stream: system-a.events.>}] } system-a: { users: [ { user:sa, password: sa } ] imports: [ {stream: {subject: system-a.events.>, account: tenant}, prefix: tenant} ] } $SYS: { users = [ {user: "s", password: "s"} ] } } leafnodes { remotes: [{ urls: [ "nats-leaf://sa:sa@127.0.0.1:17422" ] account: system-a }] }` confCluster0 := createConfFile(t, []byte(fmt.Sprintf(` server_name: a-0 port: -1 cluster: { name: cluster-a listen: 127.0.0.1:16122 routes = [ nats://127.0.0.1:16123 ] pool_size: -1 } %s`, clusterCommonConf))) confCluster1 := createConfFile(t, []byte(fmt.Sprintf(` server_name: a-1 port: -1 cluster: { name: cluster-a listen: 127.0.0.1:16123 routes = [ nats://127.0.0.1:16122 ] pool_size: -1 } %s`, clusterCommonConf))) serverB := createConfFile(t, []byte(` server_name: b port: -1 leafnodes: { port: 17422 } accounts: { system-a: { users: [ { user: sa, password: sa } ] exports: [{stream: *.system-a.>}] } system-b: { users: [ { user: sb, password: sb } ] imports: [ {stream: {subject: *.system-a.>, account: system-a }}] } $SYS: { users = [ {user: "s", password: "s"} ] } }`)) // Start server B srvB, _ := RunServerWithConfig(serverB) defer srvB.Shutdown() // Start the cluster servers. srvA0, _ := RunServerWithConfig(confCluster0) defer srvA0.Shutdown() // Make sure this is connected first before starting the second server in cluster A. checkLeafNodeConnectedCount(t, srvB, 1) // Start second A server. srvA1, _ := RunServerWithConfig(confCluster1) defer srvA1.Shutdown() // Make sure they are routed together. checkNumRoutes(t, srvA0, 1) checkNumRoutes(t, srvA1, 1) // Make sure each cluster server is connected to server B. checkLeafNodeConnectedCount(t, srvB, 2) // Create plain subscriber on server B attached to system-b account. ncB := natsConnect(t, srvB.ClientURL(), nats.UserInfo("sb", "sb")) defer ncB.Close() sub, err := ncB.SubscribeSync("*.system-a.events.>") require_NoError(t, err) // Create a new sub that has a queue group as well. subq, err := ncB.QueueSubscribeSync("*.system-a.events.objectnotfound", "SYSB") require_NoError(t, err) ncB.Flush() time.Sleep(250 * time.Millisecond) // Connect to cluster A ncA := natsConnect(t, srvA0.ClientURL(), nats.UserInfo("t", "t")) defer ncA.Close() err = ncA.Publish("system-a.events.objectnotfound", []byte("EventA")) require_NoError(t, err) ncA.Flush() // Wait for them to be received. time.Sleep(250 * time.Millisecond) n, _, err := sub.Pending() require_NoError(t, err) require_Equal(t, n, 1) n, _, err = subq.Pending() require_NoError(t, err) require_Equal(t, n, 1) } func TestLeafNodeServerKickClient(t *testing.T) { stmpl := ` listen: 127.0.0.1:-1 server_name: test-server leaf { listen: 127.0.0.1:-1 } ` conf := createConfFile(t, []byte(stmpl)) s, o := RunServerWithConfig(conf) defer s.Shutdown() tmpl := ` listen: 127.0.0.1:-1 server_name: test-leaf leaf { remotes: [ { urls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ] } ] } ` tmpl = strings.Replace(tmpl, "{LEAF_PORT}", fmt.Sprintf("%d", o.LeafNode.Port), 1) lConf := createConfFile(t, []byte(tmpl)) l, _ := RunServerWithConfig(lConf) defer l.Shutdown() checkLeafNodeConnected(t, l) // We want to make sure we can kick the leafnode connections as well as client connections. conns, err := s.Connz(&ConnzOptions{Account: globalAccountName}) require_NoError(t, err) require_Equal(t, len(conns.Conns), 1) lid := conns.Conns[0].Cid disconnectTime := time.Now() err = s.DisconnectClientByID(lid) require_NoError(t, err) // Wait until we are reconnected. checkLeafNodeConnected(t, s) // Look back up again and make sure start time indicates a restart, meaning kick worked. conns, err = s.Connz(&ConnzOptions{Account: globalAccountName}) require_NoError(t, err) require_Equal(t, len(conns.Conns), 1) ln := conns.Conns[0] require_True(t, lid != ln.Cid) require_True(t, ln.Start.After(disconnectTime)) } func TestLeafCredFormatting(t *testing.T) { //create the operator/sys/account tree oKP, err := nkeys.CreateOperator() require_NoError(t, err) oPK, err := oKP.PublicKey() require_NoError(t, err) oc := jwt.NewOperatorClaims(oPK) oc.Name = "O" oJWT, err := oc.Encode(oKP) require_NoError(t, err) sysKP, err := nkeys.CreateAccount() require_NoError(t, err) sysPK, err := sysKP.PublicKey() require_NoError(t, err) sys := jwt.NewAccountClaims(sysPK) sys.Name = "SYS" sysJWT, err := sys.Encode(oKP) require_NoError(t, err) aKP, err := nkeys.CreateAccount() require_NoError(t, err) aPK, err := aKP.PublicKey() require_NoError(t, err) ac := jwt.NewAccountClaims(aPK) ac.Name = "A" aJWT, err := ac.Encode(oKP) require_NoError(t, err) uKP, err := nkeys.CreateUser() require_NoError(t, err) uSeed, err := uKP.Seed() require_NoError(t, err) uPK, err := uKP.PublicKey() require_NoError(t, err) // build the config stmpl := fmt.Sprintf(` listen: 127.0.0.1:-1 operator: %s system_account: %s resolver: MEM resolver_preload: { %s: %s %s: %s } leaf { listen: 127.0.0.1:-1 } `, oJWT, sysPK, sysPK, sysJWT, aPK, aJWT) conf := createConfFile(t, []byte(stmpl)) s, o := RunServerWithConfig(conf) defer s.Shutdown() // create the leaf node // generate the user credentials uc := jwt.NewUserClaims(uPK) uc.Name = "U" uc.Limits.Data = -1 uc.Limits.Payload = -1 uc.Permissions.Pub.Allow.Add(">") uc.Permissions.Sub.Allow.Add(">") uJWT, err := uc.Encode(aKP) require_NoError(t, err) runLeaf := func(t *testing.T, creds []byte) { file, err := os.CreateTemp("", "tmp-*.creds") require_NoError(t, err) _, err = file.Write(creds) require_NoError(t, err) require_NoError(t, file.Close()) template := fmt.Sprintf(` listen: 127.0.0.1:-1 leaf { remotes: [ { urls: [ nats-leaf://127.0.0.1:%d ] credentials: "%s" } ] }`, o.LeafNode.Port, file.Name()) conf := createConfFile(t, []byte(template)) leaf, _ := RunServerWithConfig(conf) defer leaf.Shutdown() defer os.Remove(file.Name()) checkLeafNodeConnected(t, leaf) } creds, err := jwt.FormatUserConfig(uJWT, uSeed) require_NoError(t, err) runLeaf(t, creds) runLeaf(t, bytes.ReplaceAll(creds, []byte{'\n'}, []byte{'\r', '\n'})) } func TestLeafNodePermissionWithLiteralSubjectAndQueueInterest(t *testing.T) { hconf := createConfFile(t, []byte(` server_name: "HUB" listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" } accounts { A { users: [ { user: "user", password: "pwd", permissions: { subscribe: { allow: ["_INBOX.>", "my.subject"] } publish: {allow: [">"]} } } ] } } `)) hub, ohub := RunServerWithConfig(hconf) defer hub.Shutdown() lconf := createConfFile(t, []byte(fmt.Sprintf(` server_name: "LEAF" listen: "127.0.0.1:-1" leafnodes { remotes: [ {url: "nats://user:pwd@127.0.0.1:%d", account: A} ] } accounts { A { users: [{user: user, password: pwd}] } } `, ohub.LeafNode.Port))) leaf, _ := RunServerWithConfig(lconf) defer leaf.Shutdown() checkLeafNodeConnected(t, hub) checkLeafNodeConnected(t, leaf) ncLeaf := natsConnect(t, leaf.ClientURL(), nats.UserInfo("user", "pwd")) defer ncLeaf.Close() natsQueueSub(t, ncLeaf, "my.subject", "queue", func(m *nats.Msg) { m.Respond([]byte("OK")) }) natsFlush(t, ncLeaf) ncHub := natsConnect(t, hub.ClientURL(), nats.UserInfo("user", "pwd")) defer ncHub.Close() resp, err := ncHub.Request("my.subject", []byte("hello"), time.Second) require_NoError(t, err) require_Equal(t, "OK", string(resp.Data)) } func TestLeafNodeConnectionSucceedsEvenWithDelayedFirstINFO(t *testing.T) { for _, test := range []struct { name string websocket bool }{ {"regular", false}, {"websocket", true}, } { t.Run(test.name, func(t *testing.T) { ob := DefaultOptions() ob.ServerName = "HUB" ob.LeafNode.Host = "127.0.0.1" ob.LeafNode.Port = -1 ob.LeafNode.AuthTimeout = 10 if test.websocket { ob.Websocket.Host = "127.0.0.1" ob.Websocket.Port = -1 ob.Websocket.HandshakeTimeout = 10 * time.Second ob.Websocket.AuthTimeout = 10 ob.Websocket.NoTLS = true } sb := RunServer(ob) defer sb.Shutdown() var port int var scheme string if test.websocket { port = ob.Websocket.Port scheme = wsSchemePrefix } else { port = ob.LeafNode.Port scheme = "nats" } urlStr := fmt.Sprintf("%s://127.0.0.1:%d", scheme, port) proxy := createNetProxy(1100*time.Millisecond, 1024*1024*1024, 1024*1024*1024, urlStr, true) defer proxy.stop() proxyURL := proxy.clientURL() _, proxyPort, err := net.SplitHostPort(proxyURL[len(scheme)+3:]) require_NoError(t, err) lnBURL, err := url.Parse(fmt.Sprintf("%s://127.0.0.1:%s", scheme, proxyPort)) require_NoError(t, err) oa := DefaultOptions() oa.ServerName = "SPOKE" oa.Cluster.Name = "xyz" remote := &RemoteLeafOpts{ URLs: []*url.URL{lnBURL}, FirstInfoTimeout: 3 * time.Second, } oa.LeafNode.Remotes = []*RemoteLeafOpts{remote} sa := RunServer(oa) defer sa.Shutdown() checkLeafNodeConnected(t, sa) }) } } type captureLeafConnClosed struct { DummyLogger ch chan struct{} } func (l *captureLeafConnClosed) Noticef(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "Leafnode connection closed: Read Error") { select { case l.ch <- struct{}{}: default: } } } func TestLeafNodeDetectsStaleConnectionIfNoInfo(t *testing.T) { for _, test := range []struct { name string websocket bool }{ {"regular", false}, {"websocket", true}, } { t.Run(test.name, func(t *testing.T) { l, err := net.Listen("tcp", "127.0.0.1:0") require_NoError(t, err) defer l.Close() ch := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() c, err := l.Accept() if err != nil { return } defer c.Close() <-ch }() var scheme string if test.websocket { scheme = wsSchemePrefix } else { scheme = "nats" } urlStr := fmt.Sprintf("%s://%s", scheme, l.Addr()) lnBURL, err := url.Parse(urlStr) require_NoError(t, err) oa := DefaultOptions() oa.ServerName = "SPOKE" oa.Cluster.Name = "xyz" remote := &RemoteLeafOpts{ URLs: []*url.URL{lnBURL}, FirstInfoTimeout: 250 * time.Millisecond, } oa.LeafNode.Remotes = []*RemoteLeafOpts{remote} oa.DisableShortFirstPing = false oa.NoLog = false sa, err := NewServer(oa) require_NoError(t, err) defer sa.Shutdown() log := &captureLeafConnClosed{ch: make(chan struct{}, 1)} sa.SetLogger(log, false, false) sa.Start() select { case <-log.ch: // OK case <-time.After(750 * time.Millisecond): t.Fatalf("Connection was not closed") } sa.Shutdown() close(ch) wg.Wait() sa.WaitForShutdown() }) } } nats-server-2.10.27/server/log.go000066400000000000000000000171111477524627100165720ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "io" "os" "sync/atomic" "time" srvlog "github.com/nats-io/nats-server/v2/logger" ) // Logger interface of the NATS Server type Logger interface { // Log a notice statement Noticef(format string, v ...any) // Log a warning statement Warnf(format string, v ...any) // Log a fatal error Fatalf(format string, v ...any) // Log an error Errorf(format string, v ...any) // Log a debug statement Debugf(format string, v ...any) // Log a trace statement Tracef(format string, v ...any) } // ConfigureLogger configures and sets the logger for the server. func (s *Server) ConfigureLogger() { var ( log Logger // Snapshot server options. opts = s.getOpts() ) if opts.NoLog { return } syslog := opts.Syslog if isWindowsService() && opts.LogFile == "" { // Enable syslog if no log file is specified and we're running as a // Windows service so that logs are written to the Windows event log. syslog = true } if opts.LogFile != "" { log = srvlog.NewFileLogger(opts.LogFile, opts.Logtime, opts.Debug, opts.Trace, true, srvlog.LogUTC(opts.LogtimeUTC)) if opts.LogSizeLimit > 0 { if l, ok := log.(*srvlog.Logger); ok { l.SetSizeLimit(opts.LogSizeLimit) } } if opts.LogMaxFiles > 0 { if l, ok := log.(*srvlog.Logger); ok { al := int(opts.LogMaxFiles) if int64(al) != opts.LogMaxFiles { // set to default (no max) on overflow al = 0 } l.SetMaxNumFiles(al) } } } else if opts.RemoteSyslog != "" { log = srvlog.NewRemoteSysLogger(opts.RemoteSyslog, opts.Debug, opts.Trace) } else if syslog { log = srvlog.NewSysLogger(opts.Debug, opts.Trace) } else { colors := true // Check to see if stderr is being redirected and if so turn off color // Also turn off colors if we're running on Windows where os.Stderr.Stat() returns an invalid handle-error stat, err := os.Stderr.Stat() if err != nil || (stat.Mode()&os.ModeCharDevice) == 0 { colors = false } log = srvlog.NewStdLogger(opts.Logtime, opts.Debug, opts.Trace, colors, true, srvlog.LogUTC(opts.LogtimeUTC)) } s.SetLoggerV2(log, opts.Debug, opts.Trace, opts.TraceVerbose) } // Returns our current logger. func (s *Server) Logger() Logger { s.logging.Lock() defer s.logging.Unlock() return s.logging.logger } // SetLogger sets the logger of the server func (s *Server) SetLogger(logger Logger, debugFlag, traceFlag bool) { s.SetLoggerV2(logger, debugFlag, traceFlag, false) } // SetLogger sets the logger of the server func (s *Server) SetLoggerV2(logger Logger, debugFlag, traceFlag, sysTrace bool) { if debugFlag { atomic.StoreInt32(&s.logging.debug, 1) } else { atomic.StoreInt32(&s.logging.debug, 0) } if traceFlag { atomic.StoreInt32(&s.logging.trace, 1) } else { atomic.StoreInt32(&s.logging.trace, 0) } if sysTrace { atomic.StoreInt32(&s.logging.traceSysAcc, 1) } else { atomic.StoreInt32(&s.logging.traceSysAcc, 0) } s.logging.Lock() if s.logging.logger != nil { // Check to see if the logger implements io.Closer. This could be a // logger from another process embedding the NATS server or a dummy // test logger that may not implement that interface. if l, ok := s.logging.logger.(io.Closer); ok { if err := l.Close(); err != nil { s.Errorf("Error closing logger: %v", err) } } } s.logging.logger = logger s.logging.Unlock() } // ReOpenLogFile if the logger is a file based logger, close and re-open the file. // This allows for file rotation by 'mv'ing the file then signaling // the process to trigger this function. func (s *Server) ReOpenLogFile() { // Check to make sure this is a file logger. s.logging.RLock() ll := s.logging.logger s.logging.RUnlock() if ll == nil { s.Noticef("File log re-open ignored, no logger") return } // Snapshot server options. opts := s.getOpts() if opts.LogFile == "" { s.Noticef("File log re-open ignored, not a file logger") } else { fileLog := srvlog.NewFileLogger( opts.LogFile, opts.Logtime, opts.Debug, opts.Trace, true, srvlog.LogUTC(opts.LogtimeUTC), ) s.SetLogger(fileLog, opts.Debug, opts.Trace) if opts.LogSizeLimit > 0 { fileLog.SetSizeLimit(opts.LogSizeLimit) } s.Noticef("File log re-opened") } } // Noticef logs a notice statement func (s *Server) Noticef(format string, v ...any) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Noticef(format, v...) }, format, v...) } // Errorf logs an error func (s *Server) Errorf(format string, v ...any) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Errorf(format, v...) }, format, v...) } // Error logs an error with a scope func (s *Server) Errors(scope any, e error) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Errorf(format, v...) }, "%s - %s", scope, UnpackIfErrorCtx(e)) } // Error logs an error with a context func (s *Server) Errorc(ctx string, e error) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Errorf(format, v...) }, "%s: %s", ctx, UnpackIfErrorCtx(e)) } // Error logs an error with a scope and context func (s *Server) Errorsc(scope any, ctx string, e error) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Errorf(format, v...) }, "%s - %s: %s", scope, ctx, UnpackIfErrorCtx(e)) } // Warnf logs a warning error func (s *Server) Warnf(format string, v ...any) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Warnf(format, v...) }, format, v...) } func (s *Server) rateLimitFormatWarnf(format string, v ...any) { if _, loaded := s.rateLimitLogging.LoadOrStore(format, time.Now()); loaded { return } statement := fmt.Sprintf(format, v...) s.Warnf("%s", statement) } func (s *Server) RateLimitWarnf(format string, v ...any) { statement := fmt.Sprintf(format, v...) if _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { return } s.Warnf("%s", statement) } func (s *Server) RateLimitDebugf(format string, v ...any) { statement := fmt.Sprintf(format, v...) if _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded { return } s.Debugf("%s", statement) } // Fatalf logs a fatal error func (s *Server) Fatalf(format string, v ...any) { s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Fatalf(format, v...) }, format, v...) } // Debugf logs a debug statement func (s *Server) Debugf(format string, v ...any) { if atomic.LoadInt32(&s.logging.debug) == 0 { return } s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Debugf(format, v...) }, format, v...) } // Tracef logs a trace statement func (s *Server) Tracef(format string, v ...any) { if atomic.LoadInt32(&s.logging.trace) == 0 { return } s.executeLogCall(func(logger Logger, format string, v ...any) { logger.Tracef(format, v...) }, format, v...) } func (s *Server) executeLogCall(f func(logger Logger, format string, v ...any), format string, args ...any) { s.logging.RLock() defer s.logging.RUnlock() if s.logging.logger == nil { return } f(s.logging.logger, format, args...) } nats-server-2.10.27/server/log_test.go000066400000000000000000000264761477524627100176470ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "os" "path/filepath" "runtime" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/internal/testhelper" "github.com/nats-io/nats-server/v2/logger" ) func TestSetLogger(t *testing.T) { server := &Server{} defer server.SetLogger(nil, false, false) dl := &DummyLogger{} server.SetLogger(dl, true, true) // We assert that the logger has change to the DummyLogger _ = server.logging.logger.(*DummyLogger) if server.logging.debug != 1 { t.Fatalf("Expected debug 1, received value %d\n", server.logging.debug) } if server.logging.trace != 1 { t.Fatalf("Expected trace 1, received value %d\n", server.logging.trace) } // Check traces expectedStr := "This is a Notice" server.Noticef(expectedStr) dl.CheckContent(t, expectedStr) expectedStr = "This is an Error" server.Errorf(expectedStr) dl.CheckContent(t, expectedStr) expectedStr = "This is a Fatal" server.Fatalf(expectedStr) dl.CheckContent(t, expectedStr) expectedStr = "This is a Debug" server.Debugf(expectedStr) dl.CheckContent(t, expectedStr) expectedStr = "This is a Trace" server.Tracef(expectedStr) dl.CheckContent(t, expectedStr) expectedStr = "This is a Warning" server.Tracef(expectedStr) dl.CheckContent(t, expectedStr) // Make sure that we can reset to fal server.SetLogger(dl, false, false) if server.logging.debug != 0 { t.Fatalf("Expected debug 0, got %v", server.logging.debug) } if server.logging.trace != 0 { t.Fatalf("Expected trace 0, got %v", server.logging.trace) } // Now, Debug and Trace should not produce anything dl.Msg = "" server.Debugf("This Debug should not be traced") dl.CheckContent(t, "") server.Tracef("This Trace should not be traced") dl.CheckContent(t, "") } type DummyLogger = testhelper.DummyLogger func TestReOpenLogFile(t *testing.T) { // We can't rename the file log when still opened on Windows, so skip if runtime.GOOS == "windows" { t.SkipNow() } s := &Server{opts: &Options{}} defer s.SetLogger(nil, false, false) // First check with no logger s.SetLogger(nil, false, false) s.ReOpenLogFile() // Then when LogFile is not provided. dl := &DummyLogger{} s.SetLogger(dl, false, false) s.ReOpenLogFile() dl.CheckContent(t, "File log re-open ignored, not a file logger") // Set a File log s.opts.LogFile = filepath.Join(t.TempDir(), "test.log") fileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC)) s.SetLogger(fileLog, false, false) // Add some log expectedStr := "This is a Notice" s.Noticef(expectedStr) // Check content of log buf, err := os.ReadFile(s.opts.LogFile) if err != nil { t.Fatalf("Error reading file: %v", err) } if !strings.Contains(string(buf), expectedStr) { t.Fatalf("Expected log to contain: %q, got %q", expectedStr, string(buf)) } // Close the file and rename it if err := os.Rename(s.opts.LogFile, s.opts.LogFile+".bak"); err != nil { t.Fatalf("Unable to rename log file: %v", err) } // Now re-open LogFile s.ReOpenLogFile() // Content should indicate that we have re-opened the log buf, err = os.ReadFile(s.opts.LogFile) if err != nil { t.Fatalf("Error reading file: %v", err) } if strings.HasSuffix(string(buf), "File log-reopened") { t.Fatalf("File should indicate that file log was re-opened, got: %v", string(buf)) } // Make sure we can append to the log s.Noticef("New message") buf, err = os.ReadFile(s.opts.LogFile) if err != nil { t.Fatalf("Error reading file: %v", err) } if strings.HasSuffix(string(buf), "New message") { t.Fatalf("New message was not appended after file was re-opened, got: %v", string(buf)) } } func TestFileLoggerSizeLimitAndReopen(t *testing.T) { file := createTempFile(t, "log_") file.Close() s := &Server{opts: &Options{}} defer s.SetLogger(nil, false, false) // Set a File log s.opts.LogFile = file.Name() s.opts.Logtime = true s.opts.LogSizeLimit = 1000 s.ConfigureLogger() // Add a trace s.Noticef("this is a notice") // Do a re-open... s.ReOpenLogFile() // Content should indicate that we have re-opened the log buf, err := os.ReadFile(s.opts.LogFile) if err != nil { t.Fatalf("Error reading file: %v", err) } if strings.HasSuffix(string(buf), "File log-reopened") { t.Fatalf("File should indicate that file log was re-opened, got: %v", string(buf)) } // Now make sure that the limit is still honored. txt := make([]byte, 800) for i := 0; i < len(txt); i++ { txt[i] = 'A' } s.Noticef(string(txt)) for i := 0; i < len(txt); i++ { txt[i] = 'B' } s.Noticef(string(txt)) buf, err = os.ReadFile(s.opts.LogFile) if err != nil { t.Fatalf("Error reading file: %v", err) } sbuf := string(buf) if strings.Contains(sbuf, "AAAAA") || strings.Contains(sbuf, "BBBBB") { t.Fatalf("Looks like file was not rotated: %s", sbuf) } if !strings.Contains(sbuf, "Rotated log, backup saved") { t.Fatalf("File should have been rotated, was not: %s", sbuf) } } func TestNoPasswordsFromConnectTrace(t *testing.T) { opts := DefaultOptions() opts.NoLog = false opts.Trace = true opts.Username = "derek" opts.Password = "s3cr3t" opts.PingInterval = 2 * time.Minute setBaselineOptions(opts) s := &Server{opts: opts} dl := testhelper.NewDummyLogger(100) s.SetLogger(dl, false, true) _ = s.logging.logger.(*DummyLogger) if s.logging.trace != 1 { t.Fatalf("Expected trace 1, received value %d\n", s.logging.trace) } defer s.SetLogger(nil, false, false) c, _, _ := newClientForServer(s) defer c.close() connectOp := []byte("CONNECT {\"user\":\"derek\",\"pass\":\"s3cr3t\"}\r\n") err := c.parse(connectOp) if err != nil { t.Fatalf("Received error: %v\n", err) } dl.CheckForProhibited(t, "password found", "s3cr3t") } func TestRemovePassFromTrace(t *testing.T) { tests := []struct { name string input string expected string }{ { "user and pass", "CONNECT {\"user\":\"derek\",\"pass\":\"s3cr3t\"}\r\n", "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", }, { "user and pass extra space", "CONNECT {\"user\":\"derek\",\"pass\": \"s3cr3t\"}\r\n", "CONNECT {\"user\":\"derek\",\"pass\": \"[REDACTED]\"}\r\n", }, { "user and pass is empty", "CONNECT {\"user\":\"derek\",\"pass\":\"\"}\r\n", "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", }, { "user and pass is empty whitespace", "CONNECT {\"user\":\"derek\",\"pass\":\" \"}\r\n", "CONNECT {\"user\":\"derek\",\"pass\":\"[REDACTED]\"}\r\n", }, { "user and pass whitespace", "CONNECT {\"user\":\"derek\",\"pass\": \"s3cr3t\" }\r\n", "CONNECT {\"user\":\"derek\",\"pass\": \"[REDACTED]\" }\r\n", }, { "only pass", "CONNECT {\"pass\":\"s3cr3t\",}\r\n", "CONNECT {\"pass\":\"[REDACTED]\",}\r\n", }, { "invalid json", "CONNECT {pass:s3cr3t , password = s3cr3t}", "CONNECT {pass:[REDACTED], password = s3cr3t}", }, { "invalid json no whitespace after key", "CONNECT {pass:s3cr3t , password= s3cr3t}", "CONNECT {pass:[REDACTED], password= s3cr3t}", }, { "both pass and wrong password key", `CONNECT {"pass":"s3cr3t4", "password": "s3cr3t4"}`, `CONNECT {"pass":"[REDACTED]", "password": "s3cr3t4"}`, }, { "invalid json", "CONNECT {user = hello, password = s3cr3t}", "CONNECT {user = hello, password = [REDACTED]}", }, { "complete connect", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", }, { "invalid json with only pass key", "CONNECT {pass:s3cr3t\r\n", "CONNECT {pass:[REDACTED]\r\n", }, { "invalid password key also filtered", "CONNECT {\"password\":\"s3cr3t\",}\r\n", "CONNECT {\"password\":\"[REDACTED]\",}\r\n", }, { "single long password with whitespace", "CONNECT {\"pass\":\"secret password which is very long\",}\r\n", "CONNECT {\"pass\":\"[REDACTED]\",}\r\n", }, { "single long pass key is filtered", "CONNECT {\"pass\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"}\r\n", "CONNECT {\"pass\":\"[REDACTED]\"}\r\n", }, { "duplicate keys only filtered once", "CONNECT {\"pass\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",\"pass\":\"BBBBBBBBBBBBBBBBBBBB\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n", "CONNECT {\"pass\":\"[REDACTED]\",\"pass\":\"BBBBBBBBBBBBBBBBBBBB\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n", }, { "invalid json with multiple keys only one is filtered", "CONNECT {pass = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",pass= \"BBBBBBBBBBBBBBBBBBBB\",password =\"CCCCCCCCCCCCCCCC\"}\r\n", "CONNECT {pass = \"[REDACTED]\",pass= \"BBBBBBBBBBBBBBBBBBBB\",password =\"CCCCCCCCCCCCCCCC\"}\r\n", }, { "complete connect protocol", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n", }, { "user and pass are filterered", "CONNECT {\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\"}\r\n", "CONNECT {\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\"}\r\n", }, { "complete connect using password key with user and password being the same", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"...\"}\r\n", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"...\"}\r\n", }, { "complete connect with user password and name all the same", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n", }, { "complete connect extra white space at the beginning", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"s3cr3t\",\"tls_required\":false,\"name\":\"foo\"}\r\n", "CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"pass\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"foo\"}\r\n", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { output := removePassFromTrace([]byte(test.input)) if !bytes.Equal(output, []byte(test.expected)) { t.Errorf("\nExpected %q\n got: %q", test.expected, string(output)) } }) } } nats-server-2.10.27/server/memstore.go000066400000000000000000001415711477524627100176540ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( crand "crypto/rand" "encoding/binary" "fmt" "sort" "sync" "time" "github.com/nats-io/nats-server/v2/server/avl" "github.com/nats-io/nats-server/v2/server/stree" ) // TODO(dlc) - This is a fairly simplistic approach but should do for now. type memStore struct { mu sync.RWMutex cfg StreamConfig state StreamState msgs map[uint64]*StoreMsg fss *stree.SubjectTree[SimpleState] dmap avl.SequenceSet maxp int64 scb StorageUpdateHandler ageChk *time.Timer consumers int receivedAny bool } func newMemStore(cfg *StreamConfig) (*memStore, error) { if cfg == nil { return nil, fmt.Errorf("config required") } if cfg.Storage != MemoryStorage { return nil, fmt.Errorf("memStore requires memory storage type in config") } ms := &memStore{ msgs: make(map[uint64]*StoreMsg), fss: stree.NewSubjectTree[SimpleState](), maxp: cfg.MaxMsgsPer, cfg: *cfg, } if cfg.FirstSeq > 0 { if _, err := ms.purge(cfg.FirstSeq); err != nil { return nil, err } } return ms, nil } func (ms *memStore) UpdateConfig(cfg *StreamConfig) error { if cfg == nil { return fmt.Errorf("config required") } if cfg.Storage != MemoryStorage { return fmt.Errorf("memStore requires memory storage type in config") } ms.mu.Lock() ms.cfg = *cfg // Limits checks and enforcement. ms.enforceMsgLimit() ms.enforceBytesLimit() // Do age timers. if ms.ageChk == nil && ms.cfg.MaxAge != 0 { ms.startAgeChk() } if ms.ageChk != nil && ms.cfg.MaxAge == 0 { ms.ageChk.Stop() ms.ageChk = nil } // Make sure to update MaxMsgsPer if cfg.MaxMsgsPer < -1 { cfg.MaxMsgsPer = -1 } maxp := ms.maxp ms.maxp = cfg.MaxMsgsPer // If the value is smaller, or was unset before, we need to enforce that. if ms.maxp > 0 && (maxp == 0 || ms.maxp < maxp) { lm := uint64(ms.maxp) ms.fss.IterFast(func(subj []byte, ss *SimpleState) bool { if ss.Msgs > lm { ms.enforcePerSubjectLimit(bytesToString(subj), ss) } return true }) } ms.mu.Unlock() if cfg.MaxAge != 0 { ms.expireMsgs() } return nil } // Stores a raw message with expected sequence number and timestamp. // Lock should be held. func (ms *memStore) storeRawMsg(subj string, hdr, msg []byte, seq uint64, ts int64) error { if ms.msgs == nil { return ErrStoreClosed } // Tracking by subject. var ss *SimpleState var asl bool if len(subj) > 0 { var ok bool if ss, ok = ms.fss.Find(stringToBytes(subj)); ok { asl = ms.maxp > 0 && ss.Msgs >= uint64(ms.maxp) } } // Check if we are discarding new messages when we reach the limit. if ms.cfg.Discard == DiscardNew { if asl && ms.cfg.DiscardNewPer { return ErrMaxMsgsPerSubject } // If we are discard new and limits policy and clustered, we do the enforcement // above and should not disqualify the message here since it could cause replicas to drift. if ms.cfg.Retention == LimitsPolicy || ms.cfg.Replicas == 1 { if ms.cfg.MaxMsgs > 0 && ms.state.Msgs >= uint64(ms.cfg.MaxMsgs) { // If we are tracking max messages per subject and are at the limit we will replace, so this is ok. if !asl { return ErrMaxMsgs } } if ms.cfg.MaxBytes > 0 && ms.state.Bytes+memStoreMsgSize(subj, hdr, msg) >= uint64(ms.cfg.MaxBytes) { if !asl { return ErrMaxBytes } // If we are here we are at a subject maximum, need to determine if dropping last message gives us enough room. if ss.firstNeedsUpdate || ss.lastNeedsUpdate { ms.recalculateForSubj(subj, ss) } sm, ok := ms.msgs[ss.First] if !ok || memStoreMsgSize(sm.subj, sm.hdr, sm.msg) < memStoreMsgSize(subj, hdr, msg) { return ErrMaxBytes } } } } if seq != ms.state.LastSeq+1 { if seq > 0 { return ErrSequenceMismatch } seq = ms.state.LastSeq + 1 } // Adjust first if needed. now := time.Unix(0, ts).UTC() if ms.state.Msgs == 0 { ms.state.FirstSeq = seq ms.state.FirstTime = now } // Make copies // TODO(dlc) - Maybe be smarter here. if len(msg) > 0 { msg = copyBytes(msg) } if len(hdr) > 0 { hdr = copyBytes(hdr) } // FIXME(dlc) - Could pool at this level? sm := &StoreMsg{subj, nil, nil, make([]byte, 0, len(hdr)+len(msg)), seq, ts} sm.buf = append(sm.buf, hdr...) sm.buf = append(sm.buf, msg...) if len(hdr) > 0 { sm.hdr = sm.buf[:len(hdr)] } sm.msg = sm.buf[len(hdr):] ms.msgs[seq] = sm ms.state.Msgs++ ms.state.Bytes += memStoreMsgSize(subj, hdr, msg) ms.state.LastSeq = seq ms.state.LastTime = now // Track per subject. if len(subj) > 0 { if ss != nil { ss.Msgs++ ss.Last = seq ss.lastNeedsUpdate = false // Check per subject limits. if ms.maxp > 0 && ss.Msgs > uint64(ms.maxp) { ms.enforcePerSubjectLimit(subj, ss) } } else { ms.fss.Insert([]byte(subj), SimpleState{Msgs: 1, First: seq, Last: seq}) } } // Limits checks and enforcement. ms.enforceMsgLimit() ms.enforceBytesLimit() // Check if we have and need the age expiration timer running. if ms.ageChk == nil && ms.cfg.MaxAge != 0 { ms.startAgeChk() } return nil } // StoreRawMsg stores a raw message with expected sequence number and timestamp. func (ms *memStore) StoreRawMsg(subj string, hdr, msg []byte, seq uint64, ts int64) error { ms.mu.Lock() err := ms.storeRawMsg(subj, hdr, msg, seq, ts) cb := ms.scb // Check if first message timestamp requires expiry // sooner than initial replica expiry timer set to MaxAge when initializing. if !ms.receivedAny && ms.cfg.MaxAge != 0 && ts > 0 { ms.receivedAny = true // Calculate duration when the next expireMsgs should be called. ms.resetAgeChk(int64(time.Millisecond) * 50) } ms.mu.Unlock() if err == nil && cb != nil { cb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj) } return err } // Store stores a message. func (ms *memStore) StoreMsg(subj string, hdr, msg []byte) (uint64, int64, error) { ms.mu.Lock() seq, ts := ms.state.LastSeq+1, time.Now().UnixNano() err := ms.storeRawMsg(subj, hdr, msg, seq, ts) cb := ms.scb ms.mu.Unlock() if err != nil { seq, ts = 0, 0 } else if cb != nil { cb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj) } return seq, ts, err } // SkipMsg will use the next sequence number but not store anything. func (ms *memStore) SkipMsg() uint64 { // Grab time. now := time.Now().UTC() ms.mu.Lock() seq := ms.state.LastSeq + 1 ms.state.LastSeq = seq ms.state.LastTime = now if ms.state.Msgs == 0 { ms.state.FirstSeq = seq + 1 ms.state.FirstTime = now } else { ms.dmap.Insert(seq) } ms.mu.Unlock() return seq } // Skip multiple msgs. func (ms *memStore) SkipMsgs(seq uint64, num uint64) error { // Grab time. now := time.Now().UTC() ms.mu.Lock() defer ms.mu.Unlock() // Check sequence matches our last sequence. if seq != ms.state.LastSeq+1 { if seq > 0 { return ErrSequenceMismatch } seq = ms.state.LastSeq + 1 } lseq := seq + num - 1 ms.state.LastSeq = lseq ms.state.LastTime = now if ms.state.Msgs == 0 { ms.state.FirstSeq, ms.state.FirstTime = lseq+1, now } else { for ; seq <= lseq; seq++ { ms.dmap.Insert(seq) } } return nil } // RegisterStorageUpdates registers a callback for updates to storage changes. // It will present number of messages and bytes as a signed integer and an // optional sequence number of the message if a single. func (ms *memStore) RegisterStorageUpdates(cb StorageUpdateHandler) { ms.mu.Lock() ms.scb = cb ms.mu.Unlock() } // GetSeqFromTime looks for the first sequence number that has the message // with >= timestamp. // FIXME(dlc) - inefficient. func (ms *memStore) GetSeqFromTime(t time.Time) uint64 { ts := t.UnixNano() ms.mu.RLock() defer ms.mu.RUnlock() if len(ms.msgs) == 0 { return ms.state.LastSeq + 1 } if ts <= ms.msgs[ms.state.FirstSeq].ts { return ms.state.FirstSeq } // LastSeq is not guaranteed to be present since last does not go backwards. var lmsg *StoreMsg for lseq := ms.state.LastSeq; lseq > ms.state.FirstSeq; lseq-- { if lmsg = ms.msgs[lseq]; lmsg != nil { break } } if lmsg == nil { return ms.state.FirstSeq } last := lmsg.ts if ts == last { return ms.state.LastSeq } if ts > last { return ms.state.LastSeq + 1 } index := sort.Search(len(ms.msgs), func(i int) bool { if msg := ms.msgs[ms.state.FirstSeq+uint64(i)]; msg != nil { return msg.ts >= ts } return false }) return uint64(index) + ms.state.FirstSeq } // FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence. func (ms *memStore) FilteredState(sseq uint64, subj string) SimpleState { // This needs to be a write lock, as filteredStateLocked can // mutate the per-subject state. ms.mu.Lock() defer ms.mu.Unlock() return ms.filteredStateLocked(sseq, subj, false) } func (ms *memStore) filteredStateLocked(sseq uint64, filter string, lastPerSubject bool) SimpleState { if sseq < ms.state.FirstSeq { sseq = ms.state.FirstSeq } // If past the end no results. if sseq > ms.state.LastSeq { return SimpleState{} } if filter == _EMPTY_ { filter = fwcs } isAll := filter == fwcs // First check if we can optimize this part. // This means we want all and the starting sequence was before this block. if isAll && sseq <= ms.state.FirstSeq { total := ms.state.Msgs if lastPerSubject { total = uint64(ms.fss.Size()) } return SimpleState{ Msgs: total, First: ms.state.FirstSeq, Last: ms.state.LastSeq, } } _tsa, _fsa := [32]string{}, [32]string{} tsa, fsa := _tsa[:0], _fsa[:0] wc := subjectHasWildcard(filter) if wc { fsa = tokenizeSubjectIntoSlice(fsa[:0], filter) } // 1. See if we match any subs from fss. // 2. If we match and the sseq is past ss.Last then we can use meta only. // 3. If we match we need to do a partial, break and clear any totals and do a full scan like num pending. isMatch := func(subj string) bool { if isAll { return true } if !wc { return subj == filter } tsa = tokenizeSubjectIntoSlice(tsa[:0], subj) return isSubsetMatchTokenized(tsa, fsa) } var ss SimpleState update := func(fss *SimpleState) { msgs, first, last := fss.Msgs, fss.First, fss.Last if lastPerSubject { msgs, first = 1, last } ss.Msgs += msgs if ss.First == 0 || first < ss.First { ss.First = first } if last > ss.Last { ss.Last = last } } var havePartial bool var totalSkipped uint64 // We will track start and end sequences as we go. ms.fss.Match(stringToBytes(filter), func(subj []byte, fss *SimpleState) { if fss.firstNeedsUpdate || fss.lastNeedsUpdate { ms.recalculateForSubj(bytesToString(subj), fss) } if sseq <= fss.First { update(fss) } else if sseq <= fss.Last { // We matched but it is a partial. havePartial = true // Don't break here, we will update to keep tracking last. update(fss) } else { totalSkipped += fss.Msgs } }) // If we did not encounter any partials we can return here. if !havePartial { return ss } // If we are here we need to scan the msgs. // Capture first and last sequences for scan and then clear what we had. first, last := ss.First, ss.Last // To track if we decide to exclude we need to calculate first. var needScanFirst bool if first < sseq { first = sseq needScanFirst = true } // Now we want to check if it is better to scan inclusive and recalculate that way // or leave and scan exclusive and adjust our totals. // ss.Last is always correct here. toScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last var seen map[string]bool if lastPerSubject { seen = make(map[string]bool) } if toScan < toExclude { ss.Msgs, ss.First = 0, 0 update := func(sm *StoreMsg) { ss.Msgs++ if ss.First == 0 { ss.First = sm.seq } if seen != nil { seen[sm.subj] = true } } // Check if easier to just scan msgs vs the sequence range. // This can happen with lots of interior deletes. if last-first > uint64(len(ms.msgs)) { for _, sm := range ms.msgs { if sm.seq >= first && sm.seq <= last && !seen[sm.subj] && isMatch(sm.subj) { update(sm) } } } else { for seq := first; seq <= last; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { update(sm) } } } } else { // We will adjust from the totals above by scanning what we need to exclude. ss.First = first ss.Msgs += totalSkipped var adjust uint64 var tss *SimpleState update := func(sm *StoreMsg) { if lastPerSubject { tss, _ = ms.fss.Find(stringToBytes(sm.subj)) } // If we are last per subject, make sure to only adjust if all messages are before our first. if tss == nil || tss.Last < first { adjust++ } if seen != nil { seen[sm.subj] = true } } // Check if easier to just scan msgs vs the sequence range. if first-ms.state.FirstSeq > uint64(len(ms.msgs)) { for _, sm := range ms.msgs { if sm.seq < first && !seen[sm.subj] && isMatch(sm.subj) { update(sm) } } } else { for seq := ms.state.FirstSeq; seq < first; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { update(sm) } } } // Now do range at end. for seq := last + 1; seq < ms.state.LastSeq; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) { adjust++ if seen != nil { seen[sm.subj] = true } } } ss.Msgs -= adjust if needScanFirst { // Check if easier to just scan msgs vs the sequence range. // Since we will need to scan all of the msgs vs below where we break on the first match, // we will only do so if a few orders of magnitude lower. if last-first > 100*uint64(len(ms.msgs)) { low := ms.state.LastSeq for _, sm := range ms.msgs { if sm.seq >= first && sm.seq < last && isMatch(sm.subj) { if sm.seq < low { low = sm.seq } } } if low < ms.state.LastSeq { ss.First = low } } else { for seq := first; seq < last; seq++ { if sm, ok := ms.msgs[seq]; ok && isMatch(sm.subj) { ss.First = seq break } } } } } return ss } // SubjectsState returns a map of SimpleState for all matching subjects. func (ms *memStore) SubjectsState(subject string) map[string]SimpleState { // This needs to be a write lock, as we can mutate the per-subject state. ms.mu.Lock() defer ms.mu.Unlock() if ms.fss.Size() == 0 { return nil } if subject == _EMPTY_ { subject = fwcs } fss := make(map[string]SimpleState) ms.fss.Match(stringToBytes(subject), func(subj []byte, ss *SimpleState) { subjs := string(subj) if ss.firstNeedsUpdate || ss.lastNeedsUpdate { ms.recalculateForSubj(subjs, ss) } oss := fss[subjs] if oss.First == 0 { // New fss[subjs] = *ss } else { // Merge here. oss.Last, oss.Msgs = ss.Last, oss.Msgs+ss.Msgs fss[subjs] = oss } }) return fss } // SubjectsTotal return message totals per subject. func (ms *memStore) SubjectsTotals(filterSubject string) map[string]uint64 { ms.mu.RLock() defer ms.mu.RUnlock() if ms.fss.Size() == 0 { return nil } _tsa, _fsa := [32]string{}, [32]string{} tsa, fsa := _tsa[:0], _fsa[:0] fsa = tokenizeSubjectIntoSlice(fsa[:0], filterSubject) isAll := filterSubject == _EMPTY_ || filterSubject == fwcs fst := make(map[string]uint64) ms.fss.Match(stringToBytes(filterSubject), func(subj []byte, ss *SimpleState) { subjs := string(subj) if isAll { fst[subjs] = ss.Msgs } else { if tsa = tokenizeSubjectIntoSlice(tsa[:0], subjs); isSubsetMatchTokenized(tsa, fsa) { fst[subjs] = ss.Msgs } } }) return fst } // NumPending will return the number of pending messages matching the filter subject starting at sequence. func (ms *memStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64) { // This needs to be a write lock, as filteredStateLocked can mutate the per-subject state. ms.mu.Lock() defer ms.mu.Unlock() ss := ms.filteredStateLocked(sseq, filter, lastPerSubject) return ss.Msgs, ms.state.LastSeq } // NumPending will return the number of pending messages matching any subject in the sublist starting at sequence. func (ms *memStore) NumPendingMulti(sseq uint64, sl *Sublist, lastPerSubject bool) (total, validThrough uint64) { if sl == nil { return ms.NumPending(sseq, fwcs, lastPerSubject) } // This needs to be a write lock, as we can mutate the per-subject state. ms.mu.Lock() defer ms.mu.Unlock() var ss SimpleState if sseq < ms.state.FirstSeq { sseq = ms.state.FirstSeq } // If past the end no results. if sseq > ms.state.LastSeq { return 0, ms.state.LastSeq } update := func(fss *SimpleState) { msgs, first, last := fss.Msgs, fss.First, fss.Last if lastPerSubject { msgs, first = 1, last } ss.Msgs += msgs if ss.First == 0 || first < ss.First { ss.First = first } if last > ss.Last { ss.Last = last } } var havePartial bool var totalSkipped uint64 // We will track start and end sequences as we go. IntersectStree[SimpleState](ms.fss, sl, func(subj []byte, fss *SimpleState) { if fss.firstNeedsUpdate || fss.lastNeedsUpdate { ms.recalculateForSubj(bytesToString(subj), fss) } if sseq <= fss.First { update(fss) } else if sseq <= fss.Last { // We matched but it is a partial. havePartial = true // Don't break here, we will update to keep tracking last. update(fss) } else { totalSkipped += fss.Msgs } }) // If we did not encounter any partials we can return here. if !havePartial { return ss.Msgs, ms.state.LastSeq } // If we are here we need to scan the msgs. // Capture first and last sequences for scan and then clear what we had. first, last := ss.First, ss.Last // To track if we decide to exclude we need to calculate first. if first < sseq { first = sseq } // Now we want to check if it is better to scan inclusive and recalculate that way // or leave and scan exclusive and adjust our totals. // ss.Last is always correct here. toScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last var seen map[string]bool if lastPerSubject { seen = make(map[string]bool) } if toScan < toExclude { ss.Msgs, ss.First = 0, 0 update := func(sm *StoreMsg) { ss.Msgs++ if ss.First == 0 { ss.First = sm.seq } if seen != nil { seen[sm.subj] = true } } // Check if easier to just scan msgs vs the sequence range. // This can happen with lots of interior deletes. if last-first > uint64(len(ms.msgs)) { for _, sm := range ms.msgs { if sm.seq >= first && sm.seq <= last && !seen[sm.subj] && sl.HasInterest(sm.subj) { update(sm) } } } else { for seq := first; seq <= last; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { update(sm) } } } } else { // We will adjust from the totals above by scanning what we need to exclude. ss.First = first ss.Msgs += totalSkipped var adjust uint64 var tss *SimpleState update := func(sm *StoreMsg) { if lastPerSubject { tss, _ = ms.fss.Find(stringToBytes(sm.subj)) } // If we are last per subject, make sure to only adjust if all messages are before our first. if tss == nil || tss.Last < first { adjust++ } if seen != nil { seen[sm.subj] = true } } // Check if easier to just scan msgs vs the sequence range. if first-ms.state.FirstSeq > uint64(len(ms.msgs)) { for _, sm := range ms.msgs { if sm.seq < first && !seen[sm.subj] && sl.HasInterest(sm.subj) { update(sm) } } } else { for seq := ms.state.FirstSeq; seq < first; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { update(sm) } } } // Now do range at end. for seq := last + 1; seq < ms.state.LastSeq; seq++ { if sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) { adjust++ if seen != nil { seen[sm.subj] = true } } } ss.Msgs -= adjust } return ss.Msgs, ms.state.LastSeq } // Will check the msg limit for this tracked subject. // Lock should be held. func (ms *memStore) enforcePerSubjectLimit(subj string, ss *SimpleState) { if ms.maxp <= 0 { return } for nmsgs := ss.Msgs; nmsgs > uint64(ms.maxp); nmsgs = ss.Msgs { if ss.firstNeedsUpdate || ss.lastNeedsUpdate { ms.recalculateForSubj(subj, ss) } if !ms.removeMsg(ss.First, false) { break } } } // Will check the msg limit and drop firstSeq msg if needed. // Lock should be held. func (ms *memStore) enforceMsgLimit() { if ms.cfg.Discard != DiscardOld { return } if ms.cfg.MaxMsgs <= 0 || ms.state.Msgs <= uint64(ms.cfg.MaxMsgs) { return } for nmsgs := ms.state.Msgs; nmsgs > uint64(ms.cfg.MaxMsgs); nmsgs = ms.state.Msgs { ms.deleteFirstMsgOrPanic() } } // Will check the bytes limit and drop msgs if needed. // Lock should be held. func (ms *memStore) enforceBytesLimit() { if ms.cfg.Discard != DiscardOld { return } if ms.cfg.MaxBytes <= 0 || ms.state.Bytes <= uint64(ms.cfg.MaxBytes) { return } for bs := ms.state.Bytes; bs > uint64(ms.cfg.MaxBytes); bs = ms.state.Bytes { ms.deleteFirstMsgOrPanic() } } // Will start the age check timer. // Lock should be held. func (ms *memStore) startAgeChk() { if ms.ageChk == nil && ms.cfg.MaxAge != 0 { ms.ageChk = time.AfterFunc(ms.cfg.MaxAge, ms.expireMsgs) } } // Lock should be held. func (ms *memStore) resetAgeChk(delta int64) { if ms.cfg.MaxAge == 0 { return } fireIn := ms.cfg.MaxAge if delta > 0 && time.Duration(delta) < fireIn { if fireIn = time.Duration(delta); fireIn < 250*time.Millisecond { // Only fire at most once every 250ms. // Excessive firing can effect ingest performance. fireIn = time.Second } } if ms.ageChk != nil { ms.ageChk.Reset(fireIn) } else { ms.ageChk = time.AfterFunc(fireIn, ms.expireMsgs) } } // Will expire msgs that are too old. func (ms *memStore) expireMsgs() { ms.mu.RLock() now := time.Now().UnixNano() minAge := now - int64(ms.cfg.MaxAge) ms.mu.RUnlock() for { ms.mu.Lock() if sm, ok := ms.msgs[ms.state.FirstSeq]; ok && sm.ts <= minAge { ms.deleteFirstMsgOrPanic() // Recalculate in case we are expiring a bunch. now = time.Now().UnixNano() minAge = now - int64(ms.cfg.MaxAge) ms.mu.Unlock() } else { // We will exit here if len(ms.msgs) == 0 { if ms.ageChk != nil { ms.ageChk.Stop() ms.ageChk = nil } } else { var fireIn time.Duration if sm == nil { fireIn = ms.cfg.MaxAge } else { fireIn = time.Duration(sm.ts - minAge) } if ms.ageChk != nil { ms.ageChk.Reset(fireIn) } else { ms.ageChk = time.AfterFunc(fireIn, ms.expireMsgs) } } ms.mu.Unlock() break } } } // PurgeEx will remove messages based on subject filters, sequence and number of messages to keep. // Will return the number of purged messages. func (ms *memStore) PurgeEx(subject string, sequence, keep uint64) (purged uint64, err error) { if subject == _EMPTY_ || subject == fwcs { if keep == 0 && sequence == 0 { return ms.Purge() } if sequence > 1 { return ms.Compact(sequence) } else if keep > 0 { ms.mu.RLock() msgs, lseq := ms.state.Msgs, ms.state.LastSeq ms.mu.RUnlock() if keep >= msgs { return 0, nil } return ms.Compact(lseq - keep + 1) } return 0, nil } eq := compareFn(subject) if ss := ms.FilteredState(1, subject); ss.Msgs > 0 { if keep > 0 { if keep >= ss.Msgs { return 0, nil } ss.Msgs -= keep } last := ss.Last if sequence > 1 { last = sequence - 1 } ms.mu.Lock() for seq := ss.First; seq <= last; seq++ { if sm, ok := ms.msgs[seq]; ok && eq(sm.subj, subject) { if ok := ms.removeMsg(sm.seq, false); ok { purged++ if purged >= ss.Msgs { break } } } } ms.mu.Unlock() } return purged, nil } // Purge will remove all messages from this store. // Will return the number of purged messages. func (ms *memStore) Purge() (uint64, error) { ms.mu.RLock() first := ms.state.LastSeq + 1 ms.mu.RUnlock() return ms.purge(first) } func (ms *memStore) purge(fseq uint64) (uint64, error) { ms.mu.Lock() purged := uint64(len(ms.msgs)) cb := ms.scb bytes := int64(ms.state.Bytes) if fseq < ms.state.LastSeq { ms.mu.Unlock() return 0, fmt.Errorf("partial purges not supported on memory store") } ms.state.FirstSeq = fseq ms.state.LastSeq = fseq - 1 ms.state.FirstTime = time.Time{} ms.state.Bytes = 0 ms.state.Msgs = 0 ms.msgs = make(map[uint64]*StoreMsg) ms.fss = stree.NewSubjectTree[SimpleState]() ms.mu.Unlock() if cb != nil { cb(-int64(purged), -bytes, 0, _EMPTY_) } return purged, nil } // Compact will remove all messages from this store up to // but not including the seq parameter. // Will return the number of purged messages. func (ms *memStore) Compact(seq uint64) (uint64, error) { if seq == 0 { return ms.Purge() } var purged, bytes uint64 ms.mu.Lock() cb := ms.scb if seq <= ms.state.LastSeq { fseq := ms.state.FirstSeq // Determine new first sequence. for ; seq <= ms.state.LastSeq; seq++ { if sm, ok := ms.msgs[seq]; ok { ms.state.FirstSeq = seq ms.state.FirstTime = time.Unix(0, sm.ts).UTC() break } } for seq := seq - 1; seq >= fseq; seq-- { if sm := ms.msgs[seq]; sm != nil { bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) purged++ ms.removeSeqPerSubject(sm.subj, seq) // Must delete message after updating per-subject info, to be consistent with file store. delete(ms.msgs, seq) } else if !ms.dmap.IsEmpty() { ms.dmap.Delete(seq) } } if purged > ms.state.Msgs { purged = ms.state.Msgs } ms.state.Msgs -= purged if bytes > ms.state.Bytes { bytes = ms.state.Bytes } ms.state.Bytes -= bytes } else { // We are compacting past the end of our range. Do purge and set sequences correctly // such that the next message placed will have seq. purged = uint64(len(ms.msgs)) bytes = ms.state.Bytes ms.state.Bytes = 0 ms.state.Msgs = 0 ms.state.FirstSeq = seq ms.state.FirstTime = time.Time{} ms.state.LastSeq = seq - 1 // Reset msgs, fss and dmap. ms.msgs = make(map[uint64]*StoreMsg) ms.fss = stree.NewSubjectTree[SimpleState]() ms.dmap.Empty() } ms.mu.Unlock() if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return purged, nil } // Will completely reset our store. func (ms *memStore) reset() error { ms.mu.Lock() var purged, bytes uint64 cb := ms.scb if cb != nil { for _, sm := range ms.msgs { purged++ bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) } } // Reset ms.state.FirstSeq = 0 ms.state.FirstTime = time.Time{} ms.state.LastSeq = 0 ms.state.LastTime = time.Now().UTC() // Update msgs and bytes. ms.state.Msgs = 0 ms.state.Bytes = 0 // Reset msgs, fss and dmap. ms.msgs = make(map[uint64]*StoreMsg) ms.fss = stree.NewSubjectTree[SimpleState]() ms.dmap.Empty() ms.mu.Unlock() if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return nil } // Truncate will truncate a stream store up to seq. Sequence needs to be valid. func (ms *memStore) Truncate(seq uint64) error { // Check for request to reset. if seq == 0 { return ms.reset() } var purged, bytes uint64 ms.mu.Lock() lsm, ok := ms.msgs[seq] if !ok { ms.mu.Unlock() return ErrInvalidSequence } for i := ms.state.LastSeq; i > seq; i-- { if sm := ms.msgs[i]; sm != nil { purged++ bytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg) ms.removeSeqPerSubject(sm.subj, i) // Must delete message after updating per-subject info, to be consistent with file store. delete(ms.msgs, i) } else if !ms.dmap.IsEmpty() { ms.dmap.Delete(i) } } // Reset last. ms.state.LastSeq = lsm.seq ms.state.LastTime = time.Unix(0, lsm.ts).UTC() // Update msgs and bytes. if purged > ms.state.Msgs { purged = ms.state.Msgs } ms.state.Msgs -= purged if bytes > ms.state.Bytes { bytes = ms.state.Bytes } ms.state.Bytes -= bytes cb := ms.scb ms.mu.Unlock() if cb != nil { cb(-int64(purged), -int64(bytes), 0, _EMPTY_) } return nil } func (ms *memStore) deleteFirstMsgOrPanic() { if !ms.deleteFirstMsg() { panic("jetstream memstore has inconsistent state, can't find first seq msg") } } func (ms *memStore) deleteFirstMsg() bool { return ms.removeMsg(ms.state.FirstSeq, false) } // LoadMsg will lookup the message by sequence number and return it if found. func (ms *memStore) LoadMsg(seq uint64, smp *StoreMsg) (*StoreMsg, error) { ms.mu.RLock() sm, ok := ms.msgs[seq] last := ms.state.LastSeq ms.mu.RUnlock() if !ok || sm == nil { var err = ErrStoreEOF if seq <= last { err = ErrStoreMsgNotFound } return nil, err } if smp == nil { smp = new(StoreMsg) } sm.copy(smp) return smp, nil } // LoadLastMsg will return the last message we have that matches a given subject. // The subject can be a wildcard. func (ms *memStore) LoadLastMsg(subject string, smp *StoreMsg) (*StoreMsg, error) { var sm *StoreMsg var ok bool // This needs to be a write lock, as filteredStateLocked can // mutate the per-subject state. ms.mu.Lock() defer ms.mu.Unlock() if subject == _EMPTY_ || subject == fwcs { sm, ok = ms.msgs[ms.state.LastSeq] } else if subjectIsLiteral(subject) { var ss *SimpleState if ss, ok = ms.fss.Find(stringToBytes(subject)); ok && ss.Msgs > 0 { sm, ok = ms.msgs[ss.Last] } } else if ss := ms.filteredStateLocked(1, subject, true); ss.Msgs > 0 { sm, ok = ms.msgs[ss.Last] } if !ok || sm == nil { return nil, ErrStoreMsgNotFound } if smp == nil { smp = new(StoreMsg) } sm.copy(smp) return smp, nil } // LoadNextMsgMulti will find the next message matching any entry in the sublist. func (ms *memStore) LoadNextMsgMulti(sl *Sublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) { // TODO(dlc) - for now simple linear walk to get started. ms.mu.RLock() defer ms.mu.RUnlock() if start < ms.state.FirstSeq { start = ms.state.FirstSeq } // If past the end no results. if start > ms.state.LastSeq || ms.state.Msgs == 0 { return nil, ms.state.LastSeq, ErrStoreEOF } // Initial setup. fseq, lseq := start, ms.state.LastSeq for nseq := fseq; nseq <= lseq; nseq++ { sm, ok := ms.msgs[nseq] if !ok { continue } if sl.HasInterest(sm.subj) { if smp == nil { smp = new(StoreMsg) } sm.copy(smp) return smp, nseq, nil } } return nil, ms.state.LastSeq, ErrStoreEOF } // LoadNextMsg will find the next message matching the filter subject starting at the start sequence. // The filter subject can be a wildcard. func (ms *memStore) LoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) { ms.mu.Lock() defer ms.mu.Unlock() if start < ms.state.FirstSeq { start = ms.state.FirstSeq } // If past the end no results. if start > ms.state.LastSeq || ms.state.Msgs == 0 { return nil, ms.state.LastSeq, ErrStoreEOF } if filter == _EMPTY_ { filter = fwcs } isAll := filter == fwcs // Skip scan of ms.fss if number of messages in the block are less than // 1/2 the number of subjects in ms.fss. Or we have a wc and lots of fss entries. const linearScanMaxFSS = 256 doLinearScan := isAll || 2*int(ms.state.LastSeq-start) < ms.fss.Size() || (wc && ms.fss.Size() > linearScanMaxFSS) // Initial setup. fseq, lseq := start, ms.state.LastSeq if !doLinearScan { subs := []string{filter} if wc || isAll { subs = subs[:0] ms.fss.Match(stringToBytes(filter), func(subj []byte, val *SimpleState) { subs = append(subs, string(subj)) }) } fseq, lseq = ms.state.LastSeq, uint64(0) for _, subj := range subs { ss, ok := ms.fss.Find(stringToBytes(subj)) if !ok { continue } if ss.firstNeedsUpdate || ss.lastNeedsUpdate { ms.recalculateForSubj(subj, ss) } if ss.First < fseq { fseq = ss.First } if ss.Last > lseq { lseq = ss.Last } } if fseq < start { fseq = start } } eq := subjectsEqual if wc { eq = subjectIsSubsetMatch } for nseq := fseq; nseq <= lseq; nseq++ { if sm, ok := ms.msgs[nseq]; ok && (isAll || eq(sm.subj, filter)) { if smp == nil { smp = new(StoreMsg) } sm.copy(smp) return smp, nseq, nil } } return nil, ms.state.LastSeq, ErrStoreEOF } // Will load the next non-deleted msg starting at the start sequence and walking backwards. func (ms *memStore) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error) { ms.mu.RLock() defer ms.mu.RUnlock() if ms.msgs == nil { return nil, ErrStoreClosed } if ms.state.Msgs == 0 || start < ms.state.FirstSeq { return nil, ErrStoreEOF } if start > ms.state.LastSeq { start = ms.state.LastSeq } for seq := start; seq >= ms.state.FirstSeq; seq-- { if sm, ok := ms.msgs[seq]; ok { if smp == nil { smp = new(StoreMsg) } sm.copy(smp) return smp, nil } } return nil, ErrStoreEOF } // RemoveMsg will remove the message from this store. // Will return the number of bytes removed. func (ms *memStore) RemoveMsg(seq uint64) (bool, error) { ms.mu.Lock() removed := ms.removeMsg(seq, false) ms.mu.Unlock() return removed, nil } // EraseMsg will remove the message and rewrite its contents. func (ms *memStore) EraseMsg(seq uint64) (bool, error) { ms.mu.Lock() removed := ms.removeMsg(seq, true) ms.mu.Unlock() return removed, nil } // Performs logic to update first sequence number. // Lock should be held. func (ms *memStore) updateFirstSeq(seq uint64) { if seq != ms.state.FirstSeq { // Interior delete. return } var nsm *StoreMsg var ok bool for nseq := ms.state.FirstSeq + 1; nseq <= ms.state.LastSeq; nseq++ { if nsm, ok = ms.msgs[nseq]; ok { break } } oldFirst := ms.state.FirstSeq if nsm != nil { ms.state.FirstSeq = nsm.seq ms.state.FirstTime = time.Unix(0, nsm.ts).UTC() } else { // Like purge. ms.state.FirstSeq = ms.state.LastSeq + 1 ms.state.FirstTime = time.Time{} } if oldFirst == ms.state.FirstSeq-1 { ms.dmap.Delete(oldFirst) } else { for seq := oldFirst; seq < ms.state.FirstSeq; seq++ { ms.dmap.Delete(seq) } } } // Remove a seq from the fss and select new first. // Lock should be held. func (ms *memStore) removeSeqPerSubject(subj string, seq uint64) { ss, ok := ms.fss.Find(stringToBytes(subj)) if !ok { return } if ss.Msgs == 1 { ms.fss.Delete(stringToBytes(subj)) return } ss.Msgs-- // Only one left if ss.Msgs == 1 { if !ss.lastNeedsUpdate && seq != ss.Last { ss.First = ss.Last ss.firstNeedsUpdate = false return } if !ss.firstNeedsUpdate && seq != ss.First { ss.Last = ss.First ss.lastNeedsUpdate = false return } } // We can lazily calculate the first/last sequence when needed. ss.firstNeedsUpdate = seq == ss.First || ss.firstNeedsUpdate ss.lastNeedsUpdate = seq == ss.Last || ss.lastNeedsUpdate } // Will recalculate the first and/or last sequence for this subject. // Lock should be held. func (ms *memStore) recalculateForSubj(subj string, ss *SimpleState) { if ss.firstNeedsUpdate { tseq := ss.First + 1 if tseq < ms.state.FirstSeq { tseq = ms.state.FirstSeq } for ; tseq <= ss.Last; tseq++ { if sm := ms.msgs[tseq]; sm != nil && sm.subj == subj { ss.First = tseq ss.firstNeedsUpdate = false if ss.Msgs == 1 { ss.Last = tseq ss.lastNeedsUpdate = false return } break } } } if ss.lastNeedsUpdate { tseq := ss.Last - 1 if tseq > ms.state.LastSeq { tseq = ms.state.LastSeq } for ; tseq >= ss.First; tseq-- { if sm := ms.msgs[tseq]; sm != nil && sm.subj == subj { ss.Last = tseq ss.lastNeedsUpdate = false if ss.Msgs == 1 { ss.First = tseq ss.firstNeedsUpdate = false } return } } } } // Removes the message referenced by seq. // Lock should be held. func (ms *memStore) removeMsg(seq uint64, secure bool) bool { var ss uint64 sm, ok := ms.msgs[seq] if !ok { return false } ss = memStoreMsgSize(sm.subj, sm.hdr, sm.msg) if ms.state.Msgs > 0 { ms.state.Msgs-- if ss > ms.state.Bytes { ss = ms.state.Bytes } ms.state.Bytes -= ss } ms.dmap.Insert(seq) ms.updateFirstSeq(seq) if secure { if len(sm.hdr) > 0 { sm.hdr = make([]byte, len(sm.hdr)) crand.Read(sm.hdr) } if len(sm.msg) > 0 { sm.msg = make([]byte, len(sm.msg)) crand.Read(sm.msg) } sm.seq, sm.ts = 0, 0 } // Remove any per subject tracking. ms.removeSeqPerSubject(sm.subj, seq) // Must delete message after updating per-subject info, to be consistent with file store. delete(ms.msgs, seq) if ms.scb != nil { // We do not want to hold any locks here. ms.mu.Unlock() delta := int64(ss) ms.scb(-1, -delta, seq, sm.subj) ms.mu.Lock() } return ok } // Type returns the type of the underlying store. func (ms *memStore) Type() StorageType { return MemoryStorage } // FastState will fill in state with only the following. // Msgs, Bytes, First and Last Sequence and Time and NumDeleted. func (ms *memStore) FastState(state *StreamState) { ms.mu.RLock() state.Msgs = ms.state.Msgs state.Bytes = ms.state.Bytes state.FirstSeq = ms.state.FirstSeq state.FirstTime = ms.state.FirstTime state.LastSeq = ms.state.LastSeq state.LastTime = ms.state.LastTime if state.LastSeq > state.FirstSeq { state.NumDeleted = int((state.LastSeq - state.FirstSeq + 1) - state.Msgs) if state.NumDeleted < 0 { state.NumDeleted = 0 } } state.Consumers = ms.consumers state.NumSubjects = ms.fss.Size() ms.mu.RUnlock() } func (ms *memStore) State() StreamState { ms.mu.RLock() defer ms.mu.RUnlock() state := ms.state state.Consumers = ms.consumers state.NumSubjects = ms.fss.Size() state.Deleted = nil // Calculate interior delete details. if numDeleted := int((state.LastSeq - state.FirstSeq + 1) - state.Msgs); numDeleted > 0 { state.Deleted = make([]uint64, 0, numDeleted) fseq, lseq := state.FirstSeq, state.LastSeq ms.dmap.Range(func(seq uint64) bool { if seq < fseq || seq > lseq { ms.dmap.Delete(seq) } else { state.Deleted = append(state.Deleted, seq) } return true }) } if len(state.Deleted) > 0 { state.NumDeleted = len(state.Deleted) } return state } func (ms *memStore) Utilization() (total, reported uint64, err error) { ms.mu.RLock() defer ms.mu.RUnlock() return ms.state.Bytes, ms.state.Bytes, nil } func memStoreMsgSize(subj string, hdr, msg []byte) uint64 { return uint64(len(subj) + len(hdr) + len(msg) + 16) // 8*2 for seq + age } // Delete is same as Stop for memory store. func (ms *memStore) Delete() error { return ms.Stop() } func (ms *memStore) Stop() error { // These can't come back, so stop is same as Delete. ms.Purge() ms.mu.Lock() if ms.ageChk != nil { ms.ageChk.Stop() ms.ageChk = nil } ms.msgs = nil ms.mu.Unlock() return nil } func (ms *memStore) isClosed() bool { ms.mu.RLock() defer ms.mu.RUnlock() return ms.msgs == nil } type consumerMemStore struct { mu sync.Mutex ms StreamStore cfg ConsumerConfig state ConsumerState closed bool } func (ms *memStore) ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerStore, error) { if ms == nil { return nil, fmt.Errorf("memstore is nil") } if ms.isClosed() { return nil, ErrStoreClosed } if cfg == nil || name == _EMPTY_ { return nil, fmt.Errorf("bad consumer config") } o := &consumerMemStore{ms: ms, cfg: *cfg} ms.AddConsumer(o) return o, nil } func (ms *memStore) AddConsumer(o ConsumerStore) error { ms.mu.Lock() ms.consumers++ ms.mu.Unlock() return nil } func (ms *memStore) RemoveConsumer(o ConsumerStore) error { ms.mu.Lock() if ms.consumers > 0 { ms.consumers-- } ms.mu.Unlock() return nil } func (ms *memStore) Snapshot(_ time.Duration, _, _ bool) (*SnapshotResult, error) { return nil, fmt.Errorf("no impl") } // Binary encoded state snapshot, >= v2.10 server. func (ms *memStore) EncodedStreamState(failed uint64) ([]byte, error) { ms.mu.RLock() defer ms.mu.RUnlock() // Quick calculate num deleted. numDeleted := int((ms.state.LastSeq - ms.state.FirstSeq + 1) - ms.state.Msgs) if numDeleted < 0 { numDeleted = 0 } // Encoded is Msgs, Bytes, FirstSeq, LastSeq, Failed, NumDeleted and optional DeletedBlocks var buf [1024]byte buf[0], buf[1] = streamStateMagic, streamStateVersion n := hdrLen n += binary.PutUvarint(buf[n:], ms.state.Msgs) n += binary.PutUvarint(buf[n:], ms.state.Bytes) n += binary.PutUvarint(buf[n:], ms.state.FirstSeq) n += binary.PutUvarint(buf[n:], ms.state.LastSeq) n += binary.PutUvarint(buf[n:], failed) n += binary.PutUvarint(buf[n:], uint64(numDeleted)) b := buf[0:n] if numDeleted > 0 { buf, err := ms.dmap.Encode(nil) if err != nil { return nil, err } b = append(b, buf...) } return b, nil } // SyncDeleted will make sure this stream has same deleted state as dbs. func (ms *memStore) SyncDeleted(dbs DeleteBlocks) { ms.mu.Lock() defer ms.mu.Unlock() // For now we share one dmap, so if we have one entry here check if states are the same. // Note this will work for any DeleteBlock type, but we expect this to be a dmap too. if len(dbs) == 1 { min, max, num := ms.dmap.State() if pmin, pmax, pnum := dbs[0].State(); pmin == min && pmax == max && pnum == num { return } } lseq := ms.state.LastSeq for _, db := range dbs { // Skip if beyond our current state. if first, _, _ := db.State(); first > lseq { continue } db.Range(func(seq uint64) bool { ms.removeMsg(seq, false) return true }) } } func (o *consumerMemStore) Update(state *ConsumerState) error { // Sanity checks. if state.AckFloor.Consumer > state.Delivered.Consumer { return fmt.Errorf("bad ack floor for consumer") } if state.AckFloor.Stream > state.Delivered.Stream { return fmt.Errorf("bad ack floor for stream") } // Copy to our state. var pending map[uint64]*Pending var redelivered map[uint64]uint64 if len(state.Pending) > 0 { pending = make(map[uint64]*Pending, len(state.Pending)) for seq, p := range state.Pending { pending[seq] = &Pending{p.Sequence, p.Timestamp} if seq <= state.AckFloor.Stream || seq > state.Delivered.Stream { return fmt.Errorf("bad pending entry, sequence [%d] out of range", seq) } } } if len(state.Redelivered) > 0 { redelivered = make(map[uint64]uint64, len(state.Redelivered)) for seq, dc := range state.Redelivered { redelivered[seq] = dc } } // Replace our state. o.mu.Lock() defer o.mu.Unlock() // Check to see if this is an outdated update. if state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream { return fmt.Errorf("old update ignored") } o.state.Delivered = state.Delivered o.state.AckFloor = state.AckFloor o.state.Pending = pending o.state.Redelivered = redelivered return nil } // SetStarting sets our starting stream sequence. func (o *consumerMemStore) SetStarting(sseq uint64) error { o.mu.Lock() o.state.Delivered.Stream = sseq o.mu.Unlock() return nil } // HasState returns if this store has a recorded state. func (o *consumerMemStore) HasState() bool { return false } func (o *consumerMemStore) UpdateDelivered(dseq, sseq, dc uint64, ts int64) error { o.mu.Lock() defer o.mu.Unlock() if dc != 1 && o.cfg.AckPolicy == AckNone { return ErrNoAckPolicy } if dseq <= o.state.AckFloor.Consumer { return nil } // See if we expect an ack for this. if o.cfg.AckPolicy != AckNone { // Need to create pending records here. if o.state.Pending == nil { o.state.Pending = make(map[uint64]*Pending) } var p *Pending // Check for an update to a message already delivered. if sseq <= o.state.Delivered.Stream { if p = o.state.Pending[sseq]; p != nil { // Do not update p.Sequence, that should be the original delivery sequence. p.Timestamp = ts } } else { // Add to pending. o.state.Pending[sseq] = &Pending{dseq, ts} } // Update delivered as needed. if dseq > o.state.Delivered.Consumer { o.state.Delivered.Consumer = dseq } if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq } if dc > 1 { if maxdc := uint64(o.cfg.MaxDeliver); maxdc > 0 && dc > maxdc { // Make sure to remove from pending. delete(o.state.Pending, sseq) } if o.state.Redelivered == nil { o.state.Redelivered = make(map[uint64]uint64) } // Only update if greater than what we already have. if o.state.Redelivered[sseq] < dc-1 { o.state.Redelivered[sseq] = dc - 1 } } } else { // For AckNone just update delivered and ackfloor at the same time. if dseq > o.state.Delivered.Consumer { o.state.Delivered.Consumer = dseq o.state.AckFloor.Consumer = dseq } if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq o.state.AckFloor.Stream = sseq } } return nil } func (o *consumerMemStore) UpdateAcks(dseq, sseq uint64) error { o.mu.Lock() defer o.mu.Unlock() if o.cfg.AckPolicy == AckNone { return ErrNoAckPolicy } // On restarts the old leader may get a replay from the raft logs that are old. if dseq <= o.state.AckFloor.Consumer { return nil } // Match leader logic on checking if ack is ahead of delivered. // This could happen on a cooperative takeover with high speed deliveries. if sseq > o.state.Delivered.Stream { o.state.Delivered.Stream = sseq + 1 } if len(o.state.Pending) == 0 || o.state.Pending[sseq] == nil { delete(o.state.Redelivered, sseq) return ErrStoreMsgNotFound } // Check for AckAll here. if o.cfg.AckPolicy == AckAll { sgap := sseq - o.state.AckFloor.Stream o.state.AckFloor.Consumer = dseq o.state.AckFloor.Stream = sseq if sgap > uint64(len(o.state.Pending)) { for seq := range o.state.Pending { if seq <= sseq { delete(o.state.Pending, seq) delete(o.state.Redelivered, seq) } } } else { for seq := sseq; seq > sseq-sgap && len(o.state.Pending) > 0; seq-- { delete(o.state.Pending, seq) delete(o.state.Redelivered, seq) } } return nil } // AckExplicit // First delete from our pending state. if p, ok := o.state.Pending[sseq]; ok { delete(o.state.Pending, sseq) if dseq > p.Sequence && p.Sequence > 0 { dseq = p.Sequence // Use the original. } } if len(o.state.Pending) == 0 { o.state.AckFloor.Consumer = o.state.Delivered.Consumer o.state.AckFloor.Stream = o.state.Delivered.Stream } else if dseq == o.state.AckFloor.Consumer+1 { o.state.AckFloor.Consumer = dseq o.state.AckFloor.Stream = sseq if o.state.Delivered.Consumer > dseq { for ss := sseq + 1; ss <= o.state.Delivered.Stream; ss++ { if p, ok := o.state.Pending[ss]; ok { if p.Sequence > 0 { o.state.AckFloor.Consumer = p.Sequence - 1 o.state.AckFloor.Stream = ss - 1 } break } } } } // We do these regardless. delete(o.state.Redelivered, sseq) return nil } func (o *consumerMemStore) UpdateConfig(cfg *ConsumerConfig) error { o.mu.Lock() defer o.mu.Unlock() // This is mostly unchecked here. We are assuming the upper layers have done sanity checking. o.cfg = *cfg return nil } func (o *consumerMemStore) Stop() error { o.mu.Lock() o.closed = true ms := o.ms o.mu.Unlock() ms.RemoveConsumer(o) return nil } func (o *consumerMemStore) Delete() error { return o.Stop() } func (o *consumerMemStore) StreamDelete() error { return o.Stop() } func (o *consumerMemStore) State() (*ConsumerState, error) { return o.stateWithCopy(true) } // This will not copy pending or redelivered, so should only be done under the // consumer owner's lock. func (o *consumerMemStore) BorrowState() (*ConsumerState, error) { return o.stateWithCopy(false) } func (o *consumerMemStore) stateWithCopy(doCopy bool) (*ConsumerState, error) { o.mu.Lock() defer o.mu.Unlock() if o.closed { return nil, ErrStoreClosed } state := &ConsumerState{} state.Delivered = o.state.Delivered state.AckFloor = o.state.AckFloor if len(o.state.Pending) > 0 { if doCopy { state.Pending = o.copyPending() } else { state.Pending = o.state.Pending } } if len(o.state.Redelivered) > 0 { if doCopy { state.Redelivered = o.copyRedelivered() } else { state.Redelivered = o.state.Redelivered } } return state, nil } // EncodedState for this consumer store. func (o *consumerMemStore) EncodedState() ([]byte, error) { o.mu.Lock() defer o.mu.Unlock() if o.closed { return nil, ErrStoreClosed } return encodeConsumerState(&o.state), nil } func (o *consumerMemStore) copyPending() map[uint64]*Pending { pending := make(map[uint64]*Pending, len(o.state.Pending)) for seq, p := range o.state.Pending { pending[seq] = &Pending{p.Sequence, p.Timestamp} } return pending } func (o *consumerMemStore) copyRedelivered() map[uint64]uint64 { redelivered := make(map[uint64]uint64, len(o.state.Redelivered)) for seq, dc := range o.state.Redelivered { redelivered[seq] = dc } return redelivered } // Type returns the type of the underlying store. func (o *consumerMemStore) Type() StorageType { return MemoryStorage } // Templates type templateMemStore struct{} func newTemplateMemStore() *templateMemStore { return &templateMemStore{} } // No-ops for memstore. func (ts *templateMemStore) Store(t *streamTemplate) error { return nil } func (ts *templateMemStore) Delete(t *streamTemplate) error { return nil } nats-server-2.10.27/server/memstore_test.go000066400000000000000000000704761477524627100207200ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_store_tests // +build !skip_store_tests package server import ( "bytes" "errors" "fmt" "math/rand" "reflect" "testing" "time" "github.com/nats-io/nuid" ) func TestMemStoreBasics(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") now := time.Now().UnixNano() if seq, ts, err := ms.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } else if seq != 1 { t.Fatalf("Expected sequence to be 1, got %d", seq) } else if ts < now || ts > now+int64(time.Millisecond) { t.Fatalf("Expected timestamp to be current, got %v", ts-now) } state := ms.State() if state.Msgs != 1 { t.Fatalf("Expected 1 msg, got %d", state.Msgs) } expectedSize := memStoreMsgSize(subj, nil, msg) if state.Bytes != expectedSize { t.Fatalf("Expected %d bytes, got %d", expectedSize, state.Bytes) } sm, err := ms.LoadMsg(1, nil) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if sm.subj != subj { t.Fatalf("Subjects don't match, original %q vs %q", subj, sm.subj) } if !bytes.Equal(sm.msg, msg) { t.Fatalf("Msgs don't match, original %q vs %q", msg, sm.msg) } } func TestMemStoreMsgLimit(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxMsgs: 10}) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { ms.StoreMsg(subj, nil, msg) } state := ms.State() if state.Msgs != 10 { t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) } if _, _, err := ms.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } state = ms.State() if state.Msgs != 10 { t.Fatalf("Expected %d msgs, got %d", 10, state.Msgs) } if state.LastSeq != 11 { t.Fatalf("Expected the last sequence to be 11 now, but got %d", state.LastSeq) } if state.FirstSeq != 2 { t.Fatalf("Expected the first sequence to be 2 now, but got %d", state.FirstSeq) } // Make sure we can not lookup seq 1. if _, err := ms.LoadMsg(1, nil); err == nil { t.Fatalf("Expected error looking up seq 1 but got none") } } func TestMemStoreBytesLimit(t *testing.T) { subj, msg := "foo", make([]byte, 512) storedMsgSize := memStoreMsgSize(subj, nil, msg) toStore := uint64(1024) maxBytes := storedMsgSize * toStore ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)}) require_NoError(t, err) defer ms.Stop() for i := uint64(0); i < toStore; i++ { ms.StoreMsg(subj, nil, msg) } state := ms.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } // Now send 10 more and check that bytes limit enforced. for i := 0; i < 10; i++ { if _, _, err := ms.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } } state = ms.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } if state.FirstSeq != 11 { t.Fatalf("Expected first sequence to be 11, got %d", state.FirstSeq) } if state.LastSeq != toStore+10 { t.Fatalf("Expected last sequence to be %d, got %d", toStore+10, state.LastSeq) } } // https://github.com/nats-io/nats-server/issues/4771 func TestMemStoreBytesLimitWithDiscardNew(t *testing.T) { subj, msg := "tiny", make([]byte, 7) storedMsgSize := memStoreMsgSize(subj, nil, msg) toStore := uint64(3) maxBytes := 100 ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes), Discard: DiscardNew}) require_NoError(t, err) defer ms.Stop() // Now send 10 messages and check that bytes limit enforced. for i := 0; i < 10; i++ { _, _, err := ms.StoreMsg(subj, nil, msg) if i < int(toStore) { if err != nil { t.Fatalf("Error storing msg: %v", err) } } else if !errors.Is(err, ErrMaxBytes) { t.Fatalf("Storing msg should result in: %v", ErrMaxBytes) } } state := ms.State() if state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } if state.Bytes != storedMsgSize*toStore { t.Fatalf("Expected bytes to be %d, got %d", storedMsgSize*toStore, state.Bytes) } } func TestMemStoreAgeLimit(t *testing.T) { maxAge := 10 * time.Millisecond ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxAge: maxAge}) require_NoError(t, err) defer ms.Stop() // Store some messages. Does not really matter how many. subj, msg := "foo", []byte("Hello World") toStore := 100 for i := 0; i < toStore; i++ { ms.StoreMsg(subj, nil, msg) } state := ms.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } checkExpired := func(t *testing.T) { t.Helper() checkFor(t, time.Second, maxAge, func() error { state = ms.State() if state.Msgs != 0 { return fmt.Errorf("Expected no msgs, got %d", state.Msgs) } if state.Bytes != 0 { return fmt.Errorf("Expected no bytes, got %d", state.Bytes) } return nil }) } // Let them expire checkExpired(t) // Now add some more and make sure that timer will fire again. for i := 0; i < toStore; i++ { ms.StoreMsg(subj, nil, msg) } state = ms.State() if state.Msgs != uint64(toStore) { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } checkExpired(t) } func TestMemStoreTimeStamps(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() last := time.Now().UnixNano() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { time.Sleep(5 * time.Microsecond) ms.StoreMsg(subj, nil, msg) } var smv StoreMsg for seq := uint64(1); seq <= 10; seq++ { sm, err := ms.LoadMsg(seq, &smv) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } // These should be different if sm.ts <= last { t.Fatalf("Expected different timestamps, got %v", sm.ts) } last = sm.ts } } func TestMemStorePurge(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { ms.StoreMsg(subj, nil, msg) } if state := ms.State(); state.Msgs != 10 { t.Fatalf("Expected 10 msgs, got %d", state.Msgs) } ms.Purge() if state := ms.State(); state.Msgs != 0 { t.Fatalf("Expected no msgs, got %d", state.Msgs) } } func TestMemStoreCompact(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 10; i++ { ms.StoreMsg(subj, nil, msg) } if state := ms.State(); state.Msgs != 10 { t.Fatalf("Expected 10 msgs, got %d", state.Msgs) } n, err := ms.Compact(6) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n != 5 { t.Fatalf("Expected to have purged 5 msgs, got %d", n) } state := ms.State() if state.Msgs != 5 { t.Fatalf("Expected 5 msgs, got %d", state.Msgs) } if state.FirstSeq != 6 { t.Fatalf("Expected first seq of 6, got %d", state.FirstSeq) } // Now test that compact will also reset first if seq > last n, err = ms.Compact(100) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n != 5 { t.Fatalf("Expected to have purged 5 msgs, got %d", n) } if state = ms.State(); state.FirstSeq != 100 { t.Fatalf("Expected first seq of 100, got %d", state.FirstSeq) } } func TestMemStoreEraseMsg(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") ms.StoreMsg(subj, nil, msg) sm, err := ms.LoadMsg(1, nil) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if !bytes.Equal(msg, sm.msg) { t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) } if removed, _ := ms.EraseMsg(1); !removed { t.Fatalf("Expected erase msg to return success") } } func TestMemStoreMsgHeaders(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, hdr, msg := "foo", []byte("name:derek"), []byte("Hello World") if sz := int(memStoreMsgSize(subj, hdr, msg)); sz != (len(subj) + len(hdr) + len(msg) + 16) { t.Fatalf("Wrong size for stored msg with header") } ms.StoreMsg(subj, hdr, msg) sm, err := ms.LoadMsg(1, nil) if err != nil { t.Fatalf("Unexpected error looking up msg: %v", err) } if !bytes.Equal(msg, sm.msg) { t.Fatalf("Expected same msg, got %q vs %q", sm.msg, msg) } if !bytes.Equal(hdr, sm.hdr) { t.Fatalf("Expected same hdr, got %q vs %q", sm.hdr, hdr) } if removed, _ := ms.EraseMsg(1); !removed { t.Fatalf("Expected erase msg to return success") } } func TestMemStoreStreamStateDeleted(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() subj, toStore := "foo", uint64(10) for i := uint64(1); i <= toStore; i++ { msg := []byte(fmt.Sprintf("[%08d] Hello World!", i)) if _, _, err := ms.StoreMsg(subj, nil, msg); err != nil { t.Fatalf("Error storing msg: %v", err) } } state := ms.State() if len(state.Deleted) != 0 { t.Fatalf("Expected deleted to be empty") } // Now remove some interior messages. var expected []uint64 for seq := uint64(2); seq < toStore; seq += 2 { ms.RemoveMsg(seq) expected = append(expected, seq) } state = ms.State() if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } // Now fill the gap by deleting 1 and 3 ms.RemoveMsg(1) ms.RemoveMsg(3) expected = expected[2:] state = ms.State() if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } if state.FirstSeq != 5 { t.Fatalf("Expected first seq to be 5, got %d", state.FirstSeq) } ms.Purge() if state = ms.State(); len(state.Deleted) != 0 { t.Fatalf("Expected no deleted after purge, got %+v\n", state.Deleted) } } func TestMemStoreStreamTruncate(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() tseq := uint64(50) subj, toStore := "foo", uint64(100) for i := uint64(1); i < tseq; i++ { _, _, err := ms.StoreMsg(subj, nil, []byte("ok")) require_NoError(t, err) } subj = "bar" for i := tseq; i <= toStore; i++ { _, _, err := ms.StoreMsg(subj, nil, []byte("ok")) require_NoError(t, err) } if state := ms.State(); state.Msgs != toStore { t.Fatalf("Expected %d msgs, got %d", toStore, state.Msgs) } // Check that sequence has to be interior. if err := ms.Truncate(toStore + 1); err != ErrInvalidSequence { t.Fatalf("Expected err of '%v', got '%v'", ErrInvalidSequence, err) } if err := ms.Truncate(tseq); err != nil { t.Fatalf("Unexpected error: %v", err) } if state := ms.State(); state.Msgs != tseq { t.Fatalf("Expected %d msgs, got %d", tseq, state.Msgs) } // Now make sure we report properly if we have some deleted interior messages. ms.RemoveMsg(10) ms.RemoveMsg(20) ms.RemoveMsg(30) ms.RemoveMsg(40) tseq = uint64(25) if err := ms.Truncate(tseq); err != nil { t.Fatalf("Unexpected error: %v", err) } state := ms.State() if state.Msgs != tseq-2 { t.Fatalf("Expected %d msgs, got %d", tseq-2, state.Msgs) } if state.NumSubjects != 1 { t.Fatalf("Expected only 1 subject, got %d", state.NumSubjects) } expected := []uint64{10, 20} if !reflect.DeepEqual(state.Deleted, expected) { t.Fatalf("Expected deleted to be %+v, got %+v\n", expected, state.Deleted) } } func TestMemStorePurgeExWithSubject(t *testing.T) { ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage}) require_NoError(t, err) defer ms.Stop() for i := 0; i < 100; i++ { _, _, err = ms.StoreMsg("foo", nil, nil) require_NoError(t, err) } // This should purge all. ms.PurgeEx("foo", 1, 0) require_True(t, ms.State().Msgs == 0) } func TestMemStoreUpdateMaxMsgsPerSubject(t *testing.T) { cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"foo"}, MaxMsgsPer: 10, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() // Make sure this is honored on an update. cfg.MaxMsgsPer = 50 err = ms.UpdateConfig(cfg) require_NoError(t, err) numStored := 22 for i := 0; i < numStored; i++ { _, _, err = ms.StoreMsg("foo", nil, nil) require_NoError(t, err) } ss := ms.SubjectsState("foo")["foo"] if ss.Msgs != uint64(numStored) { t.Fatalf("Expected to have %d stored, got %d", numStored, ss.Msgs) } // Now make sure we trunk if setting to lower value. cfg.MaxMsgsPer = 10 err = ms.UpdateConfig(cfg) require_NoError(t, err) ss = ms.SubjectsState("foo")["foo"] if ss.Msgs != 10 { t.Fatalf("Expected to have %d stored, got %d", 10, ss.Msgs) } } func TestMemStoreStreamTruncateReset(t *testing.T) { cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"foo"}, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() subj, msg := "foo", []byte("Hello World") for i := 0; i < 1000; i++ { _, _, err := ms.StoreMsg(subj, nil, msg) require_NoError(t, err) } // Reset everything require_NoError(t, ms.Truncate(0)) state := ms.State() require_True(t, state.Msgs == 0) require_True(t, state.Bytes == 0) require_True(t, state.FirstSeq == 0) require_True(t, state.LastSeq == 0) require_True(t, state.NumSubjects == 0) require_True(t, state.NumDeleted == 0) for i := 0; i < 1000; i++ { _, _, err := ms.StoreMsg(subj, nil, msg) require_NoError(t, err) } state = ms.State() require_True(t, state.Msgs == 1000) require_True(t, state.Bytes == 30000) require_True(t, state.FirstSeq == 1) require_True(t, state.LastSeq == 1000) require_True(t, state.NumSubjects == 1) require_True(t, state.NumDeleted == 0) } func TestMemStoreStreamCompactMultiBlockSubjectInfo(t *testing.T) { cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"foo.*"}, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } // Compact such that we know we throw blocks away from the beginning. deleted, err := ms.Compact(501) require_NoError(t, err) require_True(t, deleted == 500) // Make sure we adjusted for subjects etc. state := ms.State() require_True(t, state.NumSubjects == 500) } func TestMemStoreSubjectsTotals(t *testing.T) { cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"*.*"}, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() fmap := make(map[int]int) bmap := make(map[int]int) var m map[int]int var ft string for i := 0; i < 10_000; i++ { // Flip coin for prefix if rand.Intn(2) == 0 { ft, m = "foo", fmap } else { ft, m = "bar", bmap } dt := rand.Intn(100) subj := fmt.Sprintf("%s.%d", ft, dt) m[dt]++ _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } // Now test SubjectsTotal for dt, total := range fmap { subj := fmt.Sprintf("foo.%d", dt) m := ms.SubjectsTotals(subj) if m[subj] != uint64(total) { t.Fatalf("Expected %q to have %d total, got %d", subj, total, m[subj]) } } // Check fmap. if st := ms.SubjectsTotals("foo.*"); len(st) != len(fmap) { t.Fatalf("Expected %d subjects for %q, got %d", len(fmap), "foo.*", len(st)) } else { expected := 0 for _, n := range fmap { expected += n } received := uint64(0) for _, n := range st { received += n } if received != uint64(expected) { t.Fatalf("Expected %d total but got %d", expected, received) } } // Check bmap. if st := ms.SubjectsTotals("bar.*"); len(st) != len(bmap) { t.Fatalf("Expected %d subjects for %q, got %d", len(bmap), "bar.*", len(st)) } else { expected := 0 for _, n := range bmap { expected += n } received := uint64(0) for _, n := range st { received += n } if received != uint64(expected) { t.Fatalf("Expected %d total but got %d", expected, received) } } // All with pwc match. if st, expected := ms.SubjectsTotals("*.*"), len(bmap)+len(fmap); len(st) != expected { t.Fatalf("Expected %d subjects for %q, got %d", expected, "*.*", len(st)) } } func TestMemStoreNumPending(t *testing.T) { cfg := &StreamConfig{ Name: "TEST", Storage: MemoryStorage, Subjects: []string{"*.*.*.*"}, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() tokens := []string{"foo", "bar", "baz"} genSubj := func() string { return fmt.Sprintf("%s.%s.%s.%s", tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], ) } for i := 0; i < 50_000; i++ { subj := genSubj() _, _, err := ms.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } state := ms.State() // Scan one by one for sanity check against other calculations. sanityCheck := func(sseq uint64, filter string) SimpleState { t.Helper() var ss SimpleState var smv StoreMsg // For here we know 0 is invalid, set to 1. if sseq == 0 { sseq = 1 } for seq := sseq; seq <= state.LastSeq; seq++ { sm, err := ms.LoadMsg(seq, &smv) if err != nil { t.Logf("Encountered error %v loading sequence: %d", err, seq) continue } if subjectIsSubsetMatch(sm.subj, filter) { ss.Msgs++ ss.Last = seq if ss.First == 0 || seq < ss.First { ss.First = seq } } } return ss } check := func(sseq uint64, filter string) { t.Helper() np, lvs := ms.NumPending(sseq, filter, false) ss := ms.FilteredState(sseq, filter) sss := sanityCheck(sseq, filter) if lvs != state.LastSeq { t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) } if ss.Msgs != np { t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) } if ss != sss { t.Fatalf("Failed sanity check, expected %+v got %+v", sss, ss) } } sanityCheckLastOnly := func(sseq uint64, filter string) SimpleState { t.Helper() var ss SimpleState var smv StoreMsg // For here we know 0 is invalid, set to 1. if sseq == 0 { sseq = 1 } seen := make(map[string]bool) for seq := state.LastSeq; seq >= sseq; seq-- { sm, err := ms.LoadMsg(seq, &smv) if err != nil { t.Logf("Encountered error %v loading sequence: %d", err, seq) continue } if !seen[sm.subj] && subjectIsSubsetMatch(sm.subj, filter) { ss.Msgs++ if ss.Last == 0 { ss.Last = seq } if ss.First == 0 || seq < ss.First { ss.First = seq } seen[sm.subj] = true } } return ss } checkLastOnly := func(sseq uint64, filter string) { t.Helper() np, lvs := ms.NumPending(sseq, filter, true) ss := sanityCheckLastOnly(sseq, filter) if lvs != state.LastSeq { t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) } if ss.Msgs != np { t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) } } startSeqs := []uint64{0, 1, 2, 200, 444, 555, 2222, 8888, 12_345, 28_222, 33_456, 44_400, 49_999} checkSubs := []string{"foo.>", "*.bar.>", "foo.bar.*.baz", "*.bar.>", "*.foo.bar.*", "foo.foo.bar.baz"} for _, filter := range checkSubs { for _, start := range startSeqs { check(start, filter) checkLastOnly(start, filter) } } } func TestMemStoreInitialFirstSeq(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Storage: MemoryStorage, FirstSeq: 1000, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() seq, _, err := ms.StoreMsg("A", nil, []byte("OK")) require_NoError(t, err) if seq != 1000 { t.Fatalf("Message should have been sequence 1000 but was %d", seq) } seq, _, err = ms.StoreMsg("B", nil, []byte("OK")) require_NoError(t, err) if seq != 1001 { t.Fatalf("Message should have been sequence 1001 but was %d", seq) } var state StreamState ms.FastState(&state) switch { case state.Msgs != 2: t.Fatalf("Expected 2 messages, got %d", state.Msgs) case state.FirstSeq != 1000: t.Fatalf("Expected first seq 1000, got %d", state.FirstSeq) case state.LastSeq != 1001: t.Fatalf("Expected last seq 1001, got %d", state.LastSeq) } } func TestMemStoreDeleteBlocks(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() // Put in 10_000 msgs. total := 10_000 for i := 0; i < total; i++ { _, _, err := ms.StoreMsg("A", nil, []byte("OK")) require_NoError(t, err) } // Now pick 5k random sequences. delete := 5000 deleteMap := make(map[int]struct{}, delete) for len(deleteMap) < delete { deleteMap[rand.Intn(total)+1] = struct{}{} } // Now remove? for seq := range deleteMap { ms.RemoveMsg(uint64(seq)) } var state StreamState ms.FastState(&state) // For now we just track via one dmap. ms.mu.RLock() dmap := ms.dmap.Clone() ms.mu.RUnlock() require_True(t, dmap.Size() == state.NumDeleted) } // https://github.com/nats-io/nats-server/issues/4850 func TestMemStoreGetSeqFromTimeWithLastDeleted(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) // Put in 1000 msgs. total := 1000 var st time.Time for i := 1; i <= total; i++ { _, _, err := ms.StoreMsg("A", nil, []byte("OK")) require_NoError(t, err) if i == total/2 { time.Sleep(100 * time.Millisecond) st = time.Now() } } // Delete last 100 for seq := total - 100; seq <= total; seq++ { ms.RemoveMsg(uint64(seq)) } // Make sure this does not panic with last sequence no longer accessible. seq := ms.GetSeqFromTime(st) // Make sure we get the right value. require_Equal(t, seq, 501) } func TestMemStoreSkipMsgs(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) // Test on empty FS first. // Make sure wrong starting sequence fails. err = ms.SkipMsgs(10, 100) require_Error(t, err, ErrSequenceMismatch) err = ms.SkipMsgs(1, 100) require_NoError(t, err) state := ms.State() require_Equal(t, state.FirstSeq, 101) require_Equal(t, state.LastSeq, 100) // Now add alot. err = ms.SkipMsgs(101, 100_000) require_NoError(t, err) state = ms.State() require_Equal(t, state.FirstSeq, 100_101) require_Equal(t, state.LastSeq, 100_100) // Now add in a message, and then skip to check dmap. ms, err = newMemStore(cfg) require_NoError(t, err) ms.StoreMsg("foo", nil, nil) err = ms.SkipMsgs(2, 10) require_NoError(t, err) state = ms.State() require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 11) require_Equal(t, state.Msgs, 1) require_Equal(t, state.NumDeleted, 10) require_Equal(t, len(state.Deleted), 10) // Check Fast State too. state.Deleted = nil ms.FastState(&state) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 11) require_Equal(t, state.Msgs, 1) require_Equal(t, state.NumDeleted, 10) } // Bug would cause PurgeEx to fail if it encountered a deleted msg at sequence to delete up to. func TestMemStorePurgeExWithDeletedMsgs(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"foo"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { ms.StoreMsg("foo", nil, msg) } ms.RemoveMsg(2) ms.RemoveMsg(9) // This was the bug n, err := ms.PurgeEx(_EMPTY_, 9, 0) require_NoError(t, err) require_Equal(t, n, 7) var state StreamState ms.FastState(&state) require_Equal(t, state.FirstSeq, 10) require_Equal(t, state.LastSeq, 10) require_Equal(t, state.Msgs, 1) } // When all messages are deleted we should have a state of first = last + 1. func TestMemStoreDeleteAllFirstSequenceCheck(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"foo"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() msg := []byte("abc") for i := 1; i <= 10; i++ { ms.StoreMsg("foo", nil, msg) } for seq := uint64(1); seq <= 10; seq++ { ms.RemoveMsg(seq) } var state StreamState ms.FastState(&state) require_Equal(t, state.FirstSeq, 11) require_Equal(t, state.LastSeq, 10) require_Equal(t, state.Msgs, 0) } func TestMemStoreNumPendingMulti(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"ev.*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() totalMsgs := 100_000 totalSubjects := 10_000 numFiltered := 5000 startSeq := uint64(5_000 + rand.Intn(90_000)) subjects := make([]string, 0, totalSubjects) for i := 0; i < totalSubjects; i++ { subjects = append(subjects, fmt.Sprintf("ev.%s", nuid.Next())) } // Put in 100k msgs with random subjects. msg := bytes.Repeat([]byte("ZZZ"), 333) for i := 0; i < totalMsgs; i++ { _, _, err = ms.StoreMsg(subjects[rand.Intn(totalSubjects)], nil, msg) require_NoError(t, err) } // Now we want to do a calculate NumPendingMulti. filters := NewSublistNoCache() for filters.Count() < uint32(numFiltered) { filter := subjects[rand.Intn(totalSubjects)] if !filters.HasInterest(filter) { filters.Insert(&subscription{subject: []byte(filter)}) } } // Use new function. total, _ := ms.NumPendingMulti(startSeq, filters, false) // Check our results. var checkTotal uint64 var smv StoreMsg for seq := startSeq; seq <= uint64(totalMsgs); seq++ { sm, err := ms.LoadMsg(seq, &smv) require_NoError(t, err) if filters.HasInterest(sm.subj) { checkTotal++ } } require_Equal(t, total, checkTotal) } func TestMemStoreNumPendingBug(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"foo.*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() // 12 msgs total for _, subj := range []string{"foo.foo", "foo.bar", "foo.baz", "foo.zzz"} { ms.StoreMsg("foo.aaa", nil, nil) ms.StoreMsg(subj, nil, nil) ms.StoreMsg(subj, nil, nil) } total, _ := ms.NumPending(4, "foo.*", false) var checkTotal uint64 var smv StoreMsg for seq := 4; seq <= 12; seq++ { sm, err := ms.LoadMsg(uint64(seq), &smv) require_NoError(t, err) if subjectIsSubsetMatch(sm.subj, "foo.*") { checkTotal++ } } require_Equal(t, total, checkTotal) } /////////////////////////////////////////////////////////////////////////// // Benchmarks /////////////////////////////////////////////////////////////////////////// func Benchmark_MemStoreNumPendingWithLargeInteriorDeletesScan(b *testing.B) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(b, err) defer ms.Stop() msg := []byte("abc") ms.StoreMsg("foo.bar.baz", nil, msg) for i := 1; i <= 1_000_000; i++ { ms.SkipMsg() } ms.StoreMsg("foo.bar.baz", nil, msg) b.ResetTimer() for i := 0; i < b.N; i++ { total, _ := ms.NumPending(600_000, "foo.*.baz", false) if total != 1 { b.Fatalf("Expected total of 2 got %d", total) } } } func Benchmark_MemStoreNumPendingWithLargeInteriorDeletesExclude(b *testing.B) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"foo.*.*"}, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(b, err) defer ms.Stop() msg := []byte("abc") ms.StoreMsg("foo.bar.baz", nil, msg) for i := 1; i <= 1_000_000; i++ { ms.SkipMsg() } ms.StoreMsg("foo.bar.baz", nil, msg) b.ResetTimer() for i := 0; i < b.N; i++ { total, _ := ms.NumPending(400_000, "foo.*.baz", false) if total != 1 { b.Fatalf("Expected total of 2 got %d", total) } } } nats-server-2.10.27/server/monitor.go000066400000000000000000003503311477524627100175040ustar00rootroot00000000000000// Copyright 2013-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/hex" "encoding/json" "expvar" "fmt" "net" "net/http" "net/url" "os" "path/filepath" "runtime" "runtime/pprof" "slices" "sort" "strconv" "strings" "sync/atomic" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/pse" ) // Connz represents detailed information on current client connections. type Connz struct { ID string `json:"server_id"` Now time.Time `json:"now"` NumConns int `json:"num_connections"` Total int `json:"total"` Offset int `json:"offset"` Limit int `json:"limit"` Conns []*ConnInfo `json:"connections"` } // ConnzOptions are the options passed to Connz() type ConnzOptions struct { // Sort indicates how the results will be sorted. Check SortOpt for possible values. // Only the sort by connection ID (ByCid) is ascending, all others are descending. Sort SortOpt `json:"sort"` // Username indicates if user names should be included in the results. Username bool `json:"auth"` // Subscriptions indicates if subscriptions should be included in the results. Subscriptions bool `json:"subscriptions"` // SubscriptionsDetail indicates if subscription details should be included in the results SubscriptionsDetail bool `json:"subscriptions_detail"` // Offset is used for pagination. Connz() only returns connections starting at this // offset from the global results. Offset int `json:"offset"` // Limit is the maximum number of connections that should be returned by Connz(). Limit int `json:"limit"` // Filter for this explicit client connection. CID uint64 `json:"cid"` // Filter for this explicit client connection based on the MQTT client ID MQTTClient string `json:"mqtt_client"` // Filter by connection state. State ConnState `json:"state"` // The below options only apply if auth is true. // Filter by username. User string `json:"user"` // Filter by account. Account string `json:"acc"` // Filter by subject interest FilterSubject string `json:"filter_subject"` } // ConnState is for filtering states of connections. We will only have two, open and closed. type ConnState int const ( // ConnOpen filters on open clients. ConnOpen = ConnState(iota) // ConnClosed filters on closed clients. ConnClosed // ConnAll returns all clients. ConnAll ) // ConnInfo has detailed information on a per connection basis. type ConnInfo struct { Cid uint64 `json:"cid"` Kind string `json:"kind,omitempty"` Type string `json:"type,omitempty"` IP string `json:"ip"` Port int `json:"port"` Start time.Time `json:"start"` LastActivity time.Time `json:"last_activity"` Stop *time.Time `json:"stop,omitempty"` Reason string `json:"reason,omitempty"` RTT string `json:"rtt,omitempty"` Uptime string `json:"uptime"` Idle string `json:"idle"` Pending int `json:"pending_bytes"` InMsgs int64 `json:"in_msgs"` OutMsgs int64 `json:"out_msgs"` InBytes int64 `json:"in_bytes"` OutBytes int64 `json:"out_bytes"` NumSubs uint32 `json:"subscriptions"` Name string `json:"name,omitempty"` Lang string `json:"lang,omitempty"` Version string `json:"version,omitempty"` TLSVersion string `json:"tls_version,omitempty"` TLSCipher string `json:"tls_cipher_suite,omitempty"` TLSPeerCerts []*TLSPeerCert `json:"tls_peer_certs,omitempty"` TLSFirst bool `json:"tls_first,omitempty"` AuthorizedUser string `json:"authorized_user,omitempty"` Account string `json:"account,omitempty"` Subs []string `json:"subscriptions_list,omitempty"` SubsDetail []SubDetail `json:"subscriptions_list_detail,omitempty"` JWT string `json:"jwt,omitempty"` IssuerKey string `json:"issuer_key,omitempty"` NameTag string `json:"name_tag,omitempty"` Tags jwt.TagList `json:"tags,omitempty"` MQTTClient string `json:"mqtt_client,omitempty"` // This is the MQTT client id // Internal rtt int64 // For fast sorting } // TLSPeerCert contains basic information about a TLS peer certificate type TLSPeerCert struct { Subject string `json:"subject,omitempty"` SubjectPKISha256 string `json:"spki_sha256,omitempty"` CertSha256 string `json:"cert_sha256,omitempty"` } // DefaultConnListSize is the default size of the connection list. const DefaultConnListSize = 1024 // DefaultSubListSize is the default size of the subscriptions list. const DefaultSubListSize = 1024 const defaultStackBufSize = 10000 func newSubsDetailList(client *client) []SubDetail { subsDetail := make([]SubDetail, 0, len(client.subs)) for _, sub := range client.subs { subsDetail = append(subsDetail, newClientSubDetail(sub)) } return subsDetail } func newSubsList(client *client) []string { subs := make([]string, 0, len(client.subs)) for _, sub := range client.subs { subs = append(subs, string(sub.subject)) } return subs } // Connz returns a Connz struct containing information about connections. func (s *Server) Connz(opts *ConnzOptions) (*Connz, error) { var ( sortOpt = ByCid auth bool subs bool subsDet bool offset int limit = DefaultConnListSize cid = uint64(0) state = ConnOpen user string acc string a *Account filter string mqttCID string ) if opts != nil { // If no sort option given or sort is by uptime, then sort by cid if opts.Sort != _EMPTY_ { sortOpt = opts.Sort if !sortOpt.IsValid() { return nil, fmt.Errorf("invalid sorting option: %s", sortOpt) } } // Auth specifics. auth = opts.Username user = opts.User acc = opts.Account mqttCID = opts.MQTTClient subs = opts.Subscriptions subsDet = opts.SubscriptionsDetail offset = opts.Offset if offset < 0 { offset = 0 } limit = opts.Limit if limit <= 0 { limit = DefaultConnListSize } // state state = opts.State // ByStop only makes sense on closed connections if sortOpt == ByStop && state != ConnClosed { return nil, fmt.Errorf("sort by stop only valid on closed connections") } // ByReason is the same. if sortOpt == ByReason && state != ConnClosed { return nil, fmt.Errorf("sort by reason only valid on closed connections") } // If searching by CID if opts.CID > 0 { cid = opts.CID limit = 1 } // If filtering by subject. if opts.FilterSubject != _EMPTY_ && opts.FilterSubject != fwcs { if acc == _EMPTY_ { return nil, fmt.Errorf("filter by subject only valid with account filtering") } filter = opts.FilterSubject } } c := &Connz{ Offset: offset, Limit: limit, Now: time.Now().UTC(), } // Open clients var openClients []*client // Hold for closed clients if requested. var closedClients []*closedClient var clist map[uint64]*client if acc != _EMPTY_ { var err error a, err = s.lookupAccount(acc) if err != nil { return c, nil } a.mu.RLock() clist = make(map[uint64]*client, a.numLocalConnections()) for c := range a.clients { if c.kind == CLIENT || c.kind == LEAF { clist[c.cid] = c } } a.mu.RUnlock() } // Walk the open client list with server lock held. s.mu.RLock() // Default to all client unless filled in above. if clist == nil { clist = s.clients } // copy the server id for monitoring c.ID = s.info.ID // Number of total clients. The resulting ConnInfo array // may be smaller if pagination is used. switch state { case ConnOpen: c.Total = len(clist) case ConnClosed: closedClients = s.closed.closedClients() c.Total = len(closedClients) case ConnAll: c.Total = len(clist) closedClients = s.closed.closedClients() c.Total += len(closedClients) } // We may need to filter these connections. if acc != _EMPTY_ && len(closedClients) > 0 { var ccc []*closedClient for _, cc := range closedClients { if cc.acc != acc { continue } ccc = append(ccc, cc) } c.Total -= (len(closedClients) - len(ccc)) closedClients = ccc } totalClients := c.Total if cid > 0 { // Meaning we only want 1. totalClients = 1 } if state == ConnOpen || state == ConnAll { openClients = make([]*client, 0, totalClients) } // Data structures for results. var conns []ConnInfo // Limits allocs for actual ConnInfos. var pconns ConnInfos switch state { case ConnOpen: conns = make([]ConnInfo, totalClients) pconns = make(ConnInfos, totalClients) case ConnClosed: pconns = make(ConnInfos, totalClients) case ConnAll: conns = make([]ConnInfo, cap(openClients)) pconns = make(ConnInfos, totalClients) } // Search by individual CID. if cid > 0 { if state == ConnClosed || state == ConnAll { copyClosed := closedClients closedClients = nil for _, cc := range copyClosed { if cc.Cid == cid { closedClients = []*closedClient{cc} break } } } else if state == ConnOpen || state == ConnAll { client := s.clients[cid] if client != nil { openClients = append(openClients, client) } } } else { // Gather all open clients. if state == ConnOpen || state == ConnAll { for _, client := range clist { // If we have an account specified we need to filter. if acc != _EMPTY_ && (client.acc == nil || client.acc.Name != acc) { continue } // Do user filtering second if user != _EMPTY_ && client.getRawAuthUserLock() != user { continue } // Do mqtt client ID filtering next if mqttCID != _EMPTY_ && client.getMQTTClientID() != mqttCID { continue } openClients = append(openClients, client) } } } s.mu.RUnlock() // Filter by subject now if needed. We do this outside of server lock. if filter != _EMPTY_ { var oc []*client for _, c := range openClients { c.mu.Lock() for _, sub := range c.subs { if SubjectsCollide(filter, string(sub.subject)) { oc = append(oc, c) break } } c.mu.Unlock() openClients = oc } } // Just return with empty array if nothing here. if len(openClients) == 0 && len(closedClients) == 0 { c.Conns = ConnInfos{} return c, nil } // Now whip through and generate ConnInfo entries // Open Clients i := 0 for _, client := range openClients { client.mu.Lock() ci := &conns[i] ci.fill(client, client.nc, c.Now, auth) // Fill in subscription data if requested. if len(client.subs) > 0 { if subsDet { ci.SubsDetail = newSubsDetailList(client) } else if subs { ci.Subs = newSubsList(client) } } // Fill in user if auth requested. if auth { ci.AuthorizedUser = client.getRawAuthUser() // Add in account iff not the global account. if client.acc != nil && (client.acc.Name != globalAccountName) { ci.Account = client.acc.Name } ci.JWT = client.opts.JWT ci.IssuerKey = issuerForClient(client) ci.Tags = client.tags ci.NameTag = client.nameTag } client.mu.Unlock() pconns[i] = ci i++ } // Closed Clients var needCopy bool if subs || auth { needCopy = true } for _, cc := range closedClients { // If we have an account specified we need to filter. if acc != _EMPTY_ && cc.acc != acc { continue } // Do user filtering second if user != _EMPTY_ && cc.user != user { continue } // Do mqtt client ID filtering next if mqttCID != _EMPTY_ && cc.MQTTClient != mqttCID { continue } // Copy if needed for any changes to the ConnInfo if needCopy { cx := *cc cc = &cx } // Fill in subscription data if requested. if len(cc.subs) > 0 { if subsDet { cc.SubsDetail = cc.subs } else if subs { cc.Subs = make([]string, 0, len(cc.subs)) for _, sub := range cc.subs { cc.Subs = append(cc.Subs, sub.Subject) } } } // Fill in user if auth requested. if auth { cc.AuthorizedUser = cc.user // Add in account iff not the global account. if cc.acc != _EMPTY_ && (cc.acc != globalAccountName) { cc.Account = cc.acc } } pconns[i] = &cc.ConnInfo i++ } // This will trip if we have filtered out client connections. if len(pconns) != i { pconns = pconns[:i] totalClients = i } switch sortOpt { case ByCid, ByStart: sort.Sort(byCid{pconns}) case BySubs: sort.Sort(sort.Reverse(bySubs{pconns})) case ByPending: sort.Sort(sort.Reverse(byPending{pconns})) case ByOutMsgs: sort.Sort(sort.Reverse(byOutMsgs{pconns})) case ByInMsgs: sort.Sort(sort.Reverse(byInMsgs{pconns})) case ByOutBytes: sort.Sort(sort.Reverse(byOutBytes{pconns})) case ByInBytes: sort.Sort(sort.Reverse(byInBytes{pconns})) case ByLast: sort.Sort(sort.Reverse(byLast{pconns})) case ByIdle: sort.Sort(sort.Reverse(byIdle{pconns, c.Now})) case ByUptime: sort.Sort(byUptime{pconns, time.Now()}) case ByStop: sort.Sort(sort.Reverse(byStop{pconns})) case ByReason: sort.Sort(byReason{pconns}) case ByRTT: sort.Sort(sort.Reverse(byRTT{pconns})) } minoff := c.Offset maxoff := c.Offset + c.Limit maxIndex := totalClients // Make sure these are sane. if minoff > maxIndex { minoff = maxIndex } if maxoff > maxIndex { maxoff = maxIndex } // Now pare down to the requested size. // TODO(dlc) - for very large number of connections we // could save the whole list in a hash, send hash on first // request and allow users to use has for subsequent pages. // Low TTL, say < 1sec. c.Conns = pconns[minoff:maxoff] c.NumConns = len(c.Conns) return c, nil } // Fills in the ConnInfo from the client. // client should be locked. func (ci *ConnInfo) fill(client *client, nc net.Conn, now time.Time, auth bool) { // For fast sort if required. rtt := client.getRTT() ci.rtt = int64(rtt) ci.Cid = client.cid ci.MQTTClient = client.getMQTTClientID() ci.Kind = client.kindString() ci.Type = client.clientTypeString() ci.Start = client.start ci.LastActivity = client.last ci.Uptime = myUptime(now.Sub(client.start)) ci.Idle = myUptime(now.Sub(client.last)) ci.RTT = rtt.String() ci.OutMsgs = client.outMsgs ci.OutBytes = client.outBytes ci.NumSubs = uint32(len(client.subs)) ci.Pending = int(client.out.pb) ci.Name = client.opts.Name ci.Lang = client.opts.Lang ci.Version = client.opts.Version // inMsgs and inBytes are updated outside of the client's lock, so // we need to use atomic here. ci.InMsgs = atomic.LoadInt64(&client.inMsgs) ci.InBytes = atomic.LoadInt64(&client.inBytes) // If the connection is gone, too bad, we won't set TLSVersion and TLSCipher. // Exclude clients that are still doing handshake so we don't block in // ConnectionState(). if client.flags.isSet(handshakeComplete) && nc != nil { if conn, ok := nc.(*tls.Conn); ok { cs := conn.ConnectionState() ci.TLSVersion = tlsVersion(cs.Version) ci.TLSCipher = tlsCipher(cs.CipherSuite) if auth && len(cs.PeerCertificates) > 0 { ci.TLSPeerCerts = makePeerCerts(cs.PeerCertificates) } ci.TLSFirst = client.flags.isSet(didTLSFirst) } } if client.port != 0 { ci.Port = int(client.port) ci.IP = client.host } } func makePeerCerts(pc []*x509.Certificate) []*TLSPeerCert { res := make([]*TLSPeerCert, len(pc)) for i, c := range pc { tmp := sha256.Sum256(c.RawSubjectPublicKeyInfo) ssha := hex.EncodeToString(tmp[:]) tmp = sha256.Sum256(c.Raw) csha := hex.EncodeToString(tmp[:]) res[i] = &TLSPeerCert{Subject: c.Subject.String(), SubjectPKISha256: ssha, CertSha256: csha} } return res } // Assume lock is held func (c *client) getRTT() time.Duration { if c.rtt == 0 { // If a real client, go ahead and send ping now to get a value // for RTT. For tests and telnet, or if client is closing, etc skip. if c.opts.Lang != _EMPTY_ { c.sendRTTPingLocked() } return 0 } var rtt time.Duration if c.rtt > time.Microsecond && c.rtt < time.Millisecond { rtt = c.rtt.Truncate(time.Microsecond) } else { rtt = c.rtt.Truncate(time.Nanosecond) } return rtt } func decodeBool(w http.ResponseWriter, r *http.Request, param string) (bool, error) { str := r.URL.Query().Get(param) if str == _EMPTY_ { return false, nil } val, err := strconv.ParseBool(str) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("Error decoding boolean for '%s': %v", param, err))) return false, err } return val, nil } func decodeUint64(w http.ResponseWriter, r *http.Request, param string) (uint64, error) { str := r.URL.Query().Get(param) if str == _EMPTY_ { return 0, nil } val, err := strconv.ParseUint(str, 10, 64) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("Error decoding uint64 for '%s': %v", param, err))) return 0, err } return val, nil } func decodeInt(w http.ResponseWriter, r *http.Request, param string) (int, error) { str := r.URL.Query().Get(param) if str == _EMPTY_ { return 0, nil } val, err := strconv.Atoi(str) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf("Error decoding int for '%s': %v", param, err))) return 0, err } return val, nil } func decodeState(w http.ResponseWriter, r *http.Request) (ConnState, error) { str := r.URL.Query().Get("state") if str == _EMPTY_ { return ConnOpen, nil } switch strings.ToLower(str) { case "open": return ConnOpen, nil case "closed": return ConnClosed, nil case "any", "all": return ConnAll, nil } // We do not understand intended state here. w.WriteHeader(http.StatusBadRequest) err := fmt.Errorf("Error decoding state for %s", str) w.Write([]byte(err.Error())) return 0, err } func decodeSubs(w http.ResponseWriter, r *http.Request) (subs bool, subsDet bool, err error) { subsDet = strings.ToLower(r.URL.Query().Get("subs")) == "detail" if !subsDet { subs, err = decodeBool(w, r, "subs") } return } // HandleConnz process HTTP requests for connection information. func (s *Server) HandleConnz(w http.ResponseWriter, r *http.Request) { sortOpt := SortOpt(r.URL.Query().Get("sort")) auth, err := decodeBool(w, r, "auth") if err != nil { return } subs, subsDet, err := decodeSubs(w, r) if err != nil { return } offset, err := decodeInt(w, r, "offset") if err != nil { return } limit, err := decodeInt(w, r, "limit") if err != nil { return } cid, err := decodeUint64(w, r, "cid") if err != nil { return } state, err := decodeState(w, r) if err != nil { return } user := r.URL.Query().Get("user") acc := r.URL.Query().Get("acc") mqttCID := r.URL.Query().Get("mqtt_client") connzOpts := &ConnzOptions{ Sort: sortOpt, Username: auth, Subscriptions: subs, SubscriptionsDetail: subsDet, Offset: offset, Limit: limit, CID: cid, MQTTClient: mqttCID, State: state, User: user, Account: acc, } s.mu.Lock() s.httpReqStats[ConnzPath]++ s.mu.Unlock() c, err := s.Connz(connzOpts) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } b, err := json.MarshalIndent(c, "", " ") if err != nil { s.Errorf("Error marshaling response to /connz request: %v", err) } // Handle response ResponseHandler(w, r, b) } // Routez represents detailed information on current client connections. type Routez struct { ID string `json:"server_id"` Name string `json:"server_name"` Now time.Time `json:"now"` Import *SubjectPermission `json:"import,omitempty"` Export *SubjectPermission `json:"export,omitempty"` NumRoutes int `json:"num_routes"` Routes []*RouteInfo `json:"routes"` } // RoutezOptions are options passed to Routez type RoutezOptions struct { // Subscriptions indicates that Routez will return a route's subscriptions Subscriptions bool `json:"subscriptions"` // SubscriptionsDetail indicates if subscription details should be included in the results SubscriptionsDetail bool `json:"subscriptions_detail"` } // RouteInfo has detailed information on a per connection basis. type RouteInfo struct { Rid uint64 `json:"rid"` RemoteID string `json:"remote_id"` RemoteName string `json:"remote_name"` DidSolicit bool `json:"did_solicit"` IsConfigured bool `json:"is_configured"` IP string `json:"ip"` Port int `json:"port"` Start time.Time `json:"start"` LastActivity time.Time `json:"last_activity"` RTT string `json:"rtt,omitempty"` Uptime string `json:"uptime"` Idle string `json:"idle"` Import *SubjectPermission `json:"import,omitempty"` Export *SubjectPermission `json:"export,omitempty"` Pending int `json:"pending_size"` InMsgs int64 `json:"in_msgs"` OutMsgs int64 `json:"out_msgs"` InBytes int64 `json:"in_bytes"` OutBytes int64 `json:"out_bytes"` NumSubs uint32 `json:"subscriptions"` Subs []string `json:"subscriptions_list,omitempty"` SubsDetail []SubDetail `json:"subscriptions_list_detail,omitempty"` Account string `json:"account,omitempty"` Compression string `json:"compression,omitempty"` } // Routez returns a Routez struct containing information about routes. func (s *Server) Routez(routezOpts *RoutezOptions) (*Routez, error) { rs := &Routez{Routes: []*RouteInfo{}} rs.Now = time.Now().UTC() if routezOpts == nil { routezOpts = &RoutezOptions{} } s.mu.Lock() rs.NumRoutes = s.numRoutes() // copy the server id for monitoring rs.ID = s.info.ID // Check for defined permissions for all connected routes. if perms := s.getOpts().Cluster.Permissions; perms != nil { rs.Import = perms.Import rs.Export = perms.Export } rs.Name = s.getOpts().ServerName addRoute := func(r *client) { r.mu.Lock() ri := &RouteInfo{ Rid: r.cid, RemoteID: r.route.remoteID, RemoteName: r.route.remoteName, DidSolicit: r.route.didSolicit, IsConfigured: r.route.routeType == Explicit, InMsgs: atomic.LoadInt64(&r.inMsgs), OutMsgs: r.outMsgs, InBytes: atomic.LoadInt64(&r.inBytes), OutBytes: r.outBytes, NumSubs: uint32(len(r.subs)), Import: r.opts.Import, Pending: int(r.out.pb), Export: r.opts.Export, RTT: r.getRTT().String(), Start: r.start, LastActivity: r.last, Uptime: myUptime(rs.Now.Sub(r.start)), Idle: myUptime(rs.Now.Sub(r.last)), Account: string(r.route.accName), Compression: r.route.compression, } if len(r.subs) > 0 { if routezOpts.SubscriptionsDetail { ri.SubsDetail = newSubsDetailList(r) } else if routezOpts.Subscriptions { ri.Subs = newSubsList(r) } } switch conn := r.nc.(type) { case *net.TCPConn, *tls.Conn: addr := conn.RemoteAddr().(*net.TCPAddr) ri.Port = addr.Port ri.IP = addr.IP.String() } r.mu.Unlock() rs.Routes = append(rs.Routes, ri) } // Walk the list s.forEachRoute(func(r *client) { addRoute(r) }) s.mu.Unlock() return rs, nil } // HandleRoutez process HTTP requests for route information. func (s *Server) HandleRoutez(w http.ResponseWriter, r *http.Request) { subs, subsDetail, err := decodeSubs(w, r) if err != nil { return } opts := RoutezOptions{Subscriptions: subs, SubscriptionsDetail: subsDetail} s.mu.Lock() s.httpReqStats[RoutezPath]++ s.mu.Unlock() // As of now, no error is ever returned. rs, _ := s.Routez(&opts) b, err := json.MarshalIndent(rs, "", " ") if err != nil { s.Errorf("Error marshaling response to /routez request: %v", err) } // Handle response ResponseHandler(w, r, b) } // Subsz represents detail information on current connections. type Subsz struct { ID string `json:"server_id"` Now time.Time `json:"now"` *SublistStats Total int `json:"total"` Offset int `json:"offset"` Limit int `json:"limit"` Subs []SubDetail `json:"subscriptions_list,omitempty"` } // SubszOptions are the options passed to Subsz. // As of now, there are no options defined. type SubszOptions struct { // Offset is used for pagination. Subsz() only returns connections starting at this // offset from the global results. Offset int `json:"offset"` // Limit is the maximum number of subscriptions that should be returned by Subsz(). Limit int `json:"limit"` // Subscriptions indicates if subscription details should be included in the results. Subscriptions bool `json:"subscriptions"` // Filter based on this account name. Account string `json:"account,omitempty"` // Test the list against this subject. Needs to be literal since it signifies a publish subject. // We will only return subscriptions that would match if a message was sent to this subject. Test string `json:"test,omitempty"` } // SubDetail is for verbose information for subscriptions. type SubDetail struct { Account string `json:"account,omitempty"` Subject string `json:"subject"` Queue string `json:"qgroup,omitempty"` Sid string `json:"sid"` Msgs int64 `json:"msgs"` Max int64 `json:"max,omitempty"` Cid uint64 `json:"cid"` } // Subscription client should be locked and guaranteed to be present. func newSubDetail(sub *subscription) SubDetail { sd := newClientSubDetail(sub) if sub.client.acc != nil { sd.Account = sub.client.acc.Name } return sd } // For subs details under clients. func newClientSubDetail(sub *subscription) SubDetail { return SubDetail{ Subject: string(sub.subject), Queue: string(sub.queue), Sid: string(sub.sid), Msgs: sub.nm, Max: sub.max, Cid: sub.client.cid, } } // Subsz returns a Subsz struct containing subjects statistics func (s *Server) Subsz(opts *SubszOptions) (*Subsz, error) { var ( subdetail bool test bool offset int testSub string filterAcc string limit = DefaultSubListSize ) if opts != nil { subdetail = opts.Subscriptions offset = opts.Offset if offset < 0 { offset = 0 } limit = opts.Limit if limit <= 0 { limit = DefaultSubListSize } if opts.Test != _EMPTY_ { testSub = opts.Test test = true if !IsValidLiteralSubject(testSub) { return nil, fmt.Errorf("invalid test subject, must be valid publish subject: %s", testSub) } } if opts.Account != _EMPTY_ { filterAcc = opts.Account } } slStats := &SublistStats{} // FIXME(dlc) - Make account aware. sz := &Subsz{s.info.ID, time.Now().UTC(), slStats, 0, offset, limit, nil} if subdetail { var raw [4096]*subscription subs := raw[:0] s.accounts.Range(func(k, v any) bool { acc := v.(*Account) if filterAcc != _EMPTY_ && acc.GetName() != filterAcc { return true } slStats.add(acc.sl.Stats()) acc.sl.localSubs(&subs, false) return true }) details := make([]SubDetail, len(subs)) i := 0 // TODO(dlc) - may be inefficient and could just do normal match when total subs is large and filtering. for _, sub := range subs { // Check for filter if test && !matchLiteral(testSub, string(sub.subject)) { continue } if sub.client == nil { continue } sub.client.mu.Lock() details[i] = newSubDetail(sub) sub.client.mu.Unlock() i++ } minoff := sz.Offset maxoff := sz.Offset + sz.Limit maxIndex := i // Make sure these are sane. if minoff > maxIndex { minoff = maxIndex } if maxoff > maxIndex { maxoff = maxIndex } sz.Subs = details[minoff:maxoff] sz.Total = len(sz.Subs) } else { s.accounts.Range(func(k, v any) bool { acc := v.(*Account) if filterAcc != _EMPTY_ && acc.GetName() != filterAcc { return true } slStats.add(acc.sl.Stats()) return true }) } return sz, nil } // HandleSubsz processes HTTP requests for subjects stats. func (s *Server) HandleSubsz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[SubszPath]++ s.mu.Unlock() subs, err := decodeBool(w, r, "subs") if err != nil { return } offset, err := decodeInt(w, r, "offset") if err != nil { return } limit, err := decodeInt(w, r, "limit") if err != nil { return } testSub := r.URL.Query().Get("test") // Filtered account. filterAcc := r.URL.Query().Get("acc") subszOpts := &SubszOptions{ Subscriptions: subs, Offset: offset, Limit: limit, Account: filterAcc, Test: testSub, } st, err := s.Subsz(subszOpts) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } var b []byte if len(st.Subs) == 0 { b, err = json.MarshalIndent(st.SublistStats, "", " ") } else { b, err = json.MarshalIndent(st, "", " ") } if err != nil { s.Errorf("Error marshaling response to /subscriptionsz request: %v", err) } // Handle response ResponseHandler(w, r, b) } // HandleStacksz processes HTTP requests for getting stacks func (s *Server) HandleStacksz(w http.ResponseWriter, r *http.Request) { // Do not get any lock here that would prevent getting the stacks // if we were to have a deadlock somewhere. var defaultBuf [defaultStackBufSize]byte size := defaultStackBufSize buf := defaultBuf[:size] n := 0 for { n = runtime.Stack(buf, true) if n < size { break } size *= 2 buf = make([]byte, size) } // Handle response ResponseHandler(w, r, buf[:n]) } type IpqueueszStatusIPQ struct { Pending int `json:"pending"` InProgress int `json:"in_progress,omitempty"` } type IpqueueszStatus map[string]IpqueueszStatusIPQ func (s *Server) Ipqueuesz(opts *IpqueueszOptions) *IpqueueszStatus { all, qfilter := opts.All, opts.Filter queues := IpqueueszStatus{} s.ipQueues.Range(func(k, v any) bool { var pending, inProgress int name := k.(string) queue, ok := v.(interface { len() int inProgress() int64 }) if ok { pending = queue.len() inProgress = int(queue.inProgress()) } if !all && (pending == 0 && inProgress == 0) { return true } else if qfilter != _EMPTY_ && !strings.Contains(name, qfilter) { return true } queues[name] = IpqueueszStatusIPQ{Pending: pending, InProgress: inProgress} return true }) return &queues } func (s *Server) HandleIPQueuesz(w http.ResponseWriter, r *http.Request) { all, err := decodeBool(w, r, "all") if err != nil { return } qfilter := r.URL.Query().Get("queues") queues := s.Ipqueuesz(&IpqueueszOptions{ All: all, Filter: qfilter, }) b, _ := json.MarshalIndent(queues, "", " ") ResponseHandler(w, r, b) } // Varz will output server information on the monitoring port at /varz. type Varz struct { ID string `json:"server_id"` Name string `json:"server_name"` Version string `json:"version"` Proto int `json:"proto"` GitCommit string `json:"git_commit,omitempty"` GoVersion string `json:"go"` Host string `json:"host"` Port int `json:"port"` AuthRequired bool `json:"auth_required,omitempty"` TLSRequired bool `json:"tls_required,omitempty"` TLSVerify bool `json:"tls_verify,omitempty"` TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` IP string `json:"ip,omitempty"` ClientConnectURLs []string `json:"connect_urls,omitempty"` WSConnectURLs []string `json:"ws_connect_urls,omitempty"` MaxConn int `json:"max_connections"` MaxSubs int `json:"max_subscriptions,omitempty"` PingInterval time.Duration `json:"ping_interval"` MaxPingsOut int `json:"ping_max"` HTTPHost string `json:"http_host"` HTTPPort int `json:"http_port"` HTTPBasePath string `json:"http_base_path"` HTTPSPort int `json:"https_port"` AuthTimeout float64 `json:"auth_timeout"` MaxControlLine int32 `json:"max_control_line"` MaxPayload int `json:"max_payload"` MaxPending int64 `json:"max_pending"` Cluster ClusterOptsVarz `json:"cluster,omitempty"` Gateway GatewayOptsVarz `json:"gateway,omitempty"` LeafNode LeafNodeOptsVarz `json:"leaf,omitempty"` MQTT MQTTOptsVarz `json:"mqtt,omitempty"` Websocket WebsocketOptsVarz `json:"websocket,omitempty"` JetStream JetStreamVarz `json:"jetstream,omitempty"` TLSTimeout float64 `json:"tls_timeout"` WriteDeadline time.Duration `json:"write_deadline"` Start time.Time `json:"start"` Now time.Time `json:"now"` Uptime string `json:"uptime"` Mem int64 `json:"mem"` Cores int `json:"cores"` MaxProcs int `json:"gomaxprocs"` CPU float64 `json:"cpu"` Connections int `json:"connections"` TotalConnections uint64 `json:"total_connections"` Routes int `json:"routes"` Remotes int `json:"remotes"` Leafs int `json:"leafnodes"` InMsgs int64 `json:"in_msgs"` OutMsgs int64 `json:"out_msgs"` InBytes int64 `json:"in_bytes"` OutBytes int64 `json:"out_bytes"` SlowConsumers int64 `json:"slow_consumers"` Subscriptions uint32 `json:"subscriptions"` HTTPReqStats map[string]uint64 `json:"http_req_stats"` ConfigLoadTime time.Time `json:"config_load_time"` Tags jwt.TagList `json:"tags,omitempty"` TrustedOperatorsJwt []string `json:"trusted_operators_jwt,omitempty"` TrustedOperatorsClaim []*jwt.OperatorClaims `json:"trusted_operators_claim,omitempty"` SystemAccount string `json:"system_account,omitempty"` PinnedAccountFail uint64 `json:"pinned_account_fails,omitempty"` OCSPResponseCache *OCSPResponseCacheVarz `json:"ocsp_peer_cache,omitempty"` SlowConsumersStats *SlowConsumersStats `json:"slow_consumer_stats"` } // JetStreamVarz contains basic runtime information about jetstream type JetStreamVarz struct { Config *JetStreamConfig `json:"config,omitempty"` Stats *JetStreamStats `json:"stats,omitempty"` Meta *MetaClusterInfo `json:"meta,omitempty"` } // ClusterOptsVarz contains monitoring cluster information type ClusterOptsVarz struct { Name string `json:"name,omitempty"` Host string `json:"addr,omitempty"` Port int `json:"cluster_port,omitempty"` AuthTimeout float64 `json:"auth_timeout,omitempty"` URLs []string `json:"urls,omitempty"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSRequired bool `json:"tls_required,omitempty"` TLSVerify bool `json:"tls_verify,omitempty"` PoolSize int `json:"pool_size,omitempty"` } // GatewayOptsVarz contains monitoring gateway information type GatewayOptsVarz struct { Name string `json:"name,omitempty"` Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` AuthTimeout float64 `json:"auth_timeout,omitempty"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSRequired bool `json:"tls_required,omitempty"` TLSVerify bool `json:"tls_verify,omitempty"` Advertise string `json:"advertise,omitempty"` ConnectRetries int `json:"connect_retries,omitempty"` Gateways []RemoteGatewayOptsVarz `json:"gateways,omitempty"` RejectUnknown bool `json:"reject_unknown,omitempty"` // config got renamed to reject_unknown_cluster } // RemoteGatewayOptsVarz contains monitoring remote gateway information type RemoteGatewayOptsVarz struct { Name string `json:"name"` TLSTimeout float64 `json:"tls_timeout,omitempty"` URLs []string `json:"urls,omitempty"` } // LeafNodeOptsVarz contains monitoring leaf node information type LeafNodeOptsVarz struct { Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` AuthTimeout float64 `json:"auth_timeout,omitempty"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSRequired bool `json:"tls_required,omitempty"` TLSVerify bool `json:"tls_verify,omitempty"` Remotes []RemoteLeafOptsVarz `json:"remotes,omitempty"` TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` } // DenyRules Contains lists of subjects not allowed to be imported/exported type DenyRules struct { Exports []string `json:"exports,omitempty"` Imports []string `json:"imports,omitempty"` } // RemoteLeafOptsVarz contains monitoring remote leaf node information type RemoteLeafOptsVarz struct { LocalAccount string `json:"local_account,omitempty"` TLSTimeout float64 `json:"tls_timeout,omitempty"` URLs []string `json:"urls,omitempty"` Deny *DenyRules `json:"deny,omitempty"` TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` } // MQTTOptsVarz contains monitoring MQTT information type MQTTOptsVarz struct { Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` NoAuthUser string `json:"no_auth_user,omitempty"` AuthTimeout float64 `json:"auth_timeout,omitempty"` TLSMap bool `json:"tls_map,omitempty"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSPinnedCerts []string `json:"tls_pinned_certs,omitempty"` JsDomain string `json:"js_domain,omitempty"` AckWait time.Duration `json:"ack_wait,omitempty"` MaxAckPending uint16 `json:"max_ack_pending,omitempty"` TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` } // WebsocketOptsVarz contains monitoring websocket information type WebsocketOptsVarz struct { Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` Advertise string `json:"advertise,omitempty"` NoAuthUser string `json:"no_auth_user,omitempty"` JWTCookie string `json:"jwt_cookie,omitempty"` HandshakeTimeout time.Duration `json:"handshake_timeout,omitempty"` AuthTimeout float64 `json:"auth_timeout,omitempty"` NoTLS bool `json:"no_tls,omitempty"` TLSMap bool `json:"tls_map,omitempty"` TLSPinnedCerts []string `json:"tls_pinned_certs,omitempty"` SameOrigin bool `json:"same_origin,omitempty"` AllowedOrigins []string `json:"allowed_origins,omitempty"` Compression bool `json:"compression,omitempty"` TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` } // OCSPResponseCacheVarz contains OCSP response cache information type OCSPResponseCacheVarz struct { Type string `json:"cache_type,omitempty"` Hits int64 `json:"cache_hits,omitempty"` Misses int64 `json:"cache_misses,omitempty"` Responses int64 `json:"cached_responses,omitempty"` Revokes int64 `json:"cached_revoked_responses,omitempty"` Goods int64 `json:"cached_good_responses,omitempty"` Unknowns int64 `json:"cached_unknown_responses,omitempty"` } // VarzOptions are the options passed to Varz(). // Currently, there are no options defined. type VarzOptions struct{} // SlowConsumersStats contains information about the slow consumers from different type of connections. type SlowConsumersStats struct { Clients uint64 `json:"clients"` Routes uint64 `json:"routes"` Gateways uint64 `json:"gateways"` Leafs uint64 `json:"leafs"` } func myUptime(d time.Duration) string { // Just use total seconds for uptime, and display days / years tsecs := d / time.Second tmins := tsecs / 60 thrs := tmins / 60 tdays := thrs / 24 tyrs := tdays / 365 if tyrs > 0 { return fmt.Sprintf("%dy%dd%dh%dm%ds", tyrs, tdays%365, thrs%24, tmins%60, tsecs%60) } if tdays > 0 { return fmt.Sprintf("%dd%dh%dm%ds", tdays, thrs%24, tmins%60, tsecs%60) } if thrs > 0 { return fmt.Sprintf("%dh%dm%ds", thrs, tmins%60, tsecs%60) } if tmins > 0 { return fmt.Sprintf("%dm%ds", tmins, tsecs%60) } return fmt.Sprintf("%ds", tsecs) } // HandleRoot will show basic info and links to others handlers. func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) { // This feels dumb to me, but is required: https://code.google.com/p/go/issues/detail?id=4799 if r.URL.Path != s.httpBasePath { http.NotFound(w, r) return } s.mu.Lock() s.httpReqStats[RootPath]++ s.mu.Unlock() // Calculate source url. If git set go directly to that tag, otherwise just main. var srcUrl string if gitCommit == _EMPTY_ { srcUrl = "https://github.com/nats-io/nats-server" } else if serverVersion != _EMPTY_ { srcUrl = fmt.Sprintf("https://github.com/nats-io/nats-server/tree/%s", serverVersion) } else { srcUrl = fmt.Sprintf("https://github.com/nats-io/nats-server/tree/%s", gitCommit) } fmt.Fprintf(w, ` v%s
General JetStream Connections Accounts Account Stats Subscriptions Routes LeafNodes Gateways Raft Groups Health Probe Help `, srcUrl, VERSION, s.basePath(VarzPath), s.basePath(JszPath), s.basePath(ConnzPath), s.basePath(AccountzPath), s.basePath(AccountStatzPath), s.basePath(SubszPath), s.basePath(RoutezPath), s.basePath(LeafzPath), s.basePath(GatewayzPath), s.basePath(RaftzPath), s.basePath(HealthzPath), ) } func (s *Server) updateJszVarz(js *jetStream, v *JetStreamVarz, doConfig bool) { if doConfig { js.mu.RLock() // We want to snapshot the config since it will then be available outside // of the js lock. So make a copy first, then point to this copy. cfg := js.config v.Config = &cfg js.mu.RUnlock() } v.Stats = js.usageStats() if mg := js.getMetaGroup(); mg != nil { if ci := s.raftNodeToClusterInfo(mg); ci != nil { v.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Size: mg.ClusterSize()} if ci.Leader == s.info.Name { v.Meta.Replicas = ci.Replicas } if ipq := s.jsAPIRoutedReqs; ipq != nil { v.Meta.Pending = ipq.len() } } } } // Varz returns a Varz struct containing the server information. func (s *Server) Varz(varzOpts *VarzOptions) (*Varz, error) { var rss, vss int64 var pcpu float64 // We want to do that outside of the lock. pse.ProcUsage(&pcpu, &rss, &vss) s.mu.RLock() // We need to create a new instance of Varz (with no reference // whatsoever to anything stored in the server) since the user // has access to the returned value. v := s.createVarz(pcpu, rss) s.mu.RUnlock() if js := s.getJetStream(); js != nil { s.updateJszVarz(js, &v.JetStream, true) } return v, nil } // Returns a Varz instance. // Server lock is held on entry. func (s *Server) createVarz(pcpu float64, rss int64) *Varz { info := s.info opts := s.getOpts() c := &opts.Cluster gw := &opts.Gateway ln := &opts.LeafNode mqtt := &opts.MQTT ws := &opts.Websocket clustTlsReq := c.TLSConfig != nil gatewayTlsReq := gw.TLSConfig != nil leafTlsReq := ln.TLSConfig != nil leafTlsVerify := leafTlsReq && ln.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert leafTlsOCSPPeerVerify := s.ocspPeerVerify && leafTlsReq && ln.tlsConfigOpts.OCSPPeerConfig != nil && ln.tlsConfigOpts.OCSPPeerConfig.Verify mqttTlsOCSPPeerVerify := s.ocspPeerVerify && mqtt.TLSConfig != nil && mqtt.tlsConfigOpts.OCSPPeerConfig != nil && mqtt.tlsConfigOpts.OCSPPeerConfig.Verify wsTlsOCSPPeerVerify := s.ocspPeerVerify && ws.TLSConfig != nil && ws.tlsConfigOpts.OCSPPeerConfig != nil && ws.tlsConfigOpts.OCSPPeerConfig.Verify varz := &Varz{ ID: info.ID, Version: info.Version, Proto: info.Proto, GitCommit: info.GitCommit, GoVersion: info.GoVersion, Name: info.Name, Host: info.Host, Port: info.Port, IP: info.IP, HTTPHost: opts.HTTPHost, HTTPPort: opts.HTTPPort, HTTPBasePath: opts.HTTPBasePath, HTTPSPort: opts.HTTPSPort, Cluster: ClusterOptsVarz{ Name: info.Cluster, Host: c.Host, Port: c.Port, AuthTimeout: c.AuthTimeout, TLSTimeout: c.TLSTimeout, TLSRequired: clustTlsReq, TLSVerify: clustTlsReq, PoolSize: opts.Cluster.PoolSize, }, Gateway: GatewayOptsVarz{ Name: gw.Name, Host: gw.Host, Port: gw.Port, AuthTimeout: gw.AuthTimeout, TLSTimeout: gw.TLSTimeout, TLSRequired: gatewayTlsReq, TLSVerify: gatewayTlsReq, Advertise: gw.Advertise, ConnectRetries: gw.ConnectRetries, Gateways: []RemoteGatewayOptsVarz{}, RejectUnknown: gw.RejectUnknown, }, LeafNode: LeafNodeOptsVarz{ Host: ln.Host, Port: ln.Port, AuthTimeout: ln.AuthTimeout, TLSTimeout: ln.TLSTimeout, TLSRequired: leafTlsReq, TLSVerify: leafTlsVerify, TLSOCSPPeerVerify: leafTlsOCSPPeerVerify, Remotes: []RemoteLeafOptsVarz{}, }, MQTT: MQTTOptsVarz{ Host: mqtt.Host, Port: mqtt.Port, NoAuthUser: mqtt.NoAuthUser, AuthTimeout: mqtt.AuthTimeout, TLSMap: mqtt.TLSMap, TLSTimeout: mqtt.TLSTimeout, JsDomain: mqtt.JsDomain, AckWait: mqtt.AckWait, MaxAckPending: mqtt.MaxAckPending, TLSOCSPPeerVerify: mqttTlsOCSPPeerVerify, }, Websocket: WebsocketOptsVarz{ Host: ws.Host, Port: ws.Port, Advertise: ws.Advertise, NoAuthUser: ws.NoAuthUser, JWTCookie: ws.JWTCookie, AuthTimeout: ws.AuthTimeout, NoTLS: ws.NoTLS, TLSMap: ws.TLSMap, SameOrigin: ws.SameOrigin, AllowedOrigins: copyStrings(ws.AllowedOrigins), Compression: ws.Compression, HandshakeTimeout: ws.HandshakeTimeout, TLSOCSPPeerVerify: wsTlsOCSPPeerVerify, }, Start: s.start.UTC(), MaxSubs: opts.MaxSubs, Cores: runtime.NumCPU(), MaxProcs: runtime.GOMAXPROCS(0), Tags: opts.Tags, TrustedOperatorsJwt: opts.operatorJWT, TrustedOperatorsClaim: opts.TrustedOperators, } if len(opts.Routes) > 0 { varz.Cluster.URLs = urlsToStrings(opts.Routes) } if l := len(gw.Gateways); l > 0 { rgwa := make([]RemoteGatewayOptsVarz, l) for i, r := range gw.Gateways { rgwa[i] = RemoteGatewayOptsVarz{ Name: r.Name, TLSTimeout: r.TLSTimeout, } } varz.Gateway.Gateways = rgwa } if l := len(ln.Remotes); l > 0 { rlna := make([]RemoteLeafOptsVarz, l) for i, r := range ln.Remotes { var deny *DenyRules if len(r.DenyImports) > 0 || len(r.DenyExports) > 0 { deny = &DenyRules{ Imports: r.DenyImports, Exports: r.DenyExports, } } remoteTlsOCSPPeerVerify := s.ocspPeerVerify && r.tlsConfigOpts != nil && r.tlsConfigOpts.OCSPPeerConfig != nil && r.tlsConfigOpts.OCSPPeerConfig.Verify rlna[i] = RemoteLeafOptsVarz{ LocalAccount: r.LocalAccount, URLs: urlsToStrings(r.URLs), TLSTimeout: r.TLSTimeout, Deny: deny, TLSOCSPPeerVerify: remoteTlsOCSPPeerVerify, } } varz.LeafNode.Remotes = rlna } // Finish setting it up with fields that can be updated during // configuration reload and runtime. s.updateVarzConfigReloadableFields(varz) s.updateVarzRuntimeFields(varz, true, pcpu, rss) return varz } func urlsToStrings(urls []*url.URL) []string { sURLs := make([]string, len(urls)) for i, u := range urls { sURLs[i] = u.Host } return sURLs } // Invoked during configuration reload once options have possibly be changed // and config load time has been set. If s.varz has not been initialized yet // (because no pooling of /varz has been made), this function does nothing. // Server lock is held on entry. func (s *Server) updateVarzConfigReloadableFields(v *Varz) { if v == nil { return } opts := s.getOpts() info := &s.info v.AuthRequired = info.AuthRequired v.TLSRequired = info.TLSRequired v.TLSVerify = info.TLSVerify v.MaxConn = opts.MaxConn v.PingInterval = opts.PingInterval v.MaxPingsOut = opts.MaxPingsOut v.AuthTimeout = opts.AuthTimeout v.MaxControlLine = opts.MaxControlLine v.MaxPayload = int(opts.MaxPayload) v.MaxPending = opts.MaxPending v.TLSTimeout = opts.TLSTimeout v.WriteDeadline = opts.WriteDeadline v.ConfigLoadTime = s.configTime.UTC() // Update route URLs if applicable if s.varzUpdateRouteURLs { v.Cluster.URLs = urlsToStrings(opts.Routes) s.varzUpdateRouteURLs = false } if s.sys != nil && s.sys.account != nil { v.SystemAccount = s.sys.account.GetName() } v.MQTT.TLSPinnedCerts = getPinnedCertsAsSlice(opts.MQTT.TLSPinnedCerts) v.Websocket.TLSPinnedCerts = getPinnedCertsAsSlice(opts.Websocket.TLSPinnedCerts) v.TLSOCSPPeerVerify = s.ocspPeerVerify && v.TLSRequired && s.opts.tlsConfigOpts != nil && s.opts.tlsConfigOpts.OCSPPeerConfig != nil && s.opts.tlsConfigOpts.OCSPPeerConfig.Verify } func getPinnedCertsAsSlice(certs PinnedCertSet) []string { if len(certs) == 0 { return nil } res := make([]string, 0, len(certs)) for cn := range certs { res = append(res, cn) } return res } // Updates the runtime Varz fields, that is, fields that change during // runtime and that should be updated any time Varz() or polling of /varz // is done. // Server lock is held on entry. func (s *Server) updateVarzRuntimeFields(v *Varz, forceUpdate bool, pcpu float64, rss int64) { v.Now = time.Now().UTC() v.Uptime = myUptime(time.Since(s.start)) v.Mem = rss v.CPU = pcpu if l := len(s.info.ClientConnectURLs); l > 0 { v.ClientConnectURLs = append([]string(nil), s.info.ClientConnectURLs...) } if l := len(s.info.WSConnectURLs); l > 0 { v.WSConnectURLs = append([]string(nil), s.info.WSConnectURLs...) } v.Connections = len(s.clients) v.TotalConnections = s.totalClients v.Routes = s.numRoutes() v.Remotes = s.numRemotes() v.Leafs = len(s.leafs) v.InMsgs = atomic.LoadInt64(&s.inMsgs) v.InBytes = atomic.LoadInt64(&s.inBytes) v.OutMsgs = atomic.LoadInt64(&s.outMsgs) v.OutBytes = atomic.LoadInt64(&s.outBytes) v.SlowConsumers = atomic.LoadInt64(&s.slowConsumers) v.SlowConsumersStats = &SlowConsumersStats{ Clients: s.NumSlowConsumersClients(), Routes: s.NumSlowConsumersRoutes(), Gateways: s.NumSlowConsumersGateways(), Leafs: s.NumSlowConsumersLeafs(), } v.PinnedAccountFail = atomic.LoadUint64(&s.pinnedAccFail) // Make sure to reset in case we are re-using. v.Subscriptions = 0 s.accounts.Range(func(k, val any) bool { acc := val.(*Account) v.Subscriptions += acc.sl.Count() return true }) v.HTTPReqStats = make(map[string]uint64, len(s.httpReqStats)) for key, val := range s.httpReqStats { v.HTTPReqStats[key] = val } // Update Gateway remote urls if applicable gw := s.gateway gw.RLock() if gw.enabled { for i := 0; i < len(v.Gateway.Gateways); i++ { g := &v.Gateway.Gateways[i] rgw := gw.remotes[g.Name] if rgw != nil { rgw.RLock() // forceUpdate is needed if user calls Varz() programmatically, // since we need to create a new instance every time and the // gateway's varzUpdateURLs may have been set to false after // a web /varz inspection. if forceUpdate || rgw.varzUpdateURLs { // Make reuse of backend array g.URLs = g.URLs[:0] // rgw.urls is a map[string]*url.URL where the key is // already in the right format (host:port, without any // user info present). for u := range rgw.urls { g.URLs = append(g.URLs, u) } rgw.varzUpdateURLs = false } rgw.RUnlock() } else if g.Name == gw.name && len(gw.ownCfgURLs) > 0 { // This is a remote that correspond to this very same server. // We report the URLs that were configured (if any). // Since we don't support changes to the gateway configuration // at this time, we could do this only if g.URLs has not been already // set, but let's do it regardless in case we add support for // gateway config reload. g.URLs = g.URLs[:0] g.URLs = append(g.URLs, gw.ownCfgURLs...) } } } gw.RUnlock() if s.ocsprc != nil && s.ocsprc.Type() != "none" { stats := s.ocsprc.Stats() if stats != nil { v.OCSPResponseCache = &OCSPResponseCacheVarz{ s.ocsprc.Type(), stats.Hits, stats.Misses, stats.Responses, stats.Revokes, stats.Goods, stats.Unknowns, } } } } // HandleVarz will process HTTP requests for server information. func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) { var rss, vss int64 var pcpu float64 // We want to do that outside of the lock. pse.ProcUsage(&pcpu, &rss, &vss) // In response to http requests, we want to minimize mem copies // so we use an object stored in the server. Creating/collecting // server metrics is done under server lock, but we don't want // to marshal under that lock. Still, we need to prevent concurrent // http requests to /varz to update s.varz while marshal is // happening, so we need a new lock that serialize those http // requests and include marshaling. s.varzMu.Lock() // Use server lock to create/update the server's varz object. s.mu.Lock() var created bool s.httpReqStats[VarzPath]++ if s.varz == nil { s.varz = s.createVarz(pcpu, rss) created = true } else { s.updateVarzRuntimeFields(s.varz, false, pcpu, rss) } s.mu.Unlock() // Since locking is jetStream -> Server, need to update jetstream // varz outside of server lock. if js := s.getJetStream(); js != nil { var v JetStreamVarz // Work on stack variable s.updateJszVarz(js, &v, created) // Now update server's varz s.mu.RLock() sv := &s.varz.JetStream if created { sv.Config = v.Config } sv.Stats = v.Stats sv.Meta = v.Meta s.mu.RUnlock() } // Do the marshaling outside of server lock, but under varzMu lock. b, err := json.MarshalIndent(s.varz, "", " ") s.varzMu.Unlock() if err != nil { s.Errorf("Error marshaling response to /varz request: %v", err) } // Handle response ResponseHandler(w, r, b) } // GatewayzOptions are the options passed to Gatewayz() type GatewayzOptions struct { // Name will output only remote gateways with this name Name string `json:"name"` // Accounts indicates if accounts with its interest should be included in the results. Accounts bool `json:"accounts"` // AccountName will limit the list of accounts to that account name (makes Accounts implicit) AccountName string `json:"account_name"` // AccountSubscriptions indicates if subscriptions should be included in the results. // Note: This is used only if `Accounts` or `AccountName` are specified. AccountSubscriptions bool `json:"subscriptions"` // AccountSubscriptionsDetail indicates if subscription details should be included in the results. // Note: This is used only if `Accounts` or `AccountName` are specified. AccountSubscriptionsDetail bool `json:"subscriptions_detail"` } // Gatewayz represents detailed information on Gateways type Gatewayz struct { ID string `json:"server_id"` Now time.Time `json:"now"` Name string `json:"name,omitempty"` Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` OutboundGateways map[string]*RemoteGatewayz `json:"outbound_gateways"` InboundGateways map[string][]*RemoteGatewayz `json:"inbound_gateways"` } // RemoteGatewayz represents information about an outbound connection to a gateway type RemoteGatewayz struct { IsConfigured bool `json:"configured"` Connection *ConnInfo `json:"connection,omitempty"` Accounts []*AccountGatewayz `json:"accounts,omitempty"` } // AccountGatewayz represents interest mode for this account type AccountGatewayz struct { Name string `json:"name"` InterestMode string `json:"interest_mode"` NoInterestCount int `json:"no_interest_count,omitempty"` InterestOnlyThreshold int `json:"interest_only_threshold,omitempty"` TotalSubscriptions int `json:"num_subs,omitempty"` NumQueueSubscriptions int `json:"num_queue_subs,omitempty"` Subs []string `json:"subscriptions_list,omitempty"` SubsDetail []SubDetail `json:"subscriptions_list_detail,omitempty"` } // Gatewayz returns a Gatewayz struct containing information about gateways. func (s *Server) Gatewayz(opts *GatewayzOptions) (*Gatewayz, error) { srvID := s.ID() now := time.Now().UTC() gw := s.gateway gw.RLock() if !gw.enabled || gw.info == nil { gw.RUnlock() gwz := &Gatewayz{ ID: srvID, Now: now, OutboundGateways: map[string]*RemoteGatewayz{}, InboundGateways: map[string][]*RemoteGatewayz{}, } return gwz, nil } // Here gateways are enabled, so fill up more. gwz := &Gatewayz{ ID: srvID, Now: now, Name: gw.name, Host: gw.info.Host, Port: gw.info.Port, } gw.RUnlock() gwz.OutboundGateways = s.createOutboundsRemoteGatewayz(opts, now) gwz.InboundGateways = s.createInboundsRemoteGatewayz(opts, now) return gwz, nil } // Based on give options struct, returns if there is a filtered // Gateway Name and if we should do report Accounts. // Note that if Accounts is false but AccountName is not empty, // then Accounts is implicitly set to true. func getMonitorGWOptions(opts *GatewayzOptions) (string, bool) { var name string var accs bool if opts != nil { if opts.Name != _EMPTY_ { name = opts.Name } accs = opts.Accounts if !accs && opts.AccountName != _EMPTY_ { accs = true } } return name, accs } // Returns a map of gateways outbound connections. // Based on options, will include a single or all gateways, // with no/single/or all accounts interest information. func (s *Server) createOutboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string]*RemoteGatewayz { targetGWName, doAccs := getMonitorGWOptions(opts) if targetGWName != _EMPTY_ { c := s.getOutboundGatewayConnection(targetGWName) if c == nil { return nil } outbounds := make(map[string]*RemoteGatewayz, 1) _, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs) outbounds[targetGWName] = rgw return outbounds } var connsa [16]*client var conns = connsa[:0] s.getOutboundGatewayConnections(&conns) outbounds := make(map[string]*RemoteGatewayz, len(conns)) for _, c := range conns { name, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs) if rgw != nil { outbounds[name] = rgw } } return outbounds } // Returns a RemoteGatewayz for a given outbound gw connection func createOutboundRemoteGatewayz(c *client, opts *GatewayzOptions, now time.Time, doAccs bool) (string, *RemoteGatewayz) { var name string var rgw *RemoteGatewayz c.mu.Lock() if c.gw != nil { rgw = &RemoteGatewayz{} if doAccs { rgw.Accounts = createOutboundAccountsGatewayz(opts, c.gw) } if c.gw.cfg != nil { rgw.IsConfigured = !c.gw.cfg.isImplicit() } rgw.Connection = &ConnInfo{} rgw.Connection.fill(c, c.nc, now, false) name = c.gw.name } c.mu.Unlock() return name, rgw } // Returns the list of accounts for this outbound gateway connection. // Based on the options, it will be a single or all accounts for // this outbound. func createOutboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz { if gw.outsim == nil { return nil } var accName string if opts != nil { accName = opts.AccountName } if accName != _EMPTY_ { ei, ok := gw.outsim.Load(accName) if !ok { return nil } a := createAccountOutboundGatewayz(opts, accName, ei) return []*AccountGatewayz{a} } accs := make([]*AccountGatewayz, 0, 4) gw.outsim.Range(func(k, v any) bool { name := k.(string) a := createAccountOutboundGatewayz(opts, name, v) accs = append(accs, a) return true }) return accs } // Returns an AccountGatewayz for this gateway outbound connection func createAccountOutboundGatewayz(opts *GatewayzOptions, name string, ei any) *AccountGatewayz { a := &AccountGatewayz{ Name: name, InterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch, } if ei != nil { e := ei.(*outsie) e.RLock() a.InterestMode = e.mode.String() a.NoInterestCount = len(e.ni) a.NumQueueSubscriptions = e.qsubs a.TotalSubscriptions = int(e.sl.Count()) if opts.AccountSubscriptions || opts.AccountSubscriptionsDetail { var subsa [4096]*subscription subs := subsa[:0] e.sl.All(&subs) if opts.AccountSubscriptions { a.Subs = make([]string, 0, len(subs)) } else { a.SubsDetail = make([]SubDetail, 0, len(subs)) } for _, sub := range subs { if opts.AccountSubscriptions { a.Subs = append(a.Subs, string(sub.subject)) } else { a.SubsDetail = append(a.SubsDetail, newClientSubDetail(sub)) } } } e.RUnlock() } else { a.InterestMode = Optimistic.String() } return a } // Returns a map of gateways inbound connections. // Each entry is an array of RemoteGatewayz since a given server // may have more than one inbound from the same remote gateway. // Based on options, will include a single or all gateways, // with no/single/or all accounts interest information. func (s *Server) createInboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string][]*RemoteGatewayz { targetGWName, doAccs := getMonitorGWOptions(opts) var connsa [16]*client var conns = connsa[:0] s.getInboundGatewayConnections(&conns) m := make(map[string][]*RemoteGatewayz) for _, c := range conns { c.mu.Lock() if c.gw != nil && (targetGWName == _EMPTY_ || targetGWName == c.gw.name) { igws := m[c.gw.name] if igws == nil { igws = make([]*RemoteGatewayz, 0, 2) } rgw := &RemoteGatewayz{} if doAccs { rgw.Accounts = createInboundAccountsGatewayz(opts, c.gw) } rgw.Connection = &ConnInfo{} rgw.Connection.fill(c, c.nc, now, false) igws = append(igws, rgw) m[c.gw.name] = igws } c.mu.Unlock() } return m } // Returns the list of accounts for this inbound gateway connection. // Based on the options, it will be a single or all accounts for // this inbound. func createInboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz { if gw.insim == nil { return nil } var accName string if opts != nil { accName = opts.AccountName } if accName != _EMPTY_ { e, ok := gw.insim[accName] if !ok { return nil } a := createInboundAccountGatewayz(accName, e) return []*AccountGatewayz{a} } accs := make([]*AccountGatewayz, 0, 4) for name, e := range gw.insim { a := createInboundAccountGatewayz(name, e) accs = append(accs, a) } return accs } // Returns an AccountGatewayz for this gateway inbound connection func createInboundAccountGatewayz(name string, e *insie) *AccountGatewayz { a := &AccountGatewayz{ Name: name, InterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch, } if e != nil { a.InterestMode = e.mode.String() a.NoInterestCount = len(e.ni) } else { a.InterestMode = Optimistic.String() } return a } // HandleGatewayz process HTTP requests for route information. func (s *Server) HandleGatewayz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[GatewayzPath]++ s.mu.Unlock() subs, subsDet, err := decodeSubs(w, r) if err != nil { return } accs, err := decodeBool(w, r, "accs") if err != nil { return } gwName := r.URL.Query().Get("gw_name") accName := r.URL.Query().Get("acc_name") if accName != _EMPTY_ { accs = true } opts := &GatewayzOptions{ Name: gwName, Accounts: accs, AccountName: accName, AccountSubscriptions: subs, AccountSubscriptionsDetail: subsDet, } gw, err := s.Gatewayz(opts) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } b, err := json.MarshalIndent(gw, "", " ") if err != nil { s.Errorf("Error marshaling response to /gatewayz request: %v", err) } // Handle response ResponseHandler(w, r, b) } // Leafz represents detailed information on Leafnodes. type Leafz struct { ID string `json:"server_id"` Now time.Time `json:"now"` NumLeafs int `json:"leafnodes"` Leafs []*LeafInfo `json:"leafs"` } // LeafzOptions are options passed to Leafz type LeafzOptions struct { // Subscriptions indicates that Leafz will return a leafnode's subscriptions Subscriptions bool `json:"subscriptions"` Account string `json:"account"` } // LeafInfo has detailed information on each remote leafnode connection. type LeafInfo struct { Name string `json:"name"` IsSpoke bool `json:"is_spoke"` Account string `json:"account"` IP string `json:"ip"` Port int `json:"port"` RTT string `json:"rtt,omitempty"` InMsgs int64 `json:"in_msgs"` OutMsgs int64 `json:"out_msgs"` InBytes int64 `json:"in_bytes"` OutBytes int64 `json:"out_bytes"` NumSubs uint32 `json:"subscriptions"` Subs []string `json:"subscriptions_list,omitempty"` Compression string `json:"compression,omitempty"` } // Leafz returns a Leafz structure containing information about leafnodes. func (s *Server) Leafz(opts *LeafzOptions) (*Leafz, error) { // Grab leafnodes var lconns []*client s.mu.Lock() if len(s.leafs) > 0 { lconns = make([]*client, 0, len(s.leafs)) for _, ln := range s.leafs { if opts != nil && opts.Account != _EMPTY_ { ln.mu.Lock() ok := ln.acc.Name == opts.Account ln.mu.Unlock() if !ok { continue } } lconns = append(lconns, ln) } } s.mu.Unlock() leafnodes := make([]*LeafInfo, 0, len(lconns)) if len(lconns) > 0 { for _, ln := range lconns { ln.mu.Lock() lni := &LeafInfo{ Name: ln.leaf.remoteServer, IsSpoke: ln.isSpokeLeafNode(), Account: ln.acc.Name, IP: ln.host, Port: int(ln.port), RTT: ln.getRTT().String(), InMsgs: atomic.LoadInt64(&ln.inMsgs), OutMsgs: ln.outMsgs, InBytes: atomic.LoadInt64(&ln.inBytes), OutBytes: ln.outBytes, NumSubs: uint32(len(ln.subs)), Compression: ln.leaf.compression, } if opts != nil && opts.Subscriptions { lni.Subs = make([]string, 0, len(ln.subs)) for _, sub := range ln.subs { lni.Subs = append(lni.Subs, string(sub.subject)) } } ln.mu.Unlock() leafnodes = append(leafnodes, lni) } } return &Leafz{ ID: s.ID(), Now: time.Now().UTC(), NumLeafs: len(leafnodes), Leafs: leafnodes, }, nil } // HandleLeafz process HTTP requests for leafnode information. func (s *Server) HandleLeafz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[LeafzPath]++ s.mu.Unlock() subs, err := decodeBool(w, r, "subs") if err != nil { return } l, err := s.Leafz(&LeafzOptions{subs, r.URL.Query().Get("acc")}) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } b, err := json.MarshalIndent(l, "", " ") if err != nil { s.Errorf("Error marshaling response to /leafz request: %v", err) } // Handle response ResponseHandler(w, r, b) } // Leafz represents detailed information on Leafnodes. type AccountStatz struct { ID string `json:"server_id"` Now time.Time `json:"now"` Accounts []*AccountStat `json:"account_statz"` } // AccountStatzOptions are options passed to account stats requests. type AccountStatzOptions struct { Accounts []string `json:"accounts"` IncludeUnused bool `json:"include_unused"` } // Leafz returns a AccountStatz structure containing summary information about accounts. func (s *Server) AccountStatz(opts *AccountStatzOptions) (*AccountStatz, error) { stz := &AccountStatz{ ID: s.ID(), Now: time.Now().UTC(), Accounts: []*AccountStat{}, } if opts == nil || len(opts.Accounts) == 0 { s.accounts.Range(func(key, a any) bool { acc := a.(*Account) acc.mu.RLock() if opts.IncludeUnused || acc.numLocalConnections() != 0 { stz.Accounts = append(stz.Accounts, acc.statz()) } acc.mu.RUnlock() return true }) } else { for _, a := range opts.Accounts { if acc, ok := s.accounts.Load(a); ok { acc := acc.(*Account) acc.mu.RLock() if opts.IncludeUnused || acc.numLocalConnections() != 0 { stz.Accounts = append(stz.Accounts, acc.statz()) } acc.mu.RUnlock() } } } return stz, nil } // HandleAccountStatz process HTTP requests for statz information of all accounts. func (s *Server) HandleAccountStatz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[AccountStatzPath]++ s.mu.Unlock() unused, err := decodeBool(w, r, "unused") if err != nil { return } l, err := s.AccountStatz(&AccountStatzOptions{IncludeUnused: unused}) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } b, err := json.MarshalIndent(l, "", " ") if err != nil { s.Errorf("Error marshaling response to %s request: %v", AccountStatzPath, err) return } // Handle response ResponseHandler(w, r, b) } // ResponseHandler handles responses for monitoring routes. func ResponseHandler(w http.ResponseWriter, r *http.Request, data []byte) { handleResponse(http.StatusOK, w, r, data) } // handleResponse handles responses for monitoring routes with a specific HTTP status code. func handleResponse(code int, w http.ResponseWriter, r *http.Request, data []byte) { // Get callback from request callback := r.URL.Query().Get("callback") if callback != _EMPTY_ { // Response for JSONP w.Header().Set("Content-Type", "application/javascript") w.WriteHeader(code) fmt.Fprintf(w, "%s(%s)", callback, data) } else { // Otherwise JSON w.Header().Set("Content-Type", "application/json") w.Header().Set("Access-Control-Allow-Origin", "*") w.WriteHeader(code) w.Write(data) } } func (reason ClosedState) String() string { switch reason { case ClientClosed: return "Client Closed" case AuthenticationTimeout: return "Authentication Timeout" case AuthenticationViolation: return "Authentication Failure" case TLSHandshakeError: return "TLS Handshake Failure" case SlowConsumerPendingBytes: return "Slow Consumer (Pending Bytes)" case SlowConsumerWriteDeadline: return "Slow Consumer (Write Deadline)" case WriteError: return "Write Error" case ReadError: return "Read Error" case ParseError: return "Parse Error" case StaleConnection: return "Stale Connection" case ProtocolViolation: return "Protocol Violation" case BadClientProtocolVersion: return "Bad Client Protocol Version" case WrongPort: return "Incorrect Port" case MaxConnectionsExceeded: return "Maximum Connections Exceeded" case MaxAccountConnectionsExceeded: return "Maximum Account Connections Exceeded" case MaxPayloadExceeded: return "Maximum Message Payload Exceeded" case MaxControlLineExceeded: return "Maximum Control Line Exceeded" case MaxSubscriptionsExceeded: return "Maximum Subscriptions Exceeded" case DuplicateRoute: return "Duplicate Route" case RouteRemoved: return "Route Removed" case ServerShutdown: return "Server Shutdown" case AuthenticationExpired: return "Authentication Expired" case WrongGateway: return "Wrong Gateway" case MissingAccount: return "Missing Account" case Revocation: return "Credentials Revoked" case InternalClient: return "Internal Client" case MsgHeaderViolation: return "Message Header Violation" case NoRespondersRequiresHeaders: return "No Responders Requires Headers" case ClusterNameConflict: return "Cluster Name Conflict" case DuplicateRemoteLeafnodeConnection: return "Duplicate Remote LeafNode Connection" case DuplicateClientID: return "Duplicate Client ID" case DuplicateServerName: return "Duplicate Server Name" case MinimumVersionRequired: return "Minimum Version Required" case ClusterNamesIdentical: return "Cluster Names Identical" case Kicked: return "Kicked" } return "Unknown State" } // AccountzOptions are options passed to Accountz type AccountzOptions struct { // Account indicates that Accountz will return details for the account Account string `json:"account"` } func newExtServiceLatency(l *serviceLatency) *jwt.ServiceLatency { if l == nil { return nil } return &jwt.ServiceLatency{ Sampling: jwt.SamplingRate(l.sampling), Results: jwt.Subject(l.subject), } } type ExtImport struct { jwt.Import Invalid bool `json:"invalid"` Share bool `json:"share"` Tracking bool `json:"tracking"` TrackingHdr http.Header `json:"tracking_header,omitempty"` Latency *jwt.ServiceLatency `json:"latency,omitempty"` M1 *ServiceLatency `json:"m1,omitempty"` } type ExtExport struct { jwt.Export ApprovedAccounts []string `json:"approved_accounts,omitempty"` RevokedAct map[string]time.Time `json:"revoked_activations,omitempty"` } type ExtVrIssues struct { Description string `json:"description"` Blocking bool `json:"blocking"` Time bool `json:"time_check"` } type ExtMap map[string][]*MapDest type AccountInfo struct { AccountName string `json:"account_name"` LastUpdate time.Time `json:"update_time,omitempty"` IsSystem bool `json:"is_system,omitempty"` Expired bool `json:"expired"` Complete bool `json:"complete"` JetStream bool `json:"jetstream_enabled"` LeafCnt int `json:"leafnode_connections"` ClientCnt int `json:"client_connections"` SubCnt uint32 `json:"subscriptions"` Mappings ExtMap `json:"mappings,omitempty"` Exports []ExtExport `json:"exports,omitempty"` Imports []ExtImport `json:"imports,omitempty"` Jwt string `json:"jwt,omitempty"` IssuerKey string `json:"issuer_key,omitempty"` NameTag string `json:"name_tag,omitempty"` Tags jwt.TagList `json:"tags,omitempty"` Claim *jwt.AccountClaims `json:"decoded_jwt,omitempty"` Vr []ExtVrIssues `json:"validation_result_jwt,omitempty"` RevokedUser map[string]time.Time `json:"revoked_user,omitempty"` Sublist *SublistStats `json:"sublist_stats,omitempty"` Responses map[string]ExtImport `json:"responses,omitempty"` } type Accountz struct { ID string `json:"server_id"` Now time.Time `json:"now"` SystemAccount string `json:"system_account,omitempty"` Accounts []string `json:"accounts,omitempty"` Account *AccountInfo `json:"account_detail,omitempty"` } // HandleAccountz process HTTP requests for account information. func (s *Server) HandleAccountz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[AccountzPath]++ s.mu.Unlock() if l, err := s.Accountz(&AccountzOptions{r.URL.Query().Get("acc")}); err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) } else if b, err := json.MarshalIndent(l, "", " "); err != nil { s.Errorf("Error marshaling response to %s request: %v", AccountzPath, err) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) } else { ResponseHandler(w, r, b) // Handle response } } func (s *Server) Accountz(optz *AccountzOptions) (*Accountz, error) { a := &Accountz{ ID: s.ID(), Now: time.Now().UTC(), } if sacc := s.SystemAccount(); sacc != nil { a.SystemAccount = sacc.GetName() } if optz == nil || optz.Account == _EMPTY_ { a.Accounts = []string{} s.accounts.Range(func(key, value any) bool { a.Accounts = append(a.Accounts, key.(string)) return true }) return a, nil } aInfo, err := s.accountInfo(optz.Account) if err != nil { return nil, err } a.Account = aInfo return a, nil } func newExtImport(v *serviceImport) ExtImport { imp := ExtImport{ Invalid: true, Import: jwt.Import{Type: jwt.Service}, } if v != nil { imp.Share = v.share imp.Tracking = v.tracking imp.Invalid = v.invalid imp.Import = jwt.Import{ Subject: jwt.Subject(v.to), Account: v.acc.Name, Type: jwt.Service, // Deprecated so we duplicate. Use LocalSubject. To: jwt.Subject(v.from), LocalSubject: jwt.RenamingSubject(v.from), } imp.TrackingHdr = v.trackingHdr imp.Latency = newExtServiceLatency(v.latency) if v.m1 != nil { m1 := *v.m1 imp.M1 = &m1 } } return imp } func (s *Server) accountInfo(accName string) (*AccountInfo, error) { var a *Account if v, ok := s.accounts.Load(accName); !ok { return nil, fmt.Errorf("Account %s does not exist", accName) } else { a = v.(*Account) } isSys := a == s.SystemAccount() a.mu.RLock() defer a.mu.RUnlock() var vrIssues []ExtVrIssues claim, _ := jwt.DecodeAccountClaims(a.claimJWT) // ignore error if claim != nil { vr := jwt.ValidationResults{} claim.Validate(&vr) vrIssues = make([]ExtVrIssues, len(vr.Issues)) for i, v := range vr.Issues { vrIssues[i] = ExtVrIssues{v.Description, v.Blocking, v.TimeCheck} } } collectRevocations := func(revocations map[string]int64) map[string]time.Time { l := len(revocations) if l == 0 { return nil } rev := make(map[string]time.Time, l) for k, v := range revocations { rev[k] = time.Unix(v, 0) } return rev } exports := []ExtExport{} for k, v := range a.exports.services { e := ExtExport{ Export: jwt.Export{ Subject: jwt.Subject(k), Type: jwt.Service, }, ApprovedAccounts: []string{}, } if v != nil { e.Latency = newExtServiceLatency(v.latency) e.TokenReq = v.tokenReq e.ResponseType = jwt.ResponseType(v.respType.String()) for name := range v.approved { e.ApprovedAccounts = append(e.ApprovedAccounts, name) } e.RevokedAct = collectRevocations(v.actsRevoked) } exports = append(exports, e) } for k, v := range a.exports.streams { e := ExtExport{ Export: jwt.Export{ Subject: jwt.Subject(k), Type: jwt.Stream, }, ApprovedAccounts: []string{}, } if v != nil { e.TokenReq = v.tokenReq for name := range v.approved { e.ApprovedAccounts = append(e.ApprovedAccounts, name) } e.RevokedAct = collectRevocations(v.actsRevoked) } exports = append(exports, e) } imports := []ExtImport{} for _, v := range a.imports.streams { imp := ExtImport{ Invalid: true, Import: jwt.Import{Type: jwt.Stream}, } if v != nil { imp.Invalid = v.invalid imp.Import = jwt.Import{ Subject: jwt.Subject(v.from), Account: v.acc.Name, Type: jwt.Stream, LocalSubject: jwt.RenamingSubject(v.to), } } imports = append(imports, imp) } for _, v := range a.imports.services { imports = append(imports, newExtImport(v)) } responses := map[string]ExtImport{} for k, v := range a.exports.responses { responses[k] = newExtImport(v) } mappings := ExtMap{} for _, m := range a.mappings { var dests []*MapDest var src string if m == nil { src = "nil" if _, ok := mappings[src]; ok { // only set if not present (keep orig in case nil is used) continue } dests = append(dests, &MapDest{}) } else { src = m.src for _, d := range m.dests { dests = append(dests, &MapDest{d.tr.dest, d.weight, _EMPTY_}) } for c, cd := range m.cdests { for _, d := range cd { dests = append(dests, &MapDest{d.tr.dest, d.weight, c}) } } } mappings[src] = dests } return &AccountInfo{ accName, a.updated.UTC(), isSys, a.expired.Load(), !a.incomplete, a.js != nil, a.numLocalLeafNodes(), a.numLocalConnections(), a.sl.Count(), mappings, exports, imports, a.claimJWT, a.Issuer, a.nameTag, a.tags, claim, vrIssues, collectRevocations(a.usersRevoked), a.sl.Stats(), responses, }, nil } // JSzOptions are options passed to Jsz type JSzOptions struct { Account string `json:"account,omitempty"` Accounts bool `json:"accounts,omitempty"` Streams bool `json:"streams,omitempty"` Consumer bool `json:"consumer,omitempty"` Config bool `json:"config,omitempty"` LeaderOnly bool `json:"leader_only,omitempty"` Offset int `json:"offset,omitempty"` Limit int `json:"limit,omitempty"` RaftGroups bool `json:"raft,omitempty"` StreamLeaderOnly bool `json:"stream_leader_only,omitempty"` } // HealthzOptions are options passed to Healthz type HealthzOptions struct { // Deprecated: Use JSEnabledOnly instead JSEnabled bool `json:"js-enabled,omitempty"` JSEnabledOnly bool `json:"js-enabled-only,omitempty"` JSServerOnly bool `json:"js-server-only,omitempty"` Account string `json:"account,omitempty"` Stream string `json:"stream,omitempty"` Consumer string `json:"consumer,omitempty"` Details bool `json:"details,omitempty"` } // ProfilezOptions are options passed to Profilez type ProfilezOptions struct { Name string `json:"name"` Debug int `json:"debug"` Duration time.Duration `json:"duration,omitempty"` } // IpqueueszOptions are options passed to Ipqueuesz type IpqueueszOptions struct { All bool `json:"all"` Filter string `json:"filter"` } // RaftzOptions are options passed to Raftz type RaftzOptions struct { AccountFilter string `json:"account"` GroupFilter string `json:"group"` } // StreamDetail shows information about the stream state and its consumers. type StreamDetail struct { Name string `json:"name"` Created time.Time `json:"created"` Cluster *ClusterInfo `json:"cluster,omitempty"` Config *StreamConfig `json:"config,omitempty"` State StreamState `json:"state,omitempty"` Consumer []*ConsumerInfo `json:"consumer_detail,omitempty"` Mirror *StreamSourceInfo `json:"mirror,omitempty"` Sources []*StreamSourceInfo `json:"sources,omitempty"` RaftGroup string `json:"stream_raft_group,omitempty"` ConsumerRaftGroups []*RaftGroupDetail `json:"consumer_raft_groups,omitempty"` } // RaftGroupDetail shows information details about the Raft group. type RaftGroupDetail struct { Name string `json:"name"` RaftGroup string `json:"raft_group,omitempty"` } type AccountDetail struct { Name string `json:"name"` Id string `json:"id"` JetStreamStats Streams []StreamDetail `json:"stream_detail,omitempty"` } // MetaClusterInfo shows information about the meta group. type MetaClusterInfo struct { Name string `json:"name,omitempty"` Leader string `json:"leader,omitempty"` Peer string `json:"peer,omitempty"` Replicas []*PeerInfo `json:"replicas,omitempty"` Size int `json:"cluster_size"` Pending int `json:"pending"` } // JSInfo has detailed information on JetStream. type JSInfo struct { ID string `json:"server_id"` Now time.Time `json:"now"` Disabled bool `json:"disabled,omitempty"` Config JetStreamConfig `json:"config,omitempty"` JetStreamStats Streams int `json:"streams"` Consumers int `json:"consumers"` Messages uint64 `json:"messages"` Bytes uint64 `json:"bytes"` Meta *MetaClusterInfo `json:"meta_cluster,omitempty"` // aggregate raft info AccountDetails []*AccountDetail `json:"account_details,omitempty"` } func (s *Server) accountDetail(jsa *jsAccount, optStreams, optConsumers, optCfg, optRaft, optStreamLeader bool) *AccountDetail { jsa.mu.RLock() acc := jsa.account name := acc.GetName() id := name if acc.nameTag != _EMPTY_ { name = acc.nameTag } jsa.usageMu.RLock() totalMem, totalStore := jsa.storageTotals() detail := AccountDetail{ Name: name, Id: id, JetStreamStats: JetStreamStats{ Memory: totalMem, Store: totalStore, API: JetStreamAPIStats{ Total: jsa.apiTotal, Errors: jsa.apiErrors, }, }, Streams: make([]StreamDetail, 0, len(jsa.streams)), } if reserved, ok := jsa.limits[_EMPTY_]; ok { detail.JetStreamStats.ReservedMemory = uint64(reserved.MaxMemory) detail.JetStreamStats.ReservedStore = uint64(reserved.MaxStore) } jsa.usageMu.RUnlock() var streams []*stream if optStreams { for _, stream := range jsa.streams { streams = append(streams, stream) } } jsa.mu.RUnlock() if js := s.getJetStream(); js != nil && optStreams { for _, stream := range streams { rgroup := stream.raftGroup() ci := js.clusterInfo(rgroup) var cfg *StreamConfig if optCfg { c := stream.config() cfg = &c } // Skip if we are only looking for stream leaders. if optStreamLeader && ci != nil && ci.Leader != s.Name() { continue } sdet := StreamDetail{ Name: stream.name(), Created: stream.createdTime(), State: stream.state(), Cluster: ci, Config: cfg, Mirror: stream.mirrorInfo(), Sources: stream.sourcesInfo(), } if optRaft && rgroup != nil { sdet.RaftGroup = rgroup.Name sdet.ConsumerRaftGroups = make([]*RaftGroupDetail, 0) } if optConsumers { for _, consumer := range stream.getPublicConsumers() { cInfo := consumer.info() if cInfo == nil { continue } if !optCfg { cInfo.Config = nil } sdet.Consumer = append(sdet.Consumer, cInfo) if optRaft { crgroup := consumer.raftGroup() if crgroup != nil { sdet.ConsumerRaftGroups = append(sdet.ConsumerRaftGroups, &RaftGroupDetail{cInfo.Name, crgroup.Name}, ) } } } } detail.Streams = append(detail.Streams, sdet) } } return &detail } func (s *Server) JszAccount(opts *JSzOptions) (*AccountDetail, error) { js := s.getJetStream() if js == nil { return nil, fmt.Errorf("jetstream not enabled") } acc := opts.Account account, ok := s.accounts.Load(acc) if !ok { return nil, fmt.Errorf("account %q not found", acc) } js.mu.RLock() jsa, ok := js.accounts[account.(*Account).Name] js.mu.RUnlock() if !ok { return nil, fmt.Errorf("account %q not jetstream enabled", acc) } return s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.Config, opts.RaftGroups, opts.StreamLeaderOnly), nil } // helper to get cluster info from node via dummy group func (s *Server) raftNodeToClusterInfo(node RaftNode) *ClusterInfo { if node == nil { return nil } peers := node.Peers() peerList := make([]string, len(peers)) for i, p := range peers { peerList[i] = p.ID } group := &raftGroup{ Name: _EMPTY_, Peers: peerList, node: node, } return s.getJetStream().clusterInfo(group) } // Jsz returns a Jsz structure containing information about JetStream. func (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) { // set option defaults if opts == nil { opts = &JSzOptions{} } if opts.Limit == 0 { opts.Limit = 1024 } if opts.Consumer { opts.Streams = true } if opts.Streams && opts.Account == _EMPTY_ { opts.Accounts = true } jsi := &JSInfo{ ID: s.ID(), Now: time.Now().UTC(), } js := s.getJetStream() if js == nil || !js.isEnabled() { if opts.LeaderOnly { return nil, fmt.Errorf("%w: not leader", errSkipZreq) } jsi.Disabled = true return jsi, nil } js.mu.RLock() isLeader := js.cluster == nil || js.cluster.isLeader() js.mu.RUnlock() if opts.LeaderOnly && !isLeader { return nil, fmt.Errorf("%w: not leader", errSkipZreq) } var accounts []*jsAccount js.mu.RLock() jsi.Config = js.config for _, info := range js.accounts { accounts = append(accounts, info) } js.mu.RUnlock() if mg := js.getMetaGroup(); mg != nil { if ci := s.raftNodeToClusterInfo(mg); ci != nil { jsi.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Size: mg.ClusterSize()} if isLeader { jsi.Meta.Replicas = ci.Replicas } if ipq := s.jsAPIRoutedReqs; ipq != nil { jsi.Meta.Pending = ipq.len() } } } jsi.JetStreamStats = *js.usageStats() filterIdx := -1 for i, jsa := range accounts { if jsa.acc().GetName() == opts.Account { filterIdx = i } jsa.mu.RLock() streams := make([]*stream, 0, len(jsa.streams)) for _, stream := range jsa.streams { streams = append(streams, stream) } jsa.mu.RUnlock() jsi.Streams += len(streams) for _, stream := range streams { streamState := stream.state() jsi.Messages += streamState.Msgs jsi.Bytes += streamState.Bytes jsi.Consumers += streamState.Consumers } } // filter logic if filterIdx != -1 { accounts = []*jsAccount{accounts[filterIdx]} } else if opts.Accounts { if opts.Offset != 0 { slices.SortFunc(accounts, func(i, j *jsAccount) int { return cmp.Compare(i.acc().Name, j.acc().Name) }) if opts.Offset > len(accounts) { accounts = []*jsAccount{} } else { accounts = accounts[opts.Offset:] } } if opts.Limit != 0 { if opts.Limit < len(accounts) { accounts = accounts[:opts.Limit] } } } else { accounts = []*jsAccount{} } if len(accounts) > 0 { jsi.AccountDetails = make([]*AccountDetail, 0, len(accounts)) } // if wanted, obtain accounts/streams/consumer for _, jsa := range accounts { detail := s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.Config, opts.RaftGroups, opts.StreamLeaderOnly) jsi.AccountDetails = append(jsi.AccountDetails, detail) } return jsi, nil } // HandleJsz process HTTP requests for jetstream information. func (s *Server) HandleJsz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[JszPath]++ s.mu.Unlock() accounts, err := decodeBool(w, r, "accounts") if err != nil { return } streams, err := decodeBool(w, r, "streams") if err != nil { return } consumers, err := decodeBool(w, r, "consumers") if err != nil { return } config, err := decodeBool(w, r, "config") if err != nil { return } offset, err := decodeInt(w, r, "offset") if err != nil { return } limit, err := decodeInt(w, r, "limit") if err != nil { return } leader, err := decodeBool(w, r, "leader-only") if err != nil { return } rgroups, err := decodeBool(w, r, "raft") if err != nil { return } sleader, err := decodeBool(w, r, "stream-leader-only") if err != nil { return } l, err := s.Jsz(&JSzOptions{ Account: r.URL.Query().Get("acc"), Accounts: accounts, Streams: streams, Consumer: consumers, Config: config, LeaderOnly: leader, Offset: offset, Limit: limit, RaftGroups: rgroups, StreamLeaderOnly: sleader, }) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } b, err := json.MarshalIndent(l, "", " ") if err != nil { s.Errorf("Error marshaling response to /jsz request: %v", err) } // Handle response ResponseHandler(w, r, b) } type HealthStatus struct { Status string `json:"status"` StatusCode int `json:"status_code,omitempty"` Error string `json:"error,omitempty"` Errors []HealthzError `json:"errors,omitempty"` } type HealthzError struct { Type HealthZErrorType `json:"type"` Account string `json:"account,omitempty"` Stream string `json:"stream,omitempty"` Consumer string `json:"consumer,omitempty"` Error string `json:"error,omitempty"` } type HealthZErrorType int const ( HealthzErrorConn HealthZErrorType = iota HealthzErrorBadRequest HealthzErrorJetStream HealthzErrorAccount HealthzErrorStream HealthzErrorConsumer ) func (t HealthZErrorType) String() string { switch t { case HealthzErrorConn: return "CONNECTION" case HealthzErrorBadRequest: return "BAD_REQUEST" case HealthzErrorJetStream: return "JETSTREAM" case HealthzErrorAccount: return "ACCOUNT" case HealthzErrorStream: return "STREAM" case HealthzErrorConsumer: return "CONSUMER" default: return "unknown" } } func (t HealthZErrorType) MarshalJSON() ([]byte, error) { return json.Marshal(t.String()) } func (t *HealthZErrorType) UnmarshalJSON(data []byte) error { switch string(data) { case `"CONNECTION"`: *t = HealthzErrorConn case `"BAD_REQUEST"`: *t = HealthzErrorBadRequest case `"JETSTREAM"`: *t = HealthzErrorJetStream case `"ACCOUNT"`: *t = HealthzErrorAccount case `"STREAM"`: *t = HealthzErrorStream case `"CONSUMER"`: *t = HealthzErrorConsumer default: return fmt.Errorf("unknown healthz error type %q", data) } return nil } // https://datatracker.ietf.org/doc/html/draft-inadarei-api-health-check func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.httpReqStats[HealthzPath]++ s.mu.Unlock() jsEnabled, err := decodeBool(w, r, "js-enabled") if err != nil { return } if jsEnabled { s.Warnf("Healthcheck: js-enabled deprecated, use js-enabled-only instead") } jsEnabledOnly, err := decodeBool(w, r, "js-enabled-only") if err != nil { return } jsServerOnly, err := decodeBool(w, r, "js-server-only") if err != nil { return } includeDetails, err := decodeBool(w, r, "details") if err != nil { return } hs := s.healthz(&HealthzOptions{ JSEnabled: jsEnabled, JSEnabledOnly: jsEnabledOnly, JSServerOnly: jsServerOnly, Account: r.URL.Query().Get("account"), Stream: r.URL.Query().Get("stream"), Consumer: r.URL.Query().Get("consumer"), Details: includeDetails, }) code := hs.StatusCode if hs.Error != _EMPTY_ { s.Warnf("Healthcheck failed: %q", hs.Error) } else if len(hs.Errors) != 0 { s.Warnf("Healthcheck failed: %d errors", len(hs.Errors)) } // Remove StatusCode from JSON representation when responding via HTTP // since this is already in the response. hs.StatusCode = 0 b, err := json.Marshal(hs) if err != nil { s.Errorf("Error marshaling response to /healthz request: %v", err) } handleResponse(code, w, r, b) } // Generate health status. func (s *Server) healthz(opts *HealthzOptions) *HealthStatus { var health = &HealthStatus{Status: "ok"} // set option defaults if opts == nil { opts = &HealthzOptions{} } details := opts.Details defer func() { // for response with details enabled, set status to either "error" or "ok" if details { if len(health.Errors) != 0 { health.Status = "error" } else { health.Status = "ok" } } // if no specific status code was set, set it based on the presence of errors if health.StatusCode == 0 { if health.Error != _EMPTY_ || len(health.Errors) != 0 { health.StatusCode = http.StatusServiceUnavailable } else { health.StatusCode = http.StatusOK } } }() if opts.Account == _EMPTY_ && opts.Stream != _EMPTY_ { health.StatusCode = http.StatusBadRequest if !details { health.Status = "error" health.Error = fmt.Sprintf("%q must not be empty when checking stream health", "account") } else { health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorBadRequest, Error: fmt.Sprintf("%q must not be empty when checking stream health", "account"), }) } return health } if opts.Stream == _EMPTY_ && opts.Consumer != _EMPTY_ { health.StatusCode = http.StatusBadRequest if !details { health.Status = "error" health.Error = fmt.Sprintf("%q must not be empty when checking consumer health", "stream") } else { health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorBadRequest, Error: fmt.Sprintf("%q must not be empty when checking consumer health", "stream"), }) } return health } if err := s.readyForConnections(time.Millisecond); err != nil { health.StatusCode = http.StatusInternalServerError health.Status = "error" if !details { health.Error = err.Error() } else { health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorConn, Error: err.Error(), }) } return health } sopts := s.getOpts() // If JS is not enabled in the config, we stop. if !sopts.JetStream { return health } // Access the Jetstream state to perform additional checks. js := s.getJetStream() const na = "unavailable" if !js.isEnabled() { health.StatusCode = http.StatusServiceUnavailable health.Status = na if !details { health.Error = NewJSNotEnabledError().Error() } else { health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorJetStream, Error: NewJSNotEnabledError().Error(), }) } return health } // Only check if JS is enabled, skip meta and asset check. if opts.JSEnabledOnly || opts.JSEnabled { return health } // Clustered JetStream js.mu.RLock() cc := js.cluster js.mu.RUnlock() // Currently single server we make sure the streams were recovered. if cc == nil { sdir := js.config.StoreDir // Whip through account folders and pull each stream name. fis, _ := os.ReadDir(sdir) var accFound, streamFound, consumerFound bool for _, fi := range fis { if fi.Name() == snapStagingDir { continue } if opts.Account != _EMPTY_ { if fi.Name() != opts.Account { continue } accFound = true } acc, err := s.LookupAccount(fi.Name()) if err != nil { if !details { health.Status = na health.Error = fmt.Sprintf("JetStream account '%s' could not be resolved", fi.Name()) return health } health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorAccount, Account: fi.Name(), Error: fmt.Sprintf("JetStream account '%s' could not be resolved", fi.Name()), }) continue } sfis, _ := os.ReadDir(filepath.Join(sdir, fi.Name(), "streams")) for _, sfi := range sfis { if opts.Stream != _EMPTY_ { if sfi.Name() != opts.Stream { continue } streamFound = true } stream := sfi.Name() s, err := acc.lookupStream(stream) if err != nil { if !details { health.Status = na health.Error = fmt.Sprintf("JetStream stream '%s > %s' could not be recovered", acc, stream) return health } health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorStream, Account: acc.Name, Stream: stream, Error: fmt.Sprintf("JetStream stream '%s > %s' could not be recovered", acc, stream), }) continue } if streamFound { // if consumer option is passed, verify that the consumer exists on stream if opts.Consumer != _EMPTY_ { for _, cons := range s.consumers { if cons.name == opts.Consumer { consumerFound = true break } } } break } } if accFound { break } } if opts.Account != _EMPTY_ && !accFound { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream account %q not found", opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorAccount, Account: opts.Account, Error: fmt.Sprintf("JetStream account %q not found", opts.Account), }, } } return health } if opts.Stream != _EMPTY_ && !streamFound { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream stream %q not found on account %q", opts.Stream, opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorStream, Account: opts.Account, Stream: opts.Stream, Error: fmt.Sprintf("JetStream stream %q not found on account %q", opts.Stream, opts.Account), }, } } return health } if opts.Consumer != _EMPTY_ && !consumerFound { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream consumer %q not found for stream %q on account %q", opts.Consumer, opts.Stream, opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorConsumer, Account: opts.Account, Stream: opts.Stream, Consumer: opts.Consumer, Error: fmt.Sprintf("JetStream consumer %q not found for stream %q on account %q", opts.Consumer, opts.Stream, opts.Account), }, } } } return health } // If we are here we want to check for any assets assigned to us. var meta RaftNode js.mu.RLock() meta = cc.meta js.mu.RUnlock() // If no meta leader. if meta == nil || meta.GroupLeader() == _EMPTY_ { if !details { health.Status = na health.Error = "JetStream has not established contact with a meta leader" } else { health.Errors = []HealthzError{ { Type: HealthzErrorJetStream, Error: "JetStream has not established contact with a meta leader", }, } } return health } // If we are not current with the meta leader. if !meta.Healthy() { if !details { health.Status = na health.Error = "JetStream is not current with the meta leader" } else { health.Errors = []HealthzError{ { Type: HealthzErrorJetStream, Error: "JetStream is not current with the meta leader", }, } } return health } // If JSServerOnly is true, then do not check further accounts, streams and consumers. if opts.JSServerOnly { return health } // Are we still recovering meta layer? if js.isMetaRecovering() { if !details { health.Status = na health.Error = "JetStream is still recovering meta layer" } else { health.Errors = []HealthzError{ { Type: HealthzErrorJetStream, Error: "JetStream is still recovering meta layer", }, } } return health } // Range across all accounts, the streams assigned to them, and the consumers. // If they are assigned to this server check their status. ourID := meta.ID() // Copy the meta layer so we do not need to hold the js read lock for an extended period of time. var streams map[string]map[string]*streamAssignment js.mu.RLock() if opts.Account == _EMPTY_ { // Collect all relevant streams and consumers. streams = make(map[string]map[string]*streamAssignment, len(cc.streams)) for acc, asa := range cc.streams { nasa := make(map[string]*streamAssignment) for stream, sa := range asa { // If we are a member and we are not being restored, select for check. if sa.Group.isMember(ourID) && sa.Restore == nil { csa := sa.copyGroup() csa.consumers = make(map[string]*consumerAssignment) for consumer, ca := range sa.consumers { if ca.Group.isMember(ourID) { // Use original here. Not a copy. csa.consumers[consumer] = ca } } nasa[stream] = csa } } streams[acc] = nasa } } else { streams = make(map[string]map[string]*streamAssignment, 1) asa, ok := cc.streams[opts.Account] if !ok { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream account %q not found", opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorAccount, Account: opts.Account, Error: fmt.Sprintf("JetStream account %q not found", opts.Account), }, } } js.mu.RUnlock() return health } nasa := make(map[string]*streamAssignment) if opts.Stream != _EMPTY_ { sa, ok := asa[opts.Stream] if !ok || !sa.Group.isMember(ourID) { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream stream %q not found on account %q", opts.Stream, opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorStream, Account: opts.Account, Stream: opts.Stream, Error: fmt.Sprintf("JetStream stream %q not found on account %q", opts.Stream, opts.Account), }, } } js.mu.RUnlock() return health } csa := sa.copyGroup() csa.consumers = make(map[string]*consumerAssignment) var consumerFound bool for consumer, ca := range sa.consumers { if opts.Consumer != _EMPTY_ { if consumer != opts.Consumer || !ca.Group.isMember(ourID) { continue } consumerFound = true } // If we are a member and we are not being restored, select for check. if sa.Group.isMember(ourID) && sa.Restore == nil { csa.consumers[consumer] = ca } if consumerFound { break } } if opts.Consumer != _EMPTY_ && !consumerFound { health.StatusCode = http.StatusNotFound if !details { health.Status = na health.Error = fmt.Sprintf("JetStream consumer %q not found for stream %q on account %q", opts.Consumer, opts.Stream, opts.Account) } else { health.Errors = []HealthzError{ { Type: HealthzErrorConsumer, Account: opts.Account, Stream: opts.Stream, Consumer: opts.Consumer, Error: fmt.Sprintf("JetStream consumer %q not found for stream %q on account %q", opts.Consumer, opts.Stream, opts.Account), }, } } js.mu.RUnlock() return health } nasa[opts.Stream] = csa } else { for stream, sa := range asa { // If we are a member and we are not being restored, select for check. if sa.Group.isMember(ourID) && sa.Restore == nil { csa := sa.copyGroup() csa.consumers = make(map[string]*consumerAssignment) for consumer, ca := range sa.consumers { if ca.Group.isMember(ourID) { csa.consumers[consumer] = ca } } nasa[stream] = csa } } } streams[opts.Account] = nasa } js.mu.RUnlock() // Use our copy to traverse so we do not need to hold the js lock. for accName, asa := range streams { acc, err := s.LookupAccount(accName) if err != nil && len(asa) > 0 { if !details { health.Status = na health.Error = fmt.Sprintf("JetStream can not lookup account %q: %v", accName, err) return health } health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorAccount, Account: accName, Error: fmt.Sprintf("JetStream can not lookup account %q: %v", accName, err), }) continue } for stream, sa := range asa { // Make sure we can look up if err := js.isStreamHealthy(acc, sa); err != nil { if !details { health.Status = na health.Error = fmt.Sprintf("JetStream stream '%s > %s' is not current: %s", accName, stream, err) return health } health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorStream, Account: accName, Stream: stream, Error: fmt.Sprintf("JetStream stream '%s > %s' is not current: %s", accName, stream, err), }) continue } mset, _ := acc.lookupStream(stream) // Now check consumers. for consumer, ca := range sa.consumers { if err := js.isConsumerHealthy(mset, consumer, ca); err != nil { if !details { health.Status = na health.Error = fmt.Sprintf("JetStream consumer '%s > %s > %s' is not current: %s", acc, stream, consumer, err) return health } health.Errors = append(health.Errors, HealthzError{ Type: HealthzErrorConsumer, Account: accName, Stream: stream, Consumer: consumer, Error: fmt.Sprintf("JetStream consumer '%s > %s > %s' is not current: %s", acc, stream, consumer, err), }) } } } } // Success. return health } type ExpvarzStatus struct { Memstats json.RawMessage `json:"memstats"` Cmdline json.RawMessage `json:"cmdline"` } func (s *Server) expvarz(_ *ExpvarzEventOptions) *ExpvarzStatus { var stat ExpvarzStatus const memStatsKey = "memstats" const cmdLineKey = "cmdline" expvar.Do(func(v expvar.KeyValue) { switch v.Key { case memStatsKey: stat.Memstats = json.RawMessage(v.Value.String()) case cmdLineKey: stat.Cmdline = json.RawMessage(v.Value.String()) } }) return &stat } type ProfilezStatus struct { Profile []byte `json:"profile"` Error string `json:"error"` } func (s *Server) profilez(opts *ProfilezOptions) *ProfilezStatus { var buffer bytes.Buffer switch opts.Name { case _EMPTY_: return &ProfilezStatus{ Error: "Profile name not specified", } case "cpu": if opts.Duration <= 0 || opts.Duration > 15*time.Second { return &ProfilezStatus{ Error: fmt.Sprintf("Duration %s should be between 0s and 15s", opts.Duration), } } if err := pprof.StartCPUProfile(&buffer); err != nil { return &ProfilezStatus{ Error: fmt.Sprintf("Failed to start CPU profile: %s", err), } } time.Sleep(opts.Duration) pprof.StopCPUProfile() default: profile := pprof.Lookup(opts.Name) if profile == nil { return &ProfilezStatus{ Error: fmt.Sprintf("Profile %q not found", opts.Name), } } if err := profile.WriteTo(&buffer, opts.Debug); err != nil { return &ProfilezStatus{ Error: fmt.Sprintf("Profile %q error: %s", opts.Name, err), } } } return &ProfilezStatus{ Profile: buffer.Bytes(), } } type RaftzGroup struct { ID string `json:"id"` State string `json:"state"` Size int `json:"size"` QuorumNeeded int `json:"quorum_needed"` Observer bool `json:"observer,omitempty"` Paused bool `json:"paused,omitempty"` Committed uint64 `json:"committed"` Applied uint64 `json:"applied"` CatchingUp bool `json:"catching_up,omitempty"` Leader string `json:"leader,omitempty"` EverHadLeader bool `json:"ever_had_leader"` Term uint64 `json:"term"` Vote string `json:"voted_for,omitempty"` PTerm uint64 `json:"pterm"` PIndex uint64 `json:"pindex"` IPQPropLen int `json:"ipq_proposal_len"` IPQEntryLen int `json:"ipq_entry_len"` IPQRespLen int `json:"ipq_resp_len"` IPQApplyLen int `json:"ipq_apply_len"` WAL StreamState `json:"wal"` WALError error `json:"wal_error,omitempty"` Peers map[string]RaftzGroupPeer `json:"peers"` } type RaftzGroupPeer struct { Name string `json:"name"` Known bool `json:"known"` LastReplicatedIndex uint64 `json:"last_replicated_index,omitempty"` LastSeen string `json:"last_seen,omitempty"` } type RaftzStatus map[string]map[string]RaftzGroup func (s *Server) HandleRaftz(w http.ResponseWriter, r *http.Request) { if s.raftNodes == nil { w.WriteHeader(404) w.Write([]byte("No Raft nodes registered")) return } groups := s.Raftz(&RaftzOptions{ AccountFilter: r.URL.Query().Get("acc"), GroupFilter: r.URL.Query().Get("group"), }) if groups == nil { w.WriteHeader(404) w.Write([]byte("No Raft nodes returned, check supplied filters")) return } b, _ := json.MarshalIndent(groups, "", " ") ResponseHandler(w, r, b) } func (s *Server) Raftz(opts *RaftzOptions) *RaftzStatus { afilter, gfilter := opts.AccountFilter, opts.GroupFilter if afilter == _EMPTY_ { if sys := s.SystemAccount(); sys != nil { afilter = sys.Name } else { return nil } } groups := map[string]RaftNode{} infos := RaftzStatus{} // account -> group ID s.rnMu.RLock() if gfilter != _EMPTY_ { if rg, ok := s.raftNodes[gfilter]; ok && rg != nil { if n, ok := rg.(*raft); ok { if n.accName == afilter { groups[gfilter] = rg } } } } else { for name, rg := range s.raftNodes { if rg == nil { continue } if n, ok := rg.(*raft); ok { if n.accName != afilter { continue } groups[name] = rg } } } s.rnMu.RUnlock() for name, rg := range groups { n, ok := rg.(*raft) if n == nil || !ok { continue } if _, ok := infos[n.accName]; !ok { infos[n.accName] = map[string]RaftzGroup{} } // Only take the lock once, using the public RaftNode functions would // cause us to take and release the locks over and over again. n.RLock() info := RaftzGroup{ ID: n.id, State: RaftState(n.state.Load()).String(), Size: n.csz, QuorumNeeded: n.qn, Observer: n.observer, Paused: n.paused, Committed: n.commit, Applied: n.applied, CatchingUp: n.catchup != nil, Leader: n.leader, EverHadLeader: n.pleader.Load(), Term: n.term, Vote: n.vote, PTerm: n.pterm, PIndex: n.pindex, IPQPropLen: n.prop.len(), IPQEntryLen: n.entry.len(), IPQRespLen: n.resp.len(), IPQApplyLen: n.apply.len(), WALError: n.werr, Peers: map[string]RaftzGroupPeer{}, } n.wal.FastState(&info.WAL) for id, p := range n.peers { if id == n.id { continue } peer := RaftzGroupPeer{ Name: s.serverNameForNode(id), Known: p.kp, LastReplicatedIndex: p.li, } if p.ts > 0 { peer.LastSeen = time.Since(time.Unix(0, p.ts)).String() } info.Peers[id] = peer } n.RUnlock() infos[n.accName][name] = info } return &infos } nats-server-2.10.27/server/monitor_sort_opts.go000066400000000000000000000111231477524627100216110ustar00rootroot00000000000000// Copyright 2013-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "time" ) // ConnInfos represents a connection info list. We use pointers since it will be sorted. type ConnInfos []*ConnInfo // For sorting // Len returns length for sorting. func (cl ConnInfos) Len() int { return len(cl) } // Swap will sawap the elements. func (cl ConnInfos) Swap(i, j int) { cl[i], cl[j] = cl[j], cl[i] } // SortOpt is a helper type to sort clients type SortOpt string // Possible sort options const ( ByCid SortOpt = "cid" // By connection ID ByStart SortOpt = "start" // By connection start time, same as CID BySubs SortOpt = "subs" // By number of subscriptions ByPending SortOpt = "pending" // By amount of data in bytes waiting to be sent to client ByOutMsgs SortOpt = "msgs_to" // By number of messages sent ByInMsgs SortOpt = "msgs_from" // By number of messages received ByOutBytes SortOpt = "bytes_to" // By amount of bytes sent ByInBytes SortOpt = "bytes_from" // By amount of bytes received ByLast SortOpt = "last" // By the last activity ByIdle SortOpt = "idle" // By the amount of inactivity ByUptime SortOpt = "uptime" // By the amount of time connections exist ByStop SortOpt = "stop" // By the stop time for a closed connection ByReason SortOpt = "reason" // By the reason for a closed connection ByRTT SortOpt = "rtt" // By the round trip time ) // Individual sort options provide the Less for sort.Interface. Len and Swap are on cList. // CID type byCid struct{ ConnInfos } func (l byCid) Less(i, j int) bool { return l.ConnInfos[i].Cid < l.ConnInfos[j].Cid } // Number of Subscriptions type bySubs struct{ ConnInfos } func (l bySubs) Less(i, j int) bool { return l.ConnInfos[i].NumSubs < l.ConnInfos[j].NumSubs } // Pending Bytes type byPending struct{ ConnInfos } func (l byPending) Less(i, j int) bool { return l.ConnInfos[i].Pending < l.ConnInfos[j].Pending } // Outbound Msgs type byOutMsgs struct{ ConnInfos } func (l byOutMsgs) Less(i, j int) bool { return l.ConnInfos[i].OutMsgs < l.ConnInfos[j].OutMsgs } // Inbound Msgs type byInMsgs struct{ ConnInfos } func (l byInMsgs) Less(i, j int) bool { return l.ConnInfos[i].InMsgs < l.ConnInfos[j].InMsgs } // Outbound Bytes type byOutBytes struct{ ConnInfos } func (l byOutBytes) Less(i, j int) bool { return l.ConnInfos[i].OutBytes < l.ConnInfos[j].OutBytes } // Inbound Bytes type byInBytes struct{ ConnInfos } func (l byInBytes) Less(i, j int) bool { return l.ConnInfos[i].InBytes < l.ConnInfos[j].InBytes } // Last Activity type byLast struct{ ConnInfos } func (l byLast) Less(i, j int) bool { return l.ConnInfos[i].LastActivity.UnixNano() < l.ConnInfos[j].LastActivity.UnixNano() } // Idle time type byIdle struct { ConnInfos now time.Time } func (l byIdle) Less(i, j int) bool { return l.now.Sub(l.ConnInfos[i].LastActivity) < l.now.Sub(l.ConnInfos[j].LastActivity) } // Uptime type byUptime struct { ConnInfos now time.Time } func (l byUptime) Less(i, j int) bool { ci := l.ConnInfos[i] cj := l.ConnInfos[j] var upi, upj time.Duration if ci.Stop == nil || ci.Stop.IsZero() { upi = l.now.Sub(ci.Start) } else { upi = ci.Stop.Sub(ci.Start) } if cj.Stop == nil || cj.Stop.IsZero() { upj = l.now.Sub(cj.Start) } else { upj = cj.Stop.Sub(cj.Start) } return upi < upj } // Stop type byStop struct{ ConnInfos } func (l byStop) Less(i, j int) bool { ciStop := l.ConnInfos[i].Stop cjStop := l.ConnInfos[j].Stop return ciStop.Before(*cjStop) } // Reason type byReason struct{ ConnInfos } func (l byReason) Less(i, j int) bool { return l.ConnInfos[i].Reason < l.ConnInfos[j].Reason } // RTT - Default is descending type byRTT struct{ ConnInfos } func (l byRTT) Less(i, j int) bool { return l.ConnInfos[i].rtt < l.ConnInfos[j].rtt } // IsValid determines if a sort option is valid func (s SortOpt) IsValid() bool { switch s { case _EMPTY_, ByCid, ByStart, BySubs, ByPending, ByOutMsgs, ByInMsgs, ByOutBytes, ByInBytes, ByLast, ByIdle, ByUptime, ByStop, ByReason, ByRTT: return true default: return false } } nats-server-2.10.27/server/monitor_test.go000066400000000000000000005037211477524627100205460ustar00rootroot00000000000000// Copyright 2013-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "encoding/json" "errors" "fmt" "io" "math/rand" "net" "net/http" "net/url" "os" "reflect" "runtime" "sort" "strings" "sync" "testing" "time" "unicode" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) const CLIENT_PORT = -1 const MONITOR_PORT = -1 const CLUSTER_PORT = -1 func DefaultMonitorOptions() *Options { return &Options{ Host: "127.0.0.1", Port: CLIENT_PORT, HTTPHost: "127.0.0.1", HTTPPort: MONITOR_PORT, HTTPBasePath: "/", ServerName: "monitor_server", NoLog: true, NoSigs: true, Tags: []string{"tag"}, } } func runMonitorServer() *Server { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true return RunServer(opts) } func runMonitorServerWithAccounts() *Server { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true aA := NewAccount("A") aB := NewAccount("B") opts.Accounts = append(opts.Accounts, aA, aB) opts.Users = append(opts.Users, &User{Username: "a", Password: "a", Account: aA}, &User{Username: "b", Password: "b", Account: aB}) return RunServer(opts) } func runMonitorServerNoHTTPPort() *Server { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.HTTPPort = 0 return RunServer(opts) } func resetPreviousHTTPConnections() { http.DefaultTransport.(*http.Transport).CloseIdleConnections() } func TestMyUptime(t *testing.T) { // Make sure we print this stuff right. var d time.Duration var s string d = 22 * time.Second s = myUptime(d) if s != "22s" { t.Fatalf("Expected `22s`, go ``%s`", s) } d = 4*time.Minute + d s = myUptime(d) if s != "4m22s" { t.Fatalf("Expected `4m22s`, go ``%s`", s) } d = 4*time.Hour + d s = myUptime(d) if s != "4h4m22s" { t.Fatalf("Expected `4h4m22s`, go ``%s`", s) } d = 32*24*time.Hour + d s = myUptime(d) if s != "32d4h4m22s" { t.Fatalf("Expected `32d4h4m22s`, go ``%s`", s) } d = 22*365*24*time.Hour + d s = myUptime(d) if s != "22y32d4h4m22s" { t.Fatalf("Expected `22y32d4h4m22s`, go ``%s`", s) } } // Make sure that we do not run the http server for monitoring unless asked. func TestNoMonitorPort(t *testing.T) { s := runMonitorServerNoHTTPPort() defer s.Shutdown() // this test might be meaningless now that we're testing with random ports? url := fmt.Sprintf("http://127.0.0.1:%d/", 11245) if resp, err := http.Get(url + "varz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } if resp, err := http.Get(url + "healthz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } if resp, err := http.Get(url + "connz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } } var ( appJSONContent = "application/json" appJSContent = "application/javascript" textPlain = "text/plain; charset=utf-8" textHTML = "text/html; charset=utf-8" ) func readBodyEx(t *testing.T, url string, status int, content string) []byte { t.Helper() resp, err := http.Get(url) if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } defer resp.Body.Close() if resp.StatusCode != status { t.Fatalf("Expected a %d response, got %d\n", status, resp.StatusCode) } ct := resp.Header.Get("Content-Type") if ct != content { t.Fatalf("Expected %q content-type, got %q\n", content, ct) } // Check the CORS header for "application/json" requests only. if ct == appJSONContent { acao := resp.Header.Get("Access-Control-Allow-Origin") if acao != "*" { t.Fatalf("Expected with %q Content-Type an Access-Control-Allow-Origin header with value %q, got %q\n", appJSONContent, "*", acao) } } body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } return body } func TestHTTPBasePath(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.HTTPBasePath = "/nats" s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/nats", s.MonitorAddr().Port) readBodyEx(t, url, http.StatusOK, textHTML) } func readBody(t *testing.T, url string) []byte { return readBodyEx(t, url, http.StatusOK, appJSONContent) } func pollVarz(t *testing.T, s *Server, mode int, url string, opts *VarzOptions) *Varz { t.Helper() if mode == 0 { v := &Varz{} body := readBody(t, url) if err := json.Unmarshal(body, v); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return v } v, err := s.Varz(opts) if err != nil { t.Fatalf("Error on Varz: %v", err) } return v } // https://github.com/nats-io/nats-server/issues/2170 // Just the ever increasing subs part. func TestVarzSubscriptionsResetProperly(t *testing.T) { // Run with JS to create a bunch of subs to start. resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.JetStream = true s := RunServer(opts) defer s.Shutdown() // This bug seems to only happen via the http endpoint, not direct calls. // Every time you call it doubles. url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) osubs := pollVarz(t, s, 0, url, nil).Subscriptions // Make sure we get same number back. if v := pollVarz(t, s, 0, url, nil); v.Subscriptions != osubs { t.Fatalf("Expected subs to stay the same, %d vs %d", osubs, v.Subscriptions) } } func TestHandleVarz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url+"varz", nil) // Do some sanity checks on values if time.Since(v.Start) > 10*time.Second { t.Fatal("Expected start time to be within 10 seconds.") } } time.Sleep(100 * time.Millisecond) nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url+"varz", nil) if v.Connections != 1 { t.Fatalf("Expected Connections of 1, got %v\n", v.Connections) } if v.TotalConnections < 1 { t.Fatalf("Expected Total Connections of at least 1, got %v\n", v.TotalConnections) } if v.InMsgs != 1 { t.Fatalf("Expected InMsgs of 1, got %v\n", v.InMsgs) } if v.OutMsgs != 1 { t.Fatalf("Expected OutMsgs of 1, got %v\n", v.OutMsgs) } if v.InBytes != 5 { t.Fatalf("Expected InBytes of 5, got %v\n", v.InBytes) } if v.OutBytes != 5 { t.Fatalf("Expected OutBytes of 5, got %v\n", v.OutBytes) } if v.Subscriptions != 0 { t.Fatalf("Expected Subscriptions of 0, got %v\n", v.Subscriptions) } if v.Name != "monitor_server" { t.Fatal("Expected ServerName to be 'monitor_server'") } if !v.Tags.Contains("tag") { t.Fatal("Expected tags to be 'tag'") } } // Test JSONP readBodyEx(t, url+"varz?callback=callback", http.StatusOK, appJSContent) } func pollConz(t *testing.T, s *Server, mode int, url string, opts *ConnzOptions) *Connz { t.Helper() if mode == 0 { body := readBody(t, url) c := &Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return c } c, err := s.Connz(opts) if err != nil { t.Fatalf("Error on Connz(): %v", err) } return c } func TestConnz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) testConnz := func(mode int) { c := pollConz(t, s, mode, url+"connz", nil) // Test contents.. if c.NumConns != 0 { t.Fatalf("Expected 0 connections, got %d\n", c.NumConns) } if c.Total != 0 { t.Fatalf("Expected 0 live connections, got %d\n", c.Total) } if c.Conns == nil || len(c.Conns) != 0 { t.Fatalf("Expected 0 connections in array, got %p\n", c.Conns) } // Test with connections. nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() time.Sleep(50 * time.Millisecond) c = pollConz(t, s, mode, url+"connz", nil) if c.NumConns != 1 { t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) } if c.Total != 1 { t.Fatalf("Expected 1 live connection, got %d\n", c.Total) } if c.Conns == nil || len(c.Conns) != 1 { t.Fatalf("Expected 1 connection in array, got %d\n", len(c.Conns)) } if c.Limit != DefaultConnListSize { t.Fatalf("Expected limit of %d, got %v\n", DefaultConnListSize, c.Limit) } if c.Offset != 0 { t.Fatalf("Expected offset of 0, got %v\n", c.Offset) } // Test inside details of each connection ci := c.Conns[0] if ci.Cid == 0 { t.Fatalf("Expected non-zero cid, got %v\n", ci.Cid) } if ci.IP != "127.0.0.1" { t.Fatalf("Expected \"127.0.0.1\" for IP, got %v\n", ci.IP) } if ci.Port == 0 { t.Fatalf("Expected non-zero port, got %v\n", ci.Port) } if ci.NumSubs != 0 { t.Fatalf("Expected num_subs of 0, got %v\n", ci.NumSubs) } if len(ci.Subs) != 0 { t.Fatalf("Expected subs of 0, got %v\n", ci.Subs) } if len(ci.SubsDetail) != 0 { t.Fatalf("Expected subsdetail of 0, got %v\n", ci.SubsDetail) } if ci.InMsgs != 1 { t.Fatalf("Expected InMsgs of 1, got %v\n", ci.InMsgs) } if ci.OutMsgs != 1 { t.Fatalf("Expected OutMsgs of 1, got %v\n", ci.OutMsgs) } if ci.InBytes != 5 { t.Fatalf("Expected InBytes of 1, got %v\n", ci.InBytes) } if ci.OutBytes != 5 { t.Fatalf("Expected OutBytes of 1, got %v\n", ci.OutBytes) } if ci.Start.IsZero() { t.Fatal("Expected Start to be valid\n") } if ci.Uptime == "" { t.Fatal("Expected Uptime to be valid\n") } if ci.LastActivity.IsZero() { t.Fatal("Expected LastActivity to be valid\n") } if ci.LastActivity.UnixNano() < ci.Start.UnixNano() { t.Fatalf("Expected LastActivity [%v] to be > Start [%v]\n", ci.LastActivity, ci.Start) } if ci.Idle == "" { t.Fatal("Expected Idle to be valid\n") } // This is a change, we now expect them to be set for connections when the // client sends a connect. if ci.RTT == "" { t.Fatal("Expected RTT to be set for new connection\n") } } for mode := 0; mode < 2; mode++ { testConnz(mode) checkClientsCount(t, s, 0) } // Test JSONP readBodyEx(t, url+"connz?callback=callback", http.StatusOK, appJSContent) } func TestConnzBadParams(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/connz?", s.MonitorAddr().Port) readBodyEx(t, url+"auth=xxx", http.StatusBadRequest, textPlain) readBodyEx(t, url+"subs=xxx", http.StatusBadRequest, textPlain) readBodyEx(t, url+"offset=xxx", http.StatusBadRequest, textPlain) readBodyEx(t, url+"limit=xxx", http.StatusBadRequest, textPlain) readBodyEx(t, url+"state=xxx", http.StatusBadRequest, textPlain) } func TestConnzWithSubs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() nc.Subscribe("hello.foo", func(m *nats.Msg) {}) ensureServerActivityRecorded(t, nc) url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?subs=1", &ConnzOptions{Subscriptions: true}) // Test inside details of each connection ci := c.Conns[0] if len(ci.Subs) != 1 || ci.Subs[0] != "hello.foo" { t.Fatalf("Expected subs of 1, got %v\n", ci.Subs) } } } func TestConnzWithSubsDetail(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() nc.Subscribe("hello.foo", func(m *nats.Msg) {}) ensureServerActivityRecorded(t, nc) url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?subs=detail", &ConnzOptions{SubscriptionsDetail: true}) // Test inside details of each connection ci := c.Conns[0] if len(ci.SubsDetail) != 1 || ci.SubsDetail[0].Subject != "hello.foo" { t.Fatalf("Expected subsdetail of 1, got %v\n", ci.Subs) } } } func TestClosedConnzWithSubsDetail(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) nc.Subscribe("hello.foo", func(m *nats.Msg) {}) ensureServerActivityRecorded(t, nc) nc.Close() s.mu.Lock() for len(s.clients) != 0 { s.mu.Unlock() <-time.After(100 * time.Millisecond) s.mu.Lock() } s.mu.Unlock() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?state=closed&subs=detail", &ConnzOptions{State: ConnClosed, SubscriptionsDetail: true}) // Test inside details of each connection ci := c.Conns[0] if len(ci.SubsDetail) != 1 || ci.SubsDetail[0].Subject != "hello.foo" { t.Fatalf("Expected subsdetail of 1, got %v\n", ci.Subs) } } } func TestConnzWithCID(t *testing.T) { s := runMonitorServer() defer s.Shutdown() // The one we will request cid := 5 total := 10 // Create 10 for i := 1; i <= total; i++ { nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() if i == cid { nc.Subscribe("hello.foo", func(m *nats.Msg) {}) nc.Subscribe("hello.bar", func(m *nats.Msg) {}) ensureServerActivityRecorded(t, nc) } } url := fmt.Sprintf("http://127.0.0.1:%d/connz?cid=%d", s.MonitorAddr().Port, cid) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url, &ConnzOptions{CID: uint64(cid)}) // Test inside details of each connection if len(c.Conns) != 1 { t.Fatalf("Expected only one connection, but got %d\n", len(c.Conns)) } if c.NumConns != 1 { t.Fatalf("Expected NumConns to be 1, but got %d\n", c.NumConns) } ci := c.Conns[0] if ci.Cid != uint64(cid) { t.Fatalf("Expected to receive connection %v, but received %v\n", cid, ci.Cid) } if ci.NumSubs != 2 { t.Fatalf("Expected to receive connection with %d subs, but received %d\n", 2, ci.NumSubs) } // Now test a miss badUrl := fmt.Sprintf("http://127.0.0.1:%d/connz?cid=%d", s.MonitorAddr().Port, 100) c = pollConz(t, s, mode, badUrl, &ConnzOptions{CID: uint64(100)}) if len(c.Conns) != 0 { t.Fatalf("Expected no connections, got %d\n", len(c.Conns)) } if c.NumConns != 0 { t.Fatalf("Expected NumConns of 0, got %d\n", c.NumConns) } } } // Helper to map to connection name func createConnMap(cz *Connz) map[string]*ConnInfo { cm := make(map[string]*ConnInfo) for _, c := range cz.Conns { cm[c.Name] = c } return cm } func getFooAndBar(cm map[string]*ConnInfo) (*ConnInfo, *ConnInfo) { return cm["foo"], cm["bar"] } func ensureServerActivityRecorded(t *testing.T, nc *nats.Conn) { nc.Flush() err := nc.Flush() if err != nil { t.Fatalf("Error flushing: %v\n", err) } } func TestConnzRTT(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) testRTT := func(mode int) { // Test with connections. nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() c := pollConz(t, s, mode, url+"connz", nil) if c.NumConns != 1 { t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) } // Send a server side PING to record RTT s.mu.Lock() ci := c.Conns[0] sc := s.clients[ci.Cid] if sc == nil { t.Fatalf("Error looking up client %v\n", ci.Cid) } s.mu.Unlock() sc.mu.Lock() sc.sendPing() sc.mu.Unlock() // Wait for client to respond with PONG time.Sleep(20 * time.Millisecond) // Repoll for updated information. c = pollConz(t, s, mode, url+"connz", nil) ci = c.Conns[0] rtt, err := time.ParseDuration(ci.RTT) if err != nil { t.Fatalf("Could not parse RTT properly, %v (ci.RTT=%v)", err, ci.RTT) } if rtt <= 0 { t.Fatal("Expected RTT to be valid and non-zero\n") } if (runtime.GOOS == "windows" && rtt > 20*time.Millisecond) || rtt > 20*time.Millisecond || rtt < 100*time.Nanosecond { t.Fatalf("Invalid RTT of %s\n", ci.RTT) } } for mode := 0; mode < 2; mode++ { testRTT(mode) checkClientsCount(t, s, 0) } } func TestConnzLastActivity(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) url += "connz?subs=1" opts := &ConnzOptions{Subscriptions: true} var sleepTime time.Duration if runtime.GOOS == "windows" { sleepTime = 10 * time.Millisecond } testActivity := func(mode int) { ncFoo := createClientConnWithName(t, "foo", s) defer ncFoo.Close() ncBar := createClientConnWithName(t, "bar", s) defer ncBar.Close() // Test inside details of each connection ciFoo, ciBar := getFooAndBar(createConnMap(pollConz(t, s, mode, url, opts))) // Test that LastActivity is non-zero if ciFoo.LastActivity.IsZero() { t.Fatalf("Expected LastActivity for connection '%s'to be valid\n", ciFoo.Name) } if ciBar.LastActivity.IsZero() { t.Fatalf("Expected LastActivity for connection '%s'to be valid\n", ciBar.Name) } // Foo should be older than Bar if ciFoo.LastActivity.After(ciBar.LastActivity) { t.Fatal("Expected connection 'foo' to be older than 'bar'\n") } fooLA := ciFoo.LastActivity barLA := ciBar.LastActivity ensureServerActivityRecorded(t, ncFoo) ensureServerActivityRecorded(t, ncBar) time.Sleep(sleepTime) // Sub should trigger update. sub, _ := ncFoo.Subscribe("hello.world", func(m *nats.Msg) {}) ensureServerActivityRecorded(t, ncFoo) ciFoo, _ = getFooAndBar(createConnMap(pollConz(t, s, mode, url, opts))) nextLA := ciFoo.LastActivity if fooLA.Equal(nextLA) { t.Fatalf("Subscribe should have triggered update to LastActivity %+v\n", ciFoo) } fooLA = nextLA time.Sleep(sleepTime) // Publish and Message Delivery should trigger as well. So both connections // should have updates. ncBar.Publish("hello.world", []byte("Hello")) ensureServerActivityRecorded(t, ncFoo) ensureServerActivityRecorded(t, ncBar) ciFoo, ciBar = getFooAndBar(createConnMap(pollConz(t, s, mode, url, opts))) nextLA = ciBar.LastActivity if barLA.Equal(nextLA) { t.Fatalf("Publish should have triggered update to LastActivity\n") } // Message delivery on ncFoo should have triggered as well. nextLA = ciFoo.LastActivity if fooLA.Equal(nextLA) { t.Fatalf("Message delivery should have triggered update to LastActivity\n") } fooLA = nextLA time.Sleep(sleepTime) // Unsub should trigger as well sub.Unsubscribe() ensureServerActivityRecorded(t, ncFoo) ciFoo, _ = getFooAndBar(createConnMap(pollConz(t, s, mode, url, opts))) nextLA = ciFoo.LastActivity if fooLA.Equal(nextLA) { t.Fatalf("Message delivery should have triggered update to LastActivity\n") } } for mode := 0; mode < 2; mode++ { testActivity(mode) } } func TestConnzWithOffsetAndLimit(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?offset=1&limit=1", &ConnzOptions{Offset: 1, Limit: 1}) if c.Conns == nil || len(c.Conns) != 0 { t.Fatalf("Expected 0 connections in array, got %p\n", c.Conns) } // Test that when given negative values, 0 or default is used c = pollConz(t, s, mode, url+"connz?offset=-1&limit=-1", &ConnzOptions{Offset: -11, Limit: -11}) if c.Conns == nil || len(c.Conns) != 0 { t.Fatalf("Expected 0 connections in array, got %p\n", c.Conns) } if c.Offset != 0 { t.Fatalf("Expected offset to be 0, and limit to be %v, got %v and %v", DefaultConnListSize, c.Offset, c.Limit) } } cl1 := createClientConnSubscribeAndPublish(t, s) defer cl1.Close() cl2 := createClientConnSubscribeAndPublish(t, s) defer cl2.Close() for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?offset=1&limit=1", &ConnzOptions{Offset: 1, Limit: 1}) if c.Limit != 1 { t.Fatalf("Expected limit of 1, got %v\n", c.Limit) } if c.Offset != 1 { t.Fatalf("Expected offset of 1, got %v\n", c.Offset) } if len(c.Conns) != 1 { t.Fatalf("Expected conns of 1, got %v\n", len(c.Conns)) } if c.NumConns != 1 { t.Fatalf("Expected NumConns to be 1, got %v\n", c.NumConns) } if c.Total != 2 { t.Fatalf("Expected Total to be at least 2, got %v", c.Total) } c = pollConz(t, s, mode, url+"connz?offset=2&limit=1", &ConnzOptions{Offset: 2, Limit: 1}) if c.Limit != 1 { t.Fatalf("Expected limit of 1, got %v\n", c.Limit) } if c.Offset != 2 { t.Fatalf("Expected offset of 2, got %v\n", c.Offset) } if len(c.Conns) != 0 { t.Fatalf("Expected conns of 0, got %v\n", len(c.Conns)) } if c.NumConns != 0 { t.Fatalf("Expected NumConns to be 0, got %v\n", c.NumConns) } if c.Total != 2 { t.Fatalf("Expected Total to be 2, got %v", c.Total) } } } func TestConnzDefaultSorted(t *testing.T) { s := runMonitorServer() defer s.Shutdown() clients := make([]*nats.Conn, 4) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz", nil) if c.Conns[0].Cid > c.Conns[1].Cid || c.Conns[1].Cid > c.Conns[2].Cid || c.Conns[2].Cid > c.Conns[3].Cid { t.Fatalf("Expected conns sorted in ascending order by cid, got %v < %v\n", c.Conns[0].Cid, c.Conns[3].Cid) } } } func TestConnzSortedByCid(t *testing.T) { s := runMonitorServer() defer s.Shutdown() clients := make([]*nats.Conn, 4) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=cid", &ConnzOptions{Sort: ByCid}) if c.Conns[0].Cid > c.Conns[1].Cid || c.Conns[1].Cid > c.Conns[2].Cid || c.Conns[2].Cid > c.Conns[3].Cid { t.Fatalf("Expected conns sorted in ascending order by cid, got [%v, %v, %v, %v]\n", c.Conns[0].Cid, c.Conns[1].Cid, c.Conns[2].Cid, c.Conns[3].Cid) } } } func TestConnzSortedByStart(t *testing.T) { s := runMonitorServer() defer s.Shutdown() clients := make([]*nats.Conn, 4) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=start", &ConnzOptions{Sort: ByStart}) if c.Conns[0].Start.After(c.Conns[1].Start) || c.Conns[1].Start.After(c.Conns[2].Start) || c.Conns[2].Start.After(c.Conns[3].Start) { t.Fatalf("Expected conns sorted in ascending order by startime, got [%v, %v, %v, %v]\n", c.Conns[0].Start, c.Conns[1].Start, c.Conns[2].Start, c.Conns[3].Start) } } } func TestConnzSortedByBytesAndMsgs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() // Create a connection and make it send more messages than others firstClient := createClientConnSubscribeAndPublish(t, s) for i := 0; i < 100; i++ { firstClient.Publish("foo", []byte("Hello World")) } defer firstClient.Close() firstClient.Flush() clients := make([]*nats.Conn, 3) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=bytes_to", &ConnzOptions{Sort: ByOutBytes}) if c.Conns[0].OutBytes < c.Conns[1].OutBytes || c.Conns[0].OutBytes < c.Conns[2].OutBytes || c.Conns[0].OutBytes < c.Conns[3].OutBytes { t.Fatalf("Expected conns sorted in descending order by bytes to, got %v < one of [%v, %v, %v]\n", c.Conns[0].OutBytes, c.Conns[1].OutBytes, c.Conns[2].OutBytes, c.Conns[3].OutBytes) } c = pollConz(t, s, mode, url+"connz?sort=msgs_to", &ConnzOptions{Sort: ByOutMsgs}) if c.Conns[0].OutMsgs < c.Conns[1].OutMsgs || c.Conns[0].OutMsgs < c.Conns[2].OutMsgs || c.Conns[0].OutMsgs < c.Conns[3].OutMsgs { t.Fatalf("Expected conns sorted in descending order by msgs from, got %v < one of [%v, %v, %v]\n", c.Conns[0].OutMsgs, c.Conns[1].OutMsgs, c.Conns[2].OutMsgs, c.Conns[3].OutMsgs) } c = pollConz(t, s, mode, url+"connz?sort=bytes_from", &ConnzOptions{Sort: ByInBytes}) if c.Conns[0].InBytes < c.Conns[1].InBytes || c.Conns[0].InBytes < c.Conns[2].InBytes || c.Conns[0].InBytes < c.Conns[3].InBytes { t.Fatalf("Expected conns sorted in descending order by bytes from, got %v < one of [%v, %v, %v]\n", c.Conns[0].InBytes, c.Conns[1].InBytes, c.Conns[2].InBytes, c.Conns[3].InBytes) } c = pollConz(t, s, mode, url+"connz?sort=msgs_from", &ConnzOptions{Sort: ByInMsgs}) if c.Conns[0].InMsgs < c.Conns[1].InMsgs || c.Conns[0].InMsgs < c.Conns[2].InMsgs || c.Conns[0].InMsgs < c.Conns[3].InMsgs { t.Fatalf("Expected conns sorted in descending order by msgs from, got %v < one of [%v, %v, %v]\n", c.Conns[0].InMsgs, c.Conns[1].InMsgs, c.Conns[2].InMsgs, c.Conns[3].InMsgs) } } } func TestConnzSortedByPending(t *testing.T) { s := runMonitorServer() defer s.Shutdown() firstClient := createClientConnSubscribeAndPublish(t, s) firstClient.Subscribe("hello.world", func(m *nats.Msg) {}) clients := make([]*nats.Conn, 3) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } defer firstClient.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=pending", &ConnzOptions{Sort: ByPending}) if c.Conns[0].Pending < c.Conns[1].Pending || c.Conns[0].Pending < c.Conns[2].Pending || c.Conns[0].Pending < c.Conns[3].Pending { t.Fatalf("Expected conns sorted in descending order by number of pending, got %v < one of [%v, %v, %v]\n", c.Conns[0].Pending, c.Conns[1].Pending, c.Conns[2].Pending, c.Conns[3].Pending) } } } func TestConnzSortedBySubs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() firstClient := createClientConnSubscribeAndPublish(t, s) firstClient.Subscribe("hello.world", func(m *nats.Msg) {}) defer firstClient.Close() clients := make([]*nats.Conn, 3) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=subs", &ConnzOptions{Sort: BySubs}) if c.Conns[0].NumSubs < c.Conns[1].NumSubs || c.Conns[0].NumSubs < c.Conns[2].NumSubs || c.Conns[0].NumSubs < c.Conns[3].NumSubs { t.Fatalf("Expected conns sorted in descending order by number of subs, got %v < one of [%v, %v, %v]\n", c.Conns[0].NumSubs, c.Conns[1].NumSubs, c.Conns[2].NumSubs, c.Conns[3].NumSubs) } } } func TestConnzSortedByLast(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true s := RunServer(opts) defer s.Shutdown() firstClient := createClientConnSubscribeAndPublish(t, s) defer firstClient.Close() firstClient.Subscribe("hello.world", func(m *nats.Msg) {}) firstClient.Flush() clients := make([]*nats.Conn, 3) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() clients[i].Flush() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=last", &ConnzOptions{Sort: ByLast}) if c.Conns[0].LastActivity.UnixNano() < c.Conns[1].LastActivity.UnixNano() || c.Conns[1].LastActivity.UnixNano() < c.Conns[2].LastActivity.UnixNano() || c.Conns[2].LastActivity.UnixNano() < c.Conns[3].LastActivity.UnixNano() { t.Fatalf("Expected conns sorted in descending order by lastActivity, got %v < one of [%v, %v, %v]\n", c.Conns[0].LastActivity, c.Conns[1].LastActivity, c.Conns[2].LastActivity, c.Conns[3].LastActivity) } } } func TestConnzSortedByUptime(t *testing.T) { s := runMonitorServer() defer s.Shutdown() for i := 0; i < 4; i++ { client := createClientConnSubscribeAndPublish(t, s) defer client.Close() // Since we check times (now-start) does not have to be big. time.Sleep(50 * time.Millisecond) } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?sort=uptime", &ConnzOptions{Sort: ByUptime}) now := time.Now() ups := make([]int, 4) for i := 0; i < 4; i++ { ups[i] = int(now.Sub(c.Conns[i].Start)) } if !sort.IntsAreSorted(ups) { d := make([]time.Duration, 4) for i := 0; i < 4; i++ { d[i] = time.Duration(ups[i]) } t.Fatalf("Expected conns sorted in ascending order by uptime (now-Start), got %+v\n", d) } } } func TestConnzSortedByUptimeClosedConn(t *testing.T) { s := runMonitorServer() defer s.Shutdown() for i := time.Duration(1); i <= 4; i++ { c := createClientConnSubscribeAndPublish(t, s) // Grab client and asjust start time such that client := s.getClient(uint64(i)) if client == nil { t.Fatalf("Could nopt retrieve client for %d\n", i) } client.mu.Lock() client.start = client.start.Add(-10 * (4 - i) * time.Second) client.mu.Unlock() c.Close() } checkClosedConns(t, s, 4, time.Second) url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?state=closed&sort=uptime", &ConnzOptions{State: ConnClosed, Sort: ByUptime}) ups := make([]int, 4) for i := 0; i < 4; i++ { ups[i] = int(c.Conns[i].Stop.Sub(c.Conns[i].Start)) } if !sort.IntsAreSorted(ups) { d := make([]time.Duration, 4) for i := 0; i < 4; i++ { d[i] = time.Duration(ups[i]) } t.Fatalf("Expected conns sorted in ascending order by uptime, got %+v\n", d) } } } func TestConnzSortedByStopOnOpen(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // 4 clients for i := 0; i < 4; i++ { c, err := nats.Connect(url) if err != nil { t.Fatalf("Could not create client: %v\n", err) } defer c.Close() } c, err := s.Connz(&ConnzOptions{Sort: ByStop}) if err == nil { t.Fatalf("Expected err to be non-nil, got %+v\n", c) } } func TestConnzSortedByStopTimeClosedConn(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // 4 clients for i := 0; i < 4; i++ { c, err := nats.Connect(url) if err != nil { t.Fatalf("Could not create client: %v\n", err) } c.Close() } checkClosedConns(t, s, 4, time.Second) // Now adjust the Stop times for these with some random values. s.mu.Lock() now := time.Now().UTC() ccs := s.closed.closedClients() for _, cc := range ccs { newStop := now.Add(time.Duration(rand.Int()%120) * -time.Minute) cc.Stop = &newStop } s.mu.Unlock() url = fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?state=closed&sort=stop", &ConnzOptions{State: ConnClosed, Sort: ByStop}) ups := make([]int, 4) nowU := time.Now().UnixNano() for i := 0; i < 4; i++ { ups[i] = int(nowU - c.Conns[i].Stop.UnixNano()) } if !sort.IntsAreSorted(ups) { d := make([]time.Duration, 4) for i := 0; i < 4; i++ { d[i] = time.Duration(ups[i]) } t.Fatalf("Expected conns sorted in ascending order by stop time, got %+v\n", d) } } } func TestConnzSortedByReason(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // 20 clients for i := 0; i < 20; i++ { c, err := nats.Connect(url) if err != nil { t.Fatalf("Could not create client: %v\n", err) } c.Close() } checkClosedConns(t, s, 20, time.Second) // Now adjust the Reasons for these with some random values. s.mu.Lock() ccs := s.closed.closedClients() max := int(ServerShutdown) for _, cc := range ccs { cc.Reason = ClosedState(rand.Int() % max).String() } s.mu.Unlock() url = fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz?state=closed&sort=reason", &ConnzOptions{State: ConnClosed, Sort: ByReason}) rs := make([]string, 20) for i := 0; i < 20; i++ { rs[i] = c.Conns[i].Reason } if !sort.StringsAreSorted(rs) { t.Fatalf("Expected conns sorted in order by stop reason, got %#v\n", rs) } } } func TestConnzSortedByReasonOnOpen(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // 4 clients for i := 0; i < 4; i++ { c, err := nats.Connect(url) if err != nil { t.Fatalf("Could not create client: %v\n", err) } defer c.Close() } c, err := s.Connz(&ConnzOptions{Sort: ByReason}) if err == nil { t.Fatalf("Expected err to be non-nil, got %+v\n", c) } } func TestConnzSortedByIdle(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://%s/connz?sort=idle", s.MonitorAddr()) now := time.Now() clients := []struct { start time.Time // Client start time. last time.Time // Client last activity time. }{ {start: now.Add(-10 * time.Second), last: now.Add(-5 * time.Second)}, {start: now.Add(-20 * time.Second), last: now.Add(-10 * time.Second)}, {start: now.Add(-3 * time.Second), last: now.Add(-2 * time.Second)}, {start: now.Add(-30 * time.Second), last: now.Add(-20 * time.Second)}, } testIdle := func(mode int) { // Connect the specified number of clients. for _, c := range clients { clientConn := createClientConnSubscribeAndPublish(t, s) defer clientConn.Close() cid, err := clientConn.GetClientID() if err != nil { t.Fatalf("error getting the client CID: %v", err) } client := s.getClient(cid) if client == nil { t.Fatalf("error looking up client %d", cid) } // Change the client's start and last activity times. client.mu.Lock() client.start = c.start client.last = c.last client.mu.Unlock() } connz := pollConz(t, s, mode, url, &ConnzOptions{Sort: ByIdle}) wantConns := len(clients) gotConns := len(connz.Conns) if gotConns != wantConns { t.Fatalf("want %d connections, got %d", wantConns, gotConns) } idleDurations := getConnsIdleDurations(t, connz.Conns) if !sortedDurationsDesc(idleDurations) { t.Errorf("want durations sorted in descending order, got %v", idleDurations) } } for mode := 0; mode < 2; mode++ { testIdle(mode) } } // getConnsIdleDurations returns a slice of parsed idle durations from a connection info slice. func getConnsIdleDurations(t *testing.T, conns []*ConnInfo) []time.Duration { t.Helper() durations := make([]time.Duration, 0, len(conns)) for _, conn := range conns { idle, err := time.ParseDuration(conn.Idle) if err != nil { t.Fatalf("error parsing duration %q: %v", conn.Idle, err) } durations = append(durations, idle) } return durations } // sortedDurationsDesc checks if a time.Duration slice is sorted in descending order. func sortedDurationsDesc(durations []time.Duration) bool { return sort.SliceIsSorted(durations, func(i, j int) bool { // Must be longer than the next duration. return durations[i] > durations[j] }) } func TestConnzSortByIdleTime(t *testing.T) { now := time.Now().UTC() cases := map[string]ConnInfos{ "zero values": {{}, {}, {}, {}}, "equal last activity times": { {Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-time.Minute)}, {Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-time.Minute)}, {Start: now.Add(-10 * time.Second), LastActivity: now.Add(-time.Minute)}, {Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)}, }, "last activity in the future": { {Start: now.Add(-50 * time.Minute), LastActivity: now.Add(10 * time.Minute)}, // +10m {Start: now.Add(-30 * time.Minute), LastActivity: now.Add(5 * time.Minute)}, // +5m {Start: now.Add(-24 * time.Hour), LastActivity: now.Add(2 * time.Second)}, // +2s {Start: now.Add(-10 * time.Second), LastActivity: now.Add(15 * time.Minute)}, // +15m {Start: now.Add(-2 * time.Hour), LastActivity: now.Add(time.Minute)}, // +1m }, "unsorted": { {Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-10 * time.Minute)}, // 10m ago {Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-5 * time.Minute)}, // 5m ago {Start: now.Add(-24 * time.Hour), LastActivity: now.Add(-2 * time.Second)}, // 2s ago {Start: now.Add(-10 * time.Second), LastActivity: now.Add(-15 * time.Minute)}, // 15m ago {Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)}, // 1m ago }, "unsorted with zero value start time": { {LastActivity: now.Add(-10 * time.Minute)}, // 10m ago {LastActivity: now.Add(-5 * time.Minute)}, // 5m ago {LastActivity: now.Add(-2 * time.Second)}, // 2s ago {LastActivity: now.Add(-15 * time.Minute)}, // 15m ago {LastActivity: now.Add(-time.Minute)}, // 1m ago }, "sorted": { {Start: now.Add(-24 * time.Hour), LastActivity: now.Add(-2 * time.Second)}, // 2s ago {Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)}, // 1m ago {Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-5 * time.Minute)}, // 5m ago {Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-10 * time.Minute)}, // 10m ago {Start: now.Add(-10 * time.Second), LastActivity: now.Add(-15 * time.Minute)}, // 15m ago }, "sorted with zero value start time": { {LastActivity: now.Add(-2 * time.Second)}, // 2s ago {LastActivity: now.Add(-time.Minute)}, // 1m ago {LastActivity: now.Add(-5 * time.Minute)}, // 5m ago {LastActivity: now.Add(-10 * time.Minute)}, // 10m ago {LastActivity: now.Add(-15 * time.Minute)}, // 15m ago }, } for name, conns := range cases { t.Run(name, func(t *testing.T) { sort.Sort(byIdle{conns, now}) idleDurations := getIdleDurations(conns, now) if !sortedDurationsAsc(idleDurations) { t.Errorf("want durations sorted in ascending order, got %v", idleDurations) } }) } } // getIdleDurations returns a slice of idle durations from a connection info list up until now time. func getIdleDurations(conns ConnInfos, now time.Time) []time.Duration { durations := make([]time.Duration, 0, len(conns)) for _, conn := range conns { durations = append(durations, now.Sub(conn.LastActivity)) } return durations } // sortedDurationsAsc checks if a time.Duration slice is sorted in ascending order. func sortedDurationsAsc(durations []time.Duration) bool { return sort.SliceIsSorted(durations, func(i, j int) bool { return durations[i] < durations[j] }) } func TestConnzSortBadRequest(t *testing.T) { s := runMonitorServer() defer s.Shutdown() firstClient := createClientConnSubscribeAndPublish(t, s) firstClient.Subscribe("hello.world", func(m *nats.Msg) {}) clients := make([]*nats.Conn, 3) for i := range clients { clients[i] = createClientConnSubscribeAndPublish(t, s) defer clients[i].Close() } defer firstClient.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) readBodyEx(t, url+"connz?sort=foo", http.StatusBadRequest, textPlain) if _, err := s.Connz(&ConnzOptions{Sort: "foo"}); err == nil { t.Fatal("Expected error, got none") } } func pollRoutez(t *testing.T, s *Server, mode int, url string, opts *RoutezOptions) *Routez { t.Helper() if mode == 0 { rz := &Routez{} body := readBody(t, url) if err := json.Unmarshal(body, rz); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return rz } rz, err := s.Routez(opts) if err != nil { t.Fatalf("Error on Routez: %v", err) } return rz } func TestConnzWithRoutes(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.Cluster.Name = "A" opts.Cluster.Host = "127.0.0.1" opts.Cluster.Port = CLUSTER_PORT s := RunServer(opts) defer s.Shutdown() opts = &Options{ Host: "127.0.0.1", Port: -1, Cluster: ClusterOpts{ Name: "A", Host: "127.0.0.1", Port: -1, }, NoLog: true, NoSigs: true, NoSystemAccount: true, } routeURL, _ := url.Parse(fmt.Sprintf("nats-route://127.0.0.1:%d", s.ClusterAddr().Port)) opts.Routes = []*url.URL{routeURL} start := time.Now() sc := RunServer(opts) defer sc.Shutdown() checkClusterFormed(t, s, sc) url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz", nil) // Test contents.. // Make sure routes don't show up under connz, but do under routez if c.NumConns != 0 { t.Fatalf("Expected 0 connections, got %d", c.NumConns) } if c.Conns == nil || len(c.Conns) != 0 { t.Fatalf("Expected 0 connections in array, got %p", c.Conns) } } nc := createClientConnSubscribeAndPublish(t, sc) defer nc.Close() nc.Subscribe("hello.bar", func(m *nats.Msg) {}) nc.Flush() checkExpectedSubs(t, 1, s, sc) // Now check routez urls := []string{"routez", "routez?subs=1", "routez?subs=detail"} for subs, urlSuffix := range urls { for mode := 0; mode < 2; mode++ { rz := pollRoutez(t, s, mode, url+urlSuffix, &RoutezOptions{Subscriptions: subs == 1, SubscriptionsDetail: subs == 2}) if rz.NumRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, rz.NumRoutes) } if len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected route array of %d, got %v", DEFAULT_ROUTE_POOL_SIZE, len(rz.Routes)) } route := rz.Routes[0] if route.DidSolicit { t.Fatalf("Expected unsolicited route, got %v", route.DidSolicit) } if route.Start.IsZero() { t.Fatalf("Expected Start to be set, got %+v", route) } else if route.Start.Before(start) { t.Fatalf("Unexpected start time: route was started around %v, got %v", start, route.Start) } if route.LastActivity.IsZero() { t.Fatalf("Expected LastActivity to be set, got %+v", route) } if route.Uptime == _EMPTY_ { t.Fatalf("Expected Uptime to be set, it was not") } if route.Idle == _EMPTY_ { t.Fatalf("Expected Idle to be set, it was not") } // Don't ask for subs, so there should not be any if subs == 0 { if len(route.Subs) != 0 { t.Fatalf("There should not be subs, got %v", len(route.Subs)) } } else if subs == 1 { if len(route.Subs) != 1 && len(route.SubsDetail) != 0 { t.Fatalf("There should be 1 sub, got %v", len(route.Subs)) } } else if subs == 2 { if len(route.SubsDetail) != 1 && len(route.Subs) != 0 { t.Fatalf("There should be 1 sub, got %v", len(route.SubsDetail)) } } } } // Test JSONP readBodyEx(t, url+"routez?callback=callback", http.StatusOK, appJSContent) } func TestRoutezWithBadParams(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/routez?", s.MonitorAddr().Port) readBodyEx(t, url+"subs=xxx", http.StatusBadRequest, textPlain) } func pollSubsz(t *testing.T, s *Server, mode int, url string, opts *SubszOptions) *Subsz { t.Helper() if mode == 0 { body := readBody(t, url) sz := &Subsz{} if err := json.Unmarshal(body, sz); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return sz } sz, err := s.Subsz(opts) if err != nil { t.Fatalf("Error on Subsz: %v", err) } return sz } func TestSubsz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz", nil) if sl.NumSubs != 0 { t.Fatalf("Expected NumSubs of 0, got %d\n", sl.NumSubs) } if sl.NumInserts != 1 { t.Fatalf("Expected NumInserts of 1, got %d\n", sl.NumInserts) } if sl.NumMatches != 1 { t.Fatalf("Expected NumMatches of 1, got %d\n", sl.NumMatches) } } // Test JSONP readBodyEx(t, url+"subsz?callback=callback", http.StatusOK, appJSContent) } func TestSubszDetails(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() nc.Subscribe("foo.*", func(m *nats.Msg) {}) nc.Subscribe("foo.bar", func(m *nats.Msg) {}) nc.Subscribe("foo.foo", func(m *nats.Msg) {}) nc.Publish("foo.bar", []byte("Hello")) nc.Publish("foo.baz", []byte("Hello")) nc.Publish("foo.foo", []byte("Hello")) nc.Flush() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz?subs=1", &SubszOptions{Subscriptions: true}) if sl.NumSubs != 3 { t.Fatalf("Expected NumSubs of 3, got %d\n", sl.NumSubs) } if sl.Total != 3 { t.Fatalf("Expected Total of 3, got %d\n", sl.Total) } if len(sl.Subs) != 3 { t.Fatalf("Expected subscription details for 3 subs, got %d\n", len(sl.Subs)) } } } func TestSubszWithOffsetAndLimit(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() for i := 0; i < 200; i++ { nc.Subscribe(fmt.Sprintf("foo.%d", i), func(m *nats.Msg) {}) } nc.Flush() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz?subs=1&offset=10&limit=100", &SubszOptions{Subscriptions: true, Offset: 10, Limit: 100}) if sl.NumSubs != 200 { t.Fatalf("Expected NumSubs of 200, got %d\n", sl.NumSubs) } if sl.Total != 100 { t.Fatalf("Expected Total of 100, got %d\n", sl.Total) } if sl.Offset != 10 { t.Fatalf("Expected Offset of 10, got %d\n", sl.Offset) } if sl.Limit != 100 { t.Fatalf("Expected Total of 100, got %d\n", sl.Limit) } if len(sl.Subs) != 100 { t.Fatalf("Expected subscription details for 100 subs, got %d\n", len(sl.Subs)) } } } func TestSubszTestPubSubject(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() nc.Subscribe("foo.*", func(m *nats.Msg) {}) nc.Subscribe("foo.bar", func(m *nats.Msg) {}) nc.Subscribe("foo.foo", func(m *nats.Msg) {}) nc.Flush() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz?subs=1&test=foo.foo", &SubszOptions{Subscriptions: true, Test: "foo.foo"}) if sl.Total != 2 { t.Fatalf("Expected Total of 2 match, got %d\n", sl.Total) } if len(sl.Subs) != 2 { t.Fatalf("Expected subscription details for 2 matching subs, got %d\n", len(sl.Subs)) } sl = pollSubsz(t, s, mode, url+"subsz?subs=1&test=foo", &SubszOptions{Subscriptions: true, Test: "foo"}) if len(sl.Subs) != 0 { t.Fatalf("Expected no matching subs, got %d\n", len(sl.Subs)) } } // Make sure we get an error with invalid test subject. testUrl := url + "subsz?subs=1&" readBodyEx(t, testUrl+"test=*", http.StatusBadRequest, textPlain) readBodyEx(t, testUrl+"test=foo.*", http.StatusBadRequest, textPlain) readBodyEx(t, testUrl+"test=foo.>", http.StatusBadRequest, textPlain) readBodyEx(t, testUrl+"test=foo..bar", http.StatusBadRequest, textPlain) } func TestSubszMultiAccount(t *testing.T) { s := runMonitorServerWithAccounts() defer s.Shutdown() ncA := createClientConnWithUserSubscribeAndPublish(t, s, "a", "a") defer ncA.Close() ncA.Subscribe("foo.*", func(m *nats.Msg) {}) ncA.Subscribe("foo.bar", func(m *nats.Msg) {}) ncA.Subscribe("foo.foo", func(m *nats.Msg) {}) ncA.Publish("foo.bar", []byte("Hello")) ncA.Publish("foo.baz", []byte("Hello")) ncA.Publish("foo.foo", []byte("Hello")) ncA.Flush() ncB := createClientConnWithUserSubscribeAndPublish(t, s, "b", "b") defer ncB.Close() ncB.Subscribe("foo.*", func(m *nats.Msg) {}) ncB.Subscribe("foo.bar", func(m *nats.Msg) {}) ncB.Subscribe("foo.foo", func(m *nats.Msg) {}) ncB.Publish("foo.bar", []byte("Hello")) ncB.Publish("foo.baz", []byte("Hello")) ncB.Publish("foo.foo", []byte("Hello")) ncB.Flush() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz?subs=1", &SubszOptions{Subscriptions: true}) if sl.NumSubs != 6 { t.Fatalf("Expected NumSubs of 6, got %d\n", sl.NumSubs) } if sl.Total != 6 { t.Fatalf("Expected Total of 6, got %d\n", sl.Total) } if len(sl.Subs) != 6 { t.Fatalf("Expected subscription details for 6 subs, got %d\n", len(sl.Subs)) } for _, sd := range sl.Subs { if sd.Account != "A" && sd.Account != "B" { t.Fatalf("Expected account information to be present and be 'A' or 'B', got %q", sd.Account) } } // Now make sure we can filter on account. sl = pollSubsz(t, s, mode, url+"subsz?subs=1&acc=A", &SubszOptions{Account: "A", Subscriptions: true}) if sl.NumSubs != 3 { t.Fatalf("Expected NumSubs of 3, got %d\n", sl.NumSubs) } if sl.Total != 3 { t.Fatalf("Expected Total of 6, got %d\n", sl.Total) } if len(sl.Subs) != 3 { t.Fatalf("Expected subscription details for 6 subs, got %d\n", len(sl.Subs)) } for _, sd := range sl.Subs { if sd.Account != "A" { t.Fatalf("Expected account information to be present and be 'A', got %q", sd.Account) } } } } func TestSubszMultiAccountWithOffsetAndLimit(t *testing.T) { s := runMonitorServer() defer s.Shutdown() ncA := createClientConnWithUserSubscribeAndPublish(t, s, "a", "a") defer ncA.Close() for i := 0; i < 200; i++ { ncA.Subscribe(fmt.Sprintf("foo.%d", i), func(m *nats.Msg) {}) } ncA.Flush() ncB := createClientConnWithUserSubscribeAndPublish(t, s, "b", "b") defer ncB.Close() for i := 0; i < 200; i++ { ncB.Subscribe(fmt.Sprintf("foo.%d", i), func(m *nats.Msg) {}) } ncB.Flush() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { sl := pollSubsz(t, s, mode, url+"subsz?subs=1&offset=10&limit=100", &SubszOptions{Subscriptions: true, Offset: 10, Limit: 100}) if sl.NumSubs != 400 { t.Fatalf("Expected NumSubs of 200, got %d\n", sl.NumSubs) } if sl.Total != 100 { t.Fatalf("Expected Total of 100, got %d\n", sl.Total) } if sl.Offset != 10 { t.Fatalf("Expected Offset of 10, got %d\n", sl.Offset) } if sl.Limit != 100 { t.Fatalf("Expected Total of 100, got %d\n", sl.Limit) } if len(sl.Subs) != 100 { t.Fatalf("Expected subscription details for 100 subs, got %d\n", len(sl.Subs)) } } } // Tests handle root func TestHandleRoot(t *testing.T) { s := runMonitorServer() defer s.Shutdown() nc := createClientConnSubscribeAndPublish(t, s) defer nc.Close() resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port)) if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("Expected a %d response, got %d\n", http.StatusOK, resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Expected no error reading body: Got %v\n", err) } for _, b := range body { if b > unicode.MaxASCII { t.Fatalf("Expected body to contain only ASCII characters, but got %v\n", b) } } ct := resp.Header.Get("Content-Type") if !strings.Contains(ct, "text/html") { t.Fatalf("Expected text/html response, got %s\n", ct) } } func TestConnzWithNamedClient(t *testing.T) { s := runMonitorServer() defer s.Shutdown() clientName := "test-client" nc := createClientConnWithName(t, clientName, s) defer nc.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { // Confirm server is exposing client name in monitoring endpoint. c := pollConz(t, s, mode, url+"connz", nil) got := len(c.Conns) expected := 1 if got != expected { t.Fatalf("Expected %d connection in array, got %d\n", expected, got) } conn := c.Conns[0] if conn.Name != clientName { t.Fatalf("Expected client to have name %q. got %q", clientName, conn.Name) } } } func TestConnzWithStateForClosedConns(t *testing.T) { s := runMonitorServer() defer s.Shutdown() numEach := 10 // Create 10 closed, and 10 to leave open. for i := 0; i < numEach; i++ { nc := createClientConnSubscribeAndPublish(t, s) nc.Subscribe("hello.closed.conns", func(m *nats.Msg) {}) nc.Close() nc = createClientConnSubscribeAndPublish(t, s) nc.Subscribe("hello.open.conns", func(m *nats.Msg) {}) defer nc.Close() } url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { // Look at all open c := pollConz(t, s, mode, url+"connz?state=open", &ConnzOptions{State: ConnOpen}) if lc := len(c.Conns); lc != numEach { return fmt.Errorf("Expected %d connections in array, got %d", numEach, lc) } // Look at all closed c = pollConz(t, s, mode, url+"connz?state=closed", &ConnzOptions{State: ConnClosed}) if lc := len(c.Conns); lc != numEach { return fmt.Errorf("Expected %d connections in array, got %d", numEach, lc) } // Look at all c = pollConz(t, s, mode, url+"connz?state=ALL", &ConnzOptions{State: ConnAll}) if lc := len(c.Conns); lc != numEach*2 { return fmt.Errorf("Expected %d connections in array, got %d", 2*numEach, lc) } // Look at CID #1, which is in closed. c = pollConz(t, s, mode, url+"connz?cid=1&state=open", &ConnzOptions{CID: 1, State: ConnOpen}) if lc := len(c.Conns); lc != 0 { return fmt.Errorf("Expected no connections in open array, got %d", lc) } c = pollConz(t, s, mode, url+"connz?cid=1&state=closed", &ConnzOptions{CID: 1, State: ConnClosed}) if lc := len(c.Conns); lc != 1 { return fmt.Errorf("Expected a connection in closed array, got %d", lc) } c = pollConz(t, s, mode, url+"connz?cid=1&state=ALL", &ConnzOptions{CID: 1, State: ConnAll}) if lc := len(c.Conns); lc != 1 { return fmt.Errorf("Expected a connection in closed array, got %d", lc) } c = pollConz(t, s, mode, url+"connz?cid=1&state=closed&subs=true", &ConnzOptions{CID: 1, State: ConnClosed, Subscriptions: true}) if lc := len(c.Conns); lc != 1 { return fmt.Errorf("Expected a connection in closed array, got %d", lc) } ci := c.Conns[0] if ci.NumSubs != 1 { return fmt.Errorf("Expected NumSubs to be 1, got %d", ci.NumSubs) } if len(ci.Subs) != 1 { return fmt.Errorf("Expected len(ci.Subs) to be 1 also, got %d", len(ci.Subs)) } // Now ask for same thing without subs and make sure they are not returned. c = pollConz(t, s, mode, url+"connz?cid=1&state=closed&subs=false", &ConnzOptions{CID: 1, State: ConnClosed, Subscriptions: false}) if lc := len(c.Conns); lc != 1 { return fmt.Errorf("Expected a connection in closed array, got %d", lc) } ci = c.Conns[0] if ci.NumSubs != 1 { return fmt.Errorf("Expected NumSubs to be 1, got %d", ci.NumSubs) } if len(ci.Subs) != 0 { return fmt.Errorf("Expected len(ci.Subs) to be 0 since subs=false, got %d", len(ci.Subs)) } // CID #2 is in open c = pollConz(t, s, mode, url+"connz?cid=2&state=open", &ConnzOptions{CID: 2, State: ConnOpen}) if lc := len(c.Conns); lc != 1 { return fmt.Errorf("Expected a connection in open array, got %d", lc) } c = pollConz(t, s, mode, url+"connz?cid=2&state=closed", &ConnzOptions{CID: 2, State: ConnClosed}) if lc := len(c.Conns); lc != 0 { return fmt.Errorf("Expected no connections in closed array, got %d", lc) } return nil }) } } // Make sure options for ConnInfo like subs=1, authuser, etc do not cause a race. func TestConnzClosedConnsRace(t *testing.T) { s := runMonitorServer() defer s.Shutdown() // Create 100 closed connections. for i := 0; i < 100; i++ { nc := createClientConnSubscribeAndPublish(t, s) nc.Close() } urlWithoutSubs := fmt.Sprintf("http://127.0.0.1:%d/connz?state=closed", s.MonitorAddr().Port) urlWithSubs := urlWithoutSubs + "&subs=true" checkClosedConns(t, s, 100, 2*time.Second) wg := &sync.WaitGroup{} fn := func(url string) { deadline := time.Now().Add(1 * time.Second) for time.Now().Before(deadline) { c := pollConz(t, s, 0, url, nil) if len(c.Conns) != 100 { t.Errorf("Incorrect Results: %+v\n", c) } } wg.Done() } wg.Add(2) go fn(urlWithSubs) go fn(urlWithoutSubs) wg.Wait() } // Make sure a bad client that is disconnected right away has proper values. func TestConnzClosedConnsBadClient(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := s.getOpts() rc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on dial: %v", err) } rc.Close() checkClosedConns(t, s, 1, 2*time.Second) c := pollConz(t, s, 1, "", &ConnzOptions{State: ConnClosed}) if len(c.Conns) != 1 { t.Errorf("Incorrect Results: %+v\n", c) } ci := c.Conns[0] uptime := ci.Stop.Sub(ci.Start) idle, err := time.ParseDuration(ci.Idle) if err != nil { t.Fatalf("Could not parse Idle: %v\n", err) } if idle > uptime { t.Fatalf("Idle can't be larger then uptime, %v vs %v\n", idle, uptime) } if ci.LastActivity.IsZero() { t.Fatalf("LastActivity should not be Zero\n") } } // Make sure a bad client that tries to connect plain to TLS has proper values. func TestConnzClosedConnsBadTLSClient(t *testing.T) { resetPreviousHTTPConnections() tc := &TLSConfigOpts{} tc.CertFile = "configs/certs/server.pem" tc.KeyFile = "configs/certs/key.pem" var err error opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.TLSTimeout = 1.5 // 1.5 seconds opts.TLSConfig, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating TSL config: %v", err) } s := RunServer(opts) defer s.Shutdown() opts = s.getOpts() rc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on dial: %v", err) } rc.Write([]byte("CONNECT {}\r\n")) rc.Close() checkClosedConns(t, s, 1, 2*time.Second) c := pollConz(t, s, 1, "", &ConnzOptions{State: ConnClosed}) if len(c.Conns) != 1 { t.Errorf("Incorrect Results: %+v\n", c) } ci := c.Conns[0] uptime := ci.Stop.Sub(ci.Start) idle, err := time.ParseDuration(ci.Idle) if err != nil { t.Fatalf("Could not parse Idle: %v\n", err) } if idle > uptime { t.Fatalf("Idle can't be larger then uptime, %v vs %v\n", idle, uptime) } if ci.LastActivity.IsZero() { t.Fatalf("LastActivity should not be Zero\n") } } // Create a connection to test ConnInfo func createClientConnWithUserSubscribeAndPublish(t *testing.T, s *Server, user, pwd string) *nats.Conn { natsURL := "" if user == "" { natsURL = fmt.Sprintf("nats://127.0.0.1:%d", s.Addr().(*net.TCPAddr).Port) } else { natsURL = fmt.Sprintf("nats://%s:%s@127.0.0.1:%d", user, pwd, s.Addr().(*net.TCPAddr).Port) } client := nats.GetDefaultOptions() client.Servers = []string{natsURL} nc, err := client.Connect() if err != nil { t.Fatalf("Error creating client: %v to: %s\n", err, natsURL) } ch := make(chan bool) inbox := nats.NewInbox() sub, err := nc.Subscribe(inbox, func(m *nats.Msg) { ch <- true }) if err != nil { t.Fatalf("Error subscribing to `%s`: %v\n", inbox, err) } nc.Publish(inbox, []byte("Hello")) // Wait for message <-ch sub.Unsubscribe() close(ch) nc.Flush() return nc } func createClientConnSubscribeAndPublish(t *testing.T, s *Server) *nats.Conn { return createClientConnWithUserSubscribeAndPublish(t, s, "", "") } func createClientConnWithName(t *testing.T, name string, s *Server) *nats.Conn { natsURI := fmt.Sprintf("nats://127.0.0.1:%d", s.Addr().(*net.TCPAddr).Port) client := nats.GetDefaultOptions() client.Servers = []string{natsURI} client.Name = name nc, err := client.Connect() if err != nil { t.Fatalf("Error creating client: %v\n", err) } return nc } func TestStacksz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) body := readBody(t, url+"stacksz") // Check content str := string(body) if !strings.Contains(str, "HandleStacksz") { t.Fatalf("Result does not seem to contain server's stacks:\n%v", str) } } func TestConcurrentMonitoring(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) // Get some endpoints. Make sure we have at least varz, // and the more the merrier. endpoints := []string{"varz", "varz", "varz", "connz", "connz", "subsz", "subsz", "routez", "routez"} wg := &sync.WaitGroup{} wg.Add(len(endpoints)) ech := make(chan string, len(endpoints)) for _, e := range endpoints { go func(endpoint string) { defer wg.Done() for i := 0; i < 50; i++ { resp, err := http.Get(url + endpoint) if err != nil { ech <- fmt.Sprintf("Expected no error: Got %v\n", err) return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { ech <- fmt.Sprintf("Expected a %v response, got %d\n", http.StatusOK, resp.StatusCode) return } ct := resp.Header.Get("Content-Type") if ct != "application/json" { ech <- fmt.Sprintf("Expected application/json content-type, got %s\n", ct) return } if _, err := io.ReadAll(resp.Body); err != nil { ech <- fmt.Sprintf("Got an error reading the body: %v\n", err) return } resp.Body.Close() } }(e) } wg.Wait() // Check for any errors select { case err := <-ech: t.Fatal(err) default: } } func TestMonitorHandler(t *testing.T) { s := runMonitorServer() defer s.Shutdown() handler := s.HTTPHandler() if handler == nil { t.Fatal("HTTP Handler should be set") } s.Shutdown() handler = s.HTTPHandler() if handler != nil { t.Fatal("HTTP Handler should be nil") } } func TestMonitorRoutezRace(t *testing.T) { resetPreviousHTTPConnections() srvAOpts := DefaultMonitorOptions() srvAOpts.NoSystemAccount = true srvAOpts.Cluster.Name = "B" srvAOpts.Cluster.Port = -1 srvA := RunServer(srvAOpts) defer srvA.Shutdown() srvBOpts := nextServerOpts(srvAOpts) srvBOpts.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) doneCh := make(chan struct{}) go func() { defer func() { doneCh <- struct{}{} }() for i := 0; i < 10; i++ { time.Sleep(10 * time.Millisecond) // Reset ports srvBOpts.Port = -1 srvBOpts.Cluster.Port = -1 srvB := RunServer(srvBOpts) time.Sleep(20 * time.Millisecond) srvB.Shutdown() } }() done := false for !done { if _, err := srvA.Routez(nil); err != nil { time.Sleep(10 * time.Millisecond) } select { case <-doneCh: done = true default: } } } func TestConnzTLSInHandshake(t *testing.T) { resetPreviousHTTPConnections() tc := &TLSConfigOpts{} tc.CertFile = "configs/certs/server.pem" tc.KeyFile = "configs/certs/key.pem" var err error opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.TLSTimeout = 1.5 // 1.5 seconds opts.TLSConfig, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating TSL config: %v", err) } s := RunServer(opts) defer s.Shutdown() // Create bare TCP connection to delay client TLS handshake c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on dial: %v", err) } defer c.Close() // Wait for the connection to be registered checkClientsCount(t, s, 1) start := time.Now() endpoint := fmt.Sprintf("http://%s:%d/connz", opts.HTTPHost, s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { connz := pollConz(t, s, mode, endpoint, nil) duration := time.Since(start) if duration >= 1500*time.Millisecond { t.Fatalf("Looks like connz blocked on handshake, took %v", duration) } if len(connz.Conns) != 1 { t.Fatalf("Expected 1 conn, got %v", len(connz.Conns)) } conn := connz.Conns[0] // TLS fields should be not set if conn.TLSVersion != "" || conn.TLSCipher != "" { t.Fatalf("Expected TLS fields to not be set, got version:%v cipher:%v", conn.TLSVersion, conn.TLSCipher) } } } func TestConnzTLSCfg(t *testing.T) { resetPreviousHTTPConnections() tc := &TLSConfigOpts{} tc.CertFile = "configs/certs/server.pem" tc.KeyFile = "configs/certs/key.pem" var err error opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.TLSTimeout = 1.5 // 1.5 seconds opts.TLSConfig, err = GenTLSConfig(tc) require_NoError(t, err) opts.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert opts.Gateway.TLSConfig, err = GenTLSConfig(tc) require_NoError(t, err) opts.Gateway.TLSTimeout = 1.5 opts.LeafNode.TLSConfig, err = GenTLSConfig(tc) require_NoError(t, err) opts.LeafNode.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert opts.LeafNode.TLSTimeout = 1.5 opts.Cluster.TLSConfig, err = GenTLSConfig(tc) require_NoError(t, err) opts.Cluster.TLSTimeout = 1.5 s := RunServer(opts) defer s.Shutdown() check := func(verify, required bool, timeout float64) { t.Helper() if !verify { t.Fatalf("Expected tls_verify to be true") } if !required { t.Fatalf("Expected tls_required to be true") } if timeout != 1.5 { t.Fatalf("Expected tls_timeout to be 1.5") } } start := time.Now() endpoint := fmt.Sprintf("http://%s:%d/varz", opts.HTTPHost, s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { varz := pollVarz(t, s, mode, endpoint, nil) duration := time.Since(start) if duration >= 1500*time.Millisecond { t.Fatalf("Looks like varz blocked on handshake, took %v", duration) } check(varz.TLSVerify, varz.TLSRequired, varz.TLSTimeout) check(varz.Cluster.TLSVerify, varz.Cluster.TLSRequired, varz.Cluster.TLSTimeout) check(varz.Gateway.TLSVerify, varz.Gateway.TLSRequired, varz.Gateway.TLSTimeout) check(varz.LeafNode.TLSVerify, varz.LeafNode.TLSRequired, varz.LeafNode.TLSTimeout) } } func TestConnzTLSPeerCerts(t *testing.T) { resetPreviousHTTPConnections() tc := &TLSConfigOpts{} tc.CertFile = "../test/configs/certs/server-cert.pem" tc.KeyFile = "../test/configs/certs/server-key.pem" tc.CaFile = "../test/configs/certs/ca.pem" tc.Verify = true tc.Timeout = 2.0 var err error opts := DefaultMonitorOptions() opts.TLSConfig, err = GenTLSConfig(tc) require_NoError(t, err) s := RunServer(opts) defer s.Shutdown() nc := natsConnect(t, s.ClientURL(), nats.ClientCert("../test/configs/certs/client-cert.pem", "../test/configs/certs/client-key.pem"), nats.RootCAs("../test/configs/certs/ca.pem")) defer nc.Close() endpoint := fmt.Sprintf("http://%s:%d/connz", opts.HTTPHost, s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { // Without "auth" option, we should not get the details connz := pollConz(t, s, mode, endpoint, nil) require_True(t, len(connz.Conns) == 1) c := connz.Conns[0] if c.TLSPeerCerts != nil { t.Fatalf("Did not expect TLSPeerCerts when auth is not specified: %+v", c.TLSPeerCerts) } // Now specify "auth" option connz = pollConz(t, s, mode, endpoint+"?auth=1", &ConnzOptions{Username: true}) require_True(t, len(connz.Conns) == 1) c = connz.Conns[0] if c.TLSPeerCerts == nil { t.Fatal("Expected TLSPeerCerts to be set, was not") } else if len(c.TLSPeerCerts) != 1 { t.Fatalf("Unexpected peer certificates: %+v", c.TLSPeerCerts) } else { for _, d := range c.TLSPeerCerts { if d.Subject != "CN=localhost,OU=nats.io,O=Synadia,ST=California,C=US" { t.Fatalf("Unexpected subject: %s", d.Subject) } if len(d.SubjectPKISha256) != 64 { t.Fatalf("Unexpected spki_sha256: %s", d.SubjectPKISha256) } if len(d.CertSha256) != 64 { t.Fatalf("Unexpected cert_sha256: %s", d.CertSha256) } } } } } func TestServerIDs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() murl := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, murl+"varz", nil) if v.ID == _EMPTY_ { t.Fatal("Varz ID is empty") } c := pollConz(t, s, mode, murl+"connz", nil) if c.ID == _EMPTY_ { t.Fatal("Connz ID is empty") } r := pollRoutez(t, s, mode, murl+"routez", nil) if r.ID == _EMPTY_ { t.Fatal("Routez ID is empty") } if v.ID != c.ID || v.ID != r.ID { t.Fatalf("Varz ID [%s] is not equal to Connz ID [%s] or Routez ID [%s]", v.ID, c.ID, r.ID) } } } func TestHttpStatsNoUpdatedWhenUsingServerFuncs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() for i := 0; i < 10; i++ { s.Varz(nil) s.Connz(nil) s.Routez(nil) s.Subsz(nil) } v, _ := s.Varz(nil) endpoints := []string{VarzPath, ConnzPath, RoutezPath, SubszPath} for _, e := range endpoints { stats := v.HTTPReqStats[e] if stats != 0 { t.Fatalf("Expected HTTPReqStats for %q to be 0, got %v", e, stats) } } } func TestClusterEmptyWhenNotDefined(t *testing.T) { s := runMonitorServer() defer s.Shutdown() body := readBody(t, fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port)) var v map[string]any if err := json.Unmarshal(body, &v); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Cluster can empty, or be defined but that needs to be empty. c, ok := v["cluster"] if !ok { return } if len(c.(map[string]any)) != 0 { t.Fatalf("Expected an empty cluster definition, instead got %+v\n", c) } } func TestRoutezPermissions(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.Cluster.Name = "A" opts.Cluster.Host = "127.0.0.1" opts.Cluster.Port = -1 opts.Cluster.Permissions = &RoutePermissions{ Import: &SubjectPermission{ Allow: []string{"foo"}, }, Export: &SubjectPermission{ Allow: []string{"*"}, Deny: []string{"foo", "nats"}, }, } s1 := RunServer(opts) defer s1.Shutdown() opts = DefaultMonitorOptions() opts.NoSystemAccount = true opts.ServerName = "monitor_server_2" opts.Cluster.Host = "127.0.0.1" opts.Cluster.Name = "A" opts.Cluster.Port = -1 routeURL, _ := url.Parse(fmt.Sprintf("nats-route://127.0.0.1:%d", s1.ClusterAddr().Port)) opts.Routes = []*url.URL{routeURL} opts.HTTPPort = -1 s2 := RunServer(opts) defer s2.Shutdown() checkClusterFormed(t, s1, s2) urls := []string{ fmt.Sprintf("http://127.0.0.1:%d/routez", s1.MonitorAddr().Port), fmt.Sprintf("http://127.0.0.1:%d/routez", s2.MonitorAddr().Port), } servers := []*Server{s1, s2} for i, url := range urls { for mode := 0; mode < 2; mode++ { rz := pollRoutez(t, servers[i], mode, url, nil) // For server 1, we expect to see imports and exports if i == 0 { if rz.Import == nil || rz.Import.Allow == nil || len(rz.Import.Allow) != 1 || rz.Import.Allow[0] != "foo" || rz.Import.Deny != nil { t.Fatalf("Unexpected Import %v", rz.Import) } if rz.Export == nil || rz.Export.Allow == nil || rz.Export.Deny == nil || len(rz.Export.Allow) != 1 || rz.Export.Allow[0] != "*" || len(rz.Export.Deny) != 2 || rz.Export.Deny[0] != "foo" || rz.Export.Deny[1] != "nats" { t.Fatalf("Unexpected Export %v", rz.Export) } } else { // We expect to see NO imports and exports for server B by default. if rz.Import != nil { t.Fatal("Routez body should NOT contain \"import\" information.") } if rz.Export != nil { t.Fatal("Routez body should NOT contain \"export\" information.") } // We do expect to see them show up for the information we have on Server A though. if len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected route array of %d, got %v\n", DEFAULT_ROUTE_POOL_SIZE, len(rz.Routes)) } route := rz.Routes[0] if route.Import == nil || route.Import.Allow == nil || len(route.Import.Allow) != 1 || route.Import.Allow[0] != "foo" || route.Import.Deny != nil { t.Fatalf("Unexpected Import %v", route.Import) } if route.Export == nil || route.Export.Allow == nil || route.Export.Deny == nil || len(route.Export.Allow) != 1 || route.Export.Allow[0] != "*" || len(route.Export.Deny) != 2 || route.Export.Deny[0] != "foo" || route.Export.Deny[1] != "nats" { t.Fatalf("Unexpected Export %v", route.Export) } } } } } // Benchmark our Connz generation. Don't use HTTP here, just measure server endpoint. func Benchmark_Connz(b *testing.B) { runtime.MemProfileRate = 0 s := runMonitorServerNoHTTPPort() defer s.Shutdown() opts := s.getOpts() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // Create 250 connections with 100 subs each. for i := 0; i < 250; i++ { nc, err := nats.Connect(url) if err != nil { b.Fatalf("Error on connection[%d] to %s: %v", i, url, err) } for x := 0; x < 100; x++ { subj := fmt.Sprintf("foo.%d", x) nc.Subscribe(subj, func(m *nats.Msg) {}) } nc.Flush() defer nc.Close() } b.ResetTimer() runtime.MemProfileRate = 1 copts := &ConnzOptions{Subscriptions: false} for i := 0; i < b.N; i++ { _, err := s.Connz(copts) if err != nil { b.Fatalf("Error on Connz(): %v", err) } } } func Benchmark_Varz(b *testing.B) { runtime.MemProfileRate = 0 s := runMonitorServerNoHTTPPort() defer s.Shutdown() b.ResetTimer() runtime.MemProfileRate = 1 for i := 0; i < b.N; i++ { _, err := s.Varz(nil) if err != nil { b.Fatalf("Error on Connz(): %v", err) } } } func Benchmark_VarzHttp(b *testing.B) { runtime.MemProfileRate = 0 s := runMonitorServer() defer s.Shutdown() murl := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) b.ResetTimer() runtime.MemProfileRate = 1 for i := 0; i < b.N; i++ { v := &Varz{} resp, err := http.Get(murl) if err != nil { b.Fatalf("Expected no error: Got %v\n", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { b.Fatalf("Got an error reading the body: %v\n", err) } if err := json.Unmarshal(body, v); err != nil { b.Fatalf("Got an error unmarshalling the body: %v\n", err) } resp.Body.Close() } } func TestVarzRaces(t *testing.T) { s := runMonitorServer() defer s.Shutdown() murl := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) done := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { for i := 0; i < 2; i++ { v := pollVarz(t, s, i, murl, nil) // Check the field that we are setting in main thread // to ensure that we have a copy and there is no // race with fields set in s.info and s.opts if v.ID == "abc" || v.MaxConn == -1 { // We will not get there. Need to have something // otherwise staticcheck will report empty branch return } select { case <-done: return default: } } } }() for i := 0; i < 1000; i++ { // Simulate a change in server's info and options // by changing something. s.mu.Lock() s.info.ID = fmt.Sprintf("serverid_%d", i) s.opts.MaxConn = 100 + i s.mu.Unlock() time.Sleep(time.Nanosecond) } close(done) wg.Wait() // Now check that there is no race doing parallel polling wg.Add(3) done = make(chan struct{}) poll := func() { defer wg.Done() for { for mode := 0; mode < 2; mode++ { pollVarz(t, s, mode, murl, nil) } select { case <-done: return default: } } } for i := 0; i < 3; i++ { go poll() } time.Sleep(500 * time.Millisecond) close(done) wg.Wait() } func testMonitorStructPresent(t *testing.T, tag string) { t.Helper() resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true s := RunServer(opts) defer s.Shutdown() varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) body := readBody(t, varzURL) if !bytes.Contains(body, []byte(`"`+tag+`": {}`)) { t.Fatalf("%s should be present and empty, got %s", tag, body) } } func TestMonitorCluster(t *testing.T) { testMonitorStructPresent(t, "cluster") resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.Cluster.Name = "A" opts.Cluster.Port = -1 opts.Cluster.AuthTimeout = 1 opts.Routes = RoutesFromStr("nats://127.0.0.1:1234") s := RunServer(opts) defer s.Shutdown() expected := ClusterOptsVarz{ "A", opts.Cluster.Host, opts.Cluster.Port, opts.Cluster.AuthTimeout, []string{"127.0.0.1:1234"}, opts.Cluster.TLSTimeout, opts.Cluster.TLSConfig != nil, opts.Cluster.TLSConfig != nil, DEFAULT_ROUTE_POOL_SIZE, } varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { check := func(t *testing.T, v *Varz) { t.Helper() if !reflect.DeepEqual(v.Cluster, expected) { t.Fatalf("mode=%v - expected %+v, got %+v", mode, expected, v.Cluster) } } v := pollVarz(t, s, mode, varzURL, nil) check(t, v) // Having this here to make sure that if fields are added in ClusterOptsVarz, // we make sure to update this test (compiler will report an error if we don't) _ = ClusterOptsVarz{"", "", 0, 0, nil, 2, false, false, 0} // Alter the fields to make sure that we have a proper deep copy // of what may be stored in the server. Anything we change here // should not affect the next returned value. v.Cluster.Name = "wrong" v.Cluster.Host = "wrong" v.Cluster.Port = 0 v.Cluster.AuthTimeout = 0 v.Cluster.URLs = []string{"wrong"} v = pollVarz(t, s, mode, varzURL, nil) check(t, v) } } func TestMonitorClusterURLs(t *testing.T) { resetPreviousHTTPConnections() o2 := DefaultOptions() o2.Cluster.Host = "127.0.0.1" o2.Cluster.Name = "A" s2 := RunServer(o2) defer s2.Shutdown() s2ClusterHostPort := fmt.Sprintf("127.0.0.1:%d", s2.ClusterAddr().Port) template := ` port: -1 http: -1 cluster: { name: "A" port: -1 routes [ %s %s ] } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, "nats://"+s2ClusterHostPort, ""))) s1, _ := RunServerWithConfig(conf) defer s1.Shutdown() checkClusterFormed(t, s1, s2) // Check /varz cluster{} to see the URLs from s1 to s2 varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", s1.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s1, mode, varzURL, nil) if n := len(v.Cluster.URLs); n != 1 { t.Fatalf("mode=%v - Expected 1 URL, got %v", mode, n) } if v.Cluster.URLs[0] != s2ClusterHostPort { t.Fatalf("mode=%v - Expected url %q, got %q", mode, s2ClusterHostPort, v.Cluster.URLs[0]) } } otherClusterHostPort := "127.0.0.1:1234" // Now update the config and add a route changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, "nats://"+s2ClusterHostPort, "nats://"+otherClusterHostPort))) if err := s1.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } // Verify cluster still ok checkClusterFormed(t, s1, s2) // Now verify that s1 reports in /varz the new URL checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for mode := 0; mode < 2; mode++ { v := pollVarz(t, s1, mode, varzURL, nil) if n := len(v.Cluster.URLs); n != 2 { t.Fatalf("mode=%v - Expected 2 URL, got %v", mode, n) } gotS2 := false gotOther := false for _, u := range v.Cluster.URLs { if u == s2ClusterHostPort { gotS2 = true } else if u == otherClusterHostPort { gotOther = true } else { t.Fatalf("mode=%v - Incorrect url: %q", mode, u) } } if !gotS2 { t.Fatalf("mode=%v - Did not get cluster URL for s2", mode) } if !gotOther { t.Fatalf("mode=%v - Did not get the new cluster URL", mode) } } return nil }) // Remove all routes from config changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, "", ""))) if err := s1.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } // Now verify that s1 reports no ULRs in /varz checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for mode := 0; mode < 2; mode++ { v := pollVarz(t, s1, mode, varzURL, nil) if n := len(v.Cluster.URLs); n != 0 { t.Fatalf("mode=%v - Expected 0 URL, got %v", mode, n) } } return nil }) } func TestMonitorGateway(t *testing.T) { testMonitorStructPresent(t, "gateway") resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.Gateway.Name = "A" opts.Gateway.Port = -1 opts.Gateway.AuthTimeout = 1 opts.Gateway.TLSTimeout = 1 opts.Gateway.Advertise = "127.0.0.1" opts.Gateway.ConnectRetries = 1 opts.Gateway.RejectUnknown = false u1, _ := url.Parse("nats://ivan:pwd@localhost:1234") u2, _ := url.Parse("nats://localhost:1235") opts.Gateway.Gateways = []*RemoteGatewayOpts{ { Name: "B", TLSTimeout: 1, URLs: []*url.URL{ u1, u2, }, }, } s := RunServer(opts) defer s.Shutdown() expected := GatewayOptsVarz{ "A", opts.Gateway.Host, opts.Gateway.Port, opts.Gateway.AuthTimeout, opts.Gateway.TLSTimeout, opts.Gateway.TLSConfig != nil, opts.Gateway.TLSConfig != nil, opts.Gateway.Advertise, opts.Gateway.ConnectRetries, []RemoteGatewayOptsVarz{{"B", 1, nil}}, opts.Gateway.RejectUnknown, } // Since URLs array is not guaranteed to be always the same order, // we don't add it in the expected GatewayOptsVarz, instead we // maintain here. expectedURLs := []string{"localhost:1234", "localhost:1235"} varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { check := func(t *testing.T, v *Varz) { t.Helper() var urls []string if len(v.Gateway.Gateways) == 1 { urls = v.Gateway.Gateways[0].URLs v.Gateway.Gateways[0].URLs = nil } if !reflect.DeepEqual(v.Gateway, expected) { t.Fatalf("mode=%v - expected %+v, got %+v", mode, expected, v.Gateway) } // Now compare urls for _, u := range expectedURLs { ok := false for _, u2 := range urls { if u == u2 { ok = true break } } if !ok { t.Fatalf("mode=%v - expected urls to be %v, got %v", mode, expected.Gateways[0].URLs, urls) } } } v := pollVarz(t, s, mode, varzURL, nil) check(t, v) // Having this here to make sure that if fields are added in GatewayOptsVarz, // we make sure to update this test (compiler will report an error if we don't) _ = GatewayOptsVarz{"", "", 0, 0, 0, false, false, "", 0, []RemoteGatewayOptsVarz{{"", 0, nil}}, false} // Alter the fields to make sure that we have a proper deep copy // of what may be stored in the server. Anything we change here // should not affect the next returned value. v.Gateway.Name = "wrong" v.Gateway.Host = "wrong" v.Gateway.Port = 0 v.Gateway.AuthTimeout = 1234.5 v.Gateway.TLSTimeout = 1234.5 v.Gateway.Advertise = "wrong" v.Gateway.ConnectRetries = 1234 v.Gateway.Gateways[0].Name = "wrong" v.Gateway.Gateways[0].TLSTimeout = 1234.5 v.Gateway.Gateways[0].URLs = []string{"wrong"} v.Gateway.RejectUnknown = true v = pollVarz(t, s, mode, varzURL, nil) check(t, v) } } func TestMonitorGatewayURLsUpdated(t *testing.T) { resetPreviousHTTPConnections() ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() // Start a1 that has a single URL to sb1. oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) oa.HTTPHost = "127.0.0.1" oa.HTTPPort = MONITOR_PORT sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", sa.MonitorAddr().Port) // Check the /varz gateway's URLs for mode := 0; mode < 2; mode++ { v := pollVarz(t, sa, mode, varzURL, nil) if n := len(v.Gateway.Gateways); n != 1 { t.Fatalf("mode=%v - Expected 1 remote gateway, got %v", mode, n) } gw := v.Gateway.Gateways[0] if n := len(gw.URLs); n != 1 { t.Fatalf("mode=%v - Expected 1 url, got %v", mode, n) } expected := oa.Gateway.Gateways[0].URLs[0].Host if u := gw.URLs[0]; u != expected { t.Fatalf("mode=%v - Expected URL %q, got %q", mode, expected, u) } } // Now start sb2 that clusters with sb1. sa should add to its list of URLs // sb2 gateway's connect URL. ob2 := testDefaultOptionsForGateway("B") ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) sb2 := runGatewayServer(ob2) defer sb2.Shutdown() // Wait for sb1 and sb2 to connect checkClusterFormed(t, sb1, sb2) // sb2 should be made aware of gateway A and connect to sa waitForInboundGateways(t, sa, 2, 2*time.Second) // Now check that URLs in /varz get updated checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for mode := 0; mode < 2; mode++ { v := pollVarz(t, sa, mode, varzURL, nil) if n := len(v.Gateway.Gateways); n != 1 { return fmt.Errorf("mode=%v - Expected 1 remote gateway, got %v", mode, n) } gw := v.Gateway.Gateways[0] if n := len(gw.URLs); n != 2 { return fmt.Errorf("mode=%v - Expected 2 urls, got %v", mode, n) } gotSB1 := false gotSB2 := false for _, u := range gw.URLs { if u == fmt.Sprintf("127.0.0.1:%d", sb1.GatewayAddr().Port) { gotSB1 = true } else if u == fmt.Sprintf("127.0.0.1:%d", sb2.GatewayAddr().Port) { gotSB2 = true } else { return fmt.Errorf("mode=%v - Incorrect URL to gateway B: %v", mode, u) } } if !gotSB1 { return fmt.Errorf("mode=%v - Did not get URL to sb1", mode) } if !gotSB2 { return fmt.Errorf("mode=%v - Did not get URL to sb2", mode) } } return nil }) // Now stop sb2 and make sure that its removal is reflected in varz. sb2.Shutdown() // Wait for it to disappear from sa. waitForInboundGateways(t, sa, 1, 2*time.Second) // Now check that URLs in /varz get updated. checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for mode := 0; mode < 2; mode++ { v := pollVarz(t, sa, mode, varzURL, nil) if n := len(v.Gateway.Gateways); n != 1 { return fmt.Errorf("mode=%v - Expected 1 remote gateway, got %v", mode, n) } gw := v.Gateway.Gateways[0] if n := len(gw.URLs); n != 1 { return fmt.Errorf("mode=%v - Expected 1 url, got %v", mode, n) } u := gw.URLs[0] if u != fmt.Sprintf("127.0.0.1:%d", sb1.GatewayAddr().Port) { return fmt.Errorf("mode=%v - Did not get URL to sb1", mode) } } return nil }) } func TestMonitorGatewayReportItsOwnURLs(t *testing.T) { resetPreviousHTTPConnections() // In this test, we show that if a server has its own gateway information // as a remote (which is the case when remote gateway definitions is copied // on all clusters), we display the defined URLs. oa := testGatewayOptionsFromToWithURLs(t, "A", "A", []string{"nats://127.0.0.1:1234", "nats://127.0.0.1:1235"}) oa.HTTPHost = "127.0.0.1" oa.HTTPPort = MONITOR_PORT sa := runGatewayServer(oa) defer sa.Shutdown() varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", sa.MonitorAddr().Port) // Check the /varz gateway's URLs for mode := 0; mode < 2; mode++ { v := pollVarz(t, sa, mode, varzURL, nil) if n := len(v.Gateway.Gateways); n != 1 { t.Fatalf("mode=%v - Expected 1 remote gateway, got %v", mode, n) } gw := v.Gateway.Gateways[0] if n := len(gw.URLs); n != 2 { t.Fatalf("mode=%v - Expected 2 urls, got %v", mode, gw.URLs) } expected := []string{"127.0.0.1:1234", "127.0.0.1:1235"} if !reflect.DeepEqual(gw.URLs, expected) { t.Fatalf("mode=%v - Expected URLs %q, got %q", mode, expected, gw.URLs) } } } func TestMonitorLeafNode(t *testing.T) { testMonitorStructPresent(t, "leaf") resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.NoSystemAccount = true opts.LeafNode.Port = -1 opts.LeafNode.AuthTimeout = 1 opts.LeafNode.TLSTimeout = 1 opts.Accounts = []*Account{NewAccount("acc")} u, _ := url.Parse("nats://ivan:pwd@localhost:1234") opts.LeafNode.Remotes = []*RemoteLeafOpts{ { LocalAccount: "acc", URLs: []*url.URL{u}, TLSTimeout: 1, }, } s := RunServer(opts) defer s.Shutdown() expected := LeafNodeOptsVarz{ opts.LeafNode.Host, opts.LeafNode.Port, opts.LeafNode.AuthTimeout, opts.LeafNode.TLSTimeout, opts.LeafNode.TLSConfig != nil, opts.LeafNode.TLSConfig != nil, []RemoteLeafOptsVarz{ { "acc", 1, []string{"localhost:1234"}, nil, false, }, }, false, } varzURL := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { check := func(t *testing.T, v *Varz) { t.Helper() if !reflect.DeepEqual(v.LeafNode, expected) { t.Fatalf("mode=%v - expected %+v, got %+v", mode, expected, v.LeafNode) } } v := pollVarz(t, s, mode, varzURL, nil) check(t, v) // Having this here to make sure that if fields are added in ClusterOptsVarz, // we make sure to update this test (compiler will report an error if we don't) _ = LeafNodeOptsVarz{"", 0, 0, 0, false, false, []RemoteLeafOptsVarz{{"", 0, nil, nil, false}}, false} // Alter the fields to make sure that we have a proper deep copy // of what may be stored in the server. Anything we change here // should not affect the next returned value. v.LeafNode.Host = "wrong" v.LeafNode.Port = 0 v.LeafNode.AuthTimeout = 1234.5 v.LeafNode.TLSTimeout = 1234.5 v.LeafNode.Remotes[0].LocalAccount = "wrong" v.LeafNode.Remotes[0].URLs = append(v.LeafNode.Remotes[0].URLs, "wrong") v.LeafNode.Remotes[0].TLSTimeout = 1234.5 v = pollVarz(t, s, mode, varzURL, nil) check(t, v) } } func pollGatewayz(t *testing.T, s *Server, mode int, url string, opts *GatewayzOptions) *Gatewayz { t.Helper() if mode == 0 { g := &Gatewayz{} body := readBody(t, url) if err := json.Unmarshal(body, g); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return g } g, err := s.Gatewayz(opts) if err != nil { t.Fatalf("Error on Gatewayz: %v", err) } return g } func TestMonitorGatewayz(t *testing.T) { resetPreviousHTTPConnections() // First check that without gateway configured s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/gatewayz", s.MonitorAddr().Port) for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, s, pollMode, url, nil) // Expect Name and port to be empty if g.Name != _EMPTY_ || g.Port != 0 { t.Fatalf("Expected no gateway, got %+v", g) } } s.Shutdown() ob1 := testDefaultOptionsForGateway("B") sb1 := runGatewayServer(ob1) defer sb1.Shutdown() // Start a1 that has a single URL to sb1. oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb1) oa.HTTPHost = "127.0.0.1" oa.HTTPPort = MONITOR_PORT sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) waitForInboundGateways(t, sb1, 1, 2*time.Second) gatewayzURL := fmt.Sprintf("http://127.0.0.1:%d/gatewayz", sa.MonitorAddr().Port) for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sa, pollMode, gatewayzURL, nil) if g.Host != oa.Gateway.Host { t.Fatalf("mode=%v - Expected host to be %q, got %q", pollMode, oa.Gateway.Host, g.Host) } if g.Port != oa.Gateway.Port { t.Fatalf("mode=%v - Expected port to be %v, got %v", pollMode, oa.Gateway.Port, g.Port) } if n := len(g.OutboundGateways); n != 1 { t.Fatalf("mode=%v - Expected outbound to 1 gateway, got %v", pollMode, n) } if n := len(g.InboundGateways); n != 1 { t.Fatalf("mode=%v - Expected inbound from 1 gateway, got %v", pollMode, n) } og := g.OutboundGateways["B"] if og == nil { t.Fatalf("mode=%v - Expected to find outbound connection to B, got none", pollMode) } if !og.IsConfigured { t.Fatalf("mode=%v - Expected gw connection to be configured, was not", pollMode) } if og.Connection == nil { t.Fatalf("mode=%v - Expected outbound connection to B to be set, wat not", pollMode) } if og.Connection.Name != sb1.ID() { t.Fatalf("mode=%v - Expected outbound connection to B to have name %q, got %q", pollMode, sb1.ID(), og.Connection.Name) } if n := len(og.Accounts); n != 0 { t.Fatalf("mode=%v - Expected no account, got %v", pollMode, n) } ig := g.InboundGateways["B"] if ig == nil { t.Fatalf("mode=%v - Expected to find inbound connection from B, got none", pollMode) } if n := len(ig); n != 1 { t.Fatalf("mode=%v - Expected 1 inbound connection, got %v", pollMode, n) } igc := ig[0] if igc.Connection == nil { t.Fatalf("mode=%v - Expected inbound connection to B to be set, wat not", pollMode) } if igc.Connection.Name != sb1.ID() { t.Fatalf("mode=%v - Expected inbound connection to B to have name %q, got %q", pollMode, sb1.ID(), igc.Connection.Name) } } // Now start sb2 that clusters with sb1. sa should add to its list of URLs // sb2 gateway's connect URL. ob2 := testDefaultOptionsForGateway("B") ob2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", sb1.ClusterAddr().Port)) sb2 := runGatewayServer(ob2) defer sb2.Shutdown() // Wait for sb1 and sb2 to connect checkClusterFormed(t, sb1, sb2) // sb2 should be made aware of gateway A and connect to sa waitForInboundGateways(t, sa, 2, 2*time.Second) // Now check that URLs in /varz get updated checkGatewayB := func(t *testing.T, url string, opts *GatewayzOptions) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sa, pollMode, url, opts) if n := len(g.OutboundGateways); n != 1 { t.Fatalf("mode=%v - Expected outbound to 1 gateway, got %v", pollMode, n) } // The InboundGateways is a map with key the gateway names, // then value is array of connections. So should be 1 here. if n := len(g.InboundGateways); n != 1 { t.Fatalf("mode=%v - Expected inbound from 1 gateway, got %v", pollMode, n) } ig := g.InboundGateways["B"] if ig == nil { t.Fatalf("mode=%v - Expected to find inbound connection from B, got none", pollMode) } if n := len(ig); n != 2 { t.Fatalf("mode=%v - Expected 2 inbound connections from gateway B, got %v", pollMode, n) } gotSB1 := false gotSB2 := false for _, rg := range ig { if rg.Connection != nil { if rg.Connection.Name == sb1.ID() { gotSB1 = true } else if rg.Connection.Name == sb2.ID() { gotSB2 = true } } } if !gotSB1 { t.Fatalf("mode=%v - Missing inbound connection from sb1", pollMode) } if !gotSB2 { t.Fatalf("mode=%v - Missing inbound connection from sb2", pollMode) } } return nil }) } checkGatewayB(t, gatewayzURL, nil) // Start a new cluser C that connects to B. A should see it as // a non-configured gateway. oc := testGatewayOptionsFromToWithServers(t, "C", "B", sb1) sc := runGatewayServer(oc) defer sc.Shutdown() // All servers should have 2 outbound connections (one for each other cluster) waitForOutboundGateways(t, sa, 2, 2*time.Second) waitForOutboundGateways(t, sb1, 2, 2*time.Second) waitForOutboundGateways(t, sb2, 2, 2*time.Second) waitForOutboundGateways(t, sc, 2, 2*time.Second) // Server sa should have 3 inbounds now waitForInboundGateways(t, sa, 3, 2*time.Second) // Check gatewayz again to see that we have C now. checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sa, pollMode, gatewayzURL, nil) if n := len(g.OutboundGateways); n != 2 { t.Fatalf("mode=%v - Expected outbound to 2 gateways, got %v", pollMode, n) } // The InboundGateways is a map with key the gateway names, // then value is array of connections. So should be 2 here. if n := len(g.InboundGateways); n != 2 { t.Fatalf("mode=%v - Expected inbound from 2 gateways, got %v", pollMode, n) } og := g.OutboundGateways["C"] if og == nil { t.Fatalf("mode=%v - Expected to find outbound connection to C, got none", pollMode) } if og.IsConfigured { t.Fatalf("mode=%v - Expected IsConfigured for gateway C to be false, was true", pollMode) } if og.Connection == nil { t.Fatalf("mode=%v - Expected connection to C, got none", pollMode) } if og.Connection.Name != sc.ID() { t.Fatalf("mode=%v - Expected outbound connection to C to have name %q, got %q", pollMode, sc.ID(), og.Connection.Name) } ig := g.InboundGateways["C"] if ig == nil { t.Fatalf("mode=%v - Expected to find inbound connection from C, got none", pollMode) } if n := len(ig); n != 1 { t.Fatalf("mode=%v - Expected 1 inbound connections from gateway C, got %v", pollMode, n) } igc := ig[0] if igc.Connection == nil { t.Fatalf("mode=%v - Expected connection to C, got none", pollMode) } if igc.Connection.Name != sc.ID() { t.Fatalf("mode=%v - Expected outbound connection to C to have name %q, got %q", pollMode, sc.ID(), og.Connection.Name) } } return nil }) // Select only 1 gateway by passing the name to option/url opts := &GatewayzOptions{Name: "B"} checkGatewayB(t, gatewayzURL+"?gw_name=B", opts) // Stop gateway C and check that we have only B, with and without filter. sc.Shutdown() checkGatewayB(t, gatewayzURL+"?gw_name=B", opts) checkGatewayB(t, gatewayzURL, nil) } func TestMonitorGatewayzAccounts(t *testing.T) { GatewayDoNotForceInterestOnlyMode(true) defer GatewayDoNotForceInterestOnlyMode(false) resetPreviousHTTPConnections() // Create bunch of Accounts totalAccounts := 15 accounts := "" for i := 0; i < totalAccounts; i++ { acc := fmt.Sprintf(" acc_%d: { users=[{user:user_%d, password: pwd}] }\n", i, i) accounts += acc } bConf := createConfFile(t, []byte(fmt.Sprintf(` accounts { %s } port: -1 http: -1 gateway: { name: "B" port: -1 } no_sys_acc = true `, accounts))) sb, ob := RunServerWithConfig(bConf) defer sb.Shutdown() sb.SetLogger(&DummyLogger{}, true, true) // Start a1 that has a single URL to sb1. aConf := createConfFile(t, []byte(fmt.Sprintf(` accounts { %s } port: -1 http: -1 gateway: { name: "A" port: -1 gateways [ { name: "B" url: "nats://127.0.0.1:%d" } ] } no_sys_acc = true `, accounts, sb.GatewayAddr().Port))) sa, oa := RunServerWithConfig(aConf) defer sa.Shutdown() sa.SetLogger(&DummyLogger{}, true, true) waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) // Create clients for each account on A and publish a message // so that list of accounts appear in gatewayz produceMsgsFromA := func(t *testing.T) { t.Helper() for i := 0; i < totalAccounts; i++ { nc, err := nats.Connect(fmt.Sprintf("nats://user_%d:pwd@%s:%d", i, oa.Host, oa.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Publish("foo", []byte("hello")) nc.Flush() nc.Close() } } produceMsgsFromA(t) // Wait for A- for all accounts gwc := sa.getOutboundGatewayConnection("B") for i := 0; i < totalAccounts; i++ { checkForAccountNoInterest(t, gwc, fmt.Sprintf("acc_%d", i), true, 2*time.Second) } // Check accounts... gatewayzURL := fmt.Sprintf("http://127.0.0.1:%d/gatewayz", sa.MonitorAddr().Port) for pollMode := 0; pollMode < 2; pollMode++ { // First, without asking for it, they should not be present. g := pollGatewayz(t, sa, pollMode, gatewayzURL, nil) og := g.OutboundGateways["B"] if og == nil { t.Fatalf("mode=%v - Expected outbound gateway to B, got none", pollMode) } if n := len(og.Accounts); n != 0 { t.Fatalf("mode=%v - Expected accounts list to not be present by default, got %v", pollMode, n) } // Now ask for the accounts g = pollGatewayz(t, sa, pollMode, gatewayzURL+"?accs=1", &GatewayzOptions{Accounts: true}) og = g.OutboundGateways["B"] if og == nil { t.Fatalf("mode=%v - Expected outbound gateway to B, got none", pollMode) } if n := len(og.Accounts); n != totalAccounts { t.Fatalf("mode=%v - Expected to get all %d accounts, got %v", pollMode, totalAccounts, n) } // Now account details for _, acc := range og.Accounts { if acc.InterestMode != Optimistic.String() { t.Fatalf("mode=%v - Expected optimistic mode, got %q", pollMode, acc.InterestMode) } // Since there is no interest at all on B, the publish // will have resulted in total account no interest, so // the number of no interest (subject wise) should be 0 if acc.NoInterestCount != 0 { t.Fatalf("mode=%v - Expected 0 no-interest, got %v", pollMode, acc.NoInterestCount) } if acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 { t.Fatalf("mode=%v - Expected total subs to be 0, got %v - and num queue subs to be 0, got %v", pollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions) } } } // Check inbound on B gwURLServerB := fmt.Sprintf("http://127.0.0.1:%d/gatewayz", sb.MonitorAddr().Port) checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { // First, without asking for it, they should not be present. g := pollGatewayz(t, sb, pollMode, gwURLServerB, nil) igs := g.InboundGateways["A"] if igs == nil { return fmt.Errorf("mode=%v - Expected inbound gateway to A, got none", pollMode) } if len(igs) != 1 { return fmt.Errorf("mode=%v - Expected single inbound, got %v", pollMode, len(igs)) } ig := igs[0] if n := len(ig.Accounts); n != 0 { return fmt.Errorf("mode=%v - Expected no account, got %v", pollMode, n) } // Check that list of accounts g = pollGatewayz(t, sb, pollMode, gwURLServerB+"?accs=1", &GatewayzOptions{Accounts: true}) igs = g.InboundGateways["A"] if igs == nil { return fmt.Errorf("mode=%v - Expected inbound gateway to A, got none", pollMode) } if len(igs) != 1 { return fmt.Errorf("mode=%v - Expected single inbound, got %v", pollMode, len(igs)) } ig = igs[0] if ig.Connection == nil { return fmt.Errorf("mode=%v - Expected inbound connection from A to be set, wat not", pollMode) } if ig.Connection.Name != sa.ID() { t.Fatalf("mode=%v - Expected inbound connection from A to have name %q, got %q", pollMode, sa.ID(), ig.Connection.Name) } if n := len(ig.Accounts); n != totalAccounts { return fmt.Errorf("mode=%v - Expected to get all %d accounts, got %v", pollMode, totalAccounts, n) } // Now account details for _, acc := range ig.Accounts { if acc.InterestMode != Optimistic.String() { return fmt.Errorf("mode=%v - Expected optimistic mode, got %q", pollMode, acc.InterestMode) } // Since there is no interest at all on B, the publish // will have resulted in total account no interest, so // the number of no interest (subject wise) should be 0 if acc.NoInterestCount != 0 { t.Fatalf("mode=%v - Expected 0 no-interest, got %v", pollMode, acc.NoInterestCount) } // For inbound gateway, NumQueueSubscriptions and TotalSubscriptions // are not relevant. if acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 { return fmt.Errorf("mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v", pollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions) } } } return nil }) // Now create subscriptions on B to prevent A- and check on subject no interest for i := 0; i < totalAccounts; i++ { nc, err := nats.Connect(fmt.Sprintf("nats://user_%d:pwd@%s:%d", i, ob.Host, ob.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Create a queue sub so it shows up in gatewayz nc.QueueSubscribeSync("bar", "queue") // Create plain subscriptions on baz.0, baz.1 and baz.2. // Create to for each subject. Since gateways will send // only once per subject, the number of subs should be 3, not 6. for j := 0; j < 3; j++ { subj := fmt.Sprintf("baz.%d", j) nc.SubscribeSync(subj) nc.SubscribeSync(subj) } nc.Flush() } for i := 0; i < totalAccounts; i++ { accName := fmt.Sprintf("acc_%d", i) checkForRegisteredQSubInterest(t, sa, "B", accName, "bar", 1, 2*time.Second) } // Resend msgs from A on foo, on all accounts. There will be no interest on this subject. produceMsgsFromA(t) for i := 0; i < totalAccounts; i++ { accName := fmt.Sprintf("acc_%d", i) checkForSubjectNoInterest(t, gwc, accName, "foo", true, 2*time.Second) // Verify that we still have the queue interest registered checkForRegisteredQSubInterest(t, sa, "B", accName, "bar", 1, 2*time.Second) } // Check accounts... checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sa, pollMode, gatewayzURL+"?accs=1", &GatewayzOptions{Accounts: true}) og := g.OutboundGateways["B"] if og == nil { return fmt.Errorf("mode=%v - Expected outbound gateway to B, got none", pollMode) } if n := len(og.Accounts); n != totalAccounts { return fmt.Errorf("mode=%v - Expected to get all %d accounts, got %v", pollMode, totalAccounts, n) } // Now account details for _, acc := range og.Accounts { if acc.InterestMode != Optimistic.String() { return fmt.Errorf("mode=%v - Expected optimistic mode, got %q", pollMode, acc.InterestMode) } if acc.NoInterestCount != 1 { return fmt.Errorf("mode=%v - Expected 1 no-interest, got %v", pollMode, acc.NoInterestCount) } if acc.NumQueueSubscriptions != 1 || acc.TotalSubscriptions != 1 { return fmt.Errorf("mode=%v - Expected total subs to be 1, got %v - and num queue subs to be 1, got %v", pollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions) } } } return nil }) // Check inbound on server B checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { // Ask for accounts list g := pollGatewayz(t, sb, pollMode, gwURLServerB+"?accs=1", &GatewayzOptions{Accounts: true}) igs := g.InboundGateways["A"] if igs == nil { return fmt.Errorf("mode=%v - Expected inbound gateway to A, got none", pollMode) } if len(igs) != 1 { return fmt.Errorf("mode=%v - Expected single inbound, got %v", pollMode, len(igs)) } ig := igs[0] if ig.Connection == nil { return fmt.Errorf("mode=%v - Expected inbound connection from A to be set, wat not", pollMode) } if ig.Connection.Name != sa.ID() { t.Fatalf("mode=%v - Expected inbound connection from A to have name %q, got %q", pollMode, sa.ID(), ig.Connection.Name) } if n := len(ig.Accounts); n != totalAccounts { return fmt.Errorf("mode=%v - Expected to get all %d accounts, got %v", pollMode, totalAccounts, n) } // Now account details for _, acc := range ig.Accounts { if acc.InterestMode != Optimistic.String() { return fmt.Errorf("mode=%v - Expected optimistic mode, got %q", pollMode, acc.InterestMode) } if acc.NoInterestCount != 1 { return fmt.Errorf("mode=%v - Expected 1 no-interest, got %v", pollMode, acc.NoInterestCount) } // For inbound gateway, NumQueueSubscriptions and TotalSubscriptions // are not relevant. if acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 { return fmt.Errorf("mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v", pollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions) } } } return nil }) // Make one of the account to switch to interest only nc, err := nats.Connect(fmt.Sprintf("nats://user_1:pwd@%s:%d", oa.Host, oa.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() for i := 0; i < 1100; i++ { nc.Publish(fmt.Sprintf("foo.%d", i), []byte("hello")) } nc.Flush() nc.Close() // Check that we can select single account checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sa, pollMode, gatewayzURL+"?gw_name=B&acc_name=acc_1", &GatewayzOptions{Name: "B", AccountName: "acc_1"}) og := g.OutboundGateways["B"] if og == nil { return fmt.Errorf("mode=%v - Expected outbound gateway to B, got none", pollMode) } if n := len(og.Accounts); n != 1 { return fmt.Errorf("mode=%v - Expected to get 1 account, got %v", pollMode, n) } // Now account details acc := og.Accounts[0] if acc.InterestMode != InterestOnly.String() { return fmt.Errorf("mode=%v - Expected interest-only mode, got %q", pollMode, acc.InterestMode) } // Since we switched, this should be set to 0 if acc.NoInterestCount != 0 { return fmt.Errorf("mode=%v - Expected 0 no-interest, got %v", pollMode, acc.NoInterestCount) } // We have created 3 subs on that account on B, and 1 queue sub. // So total should be 4 and 1 for queue sub. if acc.NumQueueSubscriptions != 1 { return fmt.Errorf("mode=%v - Expected num queue subs to be 1, got %v", pollMode, acc.NumQueueSubscriptions) } if acc.TotalSubscriptions != 4 { return fmt.Errorf("mode=%v - Expected total subs to be 4, got %v", pollMode, acc.TotalSubscriptions) } } return nil }) // Check inbound on B now... checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for pollMode := 0; pollMode < 2; pollMode++ { g := pollGatewayz(t, sb, pollMode, gwURLServerB+"?gw_name=A&acc_name=acc_1", &GatewayzOptions{Name: "A", AccountName: "acc_1"}) igs := g.InboundGateways["A"] if igs == nil { return fmt.Errorf("mode=%v - Expected inbound gateway from A, got none", pollMode) } if len(igs) != 1 { return fmt.Errorf("mode=%v - Expected single inbound, got %v", pollMode, len(igs)) } ig := igs[0] if n := len(ig.Accounts); n != 1 { return fmt.Errorf("mode=%v - Expected to get 1 account, got %v", pollMode, n) } // Now account details acc := ig.Accounts[0] if acc.InterestMode != InterestOnly.String() { return fmt.Errorf("mode=%v - Expected interest-only mode, got %q", pollMode, acc.InterestMode) } if acc.InterestMode != InterestOnly.String() { return fmt.Errorf("Should be in %q mode, got %q", InterestOnly.String(), acc.InterestMode) } // Since we switched, this should be set to 0 if acc.NoInterestCount != 0 { return fmt.Errorf("mode=%v - Expected 0 no-interest, got %v", pollMode, acc.NoInterestCount) } // Again, for inbound, these should be always 0. if acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 { return fmt.Errorf("mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v", pollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions) } } return nil }) } func TestMonitorGatewayzWithSubs(t *testing.T) { resetPreviousHTTPConnections() ob := testDefaultOptionsForGateway("B") aA := NewAccount("A") aB := NewAccount("B") ob.Accounts = append(ob.Accounts, aA, aB) ob.Users = append(ob.Users, &User{Username: "a", Password: "a", Account: aA}, &User{Username: "b", Password: "b", Account: aB}) sb := runGatewayServer(ob) defer sb.Shutdown() oa := testGatewayOptionsFromToWithServers(t, "A", "B", sb) oa.HTTPHost = "127.0.0.1" oa.HTTPPort = MONITOR_PORT aA = NewAccount("A") aB = NewAccount("B") oa.Accounts = append(oa.Accounts, aA, aB) oa.Users = append(oa.Users, &User{Username: "a", Password: "a", Account: aA}, &User{Username: "b", Password: "b", Account: aB}) sa := runGatewayServer(oa) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForInboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb, 1, 2*time.Second) waitForInboundGateways(t, sb, 1, 2*time.Second) ncA := natsConnect(t, sb.ClientURL(), nats.UserInfo("a", "a")) defer ncA.Close() natsSubSync(t, ncA, "foo") natsFlush(t, ncA) ncB := natsConnect(t, sb.ClientURL(), nats.UserInfo("b", "b")) defer ncB.Close() natsSubSync(t, ncB, "foo") natsQueueSubSync(t, ncB, "bar", "baz") natsFlush(t, ncB) checkGWInterestOnlyModeInterestOn(t, sa, "B", "A", "foo") checkGWInterestOnlyModeInterestOn(t, sa, "B", "B", "foo") checkForRegisteredQSubInterest(t, sa, "B", "B", "bar", 1, time.Second) for _, test := range []struct { url string allAccs bool opts *GatewayzOptions }{ {"accs=1&subs=1", true, &GatewayzOptions{Accounts: true, AccountSubscriptions: true}}, {"accs=1&subs=detail", true, &GatewayzOptions{Accounts: true, AccountSubscriptionsDetail: true}}, {"acc_name=B&subs=1", false, &GatewayzOptions{AccountName: "B", AccountSubscriptions: true}}, {"acc_name=B&subs=detail", false, &GatewayzOptions{AccountName: "B", AccountSubscriptionsDetail: true}}, } { t.Run(test.url, func(t *testing.T) { gatewayzURL := fmt.Sprintf("http://127.0.0.1:%d/gatewayz?%s", sa.MonitorAddr().Port, test.url) for pollMode := 0; pollMode < 2; pollMode++ { gw := pollGatewayz(t, sa, pollMode, gatewayzURL, test.opts) require_Equal(t, len(gw.OutboundGateways), 1) ogw, ok := gw.OutboundGateways["B"] require_True(t, ok) require_NotNil(t, ogw) var expected int if test.allAccs { expected = 3 // A + B + $G } else { expected = 1 // B } require_Len(t, len(ogw.Accounts), expected) accs := map[string]*AccountGatewayz{} for _, a := range ogw.Accounts { // Do not include the global account there. if a.Name == globalAccountName { continue } accs[a.Name] = a } // Update the expected number of accounts if we asked for all accounts. if test.allAccs { expected-- } // The account B should always be present. _, ok = accs["B"] require_True(t, ok) if expected == 2 { _, ok = accs["A"] require_True(t, ok) } // Now that we know we have the proper account(s), check the content. for n, a := range accs { require_NotNil(t, a) require_Equal(t, a.Name, n) totalSubs := 1 var numQueueSubs int if n == "B" { totalSubs++ numQueueSubs = 1 } require_Equal(t, a.TotalSubscriptions, totalSubs) require_Equal(t, a.NumQueueSubscriptions, numQueueSubs) m := map[string]*SubDetail{} if test.opts.AccountSubscriptions { require_Len(t, len(a.Subs), totalSubs) require_Len(t, len(a.SubsDetail), 0) for _, sub := range a.Subs { m[sub] = nil } } else { require_Len(t, len(a.Subs), 0) require_Len(t, len(a.SubsDetail), totalSubs) for _, sub := range a.SubsDetail { m[sub.Subject] = &sub } } sd, ok := m["foo"] require_True(t, ok) if test.opts.AccountSubscriptionsDetail { require_NotNil(t, sd) require_Equal(t, sd.Queue, _EMPTY_) } else { require_True(t, sd == nil) } sd, ok = m["bar"] if numQueueSubs == 1 { require_True(t, ok) if test.opts.AccountSubscriptionsDetail { require_NotNil(t, sd) require_Equal(t, sd.Queue, "baz") } else { require_True(t, sd == nil) } } else { require_False(t, ok) } } } }) } } func TestMonitorRouteRTT(t *testing.T) { // Do not change default PingInterval and expect RTT to still be reported ob := DefaultOptions() sb := RunServer(ob) defer sb.Shutdown() oa := DefaultOptions() oa.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", ob.Cluster.Host, ob.Cluster.Port)) sa := RunServer(oa) defer sa.Shutdown() checkClusterFormed(t, sa, sb) checkRouteInfo := func(t *testing.T, s *Server) { t.Helper() routezURL := fmt.Sprintf("http://127.0.0.1:%d/routez", s.MonitorAddr().Port) for pollMode := 0; pollMode < 2; pollMode++ { checkFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error { rz := pollRoutez(t, s, pollMode, routezURL, nil) // Pool size + 1 for system account if len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE+1 { return fmt.Errorf("Expected %d route, got %v", DEFAULT_ROUTE_POOL_SIZE+1, len(rz.Routes)) } for _, ri := range rz.Routes { if ri.RTT == _EMPTY_ { return fmt.Errorf("Route's RTT not reported") } } return nil }) } } checkRouteInfo(t, sa) checkRouteInfo(t, sb) } func pollLeafz(t *testing.T, s *Server, mode int, url string, opts *LeafzOptions) *Leafz { t.Helper() if mode == 0 { l := &Leafz{} body := readBody(t, url) if err := json.Unmarshal(body, l); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return l } l, err := s.Leafz(opts) if err != nil { t.Fatalf("Error on Leafz: %v", err) } return l } func TestMonitorOpJWT(t *testing.T) { content := ` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" operator = "../test/configs/nkeys/op.jwt" resolver = MEMORY ` conf := createConfFile(t, []byte(content)) sa, _ := RunServerWithConfig(conf) defer sa.Shutdown() theJWT, err := os.ReadFile("../test/configs/nkeys/op.jwt") require_NoError(t, err) theJWT = []byte(strings.Split(string(theJWT), "\n")[1]) claim, err := jwt.DecodeOperatorClaims(string(theJWT)) require_NoError(t, err) pollURL := fmt.Sprintf("http://127.0.0.1:%d/varz", sa.MonitorAddr().Port) for pollMode := 1; pollMode < 2; pollMode++ { l := pollVarz(t, sa, pollMode, pollURL, nil) if len(l.TrustedOperatorsJwt) != 1 { t.Fatalf("Expected one operator jwt") } if len(l.TrustedOperatorsClaim) != 1 { t.Fatalf("Expected one operator claim") } if l.TrustedOperatorsJwt[0] != string(theJWT) { t.Fatalf("Expected operator to be identical to configuration") } if !reflect.DeepEqual(l.TrustedOperatorsClaim[0], claim) { t.Fatal("claims need to be equal") } } } func TestMonitorLeafz(t *testing.T) { content := ` server_name: "hub" listen: "127.0.0.1:-1" http: "127.0.0.1:-1" operator = "../test/configs/nkeys/op.jwt" resolver = MEMORY ping_interval = 1 leafnodes { listen: "127.0.0.1:-1" } ` conf := createConfFile(t, []byte(content)) sb, ob := RunServerWithConfig(conf) defer sb.Shutdown() createAcc := func(t *testing.T) (*Account, string) { t.Helper() acc, akp := createAccount(sb) kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() creds := genCredsFile(t, ujwt, seed) return acc, creds } acc1, mycreds1 := createAcc(t) acc2, mycreds2 := createAcc(t) leafName := "my-leaf-node" content = ` port: -1 http: "127.0.0.1:-1" ping_interval = 1 server_name: %s accounts { %s { users [ {user: user1, password: pwd} ] } %s { users [ {user: user2, password: pwd} ] } } leafnodes { remotes = [ { account: "%s" url: nats-leaf://127.0.0.1:%d credentials: '%s' } { account: "%s" url: nats-leaf://127.0.0.1:%d credentials: '%s' } ] } ` config := fmt.Sprintf(content, leafName, acc1.Name, acc2.Name, acc1.Name, ob.LeafNode.Port, mycreds1, acc2.Name, ob.LeafNode.Port, mycreds2) conf = createConfFile(t, []byte(config)) sa, oa := RunServerWithConfig(conf) defer sa.Shutdown() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := sa.NumLeafNodes(); n != 2 { return fmt.Errorf("Expected 2 leaf connections, got %v", n) } return nil }) // Wait for initial RTT to be computed time.Sleep(firstPingInterval + 500*time.Millisecond) ch := make(chan bool, 1) nc1B := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob.Port), nats.UserCredentials(mycreds1)) defer nc1B.Close() natsSub(t, nc1B, "foo", func(_ *nats.Msg) { ch <- true }) natsSub(t, nc1B, "bar", func(_ *nats.Msg) {}) natsFlush(t, nc1B) nc2B := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", ob.Port), nats.UserCredentials(mycreds2)) defer nc2B.Close() natsSub(t, nc2B, "bar", func(_ *nats.Msg) { ch <- true }) natsSub(t, nc2B, "foo", func(_ *nats.Msg) {}) natsFlush(t, nc2B) nc1A := natsConnect(t, fmt.Sprintf("nats://user1:pwd@127.0.0.1:%d", oa.Port)) defer nc1A.Close() natsPub(t, nc1A, "foo", []byte("hello")) natsFlush(t, nc1A) waitCh(t, ch, "Did not get the message") nc2A := natsConnect(t, fmt.Sprintf("nats://user2:pwd@127.0.0.1:%d", oa.Port)) defer nc2A.Close() natsPub(t, nc2A, "bar", []byte("hello")) natsPub(t, nc2A, "bar", []byte("hello")) natsFlush(t, nc2A) waitCh(t, ch, "Did not get the message") waitCh(t, ch, "Did not get the message") // Let's poll server A pollURL := fmt.Sprintf("http://127.0.0.1:%d/leafz?subs=1", sa.MonitorAddr().Port) for pollMode := 1; pollMode < 2; pollMode++ { l := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Subscriptions: true}) if l.ID != sa.ID() { t.Fatalf("Expected ID to be %q, got %q", sa.ID(), l.ID) } if l.Now.IsZero() { t.Fatalf("Expected Now to be set, was not") } if l.NumLeafs != 2 { t.Fatalf("Expected NumLeafs to be 2, got %v", l.NumLeafs) } if len(l.Leafs) != 2 { t.Fatalf("Expected array to be len 2, got %v", len(l.Leafs)) } for _, ln := range l.Leafs { if ln.Account == acc1.Name { if ln.OutMsgs != 1 || ln.OutBytes == 0 || ln.InMsgs != 0 || ln.InBytes != 0 { t.Fatalf("Expected 1 OutMsgs/Bytes and 0 InMsgs/Bytes, got %+v", ln) } } else if ln.Account == acc2.Name { if ln.OutMsgs != 2 || ln.OutBytes == 0 || ln.InMsgs != 0 || ln.InBytes != 0 { t.Fatalf("Expected 2 OutMsgs/Bytes and 0 InMsgs/Bytes, got %+v", ln) } } else { t.Fatalf("Expected account to be %q or %q, got %q", acc1.Name, acc2.Name, ln.Account) } if ln.Name != "hub" { t.Fatalf("Expected name to be %q, got %q", "hub", ln.Name) } if !ln.IsSpoke { t.Fatal("Expected leafnode connection to be spoke") } if ln.RTT == "" { t.Fatalf("RTT not tracked?") } if ln.NumSubs != 3 { t.Fatalf("Expected 3 subs, got %v", ln.NumSubs) } if len(ln.Subs) != 3 { t.Fatalf("Expected subs to be returned, got %v", len(ln.Subs)) } var foundFoo bool var foundBar bool for _, sub := range ln.Subs { if sub == "foo" { foundFoo = true } else if sub == "bar" { foundBar = true } } if !foundFoo { t.Fatal("Did not find subject foo") } if !foundBar { t.Fatal("Did not find subject bar") } } } // Make sure that if we don't ask for subs, we don't get them pollURL = fmt.Sprintf("http://127.0.0.1:%d/leafz", sa.MonitorAddr().Port) for pollMode := 1; pollMode < 2; pollMode++ { l := pollLeafz(t, sa, pollMode, pollURL, nil) for _, ln := range l.Leafs { if ln.NumSubs != 3 { t.Fatalf("Number of subs should be 3, got %v", ln.NumSubs) } if len(ln.Subs) != 0 { t.Fatalf("Subs should not have been returned, got %v", ln.Subs) } } } // Make sure that we can request per account - existing account pollURL = fmt.Sprintf("http://127.0.0.1:%d/leafz?acc=%s", sa.MonitorAddr().Port, acc1.Name) for pollMode := 1; pollMode < 2; pollMode++ { l := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Account: acc1.Name}) for _, ln := range l.Leafs { if ln.Account != acc1.Name { t.Fatalf("Expected leaf node to be from account %s, got: %v", acc1.Name, ln) } } if len(l.Leafs) != 1 { t.Fatalf("Expected only two leaf node for this account, got: %v", len(l.Leafs)) } } // Make sure that we can request per account - non existing account pollURL = fmt.Sprintf("http://127.0.0.1:%d/leafz?acc=%s", sa.MonitorAddr().Port, "DOESNOTEXIST") for pollMode := 1; pollMode < 2; pollMode++ { l := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Account: "DOESNOTEXIST"}) if len(l.Leafs) != 0 { t.Fatalf("Expected no leaf node for this account, got: %v", len(l.Leafs)) } } // Now polling server B. pollURL = fmt.Sprintf("http://127.0.0.1:%d/leafz?subs=1", sb.MonitorAddr().Port) for pollMode := 1; pollMode < 2; pollMode++ { l := pollLeafz(t, sb, pollMode, pollURL, &LeafzOptions{Subscriptions: true}) if l.ID != sb.ID() { t.Fatalf("Expected ID to be %q, got %q", sb.ID(), l.ID) } if l.Now.IsZero() { t.Fatalf("Expected Now to be set, was not") } if l.NumLeafs != 2 { t.Fatalf("Expected NumLeafs to be 1, got %v", l.NumLeafs) } if len(l.Leafs) != 2 { t.Fatalf("Expected array to be len 2, got %v", len(l.Leafs)) } for _, ln := range l.Leafs { if ln.Account == acc1.Name { if ln.OutMsgs != 0 || ln.OutBytes != 0 || ln.InMsgs != 1 || ln.InBytes == 0 { t.Fatalf("Expected 1 InMsgs/Bytes and 0 OutMsgs/Bytes, got %+v", ln) } } else if ln.Account == acc2.Name { if ln.OutMsgs != 0 || ln.OutBytes != 0 || ln.InMsgs != 2 || ln.InBytes == 0 { t.Fatalf("Expected 2 InMsgs/Bytes and 0 OutMsgs/Bytes, got %+v", ln) } } else { t.Fatalf("Expected account to be %q or %q, got %q", acc1.Name, acc2.Name, ln.Account) } if ln.Name != leafName { t.Fatalf("Expected name to be %q, got %q", leafName, ln.Name) } if ln.IsSpoke { t.Fatal("Expected leafnode connection to be hub") } if ln.RTT == "" { t.Fatalf("RTT not tracked?") } // LDS should be only one. if ln.NumSubs != 5 || len(ln.Subs) != 5 { t.Fatalf("Expected 5 subs, got %v (%v)", ln.NumSubs, ln.Subs) } } } } func TestMonitorAccountz(t *testing.T) { s := RunServer(DefaultMonitorOptions()) defer s.Shutdown() body := string(readBody(t, fmt.Sprintf("http://127.0.0.1:%d%s", s.MonitorAddr().Port, AccountzPath))) require_Contains(t, body, `$G`) require_Contains(t, body, `$SYS`) require_Contains(t, body, `"accounts": [`) require_Contains(t, body, `"system_account": "$SYS"`) body = string(readBody(t, fmt.Sprintf("http://127.0.0.1:%d%s?acc=$SYS", s.MonitorAddr().Port, AccountzPath))) require_Contains(t, body, `"account_detail": {`) require_Contains(t, body, `"account_name": "$SYS",`) require_Contains(t, body, `"subscriptions": 56,`) require_Contains(t, body, `"is_system": true,`) require_Contains(t, body, `"system_account": "$SYS"`) body = string(readBody(t, fmt.Sprintf("http://127.0.0.1:%d%s?unused=1", s.MonitorAddr().Port, AccountStatzPath))) require_Contains(t, body, `"acc": "$G"`) require_Contains(t, body, `"acc": "$SYS"`) require_Contains(t, body, `"sent": {`) require_Contains(t, body, `"received": {`) require_Contains(t, body, `"total_conns": 0,`) require_Contains(t, body, `"leafnodes": 0,`) } func TestMonitorAccountzAccountIssuerUpdate(t *testing.T) { // create an operator set of keys okp, err := nkeys.CreateOperator() require_NoError(t, err) opk, err := okp.PublicKey() require_NoError(t, err) // create the system account _, sysPK := createKey(t) sysAc := jwt.NewAccountClaims(sysPK) sysAc.Name = "SYS" sysJwt, err := sysAc.Encode(okp) require_NoError(t, err) // create the operator with the system oc := jwt.NewOperatorClaims(opk) oc.Name = "O" // add a signing keys osk1, err := nkeys.CreateOperator() require_NoError(t, err) opk1, err := osk1.PublicKey() require_NoError(t, err) // add a second signing key osk2, err := nkeys.CreateOperator() require_NoError(t, err) opk2, err := osk2.PublicKey() require_NoError(t, err) oc.SigningKeys.Add(opk1, opk2) // set the system account oc.SystemAccount = sysPK // generate oJWT, err := oc.Encode(okp) require_NoError(t, err) // create an account akp, apk := createKey(t) ac := jwt.NewAccountClaims(apk) ac.Name = "A" // sign with the signing key aJWT, err := ac.Encode(osk1) require_NoError(t, err) // build the mem-resolver conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 http: 127.0.0.1:-1 operator = %s resolver = MEMORY system_account: %s resolver_preload = { %s : %s %s : %s } `, oJWT, sysPK, sysPK, sysJwt, apk, aJWT))) // start the server s, _ := RunServerWithConfig(conf) defer s.Shutdown() // create an user for account A, or we don't see // the account in accountsz createUser := func() (string, string) { ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt, err := uclaim.Encode(akp) require_NoError(t, err) return upub, genCredsFile(t, ujwt, seed) } _, aCreds := createUser() // connect the user nc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds)) require_NoError(t, err) defer nc.Close() // lookup the account data := readBody(t, fmt.Sprintf("http://127.0.0.1:%d%s?acc=%s", s.MonitorAddr().Port, AccountzPath, apk)) var ci Accountz require_NoError(t, json.Unmarshal(data, &ci)) require_Equal(t, ci.Account.IssuerKey, opk1) // now update the account aJWT, err = ac.Encode(osk2) require_NoError(t, err) updatedConf := []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 http: 127.0.0.1:-1 operator = %s resolver = MEMORY system_account: %s resolver_preload = { %s : %s %s : %s } `, oJWT, sysPK, sysPK, sysJwt, apk, aJWT)) // update the configuration file require_NoError(t, os.WriteFile(conf, updatedConf, 0666)) // reload require_NoError(t, s.Reload()) data = readBody(t, fmt.Sprintf("http://127.0.0.1:%d%s?acc=%s", s.MonitorAddr().Port, AccountzPath, apk)) require_NoError(t, json.Unmarshal(data, &ci)) require_Equal(t, ci.Account.IssuerKey, opk2) } func TestMonitorAuthorizedUsers(t *testing.T) { kp, _ := nkeys.FromSeed(seed) usrNKey, _ := kp.PublicKey() opts := DefaultMonitorOptions() opts.Nkeys = []*NkeyUser{{Nkey: string(usrNKey)}} opts.Users = []*User{{Username: "user", Password: "pwd"}} s := RunServer(opts) defer s.Shutdown() checkAuthUser := func(expected string) { t.Helper() resetPreviousHTTPConnections() url := fmt.Sprintf("http://127.0.0.1:%d/connz?auth=true", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { connz := pollConz(t, s, mode, url, &ConnzOptions{Username: true}) if l := len(connz.Conns); l != 1 { t.Fatalf("Expected 1, got %v", l) } conn := connz.Conns[0] au := conn.AuthorizedUser if au == _EMPTY_ { t.Fatal("AuthorizedUser is empty!") } if au != expected { t.Fatalf("Expected %q, got %q", expected, au) } } } c := natsConnect(t, fmt.Sprintf("nats://user:pwd@127.0.0.1:%d", opts.Port)) defer c.Close() checkAuthUser("user") c.Close() c = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", opts.Port), nats.Nkey(usrNKey, func(nonce []byte) ([]byte, error) { return kp.Sign(nonce) })) defer c.Close() // we should get the user's NKey checkAuthUser(usrNKey) c.Close() s.Shutdown() opts = DefaultMonitorOptions() opts.Authorization = "sometoken" s = RunServer(opts) defer s.Shutdown() c = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", opts.Port), nats.Token("sometoken")) defer c.Close() // We should get the token specified by the user checkAuthUser("sometoken") c.Close() s.Shutdown() opts = DefaultMonitorOptions() // User an operator seed kp, _ = nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() opts.TrustedKeys = []string{pub} s = RunServer(opts) defer s.Shutdown() akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() upub, _ := nkp.PublicKey() nuc := jwt.NewUserClaims(upub) jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } buildMemAccResolver(s) addAccountToMemResolver(s, apub, ajwt) c = natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", opts.Port), nats.UserJWT( func() (string, error) { return jwt, nil }, func(nonce []byte) ([]byte, error) { return nkp.Sign(nonce) })) defer c.Close() // we should get the user's pubkey checkAuthUser(upub) } // Helper function to check that a JS cluster is formed func checkForJSClusterUp(t *testing.T, servers ...*Server) { t.Helper() // We will use the other JetStream helpers here. c := &cluster{t: t, servers: servers} c.checkClusterFormed() c.waitOnClusterReady() } func TestMonitorJszNonJszServer(t *testing.T) { srv := RunServer(DefaultOptions()) defer srv.Shutdown() if !srv.ReadyForConnections(5 * time.Second) { t.Fatalf("server did not become ready") } jsi, err := srv.Jsz(&JSzOptions{}) if err != nil { t.Fatalf("jsi failed: %v", err) } if jsi.ID != srv.ID() { t.Fatalf("did not receive valid info") } jsi, err = srv.Jsz(&JSzOptions{LeaderOnly: true}) if !errors.Is(err, errSkipZreq) { t.Fatalf("expected a skip z req error: %v", err) } if jsi != nil { t.Fatalf("expected no jsi: %v", jsi) } } func TestMonitorJsz(t *testing.T) { readJsInfo := func(url string) *JSInfo { t.Helper() body := readBody(t, url) info := &JSInfo{} err := json.Unmarshal(body, info) require_NoError(t, err) return info } srvs := []*Server{} for _, test := range []struct { port int mport int cport int routed int }{ {7500, 7501, 7502, 5502}, {5500, 5501, 5502, 7502}, } { tmpDir := t.TempDir() cf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:%d http: 127.0.0.1:%d system_account: SYS accounts { SYS { users [{user: sys, password: pwd}] } ACC { users [{user: usr, password: pwd}] // In clustered mode, these reservations will not impact any one server. jetstream: {max_store: 4Mb, max_memory: 5Mb} } BCC_TO_HAVE_ONE_EXTRA { users [{user: usr2, password: pwd}] jetstream: enabled } } jetstream: { max_mem_store: 10Mb max_file_store: 10Mb store_dir: '%s' unique_tag: az } cluster { name: cluster_name listen: 127.0.0.1:%d routes: [nats-route://127.0.0.1:%d] } server_name: server_%d server_tags: [ "az:%d" ] `, test.port, test.mport, tmpDir, test.cport, test.routed, test.port, test.port))) s, _ := RunServerWithConfig(cf) defer s.Shutdown() srvs = append(srvs, s) } checkClusterFormed(t, srvs...) checkForJSClusterUp(t, srvs...) nc := natsConnect(t, "nats://usr:pwd@127.0.0.1:7500") defer nc.Close() js, err := nc.JetStream(nats.MaxWait(5 * time.Second)) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "my-stream-replicated", Subjects: []string{"foo", "bar"}, Replicas: 2, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "my-stream-non-replicated", Subjects: []string{"baz"}, Replicas: 1, }) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "my-stream-mirror", Replicas: 2, Mirror: &nats.StreamSource{ Name: "my-stream-replicated", }, }) require_NoError(t, err) _, err = js.AddConsumer("my-stream-replicated", &nats.ConsumerConfig{ Durable: "my-consumer-replicated", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("my-stream-non-replicated", &nats.ConsumerConfig{ Durable: "my-consumer-non-replicated", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("my-stream-mirror", &nats.ConsumerConfig{ Durable: "my-consumer-mirror", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) nc.Flush() _, err = js.Publish("foo", nil) require_NoError(t, err) // Wait for mirror replication time.Sleep(200 * time.Millisecond) monUrl1 := fmt.Sprintf("http://127.0.0.1:%d/jsz", 7501) monUrl2 := fmt.Sprintf("http://127.0.0.1:%d/jsz", 5501) t.Run("default", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url) if len(info.AccountDetails) != 0 { t.Fatalf("expected no account to be returned by %s but got %v", url, info) } if info.Streams == 0 { t.Fatalf("expected stream count to be 3 but got %d", info.Streams) } if info.Consumers == 0 { t.Fatalf("expected consumer count to be 3 but got %d", info.Consumers) } if info.Messages != 2 { t.Fatalf("expected two message but got %d", info.Messages) } } }) t.Run("accounts", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?accounts=true") if len(info.AccountDetails) != 2 { t.Fatalf("expected both accounts to be returned by %s but got %v", url, info) } } }) t.Run("accounts reserved metrics", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?accounts=true&acc=ACC") if len(info.AccountDetails) != 1 { t.Fatalf("expected single account") } acc := info.AccountDetails[0] got := int(acc.ReservedMemory) expected := 5242880 if got != expected { t.Errorf("Expected: %v, got: %v", expected, got) } got = int(acc.ReservedStore) expected = 4194304 if got != expected { t.Errorf("Expected: %v, got: %v", expected, got) } info = readJsInfo(url + "?accounts=true&acc=BCC_TO_HAVE_ONE_EXTRA") if len(info.AccountDetails) != 1 { t.Fatalf("expected single account") } acc = info.AccountDetails[0] got = int(acc.ReservedMemory) expected = -1 if got != expected { t.Errorf("Expected: %v, got: %v", expected, got) } got = int(acc.ReservedStore) expected = -1 if got != expected { t.Errorf("Expected: %v, got: %v", expected, got) } } }) t.Run("offset-too-big", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?accounts=true&offset=10") if len(info.AccountDetails) != 0 { t.Fatalf("expected no accounts to be returned by %s but got %v", url, info) } } }) t.Run("limit", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?accounts=true&limit=1") if len(info.AccountDetails) != 1 { t.Fatalf("expected one account to be returned by %s but got %v", url, info) } if info := readJsInfo(url + "?accounts=true&offset=1&limit=1"); len(info.AccountDetails) != 1 { t.Fatalf("expected one account to be returned by %s but got %v", url, info) } } }) t.Run("offset-stable", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info1 := readJsInfo(url + "?accounts=true&offset=1&limit=1") if len(info1.AccountDetails) != 1 { t.Fatalf("expected one account to be returned by %s but got %v", url, info1) } info2 := readJsInfo(url + "?accounts=true&offset=1&limit=1") if len(info2.AccountDetails) != 1 { t.Fatalf("expected one account to be returned by %s but got %v", url, info2) } if info1.AccountDetails[0].Name != info2.AccountDetails[0].Name { t.Fatalf("absent changes, same offset should result in same account but gut: %v %v", info1.AccountDetails[0].Name, info2.AccountDetails[0].Name) } } }) t.Run("filter-account", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } if info.AccountDetails[0].Name != "ACC" { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } if len(info.AccountDetails[0].Streams) != 0 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } } }) t.Run("streams", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC&streams=true") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } if len(info.AccountDetails[0].Streams) == 0 { t.Fatalf("expected streams to be returned by %s but got %v", url, info) } if len(info.AccountDetails[0].Streams[0].Consumer) != 0 { t.Fatalf("expected no consumers to be returned by %s but got %v", url, info) } } }) t.Run("stream-leader-only", func(t *testing.T) { // First server info := readJsInfo(monUrl1 + "?streams=true&stream-leader-only=1") for _, a := range info.AccountDetails { for _, s := range a.Streams { if s.Cluster.Leader != srvs[0].serverName() { t.Fatalf("expected stream leader to be %s but got %s", srvs[0].serverName(), s.Cluster.Leader) } } } info = readJsInfo(monUrl2 + "?streams=true&stream-leader-only=1") for _, a := range info.AccountDetails { for _, s := range a.Streams { if s.Cluster.Leader != srvs[1].serverName() { t.Fatalf("expected stream leader to be %s but got %s", srvs[0].serverName(), s.Cluster.Leader) } } } }) t.Run("consumers", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC&consumers=true") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } if len(info.AccountDetails[0].Streams[0].Consumer) == 0 { t.Fatalf("expected consumers to be returned by %s but got %v", url, info) } if info.AccountDetails[0].Streams[0].Config != nil { t.Fatal("Config expected to not be present") } if info.AccountDetails[0].Streams[0].Consumer[0].Config != nil { t.Fatal("Config expected to not be present") } if len(info.AccountDetails[0].Streams[0].ConsumerRaftGroups) != 0 { t.Fatalf("expected consumer raft groups to not be returned by %s but got %v", url, info) } } }) t.Run("config", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC&consumers=true&config=true") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } if info.AccountDetails[0].Streams[0].Config == nil { t.Fatal("Config expected to be present") } if info.AccountDetails[0].Streams[0].Consumer[0].Config == nil { t.Fatal("Config expected to be present") } } }) t.Run("replication", func(t *testing.T) { // The replication lag may only be present on the leader replicationFound := false for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC&streams=true") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } streamFound := false for _, stream := range info.AccountDetails[0].Streams { if stream.Name == "my-stream-mirror" { streamFound = true if stream.Mirror != nil { replicationFound = true } } } if !streamFound { t.Fatalf("Did not locate my-stream-mirror stream in results") } } if !replicationFound { t.Fatal("ReplicationLag expected to be present for my-stream-mirror stream") } }) t.Run("cluster-info", func(t *testing.T) { found := 0 for i, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "") if info.Meta.Peer != getHash(info.Meta.Leader) { t.Fatalf("Invalid Peer: %+v", info.Meta) } if info.Meta.Replicas != nil { found++ for _, r := range info.Meta.Replicas { if r.Peer == _EMPTY_ { t.Fatalf("Replicas' Peer is empty: %+v", r) } } if info.Meta.Leader != srvs[i].Name() { t.Fatalf("received cluster info from non leader: leader %s, server: %s", info.Meta.Leader, srvs[i].Name()) } } } if found == 0 { t.Fatalf("did not receive cluster info from any node") } if found > 1 { t.Fatalf("received cluster info from multiple nodes") } }) t.Run("account-non-existing", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=DOES_NOT_EXIST") if len(info.AccountDetails) != 0 { t.Fatalf("expected no account to be returned by %s but got %v", url, info) } } }) t.Run("account-non-existing-with-stream-details", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=DOES_NOT_EXIST&streams=true") if len(info.AccountDetails) != 0 { t.Fatalf("expected no account to be returned by %s but got %v", url, info) } } }) t.Run("unique-tag-exists", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url) if len(info.Config.UniqueTag) == 0 { t.Fatalf("expected unique_tag to be returned by %s but got %v", url, info) } } }) t.Run("raftgroups", func(t *testing.T) { for _, url := range []string{monUrl1, monUrl2} { info := readJsInfo(url + "?acc=ACC&consumers=true&raft=true") if len(info.AccountDetails) != 1 { t.Fatalf("expected account ACC to be returned by %s but got %v", url, info) } // We will have two streams and order is not guaranteed. So grab the one we want. var si StreamDetail if info.AccountDetails[0].Streams[0].Name == "my-stream-replicated" { si = info.AccountDetails[0].Streams[0] } else { si = info.AccountDetails[0].Streams[1] } if len(si.Consumer) == 0 { t.Fatalf("expected consumers to be returned by %s but got %v", url, info) } if len(si.ConsumerRaftGroups) == 0 { t.Fatalf("expected consumer raft groups to be returned by %s but got %v", url, info) } if len(si.RaftGroup) == 0 { t.Fatal("expected stream raft group info to be included") } crgroup := si.ConsumerRaftGroups[0] if crgroup.Name != "my-consumer-replicated" && crgroup.Name != "my-consumer-mirror" { t.Fatalf("expected consumer name to be included in raft group info, got: %v", crgroup.Name) } if len(crgroup.RaftGroup) == 0 { t.Fatal("expected consumer raft group info to be included") } } }) } func TestMonitorReloadTLSConfig(t *testing.T) { template := ` listen: "127.0.0.1:-1" https: "127.0.0.1:-1" tls { cert_file: '%s' key_file: '%s' ca_file: '../test/configs/certs/ca.pem' # Set this to make sure that it does not impact secure monitoring # (which it did, see issue: https://github.com/nats-io/nats-server/issues/2980) verify_and_map: true } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, "../test/configs/certs/server-noip.pem", "../test/configs/certs/server-key-noip.pem"))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() addr := fmt.Sprintf("127.0.0.1:%d", s.MonitorAddr().Port) c, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer c.Close() tc := &TLSConfigOpts{CaFile: "../test/configs/certs/ca.pem"} tlsConfig, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } tlsConfig.ServerName = "127.0.0.1" tlsConfig.RootCAs = tlsConfig.ClientCAs tlsConfig.ClientCAs = nil c = tls.Client(c, tlsConfig.Clone()) if err := c.(*tls.Conn).Handshake(); err == nil || !strings.Contains(err.Error(), "SAN") { t.Fatalf("Unexpected error: %v", err) } c.Close() reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, "../test/configs/certs/server-cert.pem", "../test/configs/certs/server-key.pem")) c, err = net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer c.Close() c = tls.Client(c, tlsConfig.Clone()) if err := c.(*tls.Conn).Handshake(); err != nil { t.Fatalf("Error on TLS handshake: %v", err) } // Need to read something to see if there is a problem with the certificate or not. var buf [64]byte c.SetReadDeadline(time.Now().Add(250 * time.Millisecond)) _, err = c.Read(buf[:]) if ne, ok := err.(net.Error); !ok || !ne.Timeout() { t.Fatalf("Error: %v", err) } } func TestMonitorMQTT(t *testing.T) { o := DefaultOptions() o.HTTPHost = "127.0.0.1" o.HTTPPort = -1 o.ServerName = "mqtt_server" o.Users = []*User{{Username: "someuser"}} pinnedCerts := make(PinnedCertSet) pinnedCerts["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"] = struct{}{} o.MQTT = MQTTOpts{ Host: "127.0.0.1", Port: -1, NoAuthUser: "someuser", JsDomain: "js", AuthTimeout: 2.0, TLSMap: true, TLSTimeout: 3.0, TLSPinnedCerts: pinnedCerts, AckWait: 4 * time.Second, MaxAckPending: 256, } s := RunServer(o) defer s.Shutdown() expected := &MQTTOptsVarz{ Host: "127.0.0.1", Port: o.MQTT.Port, NoAuthUser: "someuser", JsDomain: "js", AuthTimeout: 2.0, TLSMap: true, TLSTimeout: 3.0, TLSPinnedCerts: []string{"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"}, AckWait: 4 * time.Second, MaxAckPending: 256, } url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url, nil) vm := &v.MQTT if !reflect.DeepEqual(vm, expected) { t.Fatalf("Expected\n%+v\nGot:\n%+v", expected, vm) } } } func TestMonitorWebsocket(t *testing.T) { o := DefaultOptions() o.HTTPHost = "127.0.0.1" o.HTTPPort = -1 kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() o.TrustedKeys = []string{pub} o.Users = []*User{{Username: "someuser"}} pinnedCerts := make(PinnedCertSet) pinnedCerts["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"] = struct{}{} o.Websocket = WebsocketOpts{ Host: "127.0.0.1", Port: -1, Advertise: "somehost:8080", NoAuthUser: "someuser", JWTCookie: "somecookiename", AuthTimeout: 2.0, NoTLS: true, TLSMap: true, TLSPinnedCerts: pinnedCerts, SameOrigin: true, AllowedOrigins: []string{"origin1", "origin2"}, Compression: true, HandshakeTimeout: 4 * time.Second, } s := RunServer(o) defer s.Shutdown() expected := &WebsocketOptsVarz{ Host: "127.0.0.1", Port: o.Websocket.Port, Advertise: "somehost:8080", NoAuthUser: "someuser", JWTCookie: "somecookiename", AuthTimeout: 2.0, NoTLS: true, TLSMap: true, TLSPinnedCerts: []string{"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"}, SameOrigin: true, AllowedOrigins: []string{"origin1", "origin2"}, Compression: true, HandshakeTimeout: 4 * time.Second, } url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url, nil) vw := &v.Websocket if !reflect.DeepEqual(vw, expected) { t.Fatalf("Expected\n%+v\nGot:\n%+v", expected, vw) } } } func TestServerIDZRequest(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: TEST22 # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } `)) defer removeFile(t, conf) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "s3cr3t!")) require_NoError(t, err) defer nc.Close() subject := fmt.Sprintf(serverPingReqSubj, "IDZ") resp, err := nc.Request(subject, nil, time.Second) require_NoError(t, err) var sid ServerID err = json.Unmarshal(resp.Data, &sid) require_NoError(t, err) require_True(t, sid.Name == "TEST22") require_True(t, strings.HasPrefix(sid.ID, "N")) } func TestMonitorProfilez(t *testing.T) { s := RunServer(DefaultOptions()) defer s.Shutdown() // Then start profiling. s.StartProfiler() // Now check that all of the profiles that we expect are // returning instead of erroring. for _, try := range []*ProfilezOptions{ {Name: "allocs", Debug: 0}, {Name: "allocs", Debug: 1}, {Name: "block", Debug: 0}, {Name: "goroutine", Debug: 0}, {Name: "goroutine", Debug: 1}, {Name: "goroutine", Debug: 2}, {Name: "heap", Debug: 0}, {Name: "heap", Debug: 1}, {Name: "mutex", Debug: 0}, {Name: "threadcreate", Debug: 0}, } { if ps := s.profilez(try); ps.Error != _EMPTY_ { t.Fatalf("Unexpected error on %v: %s", try, ps.Error) } } } func TestMonitorRoutePoolSize(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 http: -1 cluster { port: -1 name: "local" pool_size: 5 } no_sys_acc: true `)) defer removeFile(t, conf1) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf23 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 http: -1 cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] pool_size: 5 } no_sys_acc: true `, o1.Cluster.Port))) defer removeFile(t, conf23) s2, _ := RunServerWithConfig(conf23) defer s2.Shutdown() s3, _ := RunServerWithConfig(conf23) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) for i, s := range []*Server{s1, s2, s3} { url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url, nil) if v.Cluster.PoolSize != 5 { t.Fatalf("Expected Cluster.PoolSize==5, got %v", v.Cluster.PoolSize) } if v.Remotes != 2 { t.Fatalf("Expected Remotes==2, got %v", v.Remotes) } if v.Routes != 10 { t.Fatalf("Expected NumRoutes==10, got %v", v.Routes) } } url = fmt.Sprintf("http://127.0.0.1:%d/routez", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollRoutez(t, s, mode, url, nil) if v.NumRoutes != 10 { t.Fatalf("Expected NumRoutes==10, got %v", v.NumRoutes) } if n := len(v.Routes); n != 10 { t.Fatalf("Expected len(Routes)==10, got %v", n) } remotes := make(map[string]int) for _, r := range v.Routes { remotes[r.RemoteID]++ } if n := len(remotes); n != 2 { t.Fatalf("Expected routes for 2 different servers, got %v", n) } switch i { case 0: if n := remotes[s2.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S1 to S2, got %v", n) } if n := remotes[s3.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S1 to S3, got %v", n) } case 1: if n := remotes[s1.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S2 to S1, got %v", n) } if n := remotes[s3.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S2 to S3, got %v", n) } case 2: if n := remotes[s1.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S3 to S1, got %v", n) } if n := remotes[s2.ID()]; n != 5 { t.Fatalf("Expected 5 routes from S3 to S2, got %v", n) } } } } } func TestMonitorRoutePerAccount(t *testing.T) { conf1 := createConfFile(t, []byte(` port: -1 http: -1 accounts { A { users: [{user: "a", password: "pwd"}] } } cluster { port: -1 name: "local" accounts: ["A"] } `)) defer removeFile(t, conf1) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf23 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 http: -1 accounts { A { users: [{user: "a", password: "pwd"}] } } cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] accounts: ["A"] } `, o1.Cluster.Port))) defer removeFile(t, conf23) s2, _ := RunServerWithConfig(conf23) defer s2.Shutdown() s3, _ := RunServerWithConfig(conf23) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) for _, s := range []*Server{s1, s2, s3} { // Default pool size + account "A" + system account (added by default) enr := 2 * (DEFAULT_ROUTE_POOL_SIZE + 1 + 1) url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollVarz(t, s, mode, url, nil) if v.Remotes != 2 { t.Fatalf("Expected Remotes==2, got %v", v.Remotes) } if v.Routes != enr { t.Fatalf("Expected NumRoutes==%d, got %v", enr, v.Routes) } } url = fmt.Sprintf("http://127.0.0.1:%d/routez", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { v := pollRoutez(t, s, mode, url, nil) if v.NumRoutes != enr { t.Fatalf("Expected NumRoutes==%d, got %v", enr, v.NumRoutes) } if n := len(v.Routes); n != enr { t.Fatalf("Expected len(Routes)==%d, got %v", enr, n) } remotes := make(map[string]int) for _, r := range v.Routes { var acc int if r.Account == "A" { acc = 1 } remotes[r.RemoteID] += acc } if n := len(remotes); n != 2 { t.Fatalf("Expected routes for 2 different servers, got %v", n) } for remoteID, v := range remotes { if v != 1 { t.Fatalf("Expected one and only one connection for account A for remote %q, got %v", remoteID, v) } } } } } func TestMonitorConnzOperatorModeFilterByUser(t *testing.T) { accKp, accPub := createKey(t) accClaim := jwt.NewAccountClaims(accPub) accJwt := encodeClaim(t, accClaim, accPub) conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 http: 127.0.0.1:-1 operator = %s resolver = MEMORY resolver_preload = { %s : %s } `, ojwt, accPub, accJwt))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() createUser := func() (string, string) { ukp, _ := nkeys.CreateUser() seed, _ := ukp.Seed() upub, _ := ukp.PublicKey() uclaim := newJWTTestUserClaims() uclaim.Subject = upub ujwt, err := uclaim.Encode(accKp) require_NoError(t, err) return upub, genCredsFile(t, ujwt, seed) } // Now create 2 users. aUser, aCreds := createUser() bUser, bCreds := createUser() var users []*nats.Conn // Create 2 for A for i := 0; i < 2; i++ { nc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds)) require_NoError(t, err) defer nc.Close() users = append(users, nc) } // Create 5 for B for i := 0; i < 5; i++ { nc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(bCreds)) require_NoError(t, err) defer nc.Close() users = append(users, nc) } // Test A connz := pollConz(t, s, 1, _EMPTY_, &ConnzOptions{User: aUser, Username: true}) require_True(t, connz.NumConns == 2) for _, ci := range connz.Conns { require_True(t, ci.AuthorizedUser == aUser) } // Test B connz = pollConz(t, s, 1, _EMPTY_, &ConnzOptions{User: bUser, Username: true}) require_True(t, connz.NumConns == 5) for _, ci := range connz.Conns { require_True(t, ci.AuthorizedUser == bUser) } // Make sure URL access is the same. url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) urlFull := url + fmt.Sprintf("connz?auth=true&user=%s", aUser) connz = pollConz(t, s, 0, urlFull, nil) require_True(t, connz.NumConns == 2) for _, ci := range connz.Conns { require_True(t, ci.AuthorizedUser == aUser) } // Now test closed filtering as well. for _, nc := range users { nc.Close() } // Let them process and be moved to closed ring buffer in server. time.Sleep(100 * time.Millisecond) connz = pollConz(t, s, 1, _EMPTY_, &ConnzOptions{User: aUser, Username: true, State: ConnClosed}) require_True(t, connz.NumConns == 2) for _, ci := range connz.Conns { require_True(t, ci.AuthorizedUser == aUser) } } func TestMonitorConnzSortByRTT(t *testing.T) { s := runMonitorServer() defer s.Shutdown() for i := 0; i < 10; i++ { nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() } connz := pollConz(t, s, 1, _EMPTY_, &ConnzOptions{Sort: ByRTT}) require_True(t, connz.NumConns == 10) var rtt int64 for _, ci := range connz.Conns { if rtt == 0 { rtt = ci.rtt } else { if ci.rtt > rtt { t.Fatalf("RTT not in descending order: %v vs %v", time.Duration(rtt), time.Duration(ci.rtt)) } rtt = ci.rtt } } // Make sure url works as well. url := fmt.Sprintf("http://127.0.0.1:%d/connz?sort=rtt", s.MonitorAddr().Port) connz = pollConz(t, s, 0, url, nil) require_True(t, connz.NumConns == 10) rtt = 0 for _, ci := range connz.Conns { crttd, err := time.ParseDuration(ci.RTT) require_NoError(t, err) crtt := int64(crttd) if rtt == 0 { rtt = crtt } else { if crtt > rtt { t.Fatalf("RTT not in descending order: %v vs %v", time.Duration(rtt), time.Duration(crtt)) } rtt = ci.rtt } } } // https://github.com/nats-io/nats-server/issues/4144 func TestMonitorAccountszMappingOrderReporting(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: SR22 accounts { CLOUD { exports [ { service: "downlink.>" } ] } APP { imports [ { service: { account: CLOUD, subject: "downlink.>"}, to: "event.>"} ] } }`)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() az, err := s.Accountz(&AccountzOptions{"APP"}) require_NoError(t, err) require_NotNil(t, az.Account) require_True(t, len(az.Account.Imports) > 0) var found bool for _, si := range az.Account.Imports { if si.Import.Subject == "downlink.>" { found = true require_True(t, si.Import.LocalSubject == "event.>") break } } require_True(t, found) } // createCallbackURL adds a callback query parameter for JSONP requests. func createCallbackURL(t *testing.T, endpoint string) string { t.Helper() u, err := url.Parse(endpoint) if err != nil { t.Fatal(err) } params := u.Query() params.Set("callback", "callback") u.RawQuery = params.Encode() return u.String() } // stripCallback removes the JSONP callback function from the response. // Returns the JSON body without the wrapping callback function. // If there's no callback function, the data is returned as is. func stripCallback(data []byte) []byte { // Cut the JSONP callback function with the opening parentheses. _, after, found := bytes.Cut(data, []byte("(")) if found { return bytes.TrimSuffix(after, []byte(")")) } return data } // expectHealthStatus makes 1 regular and 1 JSONP request to the URL and checks the // HTTP status code, Content-Type header and health status string. func expectHealthStatus(t *testing.T, url string, statusCode int, wantStatus string) { t.Helper() // First check for regular requests. body := readBodyEx(t, url, statusCode, appJSONContent) checkHealthStatus(t, body, wantStatus) // Another check for JSONP requests. jsonpURL := createCallbackURL(t, url) // Adds a callback query param. jsonpBody := readBodyEx(t, jsonpURL, statusCode, appJSContent) checkHealthStatus(t, stripCallback(jsonpBody), wantStatus) } // checkHealthStatus checks the health status from a JSON response. func checkHealthStatus(t *testing.T, body []byte, wantStatus string) { t.Helper() h := &HealthStatus{} if err := json.Unmarshal(body, h); err != nil { t.Fatalf("error unmarshalling the body: %v", err) } if h.Status != wantStatus { t.Errorf("want health status %q, got %q", wantStatus, h.Status) } } // checkHealthzEndpoint makes requests to the /healthz endpoint and checks the health status. func checkHealthzEndpoint(t *testing.T, address string, statusCode int, wantStatus string) { t.Helper() cases := map[string]string{ "healthz": fmt.Sprintf("http://%s/healthz", address), "js-enabled-only": fmt.Sprintf("http://%s/healthz?js-enabled-only=true", address), "js-server-only": fmt.Sprintf("http://%s/healthz?js-server-only=true", address), } for name, url := range cases { t.Run(name, func(t *testing.T) { expectHealthStatus(t, url, statusCode, wantStatus) }) } } func TestHealthzStatusOK(t *testing.T) { s := runMonitorServer() defer s.Shutdown() checkHealthzEndpoint(t, s.MonitorAddr().String(), http.StatusOK, "ok") } func TestHealthzStatusError(t *testing.T) { s := runMonitorServer() defer s.Shutdown() // Intentionally causing an error in readyForConnections(). // Note: Private field access, taking advantage of having the tests in the same package. s.mu.Lock() sl := s.listener s.listener = nil s.mu.Unlock() checkHealthzEndpoint(t, s.MonitorAddr().String(), http.StatusInternalServerError, "error") // Restore for proper shutdown. s.mu.Lock() s.listener = sl s.mu.Unlock() } func TestHealthzStatusUnavailable(t *testing.T) { opts := DefaultMonitorOptions() opts.JetStream = true s := RunServer(opts) defer s.Shutdown() if !s.JetStreamEnabled() { t.Fatalf("want JetStream to be enabled first") } err := s.DisableJetStream() if err != nil { t.Fatalf("got an error disabling JetStream: %v", err) } checkHealthzEndpoint(t, s.MonitorAddr().String(), http.StatusServiceUnavailable, "unavailable") } // When we converted ipq to use generics we still were using sync.Map. Currently you can not convert // any or any to a generic parameterized type. So this stopped working and panics. func TestIpqzWithGenerics(t *testing.T) { opts := DefaultMonitorOptions() opts.JetStream = true s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("http://%s/ipqueuesz?all=1", s.MonitorAddr().String()) body := readBody(t, url) require_True(t, len(body) > 0) queues := IpqueueszStatus{} require_NoError(t, json.Unmarshal(body, &queues)) require_True(t, len(queues) >= 4) _, ok := queues["SendQ"] require_True(t, ok) } func TestVarzSyncInterval(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultMonitorOptions() opts.JetStream = true opts.SyncInterval = 22 * time.Second opts.SyncAlways = true s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/varz", s.MonitorAddr().Port) jscfg := pollVarz(t, s, 0, url, nil).JetStream.Config require_True(t, jscfg.SyncInterval == opts.SyncInterval) require_True(t, jscfg.SyncAlways) } nats-server-2.10.27/server/mqtt.go000066400000000000000000005012521477524627100170020ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "cmp" "crypto/tls" "encoding/binary" "encoding/json" "errors" "fmt" "io" "net" "net/http" "slices" "strconv" "strings" "sync" "time" "unicode/utf8" "github.com/nats-io/nuid" ) // References to "spec" here is from https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.pdf const ( mqttPacketConnect = byte(0x10) mqttPacketConnectAck = byte(0x20) mqttPacketPub = byte(0x30) mqttPacketPubAck = byte(0x40) mqttPacketPubRec = byte(0x50) mqttPacketPubRel = byte(0x60) mqttPacketPubComp = byte(0x70) mqttPacketSub = byte(0x80) mqttPacketSubAck = byte(0x90) mqttPacketUnsub = byte(0xa0) mqttPacketUnsubAck = byte(0xb0) mqttPacketPing = byte(0xc0) mqttPacketPingResp = byte(0xd0) mqttPacketDisconnect = byte(0xe0) mqttPacketMask = byte(0xf0) mqttPacketFlagMask = byte(0x0f) mqttProtoLevel = byte(0x4) // Connect flags mqttConnFlagReserved = byte(0x1) mqttConnFlagCleanSession = byte(0x2) mqttConnFlagWillFlag = byte(0x04) mqttConnFlagWillQoS = byte(0x18) mqttConnFlagWillRetain = byte(0x20) mqttConnFlagPasswordFlag = byte(0x40) mqttConnFlagUsernameFlag = byte(0x80) // Publish flags mqttPubFlagRetain = byte(0x01) mqttPubFlagQoS = byte(0x06) mqttPubFlagDup = byte(0x08) mqttPubQos1 = byte(0x1 << 1) mqttPubQoS2 = byte(0x2 << 1) // Subscribe flags mqttSubscribeFlags = byte(0x2) mqttSubAckFailure = byte(0x80) // Unsubscribe flags mqttUnsubscribeFlags = byte(0x2) // ConnAck returned codes mqttConnAckRCConnectionAccepted = byte(0x0) mqttConnAckRCUnacceptableProtocolVersion = byte(0x1) mqttConnAckRCIdentifierRejected = byte(0x2) mqttConnAckRCServerUnavailable = byte(0x3) mqttConnAckRCBadUserOrPassword = byte(0x4) mqttConnAckRCNotAuthorized = byte(0x5) mqttConnAckRCQoS2WillRejected = byte(0x10) // Maximum payload size of a control packet mqttMaxPayloadSize = 0xFFFFFFF // Topic/Filter characters mqttTopicLevelSep = '/' mqttSingleLevelWC = '+' mqttMultiLevelWC = '#' mqttReservedPre = '$' // This is appended to the sid of a subscription that is // created on the upper level subject because of the MQTT // wildcard '#' semantic. mqttMultiLevelSidSuffix = " fwc" // This is the prefix for NATS subscriptions subjects associated as delivery // subject of JS consumer. We want to make them unique so will prevent users // MQTT subscriptions to start with this. mqttSubPrefix = "$MQTT.sub." // Stream name for MQTT messages on a given account mqttStreamName = "$MQTT_msgs" mqttStreamSubjectPrefix = "$MQTT.msgs." // Stream name for MQTT retained messages on a given account mqttRetainedMsgsStreamName = "$MQTT_rmsgs" mqttRetainedMsgsStreamSubject = "$MQTT.rmsgs." // Stream name for MQTT sessions on a given account mqttSessStreamName = "$MQTT_sess" mqttSessStreamSubjectPrefix = "$MQTT.sess." // Stream name prefix for MQTT sessions on a given account mqttSessionsStreamNamePrefix = "$MQTT_sess_" // Stream name and subject for incoming MQTT QoS2 messages mqttQoS2IncomingMsgsStreamName = "$MQTT_qos2in" mqttQoS2IncomingMsgsStreamSubjectPrefix = "$MQTT.qos2.in." // Stream name and subjects for outgoing MQTT QoS (PUBREL) messages mqttOutStreamName = "$MQTT_out" mqttOutSubjectPrefix = "$MQTT.out." mqttPubRelSubjectPrefix = "$MQTT.out.pubrel." mqttPubRelDeliverySubjectPrefix = "$MQTT.deliver.pubrel." mqttPubRelConsumerDurablePrefix = "$MQTT_PUBREL_" // As per spec, MQTT server may not redeliver QoS 1 and 2 messages to // clients, except after client reconnects. However, NATS Server will // redeliver unacknowledged messages after this default interval. This can // be changed with the server.Options.MQTT.AckWait option. mqttDefaultAckWait = 30 * time.Second // This is the default for the outstanding number of pending QoS 1 // messages sent to a session with QoS 1 subscriptions. mqttDefaultMaxAckPending = 1024 // A session's list of subscriptions cannot have a cumulative MaxAckPending // of more than this limit. mqttMaxAckTotalLimit = 0xFFFF // Prefix of the reply subject for JS API requests. mqttJSARepliesPrefix = "$MQTT.JSA." // Those are tokens that are used for the reply subject of JS API requests. // For instance "$MQTT.JSA..SC." is the reply subject // for a request to create a stream (where is the server name hash), // while "$MQTT.JSA..SL." is for a stream lookup, etc... mqttJSAIdTokenPos = 3 mqttJSATokenPos = 4 mqttJSAClientIDPos = 5 mqttJSAStreamCreate = "SC" mqttJSAStreamUpdate = "SU" mqttJSAStreamLookup = "SL" mqttJSAStreamDel = "SD" mqttJSAConsumerCreate = "CC" mqttJSAConsumerLookup = "CL" mqttJSAConsumerDel = "CD" mqttJSAMsgStore = "MS" mqttJSAMsgLoad = "ML" mqttJSAMsgDelete = "MD" mqttJSASessPersist = "SP" mqttJSARetainedMsgDel = "RD" mqttJSAStreamNames = "SN" // This is how long to keep a client in the flappers map before closing the // connection. This prevent quick reconnect from those clients that keep // wanting to connect with a client ID already in use. mqttSessFlappingJailDur = time.Second // This is how frequently the timer to cleanup the sessions flappers map is firing. mqttSessFlappingCleanupInterval = 5 * time.Second // Default retry delay if transfer of old session streams to new one fails mqttDefaultTransferRetry = 5 * time.Second // For Websocket URLs mqttWSPath = "/mqtt" mqttInitialPubHeader = 16 // An overkill, should need 7 bytes max mqttProcessSubTooLong = 100 * time.Millisecond mqttDefaultRetainedCacheTTL = 2 * time.Minute mqttRetainedTransferTimeout = 10 * time.Second ) var ( mqttPingResponse = []byte{mqttPacketPingResp, 0x0} mqttProtoName = []byte("MQTT") mqttOldProtoName = []byte("MQIsdp") mqttSessJailDur = mqttSessFlappingJailDur mqttFlapCleanItvl = mqttSessFlappingCleanupInterval mqttJSAPITimeout = 4 * time.Second mqttRetainedCacheTTL = mqttDefaultRetainedCacheTTL ) var ( errMQTTNotWebsocketPort = errors.New("MQTT clients over websocket must connect to the Websocket port, not the MQTT port") errMQTTTopicFilterCannotBeEmpty = errors.New("topic filter cannot be empty") errMQTTMalformedVarInt = errors.New("malformed variable int") errMQTTSecondConnectPacket = errors.New("received a second CONNECT packet") errMQTTServerNameMustBeSet = errors.New("mqtt requires server name to be explicitly set") errMQTTUserMixWithUsersNKeys = errors.New("mqtt authentication username not compatible with presence of users/nkeys") errMQTTTokenMixWIthUsersNKeys = errors.New("mqtt authentication token not compatible with presence of users/nkeys") errMQTTAckWaitMustBePositive = errors.New("ack wait must be a positive value") errMQTTStandaloneNeedsJetStream = errors.New("mqtt requires JetStream to be enabled if running in standalone mode") errMQTTConnFlagReserved = errors.New("connect flags reserved bit not set to 0") errMQTTWillAndRetainFlag = errors.New("if Will flag is set to 0, Will Retain flag must be 0 too") errMQTTPasswordFlagAndNoUser = errors.New("password flag set but username flag is not") errMQTTCIDEmptyNeedsCleanFlag = errors.New("when client ID is empty, clean session flag must be set to 1") errMQTTEmptyWillTopic = errors.New("empty Will topic not allowed") errMQTTEmptyUsername = errors.New("empty user name not allowed") errMQTTTopicIsEmpty = errors.New("topic cannot be empty") errMQTTPacketIdentifierIsZero = errors.New("packet identifier cannot be 0") errMQTTUnsupportedCharacters = errors.New("character ' ' not supported for MQTT topics") errMQTTInvalidSession = errors.New("invalid MQTT session") ) type srvMQTT struct { listener net.Listener listenerErr error authOverride bool sessmgr mqttSessionManager } type mqttSessionManager struct { mu sync.RWMutex sessions map[string]*mqttAccountSessionManager // key is account name } var testDisableRMSCache = false type mqttAccountSessionManager struct { mu sync.RWMutex sessions map[string]*mqttSession // key is MQTT client ID sessByHash map[string]*mqttSession // key is MQTT client ID hash sessLocked map[string]struct{} // key is MQTT client ID and indicate that a session can not be taken by a new client at this time flappers map[string]int64 // When connection connects with client ID already in use flapTimer *time.Timer // Timer to perform some cleanup of the flappers map sl *Sublist // sublist allowing to find retained messages for given subscription retmsgs map[string]*mqttRetainedMsgRef // retained messages rmsCache *sync.Map // map[subject]mqttRetainedMsg jsa mqttJSA rrmLastSeq uint64 // Restore retained messages expected last sequence rrmDoneCh chan struct{} // To notify the caller that all retained messages have been loaded domainTk string // Domain (with trailing "."), or possibly empty. This is added to session subject. } type mqttJSAResponse struct { reply string // will be used to map to the original request in jsa.NewRequestExMulti value any } type mqttJSA struct { mu sync.Mutex id string c *client sendq *ipQueue[*mqttJSPubMsg] rplyr string replies sync.Map // [string]chan *mqttJSAResponse nuid *nuid.NUID quitCh chan struct{} domain string // Domain or possibly empty. This is added to session subject. domainSet bool // covers if domain was set, even to empty } type mqttJSPubMsg struct { subj string reply string hdr int msg []byte } type mqttRetMsgDel struct { Subject string `json:"subject"` Seq uint64 `json:"seq"` } type mqttSession struct { // subsMu is a "quick" version of the session lock, sufficient for the QoS0 // callback. It only guarantees that a new subscription is initialized, and // its retained messages if any have been queued up for delivery. The QoS12 // callback uses the session lock. mu sync.Mutex subsMu sync.RWMutex id string // client ID idHash string // client ID hash c *client jsa *mqttJSA subs map[string]byte // Key is MQTT SUBSCRIBE filter, value is the subscription QoS cons map[string]*ConsumerConfig pubRelConsumer *ConsumerConfig pubRelSubscribed bool pubRelDeliverySubject string pubRelDeliverySubjectB []byte pubRelSubject string seq uint64 // pendingPublish maps packet identifiers (PI) to JetStream ACK subjects for // QoS1 and 2 PUBLISH messages pending delivery to the session's client. pendingPublish map[uint16]*mqttPending // pendingPubRel maps PIs to JetStream ACK subjects for QoS2 PUBREL // messages pending delivery to the session's client. pendingPubRel map[uint16]*mqttPending // cpending maps delivery attempts (that come with a JS ACK subject) to // existing PIs. cpending map[string]map[uint64]uint16 // composite key: jsDur, sseq // "Last used" publish packet identifier (PI). starting point searching for the next available. last_pi uint16 // Maximum number of pending acks for this session. maxp uint16 tmaxack int clean bool domainTk string } type mqttPersistedSession struct { Origin string `json:"origin,omitempty"` ID string `json:"id,omitempty"` Clean bool `json:"clean,omitempty"` Subs map[string]byte `json:"subs,omitempty"` Cons map[string]*ConsumerConfig `json:"cons,omitempty"` PubRel *ConsumerConfig `json:"pubrel,omitempty"` } type mqttRetainedMsg struct { Origin string `json:"origin,omitempty"` Subject string `json:"subject,omitempty"` Topic string `json:"topic,omitempty"` Msg []byte `json:"msg,omitempty"` Flags byte `json:"flags,omitempty"` Source string `json:"source,omitempty"` expiresFromCache time.Time } type mqttRetainedMsgRef struct { sseq uint64 floor uint64 sub *subscription } // mqttSub contains fields associated with a MQTT subscription, and is added to // the main subscription struct for MQTT message delivery subscriptions. The // delivery callbacks may get invoked before sub.mqtt is set up, so they should // acquire either sess.mu or sess.subsMu before accessing it. type mqttSub struct { // The sub's QOS and the JS durable name. They can change when // re-subscribing, and are used in the delivery callbacks. They can be // quickly accessed using sess.subsMu.RLock, or under the main session lock. qos byte jsDur string // Pending serialization of retained messages to be sent when subscription // is registered. The sub's delivery callbacks must wait until `prm` is // ready (can block on sess.mu for that, too). prm [][]byte // If this subscription needs to be checked for being reserved. E.g. '#' or // '*' or '*/'. It is set up at the time of subscription and is immutable // after that. reserved bool } type mqtt struct { r *mqttReader cp *mqttConnectProto pp *mqttPublish asm *mqttAccountSessionManager // quick reference to account session manager, immutable after processConnect() sess *mqttSession // quick reference to session, immutable after processConnect() cid string // client ID // rejectQoS2Pub tells the MQTT client to not accept QoS2 PUBLISH, instead // error and terminate the connection. rejectQoS2Pub bool // downgradeQOS2Sub tells the MQTT client to downgrade QoS2 SUBSCRIBE // requests to QoS1. downgradeQoS2Sub bool } type mqttPending struct { sseq uint64 // stream sequence jsAckSubject string // the ACK subject to send the ack to jsDur string // JS durable name } type mqttConnectProto struct { rd time.Duration will *mqttWill flags byte } type mqttIOReader interface { io.Reader SetReadDeadline(time.Time) error } type mqttReader struct { reader mqttIOReader buf []byte pos int pstart int pbuf []byte } type mqttWriter struct { bytes.Buffer } type mqttWill struct { topic []byte subject []byte mapped []byte message []byte qos byte retain bool } type mqttFilter struct { filter string qos byte // Used only for tracing and should not be used after parsing of (un)sub protocols. ttopic []byte } type mqttPublish struct { topic []byte subject []byte mapped []byte msg []byte sz int pi uint16 flags byte } // When we re-encode incoming MQTT PUBLISH messages for NATS delivery, we add // the following headers: // - "Nmqtt-Pub" (*always) indicates that the message originated from MQTT, and // contains the original message QoS. // - "Nmqtt-Subject" contains the original MQTT subject from mqttParsePub. // - "Nmqtt-Mapped" contains the mapping during mqttParsePub. // // When we submit a PUBREL for delivery, we add a "Nmqtt-PubRel" header that // contains the PI. const ( mqttNatsHeader = "Nmqtt-Pub" mqttNatsPubRelHeader = "Nmqtt-PubRel" mqttNatsHeaderSubject = "Nmqtt-Subject" mqttNatsHeaderMapped = "Nmqtt-Mapped" ) type mqttParsedPublishNATSHeader struct { qos byte subject []byte mapped []byte } func (s *Server) startMQTT() { if s.isShuttingDown() { return } sopts := s.getOpts() o := &sopts.MQTT var hl net.Listener var err error port := o.Port if port == -1 { port = 0 } hp := net.JoinHostPort(o.Host, strconv.Itoa(port)) s.mu.Lock() s.mqtt.sessmgr.sessions = make(map[string]*mqttAccountSessionManager) hl, err = net.Listen("tcp", hp) s.mqtt.listenerErr = err if err != nil { s.mu.Unlock() s.Fatalf("Unable to listen for MQTT connections: %v", err) return } if port == 0 { o.Port = hl.Addr().(*net.TCPAddr).Port } s.mqtt.listener = hl scheme := "mqtt" if o.TLSConfig != nil { scheme = "tls" } s.Noticef("Listening for MQTT clients on %s://%s:%d", scheme, o.Host, o.Port) go s.acceptConnections(hl, "MQTT", func(conn net.Conn) { s.createMQTTClient(conn, nil) }, nil) s.mu.Unlock() } // This is similar to createClient() but has some modifications specifi to MQTT clients. // The comments have been kept to minimum to reduce code size. Check createClient() for // more details. func (s *Server) createMQTTClient(conn net.Conn, ws *websocket) *client { opts := s.getOpts() maxPay := int32(opts.MaxPayload) maxSubs := int32(opts.MaxSubs) if maxSubs == 0 { maxSubs = -1 } now := time.Now() mqtt := &mqtt{ rejectQoS2Pub: opts.MQTT.rejectQoS2Pub, downgradeQoS2Sub: opts.MQTT.downgradeQoS2Sub, } c := &client{srv: s, nc: conn, mpay: maxPay, msubs: maxSubs, start: now, last: now, mqtt: mqtt, ws: ws} c.headers = true c.mqtt.pp = &mqttPublish{} // MQTT clients don't send NATS CONNECT protocols. So make it an "echo" // client, but disable verbose and pedantic (by not setting them). c.opts.Echo = true c.registerWithAccount(s.globalAccount()) s.mu.Lock() // Check auth, override if applicable. authRequired := s.info.AuthRequired || s.mqtt.authOverride s.totalClients++ s.mu.Unlock() c.mu.Lock() if authRequired { c.flags.set(expectConnect) } c.initClient() c.Debugf("Client connection created") c.mu.Unlock() s.mu.Lock() if !s.isRunning() || s.ldm { if s.isShuttingDown() { conn.Close() } s.mu.Unlock() return c } if opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn { s.mu.Unlock() c.maxConnExceeded() return nil } s.clients[c.cid] = c // Websocket TLS handshake is already done when getting to this function. tlsRequired := opts.MQTT.TLSConfig != nil && ws == nil s.mu.Unlock() c.mu.Lock() // In case connection has already been closed if c.isClosed() { c.mu.Unlock() c.closeConnection(WriteError) return nil } var pre []byte if tlsRequired && opts.AllowNonTLS { pre = make([]byte, 4) c.nc.SetReadDeadline(time.Now().Add(secondsToDuration(opts.MQTT.TLSTimeout))) n, _ := io.ReadFull(c.nc, pre[:]) c.nc.SetReadDeadline(time.Time{}) pre = pre[:n] if n > 0 && pre[0] == 0x16 { tlsRequired = true } else { tlsRequired = false } } if tlsRequired { if len(pre) > 0 { c.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)} pre = nil } // Perform server-side TLS handshake. if err := c.doTLSServerHandshake(tlsHandshakeMQTT, opts.MQTT.TLSConfig, opts.MQTT.TLSTimeout, opts.MQTT.TLSPinnedCerts); err != nil { c.mu.Unlock() return nil } } if authRequired { timeout := opts.AuthTimeout // Possibly override with MQTT specific value. if opts.MQTT.AuthTimeout != 0 { timeout = opts.MQTT.AuthTimeout } c.setAuthTimer(secondsToDuration(timeout)) } // No Ping timer for MQTT clients... s.startGoRoutine(func() { c.readLoop(pre) }) s.startGoRoutine(func() { c.writeLoop() }) if tlsRequired { c.Debugf("TLS handshake complete") cs := c.nc.(*tls.Conn).ConnectionState() c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tlsCipher(cs.CipherSuite)) } c.mu.Unlock() return c } // Given the mqtt options, we check if any auth configuration // has been provided. If so, possibly create users/nkey users and // store them in s.mqtt.users/nkeys. // Also update a boolean that indicates if auth is required for // mqtt clients. // Server lock is held on entry. func (s *Server) mqttConfigAuth(opts *MQTTOpts) { mqtt := &s.mqtt // If any of those is specified, we consider that there is an override. mqtt.authOverride = opts.Username != _EMPTY_ || opts.Token != _EMPTY_ || opts.NoAuthUser != _EMPTY_ } // Validate the mqtt related options. func validateMQTTOptions(o *Options) error { mo := &o.MQTT // If no port is defined, we don't care about other options if mo.Port == 0 { return nil } // We have to force the server name to be explicitly set and be unique when // in cluster mode. if o.ServerName == _EMPTY_ && (o.Cluster.Port != 0 || o.Gateway.Port != 0) { return errMQTTServerNameMustBeSet } // If there is a NoAuthUser, we need to have Users defined and // the user to be present. if mo.NoAuthUser != _EMPTY_ { if err := validateNoAuthUser(o, mo.NoAuthUser); err != nil { return err } } // Token/Username not possible if there are users/nkeys if len(o.Users) > 0 || len(o.Nkeys) > 0 { if mo.Username != _EMPTY_ { return errMQTTUserMixWithUsersNKeys } if mo.Token != _EMPTY_ { return errMQTTTokenMixWIthUsersNKeys } } if mo.AckWait < 0 { return errMQTTAckWaitMustBePositive } // If strictly standalone and there is no JS enabled, then it won't work... // For leafnodes, we could either have remote(s) and it would be ok, or no // remote but accept from a remote side that has "hub" property set, which // then would ok too. So we fail only if we have no leafnode config at all. if !o.JetStream && o.Cluster.Port == 0 && o.Gateway.Port == 0 && o.LeafNode.Port == 0 && len(o.LeafNode.Remotes) == 0 { return errMQTTStandaloneNeedsJetStream } if err := validatePinnedCerts(mo.TLSPinnedCerts); err != nil { return fmt.Errorf("mqtt: %v", err) } if mo.ConsumerReplicas > 0 && mo.StreamReplicas > 0 && mo.ConsumerReplicas > mo.StreamReplicas { return fmt.Errorf("mqtt: consumer_replicas (%v) cannot be higher than stream_replicas (%v)", mo.ConsumerReplicas, mo.StreamReplicas) } return nil } // Returns true if this connection is from a MQTT client. // Lock held on entry. func (c *client) isMqtt() bool { return c.mqtt != nil } // If this is an MQTT client, returns the session client ID, // otherwise returns the empty string. // Lock held on entry func (c *client) getMQTTClientID() string { if !c.isMqtt() { return _EMPTY_ } return c.mqtt.cid } // Parse protocols inside the given buffer. // This is invoked from the readLoop. func (c *client) mqttParse(buf []byte) error { c.mu.Lock() s := c.srv trace := c.trace connected := c.flags.isSet(connectReceived) mqtt := c.mqtt r := mqtt.r var rd time.Duration if mqtt.cp != nil { rd = mqtt.cp.rd if rd > 0 { r.reader.SetReadDeadline(time.Time{}) } } hasMappings := c.in.flags.isSet(hasMappings) c.mu.Unlock() r.reset(buf) var err error var b byte var pl int var complete bool for err == nil && r.hasMore() { // Keep track of the starting of the packet, in case we have a partial r.pstart = r.pos // Read packet type and flags if b, err = r.readByte("packet type"); err != nil { break } // Packet type pt := b & mqttPacketMask // If client was not connected yet, the first packet must be // a mqttPacketConnect otherwise we fail the connection. if !connected && pt != mqttPacketConnect { // If the buffer indicates that it may be a websocket handshake // but the client is not websocket, it means that the client // connected to the MQTT port instead of the Websocket port. if bytes.HasPrefix(buf, []byte("GET ")) && !c.isWebsocket() { err = errMQTTNotWebsocketPort } else { err = fmt.Errorf("the first packet should be a CONNECT (%v), got %v", mqttPacketConnect, pt) } break } pl, complete, err = r.readPacketLen() if err != nil || !complete { break } switch pt { // Packets that we receive back when we act as the "sender": PUBACK, // PUBREC, PUBCOMP. case mqttPacketPubAck: var pi uint16 pi, err = mqttParsePIPacket(r) if trace { c.traceInOp("PUBACK", errOrTrace(err, fmt.Sprintf("pi=%v", pi))) } if err == nil { err = c.mqttProcessPubAck(pi) } case mqttPacketPubRec: var pi uint16 pi, err = mqttParsePIPacket(r) if trace { c.traceInOp("PUBREC", errOrTrace(err, fmt.Sprintf("pi=%v", pi))) } if err == nil { err = c.mqttProcessPubRec(pi) } case mqttPacketPubComp: var pi uint16 pi, err = mqttParsePIPacket(r) if trace { c.traceInOp("PUBCOMP", errOrTrace(err, fmt.Sprintf("pi=%v", pi))) } if err == nil { c.mqttProcessPubComp(pi) } // Packets where we act as the "receiver": PUBLISH, PUBREL, SUBSCRIBE, UNSUBSCRIBE. case mqttPacketPub: pp := c.mqtt.pp pp.flags = b & mqttPacketFlagMask err = c.mqttParsePub(r, pl, pp, hasMappings) if trace { c.traceInOp("PUBLISH", errOrTrace(err, mqttPubTrace(pp))) if err == nil { c.mqttTraceMsg(pp.msg) } } if err == nil { err = s.mqttProcessPub(c, pp, trace) } case mqttPacketPubRel: var pi uint16 pi, err = mqttParsePIPacket(r) if trace { c.traceInOp("PUBREL", errOrTrace(err, fmt.Sprintf("pi=%v", pi))) } if err == nil { err = s.mqttProcessPubRel(c, pi, trace) } case mqttPacketSub: var pi uint16 // packet identifier var filters []*mqttFilter var subs []*subscription pi, filters, err = c.mqttParseSubs(r, b, pl) if trace { c.traceInOp("SUBSCRIBE", errOrTrace(err, mqttSubscribeTrace(pi, filters))) } if err == nil { subs, err = c.mqttProcessSubs(filters) if err == nil && trace { c.traceOutOp("SUBACK", []byte(fmt.Sprintf("pi=%v", pi))) } } if err == nil { c.mqttEnqueueSubAck(pi, filters) c.mqttSendRetainedMsgsToNewSubs(subs) } case mqttPacketUnsub: var pi uint16 // packet identifier var filters []*mqttFilter pi, filters, err = c.mqttParseUnsubs(r, b, pl) if trace { c.traceInOp("UNSUBSCRIBE", errOrTrace(err, mqttUnsubscribeTrace(pi, filters))) } if err == nil { err = c.mqttProcessUnsubs(filters) if err == nil && trace { c.traceOutOp("UNSUBACK", []byte(fmt.Sprintf("pi=%v", pi))) } } if err == nil { c.mqttEnqueueUnsubAck(pi) } // Packets that we get both as a receiver and sender: PING, CONNECT, DISCONNECT case mqttPacketPing: if trace { c.traceInOp("PINGREQ", nil) } c.mqttEnqueuePingResp() if trace { c.traceOutOp("PINGRESP", nil) } case mqttPacketConnect: // It is an error to receive a second connect packet if connected { err = errMQTTSecondConnectPacket break } var rc byte var cp *mqttConnectProto var sessp bool rc, cp, err = c.mqttParseConnect(r, hasMappings) // Add the client id to the client's string, regardless of error. // We may still get the client_id if the call above fails somewhere // after parsing the client ID itself. c.ncs.Store(fmt.Sprintf("%s - %q", c, c.mqtt.cid)) if trace && cp != nil { c.traceInOp("CONNECT", errOrTrace(err, c.mqttConnectTrace(cp))) } if rc != 0 { c.mqttEnqueueConnAck(rc, sessp) if trace { c.traceOutOp("CONNACK", []byte(fmt.Sprintf("sp=%v rc=%v", sessp, rc))) } } else if err == nil { if err = s.mqttProcessConnect(c, cp, trace); err != nil { err = fmt.Errorf("unable to connect: %v", err) } else { // Add this debug statement so users running in Debug mode // will have the client id printed here for the first time. c.Debugf("Client connected") connected = true rd = cp.rd } } case mqttPacketDisconnect: if trace { c.traceInOp("DISCONNECT", nil) } // Normal disconnect, we need to discard the will. // Spec [MQTT-3.1.2-8] c.mu.Lock() if c.mqtt.cp != nil { c.mqtt.cp.will = nil } c.mu.Unlock() s.mqttHandleClosedClient(c) c.closeConnection(ClientClosed) return nil default: err = fmt.Errorf("received unknown packet type %d", pt>>4) } } if err == nil && rd > 0 { r.reader.SetReadDeadline(time.Now().Add(rd)) } return err } func (c *client) mqttTraceMsg(msg []byte) { maxTrace := c.srv.getOpts().MaxTracedMsgLen if maxTrace > 0 && len(msg) > maxTrace { c.Tracef("<<- MSG_PAYLOAD: [\"%s...\"]", msg[:maxTrace]) } else { c.Tracef("<<- MSG_PAYLOAD: [%q]", msg) } } // The MQTT client connection has been closed, or the DISCONNECT packet was received. // For a "clean" session, we will delete the session, otherwise, simply removing // the binding. We will also send the "will" message if applicable. // // Runs from the client's readLoop. // No lock held on entry. func (s *Server) mqttHandleClosedClient(c *client) { c.mu.Lock() asm := c.mqtt.asm sess := c.mqtt.sess c.mu.Unlock() // If asm or sess are nil, it means that we have failed a client // before it was associated with a session, so nothing more to do. if asm == nil || sess == nil { return } // Add this session to the locked map for the rest of the execution. if err := asm.lockSession(sess, c); err != nil { return } defer asm.unlockSession(sess) asm.mu.Lock() // Clear the client from the session, but session may stay. sess.mu.Lock() sess.c = nil doClean := sess.clean sess.mu.Unlock() // If it was a clean session, then we remove from the account manager, // and we will call clear() outside of any lock. if doClean { asm.removeSession(sess, false) } // Remove in case it was in the flappers map. asm.removeSessFromFlappers(sess.id) asm.mu.Unlock() // This needs to be done outside of any lock. if doClean { if err := sess.clear(true); err != nil { c.Errorf(err.Error()) } } // Now handle the "will". This function will be a no-op if there is no "will" to send. s.mqttHandleWill(c) } // Updates the MaxAckPending for all MQTT sessions, updating the // JetStream consumers and updating their max ack pending and forcing // a expiration of pending messages. // // Runs from a server configuration reload routine. // No lock held on entry. func (s *Server) mqttUpdateMaxAckPending(newmaxp uint16) { msm := &s.mqtt.sessmgr s.accounts.Range(func(k, _ any) bool { accName := k.(string) msm.mu.RLock() asm := msm.sessions[accName] msm.mu.RUnlock() if asm == nil { // Move to next account return true } asm.mu.RLock() for _, sess := range asm.sessions { sess.mu.Lock() sess.maxp = newmaxp sess.mu.Unlock() } asm.mu.RUnlock() return true }) } func (s *Server) mqttGetJSAForAccount(acc string) *mqttJSA { sm := &s.mqtt.sessmgr sm.mu.RLock() asm := sm.sessions[acc] sm.mu.RUnlock() if asm == nil { return nil } asm.mu.RLock() jsa := &asm.jsa asm.mu.RUnlock() return jsa } func (s *Server) mqttStoreQoSMsgForAccountOnNewSubject(hdr int, msg []byte, acc, subject string) { if s == nil || hdr <= 0 { return } h := mqttParsePublishNATSHeader(msg[:hdr]) if h == nil || h.qos == 0 { return } jsa := s.mqttGetJSAForAccount(acc) if jsa == nil { return } jsa.storeMsg(mqttStreamSubjectPrefix+subject, hdr, msg) } func mqttParsePublishNATSHeader(headerBytes []byte) *mqttParsedPublishNATSHeader { if len(headerBytes) == 0 { return nil } pubValue := getHeader(mqttNatsHeader, headerBytes) if len(pubValue) == 0 { return nil } return &mqttParsedPublishNATSHeader{ qos: pubValue[0] - '0', subject: getHeader(mqttNatsHeaderSubject, headerBytes), mapped: getHeader(mqttNatsHeaderMapped, headerBytes), } } func mqttParsePubRelNATSHeader(headerBytes []byte) uint16 { if len(headerBytes) == 0 { return 0 } pubrelValue := getHeader(mqttNatsPubRelHeader, headerBytes) if len(pubrelValue) == 0 { return 0 } pi, _ := strconv.Atoi(string(pubrelValue)) return uint16(pi) } // Returns the MQTT sessions manager for a given account. // If new, creates the required JetStream streams/consumers // for handling of sessions and messages. func (s *Server) getOrCreateMQTTAccountSessionManager(c *client) (*mqttAccountSessionManager, error) { sm := &s.mqtt.sessmgr c.mu.Lock() acc := c.acc c.mu.Unlock() accName := acc.GetName() sm.mu.RLock() asm, ok := sm.sessions[accName] sm.mu.RUnlock() if ok { return asm, nil } // We will pass the quitCh to the account session manager if we happen to create it. s.mu.Lock() quitCh := s.quitCh s.mu.Unlock() // Not found, now take the write lock and check again sm.mu.Lock() defer sm.mu.Unlock() asm, ok = sm.sessions[accName] if ok { return asm, nil } // Need to create one here. asm, err := s.mqttCreateAccountSessionManager(acc, quitCh) if err != nil { return nil, err } sm.sessions[accName] = asm return asm, nil } // Creates JS streams/consumers for handling of sessions and messages for this account. // // Global session manager lock is held on entry. func (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struct{}) (*mqttAccountSessionManager, error) { var err error accName := acc.GetName() opts := s.getOpts() c := s.createInternalAccountClient() c.acc = acc id := s.NodeName() replicas := opts.MQTT.StreamReplicas if replicas <= 0 { replicas = s.mqttDetermineReplicas() } qname := fmt.Sprintf("[ACC:%s] MQTT ", accName) as := &mqttAccountSessionManager{ sessions: make(map[string]*mqttSession), sessByHash: make(map[string]*mqttSession), sessLocked: make(map[string]struct{}), flappers: make(map[string]int64), jsa: mqttJSA{ id: id, c: c, rplyr: mqttJSARepliesPrefix + id + ".", sendq: newIPQueue[*mqttJSPubMsg](s, qname+"send"), nuid: nuid.New(), quitCh: quitCh, }, } if !testDisableRMSCache { as.rmsCache = &sync.Map{} } // TODO record domain name in as here // The domain to communicate with may be required for JS calls. // Search from specific (per account setting) to generic (mqtt setting) if opts.JsAccDefaultDomain != nil { if d, ok := opts.JsAccDefaultDomain[accName]; ok { if d != _EMPTY_ { as.jsa.domain = d } as.jsa.domainSet = true } // in case domain was set to empty, check if there are more generic domain overwrites } if as.jsa.domain == _EMPTY_ { if d := opts.MQTT.JsDomain; d != _EMPTY_ { as.jsa.domain = d as.jsa.domainSet = true } } // We need to include the domain in the subject prefix used to store sessions in the $MQTT_sess stream. if as.jsa.domainSet { if as.jsa.domain != _EMPTY_ { as.domainTk = as.jsa.domain + "." } } else if d := s.getOpts().JetStreamDomain; d != _EMPTY_ { as.domainTk = d + "." } if as.jsa.domainSet { s.Noticef("Creating MQTT streams/consumers with replicas %v for account %q in domain %q", replicas, accName, as.jsa.domain) } else { s.Noticef("Creating MQTT streams/consumers with replicas %v for account %q", replicas, accName) } var subs []*subscription var success bool closeCh := make(chan struct{}) defer func() { if success { return } for _, sub := range subs { c.processUnsub(sub.sid) } close(closeCh) }() // We create all subscriptions before starting the go routine that will do // sends otherwise we could get races. // Note that using two different clients (one for the subs, one for the // sends) would cause other issues such as registration of recent subs in // the "sub" client would be invisible to the check for GW routed replies // (shouldMapReplyForGatewaySend) since the client there would be the "sender". jsa := &as.jsa sid := int64(1) // This is a subscription that will process all JS API replies. We could split to // individual subscriptions if needed, but since there is a bit of common code, // that seemed like a good idea to be all in one place. if err := as.createSubscription(jsa.rplyr+">", as.processJSAPIReplies, &sid, &subs); err != nil { return nil, err } // We will listen for replies to session persist requests so that we can // detect the use of a session with the same client ID anywhere in the cluster. // `$MQTT.JSA.{js-id}.SP.{client-id-hash}.{uuid}` if err := as.createSubscription(mqttJSARepliesPrefix+"*."+mqttJSASessPersist+".*.*", as.processSessionPersist, &sid, &subs); err != nil { return nil, err } // We create the subscription on "$MQTT.sub." to limit the subjects // that a user would allow permissions on. rmsubj := mqttSubPrefix + nuid.Next() if err := as.createSubscription(rmsubj, as.processRetainedMsg, &sid, &subs); err != nil { return nil, err } // Create a subscription to be notified of retained messages delete requests. rmdelsubj := mqttJSARepliesPrefix + "*." + mqttJSARetainedMsgDel if err := as.createSubscription(rmdelsubj, as.processRetainedMsgDel, &sid, &subs); err != nil { return nil, err } // No more creation of subscriptions past this point otherwise RACEs may happen. // Start the go routine that will send JS API requests. s.startGoRoutine(func() { defer s.grWG.Done() as.sendJSAPIrequests(s, c, accName, closeCh) }) // Start the go routine that will clean up cached retained messages that expired. if as.rmsCache != nil { s.startGoRoutine(func() { defer s.grWG.Done() as.cleanupRetainedMessageCache(s, closeCh) }) } lookupStream := func(stream, txt string) (*StreamInfo, error) { si, err := jsa.lookupStream(stream) if err != nil { if IsNatsErr(err, JSStreamNotFoundErr) { return nil, nil } return nil, fmt.Errorf("lookup %s stream for account %q: %v", txt, accName, err) } if opts.MQTT.StreamReplicas == 0 { return si, nil } sr := 1 if si.Cluster != nil { sr += len(si.Cluster.Replicas) } if replicas != sr { s.Warnf("MQTT %s stream replicas mismatch: current is %v but configuration is %v for '%s > %s'", txt, sr, replicas, accName, stream) } return si, nil } if si, err := lookupStream(mqttSessStreamName, "sessions"); err != nil { return nil, err } else if si == nil { // Create the stream for the sessions. cfg := &StreamConfig{ Name: mqttSessStreamName, Subjects: []string{mqttSessStreamSubjectPrefix + as.domainTk + ">"}, Storage: FileStorage, Retention: LimitsPolicy, Replicas: replicas, MaxMsgsPer: 1, } if _, created, err := jsa.createStream(cfg); err == nil && created { as.transferUniqueSessStreamsToMuxed(s) } else if isErrorOtherThan(err, JSStreamNameExistErr) { return nil, fmt.Errorf("create sessions stream for account %q: %v", accName, err) } } if si, err := lookupStream(mqttStreamName, "messages"); err != nil { return nil, err } else if si == nil { // Create the stream for the messages. cfg := &StreamConfig{ Name: mqttStreamName, Subjects: []string{mqttStreamSubjectPrefix + ">"}, Storage: FileStorage, Retention: InterestPolicy, Replicas: replicas, } if _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) { return nil, fmt.Errorf("create messages stream for account %q: %v", accName, err) } } if si, err := lookupStream(mqttQoS2IncomingMsgsStreamName, "QoS2 incoming messages"); err != nil { return nil, err } else if si == nil { // Create the stream for the incoming QoS2 messages that have not been // PUBREL-ed by the sender. Subject is // "$MQTT.qos2..", the .PI is to achieve exactly // once for each PI. cfg := &StreamConfig{ Name: mqttQoS2IncomingMsgsStreamName, Subjects: []string{mqttQoS2IncomingMsgsStreamSubjectPrefix + ">"}, Storage: FileStorage, Retention: LimitsPolicy, Discard: DiscardNew, MaxMsgsPer: 1, DiscardNewPer: true, Replicas: replicas, } if _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) { return nil, fmt.Errorf("create QoS2 incoming messages stream for account %q: %v", accName, err) } } if si, err := lookupStream(mqttOutStreamName, "QoS2 outgoing PUBREL"); err != nil { return nil, err } else if si == nil { // Create the stream for the incoming QoS2 messages that have not been // PUBREL-ed by the sender. NATS messages are submitted as // "$MQTT.pubrel." cfg := &StreamConfig{ Name: mqttOutStreamName, Subjects: []string{mqttOutSubjectPrefix + ">"}, Storage: FileStorage, Retention: InterestPolicy, Replicas: replicas, } if _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) { return nil, fmt.Errorf("create QoS2 outgoing PUBREL stream for account %q: %v", accName, err) } } // This is the only case where we need "si" after lookup/create needToTransfer := true si, err := lookupStream(mqttRetainedMsgsStreamName, "retained messages") switch { case err != nil: return nil, err case si == nil: // Create the stream for retained messages. cfg := &StreamConfig{ Name: mqttRetainedMsgsStreamName, Subjects: []string{mqttRetainedMsgsStreamSubject + ">"}, Storage: FileStorage, Retention: LimitsPolicy, Replicas: replicas, MaxMsgsPer: 1, } // We will need "si" outside of this block. si, _, err = jsa.createStream(cfg) if err != nil { if isErrorOtherThan(err, JSStreamNameExistErr) { return nil, fmt.Errorf("create retained messages stream for account %q: %v", accName, err) } // Suppose we had a race and the stream was actually created by another // node, we really need "si" after that, so lookup the stream again here. si, err = lookupStream(mqttRetainedMsgsStreamName, "retained messages") if err != nil { return nil, err } } needToTransfer = false default: needToTransfer = si.Config.MaxMsgsPer != 1 } // Doing this check outside of above if/else due to possible race when // creating the stream. wantedSubj := mqttRetainedMsgsStreamSubject + ">" if len(si.Config.Subjects) != 1 || si.Config.Subjects[0] != wantedSubj { // Update only the Subjects at this stage, not MaxMsgsPer yet. si.Config.Subjects = []string{wantedSubj} if si, err = jsa.updateStream(&si.Config); err != nil { return nil, fmt.Errorf("failed to update stream config: %w", err) } } transferRMS := func() error { if !needToTransfer { return nil } as.transferRetainedToPerKeySubjectStream(s) // We need another lookup to have up-to-date si.State values in order // to load all retained messages. si, err = lookupStream(mqttRetainedMsgsStreamName, "retained messages") if err != nil { return err } needToTransfer = false return nil } // Attempt to transfer all "single subject" retained messages to new // subjects. It may fail, will log its own error; ignore it the first time // and proceed to updating MaxMsgsPer. Then we invoke transferRMS() again, // which will get another chance to resolve the error; if not we bail there. if err = transferRMS(); err != nil { return nil, err } // Now, if the stream does not have MaxMsgsPer set to 1, and there are no // more messages on the single $MQTT.rmsgs subject, update the stream again. if si.Config.MaxMsgsPer != 1 { si.Config.MaxMsgsPer = 1 // We will need an up-to-date si, so don't use local variable here. if si, err = jsa.updateStream(&si.Config); err != nil { return nil, fmt.Errorf("failed to update stream config: %w", err) } } // If we failed the first time, there is now at most one lingering message // in the old subject. Try again (it will be a NO-OP if succeeded the first // time). if err = transferRMS(); err != nil { return nil, err } var lastSeq uint64 var rmDoneCh chan struct{} st := si.State if st.Msgs > 0 { lastSeq = st.LastSeq if lastSeq > 0 { rmDoneCh = make(chan struct{}) as.rrmLastSeq = lastSeq as.rrmDoneCh = rmDoneCh } } // Opportunistically delete the old (legacy) consumer, from v2.10.10 and // before. Ignore any errors that might arise. rmLegacyDurName := mqttRetainedMsgsStreamName + "_" + jsa.id jsa.deleteConsumer(mqttRetainedMsgsStreamName, rmLegacyDurName, true) // Create a new, uniquely names consumer for retained messages for this // server. The prior one will expire eventually. ccfg := &CreateConsumerRequest{ Stream: mqttRetainedMsgsStreamName, Config: ConsumerConfig{ Name: mqttRetainedMsgsStreamName + "_" + nuid.Next(), FilterSubject: mqttRetainedMsgsStreamSubject + ">", DeliverSubject: rmsubj, ReplayPolicy: ReplayInstant, AckPolicy: AckNone, InactiveThreshold: 5 * time.Minute, }, } if _, err := jsa.createEphemeralConsumer(ccfg); err != nil { return nil, fmt.Errorf("create retained messages consumer for account %q: %v", accName, err) } if lastSeq > 0 { ttl := time.NewTimer(mqttJSAPITimeout) defer ttl.Stop() select { case <-rmDoneCh: case <-ttl.C: s.Warnf("Timing out waiting to load %v retained messages", st.Msgs) case <-quitCh: return nil, ErrServerNotRunning } } // Set this so that on defer we don't cleanup. success = true return as, nil } func (s *Server) mqttDetermineReplicas() int { // If not clustered, then replica will be 1. if !s.JetStreamIsClustered() { return 1 } opts := s.getOpts() replicas := 0 for _, u := range opts.Routes { host := u.Hostname() // If this is an IP just add one. if net.ParseIP(host) != nil { replicas++ } else { addrs, _ := net.LookupHost(host) replicas += len(addrs) } } if replicas < 1 { replicas = 1 } else if replicas > 3 { replicas = 3 } return replicas } ////////////////////////////////////////////////////////////////////////////// // // JS APIs related functions // ////////////////////////////////////////////////////////////////////////////// func (jsa *mqttJSA) newRequest(kind, subject string, hdr int, msg []byte) (any, error) { return jsa.newRequestEx(kind, subject, _EMPTY_, hdr, msg, mqttJSAPITimeout) } func (jsa *mqttJSA) prefixDomain(subject string) string { if jsa.domain != _EMPTY_ { // rewrite js api prefix with domain if sub := strings.TrimPrefix(subject, JSApiPrefix+"."); sub != subject { subject = fmt.Sprintf("$JS.%s.API.%s", jsa.domain, sub) } } return subject } func (jsa *mqttJSA) newRequestEx(kind, subject, cidHash string, hdr int, msg []byte, timeout time.Duration) (any, error) { responses, err := jsa.newRequestExMulti(kind, subject, cidHash, []int{hdr}, [][]byte{msg}, timeout) if err != nil { return nil, err } if len(responses) != 1 { return nil, fmt.Errorf("unreachable: invalid number of responses (%d)", len(responses)) } return responses[0].value, nil } // newRequestExMulti sends multiple messages on the same subject and waits for // all responses. It returns the same number of responses in the same order as // msgs parameter. In case of a timeout it returns an error as well as all // responses received as a sparsely populated array, matching msgs, with nils // for the values that have not yet been received. // // Note that each response may represent an error and should be inspected as // such by the caller. func (jsa *mqttJSA) newRequestExMulti(kind, subject, cidHash string, hdrs []int, msgs [][]byte, timeout time.Duration) ([]*mqttJSAResponse, error) { if len(hdrs) != len(msgs) { return nil, fmt.Errorf("unreachable: invalid number of messages (%d) or header offsets (%d)", len(msgs), len(hdrs)) } responseCh := make(chan *mqttJSAResponse, len(msgs)) // Generate and queue all outgoing requests, have all results reported to // responseCh, and store a map of reply subjects to the original subjects' // indices. r2i := map[string]int{} for i, msg := range msgs { hdr := hdrs[i] var sb strings.Builder // Either we use nuid.Next() which uses a global lock, or our own nuid object, but // then it needs to be "write" protected. This approach will reduce across account // contention since we won't use the global nuid's lock. jsa.mu.Lock() uid := jsa.nuid.Next() sb.WriteString(jsa.rplyr) jsa.mu.Unlock() sb.WriteString(kind) sb.WriteByte(btsep) if cidHash != _EMPTY_ { sb.WriteString(cidHash) sb.WriteByte(btsep) } sb.WriteString(uid) reply := sb.String() // Add responseCh to the reply channel map. It will be cleaned out on // timeout (see below), or in processJSAPIReplies upon receiving the // response. jsa.replies.Store(reply, responseCh) subject = jsa.prefixDomain(subject) jsa.sendq.push(&mqttJSPubMsg{ subj: subject, reply: reply, hdr: hdr, msg: msg, }) r2i[reply] = i } // Wait for all responses to come back, or for the timeout to expire. We // don't want to use time.After() which causes memory growth because the // timer can't be stopped and will need to expire to then be garbage // collected. c := 0 responses := make([]*mqttJSAResponse, len(msgs)) start := time.Now() t := time.NewTimer(timeout) defer t.Stop() for { select { case r := <-responseCh: i := r2i[r.reply] responses[i] = r c++ if c == len(msgs) { return responses, nil } case <-jsa.quitCh: return nil, ErrServerNotRunning case <-t.C: var reply string now := time.Now() for reply = range r2i { // preserve the last value for Errorf jsa.replies.Delete(reply) } if len(msgs) == 1 { return responses, fmt.Errorf("timeout after %v: request type %q on %q (reply=%q)", now.Sub(start), kind, subject, reply) } else { return responses, fmt.Errorf("timeout after %v: request type %q on %q: got %d out of %d", now.Sub(start), kind, subject, c, len(msgs)) } } } } func (jsa *mqttJSA) sendAck(ackSubject string) { if ackSubject == _EMPTY_ { return } // We pass -1 for the hdr so that the send loop does not need to // add the "client info" header. This is not a JS API request per se. jsa.sendq.push(&mqttJSPubMsg{subj: ackSubject, hdr: -1}) } func (jsa *mqttJSA) createEphemeralConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, err } subj := fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream) ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) if err != nil { return nil, err } ccr := ccri.(*JSApiConsumerCreateResponse) return ccr, ccr.ToError() } func (jsa *mqttJSA) createDurableConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, err } subj := fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable) ccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb) if err != nil { return nil, err } ccr := ccri.(*JSApiConsumerCreateResponse) return ccr, ccr.ToError() } func (jsa *mqttJSA) sendMsg(subj string, msg []byte) { if subj == _EMPTY_ { return } jsa.sendq.push(&mqttJSPubMsg{subj: subj, msg: msg, hdr: -1}) } // if noWait is specified, does not wait for the JS response, returns nil func (jsa *mqttJSA) deleteConsumer(streamName, consName string, noWait bool) (*JSApiConsumerDeleteResponse, error) { subj := fmt.Sprintf(JSApiConsumerDeleteT, streamName, consName) if noWait { jsa.sendMsg(subj, nil) return nil, nil } cdri, err := jsa.newRequest(mqttJSAConsumerDel, subj, 0, nil) if err != nil { return nil, err } cdr := cdri.(*JSApiConsumerDeleteResponse) return cdr, cdr.ToError() } func (jsa *mqttJSA) createStream(cfg *StreamConfig) (*StreamInfo, bool, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, false, err } scri, err := jsa.newRequest(mqttJSAStreamCreate, fmt.Sprintf(JSApiStreamCreateT, cfg.Name), 0, cfgb) if err != nil { return nil, false, err } scr := scri.(*JSApiStreamCreateResponse) return scr.StreamInfo, scr.DidCreate, scr.ToError() } func (jsa *mqttJSA) updateStream(cfg *StreamConfig) (*StreamInfo, error) { cfgb, err := json.Marshal(cfg) if err != nil { return nil, err } scri, err := jsa.newRequest(mqttJSAStreamUpdate, fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), 0, cfgb) if err != nil { return nil, err } scr := scri.(*JSApiStreamUpdateResponse) return scr.StreamInfo, scr.ToError() } func (jsa *mqttJSA) lookupStream(name string) (*StreamInfo, error) { slri, err := jsa.newRequest(mqttJSAStreamLookup, fmt.Sprintf(JSApiStreamInfoT, name), 0, nil) if err != nil { return nil, err } slr := slri.(*JSApiStreamInfoResponse) return slr.StreamInfo, slr.ToError() } func (jsa *mqttJSA) deleteStream(name string) (bool, error) { sdri, err := jsa.newRequest(mqttJSAStreamDel, fmt.Sprintf(JSApiStreamDeleteT, name), 0, nil) if err != nil { return false, err } sdr := sdri.(*JSApiStreamDeleteResponse) return sdr.Success, sdr.ToError() } func (jsa *mqttJSA) loadLastMsgFor(streamName string, subject string) (*StoredMsg, error) { mreq := &JSApiMsgGetRequest{LastFor: subject} req, err := json.Marshal(mreq) if err != nil { return nil, err } lmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req) if err != nil { return nil, err } lmr := lmri.(*JSApiMsgGetResponse) return lmr.Message, lmr.ToError() } func (jsa *mqttJSA) loadLastMsgForMulti(streamName string, subjects []string) ([]*JSApiMsgGetResponse, error) { marshaled := make([][]byte, 0, len(subjects)) headerBytes := make([]int, 0, len(subjects)) for _, subject := range subjects { mreq := &JSApiMsgGetRequest{LastFor: subject} bb, err := json.Marshal(mreq) if err != nil { return nil, err } marshaled = append(marshaled, bb) headerBytes = append(headerBytes, 0) } all, err := jsa.newRequestExMulti(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), _EMPTY_, headerBytes, marshaled, mqttJSAPITimeout) // all has the same order as subjects, preserve it as we unmarshal responses := make([]*JSApiMsgGetResponse, len(all)) for i, v := range all { if v != nil { responses[i] = v.value.(*JSApiMsgGetResponse) } } return responses, err } func (jsa *mqttJSA) loadNextMsgFor(streamName string, subject string) (*StoredMsg, error) { mreq := &JSApiMsgGetRequest{NextFor: subject} req, err := json.Marshal(mreq) if err != nil { return nil, err } lmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req) if err != nil { return nil, err } lmr := lmri.(*JSApiMsgGetResponse) return lmr.Message, lmr.ToError() } func (jsa *mqttJSA) loadMsg(streamName string, seq uint64) (*StoredMsg, error) { mreq := &JSApiMsgGetRequest{Seq: seq} req, err := json.Marshal(mreq) if err != nil { return nil, err } lmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req) if err != nil { return nil, err } lmr := lmri.(*JSApiMsgGetResponse) return lmr.Message, lmr.ToError() } func (jsa *mqttJSA) storeMsg(subject string, headers int, msg []byte) (*JSPubAckResponse, error) { return jsa.storeMsgWithKind(mqttJSAMsgStore, subject, headers, msg) } func (jsa *mqttJSA) storeMsgWithKind(kind, subject string, headers int, msg []byte) (*JSPubAckResponse, error) { smri, err := jsa.newRequest(kind, subject, headers, msg) if err != nil { return nil, err } smr := smri.(*JSPubAckResponse) return smr, smr.ToError() } func (jsa *mqttJSA) storeSessionMsg(domainTk, cidHash string, hdr int, msg []byte) (*JSPubAckResponse, error) { // Compute subject where the session is being stored subject := mqttSessStreamSubjectPrefix + domainTk + cidHash // Passing cidHash will add it to the JS reply subject, so that we can use // it in processSessionPersist. smri, err := jsa.newRequestEx(mqttJSASessPersist, subject, cidHash, hdr, msg, mqttJSAPITimeout) if err != nil { return nil, err } smr := smri.(*JSPubAckResponse) return smr, smr.ToError() } func (jsa *mqttJSA) loadSessionMsg(domainTk, cidHash string) (*StoredMsg, error) { subject := mqttSessStreamSubjectPrefix + domainTk + cidHash return jsa.loadLastMsgFor(mqttSessStreamName, subject) } func (jsa *mqttJSA) deleteMsg(stream string, seq uint64, wait bool) error { dreq := JSApiMsgDeleteRequest{Seq: seq, NoErase: true} req, _ := json.Marshal(dreq) subj := jsa.prefixDomain(fmt.Sprintf(JSApiMsgDeleteT, stream)) if !wait { jsa.sendq.push(&mqttJSPubMsg{ subj: subj, msg: req, }) return nil } dmi, err := jsa.newRequest(mqttJSAMsgDelete, subj, 0, req) if err != nil { return err } dm := dmi.(*JSApiMsgDeleteResponse) return dm.ToError() } ////////////////////////////////////////////////////////////////////////////// // // Account Sessions Manager related functions // ////////////////////////////////////////////////////////////////////////////// // Returns true if `err` is not nil and does not match the api error with ErrorIdentifier id func isErrorOtherThan(err error, id ErrorIdentifier) bool { return err != nil && !IsNatsErr(err, id) } // Process JS API replies. // // Can run from various go routines (consumer's loop, system send loop, etc..). func (as *mqttAccountSessionManager) processJSAPIReplies(_ *subscription, pc *client, _ *Account, subject, _ string, msg []byte) { token := tokenAt(subject, mqttJSATokenPos) if token == _EMPTY_ { return } jsa := &as.jsa chi, ok := jsa.replies.Load(subject) if !ok { return } jsa.replies.Delete(subject) ch := chi.(chan *mqttJSAResponse) out := func(value any) { ch <- &mqttJSAResponse{reply: subject, value: value} } switch token { case mqttJSAStreamCreate: var resp = &JSApiStreamCreateResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAStreamUpdate: var resp = &JSApiStreamUpdateResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAStreamLookup: var resp = &JSApiStreamInfoResponse{} if err := json.Unmarshal(msg, &resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAStreamDel: var resp = &JSApiStreamDeleteResponse{} if err := json.Unmarshal(msg, &resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAConsumerCreate: var resp = &JSApiConsumerCreateResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAConsumerDel: var resp = &JSApiConsumerDeleteResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAMsgStore, mqttJSASessPersist: var resp = &JSPubAckResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAMsgLoad: var resp = &JSApiMsgGetResponse{} if err := json.Unmarshal(msg, &resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAStreamNames: var resp = &JSApiStreamNamesResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) case mqttJSAMsgDelete: var resp = &JSApiMsgDeleteResponse{} if err := json.Unmarshal(msg, resp); err != nil { resp.Error = NewJSInvalidJSONError() } out(resp) default: pc.Warnf("Unknown reply code %q", token) } } // This will both load all retained messages and process updates from the cluster. // // Run from various go routines (JS consumer, etc..). // No lock held on entry. func (as *mqttAccountSessionManager) processRetainedMsg(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { _, msg := c.msgParts(rmsg) rm := &mqttRetainedMsg{} if err := json.Unmarshal(msg, rm); err != nil { return } // If lastSeq is 0 (nothing to recover, or done doing it) and this is // from our own server, ignore. as.mu.RLock() if as.rrmLastSeq == 0 && rm.Origin == as.jsa.id { as.mu.RUnlock() return } as.mu.RUnlock() // At this point we either recover from our own server, or process a remote retained message. seq, _, _ := ackReplyInfo(reply) // Handle this retained message, no need to copy the bytes. as.handleRetainedMsg(rm.Subject, &mqttRetainedMsgRef{sseq: seq}, rm, false) // If we were recovering (lastSeq > 0), then check if we are done. as.mu.Lock() if as.rrmLastSeq > 0 && seq >= as.rrmLastSeq { as.rrmLastSeq = 0 close(as.rrmDoneCh) as.rrmDoneCh = nil } as.mu.Unlock() } func (as *mqttAccountSessionManager) processRetainedMsgDel(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { idHash := tokenAt(subject, 3) if idHash == _EMPTY_ || idHash == as.jsa.id { return } _, msg := c.msgParts(rmsg) if len(msg) < LEN_CR_LF { return } var drm mqttRetMsgDel if err := json.Unmarshal(msg, &drm); err != nil { return } as.handleRetainedMsgDel(drm.Subject, drm.Seq) } // This will receive all JS API replies for a request to store a session record, // including the reply for our own server, which we will ignore. // This allows us to detect that some application somewhere else in the cluster // is connecting with the same client ID, and therefore we need to close the // connection that is currently using this client ID. // // Can run from various go routines (system send loop, etc..). // No lock held on entry. func (as *mqttAccountSessionManager) processSessionPersist(_ *subscription, pc *client, _ *Account, subject, _ string, rmsg []byte) { // Ignore our own responses here (they are handled elsewhere) if tokenAt(subject, mqttJSAIdTokenPos) == as.jsa.id { return } cIDHash := tokenAt(subject, mqttJSAClientIDPos) _, msg := pc.msgParts(rmsg) if len(msg) < LEN_CR_LF { return } var par = &JSPubAckResponse{} if err := json.Unmarshal(msg, par); err != nil { return } if err := par.Error; err != nil { return } as.mu.RLock() // Note that as.domainTk includes a terminal '.', so strip to compare to PubAck.Domain. dl := len(as.domainTk) if dl > 0 { dl-- } ignore := par.Domain != as.domainTk[:dl] as.mu.RUnlock() if ignore { return } as.mu.Lock() defer as.mu.Unlock() sess, ok := as.sessByHash[cIDHash] if !ok { return } // If our current session's stream sequence is higher, it means that this // update is stale, so we don't do anything here. sess.mu.Lock() ignore = par.Sequence < sess.seq sess.mu.Unlock() if ignore { return } as.removeSession(sess, false) sess.mu.Lock() if ec := sess.c; ec != nil { as.addSessToFlappers(sess.id) ec.Warnf("Closing because a remote connection has started with the same client ID: %q", sess.id) // Disassociate the client from the session so that on client close, // nothing will be done with regards to cleaning up the session, // such as deleting stream, etc.. sess.c = nil // Remove in separate go routine. go ec.closeConnection(DuplicateClientID) } sess.mu.Unlock() } // Adds this client ID to the flappers map, and if needed start the timer // for map cleanup. // // Lock held on entry. func (as *mqttAccountSessionManager) addSessToFlappers(clientID string) { as.flappers[clientID] = time.Now().UnixNano() if as.flapTimer == nil { as.flapTimer = time.AfterFunc(mqttFlapCleanItvl, func() { as.mu.Lock() defer as.mu.Unlock() // In case of shutdown, this will be nil if as.flapTimer == nil { return } now := time.Now().UnixNano() for cID, tm := range as.flappers { if now-tm > int64(mqttSessJailDur) { delete(as.flappers, cID) } } as.flapTimer.Reset(mqttFlapCleanItvl) }) } } // Remove this client ID from the flappers map. // // Lock held on entry. func (as *mqttAccountSessionManager) removeSessFromFlappers(clientID string) { delete(as.flappers, clientID) // Do not stop/set timer to nil here. Better leave the timer run at its // regular interval and detect that there is nothing to do. The timer // will be stopped on shutdown. } // Helper to create a subscription. It updates the sid and array of subscriptions. func (as *mqttAccountSessionManager) createSubscription(subject string, cb msgHandler, sid *int64, subs *[]*subscription) error { sub, err := as.jsa.c.processSub([]byte(subject), nil, []byte(strconv.FormatInt(*sid, 10)), cb, false) if err != nil { return err } *sid++ *subs = append(*subs, sub) return nil } // A timer loop to cleanup up expired cached retained messages for a given MQTT account. // The closeCh is used by the caller to be able to interrupt this routine // if the rest of the initialization fails, since the quitCh is really // only used when the server shutdown. // // No lock held on entry. func (as *mqttAccountSessionManager) cleanupRetainedMessageCache(s *Server, closeCh chan struct{}) { tt := time.NewTicker(mqttRetainedCacheTTL) defer tt.Stop() for { select { case <-tt.C: // Set a limit to the number of retained messages to scan since we // lock as for it. Since the map enumeration gives random order we // should eventually clean up everything. i, maxScan := 0, 10*1000 now := time.Now() as.rmsCache.Range(func(key, value any) bool { rm := value.(*mqttRetainedMsg) if now.After(rm.expiresFromCache) { as.rmsCache.Delete(key) } i++ return i < maxScan }) case <-closeCh: return case <-s.quitCh: return } } } // Loop to send JS API requests for a given MQTT account. // The closeCh is used by the caller to be able to interrupt this routine // if the rest of the initialization fails, since the quitCh is really // only used when the server shutdown. // // No lock held on entry. func (as *mqttAccountSessionManager) sendJSAPIrequests(s *Server, c *client, accName string, closeCh chan struct{}) { var cluster string if s.JetStreamEnabled() && !as.jsa.domainSet { // Only request the own cluster when it is clear that cluster = s.cachedClusterName() } as.mu.RLock() sendq := as.jsa.sendq quitCh := as.jsa.quitCh ci := ClientInfo{Account: accName, Cluster: cluster} as.mu.RUnlock() // The account session manager does not have a suhtdown API per-se, instead, // we will cleanup things when this go routine exits after detecting that the // server is shutdown or the initialization of the account manager failed. defer func() { as.mu.Lock() if as.flapTimer != nil { as.flapTimer.Stop() as.flapTimer = nil } as.mu.Unlock() }() b, _ := json.Marshal(ci) hdrStart := bytes.Buffer{} hdrStart.WriteString(hdrLine) http.Header{ClientInfoHdr: []string{string(b)}}.Write(&hdrStart) hdrStart.WriteString(CR_LF) hdrStart.WriteString(CR_LF) hdrb := hdrStart.Bytes() for { select { case <-sendq.ch: pmis := sendq.pop() for _, r := range pmis { var nsize int msg := r.msg // If r.hdr is set to -1, it means that there is no need for any header. if r.hdr != -1 { bb := bytes.Buffer{} if r.hdr > 0 { // This means that the header has been set by the caller and is // already part of `msg`, so simply set c.pa.hdr to the given value. c.pa.hdr = r.hdr nsize = len(msg) msg = append(msg, _CRLF_...) } else { // We need the ClientInfo header, so add it here. bb.Write(hdrb) c.pa.hdr = bb.Len() bb.Write(r.msg) nsize = bb.Len() bb.WriteString(_CRLF_) msg = bb.Bytes() } c.pa.hdb = []byte(strconv.Itoa(c.pa.hdr)) } else { c.pa.hdr = -1 c.pa.hdb = nil nsize = len(msg) msg = append(msg, _CRLF_...) } c.pa.subject = []byte(r.subj) c.pa.reply = []byte(r.reply) c.pa.size = nsize c.pa.szb = []byte(strconv.Itoa(nsize)) c.processInboundClientMsg(msg) c.flushClients(0) } sendq.recycle(&pmis) case <-closeCh: return case <-quitCh: return } } } // Add/Replace this message from the retained messages map. // If a message for this topic already existed, the existing record is updated // with the provided information. // Lock not held on entry. func (as *mqttAccountSessionManager) handleRetainedMsg(key string, rf *mqttRetainedMsgRef, rm *mqttRetainedMsg, copyBytesToCache bool) { as.mu.Lock() defer as.mu.Unlock() if as.retmsgs == nil { as.retmsgs = make(map[string]*mqttRetainedMsgRef) as.sl = NewSublistWithCache() } else { // Check if we already had one retained message. If so, update the existing one. if erm, exists := as.retmsgs[key]; exists { // If the new sequence is below the floor or the existing one, // then ignore the new one. if rf.sseq <= erm.sseq || rf.sseq <= erm.floor { return } // Capture existing sequence number so we can return it as the old sequence. erm.sseq = rf.sseq // Clear the floor erm.floor = 0 // If sub is nil, it means that it was removed from sublist following a // network delete. So need to add it now. if erm.sub == nil { erm.sub = &subscription{subject: []byte(key)} as.sl.Insert(erm.sub) } // Update the in-memory retained message cache but only for messages // that are already in the cache, i.e. have been (recently) used. as.setCachedRetainedMsg(key, rm, true, copyBytesToCache) return } } rf.sub = &subscription{subject: []byte(key)} as.retmsgs[key] = rf as.sl.Insert(rf.sub) } // Removes the retained message for the given `subject` if present, and returns the // stream sequence it was stored at. It will be 0 if no retained message was removed. // If a sequence is passed and not 0, then the retained message will be removed only // if the given sequence is equal or higher to what is stored. // // No lock held on entry. func (as *mqttAccountSessionManager) handleRetainedMsgDel(subject string, seq uint64) uint64 { var seqToRemove uint64 as.mu.Lock() if as.retmsgs == nil { as.retmsgs = make(map[string]*mqttRetainedMsgRef) as.sl = NewSublistWithCache() } if erm, ok := as.retmsgs[subject]; ok { if as.rmsCache != nil { as.rmsCache.Delete(subject) } if erm.sub != nil { as.sl.Remove(erm.sub) erm.sub = nil } // If processing a delete request from the network, then seq will be > 0. // If that is the case and it is greater or equal to what we have, we need // to record the floor for this subject. if seq != 0 && seq >= erm.sseq { erm.sseq = 0 erm.floor = seq } else if seq == 0 { delete(as.retmsgs, subject) seqToRemove = erm.sseq } } else if seq != 0 { rf := &mqttRetainedMsgRef{floor: seq} as.retmsgs[subject] = rf } as.mu.Unlock() return seqToRemove } // First check if this session's client ID is already in the "locked" map, // which if it is the case means that another client is now bound to this // session and this should return an error. // If not in the "locked" map, but the client is not bound with this session, // then same error is returned. // Finally, if all checks ok, then the session's ID is added to the "locked" map. // // No lock held on entry. func (as *mqttAccountSessionManager) lockSession(sess *mqttSession, c *client) error { as.mu.Lock() defer as.mu.Unlock() var fail bool if _, fail = as.sessLocked[sess.id]; !fail { sess.mu.Lock() fail = sess.c != c sess.mu.Unlock() } if fail { return fmt.Errorf("another session is in use with client ID %q", sess.id) } as.sessLocked[sess.id] = struct{}{} return nil } // Remove the session from the "locked" map. // // No lock held on entry. func (as *mqttAccountSessionManager) unlockSession(sess *mqttSession) { as.mu.Lock() delete(as.sessLocked, sess.id) as.mu.Unlock() } // Simply adds the session to the various sessions maps. // The boolean `lock` indicates if this function should acquire the lock // prior to adding to the maps. // // No lock held on entry. func (as *mqttAccountSessionManager) addSession(sess *mqttSession, lock bool) { if lock { as.mu.Lock() } as.sessions[sess.id] = sess as.sessByHash[sess.idHash] = sess if lock { as.mu.Unlock() } } // Simply removes the session from the various sessions maps. // The boolean `lock` indicates if this function should acquire the lock // prior to removing from the maps. // // No lock held on entry. func (as *mqttAccountSessionManager) removeSession(sess *mqttSession, lock bool) { if lock { as.mu.Lock() } delete(as.sessions, sess.id) delete(as.sessByHash, sess.idHash) if lock { as.mu.Unlock() } } // Helper to set the sub's mqtt fields and possibly serialize (pre-loaded) // retained messages. // // Session lock held on entry. Acquires the subs lock and holds it for // the duration. Non-MQTT messages coming into mqttDeliverMsgCbQoS0 will be // waiting. func (sess *mqttSession) processQOS12Sub( c *client, // subscribing client. subject, sid []byte, isReserved bool, qos byte, jsDurName string, h msgHandler, // subscription parameters. ) (*subscription, error) { return sess.processSub(c, subject, sid, isReserved, qos, jsDurName, h, false, nil, false, nil) } func (sess *mqttSession) processSub( c *client, // subscribing client. subject, sid []byte, isReserved bool, qos byte, jsDurName string, h msgHandler, // subscription parameters. initShadow bool, // do we need to scan for shadow subscriptions? (not for QOS1+) rms map[string]*mqttRetainedMsg, // preloaded rms (can be empty, or missing items if errors) trace bool, // trace serialized retained messages in the log? as *mqttAccountSessionManager, // needed only for rms serialization. ) (*subscription, error) { start := time.Now() defer func() { elapsed := time.Since(start) if elapsed > mqttProcessSubTooLong { c.Warnf("Took too long to process subscription for %q: %v", subject, elapsed) } }() // Hold subsMu to prevent QOS0 messages callback from doing anything until // the (MQTT) sub is initialized. sess.subsMu.Lock() defer sess.subsMu.Unlock() sub, err := c.processSub(subject, nil, sid, h, false) if err != nil { // c.processSub already called c.Errorf(), so no need here. return nil, err } subs := []*subscription{sub} if initShadow { subs = append(subs, sub.shadow...) } for _, ss := range subs { if ss.mqtt == nil { // reserved is set only once and once the subscription has been // created it can be considered immutable. ss.mqtt = &mqttSub{ reserved: isReserved, } } // QOS and jsDurName can be changed on an existing subscription, so // accessing it later requires a lock. ss.mqtt.qos = qos ss.mqtt.jsDur = jsDurName } if len(rms) > 0 { for _, ss := range subs { as.serializeRetainedMsgsForSub(rms, sess, c, ss, trace) } } return sub, nil } // Process subscriptions for the given session/client. // // When `fromSubProto` is false, it means that this is invoked from the CONNECT // protocol, when restoring subscriptions that were saved for this session. // In that case, there is no need to update the session record. // // When `fromSubProto` is true, it means that this call is invoked from the // processing of the SUBSCRIBE protocol, which means that the session needs to // be updated. It also means that if a subscription on same subject with same // QoS already exist, we should not be recreating the subscription/JS durable, // since it was already done when processing the CONNECT protocol. // // Runs from the client's readLoop. // Lock not held on entry, but session is in the locked map. func (as *mqttAccountSessionManager) processSubs(sess *mqttSession, c *client, filters []*mqttFilter, fromSubProto, trace bool) ([]*subscription, error) { c.mu.Lock() acc := c.acc c.mu.Unlock() // Helper to determine if we need to create a separate top-level // subscription for a wildcard. fwc := func(subject string) (bool, string, string) { if !mqttNeedSubForLevelUp(subject) { return false, _EMPTY_, _EMPTY_ } // Say subject is "foo.>", remove the ".>" so that it becomes "foo" fwcsubject := subject[:len(subject)-2] // Change the sid to "foo fwc" fwcsid := fwcsubject + mqttMultiLevelSidSuffix return true, fwcsubject, fwcsid } rmSubjects := map[string]struct{}{} // Preload retained messages for all requested subscriptions. Also, since // it's the first iteration over the filter list, do some cleanup. for _, f := range filters { if f.qos > 2 { f.qos = 2 } if c.mqtt.downgradeQoS2Sub && f.qos == 2 { c.Warnf("Downgrading subscription QoS2 to QoS1 for %q, as configured", f.filter) f.qos = 1 } // Do not allow subscribing to our internal subjects. // // TODO: (levb: not sure why since one can subscribe to `#` and it'll // include everything; I guess this would discourage? Otherwise another // candidate for DO NOT DELIVER prefix list). if strings.HasPrefix(f.filter, mqttSubPrefix) { f.qos = mqttSubAckFailure continue } if f.qos == 2 { if err := sess.ensurePubRelConsumerSubscription(c); err != nil { c.Errorf("failed to initialize PUBREL processing: %v", err) f.qos = mqttSubAckFailure continue } } // Find retained messages. if fromSubProto { addRMSubjects := func(subject string) error { sub := &subscription{ client: c, subject: []byte(subject), sid: []byte(subject), } if err := c.addShadowSubscriptions(acc, sub, false); err != nil { return err } for _, sub := range append([]*subscription{sub}, sub.shadow...) { as.addRetainedSubjectsForSubject(rmSubjects, bytesToString(sub.subject)) for _, ss := range sub.shadow { as.addRetainedSubjectsForSubject(rmSubjects, bytesToString(ss.subject)) } } return nil } if err := addRMSubjects(f.filter); err != nil { f.qos = mqttSubAckFailure continue } if need, subject, _ := fwc(f.filter); need { if err := addRMSubjects(subject); err != nil { f.qos = mqttSubAckFailure continue } } } } serializeRMS := len(rmSubjects) > 0 var rms map[string]*mqttRetainedMsg if serializeRMS { // Make the best effort to load retained messages. We will identify // errors in the next pass. rms = as.loadRetainedMessages(rmSubjects, c) } // Small helper to add the consumer config to the session. addJSConsToSess := func(sid string, cc *ConsumerConfig) { if cc == nil { return } if sess.cons == nil { sess.cons = make(map[string]*ConsumerConfig) } sess.cons[sid] = cc } var err error subs := make([]*subscription, 0, len(filters)) for _, f := range filters { // Skip what's already been identified as a failure. if f.qos == mqttSubAckFailure { continue } subject := f.filter bsubject := []byte(subject) sid := subject bsid := bsubject isReserved := isMQTTReservedSubscription(subject) var jscons *ConsumerConfig var jssub *subscription // Note that if a subscription already exists on this subject, the // existing sub is returned. Need to update the qos. var sub *subscription var err error const processShadowSubs = true as.mu.Lock() sess.mu.Lock() sub, err = sess.processSub(c, bsubject, bsid, isReserved, f.qos, // main subject _EMPTY_, mqttDeliverMsgCbQoS0, // no jsDur for QOS0 processShadowSubs, rms, trace, as) sess.mu.Unlock() as.mu.Unlock() if err != nil { f.qos = mqttSubAckFailure sess.cleanupFailedSub(c, sub, jscons, jssub) continue } // This will create (if not already exist) a JS consumer for // subscriptions of QoS >= 1. But if a JS consumer already exists and // the subscription for same subject is now a QoS==0, then the JS // consumer will be deleted. jscons, jssub, err = sess.processJSConsumer(c, subject, sid, f.qos, fromSubProto) if err != nil { f.qos = mqttSubAckFailure sess.cleanupFailedSub(c, sub, jscons, jssub) continue } // Process the wildcard subject if needed. if need, fwcsubject, fwcsid := fwc(subject); need { var fwjscons *ConsumerConfig var fwjssub *subscription var fwcsub *subscription // See note above about existing subscription. as.mu.Lock() sess.mu.Lock() fwcsub, err = sess.processSub(c, []byte(fwcsubject), []byte(fwcsid), isReserved, f.qos, // FWC (top-level wildcard) subject _EMPTY_, mqttDeliverMsgCbQoS0, // no jsDur for QOS0 processShadowSubs, rms, trace, as) sess.mu.Unlock() as.mu.Unlock() if err != nil { // c.processSub already called c.Errorf(), so no need here. f.qos = mqttSubAckFailure sess.cleanupFailedSub(c, sub, jscons, jssub) continue } fwjscons, fwjssub, err = sess.processJSConsumer(c, fwcsubject, fwcsid, f.qos, fromSubProto) if err != nil { // c.processSub already called c.Errorf(), so no need here. f.qos = mqttSubAckFailure sess.cleanupFailedSub(c, sub, jscons, jssub) sess.cleanupFailedSub(c, fwcsub, fwjscons, fwjssub) continue } subs = append(subs, fwcsub) addJSConsToSess(fwcsid, fwjscons) } subs = append(subs, sub) addJSConsToSess(sid, jscons) } if fromSubProto { err = sess.update(filters, true) } return subs, err } // Retained publish messages matching this subscription are serialized in the // subscription's `prm` mqtt writer. This buffer will be queued for outbound // after the subscription is processed and SUBACK is sent or possibly when // server processes an incoming published message matching the newly // registered subscription. // // Runs from the client's readLoop. // Account session manager lock held on entry. // Session lock held on entry. func (as *mqttAccountSessionManager) serializeRetainedMsgsForSub(rms map[string]*mqttRetainedMsg, sess *mqttSession, c *client, sub *subscription, trace bool) error { if len(as.retmsgs) == 0 || len(rms) == 0 { return nil } result := as.sl.ReverseMatch(string(sub.subject)) if len(result.psubs) == 0 { return nil } toTrace := []mqttPublish{} for _, psub := range result.psubs { rm := rms[string(psub.subject)] if rm == nil { // This should not happen since we pre-load messages into rms before // calling serialize. continue } var pi uint16 qos := mqttGetQoS(rm.Flags) if qos > sub.mqtt.qos { qos = sub.mqtt.qos } if c.mqtt.rejectQoS2Pub && qos == 2 { c.Warnf("Rejecting retained message with QoS2 for subscription %q, as configured", sub.subject) continue } if qos > 0 { pi = sess.trackPublishRetained() // If we failed to get a PI for this message, send it as a QoS0, the // best we can do? if pi == 0 { qos = 0 } } // Need to use the subject for the retained message, not the `sub` subject. // We can find the published retained message in rm.sub.subject. // Set the RETAIN flag: [MQTT-3.3.1-8]. flags, headerBytes := mqttMakePublishHeader(pi, qos, false, true, []byte(rm.Topic), len(rm.Msg)) c.mu.Lock() sub.mqtt.prm = append(sub.mqtt.prm, headerBytes, rm.Msg) c.mu.Unlock() if trace { toTrace = append(toTrace, mqttPublish{ topic: []byte(rm.Topic), flags: flags, pi: pi, sz: len(rm.Msg), }) } } for _, pp := range toTrace { c.traceOutOp("PUBLISH", []byte(mqttPubTrace(&pp))) } return nil } // Appends the stored message subjects for all retained message records that // match the given subscription's `subject` (which could have wildcards). // // Account session manager NOT lock held on entry. func (as *mqttAccountSessionManager) addRetainedSubjectsForSubject(list map[string]struct{}, topSubject string) bool { as.mu.RLock() if len(as.retmsgs) == 0 { as.mu.RUnlock() return false } result := as.sl.ReverseMatch(topSubject) as.mu.RUnlock() added := false for _, sub := range result.psubs { subject := string(sub.subject) if _, ok := list[subject]; ok { continue } list[subject] = struct{}{} added = true } return added } type warner interface { Warnf(format string, v ...any) } // Loads a list of retained messages given a list of stored message subjects. func (as *mqttAccountSessionManager) loadRetainedMessages(subjects map[string]struct{}, w warner) map[string]*mqttRetainedMsg { rms := make(map[string]*mqttRetainedMsg, len(subjects)) ss := []string{} for s := range subjects { if rm := as.getCachedRetainedMsg(s); rm != nil { rms[s] = rm } else { ss = append(ss, mqttRetainedMsgsStreamSubject+s) } } if len(ss) == 0 { return rms } results, err := as.jsa.loadLastMsgForMulti(mqttRetainedMsgsStreamName, ss) // If an error occurred, warn, but then proceed with what we got. if err != nil { w.Warnf("error loading retained messages: %v", err) } for i, result := range results { if result == nil { continue // skip requests that timed out } if result.ToError() != nil { w.Warnf("failed to load retained message for subject %q: %v", ss[i], err) continue } var rm mqttRetainedMsg if err := json.Unmarshal(result.Message.Data, &rm); err != nil { w.Warnf("failed to decode retained message for subject %q: %v", ss[i], err) continue } // Add the loaded retained message to the cache, and to the results map. key := ss[i][len(mqttRetainedMsgsStreamSubject):] as.setCachedRetainedMsg(key, &rm, false, false) rms[key] = &rm } return rms } // Creates the session stream (limit msgs of 1) for this client ID if it does // not already exist. If it exists, recover the single record to rebuild the // state of the session. If there is a session record but this session is not // registered in the runtime of this server, then a request is made to the // owner to close the client associated with this session since specification // [MQTT-3.1.4-2] specifies that if the ClientId represents a Client already // connected to the Server then the Server MUST disconnect the existing client. // // Runs from the client's readLoop. // Lock not held on entry, but session is in the locked map. func (as *mqttAccountSessionManager) createOrRestoreSession(clientID string, opts *Options) (*mqttSession, bool, error) { jsa := &as.jsa formatError := func(errTxt string, err error) (*mqttSession, bool, error) { accName := jsa.c.acc.GetName() return nil, false, fmt.Errorf("%s for account %q, session %q: %v", errTxt, accName, clientID, err) } hash := getHash(clientID) smsg, err := jsa.loadSessionMsg(as.domainTk, hash) if err != nil { if isErrorOtherThan(err, JSNoMessageFoundErr) { return formatError("loading session record", err) } // Message not found, so reate the session... // Create a session and indicate that this session did not exist. sess := mqttSessionCreate(jsa, clientID, hash, 0, opts) sess.domainTk = as.domainTk return sess, false, nil } // We need to recover the existing record now. ps := &mqttPersistedSession{} if err := json.Unmarshal(smsg.Data, ps); err != nil { return formatError(fmt.Sprintf("unmarshal of session record at sequence %v", smsg.Sequence), err) } // Restore this session (even if we don't own it), the caller will do the right thing. sess := mqttSessionCreate(jsa, clientID, hash, smsg.Sequence, opts) sess.domainTk = as.domainTk sess.clean = ps.Clean sess.subs = ps.Subs sess.cons = ps.Cons sess.pubRelConsumer = ps.PubRel as.addSession(sess, true) return sess, true, nil } // Sends a request to delete a message, but does not wait for the response. // // No lock held on entry. func (as *mqttAccountSessionManager) deleteRetainedMsg(seq uint64) { as.jsa.deleteMsg(mqttRetainedMsgsStreamName, seq, false) } // Sends a message indicating that a retained message on a given subject and stream sequence // is being removed. func (as *mqttAccountSessionManager) notifyRetainedMsgDeleted(subject string, seq uint64) { req := mqttRetMsgDel{ Subject: subject, Seq: seq, } b, _ := json.Marshal(&req) jsa := &as.jsa jsa.sendq.push(&mqttJSPubMsg{ subj: jsa.rplyr + mqttJSARetainedMsgDel, msg: b, }) } func (as *mqttAccountSessionManager) transferUniqueSessStreamsToMuxed(log *Server) { // Set retry to true, will be set to false on success. retry := true defer func() { if retry { next := mqttDefaultTransferRetry log.Warnf("Failed to transfer all MQTT session streams, will try again in %v", next) time.AfterFunc(next, func() { as.transferUniqueSessStreamsToMuxed(log) }) } }() jsa := &as.jsa sni, err := jsa.newRequestEx(mqttJSAStreamNames, JSApiStreams, _EMPTY_, 0, nil, 5*time.Second) if err != nil { log.Errorf("Unable to transfer MQTT session streams: %v", err) return } snames := sni.(*JSApiStreamNamesResponse) if snames.Error != nil { log.Errorf("Unable to transfer MQTT session streams: %v", snames.ToError()) return } var oldMQTTSessStreams []string for _, sn := range snames.Streams { if strings.HasPrefix(sn, mqttSessionsStreamNamePrefix) { oldMQTTSessStreams = append(oldMQTTSessStreams, sn) } } ns := len(oldMQTTSessStreams) if ns == 0 { // Nothing to do retry = false return } log.Noticef("Transferring %v MQTT session streams...", ns) for _, sn := range oldMQTTSessStreams { log.Noticef(" Transferring stream %q to %q", sn, mqttSessStreamName) smsg, err := jsa.loadLastMsgFor(sn, sn) if err != nil { log.Errorf(" Unable to load session record: %v", err) return } ps := &mqttPersistedSession{} if err := json.Unmarshal(smsg.Data, ps); err != nil { log.Warnf(" Unable to unmarshal the content of this stream, may not be a legitimate MQTT session stream, skipping") continue } // Store record to MQTT session stream if _, err := jsa.storeSessionMsg(as.domainTk, getHash(ps.ID), 0, smsg.Data); err != nil { log.Errorf(" Unable to transfer the session record: %v", err) return } jsa.deleteStream(sn) } log.Noticef("Transfer of %v MQTT session streams done!", ns) retry = false } func (as *mqttAccountSessionManager) transferRetainedToPerKeySubjectStream(log *Server) error { jsa := &as.jsa var processed int var transferred int start := time.Now() deadline := start.Add(mqttRetainedTransferTimeout) for { // Try and look up messages on the original undivided "$MQTT.rmsgs" subject. // If nothing is returned here, we assume to have migrated all old messages. smsg, err := jsa.loadNextMsgFor(mqttRetainedMsgsStreamName, "$MQTT.rmsgs") if IsNatsErr(err, JSNoMessageFoundErr) { // We've ran out of messages to transfer, done. break } if err != nil { log.Warnf(" Unable to transfer a retained message: failed to load from '$MQTT.rmsgs': %s", err) return err } // Unmarshal the message so that we can obtain the subject name. var rmsg mqttRetainedMsg if err = json.Unmarshal(smsg.Data, &rmsg); err == nil { // Store the message again, this time with the new per-key subject. subject := mqttRetainedMsgsStreamSubject + rmsg.Subject if _, err = jsa.storeMsg(subject, 0, smsg.Data); err != nil { log.Errorf(" Unable to transfer the retained message with sequence %d: %v", smsg.Sequence, err) } transferred++ } else { log.Warnf(" Unable to unmarshal retained message with sequence %d, skipping", smsg.Sequence) } // Delete the original message. if err := jsa.deleteMsg(mqttRetainedMsgsStreamName, smsg.Sequence, true); err != nil { log.Errorf(" Unable to clean up the retained message with sequence %d: %v", smsg.Sequence, err) return err } processed++ now := time.Now() if now.After(deadline) { err := fmt.Errorf("timed out while transferring retained messages from '$MQTT.rmsgs' after %v, %d processed, %d successfully transferred", now.Sub(start), processed, transferred) log.Noticef(err.Error()) return err } } if processed > 0 { log.Noticef("Processed %d messages from '$MQTT.rmsgs', successfully transferred %d in %v", processed, transferred, time.Since(start)) } else { log.Debugf("No messages found to transfer from '$MQTT.rmsgs'") } return nil } func (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttRetainedMsg { if as.rmsCache == nil { return nil } v, ok := as.rmsCache.Load(subject) if !ok { return nil } rm := v.(*mqttRetainedMsg) if rm.expiresFromCache.Before(time.Now()) { as.rmsCache.Delete(subject) return nil } return rm } func (as *mqttAccountSessionManager) setCachedRetainedMsg(subject string, rm *mqttRetainedMsg, onlyReplace bool, copyBytesToCache bool) { if as.rmsCache == nil || rm == nil { return } rm.expiresFromCache = time.Now().Add(mqttRetainedCacheTTL) if onlyReplace { if _, ok := as.rmsCache.Load(subject); !ok { return } } if copyBytesToCache { rm.Msg = copyBytes(rm.Msg) } as.rmsCache.Store(subject, rm) } ////////////////////////////////////////////////////////////////////////////// // // MQTT session related functions // ////////////////////////////////////////////////////////////////////////////// // Returns a new mqttSession object with max ack pending set based on // option or use mqttDefaultMaxAckPending if no option set. func mqttSessionCreate(jsa *mqttJSA, id, idHash string, seq uint64, opts *Options) *mqttSession { maxp := opts.MQTT.MaxAckPending if maxp == 0 { maxp = mqttDefaultMaxAckPending } return &mqttSession{ jsa: jsa, id: id, idHash: idHash, seq: seq, maxp: maxp, pubRelSubject: mqttPubRelSubjectPrefix + idHash, pubRelDeliverySubject: mqttPubRelDeliverySubjectPrefix + idHash, pubRelDeliverySubjectB: []byte(mqttPubRelDeliverySubjectPrefix + idHash), } } // Persists a session. Note that if the session's current client does not match // the given client, nothing is done. // // Lock not held on entry. func (sess *mqttSession) save() error { sess.mu.Lock() ps := mqttPersistedSession{ Origin: sess.jsa.id, ID: sess.id, Clean: sess.clean, Subs: sess.subs, Cons: sess.cons, PubRel: sess.pubRelConsumer, } b, _ := json.Marshal(&ps) domainTk, cidHash := sess.domainTk, sess.idHash seq := sess.seq sess.mu.Unlock() var hdr int if seq != 0 { bb := bytes.Buffer{} bb.WriteString(hdrLine) bb.WriteString(JSExpectedLastSubjSeq) bb.WriteString(":") bb.WriteString(strconv.FormatInt(int64(seq), 10)) bb.WriteString(CR_LF) bb.WriteString(CR_LF) hdr = bb.Len() bb.Write(b) b = bb.Bytes() } resp, err := sess.jsa.storeSessionMsg(domainTk, cidHash, hdr, b) if err != nil { return fmt.Errorf("unable to persist session %q (seq=%v): %v", ps.ID, seq, err) } sess.mu.Lock() sess.seq = resp.Sequence sess.mu.Unlock() return nil } // Clear the session. // // Runs from the client's readLoop. // Lock not held on entry, but session is in the locked map. func (sess *mqttSession) clear(noWait bool) error { var durs []string var pubRelDur string sess.mu.Lock() id := sess.id seq := sess.seq if l := len(sess.cons); l > 0 { durs = make([]string, 0, l) } for sid, cc := range sess.cons { delete(sess.cons, sid) durs = append(durs, cc.Durable) } if sess.pubRelConsumer != nil { pubRelDur = sess.pubRelConsumer.Durable } sess.subs = nil sess.pendingPublish = nil sess.pendingPubRel = nil sess.cpending = nil sess.pubRelConsumer = nil sess.seq = 0 sess.tmaxack = 0 sess.mu.Unlock() for _, dur := range durs { if _, err := sess.jsa.deleteConsumer(mqttStreamName, dur, noWait); isErrorOtherThan(err, JSConsumerNotFoundErr) { return fmt.Errorf("unable to delete consumer %q for session %q: %v", dur, sess.id, err) } } if pubRelDur != _EMPTY_ { _, err := sess.jsa.deleteConsumer(mqttOutStreamName, pubRelDur, noWait) if isErrorOtherThan(err, JSConsumerNotFoundErr) { return fmt.Errorf("unable to delete consumer %q for session %q: %v", pubRelDur, sess.id, err) } } if seq > 0 { err := sess.jsa.deleteMsg(mqttSessStreamName, seq, !noWait) // Ignore the various errors indicating that the message (or sequence) // is already deleted, can happen in a cluster. if isErrorOtherThan(err, JSSequenceNotFoundErrF) { if isErrorOtherThan(err, JSStreamMsgDeleteFailedF) || !strings.Contains(err.Error(), ErrStoreMsgNotFound.Error()) { return fmt.Errorf("unable to delete session %q record at sequence %v: %v", id, seq, err) } } } return nil } // This will update the session record for this client in the account's MQTT // sessions stream if the session had any change in the subscriptions. // // Runs from the client's readLoop. // Lock not held on entry, but session is in the locked map. func (sess *mqttSession) update(filters []*mqttFilter, add bool) error { // Evaluate if we need to persist anything. var needUpdate bool for _, f := range filters { if add { if f.qos == mqttSubAckFailure { continue } if qos, ok := sess.subs[f.filter]; !ok || qos != f.qos { if sess.subs == nil { sess.subs = make(map[string]byte) } sess.subs[f.filter] = f.qos needUpdate = true } } else { if _, ok := sess.subs[f.filter]; ok { delete(sess.subs, f.filter) needUpdate = true } } } var err error if needUpdate { err = sess.save() } return err } func (sess *mqttSession) bumpPI() uint16 { var avail bool next := sess.last_pi for i := 0; i < 0xFFFF; i++ { next++ if next == 0 { next = 1 } _, usedInPublish := sess.pendingPublish[next] _, usedInPubRel := sess.pendingPubRel[next] if !usedInPublish && !usedInPubRel { sess.last_pi = next avail = true break } } if !avail { return 0 } return sess.last_pi } // trackPublishRetained is invoked when a retained (QoS) message is published. // It need a new PI to be allocated, so we add it to the pendingPublish map, // with an empty value. Since cpending (not pending) is used to serialize the PI // mappings, we need to add this PI there as well. Make a unique key by using // mqttRetainedMsgsStreamName for the durable name, and PI for sseq. // // Lock held on entry func (sess *mqttSession) trackPublishRetained() uint16 { // Make sure we initialize the tracking maps. if sess.pendingPublish == nil { sess.pendingPublish = make(map[uint16]*mqttPending) } if sess.cpending == nil { sess.cpending = make(map[string]map[uint64]uint16) } pi := sess.bumpPI() if pi == 0 { return 0 } sess.pendingPublish[pi] = &mqttPending{} return pi } // trackPublish is invoked when a (QoS) PUBLISH message is to be delivered. It // detects an untracked (new) message based on its sequence extracted from its // delivery-time jsAckSubject, and adds it to the tracking maps. Returns a PI to // use for the message (new, or previously used), and whether this is a // duplicate delivery attempt. // // Lock held on entry func (sess *mqttSession) trackPublish(jsDur, jsAckSubject string) (uint16, bool) { var dup bool var pi uint16 if jsAckSubject == _EMPTY_ || jsDur == _EMPTY_ { return 0, false } // Make sure we initialize the tracking maps. if sess.pendingPublish == nil { sess.pendingPublish = make(map[uint16]*mqttPending) } if sess.cpending == nil { sess.cpending = make(map[string]map[uint64]uint16) } // Get the stream sequence and duplicate flag from the ack reply subject. sseq, _, dcount := ackReplyInfo(jsAckSubject) if dcount > 1 { dup = true } var ack *mqttPending sseqToPi, ok := sess.cpending[jsDur] if !ok { sseqToPi = make(map[uint64]uint16) sess.cpending[jsDur] = sseqToPi } else { pi = sseqToPi[sseq] } if pi != 0 { // There is a possible race between a PUBLISH re-delivery calling us, // and a PUBREC received already having submitting a PUBREL into JS . If // so, indicate no need for (re-)delivery by returning a PI of 0. _, usedForPubRel := sess.pendingPubRel[pi] if /*dup && */ usedForPubRel { return 0, false } // We should have a pending JS ACK for this PI. ack = sess.pendingPublish[pi] } else { // sess.maxp will always have a value > 0. if len(sess.pendingPublish) >= int(sess.maxp) { // Indicate that we did not assign a packet identifier. // The caller will not send the message to the subscription // and JS will redeliver later, based on consumer's AckWait. return 0, false } pi = sess.bumpPI() if pi == 0 { return 0, false } sseqToPi[sseq] = pi } if ack == nil { sess.pendingPublish[pi] = &mqttPending{ jsDur: jsDur, sseq: sseq, jsAckSubject: jsAckSubject, } } else { ack.jsAckSubject = jsAckSubject ack.sseq = sseq ack.jsDur = jsDur } return pi, dup } // Stops a PI from being tracked as a PUBLISH. It can still be in use for a // pending PUBREL. // // Lock held on entry func (sess *mqttSession) untrackPublish(pi uint16) (jsAckSubject string) { ack, ok := sess.pendingPublish[pi] if !ok { return _EMPTY_ } delete(sess.pendingPublish, pi) if len(sess.pendingPublish) == 0 { sess.last_pi = 0 } if len(sess.cpending) != 0 && ack.jsDur != _EMPTY_ { if sseqToPi := sess.cpending[ack.jsDur]; sseqToPi != nil { delete(sseqToPi, ack.sseq) } } return ack.jsAckSubject } // trackPubRel is invoked in 2 cases: (a) when we receive a PUBREC and we need // to change from tracking the PI as a PUBLISH to a PUBREL; and (b) when we // attempt to deliver the PUBREL to record the JS ack subject for it. // // Lock held on entry func (sess *mqttSession) trackAsPubRel(pi uint16, jsAckSubject string) { if sess.pubRelConsumer == nil { // The cosumer MUST be set up already. return } jsDur := sess.pubRelConsumer.Durable if sess.pendingPubRel == nil { sess.pendingPubRel = make(map[uint16]*mqttPending) } if jsAckSubject == _EMPTY_ { sess.pendingPubRel[pi] = &mqttPending{ jsDur: jsDur, } return } sseq, _, _ := ackReplyInfo(jsAckSubject) if sess.cpending == nil { sess.cpending = make(map[string]map[uint64]uint16) } sseqToPi := sess.cpending[jsDur] if sseqToPi == nil { sseqToPi = make(map[uint64]uint16) sess.cpending[jsDur] = sseqToPi } sseqToPi[sseq] = pi sess.pendingPubRel[pi] = &mqttPending{ jsDur: sess.pubRelConsumer.Durable, sseq: sseq, jsAckSubject: jsAckSubject, } } // Stops a PI from being tracked as a PUBREL. // // Lock held on entry func (sess *mqttSession) untrackPubRel(pi uint16) (jsAckSubject string) { ack, ok := sess.pendingPubRel[pi] if !ok { return _EMPTY_ } delete(sess.pendingPubRel, pi) if sess.pubRelConsumer != nil && len(sess.cpending) > 0 { if sseqToPi := sess.cpending[ack.jsDur]; sseqToPi != nil { delete(sseqToPi, ack.sseq) } } return ack.jsAckSubject } // Sends a consumer delete request, but does not wait for response. // // Lock not held on entry. func (sess *mqttSession) deleteConsumer(cc *ConsumerConfig) { sess.mu.Lock() sess.tmaxack -= cc.MaxAckPending sess.jsa.deleteConsumer(mqttStreamName, cc.Durable, true) sess.mu.Unlock() } ////////////////////////////////////////////////////////////////////////////// // // CONNECT protocol related functions // ////////////////////////////////////////////////////////////////////////////// // Parse the MQTT connect protocol func (c *client) mqttParseConnect(r *mqttReader, hasMappings bool) (byte, *mqttConnectProto, error) { // Protocol name proto, err := r.readBytes("protocol name", false) if err != nil { return 0, nil, err } // Spec [MQTT-3.1.2-1] if !bytes.Equal(proto, mqttProtoName) { // Check proto name against v3.1 to report better error if bytes.Equal(proto, mqttOldProtoName) { return 0, nil, fmt.Errorf("older protocol %q not supported", proto) } return 0, nil, fmt.Errorf("expected connect packet with protocol name %q, got %q", mqttProtoName, proto) } // Protocol level level, err := r.readByte("protocol level") if err != nil { return 0, nil, err } // Spec [MQTT-3.1.2-2] if level != mqttProtoLevel { return mqttConnAckRCUnacceptableProtocolVersion, nil, fmt.Errorf("unacceptable protocol version of %v", level) } cp := &mqttConnectProto{} // Connect flags cp.flags, err = r.readByte("flags") if err != nil { return 0, nil, err } // Spec [MQTT-3.1.2-3] if cp.flags&mqttConnFlagReserved != 0 { return 0, nil, errMQTTConnFlagReserved } var hasWill bool wqos := (cp.flags & mqttConnFlagWillQoS) >> 3 wretain := cp.flags&mqttConnFlagWillRetain != 0 // Spec [MQTT-3.1.2-11] if cp.flags&mqttConnFlagWillFlag == 0 { // Spec [MQTT-3.1.2-13] if wqos != 0 { return 0, nil, fmt.Errorf("if Will flag is set to 0, Will QoS must be 0 too, got %v", wqos) } // Spec [MQTT-3.1.2-15] if wretain { return 0, nil, errMQTTWillAndRetainFlag } } else { // Spec [MQTT-3.1.2-14] if wqos == 3 { return 0, nil, fmt.Errorf("if Will flag is set to 1, Will QoS can be 0, 1 or 2, got %v", wqos) } hasWill = true } if c.mqtt.rejectQoS2Pub && hasWill && wqos == 2 { return mqttConnAckRCQoS2WillRejected, nil, fmt.Errorf("server does not accept QoS2 for Will messages") } // Spec [MQTT-3.1.2-19] hasUser := cp.flags&mqttConnFlagUsernameFlag != 0 // Spec [MQTT-3.1.2-21] hasPassword := cp.flags&mqttConnFlagPasswordFlag != 0 // Spec [MQTT-3.1.2-22] if !hasUser && hasPassword { return 0, nil, errMQTTPasswordFlagAndNoUser } // Keep alive var ka uint16 ka, err = r.readUint16("keep alive") if err != nil { return 0, nil, err } // Spec [MQTT-3.1.2-24] if ka > 0 { cp.rd = time.Duration(float64(ka)*1.5) * time.Second } // Payload starts here and order is mandated by: // Spec [MQTT-3.1.3-1]: client ID, will topic, will message, username, password // Client ID c.mqtt.cid, err = r.readString("client ID") if err != nil { return 0, nil, err } // Spec [MQTT-3.1.3-7] if c.mqtt.cid == _EMPTY_ { if cp.flags&mqttConnFlagCleanSession == 0 { return mqttConnAckRCIdentifierRejected, nil, errMQTTCIDEmptyNeedsCleanFlag } // Spec [MQTT-3.1.3-6] c.mqtt.cid = nuid.Next() } // Spec [MQTT-3.1.3-4] and [MQTT-3.1.3-9] if !utf8.ValidString(c.mqtt.cid) { return mqttConnAckRCIdentifierRejected, nil, fmt.Errorf("invalid utf8 for client ID: %q", c.mqtt.cid) } if hasWill { cp.will = &mqttWill{ qos: wqos, retain: wretain, } var topic []byte // Need to make a copy since we need to hold to this topic after the // parsing of this protocol. topic, err = r.readBytes("Will topic", true) if err != nil { return 0, nil, err } if len(topic) == 0 { return 0, nil, errMQTTEmptyWillTopic } if !utf8.Valid(topic) { return 0, nil, fmt.Errorf("invalid utf8 for Will topic %q", topic) } // Convert MQTT topic to NATS subject cp.will.subject, err = mqttTopicToNATSPubSubject(topic) if err != nil { return 0, nil, err } // Check for subject mapping. if hasMappings { // For selectMappedSubject to work, we need to have c.pa.subject set. // If there is a change, c.pa.mapped will be set after the call. c.pa.subject = cp.will.subject if changed := c.selectMappedSubject(); changed { // We need to keep track of the NATS subject/mapped in the `cp` structure. cp.will.subject = c.pa.subject cp.will.mapped = c.pa.mapped // We also now need to map the original MQTT topic to the new topic // based on the new subject. topic = natsSubjectToMQTTTopic(string(cp.will.subject)) } // Reset those now. c.pa.subject, c.pa.mapped = nil, nil } cp.will.topic = topic // Now "will" message. // Ask for a copy since we need to hold to this after parsing of this protocol. cp.will.message, err = r.readBytes("Will message", true) if err != nil { return 0, nil, err } } if hasUser { c.opts.Username, err = r.readString("user name") if err != nil { return 0, nil, err } if c.opts.Username == _EMPTY_ { return mqttConnAckRCBadUserOrPassword, nil, errMQTTEmptyUsername } // Spec [MQTT-3.1.3-11] if !utf8.ValidString(c.opts.Username) { return mqttConnAckRCBadUserOrPassword, nil, fmt.Errorf("invalid utf8 for user name %q", c.opts.Username) } } if hasPassword { c.opts.Password, err = r.readString("password") if err != nil { return 0, nil, err } c.opts.Token = c.opts.Password c.opts.JWT = c.opts.Password } return 0, cp, nil } func (c *client) mqttConnectTrace(cp *mqttConnectProto) string { trace := fmt.Sprintf("clientID=%s", c.mqtt.cid) if cp.rd > 0 { trace += fmt.Sprintf(" keepAlive=%v", cp.rd) } if cp.will != nil { trace += fmt.Sprintf(" will=(topic=%s QoS=%v retain=%v)", cp.will.topic, cp.will.qos, cp.will.retain) } if cp.flags&mqttConnFlagCleanSession != 0 { trace += " clean" } if c.opts.Username != _EMPTY_ { trace += fmt.Sprintf(" username=%s", c.opts.Username) } if c.opts.Password != _EMPTY_ { trace += " password=****" } return trace } // Process the CONNECT packet. // // For the first session on the account, an account session manager will be created, // along with the JetStream streams/consumer necessary for the working of MQTT. // // The session, identified by a client ID, will be registered, or if already existing, // will be resumed. If the session exists but is associated with an existing client, // the old client is evicted, as per the specifications. // // Due to specific locking requirements around JS API requests, we cannot hold some // locks for the entire duration of processing of some protocols, therefore, we use // a map that registers the client ID in a "locked" state. If a different client tries // to connect and the server detects that the client ID is in that map, it will try // a little bit until it is not, or fail the new client, since we can't protect // processing of protocols in the original client. This is not expected to happen often. // // Runs from the client's readLoop. // No lock held on entry. func (s *Server) mqttProcessConnect(c *client, cp *mqttConnectProto, trace bool) error { sendConnAck := func(rc byte, sessp bool) { c.mqttEnqueueConnAck(rc, sessp) if trace { c.traceOutOp("CONNACK", []byte(fmt.Sprintf("sp=%v rc=%v", sessp, rc))) } } c.mu.Lock() cid := c.mqtt.cid c.clearAuthTimer() c.mu.Unlock() if !s.isClientAuthorized(c) { if trace { c.traceOutOp("CONNACK", []byte(fmt.Sprintf("sp=%v rc=%v", false, mqttConnAckRCNotAuthorized))) } c.authViolation() return ErrAuthentication } // Now that we are are authenticated, we have the client bound to the account. // Get the account's level MQTT sessions manager. If it does not exists yet, // this will create it along with the streams where sessions and messages // are stored. asm, err := s.getOrCreateMQTTAccountSessionManager(c) if err != nil { return err } // Most of the session state is altered only in the readLoop so does not // need locking. For things that can be access in the readLoop and in // callbacks, we will use explicit locking. // To prevent other clients to connect with the same client ID, we will // add the client ID to a "locked" map so that the connect somewhere else // is put on hold. // This keep track of how many times this client is detecting that its // client ID is in the locked map. After a short amount, the server will // fail this inbound client. locked := 0 CHECK: asm.mu.Lock() // Check if different applications keep trying to connect with the same // client ID at the same time. if tm, ok := asm.flappers[cid]; ok { // If the last time it tried to connect was more than 1 sec ago, // then accept and remove from flappers map. if time.Now().UnixNano()-tm > int64(mqttSessJailDur) { asm.removeSessFromFlappers(cid) } else { // Will hold this client for a second and then close it. We // do this so that if the client has a reconnect feature we // don't end-up with very rapid flapping between apps. // We need to wait in place and not schedule the connection // close because if this is a misbehaved client that does // not wait for the CONNACK and sends other protocols, the // server would not have a fully setup client and may panic. asm.mu.Unlock() select { case <-s.quitCh: case <-time.After(mqttSessJailDur): } c.closeConnection(DuplicateClientID) return ErrConnectionClosed } } // If an existing session is in the process of processing some packet, we can't // evict the old client just yet. So try again to see if the state clears, but // if it does not, then we have no choice but to fail the new client instead of // the old one. if _, ok := asm.sessLocked[cid]; ok { asm.mu.Unlock() if locked++; locked == 10 { return fmt.Errorf("other session with client ID %q is in the process of connecting", cid) } time.Sleep(100 * time.Millisecond) goto CHECK } // Register this client ID the "locked" map for the duration if this function. asm.sessLocked[cid] = struct{}{} // And remove it on exit, regardless of error or not. defer func() { asm.mu.Lock() delete(asm.sessLocked, cid) asm.mu.Unlock() }() // Is the client requesting a clean session or not. cleanSess := cp.flags&mqttConnFlagCleanSession != 0 // Session present? Assume false, will be set to true only when applicable. sessp := false // Do we have an existing session for this client ID es, exists := asm.sessions[cid] asm.mu.Unlock() // The session is not in the map, but may be on disk, so try to recover // or create the stream if not. if !exists { es, exists, err = asm.createOrRestoreSession(cid, s.getOpts()) if err != nil { return err } } if exists { // Clear the session if client wants a clean session. // Also, Spec [MQTT-3.2.2-1]: don't report session present if cleanSess || es.clean { // Spec [MQTT-3.1.2-6]: If CleanSession is set to 1, the Client and // Server MUST discard any previous Session and start a new one. // This Session lasts as long as the Network Connection. State data // associated with this Session MUST NOT be reused in any subsequent // Session. if err := es.clear(false); err != nil { asm.removeSession(es, true) return err } } else { // Report to the client that the session was present sessp = true } // Spec [MQTT-3.1.4-2]. If the ClientId represents a Client already // connected to the Server then the Server MUST disconnect the existing // client. // Bind with the new client. This needs to be protected because can be // accessed outside of the readLoop. es.mu.Lock() ec := es.c es.c = c es.clean = cleanSess es.mu.Unlock() if ec != nil { // Remove "will" of existing client before closing ec.mu.Lock() ec.mqtt.cp.will = nil ec.mu.Unlock() // Add to the map of the flappers asm.mu.Lock() asm.addSessToFlappers(cid) asm.mu.Unlock() c.Warnf("Replacing old client %q since both have the same client ID %q", ec, cid) // Close old client in separate go routine go ec.closeConnection(DuplicateClientID) } } else { // Spec [MQTT-3.2.2-3]: if the Server does not have stored Session state, // it MUST set Session Present to 0 in the CONNACK packet. es.mu.Lock() es.c, es.clean = c, cleanSess es.mu.Unlock() // Now add this new session into the account sessions asm.addSession(es, true) } // We would need to save only if it did not exist previously, but we save // always in case we are running in cluster mode. This will notify other // running servers that this session is being used. if err := es.save(); err != nil { asm.removeSession(es, true) return err } c.mu.Lock() c.flags.set(connectReceived) c.mqtt.cp = cp c.mqtt.asm = asm c.mqtt.sess = es c.mu.Unlock() // Spec [MQTT-3.2.0-1]: CONNACK must be the first protocol sent to the session. sendConnAck(mqttConnAckRCConnectionAccepted, sessp) // Process possible saved subscriptions. if l := len(es.subs); l > 0 { filters := make([]*mqttFilter, 0, l) for subject, qos := range es.subs { filters = append(filters, &mqttFilter{filter: subject, qos: qos}) } if _, err := asm.processSubs(es, c, filters, false, trace); err != nil { return err } } return nil } func (c *client) mqttEnqueueConnAck(rc byte, sessionPresent bool) { proto := [4]byte{mqttPacketConnectAck, 2, 0, rc} c.mu.Lock() // Spec [MQTT-3.2.2-4]. If return code is different from 0, then // session present flag must be set to 0. if rc == 0 { if sessionPresent { proto[2] = 1 } } c.enqueueProto(proto[:]) c.mu.Unlock() } func (s *Server) mqttHandleWill(c *client) { c.mu.Lock() if c.mqtt.cp == nil { c.mu.Unlock() return } will := c.mqtt.cp.will if will == nil { c.mu.Unlock() return } pp := c.mqtt.pp pp.topic = will.topic pp.subject = will.subject pp.mapped = will.mapped pp.msg = will.message pp.sz = len(will.message) pp.pi = 0 pp.flags = will.qos << 1 if will.retain { pp.flags |= mqttPubFlagRetain } c.mu.Unlock() s.mqttInitiateMsgDelivery(c, pp) c.flushClients(0) } ////////////////////////////////////////////////////////////////////////////// // // PUBLISH protocol related functions // ////////////////////////////////////////////////////////////////////////////// func (c *client) mqttParsePub(r *mqttReader, pl int, pp *mqttPublish, hasMappings bool) error { qos := mqttGetQoS(pp.flags) if qos > 2 { return fmt.Errorf("QoS=%v is invalid in MQTT", qos) } if c.mqtt.rejectQoS2Pub && qos == 2 { return fmt.Errorf("QoS=2 is disabled for PUBLISH messages") } // Keep track of where we are when starting to read the variable header start := r.pos var err error pp.topic, err = r.readBytes("topic", false) if err != nil { return err } if len(pp.topic) == 0 { return errMQTTTopicIsEmpty } // Convert the topic to a NATS subject. This call will also check that // there is no MQTT wildcards (Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1]) // Note that this may not result in a copy if there is no conversion. // It is good because after the message is processed we won't have a // reference to the buffer and we save a copy. pp.subject, err = mqttTopicToNATSPubSubject(pp.topic) if err != nil { return err } // Check for subject mapping. if hasMappings { // For selectMappedSubject to work, we need to have c.pa.subject set. // If there is a change, c.pa.mapped will be set after the call. c.pa.subject = pp.subject if changed := c.selectMappedSubject(); changed { // We need to keep track of the NATS subject/mapped in the `pp` structure. pp.subject = c.pa.subject pp.mapped = c.pa.mapped // We also now need to map the original MQTT topic to the new topic // based on the new subject. pp.topic = natsSubjectToMQTTTopic(string(pp.subject)) } // Reset those now. c.pa.subject, c.pa.mapped = nil, nil } if qos > 0 { pp.pi, err = r.readUint16("packet identifier") if err != nil { return err } if pp.pi == 0 { return fmt.Errorf("with QoS=%v, packet identifier cannot be 0", qos) } } else { pp.pi = 0 } // The message payload will be the total packet length minus // what we have consumed for the variable header pp.sz = pl - (r.pos - start) if pp.sz > 0 { start = r.pos r.pos += pp.sz pp.msg = r.buf[start:r.pos] } else { pp.msg = nil } return nil } func mqttPubTrace(pp *mqttPublish) string { dup := pp.flags&mqttPubFlagDup != 0 qos := mqttGetQoS(pp.flags) retain := mqttIsRetained(pp.flags) var piStr string if pp.pi > 0 { piStr = fmt.Sprintf(" pi=%v", pp.pi) } return fmt.Sprintf("%s dup=%v QoS=%v retain=%v size=%v%s", pp.topic, dup, qos, retain, pp.sz, piStr) } // Composes a NATS message from a MQTT PUBLISH packet. The message includes an // internal header containint the original packet's QoS, and for QoS2 packets // the original subject. // // Example (QoS2, subject: "foo.bar"): // // NATS/1.0\r\n // Nmqtt-Pub:2foo.bar\r\n // \r\n func mqttNewDeliverableMessage(pp *mqttPublish, encodePP bool) (natsMsg []byte, headerLen int) { size := len(hdrLine) + len(mqttNatsHeader) + 2 + 2 + // 2 for ':', and 2 for CRLF 2 + // end-of-header CRLF len(pp.msg) if encodePP { size += len(mqttNatsHeaderSubject) + 1 + // +1 for ':' len(pp.subject) + 2 // 2 for CRLF if len(pp.mapped) > 0 { size += len(mqttNatsHeaderMapped) + 1 + // +1 for ':' len(pp.mapped) + 2 // 2 for CRLF } } buf := bytes.NewBuffer(make([]byte, 0, size)) qos := mqttGetQoS(pp.flags) buf.WriteString(hdrLine) buf.WriteString(mqttNatsHeader) buf.WriteByte(':') buf.WriteByte(qos + '0') buf.WriteString(_CRLF_) if encodePP { buf.WriteString(mqttNatsHeaderSubject) buf.WriteByte(':') buf.Write(pp.subject) buf.WriteString(_CRLF_) if len(pp.mapped) > 0 { buf.WriteString(mqttNatsHeaderMapped) buf.WriteByte(':') buf.Write(pp.mapped) buf.WriteString(_CRLF_) } } // End of header buf.WriteString(_CRLF_) headerLen = buf.Len() buf.Write(pp.msg) return buf.Bytes(), headerLen } // Composes a NATS message for a pending PUBREL packet. The message includes an // internal header containing the PI for PUBREL/PUBCOMP. // // Example (PI:123): // // NATS/1.0\r\n // Nmqtt-PubRel:123\r\n // \r\n func mqttNewDeliverablePubRel(pi uint16) (natsMsg []byte, headerLen int) { size := len(hdrLine) + len(mqttNatsPubRelHeader) + 6 + 2 + // 6 for ':65535', and 2 for CRLF 2 // end-of-header CRLF buf := bytes.NewBuffer(make([]byte, 0, size)) buf.WriteString(hdrLine) buf.WriteString(mqttNatsPubRelHeader) buf.WriteByte(':') buf.WriteString(strconv.FormatInt(int64(pi), 10)) buf.WriteString(_CRLF_) buf.WriteString(_CRLF_) return buf.Bytes(), buf.Len() } // Process the PUBLISH packet. // // Runs from the client's readLoop. // No lock held on entry. func (s *Server) mqttProcessPub(c *client, pp *mqttPublish, trace bool) error { qos := mqttGetQoS(pp.flags) switch qos { case 0: return s.mqttInitiateMsgDelivery(c, pp) case 1: // [MQTT-4.3.2-2]. Initiate onward delivery of the Application Message, // Send PUBACK. // // The receiver is not required to complete delivery of the Application // Message before sending the PUBACK. When its original sender receives // the PUBACK packet, ownership of the Application Message is // transferred to the receiver. err := s.mqttInitiateMsgDelivery(c, pp) if err == nil { c.mqttEnqueuePubResponse(mqttPacketPubAck, pp.pi, trace) } return err case 2: // [MQTT-4.3.3-2]. Method A, Store message, send PUBREC. // // The receiver is not required to complete delivery of the Application // Message before sending the PUBREC or PUBCOMP. When its original // sender receives the PUBREC packet, ownership of the Application // Message is transferred to the receiver. err := s.mqttStoreQoS2MsgOnce(c, pp) if err == nil { c.mqttEnqueuePubResponse(mqttPacketPubRec, pp.pi, trace) } return err default: return fmt.Errorf("unreachable: invalid QoS in mqttProcessPub: %v", qos) } } func (s *Server) mqttInitiateMsgDelivery(c *client, pp *mqttPublish) error { natsMsg, headerLen := mqttNewDeliverableMessage(pp, false) // Set the client's pubarg for processing. c.pa.subject = pp.subject c.pa.mapped = pp.mapped c.pa.reply = nil c.pa.hdr = headerLen c.pa.hdb = []byte(strconv.FormatInt(int64(c.pa.hdr), 10)) c.pa.size = len(natsMsg) c.pa.szb = []byte(strconv.FormatInt(int64(c.pa.size), 10)) defer func() { c.pa.subject = nil c.pa.mapped = nil c.pa.reply = nil c.pa.hdr = -1 c.pa.hdb = nil c.pa.size = 0 c.pa.szb = nil }() _, permIssue := c.processInboundClientMsg(natsMsg) if permIssue { return nil } // If QoS 0 messages don't need to be stored, other (1 and 2) do. Store them // JetStream under "$MQTT.msgs." if qos := mqttGetQoS(pp.flags); qos == 0 { return nil } // We need to call flushClients now since this we may have called c.addToPCD // with destination clients (possibly a route). Without calling flushClients // the following call may then be stuck waiting for a reply that may never // come because the destination is not flushed (due to c.out.fsp > 0, // see addToPCD and writeLoop for details). c.flushClients(0) _, err := c.mqtt.sess.jsa.storeMsg(mqttStreamSubjectPrefix+string(c.pa.subject), headerLen, natsMsg) return err } var mqttMaxMsgErrPattern = fmt.Sprintf("%s (%v)", ErrMaxMsgsPerSubject.Error(), JSStreamStoreFailedF) func (s *Server) mqttStoreQoS2MsgOnce(c *client, pp *mqttPublish) error { // `true` means encode the MQTT PUBLISH packet in the NATS message header. natsMsg, headerLen := mqttNewDeliverableMessage(pp, true) // Do not broadcast the message until it has been deduplicated and released // by the sender. Instead store this QoS2 message as // "$MQTT.qos2..". If the message is a duplicate, we get back // a ErrMaxMsgsPerSubject, otherwise it does not change the flow, still need // to send a PUBREC back to the client. The original subject (translated // from MQTT topic) is included in the NATS header of the stored message to // use for latter delivery. _, err := c.mqtt.sess.jsa.storeMsg(c.mqttQoS2InternalSubject(pp.pi), headerLen, natsMsg) // TODO: would prefer a more robust and performant way of checking the // error, but it comes back wrapped as an API result. if err != nil && (isErrorOtherThan(err, JSStreamStoreFailedF) || err.Error() != mqttMaxMsgErrPattern) { return err } return nil } func (c *client) mqttQoS2InternalSubject(pi uint16) string { return mqttQoS2IncomingMsgsStreamSubjectPrefix + c.mqtt.cid + "." + strconv.FormatUint(uint64(pi), 10) } // Process a PUBREL packet (QoS2, acting as Receiver). // // Runs from the client's readLoop. // No lock held on entry. func (s *Server) mqttProcessPubRel(c *client, pi uint16, trace bool) error { // Once done with the processing, send a PUBCOMP back to the client. defer c.mqttEnqueuePubResponse(mqttPacketPubComp, pi, trace) // See if there is a message pending for this pi. All failures are treated // as "not found". asm := c.mqtt.asm stored, _ := asm.jsa.loadLastMsgFor(mqttQoS2IncomingMsgsStreamName, c.mqttQoS2InternalSubject(pi)) if stored == nil { // No message found, nothing to do. return nil } // Best attempt to delete the message from the QoS2 stream. asm.jsa.deleteMsg(mqttQoS2IncomingMsgsStreamName, stored.Sequence, true) // only MQTT QoS2 messages should be here, and they must have a subject. h := mqttParsePublishNATSHeader(stored.Header) if h == nil || h.qos != 2 || len(h.subject) == 0 { return errors.New("invalid message in QoS2 PUBREL stream") } pp := &mqttPublish{ topic: natsSubjectToMQTTTopic(string(h.subject)), subject: h.subject, mapped: h.mapped, msg: stored.Data, sz: len(stored.Data), pi: pi, flags: h.qos << 1, } return s.mqttInitiateMsgDelivery(c, pp) } // Invoked when processing an inbound client message. If the "retain" flag is // set, the message is stored so it can be later resent to (re)starting // subscriptions that match the subject. // // Invoked from the MQTT publisher's readLoop. No client lock is held on entry. func (c *client) mqttHandlePubRetain() { pp := c.mqtt.pp if !mqttIsRetained(pp.flags) { return } key := string(pp.subject) asm := c.mqtt.asm // Spec [MQTT-3.3.1-11]. Payload of size 0 removes the retained message, // but should still be delivered as a normal message. if pp.sz == 0 { if seqToRemove := asm.handleRetainedMsgDel(key, 0); seqToRemove > 0 { asm.deleteRetainedMsg(seqToRemove) asm.notifyRetainedMsgDeleted(key, seqToRemove) } } else { // Spec [MQTT-3.3.1-5]. Store the retained message with its QoS. // When coming from a publish protocol, `pp` is referencing a stack // variable that itself possibly references the read buffer. rm := &mqttRetainedMsg{ Origin: asm.jsa.id, Subject: key, Topic: string(pp.topic), Msg: pp.msg, Flags: pp.flags, Source: c.opts.Username, } rmBytes, _ := json.Marshal(rm) smr, err := asm.jsa.storeMsg(mqttRetainedMsgsStreamSubject+key, -1, rmBytes) if err == nil { // Update the new sequence rf := &mqttRetainedMsgRef{ sseq: smr.Sequence, } // Add/update the map asm.handleRetainedMsg(key, rf, rm, true) // will copy the payload bytes if needs to update rmsCache } else { c.mu.Lock() acc := c.acc c.mu.Unlock() c.Errorf("unable to store retained message for account %q, subject %q: %v", acc.GetName(), key, err) } } // Clear the retain flag for a normal published message. pp.flags &= ^mqttPubFlagRetain } // After a config reload, it is possible that the source of a publish retained // message is no longer allowed to publish on the given topic. If that is the // case, the retained message is removed from the map and will no longer be // sent to (re)starting subscriptions. // // Server lock MUST NOT be held on entry. func (s *Server) mqttCheckPubRetainedPerms() { sm := &s.mqtt.sessmgr sm.mu.RLock() done := len(sm.sessions) == 0 sm.mu.RUnlock() if done { return } s.mu.Lock() users := make(map[string]*User, len(s.users)) for un, u := range s.users { users[un] = u } s.mu.Unlock() // First get a list of all of the sessions. sm.mu.RLock() asms := make([]*mqttAccountSessionManager, 0, len(sm.sessions)) for _, asm := range sm.sessions { asms = append(asms, asm) } sm.mu.RUnlock() type retainedMsg struct { subj string rmsg *mqttRetainedMsgRef } // For each session we will obtain a list of retained messages. var _rms [128]retainedMsg rms := _rms[:0] for _, asm := range asms { // Get all of the retained messages. Then we will sort them so // that they are in sequence order, which should help the file // store to not have to load out-of-order blocks so often. asm.mu.RLock() rms = rms[:0] // reuse slice for subj, rf := range asm.retmsgs { rms = append(rms, retainedMsg{ subj: subj, rmsg: rf, }) } asm.mu.RUnlock() slices.SortFunc(rms, func(i, j retainedMsg) int { return cmp.Compare(i.rmsg.sseq, j.rmsg.sseq) }) perms := map[string]*perm{} deletes := map[string]uint64{} for _, rf := range rms { jsm, err := asm.jsa.loadMsg(mqttRetainedMsgsStreamName, rf.rmsg.sseq) if err != nil || jsm == nil { continue } var rm mqttRetainedMsg if err := json.Unmarshal(jsm.Data, &rm); err != nil { continue } if rm.Source == _EMPTY_ { continue } // Lookup source from global users. u := users[rm.Source] if u != nil { p, ok := perms[rm.Source] if !ok { p = generatePubPerms(u.Permissions) perms[rm.Source] = p } // If there is permission and no longer allowed to publish in // the subject, remove the publish retained message from the map. if p != nil && !pubAllowed(p, rf.subj) { u = nil } } // Not present or permissions have changed such that the source can't // publish on that subject anymore: remove it from the map. if u == nil { asm.mu.Lock() delete(asm.retmsgs, rf.subj) asm.sl.Remove(rf.rmsg.sub) asm.mu.Unlock() deletes[rf.subj] = rf.rmsg.sseq } } for subject, seq := range deletes { asm.deleteRetainedMsg(seq) asm.notifyRetainedMsgDeleted(subject, seq) } } } // Helper to generate only pub permissions from a Permissions object func generatePubPerms(perms *Permissions) *perm { var p *perm if perms.Publish.Allow != nil { p = &perm{} p.allow = NewSublistWithCache() for _, pubSubject := range perms.Publish.Allow { sub := &subscription{subject: []byte(pubSubject)} p.allow.Insert(sub) } } if len(perms.Publish.Deny) > 0 { if p == nil { p = &perm{} } p.deny = NewSublistWithCache() for _, pubSubject := range perms.Publish.Deny { sub := &subscription{subject: []byte(pubSubject)} p.deny.Insert(sub) } } return p } // Helper that checks if given `perms` allow to publish on the given `subject` func pubAllowed(perms *perm, subject string) bool { allowed := true if perms.allow != nil { np, _ := perms.allow.NumInterest(subject) allowed = np != 0 } // If we have a deny list and are currently allowed, check that as well. if allowed && perms.deny != nil { np, _ := perms.deny.NumInterest(subject) allowed = np == 0 } return allowed } func (c *client) mqttEnqueuePubResponse(packetType byte, pi uint16, trace bool) { proto := [4]byte{packetType, 0x2, 0, 0} proto[2] = byte(pi >> 8) proto[3] = byte(pi) // Bits 3,2,1 and 0 of the fixed header in the PUBREL Control Packet are // reserved and MUST be set to 0,0,1 and 0 respectively. The Server MUST treat // any other value as malformed and close the Network Connection [MQTT-3.6.1-1]. if packetType == mqttPacketPubRel { proto[0] |= 0x2 } c.mu.Lock() c.enqueueProto(proto[:4]) c.mu.Unlock() if trace { name := "(???)" switch packetType { case mqttPacketPubAck: name = "PUBACK" case mqttPacketPubRec: name = "PUBREC" case mqttPacketPubRel: name = "PUBREL" case mqttPacketPubComp: name = "PUBCOMP" } c.traceOutOp(name, []byte(fmt.Sprintf("pi=%v", pi))) } } func mqttParsePIPacket(r *mqttReader) (uint16, error) { pi, err := r.readUint16("packet identifier") if err != nil { return 0, err } if pi == 0 { return 0, errMQTTPacketIdentifierIsZero } return pi, nil } // Process a PUBACK (QoS1) or a PUBREC (QoS2) packet, acting as Sender. Set // isPubRec to false to process as a PUBACK. // // Runs from the client's readLoop. No lock held on entry. func (c *client) mqttProcessPublishReceived(pi uint16, isPubRec bool) (err error) { sess := c.mqtt.sess if sess == nil { return errMQTTInvalidSession } var jsAckSubject string sess.mu.Lock() // Must be the same client, and the session must have been setup for QoS2. if sess.c != c { sess.mu.Unlock() return errMQTTInvalidSession } if isPubRec { // The JS ACK subject for the PUBREL will be filled in at the delivery // attempt. sess.trackAsPubRel(pi, _EMPTY_) } jsAckSubject = sess.untrackPublish(pi) sess.mu.Unlock() if isPubRec { natsMsg, headerLen := mqttNewDeliverablePubRel(pi) _, err = sess.jsa.storeMsg(sess.pubRelSubject, headerLen, natsMsg) if err != nil { // Failure to send out PUBREL will terminate the connection. return err } } // Send the ack to JS to remove the pending message from the consumer. sess.jsa.sendAck(jsAckSubject) return nil } func (c *client) mqttProcessPubAck(pi uint16) error { return c.mqttProcessPublishReceived(pi, false) } func (c *client) mqttProcessPubRec(pi uint16) error { return c.mqttProcessPublishReceived(pi, true) } // Runs from the client's readLoop. No lock held on entry. func (c *client) mqttProcessPubComp(pi uint16) { sess := c.mqtt.sess if sess == nil { return } var jsAckSubject string sess.mu.Lock() if sess.c != c { sess.mu.Unlock() return } jsAckSubject = sess.untrackPubRel(pi) sess.mu.Unlock() // Send the ack to JS to remove the pending message from the consumer. sess.jsa.sendAck(jsAckSubject) } // Return the QoS from the given PUBLISH protocol's flags func mqttGetQoS(flags byte) byte { return flags & mqttPubFlagQoS >> 1 } func mqttIsRetained(flags byte) bool { return flags&mqttPubFlagRetain != 0 } ////////////////////////////////////////////////////////////////////////////// // // SUBSCRIBE related functions // ////////////////////////////////////////////////////////////////////////////// func (c *client) mqttParseSubs(r *mqttReader, b byte, pl int) (uint16, []*mqttFilter, error) { return c.mqttParseSubsOrUnsubs(r, b, pl, true) } func (c *client) mqttParseSubsOrUnsubs(r *mqttReader, b byte, pl int, sub bool) (uint16, []*mqttFilter, error) { var expectedFlag byte var action string if sub { expectedFlag = mqttSubscribeFlags } else { expectedFlag = mqttUnsubscribeFlags action = "un" } // Spec [MQTT-3.8.1-1], [MQTT-3.10.1-1] if rf := b & 0xf; rf != expectedFlag { return 0, nil, fmt.Errorf("wrong %ssubscribe reserved flags: %x", action, rf) } pi, err := r.readUint16("packet identifier") if err != nil { return 0, nil, fmt.Errorf("reading packet identifier: %v", err) } end := r.pos + (pl - 2) var filters []*mqttFilter for r.pos < end { // Don't make a copy now because, this will happen during conversion // or when processing the sub. topic, err := r.readBytes("topic filter", false) if err != nil { return 0, nil, err } if len(topic) == 0 { return 0, nil, errMQTTTopicFilterCannotBeEmpty } // Spec [MQTT-3.8.3-1], [MQTT-3.10.3-1] if !utf8.Valid(topic) { return 0, nil, fmt.Errorf("invalid utf8 for topic filter %q", topic) } var qos byte // We are going to report if we had an error during the conversion, // but we don't fail the parsing. When processing the sub, we will // have an error then, and the processing of subs code will send // the proper mqttSubAckFailure flag for this given subscription. filter, err := mqttFilterToNATSSubject(topic) if err != nil { c.Errorf("invalid topic %q: %v", topic, err) } if sub { qos, err = r.readByte("QoS") if err != nil { return 0, nil, err } // Spec [MQTT-3-8.3-4]. if qos > 2 { return 0, nil, fmt.Errorf("subscribe QoS value must be 0, 1 or 2, got %v", qos) } } f := &mqttFilter{ttopic: topic, filter: string(filter), qos: qos} filters = append(filters, f) } // Spec [MQTT-3.8.3-3], [MQTT-3.10.3-2] if len(filters) == 0 { return 0, nil, fmt.Errorf("%ssubscribe protocol must contain at least 1 topic filter", action) } return pi, filters, nil } func mqttSubscribeTrace(pi uint16, filters []*mqttFilter) string { var sep string sb := &strings.Builder{} sb.WriteString("[") for i, f := range filters { sb.WriteString(sep) sb.Write(f.ttopic) sb.WriteString(" (") sb.WriteString(f.filter) sb.WriteString(") QoS=") sb.WriteString(fmt.Sprintf("%v", f.qos)) if i == 0 { sep = ", " } } sb.WriteString(fmt.Sprintf("] pi=%v", pi)) return sb.String() } // For a MQTT QoS0 subscription, we create a single NATS subscription on the // actual subject, for instance "foo.bar". // // For a MQTT QoS1+ subscription, we create 2 subscriptions, one on "foo.bar" // (as for QoS0, but sub.mqtt.qos will be 1 or 2), and one on the subject // "$MQTT.sub." which is the delivery subject of the JS durable consumer // with the filter subject "$MQTT.msgs.foo.bar". // // This callback delivers messages to the client as QoS0 messages, either // because: (a) they have been produced as MQTT QoS0 messages (and therefore // only this callback can receive them); (b) they are MQTT QoS1+ published // messages but this callback is for a subscription that is QoS0; or (c) the // published messages come from (other) NATS publishers on the subject. // // This callback must reject a message if it is known to be a QoS1+ published // message and this is the callback for a QoS1+ subscription because in that // case, it will be handled by the other callback. This avoid getting duplicate // deliveries. func mqttDeliverMsgCbQoS0(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) { if pc.kind == JETSTREAM && len(reply) > 0 && strings.HasPrefix(reply, jsAckPre) { return } // This is the client associated with the subscription. cc := sub.client // This is immutable sess := cc.mqtt.sess // Lock here, otherwise we may be called with sub.mqtt == nil. Ignore // wildcard subscriptions if this subject starts with '$', per Spec // [MQTT-4.7.2-1]. sess.subsMu.RLock() subQoS := sub.mqtt.qos ignore := mqttMustIgnoreForReservedSub(sub, subject) sess.subsMu.RUnlock() if ignore { return } hdr, msg := pc.msgParts(rmsg) var topic []byte if pc.isMqtt() { // This is an MQTT publisher directly connected to this server. // Check the subscription's QoS. If the message was published with a // QoS>0 and the sub has the QoS>0 then the message will be delivered by // mqttDeliverMsgCbQoS12. msgQoS := mqttGetQoS(pc.mqtt.pp.flags) if subQoS > 0 && msgQoS > 0 { return } topic = pc.mqtt.pp.topic // Check for service imports where subject mapping is in play. if len(pc.pa.mapped) > 0 && len(pc.pa.psi) > 0 { topic = natsSubjectToMQTTTopic(subject) } } else { // Non MQTT client, could be NATS publisher, or ROUTER, etc.. h := mqttParsePublishNATSHeader(hdr) // If the message does not have the MQTT header, it is not a MQTT and // should be delivered here, at QOS0. If it does have the header, we // need to lock the session to check the sub QoS, and then ignore the // message if the Sub wants higher QOS delivery. It will be delivered by // mqttDeliverMsgCbQoS12. if subQoS > 0 && h != nil && h.qos > 0 { return } // If size is more than what a MQTT client can handle, we should probably reject, // for now just truncate. if len(msg) > mqttMaxPayloadSize { msg = msg[:mqttMaxPayloadSize] } topic = natsSubjectToMQTTTopic(subject) } // Message never has a packet identifier nor is marked as duplicate. pc.mqttEnqueuePublishMsgTo(cc, sub, 0, 0, false, topic, msg) } // This is the callback attached to a JS durable subscription for a MQTT QoS 1+ // sub. Only JETSTREAM should be sending a message to this subject (the delivery // subject associated with the JS durable consumer), but in cluster mode, this // can be coming from a route, gw, etc... We make sure that if this is the case, // the message contains a NATS/MQTT header that indicates that this is a // published QoS1+ message. func mqttDeliverMsgCbQoS12(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) { // Message on foo.bar is stored under $MQTT.msgs.foo.bar, so the subject has to be // at least as long as the stream subject prefix "$MQTT.msgs.", and after removing // the prefix, has to be at least 1 character long. if len(subject) < len(mqttStreamSubjectPrefix)+1 { return } hdr, msg := pc.msgParts(rmsg) h := mqttParsePublishNATSHeader(hdr) if pc.kind != JETSTREAM && (h == nil || h.qos == 0) { // MQTT QoS 0 messages must be ignored, they will be delivered by the // other callback, the direct NATS subscription. All JETSTREAM messages // will have the header. return } // This is the client associated with the subscription. cc := sub.client // This is immutable sess := cc.mqtt.sess // We lock to check some of the subscription's fields and if we need to keep // track of pending acks, etc. There is no need to acquire the subsMu RLock // since sess.Lock is overarching for modifying subscriptions. sess.mu.Lock() if sess.c != cc || sub.mqtt == nil { sess.mu.Unlock() return } // In this callback we handle only QoS-published messages to QoS // subscriptions. Ignore if either is 0, will be delivered by the other // callback, mqttDeliverMsgCbQos1. var qos byte if h != nil { qos = h.qos } if qos > sub.mqtt.qos { qos = sub.mqtt.qos } if qos == 0 { sess.mu.Unlock() return } // Check for reserved subject violation. If so, we will send the ack to // remove the message, and do nothing else. strippedSubj := string(subject[len(mqttStreamSubjectPrefix):]) if mqttMustIgnoreForReservedSub(sub, strippedSubj) { sess.mu.Unlock() sess.jsa.sendAck(reply) return } pi, dup := sess.trackPublish(sub.mqtt.jsDur, reply) sess.mu.Unlock() if pi == 0 { // We have reached max pending, don't send the message now. // JS will cause a redelivery and if by then the number of pending // messages has fallen below threshold, the message will be resent. return } originalTopic := natsSubjectToMQTTTopic(strippedSubj) pc.mqttEnqueuePublishMsgTo(cc, sub, pi, qos, dup, originalTopic, msg) } func mqttDeliverPubRelCb(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) { if sub.client.mqtt == nil || sub.client.mqtt.sess == nil || reply == _EMPTY_ { return } hdr, _ := pc.msgParts(rmsg) pi := mqttParsePubRelNATSHeader(hdr) if pi == 0 { return } // This is the client associated with the subscription. cc := sub.client // This is immutable sess := cc.mqtt.sess sess.mu.Lock() if sess.c != cc || sess.pubRelConsumer == nil { sess.mu.Unlock() return } sess.trackAsPubRel(pi, reply) trace := cc.trace sess.mu.Unlock() cc.mqttEnqueuePubResponse(mqttPacketPubRel, pi, trace) } // The MQTT Server MUST NOT match Topic Filters starting with a wildcard // character (# or +) with Topic Names beginning with a $ character, Spec // [MQTT-4.7.2-1]. We will return true if there is a violation. // // Session or subMu lock must be held on entry to protect access to sub.mqtt. func mqttMustIgnoreForReservedSub(sub *subscription, subject string) bool { // If the subject does not start with $ nothing to do here. if !sub.mqtt.reserved || len(subject) == 0 || subject[0] != mqttReservedPre { return false } return true } // Check if a sub is a reserved wildcard. E.g. '#', '*', or '*/" prefix. func isMQTTReservedSubscription(subject string) bool { if len(subject) == 1 && (subject[0] == fwc || subject[0] == pwc) { return true } // Match "*.<>" if len(subject) > 1 && (subject[0] == pwc && subject[1] == btsep) { return true } return false } // Common function to mqtt delivery callbacks to serialize and send the message // to the `cc` client. func (c *client) mqttEnqueuePublishMsgTo(cc *client, sub *subscription, pi uint16, qos byte, dup bool, topic, msg []byte) { flags, headerBytes := mqttMakePublishHeader(pi, qos, dup, false, topic, len(msg)) cc.mu.Lock() if sub.mqtt.prm != nil { for _, data := range sub.mqtt.prm { cc.queueOutbound(data) } sub.mqtt.prm = nil } cc.queueOutbound(headerBytes) cc.queueOutbound(msg) c.addToPCD(cc) trace := cc.trace cc.mu.Unlock() if trace { pp := mqttPublish{ topic: topic, flags: flags, pi: pi, sz: len(msg), } cc.traceOutOp("PUBLISH", []byte(mqttPubTrace(&pp))) } } // Serializes to the given writer the message for the given subject. func (w *mqttWriter) WritePublishHeader(pi uint16, qos byte, dup, retained bool, topic []byte, msgLen int) byte { // Compute len (will have to add packet id if message is sent as QoS>=1) pkLen := 2 + len(topic) + msgLen var flags byte // Set flags for dup/retained/qos1 if dup { flags |= mqttPubFlagDup } if retained { flags |= mqttPubFlagRetain } if qos > 0 { pkLen += 2 flags |= qos << 1 } w.WriteByte(mqttPacketPub | flags) w.WriteVarInt(pkLen) w.WriteBytes(topic) if qos > 0 { w.WriteUint16(pi) } return flags } // Serializes to the given writer the message for the given subject. func mqttMakePublishHeader(pi uint16, qos byte, dup, retained bool, topic []byte, msgLen int) (byte, []byte) { headerBuf := newMQTTWriter(mqttInitialPubHeader + len(topic)) flags := headerBuf.WritePublishHeader(pi, qos, dup, retained, topic, msgLen) return flags, headerBuf.Bytes() } // Process the SUBSCRIBE packet. // // Process the list of subscriptions and update the given filter // with the QoS that has been accepted (or failure). // // Spec [MQTT-3.8.4-3] says that if an exact same subscription is // found, it needs to be replaced with the new one (possibly updating // the qos) and that the flow of publications must not be interrupted, // which I read as the replacement cannot be a "remove then add" if there // is a chance that in between the 2 actions, published messages // would be "lost" because there would not be any matching subscription. // // Run from client's readLoop. // No lock held on entry. func (c *client) mqttProcessSubs(filters []*mqttFilter) ([]*subscription, error) { // Those things are immutable, but since processing subs is not // really in the fast path, let's get them under the client lock. c.mu.Lock() asm := c.mqtt.asm sess := c.mqtt.sess trace := c.trace c.mu.Unlock() if err := asm.lockSession(sess, c); err != nil { return nil, err } defer asm.unlockSession(sess) return asm.processSubs(sess, c, filters, true, trace) } // Cleanup that is performed in processSubs if there was an error. // // Runs from client's readLoop. // Lock not held on entry, but session is in the locked map. func (sess *mqttSession) cleanupFailedSub(c *client, sub *subscription, cc *ConsumerConfig, jssub *subscription) { if sub != nil { c.processUnsub(sub.sid) } if jssub != nil { c.processUnsub(jssub.sid) } if cc != nil { sess.deleteConsumer(cc) } } // Make sure we are set up to deliver PUBREL messages to this QoS2-subscribed // session. func (sess *mqttSession) ensurePubRelConsumerSubscription(c *client) error { opts := c.srv.getOpts() ackWait := opts.MQTT.AckWait if ackWait == 0 { ackWait = mqttDefaultAckWait } maxAckPending := int(opts.MQTT.MaxAckPending) if maxAckPending == 0 { maxAckPending = mqttDefaultMaxAckPending } sess.mu.Lock() pubRelSubscribed := sess.pubRelSubscribed pubRelSubject := sess.pubRelSubject pubRelDeliverySubjectB := sess.pubRelDeliverySubjectB pubRelDeliverySubject := sess.pubRelDeliverySubject pubRelConsumer := sess.pubRelConsumer tmaxack := sess.tmaxack idHash := sess.idHash id := sess.id sess.mu.Unlock() // Subscribe before the consumer is created so we don't loose any messages. if !pubRelSubscribed { _, err := c.processSub(pubRelDeliverySubjectB, nil, pubRelDeliverySubjectB, mqttDeliverPubRelCb, false) if err != nil { c.Errorf("Unable to create subscription for JetStream consumer on %q: %v", pubRelDeliverySubject, err) return err } pubRelSubscribed = true } // Create the consumer if needed. if pubRelConsumer == nil { // Check that the limit of subs' maxAckPending are not going over the limit if after := tmaxack + maxAckPending; after > mqttMaxAckTotalLimit { return fmt.Errorf("max_ack_pending for all consumers would be %v which exceeds the limit of %v", after, mqttMaxAckTotalLimit) } ccr := &CreateConsumerRequest{ Stream: mqttOutStreamName, Config: ConsumerConfig{ DeliverSubject: pubRelDeliverySubject, Durable: mqttPubRelConsumerDurablePrefix + idHash, AckPolicy: AckExplicit, DeliverPolicy: DeliverNew, FilterSubject: pubRelSubject, AckWait: ackWait, MaxAckPending: maxAckPending, MemoryStorage: opts.MQTT.ConsumerMemoryStorage, }, } if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for PUBREL for client %q: err=%v", id, err) return err } pubRelConsumer = &ccr.Config tmaxack += maxAckPending } sess.mu.Lock() sess.pubRelSubscribed = pubRelSubscribed sess.pubRelConsumer = pubRelConsumer sess.tmaxack = tmaxack sess.mu.Unlock() return nil } // When invoked with a QoS of 0, looks for an existing JS durable consumer for // the given sid and if one is found, delete the JS durable consumer and unsub // the NATS subscription on the delivery subject. // // With a QoS > 0, creates or update the existing JS durable consumer along with // its NATS subscription on a delivery subject. // // Session lock is acquired and released as needed. Session is in the locked // map. func (sess *mqttSession) processJSConsumer(c *client, subject, sid string, qos byte, fromSubProto bool) (*ConsumerConfig, *subscription, error) { sess.mu.Lock() cc, exists := sess.cons[sid] tmaxack := sess.tmaxack idHash := sess.idHash sess.mu.Unlock() // Check if we are already a JS consumer for this SID. if exists { // If current QoS is 0, it means that we need to delete the existing // one (that was QoS > 0) if qos == 0 { // The JS durable consumer's delivery subject is on a NUID of // the form: mqttSubPrefix + . It is also used as the sid // for the NATS subscription, so use that for the lookup. c.mu.Lock() sub := c.subs[cc.DeliverSubject] c.mu.Unlock() sess.mu.Lock() delete(sess.cons, sid) sess.mu.Unlock() sess.deleteConsumer(cc) if sub != nil { c.processUnsub(sub.sid) } return nil, nil, nil } // If this is called when processing SUBSCRIBE protocol, then if // the JS consumer already exists, we are done (it was created // during the processing of CONNECT). if fromSubProto { return nil, nil, nil } } // Here it means we don't have a JS consumer and if we are QoS 0, // we have nothing to do. if qos == 0 { return nil, nil, nil } var err error var inbox string if exists { inbox = cc.DeliverSubject } else { inbox = mqttSubPrefix + nuid.Next() opts := c.srv.getOpts() ackWait := opts.MQTT.AckWait if ackWait == 0 { ackWait = mqttDefaultAckWait } maxAckPending := int(opts.MQTT.MaxAckPending) if maxAckPending == 0 { maxAckPending = mqttDefaultMaxAckPending } // Check that the limit of subs' maxAckPending are not going over the limit if after := tmaxack + maxAckPending; after > mqttMaxAckTotalLimit { return nil, nil, fmt.Errorf("max_ack_pending for all consumers would be %v which exceeds the limit of %v", after, mqttMaxAckTotalLimit) } durName := idHash + "_" + nuid.Next() ccr := &CreateConsumerRequest{ Stream: mqttStreamName, Config: ConsumerConfig{ DeliverSubject: inbox, Durable: durName, AckPolicy: AckExplicit, DeliverPolicy: DeliverNew, FilterSubject: mqttStreamSubjectPrefix + subject, AckWait: ackWait, MaxAckPending: maxAckPending, MemoryStorage: opts.MQTT.ConsumerMemoryStorage, }, } if opts.MQTT.ConsumerInactiveThreshold > 0 { ccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold } if _, err := sess.jsa.createDurableConsumer(ccr); err != nil { c.Errorf("Unable to add JetStream consumer for subscription on %q: err=%v", subject, err) return nil, nil, err } cc = &ccr.Config tmaxack += maxAckPending } // This is an internal subscription on subject like "$MQTT.sub." that is setup // for the JS durable's deliver subject. sess.mu.Lock() sess.tmaxack = tmaxack sub, err := sess.processQOS12Sub(c, []byte(inbox), []byte(inbox), isMQTTReservedSubscription(subject), qos, cc.Durable, mqttDeliverMsgCbQoS12) sess.mu.Unlock() if err != nil { sess.deleteConsumer(cc) c.Errorf("Unable to create subscription for JetStream consumer on %q: %v", subject, err) return nil, nil, err } return cc, sub, nil } // Queues the published retained messages for each subscription and signals // the writeLoop. func (c *client) mqttSendRetainedMsgsToNewSubs(subs []*subscription) { c.mu.Lock() for _, sub := range subs { if sub.mqtt != nil && sub.mqtt.prm != nil { for _, data := range sub.mqtt.prm { c.queueOutbound(data) } sub.mqtt.prm = nil } } c.flushSignal() c.mu.Unlock() } func (c *client) mqttEnqueueSubAck(pi uint16, filters []*mqttFilter) { w := newMQTTWriter(7 + len(filters)) w.WriteByte(mqttPacketSubAck) // packet length is 2 (for packet identifier) and 1 byte per filter. w.WriteVarInt(2 + len(filters)) w.WriteUint16(pi) for _, f := range filters { w.WriteByte(f.qos) } c.mu.Lock() c.enqueueProto(w.Bytes()) c.mu.Unlock() } ////////////////////////////////////////////////////////////////////////////// // // UNSUBSCRIBE related functions // ////////////////////////////////////////////////////////////////////////////// func (c *client) mqttParseUnsubs(r *mqttReader, b byte, pl int) (uint16, []*mqttFilter, error) { return c.mqttParseSubsOrUnsubs(r, b, pl, false) } // Process the UNSUBSCRIBE packet. // // Given the list of topics, this is going to unsubscribe the low level NATS subscriptions // and delete the JS durable consumers when applicable. // // Runs from the client's readLoop. // No lock held on entry. func (c *client) mqttProcessUnsubs(filters []*mqttFilter) error { // Those things are immutable, but since processing unsubs is not // really in the fast path, let's get them under the client lock. c.mu.Lock() asm := c.mqtt.asm sess := c.mqtt.sess c.mu.Unlock() if err := asm.lockSession(sess, c); err != nil { return err } defer asm.unlockSession(sess) removeJSCons := func(sid string) { cc, ok := sess.cons[sid] if ok { delete(sess.cons, sid) sess.deleteConsumer(cc) // Need lock here since these are accessed by callbacks sess.mu.Lock() if seqPis, ok := sess.cpending[cc.Durable]; ok { delete(sess.cpending, cc.Durable) for _, pi := range seqPis { delete(sess.pendingPublish, pi) } if len(sess.pendingPublish) == 0 { sess.last_pi = 0 } } sess.mu.Unlock() } } for _, f := range filters { sid := f.filter // Remove JS Consumer if one exists for this sid removeJSCons(sid) if err := c.processUnsub([]byte(sid)); err != nil { c.Errorf("error unsubscribing from %q: %v", sid, err) } if mqttNeedSubForLevelUp(sid) { subject := sid[:len(sid)-2] sid = subject + mqttMultiLevelSidSuffix removeJSCons(sid) if err := c.processUnsub([]byte(sid)); err != nil { c.Errorf("error unsubscribing from %q: %v", subject, err) } } } return sess.update(filters, false) } func (c *client) mqttEnqueueUnsubAck(pi uint16) { w := newMQTTWriter(4) w.WriteByte(mqttPacketUnsubAck) w.WriteVarInt(2) w.WriteUint16(pi) c.mu.Lock() c.enqueueProto(w.Bytes()) c.mu.Unlock() } func mqttUnsubscribeTrace(pi uint16, filters []*mqttFilter) string { var sep string sb := strings.Builder{} sb.WriteString("[") for i, f := range filters { sb.WriteString(sep) sb.Write(f.ttopic) sb.WriteString(" (") sb.WriteString(f.filter) sb.WriteString(")") if i == 0 { sep = ", " } } sb.WriteString(fmt.Sprintf("] pi=%v", pi)) return sb.String() } ////////////////////////////////////////////////////////////////////////////// // // PINGREQ/PINGRESP related functions // ////////////////////////////////////////////////////////////////////////////// func (c *client) mqttEnqueuePingResp() { c.mu.Lock() c.enqueueProto(mqttPingResponse) c.mu.Unlock() } ////////////////////////////////////////////////////////////////////////////// // // Trace functions // ////////////////////////////////////////////////////////////////////////////// func errOrTrace(err error, trace string) []byte { if err != nil { return []byte(err.Error()) } return []byte(trace) } ////////////////////////////////////////////////////////////////////////////// // // Subject/Topic conversion functions // ////////////////////////////////////////////////////////////////////////////// // Converts an MQTT Topic Name to a NATS Subject (used by PUBLISH) // See mqttToNATSSubjectConversion() for details. func mqttTopicToNATSPubSubject(mt []byte) ([]byte, error) { return mqttToNATSSubjectConversion(mt, false) } // Converts an MQTT Topic Filter to a NATS Subject (used by SUBSCRIBE) // See mqttToNATSSubjectConversion() for details. func mqttFilterToNATSSubject(filter []byte) ([]byte, error) { return mqttToNATSSubjectConversion(filter, true) } // Converts an MQTT Topic Name or Filter to a NATS Subject. // In MQTT: // - a Topic Name does not have wildcard (PUBLISH uses only topic names). // - a Topic Filter can include wildcards (SUBSCRIBE uses those). // - '+' and '#' are wildcard characters (single and multiple levels respectively) // - '/' is the topic level separator. // // Conversion that occurs: // - '/' is replaced with '/.' if it is the first character in mt // - '/' is replaced with './' if the last or next character in mt is '/' // For instance, foo//bar would become foo./.bar // - '/' is replaced with '.' for all other conditions (foo/bar -> foo.bar) // - '.' is replaced with '//'. // - ' ' cause an error to be returned. // // If there is no need to convert anything (say "foo" remains "foo"), then // the no memory is allocated and the returned slice is the original `mt`. func mqttToNATSSubjectConversion(mt []byte, wcOk bool) ([]byte, error) { var cp bool var j int res := mt makeCopy := func(i int) { cp = true res = make([]byte, 0, len(mt)+10) if i > 0 { res = append(res, mt[:i]...) } } end := len(mt) - 1 for i := 0; i < len(mt); i++ { switch mt[i] { case mqttTopicLevelSep: if i == 0 || res[j-1] == btsep { if !cp { makeCopy(0) } res = append(res, mqttTopicLevelSep, btsep) j++ } else if i == end || mt[i+1] == mqttTopicLevelSep { if !cp { makeCopy(i) } res = append(res, btsep, mqttTopicLevelSep) j++ } else { if !cp { makeCopy(i) } res = append(res, btsep) } case ' ': // As of now, we cannot support ' ' in the MQTT topic/filter. return nil, errMQTTUnsupportedCharacters case btsep: if !cp { makeCopy(i) } res = append(res, mqttTopicLevelSep, mqttTopicLevelSep) j++ case mqttSingleLevelWC, mqttMultiLevelWC: if !wcOk { // Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1] // The wildcard characters can be used in Topic Filters, but MUST NOT be used within a Topic Name return nil, fmt.Errorf("wildcards not allowed in publish's topic: %q", mt) } if !cp { makeCopy(i) } if mt[i] == mqttSingleLevelWC { res = append(res, pwc) } else { res = append(res, fwc) } default: if cp { res = append(res, mt[i]) } } j++ } if cp && res[j-1] == btsep { res = append(res, mqttTopicLevelSep) j++ } return res[:j], nil } // Converts a NATS subject to MQTT topic. This is for publish // messages only, so there is no checking for wildcards. // Rules are reversed of mqttToNATSSubjectConversion. func natsSubjectToMQTTTopic(subject string) []byte { topic := []byte(subject) end := len(subject) - 1 var j int for i := 0; i < len(subject); i++ { switch subject[i] { case mqttTopicLevelSep: if i < end { switch c := subject[i+1]; c { case btsep, mqttTopicLevelSep: if c == btsep { topic[j] = mqttTopicLevelSep } else { topic[j] = btsep } j++ i++ default: } } case btsep: topic[j] = mqttTopicLevelSep j++ default: topic[j] = subject[i] j++ } } return topic[:j] } // Returns true if the subject has more than 1 token and ends with ".>" func mqttNeedSubForLevelUp(subject string) bool { if len(subject) < 3 { return false } end := len(subject) if subject[end-2] == '.' && subject[end-1] == fwc { return true } return false } ////////////////////////////////////////////////////////////////////////////// // // MQTT Reader functions // ////////////////////////////////////////////////////////////////////////////// func (r *mqttReader) reset(buf []byte) { if l := len(r.pbuf); l > 0 { tmp := make([]byte, l+len(buf)) copy(tmp, r.pbuf) copy(tmp[l:], buf) buf = tmp r.pbuf = nil } r.buf = buf r.pos = 0 r.pstart = 0 } func (r *mqttReader) hasMore() bool { return r.pos != len(r.buf) } func (r *mqttReader) readByte(field string) (byte, error) { if r.pos == len(r.buf) { return 0, fmt.Errorf("error reading %s: %v", field, io.EOF) } b := r.buf[r.pos] r.pos++ return b, nil } func (r *mqttReader) readPacketLen() (int, bool, error) { return r.readPacketLenWithCheck(true) } func (r *mqttReader) readPacketLenWithCheck(check bool) (int, bool, error) { m := 1 v := 0 for { var b byte if r.pos != len(r.buf) { b = r.buf[r.pos] r.pos++ } else { break } v += int(b&0x7f) * m if (b & 0x80) == 0 { if check && r.pos+v > len(r.buf) { break } return v, true, nil } m *= 0x80 if m > 0x200000 { return 0, false, errMQTTMalformedVarInt } } r.pbuf = make([]byte, len(r.buf)-r.pstart) copy(r.pbuf, r.buf[r.pstart:]) return 0, false, nil } func (r *mqttReader) readString(field string) (string, error) { var s string bs, err := r.readBytes(field, false) if err == nil { s = string(bs) } return s, err } func (r *mqttReader) readBytes(field string, cp bool) ([]byte, error) { luint, err := r.readUint16(field) if err != nil { return nil, err } l := int(luint) if l == 0 { return nil, nil } start := r.pos if start+l > len(r.buf) { return nil, fmt.Errorf("error reading %s: %v", field, io.ErrUnexpectedEOF) } r.pos += l b := r.buf[start:r.pos] if cp { b = copyBytes(b) } return b, nil } func (r *mqttReader) readUint16(field string) (uint16, error) { if len(r.buf)-r.pos < 2 { return 0, fmt.Errorf("error reading %s: %v", field, io.ErrUnexpectedEOF) } start := r.pos r.pos += 2 return binary.BigEndian.Uint16(r.buf[start:r.pos]), nil } ////////////////////////////////////////////////////////////////////////////// // // MQTT Writer functions // ////////////////////////////////////////////////////////////////////////////// func (w *mqttWriter) WriteUint16(i uint16) { w.WriteByte(byte(i >> 8)) w.WriteByte(byte(i)) } func (w *mqttWriter) WriteString(s string) { w.WriteBytes([]byte(s)) } func (w *mqttWriter) WriteBytes(bs []byte) { w.WriteUint16(uint16(len(bs))) w.Write(bs) } func (w *mqttWriter) WriteVarInt(value int) { for { b := byte(value & 0x7f) value >>= 7 if value > 0 { b |= 0x80 } w.WriteByte(b) if value == 0 { break } } } func newMQTTWriter(cap int) *mqttWriter { w := &mqttWriter{} w.Grow(cap) return w } nats-server-2.10.27/server/mqtt_ex_bench_test.go000066400000000000000000000204631477524627100216740ustar00rootroot00000000000000// Copyright 2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_mqtt_tests // +build !skip_mqtt_tests package server import ( "fmt" "strconv" "testing" "time" ) const ( KB = 1024 ) type mqttBenchMatrix struct { QOS []int MessageSize []int Topics []int Publishers []int Subscribers []int } type mqttBenchContext struct { QOS int MessageSize int Topics int Publishers int Subscribers int Host string Port int } var mqttBenchDefaultMatrix = mqttBenchMatrix{ QOS: []int{0, 1, 2}, MessageSize: []int{100, 1 * KB, 10 * KB}, Topics: []int{100}, Publishers: []int{1}, Subscribers: []int{1}, } type MQTTBenchmarkResult struct { Ops int `json:"ops"` NS map[string]time.Duration `json:"ns"` Bytes int64 `json:"bytes"` } func BenchmarkXMQTT(b *testing.B) { if mqttTestCommandPath == "" { b.Skip(`"mqtt-test" command is not found in $PATH.`) } bc := mqttBenchContext{} b.Run("Server", func(b *testing.B) { b.Cleanup(bc.startServer(b, false)) bc.runAll(b) }) b.Run("Cluster", func(b *testing.B) { b.Cleanup(bc.startCluster(b, false)) bc.runAll(b) }) b.Run("Server-no-RMSCache", func(b *testing.B) { b.Cleanup(bc.startServer(b, true)) bc.benchmarkSubRet(b) }) b.Run("Cluster-no-RMSCache", func(b *testing.B) { b.Cleanup(bc.startCluster(b, true)) bc.benchmarkSubRet(b) }) } func (bc mqttBenchContext) runAll(b *testing.B) { bc.benchmarkPub(b) bc.benchmarkPubRetained(b) bc.benchmarkPubSub(b) bc.benchmarkSubRet(b) } // makes a copy of bc func (bc mqttBenchContext) benchmarkPub(b *testing.B) { m := mqttBenchDefaultMatrix. NoSubscribers(). NoTopics() b.Run("PUB", func(b *testing.B) { m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { bc.runAndReport(b, "pub", "--qos", strconv.Itoa(bc.QOS), "--messages", strconv.Itoa(b.N), "--size", strconv.Itoa(bc.MessageSize), "--publishers", strconv.Itoa(bc.Publishers), ) }) }) } // makes a copy of bc func (bc mqttBenchContext) benchmarkPubRetained(b *testing.B) { // This bench is meaningless for QOS0 since the client considers the message // sent as soon as it's written out. It is also useless for QOS2 since the // flow takes a lot longer, and the difference of publishing as retained or // not is lost in the noise. m := mqttBenchDefaultMatrix. NoSubscribers(). NoTopics(). QOS1Only() b.Run("PUBRET", func(b *testing.B) { m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { bc.runAndReport(b, "pub", "--retain", "--qos", strconv.Itoa(bc.QOS), "--messages", strconv.Itoa(b.N), "--size", strconv.Itoa(bc.MessageSize), "--publishers", strconv.Itoa(bc.Publishers), ) }) }) } // makes a copy of bc func (bc mqttBenchContext) benchmarkPubSub(b *testing.B) { // This test uses a single built-in topic, and a built-in publisher, so no // reason to run it for topics and publishers. m := mqttBenchDefaultMatrix. NoTopics(). NoPublishers() b.Run("PUBSUB", func(b *testing.B) { m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { bc.runAndReport(b, "pubsub", "--qos", strconv.Itoa(bc.QOS), "--messages", strconv.Itoa(b.N), "--size", strconv.Itoa(bc.MessageSize), "--subscribers", strconv.Itoa(bc.Subscribers), ) }) }) } // makes a copy of bc func (bc mqttBenchContext) benchmarkSubRet(b *testing.B) { // This test uses a a built-in publisher, and it makes most sense to measure // the retained message delivery "overhead" on a QoS0 subscription; without // the extra time involved in actually subscribing. m := mqttBenchDefaultMatrix. NoPublishers(). QOS0Only() b.Run("SUBRET", func(b *testing.B) { m.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) { bc.runAndReport(b, "subret", "--qos", strconv.Itoa(bc.QOS), "--topics", strconv.Itoa(bc.Topics), // number of retained messages "--subscribers", strconv.Itoa(bc.Subscribers), "--size", strconv.Itoa(bc.MessageSize), "--repeat", strconv.Itoa(b.N), // number of subscribe requests ) }) }) } func (bc mqttBenchContext) runAndReport(b *testing.B, name string, extraArgs ...string) { b.Helper() r := mqttRunExCommandTest(b, name, mqttNewDial("", "", bc.Host, bc.Port, ""), extraArgs...) r.report(b) } func (bc *mqttBenchContext) startServer(b *testing.B, disableRMSCache bool) func() { b.Helper() b.StopTimer() prevDisableRMSCache := testDisableRMSCache testDisableRMSCache = disableRMSCache o := testMQTTDefaultOptions() s := testMQTTRunServer(b, o) o = s.getOpts() bc.Host = o.MQTT.Host bc.Port = o.MQTT.Port mqttInitTestServer(b, mqttNewDial("", "", bc.Host, bc.Port, "")) return func() { testMQTTShutdownServer(s) testDisableRMSCache = prevDisableRMSCache } } func (bc *mqttBenchContext) startCluster(b *testing.B, disableRMSCache bool) func() { b.Helper() b.StopTimer() prevDisableRMSCache := testDisableRMSCache testDisableRMSCache = disableRMSCache conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 stream_replicas: 3 } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(b, conf, "MQTT", 3) o := cl.randomNonLeader().getOpts() bc.Host = o.MQTT.Host bc.Port = o.MQTT.Port mqttInitTestServer(b, mqttNewDial("", "", bc.Host, bc.Port, "")) return func() { cl.shutdown() testDisableRMSCache = prevDisableRMSCache } } func mqttBenchWrapForMatrixField( vFieldPtr *int, arr []int, f func(b *testing.B, bc *mqttBenchContext), namef func(int) string, ) func(b *testing.B, bc *mqttBenchContext) { if len(arr) == 0 { return f } return func(b *testing.B, bc *mqttBenchContext) { for _, value := range arr { *vFieldPtr = value b.Run(namef(value), func(b *testing.B) { f(b, bc) }) } } } func (m mqttBenchMatrix) runMatrix(b *testing.B, bc mqttBenchContext, f func(*testing.B, *mqttBenchContext)) { b.Helper() f = mqttBenchWrapForMatrixField(&bc.MessageSize, m.MessageSize, f, func(size int) string { return sizeKB(size) }) f = mqttBenchWrapForMatrixField(&bc.Topics, m.Topics, f, func(n int) string { return fmt.Sprintf("%dtopics", n) }) f = mqttBenchWrapForMatrixField(&bc.Publishers, m.Publishers, f, func(n int) string { return fmt.Sprintf("%dpubc", n) }) f = mqttBenchWrapForMatrixField(&bc.Subscribers, m.Subscribers, f, func(n int) string { return fmt.Sprintf("%dsubc", n) }) f = mqttBenchWrapForMatrixField(&bc.QOS, m.QOS, f, func(qos int) string { return fmt.Sprintf("QOS%d", qos) }) b.ResetTimer() b.StartTimer() f(b, &bc) } func (m mqttBenchMatrix) NoSubscribers() mqttBenchMatrix { m.Subscribers = nil return m } func (m mqttBenchMatrix) NoTopics() mqttBenchMatrix { m.Topics = nil return m } func (m mqttBenchMatrix) NoPublishers() mqttBenchMatrix { m.Publishers = nil return m } func (m mqttBenchMatrix) QOS0Only() mqttBenchMatrix { m.QOS = []int{0} return m } func (m mqttBenchMatrix) QOS1Only() mqttBenchMatrix { m.QOS = []int{1} return m } func sizeKB(size int) string { unit := "B" N := size if size >= KB { unit = "KB" N = (N + KB/2) / KB } return fmt.Sprintf("%d%s", N, unit) } func (r MQTTBenchmarkResult) report(b *testing.B) { // Disable the default ns metric in favor of custom X_ms/op. b.ReportMetric(0, "ns/op") // Disable MB/s since the github benchmarking action misinterprets the sign // of the result (treats it as less is better). b.SetBytes(0) // b.SetBytes(r.Bytes) for unit, ns := range r.NS { nsOp := float64(ns) / float64(r.Ops) b.ReportMetric(nsOp/1000000, unit+"_ms/op") } // Diable ReportAllocs() since it confuses the github benchmarking action // with the noise. // b.ReportAllocs() } nats-server-2.10.27/server/mqtt_ex_test_test.go000066400000000000000000000214151477524627100215720ustar00rootroot00000000000000// Copyright 2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_mqtt_tests // +build !skip_mqtt_tests package server import ( "bytes" "encoding/json" "fmt" "io" "os" "os/exec" "strconv" "strings" "testing" "github.com/nats-io/nuid" ) type mqttDial string type mqttTarget struct { singleServers []*Server clusters []*cluster configs []mqttTestConfig all []mqttDial } type mqttTestConfig struct { name string pub []mqttDial sub []mqttDial } func TestXMQTTCompliance(t *testing.T) { if mqttCLICommandPath == _EMPTY_ { t.Skip(`"mqtt" command is not found in $PATH nor $MQTT_CLI. See https://hivemq.github.io/mqtt-cli/docs/installation/#debian-package for installation instructions`) } o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) o = s.getOpts() defer testMQTTShutdownServer(s) cmd := exec.Command(mqttCLICommandPath, "test", "-V", "3", "-p", strconv.Itoa(o.MQTT.Port)) output, err := cmd.CombinedOutput() t.Log(string(output)) if err != nil { if exitError, ok := err.(*exec.ExitError); ok { t.Fatalf("mqtt cli exited with error: %v", exitError) } } } func TestXMQTTRetainedMessages(t *testing.T) { if mqttTestCommandPath == _EMPTY_ { t.Skip(`"mqtt-test" command is not found in $PATH.`) } for _, topo := range []struct { name string makef func(testing.TB) *mqttTarget }{ { name: "single server", makef: mqttMakeTestServer, }, { name: "cluster", makef: mqttMakeTestCluster(4, ""), }, } { t.Run(topo.name, func(t *testing.T) { target := topo.makef(t) t.Cleanup(target.Shutdown) // initialize the MQTT assets by "touching" all nodes in the // cluster, but then reload to start with fresh server state. for _, dial := range target.all { mqttInitTestServer(t, dial) } numRMS := 100 strNumRMS := strconv.Itoa(numRMS) topics := make([]string, len(target.configs)) for i, tc := range target.configs { // Publish numRMS retained messages one at a time, // round-robin across pub nodes. Remember the topic for each // test config to check the subs after reload. topic := "subret_" + nuid.Next() topics[i] = topic iNode := 0 for i := 0; i < numRMS; i++ { pubTopic := fmt.Sprintf("%s/%d", topic, i) dial := tc.pub[iNode%len(tc.pub)] mqttRunExCommandTest(t, "pub", dial, "--retain", "--topic", pubTopic, "--qos", "0", "--size", "128", // message size 128 bytes ) iNode++ } } // Check all sub nodes for retained messages for i, tc := range target.configs { for _, dial := range tc.sub { mqttRunExCommandTest(t, "sub", dial, "--retained", strNumRMS, "--qos", "0", "--topic", topics[i], ) } } // Reload the target target.Reload(t) // Now check again for i, tc := range target.configs { for _, dial := range tc.sub { mqttRunExCommandTestRetry(t, 1, "sub", dial, "--retained", strNumRMS, "--qos", "0", "--topic", topics[i], ) } } }) } } func mqttInitTestServer(tb testing.TB, dial mqttDial) { tb.Helper() mqttRunExCommandTestRetry(tb, 5, "pub", dial) } func mqttNewDialForServer(s *Server, username, password string) mqttDial { o := s.getOpts().MQTT return mqttNewDial(username, password, o.Host, o.Port, s.Name()) } func mqttNewDial(username, password, host string, port int, comment string) mqttDial { d := "" switch { case username != "" && password != "": d = fmt.Sprintf("%s:%s@%s:%d", username, password, host, port) case username != "": d = fmt.Sprintf("%s@%s:%d", username, host, port) default: d = fmt.Sprintf("%s:%d", host, port) } if comment != "" { d += "#" + comment } return mqttDial(d) } func (d mqttDial) Get() (u, p, s, c string) { if d == "" { return "", "", "127.0.0.1:1883", "" } in := string(d) if i := strings.LastIndex(in, "#"); i != -1 { c = in[i+1:] in = in[:i] } if i := strings.LastIndex(in, "@"); i != -1 { up := in[:i] in = in[i+1:] u = up if i := strings.Index(up, ":"); i != -1 { u = up[:i] p = up[i+1:] } } s = in return u, p, s, c } func (d mqttDial) Name() string { _, _, _, c := d.Get() return c } func (t *mqttTarget) Reload(tb testing.TB) { tb.Helper() for _, c := range t.clusters { c.stopAll() c.restartAllSamePorts() } for i, s := range t.singleServers { o := s.getOpts() s.Shutdown() t.singleServers[i] = testMQTTRunServer(tb, o) } for _, dial := range t.all { mqttInitTestServer(tb, dial) } } func (t *mqttTarget) Shutdown() { for _, c := range t.clusters { c.shutdown() } for _, s := range t.singleServers { testMQTTShutdownServer(s) } } func mqttMakeTestServer(tb testing.TB) *mqttTarget { tb.Helper() o := testMQTTDefaultOptions() s := testMQTTRunServer(tb, o) all := []mqttDial{mqttNewDialForServer(s, "", "")} return &mqttTarget{ singleServers: []*Server{s}, all: all, configs: []mqttTestConfig{ { name: "single server", pub: all, sub: all, }, }, } } func mqttMakeTestCluster(size int, domain string) func(tb testing.TB) *mqttTarget { return func(tb testing.TB) *mqttTarget { tb.Helper() if size < 3 { tb.Fatal("cluster size must be at least 3") } if domain != "" { domain = "domain: " + domain + ", " } clusterConf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, ` + domain + `store_dir: '%s'} leafnodes { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 stream_replicas: 3 } accounts { ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(tb, clusterConf, "MQTT", size) cl.waitOnLeader() all := []mqttDial{} for _, s := range cl.servers { all = append(all, mqttNewDialForServer(s, "one", "p")) } return &mqttTarget{ clusters: []*cluster{cl}, all: all, configs: []mqttTestConfig{ { name: "publish to one", pub: []mqttDial{ mqttNewDialForServer(cl.randomServer(), "one", "p"), }, sub: all, }, { name: "publish to all", pub: all, sub: all, }, }, } } } var mqttCLICommandPath = func() string { p := os.Getenv("MQTT_CLI") if p == "" { p, _ = exec.LookPath("mqtt") } return p }() var mqttTestCommandPath = func() string { p, _ := exec.LookPath("mqtt-test") return p }() func mqttRunExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) *MQTTBenchmarkResult { tb.Helper() return mqttRunExCommandTestRetry(tb, 1, subCommand, dial, extraArgs...) } func mqttRunExCommandTestRetry(tb testing.TB, n int, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult) { tb.Helper() var err error for i := 0; i < n; i++ { if r, err = mqttTryExCommandTest(tb, subCommand, dial, extraArgs...); err == nil { return r } if i < (n - 1) { tb.Logf("failed to %q %s to %q on attempt %v, will retry.", subCommand, extraArgs, dial.Name(), i) } else { tb.Fatal(err) } } return nil } func mqttTryExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult, err error) { tb.Helper() if mqttTestCommandPath == "" { tb.Skip(`"mqtt-test" command is not found in $PATH.`) } args := []string{subCommand, // "-q", "-s", string(dial), } args = append(args, extraArgs...) cmd := exec.Command(mqttTestCommandPath, args...) stdout, err := cmd.StdoutPipe() if err != nil { return nil, fmt.Errorf("error executing %q: %v", cmd.String(), err) } defer stdout.Close() errbuf := bytes.Buffer{} cmd.Stderr = &errbuf if err = cmd.Start(); err != nil { return nil, fmt.Errorf("error executing %q: %v", cmd.String(), err) } out, err := io.ReadAll(stdout) if err != nil { return nil, fmt.Errorf("error executing %q: failed to read output: %v", cmd.String(), err) } if err = cmd.Wait(); err != nil { return nil, fmt.Errorf("error executing %q: %v\n\n%s\n\n%s", cmd.String(), err, string(out), errbuf.String()) } r = &MQTTBenchmarkResult{} if err := json.Unmarshal(out, r); err != nil { tb.Fatalf("error executing %q: failed to decode output: %v\n\n%s\n\n%s", cmd.String(), err, string(out), errbuf.String()) } return r, nil } nats-server-2.10.27/server/mqtt_test.go000066400000000000000000007427251477524627100200550ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_mqtt_tests // +build !skip_mqtt_tests package server import ( "bufio" "bytes" "crypto/tls" "encoding/json" "errors" "fmt" "io" "math/rand" "net" "os" "reflect" "strings" "sync" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) var testMQTTTimeout = 10 * time.Second var jsClusterTemplWithLeafAndMQTT = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} {{leaf}} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` type mqttWrapAsWs struct { net.Conn t testing.TB br *bufio.Reader tmp []byte } func (c *mqttWrapAsWs) Write(p []byte) (int, error) { proto := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, p) return c.Conn.Write(proto) } func (c *mqttWrapAsWs) Read(p []byte) (int, error) { for { if n := len(c.tmp); n > 0 { if len(p) < n { n = len(p) } copy(p, c.tmp[:n]) c.tmp = c.tmp[n:] return n, nil } c.tmp = testWSReadFrame(c.t, c.br) } } func testMQTTReadPacket(t testing.TB, r *mqttReader) (byte, int) { t.Helper() var b byte var pl int var err error rd := r.reader fill := func() []byte { t.Helper() var buf [512]byte n, err := rd.Read(buf[:]) if err != nil { t.Fatalf("Error reading data: %v", err) } return copyBytes(buf[:n]) } rd.SetReadDeadline(time.Now().Add(testMQTTTimeout)) for { r.pstart = r.pos if !r.hasMore() { r.reset(fill()) continue } b, err = r.readByte("packet type") if err != nil { t.Fatalf("Error reading packet: %v", err) } var complete bool pl, complete, err = r.readPacketLen() if err != nil { t.Fatalf("Error reading packet: %v", err) } if !complete { r.reset(fill()) continue } break } rd.SetReadDeadline(time.Time{}) return b, pl } func testMQTTReadPIPacket(expectedType byte, t testing.TB, r *mqttReader, expectedPI uint16) { t.Helper() b, _ := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != expectedType { t.Fatalf("Expected packet %x, got %x", expectedType, pt) } rpi, err := r.readUint16("packet identifier") if err != nil || rpi != expectedPI { t.Fatalf("Expected PI %v got: %v, err=%v", expectedPI, rpi, err) } } func TestMQTTReader(t *testing.T) { r := &mqttReader{} r.reset([]byte{0, 2, 'a', 'b'}) bs, err := r.readBytes("", false) if err != nil { t.Fatal(err) } sbs := string(bs) if sbs != "ab" { t.Fatalf(`expected "ab", got %q`, sbs) } r.reset([]byte{0, 2, 'a', 'b'}) bs, err = r.readBytes("", true) if err != nil { t.Fatal(err) } bs[0], bs[1] = 'c', 'd' if bytes.Equal(bs, r.buf[2:]) { t.Fatal("readBytes should have returned a copy") } r.reset([]byte{'a', 'b'}) if b, err := r.readByte(""); err != nil || b != 'a' { t.Fatalf("Error reading byte: b=%v err=%v", b, err) } if !r.hasMore() { t.Fatal("expected to have more, did not") } if b, err := r.readByte(""); err != nil || b != 'b' { t.Fatalf("Error reading byte: b=%v err=%v", b, err) } if r.hasMore() { t.Fatal("expected to not have more") } if _, err := r.readByte("test"); err == nil || !strings.Contains(err.Error(), "error reading test") { t.Fatalf("unexpected error: %v", err) } r.reset([]byte{0, 2, 'a', 'b'}) if s, err := r.readString(""); err != nil || s != "ab" { t.Fatalf("Error reading string: s=%q err=%v", s, err) } r.reset([]byte{10}) if _, err := r.readUint16("uint16"); err == nil || !strings.Contains(err.Error(), "error reading uint16") { t.Fatalf("unexpected error: %v", err) } r.reset([]byte{0x82, 0xff, 0x3}) l, _, err := r.readPacketLenWithCheck(false) if err != nil { t.Fatal("error getting packet len") } if l != 0xff82 { t.Fatalf("expected length 0xff82 got 0x%x", l) } r.reset([]byte{0xff, 0xff, 0xff, 0xff, 0xff}) if _, _, err := r.readPacketLenWithCheck(false); err == nil || !strings.Contains(err.Error(), "malformed") { t.Fatalf("unexpected error: %v", err) } r.reset([]byte{0, 2, 'a', 'b', mqttPacketPub, 0x82, 0xff, 0x3}) r.readString("") for i := 0; i < 2; i++ { r.pstart = r.pos b, err := r.readByte("") if err != nil { t.Fatalf("Error reading byte: %v", err) } if pt := b & mqttPacketMask; pt != mqttPacketPub { t.Fatalf("Unexpected byte: %v", b) } pl, complete, err := r.readPacketLen() if err != nil { t.Fatalf("Unexpected error: %v", err) } if complete { t.Fatal("Expected to be incomplete") } if pl != 0 { t.Fatalf("Expected pl to be 0, got %v", pl) } if i > 0 { break } if !bytes.Equal(r.pbuf, []byte{mqttPacketPub, 0x82, 0xff, 0x3}) { t.Fatalf("Invalid recorded partial: %v", r.pbuf) } r.reset([]byte{'a', 'b', 'c'}) if !bytes.Equal(r.buf, []byte{mqttPacketPub, 0x82, 0xff, 0x3, 'a', 'b', 'c'}) { t.Fatalf("Invalid buffer: %v", r.buf) } if r.pbuf != nil { t.Fatalf("Partial buffer should have been reset, got %v", r.pbuf) } if r.pos != 0 { t.Fatalf("Pos should have been reset, got %v", r.pos) } if r.pstart != 0 { t.Fatalf("Pstart should have been reset, got %v", r.pstart) } } // On second pass, the pbuf should have been extended with 'abc' if !bytes.Equal(r.pbuf, []byte{mqttPacketPub, 0x82, 0xff, 0x3, 'a', 'b', 'c'}) { t.Fatalf("Invalid recorded partial: %v", r.pbuf) } } func TestMQTTWriter(t *testing.T) { w := newMQTTWriter(0) w.WriteUint16(1234) r := &mqttReader{} r.reset(w.Bytes()) if v, err := r.readUint16(""); err != nil || v != 1234 { t.Fatalf("unexpected value: v=%v err=%v", v, err) } w.Reset() w.WriteString("test") r.reset(w.Bytes()) if len(r.buf) != 6 { t.Fatalf("Expected 2 bytes size before string, got %v", r.buf) } w.Reset() w.WriteBytes([]byte("test")) r.reset(w.Bytes()) if len(r.buf) != 6 { t.Fatalf("Expected 2 bytes size before bytes, got %v", r.buf) } ints := []int{ 0, 1, 127, 128, 16383, 16384, 2097151, 2097152, 268435455, } lens := []int{ 1, 1, 1, 2, 2, 3, 3, 4, 4, } tl := 0 w.Reset() for i, v := range ints { w.WriteVarInt(v) tl += lens[i] if tl != w.Len() { t.Fatalf("expected len %d, got %d", tl, w.Len()) } } r.reset(w.Bytes()) for _, v := range ints { x, _, _ := r.readPacketLenWithCheck(false) if v != x { t.Fatalf("expected %d, got %d", v, x) } } } func testMQTTDefaultOptions() *Options { o := DefaultOptions() o.ServerName = nuid.Next() o.Cluster.Port = 0 o.Gateway.Name = "" o.Gateway.Port = 0 o.LeafNode.Port = 0 o.Websocket.Port = 0 o.MQTT.Host = "127.0.0.1" o.MQTT.Port = -1 o.JetStream = true return o } func testMQTTRunServer(t testing.TB, o *Options) *Server { t.Helper() o.NoLog = false if o.StoreDir == _EMPTY_ { o.StoreDir = t.TempDir() } s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } l := &DummyLogger{} s.SetLogger(l, true, true) s.Start() if err := s.readyForConnections(3 * time.Second); err != nil { testMQTTShutdownServer(s) t.Fatal(err) } return s } func testMQTTShutdownRestartedServer(s **Server) { srv := *s testMQTTShutdownServer(srv) *s = nil } func testMQTTShutdownServer(s *Server) { if c := s.JetStreamConfig(); c != nil { dir := strings.TrimSuffix(c.StoreDir, JetStreamStoreDir) defer os.RemoveAll(dir) } s.Shutdown() } func testMQTTDefaultTLSOptions(t *testing.T, verify bool) *Options { t.Helper() o := testMQTTDefaultOptions() tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", Verify: verify, } var err error o.MQTT.TLSConfig, err = GenTLSConfig(tc) o.MQTT.TLSTimeout = 2.0 if err != nil { t.Fatalf("Error creating tls config: %v", err) } return o } func TestMQTTServerNameRequired(t *testing.T) { conf := createConfFile(t, []byte(` cluster { port: -1 } mqtt { port: -1 } `)) o, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } if _, err := NewServer(o); err == nil || err.Error() != errMQTTServerNameMustBeSet.Error() { t.Fatalf("Expected error about requiring server name to be set, got %v", err) } conf = createConfFile(t, []byte(` mqtt { port: -1 } `)) o, err = ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } if _, err := NewServer(o); err == nil || err.Error() != errMQTTStandaloneNeedsJetStream.Error() { t.Fatalf(`Expected errMQTTStandaloneNeedsJetStream ("next in line"), got %v`, err) } } func TestMQTTStandaloneRequiresJetStream(t *testing.T) { conf := createConfFile(t, []byte(` server_name: mqtt mqtt { port: -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" } } `)) o, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } if _, err := NewServer(o); err == nil || err.Error() != errMQTTStandaloneNeedsJetStream.Error() { t.Fatalf("Expected error about requiring JetStream in standalone mode, got %v", err) } } func TestMQTTConfig(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` jetstream { store_dir = %q } server_name: mqtt mqtt { port: -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" } } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) if o.MQTT.TLSConfig == nil { t.Fatal("expected TLS config to be set") } } func TestMQTTValidateOptions(t *testing.T) { nmqtto := DefaultOptions() mqtto := testMQTTDefaultOptions() for _, test := range []struct { name string getOpts func() *Options err error }{ {"mqtt disabled", func() *Options { return nmqtto.Clone() }, nil}, {"mqtt username not allowed if users specified", func() *Options { o := mqtto.Clone() o.Users = []*User{{Username: "abc", Password: "pwd"}} o.MQTT.Username = "b" o.MQTT.Password = "pwd" return o }, errMQTTUserMixWithUsersNKeys}, {"mqtt token not allowed if users specified", func() *Options { o := mqtto.Clone() o.Nkeys = []*NkeyUser{{Nkey: "abc"}} o.MQTT.Token = "mytoken" return o }, errMQTTTokenMixWIthUsersNKeys}, {"ack wait should be >=0", func() *Options { o := mqtto.Clone() o.MQTT.AckWait = -10 * time.Second return o }, errMQTTAckWaitMustBePositive}, } { t.Run(test.name, func(t *testing.T) { err := validateMQTTOptions(test.getOpts()) if test.err == nil && err != nil { t.Fatalf("Unexpected error: %v", err) } else if test.err != nil && (err == nil || err.Error() != test.err.Error()) { t.Fatalf("Expected error to contain %q, got %v", test.err, err) } }) } } func TestMQTTParseOptions(t *testing.T) { for _, test := range []struct { name string content string checkOpt func(*MQTTOpts) error err string }{ // Negative tests {"bad type", "mqtt: []", nil, "to be a map"}, {"bad listen", "mqtt: { listen: [] }", nil, "port or host:port"}, {"bad port", `mqtt: { port: "abc" }`, nil, "not int64"}, {"bad host", `mqtt: { host: 123 }`, nil, "not string"}, {"bad tls", `mqtt: { tls: 123 }`, nil, "not map[string]interface {}"}, {"unknown field", `mqtt: { this_does_not_exist: 123 }`, nil, "unknown"}, {"ack wait", `mqtt: {ack_wait: abc}`, nil, "invalid duration"}, {"max ack pending", `mqtt: {max_ack_pending: abc}`, nil, "not int64"}, {"max ack pending too high", `mqtt: {max_ack_pending: 12345678}`, nil, "invalid value"}, // Positive tests {"tls gen fails", ` mqtt { tls { cert_file: "./configs/certs/server.pem" } }`, nil, "missing 'key_file'"}, {"listen port only", `mqtt { listen: 1234 }`, func(o *MQTTOpts) error { if o.Port != 1234 { return fmt.Errorf("expected 1234, got %v", o.Port) } return nil }, ""}, {"listen host and port", `mqtt { listen: "localhost:1234" }`, func(o *MQTTOpts) error { if o.Host != "localhost" || o.Port != 1234 { return fmt.Errorf("expected localhost:1234, got %v:%v", o.Host, o.Port) } return nil }, ""}, {"host", `mqtt { host: "localhost" }`, func(o *MQTTOpts) error { if o.Host != "localhost" { return fmt.Errorf("expected localhost, got %v", o.Host) } return nil }, ""}, {"port", `mqtt { port: 1234 }`, func(o *MQTTOpts) error { if o.Port != 1234 { return fmt.Errorf("expected 1234, got %v", o.Port) } return nil }, ""}, {"tls config", ` mqtt { tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" } } `, func(o *MQTTOpts) error { if o.TLSConfig == nil { return fmt.Errorf("TLSConfig should have been set") } return nil }, ""}, {"no auth user", ` mqtt { no_auth_user: "noauthuser" } `, func(o *MQTTOpts) error { if o.NoAuthUser != "noauthuser" { return fmt.Errorf("Invalid NoAuthUser value: %q", o.NoAuthUser) } return nil }, ""}, {"auth block", ` mqtt { authorization { user: "mqttuser" password: "pwd" token: "token" timeout: 2.0 } } `, func(o *MQTTOpts) error { if o.Username != "mqttuser" || o.Password != "pwd" || o.Token != "token" || o.AuthTimeout != 2.0 { return fmt.Errorf("Invalid auth block: %+v", o) } return nil }, ""}, {"auth timeout as int", ` mqtt { authorization { timeout: 2 } } `, func(o *MQTTOpts) error { if o.AuthTimeout != 2.0 { return fmt.Errorf("Invalid auth timeout: %v", o.AuthTimeout) } return nil }, ""}, {"ack wait", ` mqtt { ack_wait: "10s" } `, func(o *MQTTOpts) error { if o.AckWait != 10*time.Second { return fmt.Errorf("Invalid ack wait: %v", o.AckWait) } return nil }, ""}, {"max ack pending", ` mqtt { max_ack_pending: 123 } `, func(o *MQTTOpts) error { if o.MaxAckPending != 123 { return fmt.Errorf("Invalid max ack pending: %v", o.MaxAckPending) } return nil }, ""}, {"reject_qos2_publish", ` mqtt { reject_qos2_publish: true } `, func(o *MQTTOpts) error { if !o.rejectQoS2Pub { return fmt.Errorf("Invalid: expected rejectQoS2Pub to be set") } return nil }, ""}, {"downgrade_qos2_subscribe", ` mqtt { downgrade_qos2_subscribe: true } `, func(o *MQTTOpts) error { if !o.downgradeQoS2Sub { return fmt.Errorf("Invalid: expected downgradeQoS2Sub to be set") } return nil }, ""}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(test.content)) o, err := ProcessConfigFile(conf) if test.err != _EMPTY_ { if err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("For content: %q, expected error about %q, got %v", test.content, test.err, err) } return } else if err != nil { t.Fatalf("Unexpected error for content %q: %v", test.content, err) } if err := test.checkOpt(&o.MQTT); err != nil { t.Fatalf("Incorrect option for content %q: %v", test.content, err.Error()) } }) } } func TestMQTTStart(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port)) if err != nil { t.Fatalf("Unable to create tcp connection to mqtt port: %v", err) } nc.Close() // Check failure to start due to port in use o2 := testMQTTDefaultOptions() o2.MQTT.Port = o.MQTT.Port s2, err := NewServer(o2) if err != nil { t.Fatalf("Error creating server: %v", err) } defer s2.Shutdown() l := &captureFatalLogger{fatalCh: make(chan string, 1)} s2.SetLogger(l, false, false) wg := sync.WaitGroup{} wg.Add(1) go func() { s2.Start() wg.Done() }() select { case e := <-l.fatalCh: if !strings.Contains(e, "Unable to listen for MQTT connections") { t.Fatalf("Unexpected error: %q", e) } case <-time.After(time.Second): t.Fatal("Should have gotten a fatal error") } } func TestMQTTTLS(t *testing.T) { o := testMQTTDefaultTLSOptions(t, false) s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) c, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{tls: true}, o.MQTT.Host, o.MQTT.Port, 0) if err != nil { t.Fatal(err) } c.Close() c = nil testMQTTShutdownServer(s) // Force client cert verification o = testMQTTDefaultTLSOptions(t, true) s = testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) c, _, err = testMQTTConnectRetryWithError(t, &mqttConnInfo{tls: true}, o.MQTT.Host, o.MQTT.Port, 0) if err == nil || c != nil { if c != nil { c.Close() } t.Fatal("Handshake expected to fail since client did not provide cert") } // Add client cert. tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/client-cert.pem", KeyFile: "../test/configs/certs/client-key.pem", } tlsc, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } tlsc.InsecureSkipVerify = true c, _, err = testMQTTConnectRetryWithError(t, &mqttConnInfo{ tls: true, tlsc: tlsc, }, o.MQTT.Host, o.MQTT.Port, 0) if err != nil { t.Fatal(err) } c.Close() c = nil testMQTTShutdownServer(s) // Lower TLS timeout so low that we should fail o.MQTT.TLSTimeout = 0.001 s = testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port)) if err != nil { t.Fatalf("Unable to create tcp connection to mqtt port: %v", err) } defer nc.Close() time.Sleep(100 * time.Millisecond) tlsConn := tls.Client(nc, tlsc) tlsConn.SetDeadline(time.Now().Add(time.Second)) if err := tlsConn.Handshake(); err == nil { t.Fatal("Expected failure, did not get one") } } type mqttConnInfo struct { clientID string cleanSess bool keepAlive uint16 will *mqttWill user string pass string ws bool tls bool tlsc *tls.Config } func testMQTTGetClient(t testing.TB, s *Server, clientID string) *client { t.Helper() var mc *client s.mu.Lock() for _, c := range s.clients { c.mu.Lock() if c.isMqtt() && c.mqtt.cid == clientID { mc = c } c.mu.Unlock() if mc != nil { break } } s.mu.Unlock() if mc == nil { t.Fatalf("Did not find client %q", clientID) } return mc } func testMQTTRead(c net.Conn) ([]byte, error) { var buf [512]byte // Make sure that test does not block c.SetReadDeadline(time.Now().Add(testMQTTTimeout)) n, err := c.Read(buf[:]) if err != nil { return nil, err } c.SetReadDeadline(time.Time{}) return copyBytes(buf[:n]), nil } func testMQTTWrite(c net.Conn, buf []byte) (int, error) { c.SetWriteDeadline(time.Now().Add(testMQTTTimeout)) n, err := c.Write(buf) c.SetWriteDeadline(time.Time{}) return n, err } func testMQTTConnect(t testing.TB, ci *mqttConnInfo, host string, port int) (net.Conn, *mqttReader) { t.Helper() return testMQTTConnectRetry(t, ci, host, port, 0) } func testMQTTConnectRetry(t testing.TB, ci *mqttConnInfo, host string, port int, retryCount int) (net.Conn, *mqttReader) { t.Helper() c, r, err := testMQTTConnectRetryWithError(t, ci, host, port, retryCount) if err != nil { t.Fatal(err) } return c, r } func testMQTTConnectRetryWithError(t testing.TB, ci *mqttConnInfo, host string, port int, retryCount int) (net.Conn, *mqttReader, error) { retry := func(c net.Conn) bool { if c != nil { c.Close() } if retryCount == 0 { return false } time.Sleep(time.Second) retryCount-- return true } addr := fmt.Sprintf("%s:%d", host, port) var c net.Conn var err error RETRY: if ci.ws { var br *bufio.Reader c, br, _, err = testNewWSClientWithError(t, testWSClientOptions{ host: host, port: port, noTLS: !ci.tls, path: mqttWSPath, }) if err == nil { c = &mqttWrapAsWs{Conn: c, t: t, br: br} } } else { c, err = net.Dial("tcp", addr) if err == nil && ci.tls { tc := ci.tlsc if tc == nil { tc = &tls.Config{InsecureSkipVerify: true} } c = tls.Client(c, tc) c.SetDeadline(time.Now().Add(time.Second)) err = c.(*tls.Conn).Handshake() c.SetDeadline(time.Time{}) } } if err != nil { if retry(c) { goto RETRY } return nil, nil, fmt.Errorf("Error creating mqtt connection: %v", err) } proto := mqttCreateConnectProto(ci) if _, err := testMQTTWrite(c, proto); err != nil { if retry(c) { goto RETRY } return nil, nil, fmt.Errorf("Error writing connect: %v", err) } buf, err := testMQTTRead(c) if err != nil { if retry(c) { goto RETRY } return nil, nil, fmt.Errorf("Error reading: %v", err) } mr := &mqttReader{reader: c} mr.reset(buf) return c, mr, nil } func mqttCreateConnectProto(ci *mqttConnInfo) []byte { flags := byte(0) if ci.cleanSess { flags |= mqttConnFlagCleanSession } if ci.will != nil { flags |= mqttConnFlagWillFlag | (ci.will.qos << 3) if ci.will.retain { flags |= mqttConnFlagWillRetain } } if ci.user != _EMPTY_ { flags |= mqttConnFlagUsernameFlag } if ci.pass != _EMPTY_ { flags |= mqttConnFlagPasswordFlag } pkLen := 2 + len(mqttProtoName) + 1 + // proto level 1 + // flags 2 + // keepAlive 2 + len(ci.clientID) if ci.will != nil { pkLen += 2 + len(ci.will.topic) pkLen += 2 + len(ci.will.message) } if ci.user != _EMPTY_ { pkLen += 2 + len(ci.user) } if ci.pass != _EMPTY_ { pkLen += 2 + len(ci.pass) } w := newMQTTWriter(0) w.WriteByte(mqttPacketConnect) w.WriteVarInt(pkLen) w.WriteString(string(mqttProtoName)) w.WriteByte(0x4) w.WriteByte(flags) w.WriteUint16(ci.keepAlive) w.WriteString(ci.clientID) if ci.will != nil { w.WriteBytes(ci.will.topic) w.WriteBytes(ci.will.message) } if ci.user != _EMPTY_ { w.WriteString(ci.user) } if ci.pass != _EMPTY_ { w.WriteBytes([]byte(ci.pass)) } return w.Bytes() } func testMQTTCheckConnAck(t testing.TB, r *mqttReader, rc byte, sessionPresent bool) { t.Helper() b, pl := testMQTTReadPacket(t, r) pt := b & mqttPacketMask if pt != mqttPacketConnectAck { t.Fatalf("Expected ConnAck (%x), got %x", mqttPacketConnectAck, pt) } if pl != 2 { t.Fatalf("ConnAck packet length should be 2, got %v", pl) } caf, err := r.readByte("connack flags") if err != nil { t.Fatalf("Error reading packet length: %v", err) } if caf&0xfe != 0 { t.Fatalf("ConnAck flag bits 7-1 should all be 0, got %x", caf>>1) } if sp := caf == 1; sp != sessionPresent { t.Fatalf("Expected session present flag=%v got %v", sessionPresent, sp) } carc, err := r.readByte("connack return code") if err != nil { t.Fatalf("Error reading returned code: %v", err) } if carc != rc { t.Fatalf("Expected return code to be %v, got %v", rc, carc) } } func testMQTTCheckPubAck(t testing.TB, r *mqttReader, packetType byte) { t.Helper() b, pl := testMQTTReadPacket(t, r) pt := b & mqttPacketMask if pt != packetType { t.Fatalf("Expected %x, got %x", packetType, pt) } r.pos += pl } func TestMQTTRequiresJSEnabled(t *testing.T) { o := testMQTTDefaultOptions() acc := NewAccount("mqtt") o.Accounts = []*Account{acc} o.Users = []*User{{Username: "mqtt", Account: acc}} s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) addr := fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port) c, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating mqtt connection: %v", err) } defer c.Close() proto := mqttCreateConnectProto(&mqttConnInfo{cleanSess: true, user: "mqtt"}) if _, err := testMQTTWrite(c, proto); err != nil { t.Fatalf("Error writing connect: %v", err) } if _, err := testMQTTRead(c); err == nil { t.Fatal("Expected failure, did not get one") } } func testMQTTEnableJSForAccount(t *testing.T, s *Server, accName string) { t.Helper() acc, err := s.LookupAccount(accName) if err != nil { t.Fatalf("Error looking up account: %v", err) } limits := map[string]JetStreamAccountLimits{ _EMPTY_: { MaxConsumers: -1, MaxStreams: -1, MaxMemory: 1024 * 1024, MaxStore: 1024 * 1024, }, } if err := acc.EnableJetStream(limits); err != nil { t.Fatalf("Error enabling JS: %v", err) } } func TestMQTTTLSVerifyAndMap(t *testing.T) { accName := "MyAccount" acc := NewAccount(accName) certUserName := "CN=example.com,OU=NATS.io" users := []*User{{Username: certUserName, Account: acc}} for _, test := range []struct { name string filtering bool provideCert bool }{ {"no filtering, client provides cert", false, true}, {"no filtering, client does not provide cert", false, false}, {"filtering, client provides cert", true, true}, {"filtering, client does not provide cert", true, false}, } { t.Run(test.name, func(t *testing.T) { o := testMQTTDefaultOptions() o.Host = "localhost" o.Accounts = []*Account{acc} o.Users = users if test.filtering { o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt}) } tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/server.pem", KeyFile: "../test/configs/certs/tlsauth/server-key.pem", CaFile: "../test/configs/certs/tlsauth/ca.pem", Verify: true, } tlsc, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating tls config: %v", err) } o.MQTT.TLSConfig = tlsc o.MQTT.TLSTimeout = 2.0 o.MQTT.TLSMap = true s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) testMQTTEnableJSForAccount(t, s, accName) tlscc := &tls.Config{} if test.provideCert { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/client.pem", KeyFile: "../test/configs/certs/tlsauth/client-key.pem", } var err error tlscc, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } } tlscc.InsecureSkipVerify = true if test.provideCert { tlscc.MinVersion = tls.VersionTLS13 } mc, r, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{ cleanSess: true, tls: true, tlsc: tlscc, }, o.MQTT.Host, o.MQTT.Port, 0) if !test.provideCert { if err == nil { t.Fatal("Expected error, did not get one") } else if !strings.Contains(err.Error(), "bad certificate") && !strings.Contains(err.Error(), "certificate required") { t.Fatalf("Unexpected error: %v", err) } return } if err != nil { t.Fatalf("Error reading: %v", err) } defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) var c *client s.mu.Lock() for _, sc := range s.clients { sc.mu.Lock() if sc.isMqtt() { c = sc } sc.mu.Unlock() if c != nil { break } } s.mu.Unlock() if c == nil { t.Fatal("Client not found") } var uname string var accname string c.mu.Lock() uname = c.opts.Username if c.acc != nil { accname = c.acc.GetName() } c.mu.Unlock() if uname != certUserName { t.Fatalf("Expected username %q, got %q", certUserName, uname) } if accname != accName { t.Fatalf("Expected account %q, got %v", accName, accname) } }) } } func TestMQTTBasicAuth(t *testing.T) { for _, test := range []struct { name string opts func() *Options user string pass string rc byte }{ { "top level auth, no override, wrong u/p", func() *Options { o := testMQTTDefaultOptions() o.Username = "normal" o.Password = "client" return o }, "mqtt", "client", mqttConnAckRCNotAuthorized, }, { "top level auth, no override, correct u/p", func() *Options { o := testMQTTDefaultOptions() o.Username = "normal" o.Password = "client" return o }, "normal", "client", mqttConnAckRCConnectionAccepted, }, { "no top level auth, mqtt auth, wrong u/p", func() *Options { o := testMQTTDefaultOptions() o.MQTT.Username = "mqtt" o.MQTT.Password = "client" return o }, "normal", "client", mqttConnAckRCNotAuthorized, }, { "no top level auth, mqtt auth, correct u/p", func() *Options { o := testMQTTDefaultOptions() o.MQTT.Username = "mqtt" o.MQTT.Password = "client" return o }, "mqtt", "client", mqttConnAckRCConnectionAccepted, }, { "top level auth, mqtt override, wrong u/p", func() *Options { o := testMQTTDefaultOptions() o.Username = "normal" o.Password = "client" o.MQTT.Username = "mqtt" o.MQTT.Password = "client" return o }, "normal", "client", mqttConnAckRCNotAuthorized, }, { "top level auth, mqtt override, correct u/p", func() *Options { o := testMQTTDefaultOptions() o.Username = "normal" o.Password = "client" o.MQTT.Username = "mqtt" o.MQTT.Password = "client" return o }, "mqtt", "client", mqttConnAckRCConnectionAccepted, }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ cleanSess: true, user: test.user, pass: test.pass, } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, test.rc, false) }) } } func TestMQTTAuthTimeout(t *testing.T) { for _, test := range []struct { name string at float64 mat float64 ok bool }{ {"use top-level auth timeout", 0.5, 0.0, true}, {"use mqtt auth timeout", 0.5, 0.05, false}, } { t.Run(test.name, func(t *testing.T) { o := testMQTTDefaultOptions() o.AuthTimeout = test.at o.MQTT.Username = "mqtt" o.MQTT.Password = "client" o.MQTT.AuthTimeout = test.mat s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer mc.Close() time.Sleep(100 * time.Millisecond) ci := &mqttConnInfo{ cleanSess: true, user: "mqtt", pass: "client", } proto := mqttCreateConnectProto(ci) if _, err := testMQTTWrite(mc, proto); err != nil { if test.ok { t.Fatalf("Error sending connect: %v", err) } // else it is ok since we got disconnected due to auth timeout return } buf, err := testMQTTRead(mc) if err != nil { if test.ok { t.Fatalf("Error reading: %v", err) } // else it is ok since we got disconnected due to auth timeout return } r := &mqttReader{reader: mc} r.reset(buf) testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) time.Sleep(500 * time.Millisecond) testMQTTPublish(t, mc, r, 1, false, false, "foo", 1, []byte("msg")) }) } } func TestMQTTTokenAuth(t *testing.T) { for _, test := range []struct { name string opts func() *Options token string rc byte }{ { "top level auth, no override, wrong token", func() *Options { o := testMQTTDefaultOptions() o.Authorization = "goodtoken" return o }, "badtoken", mqttConnAckRCNotAuthorized, }, { "top level auth, no override, correct token", func() *Options { o := testMQTTDefaultOptions() o.Authorization = "goodtoken" return o }, "goodtoken", mqttConnAckRCConnectionAccepted, }, { "no top level auth, mqtt auth, wrong token", func() *Options { o := testMQTTDefaultOptions() o.MQTT.Token = "goodtoken" return o }, "badtoken", mqttConnAckRCNotAuthorized, }, { "no top level auth, mqtt auth, correct token", func() *Options { o := testMQTTDefaultOptions() o.MQTT.Token = "goodtoken" return o }, "goodtoken", mqttConnAckRCConnectionAccepted, }, { "top level auth, mqtt override, wrong token", func() *Options { o := testMQTTDefaultOptions() o.Authorization = "clienttoken" o.MQTT.Token = "mqtttoken" return o }, "clienttoken", mqttConnAckRCNotAuthorized, }, { "top level auth, mqtt override, correct token", func() *Options { o := testMQTTDefaultOptions() o.Authorization = "clienttoken" o.MQTT.Token = "mqtttoken" return o }, "mqtttoken", mqttConnAckRCConnectionAccepted, }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ cleanSess: true, user: "ignore_use_token", pass: test.token, } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, test.rc, false) }) } } func TestMQTTJWTWithAllowedConnectionTypes(t *testing.T) { o := testMQTTDefaultOptions() // Create System Account syskp, _ := nkeys.CreateAccount() syspub, _ := syskp.PublicKey() sysAc := jwt.NewAccountClaims(syspub) sysjwt, err := sysAc.Encode(oKp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } // Create memory resolver and store system account mr := &MemAccResolver{} mr.Store(syspub, sysjwt) if err != nil { t.Fatalf("Error saving system account JWT to memory resolver: %v", err) } // Add system account and memory resolver to server options o.SystemAccount = syspub o.AccountResolver = mr setupAddTrusted(o) s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) for _, test := range []struct { name string connectionTypes []string rc byte }{ {"not allowed", []string{jwt.ConnectionTypeStandard}, mqttConnAckRCNotAuthorized}, {"allowed", []string{jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeMqtt)}, mqttConnAckRCConnectionAccepted}, {"allowed with unknown", []string{jwt.ConnectionTypeMqtt, "SomeNewType"}, mqttConnAckRCConnectionAccepted}, {"not allowed with unknown", []string{"SomeNewType"}, mqttConnAckRCNotAuthorized}, } { t.Run(test.name, func(t *testing.T) { nuc := newJWTTestUserClaims() nuc.AllowedConnectionTypes = test.connectionTypes nuc.BearerToken = true okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) // Enable Jetstream on account with lax limitations nac.Limits.JetStreamLimits.Consumer = -1 nac.Limits.JetStreamLimits.Streams = -1 nac.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 nac.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc.Subject = pub jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } addAccountToMemResolver(s, apub, ajwt) ci := &mqttConnInfo{ cleanSess: true, user: "ignore_use_token", pass: jwt, } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, test.rc, false) }) } } func TestMQTTUsersAuth(t *testing.T) { users := []*User{{Username: "user", Password: "pwd"}} for _, test := range []struct { name string opts func() *Options user string pass string rc byte }{ { "no filtering, wrong user", func() *Options { o := testMQTTDefaultOptions() o.Users = users return o }, "wronguser", "pwd", mqttConnAckRCNotAuthorized, }, { "no filtering, correct user", func() *Options { o := testMQTTDefaultOptions() o.Users = users return o }, "user", "pwd", mqttConnAckRCConnectionAccepted, }, { "filtering, user not allowed", func() *Options { o := testMQTTDefaultOptions() o.Users = users // Only allowed for regular clients o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard}) return o }, "user", "pwd", mqttConnAckRCNotAuthorized, }, { "filtering, user allowed", func() *Options { o := testMQTTDefaultOptions() o.Users = users o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt}) return o }, "user", "pwd", mqttConnAckRCConnectionAccepted, }, { "filtering, wrong password", func() *Options { o := testMQTTDefaultOptions() o.Users = users o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt}) return o }, "user", "badpassword", mqttConnAckRCNotAuthorized, }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ cleanSess: true, user: test.user, pass: test.pass, } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, test.rc, false) }) } } func TestMQTTNoAuthUserValidation(t *testing.T) { o := testMQTTDefaultOptions() o.Users = []*User{{Username: "user", Password: "pwd"}} // Should fail because it is not part of o.Users. o.MQTT.NoAuthUser = "notfound" if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), "not present as user") { t.Fatalf("Expected error saying not present as user, got %v", err) } // Set a valid no auth user for global options, but still should fail because // of o.MQTT.NoAuthUser o.NoAuthUser = "user" o.MQTT.NoAuthUser = "notfound" if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), "not present as user") { t.Fatalf("Expected error saying not present as user, got %v", err) } } func TestMQTTNoAuthUser(t *testing.T) { for _, test := range []struct { name string override bool useAuth bool expectedUser string expectedAcc string }{ {"no override, no user provided", false, false, "noauth", "normal"}, {"no override, user povided", false, true, "user", "normal"}, {"override, no user provided", true, false, "mqttnoauth", "mqtt"}, {"override, user provided", true, true, "mqttuser", "mqtt"}, } { t.Run(test.name, func(t *testing.T) { o := testMQTTDefaultOptions() normalAcc := NewAccount("normal") mqttAcc := NewAccount("mqtt") o.Accounts = []*Account{normalAcc, mqttAcc} o.Users = []*User{ {Username: "noauth", Password: "pwd", Account: normalAcc}, {Username: "user", Password: "pwd", Account: normalAcc}, {Username: "mqttnoauth", Password: "pwd", Account: mqttAcc}, {Username: "mqttuser", Password: "pwd", Account: mqttAcc}, } o.NoAuthUser = "noauth" if test.override { o.MQTT.NoAuthUser = "mqttnoauth" } s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) testMQTTEnableJSForAccount(t, s, "normal") testMQTTEnableJSForAccount(t, s, "mqtt") ci := &mqttConnInfo{clientID: "mqtt", cleanSess: true} if test.useAuth { ci.user = test.expectedUser ci.pass = "pwd" } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) c := testMQTTGetClient(t, s, "mqtt") c.mu.Lock() uname := c.opts.Username aname := c.acc.GetName() c.mu.Unlock() if uname != test.expectedUser { t.Fatalf("Expected selected user to be %q, got %q", test.expectedUser, uname) } if aname != test.expectedAcc { t.Fatalf("Expected selected account to be %q, got %q", test.expectedAcc, aname) } }) } } func TestMQTTConnectNotFirstPacket(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) l := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(l, false, false) c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port)) if err != nil { t.Fatalf("Error on dial: %v", err) } defer c.Close() testMQTTSendPublishPacket(t, c, 0, false, false, "foo", 0, []byte("hello")) testMQTTExpectDisconnect(t, c) select { case err := <-l.errCh: if !strings.Contains(err, "should be a CONNECT") { t.Fatalf("Expected error about first packet being a CONNECT, got %v", err) } case <-time.After(time.Second): t.Fatal("Did not log any error") } } func TestMQTTSecondConnect(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) proto := mqttCreateConnectProto(&mqttConnInfo{cleanSess: true}) if _, err := testMQTTWrite(mc, proto); err != nil { t.Fatalf("Error writing connect: %v", err) } testMQTTExpectDisconnect(t, mc) } func TestMQTTParseConnect(t *testing.T) { for _, test := range []struct { name string proto []byte err string }{ {"packet in buffer error", []byte{0}, io.ErrUnexpectedEOF.Error()}, {"bad proto name", []byte{0, 4, 'B', 'A', 'D'}, "protocol name"}, {"invalid proto name", []byte{0, 3, 'B', 'A', 'D'}, "expected connect packet with protocol name"}, {"old proto not supported", []byte{0, 6, 'M', 'Q', 'I', 's', 'd', 'p'}, "older protocol"}, {"error on protocol level", []byte{0, 4, 'M', 'Q', 'T', 'T'}, "protocol level"}, {"unacceptable protocol version", []byte{0, 4, 'M', 'Q', 'T', 'T', 10}, "unacceptable protocol version"}, {"error on flags", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel}, "flags"}, {"reserved flag", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1}, errMQTTConnFlagReserved.Error()}, {"will qos without will flag", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1 << 3}, "if Will flag is set to 0, Will QoS must be 0 too"}, {"will retain without will flag", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1 << 5}, errMQTTWillAndRetainFlag.Error()}, {"will qos", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 3<<3 | 1<<2}, "if Will flag is set to 1, Will QoS can be 0, 1 or 2"}, {"no user but password", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagPasswordFlag}, errMQTTPasswordFlagAndNoUser.Error()}, {"missing keep alive", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0}, "keep alive"}, {"missing client ID", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1}, "client ID"}, {"empty client ID", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1, 0, 0}, errMQTTCIDEmptyNeedsCleanFlag.Error()}, {"invalid utf8 client ID", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1, 0, 1, 241}, "invalid utf8 for client ID"}, {"missing will topic", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0}, "Will topic"}, {"empty will topic", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 0}, errMQTTEmptyWillTopic.Error()}, {"invalid utf8 will topic", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 241}, "invalid utf8 for Will topic"}, {"invalid wildcard will topic", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, '#'}, "wildcards not allowed"}, {"error on will message", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 'a', 0, 3}, "Will message"}, {"error on username", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0}, "user name"}, {"empty username", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 0}, errMQTTEmptyUsername.Error()}, {"invalid utf8 username", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 241}, "invalid utf8 for user name"}, {"error on password", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagPasswordFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 'a'}, "password"}, } { t.Run(test.name, func(t *testing.T) { r := &mqttReader{} r.reset(test.proto) mqtt := &mqtt{r: r} c := &client{mqtt: mqtt} if _, _, err := c.mqttParseConnect(r, false); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func TestMQTTConnectFailsOnParse(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) addr := fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port) c, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating mqtt connection: %v", err) } pkLen := 2 + len(mqttProtoName) + 1 + // proto level 1 + // flags 2 + // keepAlive 2 + len("mqtt") w := newMQTTWriter(0) w.WriteByte(mqttPacketConnect) w.WriteVarInt(pkLen) w.WriteString(string(mqttProtoName)) w.WriteByte(0x7) w.WriteByte(mqttConnFlagCleanSession) w.WriteUint16(0) w.WriteString("mqtt") c.Write(w.Bytes()) buf, err := testMQTTRead(c) if err != nil { t.Fatalf("Error reading: %v", err) } r := &mqttReader{reader: c} r.reset(buf) testMQTTCheckConnAck(t, r, mqttConnAckRCUnacceptableProtocolVersion, false) } func TestMQTTConnKeepAlive(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true, keepAlive: 1}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc, r, 0, false, false, "foo", 0, []byte("msg")) time.Sleep(2 * time.Second) testMQTTExpectDisconnect(t, mc) } func TestMQTTDontSetPinger(t *testing.T) { o := testMQTTDefaultOptions() o.PingInterval = 15 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: "mqtt", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) c := testMQTTGetClient(t, s, "mqtt") c.mu.Lock() timerSet := c.ping.tmr != nil c.mu.Unlock() if timerSet { t.Fatalf("Ping timer should not be set for MQTT clients") } // Wait a bit and expect nothing (and connection should still be valid) testMQTTExpectNothing(t, r) testMQTTPublish(t, mc, r, 0, false, false, "foo", 0, []byte("msg")) } func TestMQTTTopicAndSubjectConversion(t *testing.T) { for _, test := range []struct { name string mqttTopic string natsSubject string err string }{ {"/", "/", "/./", ""}, {"//", "//", "/././", ""}, {"///", "///", "/./././", ""}, {"////", "////", "/././././", ""}, {"foo", "foo", "foo", ""}, {"/foo", "/foo", "/.foo", ""}, {"//foo", "//foo", "/./.foo", ""}, {"///foo", "///foo", "/././.foo", ""}, {"///foo/", "///foo/", "/././.foo./", ""}, {"///foo//", "///foo//", "/././.foo././", ""}, {"///foo///", "///foo///", "/././.foo./././", ""}, {"//.foo.//", "//.foo.//", "/././/foo//././", ""}, {"foo/bar", "foo/bar", "foo.bar", ""}, {"/foo/bar", "/foo/bar", "/.foo.bar", ""}, {"/foo/bar/", "/foo/bar/", "/.foo.bar./", ""}, {"foo/bar/baz", "foo/bar/baz", "foo.bar.baz", ""}, {"/foo/bar/baz", "/foo/bar/baz", "/.foo.bar.baz", ""}, {"/foo/bar/baz/", "/foo/bar/baz/", "/.foo.bar.baz./", ""}, {"bar", "bar/", "bar./", ""}, {"bar//", "bar//", "bar././", ""}, {"bar///", "bar///", "bar./././", ""}, {"foo//bar", "foo//bar", "foo./.bar", ""}, {"foo///bar", "foo///bar", "foo././.bar", ""}, {"foo////bar", "foo////bar", "foo./././.bar", ""}, {".", ".", "//", ""}, {"..", "..", "////", ""}, {"...", "...", "//////", ""}, {"./", "./", "//./", ""}, {".//.", ".//.", "//././/", ""}, {"././.", "././.", "//.//.//", ""}, {"././/.", "././/.", "//.//././/", ""}, {".foo", ".foo", "//foo", ""}, {"foo.", "foo.", "foo//", ""}, {".foo.", ".foo.", "//foo//", ""}, {"foo../bar/", "foo../bar/", "foo////.bar./", ""}, {"foo../bar/.", "foo../bar/.", "foo////.bar.//", ""}, {"/foo/", "/foo/", "/.foo./", ""}, {"./foo/.", "./foo/.", "//.foo.//", ""}, {"foo.bar/baz", "foo.bar/baz", "foo//bar.baz", ""}, // These should produce errors {"foo/+", "foo/+", "", "wildcards not allowed in publish"}, {"foo/#", "foo/#", "", "wildcards not allowed in publish"}, {"foo bar", "foo bar", "", "not supported"}, } { t.Run(test.name, func(t *testing.T) { res, err := mqttTopicToNATSPubSubject([]byte(test.mqttTopic)) if test.err != _EMPTY_ { if err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } return } toNATS := string(res) if toNATS != test.natsSubject { t.Fatalf("Expected subject %q got %q", test.natsSubject, toNATS) } res = natsSubjectToMQTTTopic(string(res)) backToMQTT := string(res) if backToMQTT != test.mqttTopic { t.Fatalf("Expected topic %q got %q (NATS conversion was %q)", test.mqttTopic, backToMQTT, toNATS) } }) } } func TestMQTTFilterConversion(t *testing.T) { // Similar to TopicConversion test except that wildcards are OK here. // So testing only those. for _, test := range []struct { name string mqttTopic string natsSubject string }{ {"single level wildcard", "+", "*"}, {"single level wildcard", "/+", "/.*"}, {"single level wildcard", "+/", "*./"}, {"single level wildcard", "/+/", "/.*./"}, {"single level wildcard", "foo/+", "foo.*"}, {"single level wildcard", "foo/+/", "foo.*./"}, {"single level wildcard", "foo/+/bar", "foo.*.bar"}, {"single level wildcard", "foo/+/+", "foo.*.*"}, {"single level wildcard", "foo/+/+/", "foo.*.*./"}, {"single level wildcard", "foo/+/+/bar", "foo.*.*.bar"}, {"single level wildcard", "foo//+", "foo./.*"}, {"single level wildcard", "foo//+/", "foo./.*./"}, {"single level wildcard", "foo//+//", "foo./.*././"}, {"single level wildcard", "foo//+//bar", "foo./.*./.bar"}, {"single level wildcard", "foo///+///bar", "foo././.*././.bar"}, {"single level wildcard", "foo.bar///+///baz", "foo//bar././.*././.baz"}, {"multi level wildcard", "#", ">"}, {"multi level wildcard", "/#", "/.>"}, {"multi level wildcard", "/foo/#", "/.foo.>"}, {"multi level wildcard", "foo/#", "foo.>"}, {"multi level wildcard", "foo//#", "foo./.>"}, {"multi level wildcard", "foo///#", "foo././.>"}, {"multi level wildcard", "foo/bar/#", "foo.bar.>"}, {"multi level wildcard", "foo/bar.baz/#", "foo.bar//baz.>"}, } { t.Run(test.name, func(t *testing.T) { res, err := mqttFilterToNATSSubject([]byte(test.mqttTopic)) if err != nil { t.Fatalf("Error: %v", err) } if string(res) != test.natsSubject { t.Fatalf("Expected subject %q got %q", test.natsSubject, res) } }) } } func TestMQTTParseSub(t *testing.T) { for _, test := range []struct { name string proto []byte b byte pl int err string }{ {"reserved flag", nil, 3, 0, "wrong subscribe reserved flags"}, {"ensure packet loaded", []byte{1, 2}, mqttSubscribeFlags, 10, io.ErrUnexpectedEOF.Error()}, {"error reading packet id", []byte{1}, mqttSubscribeFlags, 1, "reading packet identifier"}, {"missing filters", []byte{0, 1}, mqttSubscribeFlags, 2, "subscribe protocol must contain at least 1 topic filter"}, {"error reading topic", []byte{0, 1, 0, 2, 'a'}, mqttSubscribeFlags, 5, "topic filter"}, {"empty topic", []byte{0, 1, 0, 0}, mqttSubscribeFlags, 4, errMQTTTopicFilterCannotBeEmpty.Error()}, {"invalid utf8 topic", []byte{0, 1, 0, 1, 241}, mqttSubscribeFlags, 5, "invalid utf8 for topic filter"}, {"missing qos", []byte{0, 1, 0, 1, 'a'}, mqttSubscribeFlags, 5, "QoS"}, {"invalid qos", []byte{0, 1, 0, 1, 'a', 3}, mqttSubscribeFlags, 6, "subscribe QoS value must be 0, 1 or 2"}, } { t.Run(test.name, func(t *testing.T) { r := &mqttReader{} r.reset(test.proto) mqtt := &mqtt{r: r} c := &client{mqtt: mqtt} if _, _, err := c.mqttParseSubsOrUnsubs(r, test.b, test.pl, true); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func testMQTTSub(t testing.TB, pi uint16, c net.Conn, r *mqttReader, filters []*mqttFilter, expected []byte) { t.Helper() w := newMQTTWriter(0) pkLen := 2 // for pi for i := 0; i < len(filters); i++ { f := filters[i] pkLen += 2 + len(f.filter) + 1 } w.WriteByte(mqttPacketSub | mqttSubscribeFlags) w.WriteVarInt(pkLen) w.WriteUint16(pi) for i := 0; i < len(filters); i++ { f := filters[i] w.WriteBytes([]byte(f.filter)) w.WriteByte(f.qos) } if _, err := testMQTTWrite(c, w.Bytes()); err != nil { t.Fatalf("Error writing SUBSCRIBE protocol: %v", err) } b, pl := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketSubAck { t.Fatalf("Expected SUBACK packet %x, got %x", mqttPacketSubAck, pt) } rpi, err := r.readUint16("packet identifier") if err != nil || rpi != pi { t.Fatalf("Error with packet identifier expected=%v got: %v err=%v", pi, rpi, err) } for i, rem := 0, pl-2; rem > 0; rem-- { qos, err := r.readByte("filter qos") if err != nil { t.Fatal(err) } if expected != nil && qos != expected[i] { t.Fatalf("For topic filter %q expected qos of %v, got %v", filters[i].filter, expected[i], qos) } i++ } } func TestMQTTSubAck(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) subs := []*mqttFilter{ {filter: "foo", qos: 0}, {filter: "bar", qos: 1}, {filter: "baz", qos: 2}, {filter: "foo/#/bar", qos: 0}, // Invalid sub, so we should receive a result of mqttSubAckFailure } expected := []byte{ 0, 1, 2, mqttSubAckFailure, } testMQTTSub(t, 1, mc, r, subs, expected) } func TestMQTTQoS2SubDowngrade(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.downgradeQoS2Sub = true s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) subs := []*mqttFilter{ {filter: "bar", qos: 1}, {filter: "baz", qos: 2}, } expected := []byte{ 1, 1, } testMQTTSub(t, 1, mc, r, subs, expected) } func testMQTTFlush(t testing.TB, c net.Conn, bw *bufio.Writer, r *mqttReader) { t.Helper() w := newMQTTWriter(0) w.WriteByte(mqttPacketPing) w.WriteByte(0) if bw != nil { bw.Write(w.Bytes()) bw.Flush() } else { c.Write(w.Bytes()) } ab, l := testMQTTReadPacket(t, r) if pt := ab & mqttPacketMask; pt != mqttPacketPingResp { t.Fatalf("Expected ping response got %x", pt) } if l != 0 { t.Fatalf("Expected PINGRESP length to be 0, got %v", l) } } func testMQTTExpectNothing(t testing.TB, r *mqttReader) { t.Helper() var buf [128]byte r.reader.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) if n, err := r.reader.Read(buf[:]); err == nil { t.Fatalf("Expected nothing, got %v", buf[:n]) } r.reader.SetReadDeadline(time.Time{}) } func testMQTTCheckPubMsg(t testing.TB, c net.Conn, r *mqttReader, topic string, expectedFlags byte, payload []byte) uint16 { t.Helper() pi := testMQTTCheckPubMsgNoAck(t, c, r, topic, expectedFlags, payload) if pi == 0 { return 0 } qos := mqttGetQoS(expectedFlags) switch qos { case 1: testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) case 2: testMQTTSendPIPacket(mqttPacketPubRec, t, c, pi) } return pi } func testMQTTCheckPubMsgNoAck(t testing.TB, c net.Conn, r *mqttReader, topic string, expectedFlags byte, payload []byte) uint16 { t.Helper() pflags, pi := testMQTTGetPubMsg(t, c, r, topic, payload) if pflags != expectedFlags { t.Fatalf("Expected flags to be %x, got %x", expectedFlags, pflags) } return pi } func testMQTTGetPubMsg(t testing.TB, c net.Conn, r *mqttReader, topic string, payload []byte) (byte, uint16) { t.Helper() flags, pi, _ := testMQTTGetPubMsgEx(t, c, r, topic, payload) return flags, pi } func testMQTTGetPubMsgEx(t testing.TB, _ net.Conn, r *mqttReader, topic string, payload []byte) (byte, uint16, string) { t.Helper() b, pl := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPub { t.Fatalf("Expected PUBLISH packet %x, got %x", mqttPacketPub, pt) } return testMQTTGetPubMsgExEx(t, nil, r, b, pl, topic, payload) } func testMQTTGetPubMsgExEx(t testing.TB, _ net.Conn, r *mqttReader, b byte, pl int, topic string, payload []byte) (byte, uint16, string) { t.Helper() pflags := b & mqttPacketFlagMask qos := (pflags & mqttPubFlagQoS) >> 1 start := r.pos ptopic, err := r.readString("topic name") if err != nil { t.Fatal(err) } if topic != _EMPTY_ && ptopic != topic { t.Fatalf("Expected topic %q, got %q", topic, ptopic) } var pi uint16 if qos > 0 { pi, err = r.readUint16("packet identifier") if err != nil { t.Fatal(err) } } msgLen := pl - (r.pos - start) if r.pos+msgLen > len(r.buf) { t.Fatalf("computed message length goes beyond buffer: ml=%v pos=%v lenBuf=%v", msgLen, r.pos, len(r.buf)) } ppayload := r.buf[r.pos : r.pos+msgLen] if payload != nil && !bytes.Equal(payload, ppayload) { t.Fatalf("Expected payload %q, got %q", payload, ppayload) } r.pos += msgLen return pflags, pi, ptopic } func testMQTTReadPubPacket(t testing.TB, r *mqttReader) (flags byte, pi uint16, topic string, payload []byte) { t.Helper() b, pl := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPub { t.Fatalf("Expected PUBLISH packet %x, got %x", mqttPacketPub, pt) } flags = b & mqttPacketFlagMask start := r.pos topic, err := r.readString("topic name") if err != nil { t.Fatal(err) } qos := (flags & mqttPubFlagQoS) >> 1 if qos > 0 { pi, err = r.readUint16("packet identifier") if err != nil { t.Fatal(err) } } msgLen := pl - (r.pos - start) if r.pos+msgLen > len(r.buf) { t.Fatalf("computed message length goes beyond buffer: ml=%v pos=%v lenBuf=%v", msgLen, r.pos, len(r.buf)) } payload = r.buf[r.pos : r.pos+msgLen] r.pos += msgLen return flags, pi, topic, payload } func testMQTTSendPIPacket(packetType byte, t testing.TB, c net.Conn, pi uint16) { t.Helper() w := newMQTTWriter(0) w.WriteByte(packetType) w.WriteVarInt(2) w.WriteUint16(pi) if _, err := testMQTTWrite(c, w.Bytes()); err != nil { t.Fatalf("Error writing packet type %v: %v", packetType, err) } } func testMQTTWritePublishPacket(t testing.TB, w *mqttWriter, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) { t.Helper() w.WritePublishHeader(pi, qos, dup, retain, []byte(topic), len(payload)) if _, err := w.Write(payload); err != nil { t.Fatalf("Error writing PUBLISH proto: %v", err) } } func testMQTTSendPublishPacket(t testing.TB, c net.Conn, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) { t.Helper() c.SetWriteDeadline(time.Now().Add(testMQTTTimeout)) _, header := mqttMakePublishHeader(pi, qos, dup, retain, []byte(topic), len(payload)) if _, err := c.Write(header); err != nil { t.Fatalf("Error writing PUBLISH header: %v", err) } if _, err := c.Write(payload); err != nil { t.Fatalf("Error writing PUBLISH payload: %v", err) } c.SetWriteDeadline(time.Time{}) } func testMQTTPublish(t testing.TB, c net.Conn, r *mqttReader, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) { t.Helper() testMQTTSendPublishPacket(t, c, qos, dup, retain, topic, pi, payload) switch qos { case 1: b, _ := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPubAck { t.Fatalf("Expected PUBACK packet %x, got %x", mqttPacketPubAck, pt) } rpi, err := r.readUint16("packet identifier") if err != nil || rpi != pi { t.Fatalf("Error with packet identifier expected=%v got: %v err=%v", pi, rpi, err) } case 2: b, _ := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPubRec { t.Fatalf("Expected PUBREC packet %x, got %x", mqttPacketPubRec, pt) } rpi, err := r.readUint16("packet identifier") if err != nil || rpi != pi { t.Fatalf("Error with packet identifier expected=%v got: %v err=%v", pi, rpi, err) } testMQTTSendPIPacket(mqttPacketPubRel, t, c, pi) b, _ = testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPubComp { t.Fatalf("Expected PUBCOMP packet %x, got %x", mqttPacketPubComp, pt) } rpi, err = r.readUint16("packet identifier") if err != nil || rpi != pi { t.Fatalf("Error with packet identifier expected=%v got: %v err=%v", pi, rpi, err) } testMQTTFlush(t, c, nil, r) } } func TestMQTTParsePub(t *testing.T) { for _, test := range []struct { name string flags byte proto []byte pl int err string }{ {"qos not supported", (3 << 1), nil, 0, "QoS=3 is invalid in MQTT"}, {"packet in buffer error", 0, nil, 10, io.ErrUnexpectedEOF.Error()}, {"error on topic", 0, []byte{0, 3, 'f', 'o'}, 4, "topic"}, {"empty topic", 0, []byte{0, 0}, 2, errMQTTTopicIsEmpty.Error()}, {"wildcards topic", 0, []byte{0, 1, '#'}, 3, "wildcards not allowed"}, {"error on packet identifier", mqttPubQos1, []byte{0, 3, 'f', 'o', 'o'}, 5, "packet identifier"}, {"invalid packet identifier", mqttPubQos1, []byte{0, 3, 'f', 'o', 'o', 0, 0}, 7, errMQTTPacketIdentifierIsZero.Error()}, } { t.Run(test.name, func(t *testing.T) { r := &mqttReader{} r.reset(test.proto) mqtt := &mqtt{r: r} c := &client{mqtt: mqtt} pp := &mqttPublish{flags: test.flags} if err := c.mqttParsePub(r, test.pl, pp, false); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func TestMQTTParsePIMsg(t *testing.T) { for _, test := range []struct { name string proto []byte err string }{ {"packet in buffer error", nil, io.ErrUnexpectedEOF.Error()}, {"error reading packet identifier", []byte{0}, "packet identifier"}, {"invalid packet identifier", []byte{0, 0}, errMQTTPacketIdentifierIsZero.Error()}, } { t.Run(test.name, func(t *testing.T) { r := &mqttReader{} r.reset(test.proto) if _, err := mqttParsePIPacket(r); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func TestMQTTPublish(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mcp, mpr, 0, false, false, "foo", 0, []byte("msg")) testMQTTPublish(t, mcp, mpr, 1, false, false, "foo", 1, []byte("msg")) testMQTTPublish(t, mcp, mpr, 2, false, false, "foo", 2, []byte("msg")) } func TestMQTTQoS2PubReject(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.rejectQoS2Pub = true s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mcp, mpr, 1, false, false, "foo", 1, []byte("msg")) testMQTTSendPublishPacket(t, mcp, 2, false, false, "foo", 2, []byte("msg")) testMQTTExpectDisconnect(t, mcp) } func TestMQTTSub(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) for _, test := range []struct { name string mqttSubTopic string natsPubSubject string mqttPubTopic string ok bool }{ {"1 level match", "foo", "foo", "foo", true}, {"1 level no match", "foo", "bar", "bar", false}, {"2 levels match", "foo/bar", "foo.bar", "foo/bar", true}, {"2 levels no match", "foo/bar", "foo.baz", "foo/baz", false}, {"3 levels match", "/foo/bar", "/.foo.bar", "/foo/bar", true}, {"3 levels no match", "/foo/bar", "/.foo.baz", "/foo/baz", false}, {"single level wc", "foo/+", "foo.bar.baz", "foo/bar/baz", false}, {"single level wc", "foo/+", "foo.bar./", "foo/bar/", false}, {"single level wc", "foo/+", "foo.bar", "foo/bar", true}, {"single level wc", "foo/+", "foo./", "foo/", true}, {"single level wc", "foo/+", "foo", "foo", false}, {"single level wc", "foo/+", "/.foo", "/foo", false}, {"multiple level wc", "foo/#", "foo.bar.baz./", "foo/bar/baz/", true}, {"multiple level wc", "foo/#", "foo.bar.baz", "foo/bar/baz", true}, {"multiple level wc", "foo/#", "foo.bar./", "foo/bar/", true}, {"multiple level wc", "foo/#", "foo.bar", "foo/bar", true}, {"multiple level wc", "foo/#", "foo./", "foo/", true}, {"multiple level wc", "foo/#", "foo", "foo", true}, {"multiple level wc", "foo/#", "/.foo", "/foo", false}, } { t.Run(test.name, func(t *testing.T) { mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: test.mqttSubTopic, qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) natsPub(t, nc, test.natsPubSubject, []byte("msg")) if test.ok { testMQTTCheckPubMsg(t, mc, r, test.mqttPubTopic, 0, []byte("msg")) } else { testMQTTExpectNothing(t, r) } testMQTTPublish(t, mcp, mpr, 0, false, false, test.mqttPubTopic, 0, []byte("msg")) if test.ok { testMQTTCheckPubMsg(t, mc, r, test.mqttPubTopic, 0, []byte("msg")) } else { testMQTTExpectNothing(t, r) } }) } } func TestMQTTSubQoS2(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) topic := "foo/bar/baz" mqttTopic0 := "foo/#" mqttTopic1 := "foo/bar/#" mqttTopic2 := topic testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic0, qos: 0}}, []byte{0}) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic1, qos: 1}}, []byte{1}) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic2, qos: 2}}, []byte{2}) testMQTTFlush(t, mc, nil, r) for pubQoS, expectedCounts := range map[byte]map[byte]int{ 0: {0: 3}, 1: {0: 1, 1: 2}, 2: {0: 1, 1: 1, 2: 1}, } { t.Run(fmt.Sprintf("pubQoS %v", pubQoS), func(t *testing.T) { pubPI := uint16(456) testMQTTPublish(t, mcp, mpr, pubQoS, false, false, topic, pubPI, []byte("msg")) qosCounts := map[byte]int{} delivered := map[uint16]byte{} // We have 3 subscriptions, each should receive the message, with the // QoS that maybe "trimmed" to that of the subscription. for i := 0; i < 3; i++ { flags, pi := testMQTTGetPubMsg(t, mc, r, topic, []byte("msg")) delivered[pi] = flags qosCounts[mqttGetQoS(flags)]++ } for pi, flags := range delivered { switch mqttGetQoS(flags) { case 1: testMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi) case 2: testMQTTSendPIPacket(mqttPacketPubRec, t, mc, pi) testMQTTReadPIPacket(mqttPacketPubRel, t, r, pi) testMQTTSendPIPacket(mqttPacketPubComp, t, mc, pi) } } if !reflect.DeepEqual(qosCounts, expectedCounts) { t.Fatalf("Expected QoS %#v, got %#v", expectedCounts, qosCounts) } }) } } func TestMQTTSubQoS1(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) mqttTopic := "foo/bar" // Subscribe with QoS 1 testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic, qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) // Publish from NATS, which means QoS 0 natsPub(t, nc, "foo.bar", []byte("NATS")) // Will receive as QoS 0 testMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte("NATS")) testMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte("NATS")) // Publish from MQTT with QoS 0 testMQTTPublish(t, mcp, mpr, 0, false, false, mqttTopic, 0, []byte("msg")) // Will receive as QoS 0 testMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte("msg")) // Publish from MQTT with QoS 1 testMQTTPublish(t, mcp, mpr, 1, false, false, mqttTopic, 1, []byte("msg")) pflags1, pi1 := testMQTTGetPubMsg(t, mc, r, mqttTopic, []byte("msg")) if pflags1 != 0x2 { t.Fatalf("Expected flags to be 0x2, got %v", pflags1) } pflags2, pi2 := testMQTTGetPubMsg(t, mc, r, mqttTopic, []byte("msg")) if pflags2 != 0x2 { t.Fatalf("Expected flags to be 0x2, got %v", pflags2) } if pi1 == pi2 { t.Fatalf("packet identifier for message 1: %v should be different from message 2", pi1) } testMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi1) testMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi2) } func getSubQoS(sub *subscription) int { if sub.mqtt != nil { return int(sub.mqtt.qos) } return -1 } func TestMQTTSubDups(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", user: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Test with single SUBSCRIBE protocol but multiple filters filters := []*mqttFilter{ {filter: "foo", qos: 1}, {filter: "foo", qos: 0}, } testMQTTSub(t, 1, mc, r, filters, []byte{1, 0}) testMQTTFlush(t, mc, nil, r) // And also with separate SUBSCRIBE protocols testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "bar", qos: 0}}, []byte{0}) // Ask for QoS 1 but server will downgrade to 1 testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) // Publish and test msg received only once testMQTTPublish(t, mcp, r, 0, false, false, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) testMQTTExpectNothing(t, r) testMQTTPublish(t, mcp, r, 0, false, false, "bar", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "bar", 0, []byte("msg")) testMQTTExpectNothing(t, r) // Check that the QoS for subscriptions have been updated to the latest received filter var err error subc := testMQTTGetClient(t, s, "sub") subc.mu.Lock() if subc.opts.Username != "sub" { err = fmt.Errorf("wrong user name") } if err == nil { if sub := subc.subs["foo"]; sub == nil || getSubQoS(sub) != 0 { err = fmt.Errorf("subscription foo QoS should be 0, got %v", getSubQoS(sub)) } } if err == nil { if sub := subc.subs["bar"]; sub == nil || getSubQoS(sub) != 1 { err = fmt.Errorf("subscription bar QoS should be 1, got %v", getSubQoS(sub)) } } subc.mu.Unlock() if err != nil { t.Fatal(err) } // Now subscribe on "foo/#" which means that a PUBLISH on "foo" will be received // by this subscription and also the one on "foo". testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) // Publish and test msg received twice testMQTTPublish(t, mcp, r, 0, false, false, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) checkWCSub := func(expectedQoS int) { t.Helper() subc.mu.Lock() defer subc.mu.Unlock() // When invoked with expectedQoS==1, we have the following subs: // foo (QoS-0), bar (QoS-1), foo.> (QoS-1) // which means (since QoS-1 have a JS consumer + sub for delivery // and foo.> causes a "foo fwc") that we should have the following // number of NATS subs: foo (1), bar (2), foo.> (2) and "foo fwc" (2), // so total=7. // When invoked with expectedQoS==0, it means that we have replaced // foo/# QoS-1 to QoS-0, so we should have 2 less NATS subs, // so total=5 expected := 7 if expectedQoS == 0 { expected = 5 } if lenmap := len(subc.subs); lenmap != expected { t.Fatalf("Subs map should have %v entries, got %v", expected, lenmap) } if sub, ok := subc.subs["foo.>"]; !ok { t.Fatal("Expected sub foo.> to be present but was not") } else if getSubQoS(sub) != expectedQoS { t.Fatalf("Expected sub foo.> QoS to be %v, got %v", expectedQoS, getSubQoS(sub)) } if sub, ok := subc.subs["foo fwc"]; !ok { t.Fatal("Expected sub foo fwc to be present but was not") } else if getSubQoS(sub) != expectedQoS { t.Fatalf("Expected sub foo fwc QoS to be %v, got %v", expectedQoS, getSubQoS(sub)) } // Make sure existing sub on "foo" qos was not changed. if sub, ok := subc.subs["foo"]; !ok { t.Fatal("Expected sub foo to be present but was not") } else if getSubQoS(sub) != 0 { t.Fatalf("Expected sub foo QoS to be 0, got %v", getSubQoS(sub)) } } checkWCSub(1) // Sub again on same subject with lower QoS testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) // Publish and test msg received twice testMQTTPublish(t, mcp, r, 0, false, false, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) checkWCSub(0) } func TestMQTTSubWithSpaces(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{user: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo bar", qos: 0}}, []byte{mqttSubAckFailure}) } func TestMQTTSubCaseSensitive(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{user: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "Foo/Bar", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) testMQTTPublish(t, mcp, r, 0, false, false, "Foo/Bar", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "Foo/Bar", 0, []byte("msg")) testMQTTPublish(t, mcp, r, 0, false, false, "foo/bar", 0, []byte("msg")) testMQTTExpectNothing(t, r) nc := natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "Foo.Bar", []byte("nats")) testMQTTCheckPubMsg(t, mc, r, "Foo/Bar", 0, []byte("nats")) natsPub(t, nc, "foo.bar", []byte("nats")) testMQTTExpectNothing(t, r) } func TestMQTTPubSubMatrix(t *testing.T) { for _, test := range []struct { name string natsPub bool mqttPub bool mqttPubQoS byte natsSub bool mqttSubQoS0 bool mqttSubQoS1 bool }{ {"NATS to MQTT sub QoS-0", true, false, 0, false, true, false}, {"NATS to MQTT sub QoS-1", true, false, 0, false, false, true}, {"NATS to MQTT sub QoS-0 and QoS-1", true, false, 0, false, true, true}, {"MQTT QoS-0 to NATS sub", false, true, 0, true, false, false}, {"MQTT QoS-0 to MQTT sub QoS-0", false, true, 0, false, true, false}, {"MQTT QoS-0 to MQTT sub QoS-1", false, true, 0, false, false, true}, {"MQTT QoS-0 to NATS sub and MQTT sub QoS-0", false, true, 0, true, true, false}, {"MQTT QoS-0 to NATS sub and MQTT sub QoS-1", false, true, 0, true, false, true}, {"MQTT QoS-0 to all subs", false, true, 0, true, true, true}, {"MQTT QoS-1 to NATS sub", false, true, 1, true, false, false}, {"MQTT QoS-1 to MQTT sub QoS-0", false, true, 1, false, true, false}, {"MQTT QoS-1 to MQTT sub QoS-1", false, true, 1, false, false, true}, {"MQTT QoS-1 to NATS sub and MQTT sub QoS-0", false, true, 1, true, true, false}, {"MQTT QoS-1 to NATS sub and MQTT sub QoS-1", false, true, 1, true, false, true}, {"MQTT QoS-1 to all subs", false, true, 1, true, true, true}, } { t.Run(test.name, func(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) mc1, r1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) mc2, r2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) // First setup subscriptions based on test options. var ns *nats.Subscription if test.natsSub { ns = natsSubSync(t, nc, "foo") } if test.mqttSubQoS0 { testMQTTSub(t, 1, mc1, r1, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTFlush(t, mc1, nil, r1) } if test.mqttSubQoS1 { testMQTTSub(t, 1, mc2, r2, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, r2) } // Just as a barrier natsFlush(t, nc) // Now publish if test.natsPub { natsPubReq(t, nc, "foo", "", []byte("msg")) } else { testMQTTPublish(t, mc, r, test.mqttPubQoS, false, false, "foo", 1, []byte("msg")) } // Check message received if test.natsSub { natsNexMsg(t, ns, time.Second) // Make sure no other is received if msg, err := ns.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Should not have gotten a second message, got %v", msg) } } if test.mqttSubQoS0 { testMQTTCheckPubMsg(t, mc1, r1, "foo", 0, []byte("msg")) testMQTTExpectNothing(t, r1) } if test.mqttSubQoS1 { var expectedFlag byte if test.mqttPubQoS > 0 { expectedFlag = test.mqttPubQoS << 1 } testMQTTCheckPubMsg(t, mc2, r2, "foo", expectedFlag, []byte("msg")) testMQTTExpectNothing(t, r2) } }) } } func TestMQTTPreventSubWithMQTTSubPrefix(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: strings.ReplaceAll(mqttSubPrefix, ".", "/") + "foo/bar", qos: 1}}, []byte{mqttSubAckFailure}) } func TestMQTTSubWithNATSStream(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/bar", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) mcp, rp := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTFlush(t, mcp, nil, rp) nc := natsConnect(t, s.ClientURL()) defer nc.Close() sc := &StreamConfig{ Name: "test", Storage: FileStorage, Retention: InterestPolicy, Subjects: []string{"foo.>"}, } mset, err := s.GlobalAccount().addStream(sc) if err != nil { t.Fatalf("Unable to create stream: %v", err) } sub := natsSubSync(t, nc, "bar") cc := &ConsumerConfig{ Durable: "dur", AckPolicy: AckExplicit, DeliverSubject: "bar", } if _, err := mset.addConsumer(cc); err != nil { t.Fatalf("Unable to add consumer: %v", err) } // Now send message from NATS resp, err := nc.Request("foo.bar", []byte("nats"), time.Second) if err != nil { t.Fatalf("Error publishing: %v", err) } ar := &ApiResponse{} if err := json.Unmarshal(resp.Data, ar); err != nil || ar.Error != nil { t.Fatalf("Unexpected response: err=%v resp=%+v", err, ar.Error) } // Check that message is received by both checkRecv := func(content string, flags byte) { t.Helper() if msg := natsNexMsg(t, sub, time.Second); string(msg.Data) != content { t.Fatalf("Expected %q, got %q", content, msg.Data) } testMQTTCheckPubMsg(t, mc, r, "foo/bar", flags, []byte(content)) } checkRecv("nats", 0) // Send from MQTT as a QoS0 testMQTTPublish(t, mcp, rp, 0, false, false, "foo/bar", 0, []byte("qos0")) checkRecv("qos0", 0) // Send from MQTT as a QoS1 testMQTTPublish(t, mcp, rp, 1, false, false, "foo/bar", 1, []byte("qos1")) checkRecv("qos1", mqttPubQos1) } func TestMQTTTrackPendingOverrun(t *testing.T) { sess := mqttSession{} sess.last_pi = 0xFFFF pi := sess.trackPublishRetained() if pi != 1 { t.Fatalf("Expected 1, got %v", pi) } p := &mqttPending{} for i := 1; i <= 0xFFFF; i++ { sess.pendingPublish[uint16(i)] = p } pi, _ = sess.trackPublish("test", "test") if pi != 0 { t.Fatalf("Expected 0, got %v", pi) } delete(sess.pendingPublish, 1234) pi = sess.trackPublishRetained() if pi != 1234 { t.Fatalf("Expected 1234, got %v", pi) } } func TestMQTTSubRestart(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Start an MQTT subscription QoS=1 on "foo" testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) // Now start a NATS subscription on ">" (anything that would match the JS consumer delivery subject) natsSubSync(t, nc, ">") natsFlush(t, nc) // Restart the MQTT client testMQTTDisconnect(t, mc, nil) mc, r = testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Restart an MQTT subscription QoS=1 on "foo" testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, r) pc, pr := testMQTTConnect(t, &mqttConnInfo{clientID: "pub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer pc.Close() testMQTTCheckConnAck(t, pr, mqttConnAckRCConnectionAccepted, false) // Publish a message QoS1 testMQTTPublish(t, pc, pr, 1, false, false, "foo", 1, []byte("msg1")) // Make sure we receive it testMQTTCheckPubMsg(t, mc, r, "foo", mqttPubQos1, []byte("msg1")) // Now "restart" the subscription but as a Qos0 testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) // Publish a message QoS testMQTTPublish(t, pc, pr, 1, false, false, "foo", 1, []byte("msg2")) // Make sure we receive but as a QoS0 testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg2")) } func testMQTTGetClusterTemplaceNoLeaf() string { return strings.Replace(jsClusterTemplWithLeafAndMQTT, "{{leaf}}", "", 1) } func TestMQTTSubPropagation(t *testing.T) { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", 2) defer cl.shutdown() o := cl.opts[0] s2 := cl.servers[1] nc := natsConnect(t, s2.ClientURL()) defer nc.Close() mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) // Because in MQTT foo/# means foo.> but also foo, check that this is propagated checkSubInterest(t, s2, globalAccountName, "foo", time.Second) // Publish on foo.bar, foo./ and foo and we should receive them natsPub(t, nc, "foo.bar", []byte("hello")) testMQTTCheckPubMsg(t, mc, r, "foo/bar", 0, []byte("hello")) natsPub(t, nc, "foo./", []byte("from")) testMQTTCheckPubMsg(t, mc, r, "foo/", 0, []byte("from")) natsPub(t, nc, "foo", []byte("NATS")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("NATS")) } func TestMQTTCluster(t *testing.T) { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", 2) defer cl.shutdown() for _, topTest := range []struct { name string restart bool }{ {"first_start", true}, {"restart", false}, } { t.Run(topTest.name, func(t *testing.T) { for _, test := range []struct { name string subQos byte }{ {"qos_0", 0}, {"qos_1", 1}, } { t.Run(test.name, func(t *testing.T) { clientID := nuid.Next() o := cl.opts[0] mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: clientID, cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#", qos: test.subQos}}, []byte{test.subQos}) testMQTTFlush(t, mc, nil, r) check := func(mc net.Conn, r *mqttReader, o *Options, s *Server) { t.Helper() nc := natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo.bar", []byte("fromNats")) testMQTTCheckPubMsg(t, mc, r, "foo/bar", 0, []byte("fromNats")) mpc, pr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mpc.Close() testMQTTCheckConnAck(t, pr, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mpc, pr, 0, false, false, "foo/baz", 0, []byte("mqtt_qos0")) testMQTTCheckPubMsg(t, mc, r, "foo/baz", 0, []byte("mqtt_qos0")) testMQTTPublish(t, mpc, pr, 1, false, false, "foo/bat", 1, []byte("mqtt_qos1")) expectedQoS := byte(0) if test.subQos == 1 { expectedQoS = mqttPubQos1 } testMQTTCheckPubMsg(t, mc, r, "foo/bat", expectedQoS, []byte("mqtt_qos1")) testMQTTDisconnect(t, mpc, nil) } check(mc, r, cl.opts[0], cl.servers[0]) check(mc, r, cl.opts[1], cl.servers[1]) // Start the same subscription from the other server. It should disconnect // the one connected in the first server. o = cl.opts[1] mc2, r2 := testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: false}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true) // Expect first connection to be closed. testMQTTExpectDisconnect(t, mc) // Now re-run the checks check(mc2, r2, cl.opts[0], cl.servers[0]) check(mc2, r2, cl.opts[1], cl.servers[1]) // Disconnect our sub and restart with clean session then disconnect again to clear the state. testMQTTDisconnect(t, mc2, nil) mc2, r2 = testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) testMQTTFlush(t, mc2, nil, r2) testMQTTDisconnect(t, mc2, nil) // Remove the session from the flappers so we can restart the test // without failure and have to wait for 1sec before being able to reconnect. s := cl.servers[0] sm := &s.mqtt.sessmgr sm.mu.Lock() asm := sm.sessions[globalAccountName] sm.mu.Unlock() if asm != nil { asm.mu.Lock() delete(asm.flappers, clientID) asm.mu.Unlock() } }) } if !t.Failed() && topTest.restart { cl.stopAll() cl.restartAll() streams := []string{mqttStreamName, mqttRetainedMsgsStreamName, mqttSessStreamName} for _, sn := range streams { cl.waitOnStreamLeader(globalAccountName, sn) } mset, err := cl.randomServer().GlobalAccount().lookupStream(mqttRetainedMsgsStreamName) if err != nil { t.Fatalf("Expected to find a stream for %q", mqttRetainedMsgsStreamName) } rmConsumerNames := []string{} mset.mu.RLock() for _, o := range mset.consumers { rmConsumerNames = append(rmConsumerNames, o.name) } mset.mu.RUnlock() for _, consumerName := range rmConsumerNames { cl.waitOnConsumerLeader(globalAccountName, mqttRetainedMsgsStreamName, consumerName) } } }) } } func testMQTTConnectDisconnect(t *testing.T, o *Options, clientID string, clean bool, found bool) { t.Helper() mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: clean}, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, found) testMQTTDisconnectEx(t, mc, nil, false) mc.Close() } func TestMQTTClusterConnectDisconnectClean(t *testing.T) { nServers := 3 cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", nServers) defer cl.shutdown() clientID := nuid.Next() // test runs a connect/disconnect against a random server in the cluster, as // specified. N := 100 for n := 0; n < N; n++ { testMQTTConnectDisconnect(t, cl.opts[rand.Intn(nServers)], clientID, true, false) } } func TestMQTTClusterConnectDisconnectPersist(t *testing.T) { nServers := 3 cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", nServers) defer cl.shutdown() clientID := nuid.Next() // test runs a connect/disconnect against a random server in the cluster, as // specified. N := 20 for n := 0; n < N; n++ { // First clean sessions on all servers for i := 0; i < nServers; i++ { testMQTTConnectDisconnect(t, cl.opts[i], clientID, true, false) } testMQTTConnectDisconnect(t, cl.opts[0], clientID, false, false) testMQTTConnectDisconnect(t, cl.opts[1], clientID, false, true) testMQTTConnectDisconnect(t, cl.opts[2], clientID, false, true) testMQTTConnectDisconnect(t, cl.opts[0], clientID, false, true) } } func TestMQTTClusterRetainedMsg(t *testing.T) { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", 2) defer cl.shutdown() srv1Opts := cl.opts[0] srv2Opts := cl.opts[1] // Connect subscription on server 1. mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) // Create a publisher from server 2. mp, rp := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port) defer mp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // Send retained message. testMQTTPublish(t, mp, rp, 1, false, true, "foo/bar", 1, []byte("retained")) // Check it is received. testMQTTCheckPubMsg(t, mc, rc, "foo/bar", mqttPubQos1, []byte("retained")) // Start a new subscription on server 1 and make sure we receive the retained message mc2, rc2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTCheckPubMsg(t, mc2, rc2, "foo/bar", mqttPubQos1|mqttPubFlagRetain, []byte("retained")) testMQTTDisconnect(t, mc2, nil) // Send an empty retained message which should remove it from storage, but still be delivered. testMQTTPublish(t, mp, rp, 1, false, true, "foo/bar", 1, []byte("")) testMQTTCheckPubMsg(t, mc, rc, "foo/bar", mqttPubQos1, []byte("")) // Now shutdown the consumer connection testMQTTDisconnect(t, mc, nil) mc.Close() // Reconnect to server where the retained message was published (server 2) mc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) // The retained message should not be delivered. testMQTTExpectNothing(t, rc) // Now disconnect and reconnect back to first server testMQTTDisconnect(t, mc, nil) mc.Close() // Now reconnect to the server 1, which is not where the messages were published, and check // that we don't receive the message. mc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTExpectNothing(t, rc) testMQTTDisconnect(t, mc, nil) mc.Close() // Will now test network deletes // Create a subscription on server 1 and server 2 mc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: "sub_one", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) mc2, rc2 = testMQTTConnect(t, &mqttConnInfo{clientID: "sub_two", cleanSess: false}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, rc2) // Publish 1 retained message (producer is connected to server 2) testMQTTPublish(t, mp, rp, 1, false, true, "bar", 1, []byte("msg1")) // Make sure messages are received by both testMQTTCheckPubMsg(t, mc, rc, "bar", mqttPubQos1, []byte("msg1")) testMQTTCheckPubMsg(t, mc2, rc2, "bar", mqttPubQos1, []byte("msg1")) // Now send an empty retained message that should delete it. For the one on server 1, // this will be a network delete. testMQTTPublish(t, mp, rp, 1, false, true, "bar", 1, []byte("")) testMQTTCheckPubMsg(t, mc, rc, "bar", mqttPubQos1, []byte("")) testMQTTCheckPubMsg(t, mc2, rc2, "bar", mqttPubQos1, []byte("")) // Now send a new retained message testMQTTPublish(t, mp, rp, 1, false, true, "bar", 1, []byte("msg2")) // Again, verify that they all receive it. testMQTTCheckPubMsg(t, mc, rc, "bar", mqttPubQos1, []byte("msg2")) testMQTTCheckPubMsg(t, mc2, rc2, "bar", mqttPubQos1, []byte("msg2")) // But now, restart the consumer that was in the server that processed the // original network delete. testMQTTDisconnect(t, mc, nil) mc.Close() mc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: "sub_one", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) // Expect the message to be delivered as retained testMQTTCheckPubMsg(t, mc, rc, "bar", mqttPubQos1|mqttPubFlagRetain, []byte("msg2")) } func TestMQTTRetainedMsgNetworkUpdates(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, rc := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) c := testMQTTGetClient(t, s, "sub") asm := c.mqtt.asm // For this test, we are going to simulate updates arriving in a // mixed order and verify that we have the expected outcome. check := func(t *testing.T, subject string, present bool, current, floor uint64) { t.Helper() asm.mu.RLock() defer asm.mu.RUnlock() erm, ok := asm.retmsgs[subject] if present && !ok { t.Fatalf("Subject %q not present", subject) } else if !present && ok { t.Fatalf("Subject %q should not be present", subject) } else if !present { return } if floor != erm.floor { t.Fatalf("Expected floor to be %v, got %v", floor, erm.floor) } if erm.sseq != current { t.Fatalf("Expected current sequence to be %v, got %v", current, erm.sseq) } } type action struct { add bool seq uint64 } for _, test := range []struct { subject string order []action seq uint64 floor uint64 }{ {"foo.1", []action{{true, 1}, {true, 2}, {true, 3}}, 3, 0}, {"foo.2", []action{{true, 3}, {true, 1}, {true, 2}}, 3, 0}, {"foo.3", []action{{true, 1}, {false, 1}, {true, 2}}, 2, 0}, {"foo.4", []action{{false, 2}, {true, 1}, {true, 3}, {true, 2}}, 3, 0}, {"foo.5", []action{{false, 2}, {true, 1}, {true, 2}}, 0, 2}, {"foo.6", []action{{true, 1}, {true, 2}, {false, 2}}, 0, 2}, } { t.Run(test.subject, func(t *testing.T) { for _, a := range test.order { if a.add { rf := &mqttRetainedMsgRef{sseq: a.seq} asm.handleRetainedMsg(test.subject, rf, nil, false) } else { asm.handleRetainedMsgDel(test.subject, a.seq) } } check(t, test.subject, true, test.seq, test.floor) }) } for _, subject := range []string{"foo.5", "foo.6"} { t.Run("clear_"+subject, func(t *testing.T) { // Now add a new message, which should clear the floor. rf := &mqttRetainedMsgRef{sseq: 3} asm.handleRetainedMsg(subject, rf, nil, false) check(t, subject, true, 3, 0) // Now do a non network delete and make sure it is gone. asm.handleRetainedMsgDel(subject, 0) check(t, subject, false, 0, 0) }) } } func TestMQTTRetainedMsgDel(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc, _ := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() c := testMQTTGetClient(t, s, "sub") asm := c.mqtt.asm var i uint64 for i = 0; i < 3; i++ { rf := &mqttRetainedMsgRef{sseq: i} asm.handleRetainedMsg("subject", rf, nil, false) } asm.handleRetainedMsgDel("subject", 2) if asm.sl.count > 0 { t.Fatalf("all retained messages subs should be removed, but %d still present", asm.sl.count) } } func TestMQTTRetainedMsgMigration(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc, js := jsClientConnect(t, s) defer nc.Close() // Create the retained messages stream to listen on the old subject first. // The server will correct this when the migration takes place. _, err := js.AddStream(&nats.StreamConfig{ Name: mqttRetainedMsgsStreamName, Subjects: []string{`$MQTT.rmsgs`}, Storage: nats.FileStorage, Retention: nats.LimitsPolicy, Replicas: 1, }) require_NoError(t, err) // Publish some retained messages on the old "$MQTT.rmsgs" subject. const N = 100 for i := 0; i < N; i++ { msg := fmt.Sprintf( `{"origin":"b5IQZNtG","subject":"test%d","topic":"test%d","msg":"YmFy","flags":1}`, i, i, ) _, err := js.Publish(`$MQTT.rmsgs`, []byte(msg)) require_NoError(t, err) } // Check that the old subject looks right. si, err := js.StreamInfo(mqttRetainedMsgsStreamName, &nats.StreamInfoRequest{ SubjectsFilter: `$MQTT.>`, }) require_NoError(t, err) if si.State.NumSubjects != 1 { t.Fatalf("expected 1 subject, got %d", si.State.NumSubjects) } if n := si.State.Subjects[`$MQTT.rmsgs`]; n != N { t.Fatalf("expected to find %d messages on the original subject but found %d", N, n) } // Create an MQTT client, this will cause a migration to take place. mc, rc := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "+", qos: 0}}, []byte{0}) topics := map[string]struct{}{} for i := 0; i < N; i++ { _, _, topic := testMQTTGetPubMsgEx(t, mc, rc, _EMPTY_, []byte("bar")) topics[topic] = struct{}{} } if len(topics) != N { t.Fatalf("Unexpected topics: %v", topics) } // Now look at the stream, there should be N messages on the new // divided subjects and none on the old undivided subject. si, err = js.StreamInfo(mqttRetainedMsgsStreamName, &nats.StreamInfoRequest{ SubjectsFilter: `$MQTT.>`, }) require_NoError(t, err) if si.State.NumSubjects != N { t.Fatalf("expected %d subjects, got %d", N, si.State.NumSubjects) } if n := si.State.Subjects[`$MQTT.rmsgs`]; n > 0 { t.Fatalf("expected to find no messages on the original subject but found %d", n) } // Check that the message counts look right. There should be one // retained message per key. for i := 0; i < N; i++ { expected := fmt.Sprintf(`$MQTT.rmsgs.test%d`, i) n, ok := si.State.Subjects[expected] if !ok { t.Fatalf("expected to find %q but didn't", expected) } if n != 1 { t.Fatalf("expected %q to have 1 message but had %d", expected, n) } } } func TestMQTTClusterReplicasCount(t *testing.T) { for _, test := range []struct { size int replicas int }{ {1, 1}, {2, 2}, {3, 3}, {5, 3}, } { t.Run(fmt.Sprintf("size %v", test.size), func(t *testing.T) { var s *Server var o *Options if test.size == 1 { o = testMQTTDefaultOptions() s = testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) } else { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", test.size) defer cl.shutdown() o = cl.opts[0] s = cl.randomServer() } mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) nc := natsConnect(t, s.ClientURL()) defer nc.Close() // Check the replicas of all MQTT streams js, err := nc.JetStream() if err != nil { t.Fatalf("Error getting js: %v", err) } for _, sname := range []string{ mqttStreamName, mqttRetainedMsgsStreamName, mqttSessStreamName, } { t.Run(sname, func(t *testing.T) { si, err := js.StreamInfo(sname) if err != nil { t.Fatalf("Error getting stream info: %v", err) } if si.Config.Replicas != test.replicas { t.Fatalf("Expected %v replicas, got %v", test.replicas, si.Config.Replicas) } }) } }) } } func TestMQTTClusterCanCreateSessionWithOnServerDown(t *testing.T) { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", 3) defer cl.shutdown() o := cl.opts[0] mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) mc.Close() // Shutdown one of the server. sd := cl.servers[1].StoreDir() defer os.RemoveAll(strings.TrimSuffix(sd, JetStreamStoreDir)) cl.servers[1].Shutdown() // Make sure there is a meta leader cl.waitOnPeerCount(2) cl.waitOnLeader() // Now try to create a new session. Since we use a single stream now for all sessions, // this should succeed. o = cl.opts[2] // We may still get failures because of some JS APIs may timeout while things // settle, so try again for a certain amount of times. mc, rc = testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) } func TestMQTTClusterPlacement(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() c := sc.randomCluster() lnc := c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafAndMQTT, "SPOKE", 3, 22111) defer lnc.shutdown() sc.waitOnPeerCount(9) sc.waitOnLeader() for i := 0; i < 10; i++ { mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, lnc.opts[i%3].MQTT.Host, lnc.opts[i%3].MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) } // Now check that MQTT assets have been created in the LEAF node's side, not the Hub. nc := natsConnect(t, lnc.servers[0].ClientURL()) defer nc.Close() js, err := nc.JetStream() if err != nil { t.Fatalf("Unable to get JetStream: %v", err) } count := 0 for si := range js.StreamsInfo() { if si.Cluster == nil || si.Cluster.Name != "SPOKE" { t.Fatalf("Expected asset %q to be placed on spoke cluster, was placed on %+v", si.Config.Name, si.Cluster) } for _, repl := range si.Cluster.Replicas { if !strings.HasPrefix(repl.Name, "SPOKE-") { t.Fatalf("Replica on the wrong cluster: %+v", repl) } } if si.State.Consumers > 0 { for ci := range js.ConsumersInfo(si.Config.Name) { if ci.Cluster == nil || ci.Cluster.Name != "SPOKE" { t.Fatalf("Expected asset %q to be placed on spoke cluster, was placed on %+v", ci.Name, si.Cluster) } for _, repl := range ci.Cluster.Replicas { if !strings.HasPrefix(repl.Name, "SPOKE-") { t.Fatalf("Replica on the wrong cluster: %+v", repl) } } } } count++ } if count == 0 { t.Fatal("No stream found!") } } func TestMQTTLeafnodeWithoutJSToClusterWithJSNoSharedSysAcc(t *testing.T) { test := func(t *testing.T, resolution int) { getClusterOpts := func(name string, i int) *Options { o := testMQTTDefaultOptions() o.ServerName = name o.Cluster.Name = "hub" // first two test cases rely on domain not being set in hub if resolution > 1 { o.JetStreamDomain = "DOMAIN" } o.Cluster.Host = "127.0.0.1" o.Cluster.Port = 2790 + i o.Routes = RoutesFromStr("nats://127.0.0.1:2791,nats://127.0.0.1:2792,nats://127.0.0.1:2793") o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 return o } o1 := getClusterOpts("S1", 1) s1 := testMQTTRunServer(t, o1) defer testMQTTShutdownServer(s1) o2 := getClusterOpts("S2", 2) s2 := testMQTTRunServer(t, o2) defer testMQTTShutdownServer(s2) o3 := getClusterOpts("S3", 3) s3 := testMQTTRunServer(t, o3) defer testMQTTShutdownServer(s3) cluster := []*Server{s1, s2, s3} checkClusterFormed(t, cluster...) checkFor(t, 10*time.Second, 50*time.Millisecond, func() error { for _, s := range cluster { if s.JetStreamIsLeader() { // Need to wait for usage updates now to propagate to meta leader. if len(s.JetStreamClusterPeers()) == len(cluster) { time.Sleep(100 * time.Millisecond) return nil } } } return fmt.Errorf("no leader yet") }) // Now define a leafnode that has mqtt enabled, but no JS. This should still work. lno := testMQTTDefaultOptions() // Make sure jetstream is not explicitly defined here. lno.JetStream = false switch resolution { case 0: // turn off jetstream in $G by adding another account and set mqtt domain option and set account default lno.Accounts = append(lno.Accounts, NewAccount("unused-account")) fallthrough case 1: lno.JsAccDefaultDomain = map[string]string{"$G": ""} case 2: lno.JsAccDefaultDomain = map[string]string{"$G": o1.JetStreamDomain} case 3: // turn off jetstream in $G by adding another account and set mqtt domain option lno.Accounts = append(lno.Accounts, NewAccount("unused-account")) fallthrough case 4: // actual solution lno.MQTT.JsDomain = o1.JetStreamDomain case 5: // set per account overwrite and disable js in $G lno.Accounts = append(lno.Accounts, NewAccount("unused-account")) lno.JsAccDefaultDomain = map[string]string{ "$G": o1.JetStreamDomain, } } // Whenever an account was added to disable JS in $G, enable it in the server if len(lno.Accounts) > 0 { lno.JetStream = true lno.JetStreamDomain = "OTHER" lno.StoreDir = t.TempDir() } // Use RoutesFromStr() to make an array of urls urls := RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d,nats://127.0.0.1:%d,nats://127.0.0.1:%d", o1.LeafNode.Port, o2.LeafNode.Port, o3.LeafNode.Port)) lno.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}} ln := RunServer(lno) defer testMQTTShutdownServer(ln) checkLeafNodeConnected(t, ln) // Now connect to leafnode and subscribe mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: true}, lno.MQTT.Host, lno.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) connectAndPublish := func(o *Options) { mp, rp := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mp, rp, 1, false, false, "foo", 1, []byte("msg")) } // Connect a publisher from leafnode and publish, verify message is received. connectAndPublish(lno) testMQTTCheckPubMsg(t, mc, rc, "foo", mqttPubQos1, []byte("msg")) // Connect from one server in the cluster check it works from there too. connectAndPublish(o3) testMQTTCheckPubMsg(t, mc, rc, "foo", mqttPubQos1, []byte("msg")) // Connect from a server in the hub and subscribe mc2, rc2 := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub2", cleanSess: true}, o2.MQTT.Host, o2.MQTT.Port, 5) defer mc2.Close() testMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, rc2) // Connect a publisher from leafnode and publish, verify message is received. connectAndPublish(lno) testMQTTCheckPubMsg(t, mc2, rc2, "foo", mqttPubQos1, []byte("msg")) // Connect from one server in the cluster check it works from there too. connectAndPublish(o1) testMQTTCheckPubMsg(t, mc2, rc2, "foo", mqttPubQos1, []byte("msg")) } t.Run("backwards-compatibility-default-js-enabled-in-leaf", func(t *testing.T) { test(t, 0) // test with JsAccDefaultDomain set to default (pointing at hub) but jetstream enabled in leaf node too }) t.Run("backwards-compatibility-default-js-disabled-in-leaf", func(t *testing.T) { // test with JsAccDefaultDomain set. Checks if it works with backwards compatibility code for empty domain test(t, 1) }) t.Run("backwards-compatibility-domain-js-disabled-in-leaf", func(t *testing.T) { // test with JsAccDefaultDomain set. Checks if it works with backwards compatibility code for domain set test(t, 2) // test with domain set in mqtt client }) t.Run("mqtt-explicit-js-enabled-in-leaf", func(t *testing.T) { test(t, 3) // test with domain set in mqtt client (pointing at hub) but jetstream enabled in leaf node too }) t.Run("mqtt-explicit-js-disabled-in-leaf", func(t *testing.T) { test(t, 4) // test with domain set in mqtt client }) t.Run("backwards-compatibility-domain-js-enabled-in-leaf", func(t *testing.T) { test(t, 5) // test with JsAccDefaultDomain set to domain (pointing at hub) but jetstream enabled in leaf node too }) } func TestMQTTImportExport(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } accounts { org { jetstream: enabled users: [{user: org, password: pwd}] imports = [{stream: {account: "device", subject: "foo"}, prefix: "org"}] } device { users: [{user: device, password: pwd}] exports = [{stream: "foo"}] } } mqtt { listen: "127.0.0.1:-1" } no_auth_user: device `, t.TempDir()))) defer os.Remove(conf) s, o := RunServerWithConfig(conf) defer s.Shutdown() mc1, rc1 := testMQTTConnect(t, &mqttConnInfo{clientID: "sub1", user: "org", pass: "pwd", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, rc1, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc1, rc1, []*mqttFilter{{filter: "org/foo", qos: 0}}, []byte{0}) testMQTTFlush(t, mc1, nil, rc1) mc2, rc2 := testMQTTConnect(t, &mqttConnInfo{clientID: "sub2", user: "org", pass: "pwd", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: "org/foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, rc2) nc := natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo", []byte("msg")) // Verify message is received on receiver side. testMQTTCheckPubMsg(t, mc1, rc1, "org/foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc2, rc2, "org/foo", 0, []byte("msg")) } func TestMQTTSessionMovingDomains(t *testing.T) { tmpl := strings.Replace(jsClusterTemplWithLeafAndMQTT, "{{leaf}}", `leafnodes { listen: 127.0.0.1:-1 }`, 1) tmpl = strings.Replace(tmpl, "store_dir:", "domain: HUB, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 22020, true) defer c.shutdown() c.waitOnLeader() tmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, "store_dir:", "domain: SPOKE, store_dir:", 1) lnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, "SPOKE", 3, 22111) defer lnc.shutdown() lnc.waitOnPeerCount(3) connectSubAndDisconnect := func(host string, port int, present bool) { t.Helper() mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, host, port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, present) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) testMQTTDisconnect(t, mc, nil) } // Create a session on the HUB. Make sure we don't use "clean" session so that // it is not removed when the client connection closes. for i := 0; i < 7; i++ { var present bool if i > 0 { present = true } connectSubAndDisconnect(c.opts[0].MQTT.Host, c.opts[0].MQTT.Port, present) } // Now move to the SPOKE cluster, this is a brand new session there, so should not be present. connectSubAndDisconnect(lnc.opts[1].MQTT.Host, lnc.opts[1].MQTT.Port, false) // Move back to HUB cluster. Make it interesting by connecting to a different // server in that cluster. This should work, and present flag should be true. connectSubAndDisconnect(c.opts[2].MQTT.Host, c.opts[2].MQTT.Port, true) } type remoteConnSameClientIDLogger struct { DummyLogger warn chan string } func (l *remoteConnSameClientIDLogger) Warnf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "remote connection has started with the same client ID") { l.warn <- msg } } func TestMQTTSessionsDifferentDomains(t *testing.T) { tmpl := strings.Replace(jsClusterTemplWithLeafAndMQTT, "{{leaf}}", `leafnodes { listen: 127.0.0.1:-1 }`, 1) c := createJetStreamCluster(t, tmpl, "HUB", _EMPTY_, 3, 22020, true) defer c.shutdown() c.waitOnLeader() tmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, "store_dir:", "domain: LEAF1, store_dir:", 1) lnc1 := c.createLeafNodesWithTemplateAndStartPort(tmpl, "LEAF1", 2, 22111) defer lnc1.shutdown() lnc1.waitOnPeerCount(2) l := &remoteConnSameClientIDLogger{warn: make(chan string, 10)} lnc1.servers[0].SetLogger(l, false, false) tmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, "store_dir:", "domain: LEAF2, store_dir:", 1) lnc2 := c.createLeafNodesWithTemplateAndStartPort(tmpl, "LEAF2", 2, 23111) defer lnc2.shutdown() lnc2.waitOnPeerCount(2) o := &(lnc1.opts[0].MQTT) mc1, rc1 := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, o.Host, o.Port, 5) defer mc1.Close() testMQTTCheckConnAck(t, rc1, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc1, rc1, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc1, nil, rc1) o = &(lnc2.opts[0].MQTT) connectSubAndDisconnect := func(host string, port int, present bool) { t.Helper() mc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, host, port, 5) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, present) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) testMQTTDisconnect(t, mc, nil) } for i := 0; i < 2; i++ { connectSubAndDisconnect(o.Host, o.Port, i > 0) } select { case w := <-l.warn: t.Fatalf("Got a warning: %v", w) case <-time.After(500 * time.Millisecond): // OK } } func TestMQTTParseUnsub(t *testing.T) { for _, test := range []struct { name string proto []byte b byte pl int err string }{ {"reserved flag", nil, 3, 0, "wrong unsubscribe reserved flags"}, {"ensure packet loaded", []byte{1, 2}, mqttUnsubscribeFlags, 10, io.ErrUnexpectedEOF.Error()}, {"error reading packet id", []byte{1}, mqttUnsubscribeFlags, 1, "reading packet identifier"}, {"missing filters", []byte{0, 1}, mqttUnsubscribeFlags, 2, "subscribe protocol must contain at least 1 topic filter"}, {"error reading topic", []byte{0, 1, 0, 2, 'a'}, mqttUnsubscribeFlags, 5, "topic filter"}, {"empty topic", []byte{0, 1, 0, 0}, mqttUnsubscribeFlags, 4, errMQTTTopicFilterCannotBeEmpty.Error()}, {"invalid utf8 topic", []byte{0, 1, 0, 1, 241}, mqttUnsubscribeFlags, 5, "invalid utf8 for topic filter"}, } { t.Run(test.name, func(t *testing.T) { r := &mqttReader{} r.reset(test.proto) mqtt := &mqtt{r: r} c := &client{mqtt: mqtt} if _, _, err := c.mqttParseSubsOrUnsubs(r, test.b, test.pl, false); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func testMQTTUnsub(t *testing.T, pi uint16, c net.Conn, r *mqttReader, filters []*mqttFilter) { t.Helper() w := newMQTTWriter(0) pkLen := 2 // for pi for i := 0; i < len(filters); i++ { f := filters[i] pkLen += 2 + len(f.filter) } w.WriteByte(mqttPacketUnsub | mqttUnsubscribeFlags) w.WriteVarInt(pkLen) w.WriteUint16(pi) for i := 0; i < len(filters); i++ { f := filters[i] w.WriteBytes([]byte(f.filter)) } if _, err := testMQTTWrite(c, w.Bytes()); err != nil { t.Fatalf("Error writing UNSUBSCRIBE protocol: %v", err) } b, _ := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketUnsubAck { t.Fatalf("Expected UNSUBACK packet %x, got %x", mqttPacketUnsubAck, pt) } rpi, err := r.readUint16("packet identifier") if err != nil || rpi != pi { t.Fatalf("Error with packet identifier expected=%v got: %v err=%v", pi, rpi, err) } } func TestMQTTUnsub(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false) mc, r := testMQTTConnect(t, &mqttConnInfo{user: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) // Publish and test msg received testMQTTPublish(t, mcp, r, 0, false, false, "foo", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo", 0, []byte("msg")) // Unsubscribe testMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: "foo"}}) // Publish and test msg not received testMQTTPublish(t, mcp, r, 0, false, false, "foo", 0, []byte("msg")) testMQTTExpectNothing(t, r) // Use of wildcards subs filters := []*mqttFilter{ {filter: "foo/bar", qos: 0}, {filter: "foo/#", qos: 0}, } testMQTTSub(t, 1, mc, r, filters, []byte{0, 0}) testMQTTFlush(t, mc, nil, r) // Publish and check that message received twice testMQTTPublish(t, mcp, r, 0, false, false, "foo/bar", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo/bar", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo/bar", 0, []byte("msg")) // Unsub the wildcard one testMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: "foo/#"}}) // Publish and check that message received once testMQTTPublish(t, mcp, r, 0, false, false, "foo/bar", 0, []byte("msg")) testMQTTCheckPubMsg(t, mc, r, "foo/bar", 0, []byte("msg")) testMQTTExpectNothing(t, r) // Unsub last testMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: "foo/bar"}}) // Publish and test msg not received testMQTTPublish(t, mcp, r, 0, false, false, "foo/bar", 0, []byte("msg")) testMQTTExpectNothing(t, r) } func testMQTTExpectDisconnect(t testing.TB, c net.Conn) { t.Helper() if buf, err := testMQTTRead(c); err == nil { t.Fatalf("Expected connection to be disconnected, got %s", buf) } } func TestMQTTPublishTopicErrors(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) for _, test := range []struct { name string topic string }{ {"empty", ""}, {"with single level wildcard", "foo/+"}, {"with multiple level wildcard", "foo/#"}, } { t.Run(test.name, func(t *testing.T) { mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc, r, 0, false, false, test.topic, 0, []byte("msg")) testMQTTExpectDisconnect(t, mc) }) } } func testMQTTDisconnect(t testing.TB, c net.Conn, bw *bufio.Writer) { t.Helper() testMQTTDisconnectEx(t, c, bw, true) } func testMQTTDisconnectEx(t testing.TB, c net.Conn, bw *bufio.Writer, wait bool) { t.Helper() w := newMQTTWriter(0) w.WriteByte(mqttPacketDisconnect) w.WriteByte(0) if bw != nil { bw.Write(w.Bytes()) bw.Flush() } else { c.Write(w.Bytes()) } if wait { testMQTTExpectDisconnect(t, c) } } func TestMQTTWill(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() sub := natsSubSync(t, nc, "will.topic") natsFlush(t, nc) willMsg := []byte("bye") for _, test := range []struct { name string willExpected bool willQoS byte }{ {"will qos 0", true, 0}, {"will qos 1", true, 1}, {"will qos 2", true, 2}, {"proper disconnect no will", false, 0}, } { t.Run(test.name, func(t *testing.T) { mcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "will/#", qos: 2}}, []byte{2}) testMQTTFlush(t, mcs, nil, rs) mc, r := testMQTTConnect(t, &mqttConnInfo{ cleanSess: true, will: &mqttWill{ topic: []byte("will/topic"), message: willMsg, qos: test.willQoS, }, }, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) if test.willExpected { mc.Close() testMQTTCheckPubMsg(t, mcs, rs, "will/topic", test.willQoS<<1, willMsg) wm := natsNexMsg(t, sub, time.Second) if !bytes.Equal(wm.Data, willMsg) { t.Fatalf("Expected will message to be %q, got %q", willMsg, wm.Data) } } else { testMQTTDisconnect(t, mc, nil) testMQTTExpectNothing(t, rs) if wm, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Should not have receive a message, got subj=%q data=%q", wm.Subject, wm.Data) } } }) } } func TestMQTTQoS2WillReject(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.rejectQoS2Pub = true s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "will/#", qos: 2}}, []byte{2}) testMQTTFlush(t, mcs, nil, rs) mc, r := testMQTTConnect(t, &mqttConnInfo{ cleanSess: true, will: &mqttWill{ topic: []byte("will/topic"), message: []byte("bye"), qos: 2, }, }, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCQoS2WillRejected, false) } func TestMQTTWillRetain(t *testing.T) { for _, test := range []struct { name string pubQoS byte subQoS byte }{ {"pub QoS0 sub QoS0", 0, 0}, {"pub QoS0 sub QoS1", 0, 1}, {"pub QoS1 sub QoS0", 1, 0}, {"pub QoS1 sub QoS1", 1, 1}, } { t.Run(test.name, func(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mces, res := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mces.Close() testMQTTCheckConnAck(t, res, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mces, res, []*mqttFilter{{filter: "will/#", qos: test.subQoS}}, []byte{test.subQoS}) willTopic := []byte("will/topic") willMsg := []byte("bye") mc, r := testMQTTConnect(t, &mqttConnInfo{ cleanSess: true, will: &mqttWill{ topic: willTopic, message: willMsg, qos: test.pubQoS, retain: true, }, }, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Disconnect the client mc.Close() // Wait for the server to process the connection close, which will // cause the "will" message to be published (and retained). checkClientsCount(t, s, 1) // Create subscription on will topic and expect will message. mcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "will/#", qos: test.subQoS}}, []byte{test.subQoS}) pflags, _ := testMQTTGetPubMsg(t, mcs, rs, "will/topic", willMsg) if !mqttIsRetained(pflags) { t.Fatalf("expected retain flag to be set, it was not: %v", pflags) } // Expected QoS will be the lesser of the pub/sub QoS. expectedQoS := test.pubQoS if test.subQoS == 0 { expectedQoS = 0 } if qos := mqttGetQoS(pflags); qos != expectedQoS { t.Fatalf("expected qos to be %v, got %v", expectedQoS, qos) } // The existing subscription (prior to sending the will) should receive // the will but the retain flag should not be set. pflags, _ = testMQTTGetPubMsg(t, mces, res, "will/topic", willMsg) if mqttIsRetained(pflags) { t.Fatalf("expected retain flag to not be set, it was: %v", pflags) } // Expected QoS will be the lesser of the pub/sub QoS. if qos := mqttGetQoS(pflags); qos != expectedQoS { t.Fatalf("expected qos to be %v, got %v", expectedQoS, qos) } }) } } func TestMQTTWillRetainPermViolation(t *testing.T) { template := ` port: -1 jetstream { store_dir = %q } server_name: mqtt authorization { mqtt_perms = { publish = ["%s"] subscribe = ["foo", "bar", "$MQTT.sub.>"] } users = [ {user: mqtt, password: pass, permissions: $mqtt_perms} ] } mqtt { port: -1 } ` tdir := t.TempDir() conf := createConfFile(t, []byte(fmt.Sprintf(template, tdir, "foo"))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ cleanSess: true, user: "mqtt", pass: "pass", } // We create first a connection with the Will topic that the publisher // is allowed to publish to. ci.will = &mqttWill{ topic: []byte("foo"), message: []byte("bye"), qos: 1, retain: true, } mc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Disconnect, which will cause the Will to be sent with retain flag. mc.Close() // Wait for the server to process the connection close, which will // cause the "will" message to be published (and retained). checkClientsCount(t, s, 0) // Create a subscription on the Will subject and we should receive it. ci.will = nil mcs, rs := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) pflags, _ := testMQTTGetPubMsg(t, mcs, rs, "foo", []byte("bye")) if !mqttIsRetained(pflags) { t.Fatalf("expected retain flag to be set, it was not: %v", pflags) } if qos := mqttGetQoS(pflags); qos != 1 { t.Fatalf("expected qos to be 1, got %v", qos) } testMQTTDisconnect(t, mcs, nil) // Now create another connection with a Will that client is not allowed to publish to. ci.will = &mqttWill{ topic: []byte("bar"), message: []byte("bye"), qos: 1, retain: true, } mc, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Disconnect, to cause Will to be produced, but in that case should not be stored // since user not allowed to publish on "bar". mc.Close() // Wait for the server to process the connection close, which will // cause the "will" message to be published (and retained). checkClientsCount(t, s, 0) // Create sub on "bar" which user is allowed to subscribe to. ci.will = nil mcs, rs = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) // No Will should be published since it should not have been stored in the first place. testMQTTExpectNothing(t, rs) testMQTTDisconnect(t, mcs, nil) // Now remove permission to publish on "foo" and check that a new subscription // on "foo" is now not getting the will message because the original user no // longer has permission to do so. reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, tdir, "baz")) mcs, rs = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mcs.Close() testMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTExpectNothing(t, rs) testMQTTDisconnect(t, mcs, nil) } func TestMQTTPublishRetain(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) for _, test := range []struct { name string retained bool sentValue string expectedValue string subGetsIt bool }{ {"publish retained", true, "retained", "retained", true}, {"publish not retained", false, "not retained", "retained", true}, {"remove retained", true, "", "", false}, } { t.Run(test.name, func(t *testing.T) { mc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc1, rs1, 0, false, test.retained, "foo", 0, []byte(test.sentValue)) testMQTTFlush(t, mc1, nil, rs1) mc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "foo/#", qos: 1}}, []byte{1}) if test.subGetsIt { pflags, _ := testMQTTGetPubMsg(t, mc2, rs2, "foo", []byte(test.expectedValue)) if !mqttIsRetained(pflags) { t.Fatalf("retain flag should have been set, it was not: flags=%v", pflags) } } else { testMQTTExpectNothing(t, rs2) } testMQTTDisconnect(t, mc1, nil) testMQTTDisconnect(t, mc2, nil) }) } } func TestMQTTQoS2RetainedReject(t *testing.T) { // Start the server with QOS2 enabled, and submit retained messages with QoS // 1 and 2. o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) mc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc1, rs1, 2, false, true, "foo", 1, []byte("qos2 failed")) testMQTTPublish(t, mc1, rs1, 1, false, true, "bar", 2, []byte("qos1 retained")) testMQTTFlush(t, mc1, nil, rs1) testMQTTDisconnect(t, mc1, nil) mc1.Close() s.Shutdown() // Restart the server with QOS2 disabled; we should be using the same // JetStream store, so the retained message should still be there. o.MQTT.rejectQoS2Pub = true s = testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "bar/#", qos: 2}}, []byte{2}) pflags, _ := testMQTTGetPubMsg(t, mc2, rs2, "bar", []byte("qos1 retained")) if !mqttIsRetained(pflags) { t.Fatalf("retain flag should have been set, it was not: flags=%v", pflags) } testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "foo/#", qos: 2}}, []byte{2}) testMQTTExpectNothing(t, rs2) testMQTTDisconnect(t, mc2, nil) } func TestMQTTRetainFlag(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) mc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc1, rs1, 0, false, true, "foo/0", 0, []byte("flag set")) testMQTTPublish(t, mc1, rs1, 0, false, true, "foo/1", 0, []byte("flag set")) testMQTTFlush(t, mc1, nil, rs1) mc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "foo/0", qos: 0}}, []byte{0}) pflags, _ := testMQTTGetPubMsg(t, mc2, rs2, "foo/0", []byte("flag set")) if !mqttIsRetained(pflags) { t.Fatalf("retain flag should have been set, it was not: flags=%v", pflags) } testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "foo/1", qos: 1}}, []byte{1}) pflags, _ = testMQTTGetPubMsg(t, mc2, rs2, "foo/1", []byte("flag set")) if !mqttIsRetained(pflags) { t.Fatalf("retain flag should have been set, it was not: flags=%v", pflags) } // For existing subscriptions, RETAIN flag should not be set: [MQTT-3.3.1-9]. testMQTTPublish(t, mc1, rs1, 0, false, true, "foo/0", 0, []byte("flag not set")) testMQTTFlush(t, mc1, nil, rs1) pflags, _ = testMQTTGetPubMsg(t, mc2, rs2, "foo/0", []byte("flag not set")) if mqttIsRetained(pflags) { t.Fatalf("retain flag should not have been set, it was: flags=%v", pflags) } testMQTTPublish(t, mc1, rs1, 0, false, true, "foo/1", 0, []byte("flag not set")) testMQTTFlush(t, mc1, nil, rs1) pflags, _ = testMQTTGetPubMsg(t, mc2, rs2, "foo/1", []byte("flag not set")) if mqttIsRetained(pflags) { t.Fatalf("retain flag should not have been set, it was: flags=%v", pflags) } } func TestMQTTPublishRetainPermViolation(t *testing.T) { o := testMQTTDefaultOptions() o.Users = []*User{ { Username: "mqtt", Password: "pass", Permissions: &Permissions{ Publish: &SubjectPermission{Allow: []string{"foo"}}, Subscribe: &SubjectPermission{Allow: []string{"bar", "$MQTT.sub.>"}}, }, }, } s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ cleanSess: true, user: "mqtt", pass: "pass", } mc1, rs1 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc1, rs1, 0, false, true, "bar", 0, []byte("retained")) testMQTTFlush(t, mc1, nil, rs1) mc2, rs2 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTExpectNothing(t, rs2) testMQTTDisconnect(t, mc1, nil) testMQTTDisconnect(t, mc2, nil) } func TestMQTTPublishViolation(t *testing.T) { o := testMQTTDefaultOptions() o.Users = []*User{ { Username: "mqtt", Password: "pass", Permissions: &Permissions{ Publish: &SubjectPermission{Allow: []string{"foo.bar"}}, Subscribe: &SubjectPermission{Allow: []string{"foo.*", "$MQTT.sub.>"}}, }, }, } s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ user: "mqtt", pass: "pass", } ci.clientID = "sub" mc, rc := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/+", qos: 1}}, []byte{1}) testMQTTFlush(t, mc, nil, rc) ci.clientID = "pub" ci.cleanSess = true mp, rp := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // These should be received since publisher has the right to publish on foo.bar testMQTTPublish(t, mp, rp, 0, false, false, "foo/bar", 0, []byte("msg1")) testMQTTCheckPubMsg(t, mc, rc, "foo/bar", 0, []byte("msg1")) testMQTTPublish(t, mp, rp, 1, false, false, "foo/bar", 1, []byte("msg2")) testMQTTCheckPubMsg(t, mc, rc, "foo/bar", mqttPubQos1, []byte("msg2")) // But these should not be cause pub has no permission to publish on foo.baz testMQTTPublish(t, mp, rp, 0, false, false, "foo/baz", 0, []byte("msg3")) testMQTTExpectNothing(t, rc) testMQTTPublish(t, mp, rp, 1, false, false, "foo/baz", 1, []byte("msg4")) testMQTTExpectNothing(t, rc) // Disconnect publisher testMQTTDisconnect(t, mp, nil) mp.Close() // Disconnect subscriber and restart it to make sure that it does not receive msg3/msg4 testMQTTDisconnect(t, mc, nil) mc.Close() ci.cleanSess = false ci.clientID = "sub" mc, rc = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: "foo/+", qos: 1}}, []byte{1}) testMQTTExpectNothing(t, rc) } func TestMQTTCleanSession(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ clientID: "me", cleanSess: false, } c, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTDisconnect(t, c, nil) c, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) testMQTTDisconnect(t, c, nil) ci.cleanSess = true c, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTDisconnect(t, c, nil) } func TestMQTTDuplicateClientID(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{ clientID: "me", cleanSess: false, } c1, r1 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) c2, r2 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true) // The old client should be disconnected. testMQTTExpectDisconnect(t, c1) } func TestMQTTPersistedSession(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{ {filter: "foo/#", qos: 1}, {filter: "bar", qos: 1}, {filter: "baz", qos: 0}, }, []byte{1, 1, 0}) testMQTTFlush(t, c, nil, r) // Shutdown server, close connection and restart server. It should // have restored the session and consumers. dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) s.Shutdown() c.Close() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function // Create a publisher that will send qos1 so we verify that messages // are stored for the persisted sessions. c, r = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, c, r, 1, false, false, "foo/bar", 1, []byte("msg0")) testMQTTFlush(t, c, nil, r) testMQTTDisconnect(t, c, nil) c.Close() // Recreate session c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Since consumers have been recovered, messages should be received // (MQTT does not need client to recreate consumers for a recovered // session) // Check that qos1 publish message is received. testMQTTCheckPubMsg(t, c, r, "foo/bar", mqttPubQos1, []byte("msg0")) // Flush to prevent publishes to be done too soon since we are // receiving the CONNACK before the subscriptions are restored. testMQTTFlush(t, c, nil, r) // Now publish some messages to all subscriptions. nc := natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo.bar", []byte("msg1")) testMQTTCheckPubMsg(t, c, r, "foo/bar", 0, []byte("msg1")) natsPub(t, nc, "foo", []byte("msg2")) testMQTTCheckPubMsg(t, c, r, "foo", 0, []byte("msg2")) natsPub(t, nc, "bar", []byte("msg3")) testMQTTCheckPubMsg(t, c, r, "bar", 0, []byte("msg3")) natsPub(t, nc, "baz", []byte("msg4")) testMQTTCheckPubMsg(t, c, r, "baz", 0, []byte("msg4")) // Now unsub "bar" and verify that message published on this topic // is not received. testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "bar"}}) natsPub(t, nc, "bar", []byte("msg5")) testMQTTExpectNothing(t, r) nc.Close() s.Shutdown() c.Close() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function // Recreate a client c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) nc = natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo.bar", []byte("msg6")) testMQTTCheckPubMsg(t, c, r, "foo/bar", 0, []byte("msg6")) natsPub(t, nc, "foo", []byte("msg7")) testMQTTCheckPubMsg(t, c, r, "foo", 0, []byte("msg7")) // Make sure that we did not recover bar. natsPub(t, nc, "bar", []byte("msg8")) testMQTTExpectNothing(t, r) natsPub(t, nc, "baz", []byte("msg9")) testMQTTCheckPubMsg(t, c, r, "baz", 0, []byte("msg9")) // Have the sub client send a subscription downgrading the qos1 subscription. testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo/#", qos: 0}}, []byte{0}) testMQTTFlush(t, c, nil, r) nc.Close() s.Shutdown() c.Close() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function // Recreate the sub client c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Publish as a qos1 c2, r2 := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, c2, r2, 1, false, false, "foo/bar", 1, []byte("msg10")) // Verify that it is received as qos0 which is the qos of the subscription. testMQTTCheckPubMsg(t, c, r, "foo/bar", 0, []byte("msg10")) testMQTTDisconnect(t, c, nil) c.Close() testMQTTDisconnect(t, c2, nil) c2.Close() // Finally, recreate the sub with clean session and ensure that all is gone cisub.cleanSess = true for i := 0; i < 2; i++ { c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) nc = natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo.bar", []byte("msg11")) testMQTTExpectNothing(t, r) natsPub(t, nc, "foo", []byte("msg12")) testMQTTExpectNothing(t, r) // Make sure that we did not recover bar. natsPub(t, nc, "bar", []byte("msg13")) testMQTTExpectNothing(t, r) natsPub(t, nc, "baz", []byte("msg14")) testMQTTExpectNothing(t, r) testMQTTDisconnect(t, c, nil) c.Close() nc.Close() s.Shutdown() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function } } func TestMQTTRecoverSessionAndAddNewSub(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) cisub := &mqttConnInfo{clientID: "sub1", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTDisconnect(t, c, nil) c.Close() // Shutdown server, close connection and restart server. It should // have restored the session and consumers. dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) s.Shutdown() c.Close() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // No need for defer since it is done top of function c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Now add sub and make sure it does not crash testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) // Now repeat with a new client but without server restart. cisub2 := &mqttConnInfo{clientID: "sub2", cleanSess: false} c2, r2 := testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) testMQTTDisconnect(t, c2, nil) c2.Close() c2, r2 = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, c2, r2, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, c2, nil, r2) } func TestMQTTRecoverSessionWithSubAndClientResendSub(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) cisub1 := &mqttConnInfo{clientID: "sub1", cleanSess: false} c, r := testMQTTConnect(t, cisub1, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Have a client send a SUBSCRIBE protocol for foo, QoS1 testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTDisconnect(t, c, nil) c.Close() // Restart the server now. dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) s.Shutdown() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) // No need for defer since it is done top of function // Now restart the client. Since the client was created with cleanSess==false, // the server will have recorded the subscriptions for this client. c, r = testMQTTConnect(t, cisub1, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // At this point, the server has recreated the subscription on foo, QoS1. // For applications that restart, it is possible (likely) that they // will resend their SUBSCRIBE protocols, so do so now: testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) checkNumSub := func(clientID string) { t.Helper() // Find the MQTT client... mc := testMQTTGetClient(t, s, clientID) // Check how many NATS subscriptions are registered. var fooSub int var otherSub int mc.mu.Lock() for _, sub := range mc.subs { switch string(sub.subject) { case "foo": fooSub++ default: otherSub++ } } mc.mu.Unlock() // We should have 2 subscriptions, one on "foo", and one for the JS durable // consumer's delivery subject. if fooSub != 1 { t.Fatalf("Expected 1 sub on 'foo', got %v", fooSub) } if otherSub != 1 { t.Fatalf("Expected 1 subscription for JS durable, got %v", otherSub) } } checkNumSub("sub1") c.Close() // Now same but without the server restart in-between. cisub2 := &mqttConnInfo{clientID: "sub2", cleanSess: false} c, r = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTDisconnect(t, c, nil) c.Close() // Restart client c, r = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) // Check client subs checkNumSub("sub2") } func TestMQTTFlappingSession(t *testing.T) { mqttSessJailDur = 250 * time.Millisecond mqttFlapCleanItvl = 350 * time.Millisecond defer func() { mqttSessJailDur = mqttSessFlappingJailDur mqttFlapCleanItvl = mqttSessFlappingCleanupInterval }() o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{clientID: "flapper", cleanSess: false} c, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Let's get a handle on the asm to check things later. cli := testMQTTGetClient(t, s, "flapper") asm := cli.mqtt.asm // Start a new connection with the same clientID, which should replace // the old one and put it in the flappers map. c2, r2 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true) // Should be disconnected... testMQTTExpectDisconnect(t, c) // Now try to reconnect "c" and we should fail. We have to do this manually, // since we expect it to fail. addr := fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port) c, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating mqtt connection: %v", err) } defer c.Close() proto := mqttCreateConnectProto(ci) if _, err := testMQTTWrite(c, proto); err != nil { t.Fatalf("Error writing protocols: %v", err) } // Misbehave and send a SUB protocol without waiting for the CONNACK w := newMQTTWriter(0) pkLen := 2 // for pi // Topic "foo" pkLen += 2 + 3 + 1 w.WriteByte(mqttPacketSub | mqttSubscribeFlags) w.WriteVarInt(pkLen) w.WriteUint16(1) w.WriteBytes([]byte("foo")) w.WriteByte(1) if _, err := testMQTTWrite(c, w.Bytes()); err != nil { t.Fatalf("Error writing protocols: %v", err) } // Now read the CONNACK and we should have been disconnected. if _, err := testMQTTRead(c); err == nil { t.Fatal("Expected connection to fail") } // This should be in the flappers map, but after 250ms should be cleared. for i := 0; i < 2; i++ { asm.mu.RLock() _, present := asm.flappers["flapper"] asm.mu.RUnlock() if i == 0 { if !present { t.Fatal("Did not find the client ID in the flappers map") } // Wait for more than the cleanup interval time.Sleep(mqttFlapCleanItvl + 100*time.Millisecond) } else if present { t.Fatal("The client ID should have been cleared from the map") } } } func TestMQTTLockedSession(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) subClientID := "sub" ci := &mqttConnInfo{clientID: subClientID, cleanSess: false} c, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) sm := &s.mqtt.sessmgr sm.mu.Lock() asm := sm.sessions[globalAccountName] sm.mu.Unlock() if asm == nil { t.Fatalf("account session manager not found") } // It is possible, however unlikely, to have received CONNACK while // mqttProcessConnect is still running, and the session remains locked. Wait // for it to finish. checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { asm.mu.RLock() defer asm.mu.RUnlock() if _, stillLocked := asm.sessLocked[subClientID]; stillLocked { return fmt.Errorf("session still locked") } return nil }) // Get the session for "sub" cli := testMQTTGetClient(t, s, subClientID) sess := cli.mqtt.sess // Pretend that the session above is locked. if err := asm.lockSession(sess, cli); err != nil { t.Fatalf("Unable to lock session: %v", err) } defer asm.unlockSession(sess) // Now try to connect another client that wants to use "sub". // We can't use testMQTTConnect() because it is going to fail. addr := fmt.Sprintf("%s:%d", o.MQTT.Host, o.MQTT.Port) c2, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating mqtt connection: %v", err) } defer c2.Close() proto := mqttCreateConnectProto(ci) if _, err := testMQTTWrite(c2, proto); err != nil { t.Fatalf("Error writing connect: %v", err) } if _, err := testMQTTRead(c2); err == nil { t.Fatal("Expected connection to fail") } // Now try again, but this time release the session while waiting // to connect and it should succeed. time.AfterFunc(250*time.Millisecond, func() { asm.unlockSession(sess) }) c3, r3 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c3.Close() testMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, true) } func TestMQTTPersistRetainedMsg(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} c, r := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, c, r, 1, false, true, "moo/foo", 1, []byte("foo1")) testMQTTPublish(t, c, r, 1, false, true, "moo/foo", 1, []byte("foo2")) testMQTTPublish(t, c, r, 1, false, true, "moo/bar", 1, []byte("bar1")) testMQTTPublish(t, c, r, 0, false, true, "moo/baz", 1, []byte("baz1")) // Remove bar testMQTTPublish(t, c, r, 1, false, true, "moo/bar", 1, nil) testMQTTFlush(t, c, nil, r) testMQTTDisconnect(t, c, nil) c.Close() s.Shutdown() o.Port = -1 o.MQTT.Port = -1 o.StoreDir = dir s = testMQTTRunServer(t, o) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) t.Run("many subs one topic", func(t *testing.T) { testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "moo/foo", qos: 1}}, []byte{1}) testMQTTCheckPubMsg(t, c, r, "moo/foo", mqttPubFlagRetain|mqttPubQos1, []byte("foo2")) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "moo/baz", qos: 1}}, []byte{1}) testMQTTCheckPubMsg(t, c, r, "moo/baz", mqttPubFlagRetain, []byte("baz1")) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "moo/bar", qos: 1}}, []byte{1}) testMQTTExpectNothing(t, r) }) runSingleSubscribe := func(filters []*mqttFilter) func(t *testing.T) { return func(t *testing.T) { testMQTTSub(t, 1, c, r, filters, nil) for i := 0; i < 2; i++ { flags, pi, topic, payload := testMQTTReadPubPacket(t, r) if (flags & mqttPubFlagRetain) == 0 { t.Fatalf("Expected flags to have retain set, got %v", flags) } if (flags & mqttPubFlagDup) != 0 { t.Fatalf("Expected flags to not have Dup set, got %v", flags) } var expQOS byte var expPayload []byte switch topic { case "moo/foo": expQOS = mqttPubQos1 expPayload = []byte("foo2") testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) case "moo/baz": expQOS = 0 expPayload = []byte("baz1") default: t.Fatalf("Unexpected topic: %v", topic) } if (flags & mqttPubFlagQoS) != expQOS { t.Fatalf("Expected flags to have QOS %x set, got %x", expQOS, flags) } if string(payload) != string(expPayload) { t.Fatalf("Expected payload to be %q, got %v", expPayload, string(payload)) } } testMQTTExpectNothing(t, r) } } t.Run("one sub many topics", runSingleSubscribe([]*mqttFilter{ {filter: "moo/foo", qos: 1}, {filter: "moo/baz", qos: 1}, {filter: "moo/bar", qos: 1}, })) t.Run("one sub wildcard plus", runSingleSubscribe([]*mqttFilter{ {filter: "moo/+", qos: 1}, })) t.Run("one sub wildcard hash", runSingleSubscribe([]*mqttFilter{ {filter: "moo/#", qos: 1}, })) testMQTTDisconnect(t, c, nil) c.Close() } func TestMQTTRetainedMsgCleanup(t *testing.T) { mqttRetainedCacheTTL = 250 * time.Millisecond defer func() { mqttRetainedCacheTTL = mqttDefaultRetainedCacheTTL }() o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{clientID: "cache", cleanSess: true} c, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) // Send a retained message. testMQTTPublish(t, c, r, 1, false, true, "foo", 1, []byte("msg")) testMQTTFlush(t, c, nil, r) // Start a subscription. testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1|mqttPubFlagRetain, []byte("msg")) time.Sleep(2 * mqttRetainedCacheTTL) // Make sure not in cache anymore cli := testMQTTGetClient(t, s, "cache") asm := cli.mqtt.asm if v, ok := asm.rmsCache.Load("foo"); ok { t.Fatalf("Should not be in cache, got %+v", v) } } func TestMQTTConnAckFirstPacket(t *testing.T) { o := testMQTTDefaultOptions() o.NoLog, o.Debug, o.Trace = true, false, false s := RunServer(o) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTDisconnect(t, c, nil) c.Close() nc := natsConnect(t, s.ClientURL()) defer nc.Close() wg := sync.WaitGroup{} wg.Add(1) ch := make(chan struct{}, 1) ready := make(chan struct{}) go func() { defer wg.Done() close(ready) for { nc.Publish("foo", []byte("msg")) select { case <-ch: return default: } } }() <-ready for i := 0; i < 100; i++ { c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) w := newMQTTWriter(0) w.WriteByte(mqttPacketDisconnect) w.WriteByte(0) c.Write(w.Bytes()) // Wait to be disconnected, we can't use testMQTTDisconnect() because // it would fail because we may still receive some NATS messages. var b [10]byte for { if _, err := c.Read(b[:]); err != nil { break } } c.Close() } close(ch) wg.Wait() } func TestMQTTRedeliveryAckWait(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = 250 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("foo1")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 2, []byte("foo2")) testMQTTDisconnect(t, cp, nil) cp.Close() for i := 0; i < 2; i++ { flags := mqttPubQos1 if i > 0 { flags |= mqttPubFlagDup } pi1 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", flags, []byte("foo1")) pi2 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", flags, []byte("foo2")) if pi1 != 1 || pi2 != 2 { t.Fatalf("Unexpected pi values: %v, %v", pi1, pi2) } } // Ack first message testMQTTSendPIPacket(mqttPacketPubAck, t, c, 1) // Redelivery should only be for second message now for i := 0; i < 2; i++ { flags := mqttPubQos1 | mqttPubFlagDup pi := testMQTTCheckPubMsgNoAck(t, c, r, "foo", flags, []byte("foo2")) if pi != 2 { t.Fatalf("Unexpected pi to be 2, got %v", pi) } } // Restart client, should receive second message with pi==2 c.Close() c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Check that message is received with proper pi pi := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1|mqttPubFlagDup, []byte("foo2")) if pi != 2 { t.Fatalf("Unexpected pi to be 2, got %v", pi) } // Now ack second message testMQTTSendPIPacket(mqttPacketPubAck, t, c, 2) // Flush to make sure it is processed before checking client's maps testMQTTFlush(t, c, nil, r) // Look for the sub client mc := testMQTTGetClient(t, s, "sub") mc.mu.Lock() sess := mc.mqtt.sess sess.mu.Lock() lpi := len(sess.pendingPublish) var lsseq int for _, sseqToPi := range sess.cpending { lsseq += len(sseqToPi) } sess.mu.Unlock() mc.mu.Unlock() if lpi != 0 || lsseq != 0 { t.Fatalf("Maps should be empty, got %v, %v", lpi, lsseq) } } // - [MQTT-3.10.4-3] If a Server deletes a Subscription It MUST complete the // delivery of any QoS 1 or QoS 2 messages which it has started to send to the // Client. // // Test flow: // - Subscribe to foo, publish 3 QoS2 messages. // - After one is PUBCOMP-ed, and one is PUBREC-ed, Unsubscribe. // - See that the remaining 2 are fully delivered. func TestMQTTQoS2InflightMsgsDeliveredAfterUnsubscribe(t *testing.T) { // This test has proven flaky on Travis, so skip for now. Line 4926, the 3rd // testMQTTCheckPubMsgNoAck sometimes returns `data1`, instead of `data3` // that we are expecting. It must be a retry since we did not acknowledge // `data1` until later. t.Skip() o := testMQTTDefaultOptions() o.MQTT.AckWait = 10 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) var qos2 byte = 2 cisub := &mqttConnInfo{clientID: "sub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: qos2}}, []byte{qos2}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // send 3 messages testMQTTPublish(t, cp, rp, qos2, false, false, "foo", 441, []byte("data1")) testMQTTPublish(t, cp, rp, qos2, false, false, "foo", 442, []byte("data2")) testMQTTPublish(t, cp, rp, qos2, false, false, "foo", 443, []byte("data3")) testMQTTDisconnect(t, cp, nil) cp.Close() subPI1 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data1")) subPI2 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data2")) // subPI3 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data3")) _ = testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data3")) // fully receive first message testMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI1) testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI1) testMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI1) // Do not PUBCOMP the 2nd message yet. testMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI2) testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI2) // Unsubscribe testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: qos2}}) // We expect that PI2 and PI3 will continue to be delivered, from their // respective states. gotPI2PubRel := false // TODO: Currently, we do not get the unacknowledged PUBLISH re-delivered // after an UNSUBSCRIBE. Ongoing discussion if we should/must. // gotPI3Publish := false // gotPI3PubRel := false for !gotPI2PubRel /* || !gotPI3Publish || !gotPI3PubRel */ { b, _ /* len */ := testMQTTReadPacket(t, r) switch b & mqttPacketMask { case mqttPacketPubRel: pi, err := r.readUint16("packet identifier") if err != nil { t.Fatalf("got unexpected error: %v", err) } switch pi { case subPI2: testMQTTSendPIPacket(mqttPacketPubComp, t, c, pi) gotPI2PubRel = true // case subPI3: // testMQTTSendPIPacket(mqttPacketPubComp, t, c, pi) // gotPI3PubRel = true default: t.Fatalf("Expected PI %v got: %v", subPI2, pi) } // case mqttPacketPub: // _, pi, _ := testMQTTGetPubMsgExEx(t, c, r, b, len, "foo", []byte("data3")) // if pi != subPI3 { // t.Fatalf("Expected PI %v got: %v", subPI3, pi) // } // gotPI3Publish = true // testMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI3) default: t.Fatalf("Unexpected packet type: %v", b&mqttPacketMask) } } testMQTTExpectNothing(t, r) } func TestMQTTQoS2RejectPublishDuplicates(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) var qos2 byte = 2 cisub := &mqttConnInfo{clientID: "sub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: qos2}}, []byte{qos2}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // Publish 3 different with same PI before we get any PUBREC back, then // complete the PUBREL/PUBCOMP flow as needed. Only one message (first // payload) should be delivered. PUBRECs, var pubPI uint16 = 444 testMQTTSendPublishPacket(t, cp, qos2, false, false, "foo", pubPI, []byte("data1")) testMQTTSendPublishPacket(t, cp, qos2, true, false, "foo", pubPI, []byte("data2")) testMQTTSendPublishPacket(t, cp, qos2, false, false, "foo", pubPI, []byte("data3")) for i := 0; i < 3; i++ { // [MQTT-4.3.3-1] The receiver // // - MUST respond with a PUBREC containing the Packet Identifier from // the incoming PUBLISH Packet, having accepted ownership of the // Application Message. // // - Until it has received the corresponding PUBREL packet, the Receiver // MUST acknowledge any subsequent PUBLISH packet with the same Packet // Identifier by sending a PUBREC. It MUST NOT cause duplicate messages // to be delivered to any onward recipients in this case. testMQTTReadPIPacket(mqttPacketPubRec, t, rp, pubPI) } for i := 0; i < 3; i++ { testMQTTSendPIPacket(mqttPacketPubRel, t, cp, pubPI) } for i := 0; i < 3; i++ { // [MQTT-4.3.3-1] MUST respond to a PUBREL packet by sending a PUBCOMP // packet containing the same Packet Identifier as the PUBREL. testMQTTReadPIPacket(mqttPacketPubComp, t, rp, pubPI) } // [MQTT-4.3.3-1] After it has sent a PUBCOMP, the receiver MUST treat any // subsequent PUBLISH packet that contains that Packet Identifier as being a // new publication. // // Publish another message, identical to the first one. Since the server // already sent us a PUBCOMP, it will deliver this message, for a total of 2 // delivered. testMQTTPublish(t, cp, rp, qos2, false, false, "foo", pubPI, []byte("data5")) testMQTTDisconnect(t, cp, nil) cp.Close() // Verify we got a total of 2 messages. subPI1 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data1")) subPI2 := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data5")) for _, pi := range []uint16{subPI1, subPI2} { testMQTTSendPIPacket(mqttPacketPubRec, t, c, pi) testMQTTReadPIPacket(mqttPacketPubRel, t, r, pi) testMQTTSendPIPacket(mqttPacketPubComp, t, c, pi) } testMQTTExpectNothing(t, r) } func TestMQTTQoS2RetriesPublish(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = 100 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) var qos2 byte = 2 cisub := &mqttConnInfo{clientID: "sub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: qos2}}, []byte{qos2}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // Publish a message and close the pub connection. var pubPI uint16 = 444 testMQTTPublish(t, cp, rp, qos2, false, false, "foo", pubPI, []byte("data1")) testMQTTDisconnect(t, cp, nil) cp.Close() // See that we got the message delivered to the sub, but don't PUBREC it // yet. subPI := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data1")) // See that the message is redelivered again 2 times, with the DUP on, before we PUBREC it. for i := 0; i < 2; i++ { expectedFlags := mqttPubQoS2 | mqttPubFlagDup pi := testMQTTCheckPubMsgNoAck(t, c, r, "foo", expectedFlags, []byte("data1")) if pi != subPI { t.Fatalf("Expected pi to be %v, got %v", subPI, pi) } } // Finish the exchange and make sure there are no more attempts. testMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI) testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI) testMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI) testMQTTExpectNothing(t, r) } func TestMQTTQoS2RetriesPubRel(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = 50 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) var qos2 byte = 2 cisub := &mqttConnInfo{clientID: "sub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: qos2}}, []byte{qos2}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) // Publish a message and close the pub connection. var pubPI uint16 = 444 testMQTTPublish(t, cp, rp, qos2, false, false, "foo", pubPI, []byte("data1")) testMQTTDisconnect(t, cp, nil) cp.Close() // See that we got the message delivered to the sub, PUBREC it and expect a // PUBREL from the server. subPI := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQoS2, []byte("data1")) testMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI) // See that we get PUBREL redelivered several times, there's no DUP flag to // check. testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI) testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI) testMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI) // Finish the exchange and make sure there are no more attempts. testMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI) testMQTTExpectNothing(t, r) } func TestMQTTAckWaitConfigChange(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = 250 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) sendMsg := func(topic, payload string) { t.Helper() cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, topic, 1, []byte(payload)) testMQTTDisconnect(t, cp, nil) cp.Close() } sendMsg("foo", "msg1") for i := 0; i < 2; i++ { flags := mqttPubQos1 if i > 0 { flags |= mqttPubFlagDup } testMQTTCheckPubMsgNoAck(t, c, r, "foo", flags, []byte("msg1")) } // Restart the server with a different AckWait option value. // Verify that MQTT sub restart succeeds. It will keep the // original value. c.Close() s.Shutdown() o.Port = -1 o.MQTT.Port = -1 o.MQTT.AckWait = 10 * time.Millisecond o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1|mqttPubFlagDup, []byte("msg1")) start := time.Now() testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1|mqttPubFlagDup, []byte("msg1")) if dur := time.Since(start); dur < 200*time.Millisecond { t.Fatalf("AckWait seem to have changed for existing subscription: %v", dur) } // Create new subscription testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) sendMsg("bar", "msg2") testMQTTCheckPubMsgNoAck(t, c, r, "bar", mqttPubQos1, []byte("msg2")) start = time.Now() testMQTTCheckPubMsgNoAck(t, c, r, "bar", mqttPubQos1|mqttPubFlagDup, []byte("msg2")) if dur := time.Since(start); dur > 50*time.Millisecond { t.Fatalf("AckWait new value not used by new sub: %v", dur) } c.Close() } func TestMQTTUnsubscribeWithPendingAcks(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = 250 * time.Millisecond s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg")) testMQTTDisconnect(t, cp, nil) cp.Close() for i := 0; i < 2; i++ { flags := mqttPubQos1 if i > 0 { flags |= mqttPubFlagDup } testMQTTCheckPubMsgNoAck(t, c, r, "foo", flags, []byte("msg")) } testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "foo"}}) testMQTTFlush(t, c, nil, r) mc := testMQTTGetClient(t, s, "sub") mc.mu.Lock() sess := mc.mqtt.sess sess.mu.Lock() pal := len(sess.pendingPublish) sess.mu.Unlock() mc.mu.Unlock() if pal != 0 { t.Fatalf("Expected pending ack map to be empty, got %v", pal) } } func TestMQTTMaxAckPending(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.MaxAckPending = 1 s := testMQTTRunServer(t, o) defer testMQTTShutdownRestartedServer(&s) nc, js := jsClientConnect(t, s) defer nc.Close() dir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg2")) pi := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg1")) // Check that we don't receive the second one due to max ack pending testMQTTExpectNothing(t, r) // Now ack first message testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) // Now we should receive message 2 testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1, []byte("msg2")) testMQTTDisconnect(t, c, nil) // Give a chance to the server to "close" the consumer checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { for ci := range js.ConsumersInfo("$MQTT_msgs") { if ci.PushBound { return fmt.Errorf("Consumer still connected") } } return nil }) // Send 2 messages while sub is offline testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg3")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg4")) // Restart consumer c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Should receive only message 3 pi = testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg3")) testMQTTExpectNothing(t, r) // Ack and get the next testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1, []byte("msg4")) // Make sure this message gets ack'ed mcli := testMQTTGetClient(t, s, cisub.clientID) checkFor(t, time.Second, 15*time.Millisecond, func() error { mcli.mu.Lock() sess := mcli.mqtt.sess sess.mu.Lock() np := len(sess.pendingPublish) sess.mu.Unlock() mcli.mu.Unlock() if np != 0 { return fmt.Errorf("Still %v pending messages", np) } return nil }) // Check that change to config does not prevent restart of sub. cp.Close() c.Close() s.Shutdown() o.Port = -1 o.MQTT.Port = -1 o.MQTT.MaxAckPending = 2 o.StoreDir = dir s = testMQTTRunServer(t, o) // There is already the defer for shutdown at top of function cp, rp = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg5")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg6")) // Restart consumer c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true) // Should receive only message 5 pi = testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg5")) testMQTTExpectNothing(t, r) // Ack and get the next testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1, []byte("msg6")) } func TestMQTTMaxAckPendingForMultipleSubs(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.AckWait = time.Second o.MQTT.MaxAckPending = 1 s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: true} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) pi := testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg1")) // Now send a second message but on topic bar testMQTTPublish(t, cp, rp, 1, false, false, "bar", 1, []byte("msg2")) // JS allows us to limit per consumer, but we apply the limit to the // session, so although JS will attempt to delivery this message, // the MQTT code will suppress it. testMQTTExpectNothing(t, r) // Ack the first message. testMQTTSendPIPacket(mqttPacketPubAck, t, c, pi) // Now we should get the second message testMQTTCheckPubMsg(t, c, r, "bar", mqttPubQos1|mqttPubFlagDup, []byte("msg2")) } func TestMQTTMaxAckPendingOverLimit(t *testing.T) { o := testMQTTDefaultOptions() o.MQTT.MaxAckPending = 20000 s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) checkTMax := func(sess *mqttSession, expected int) { t.Helper() sess.mu.Lock() tmax := sess.tmaxack sess.mu.Unlock() if tmax != expected { t.Fatalf("Expected current tmax to be %v, got %v", expected, tmax) } } cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) mc := testMQTTGetClient(t, s, "sub") mc.mu.Lock() sess := mc.mqtt.sess mc.mu.Unlock() // After this one, total would be 20000 testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) checkTMax(sess, 20000) // This one will count for 2, so total will be 60000 testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar/#", qos: 1}}, []byte{1}) checkTMax(sess, 60000) // This should fail testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{mqttSubAckFailure}) checkTMax(sess, 60000) // Remove the one with wildcard testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "bar/#"}}) checkTMax(sess, 20000) // Now we could add 2 more without wildcards testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) checkTMax(sess, 40000) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "baz", qos: 1}}, []byte{1}) checkTMax(sess, 60000) // Again, this one should fail testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bat", qos: 1}}, []byte{mqttSubAckFailure}) checkTMax(sess, 60000) // Now remove all and check that we are at 0 testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "foo"}}) checkTMax(sess, 40000) testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "bar"}}) checkTMax(sess, 20000) testMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: "baz"}}) checkTMax(sess, 0) } func TestMQTTConfigReload(t *testing.T) { template := ` jetstream: true server_name: mqtt mqtt { port: -1 ack_wait: %s max_ack_pending: %s } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, `"5s"`, `10000`))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) if val := o.MQTT.AckWait; val != 5*time.Second { t.Fatalf("Invalid ackwait: %v", val) } if val := o.MQTT.MaxAckPending; val != 10000 { t.Fatalf("Invalid ackwait: %v", val) } changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `"250ms"`, `1`))) if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) cipub := &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg2")) testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg1")) start := time.Now() testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1|mqttPubFlagDup, []byte("msg1")) if dur := time.Since(start); dur > 500*time.Millisecond { t.Fatalf("AckWait not applied? dur=%v", dur) } c.Close() cp.Close() testMQTTShutdownServer(s) changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `"30s"`, `1`))) s, o = RunServerWithConfig(conf) defer testMQTTShutdownServer(s) cisub.cleanSess = true c, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) cipub = &mqttConnInfo{clientID: "pub", cleanSess: true} cp, rp = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg2")) testMQTTCheckPubMsgNoAck(t, c, r, "foo", mqttPubQos1, []byte("msg1")) testMQTTExpectNothing(t, r) // Increase the max ack pending changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `"30s"`, `10`))) // Reload now if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } // Reload will have effect only on new subscriptions. // Create a new subscription, and we should not be able to get the 2 messages. testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTPublish(t, cp, rp, 1, false, false, "bar", 1, []byte("msg3")) testMQTTPublish(t, cp, rp, 1, false, false, "bar", 1, []byte("msg4")) testMQTTCheckPubMsg(t, c, r, "bar", mqttPubQos1, []byte("msg3")) testMQTTCheckPubMsg(t, c, r, "bar", mqttPubQos1, []byte("msg4")) } func TestMQTTStreamInfoReturnsNonEmptySubject(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) nc := natsConnect(t, s.ClientURL()) defer nc.Close() // Check that we can query all MQTT streams. MQTT streams are // created without subject filter, however, if we return them like this, // the 'nats' utility will fail to display them due to some xml validation. for _, sname := range []string{ mqttStreamName, mqttRetainedMsgsStreamName, } { t.Run(sname, func(t *testing.T) { resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, sname), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var bResp JSApiStreamInfoResponse if err = json.Unmarshal(resp.Data, &bResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(bResp.Config.Subjects) == 0 { t.Fatalf("No subject returned, which will cause nats tooling to fail: %+v", bResp.Config) } }) } } func TestMQTTWebsocketToMQTTPort(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } mqtt { listen: "127.0.0.1:-1" } websocket { listen: "127.0.0.1:-1" no_tls: true } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) l := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(l, false, false) ci := &mqttConnInfo{cleanSess: true, ws: true} if _, _, err := testMQTTConnectRetryWithError(t, ci, o.MQTT.Host, o.MQTT.Port, 0); err == nil { t.Fatal("Expected error during connect") } select { case e := <-l.errCh: if !strings.Contains(e, errMQTTNotWebsocketPort.Error()) { t.Fatalf("Unexpected error: %v", e) } case <-time.After(time.Second): t.Fatal("No error regarding wrong port") } } func TestMQTTWebsocket(t *testing.T) { tdir := t.TempDir() template := ` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } accounts { MQTT { jetstream: enabled users [ {user: "mqtt", pass: "pwd", connection_types: ["%s"%s]} ] } } mqtt { listen: "127.0.0.1:-1" } websocket { listen: "127.0.0.1:-1" no_tls: true } ` s, o, conf := runReloadServerWithContent(t, []byte(fmt.Sprintf(template, tdir, jwt.ConnectionTypeMqtt, ""))) defer testMQTTShutdownServer(s) cisub := &mqttConnInfo{clientID: "sub", user: "mqtt", pass: "pwd", ws: true} c, r := testMQTTConnect(t, cisub, o.Websocket.Host, o.Websocket.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCNotAuthorized, false) c.Close() ws := fmt.Sprintf(`, "%s"`, jwt.ConnectionTypeMqttWS) reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, tdir, jwt.ConnectionTypeMqtt, ws)) cisub = &mqttConnInfo{clientID: "sub", user: "mqtt", pass: "pwd", ws: true} c, r = testMQTTConnect(t, cisub, o.Websocket.Host, o.Websocket.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) cipub := &mqttConnInfo{clientID: "pub", user: "mqtt", pass: "pwd", ws: true} cp, rp := testMQTTConnect(t, cipub, o.Websocket.Host, o.Websocket.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1, []byte("msg1")) } type chunkWriteConn struct { net.Conn } func (cwc *chunkWriteConn) Write(p []byte) (int, error) { max := len(p) cs := rand.Intn(max) + 1 if cs < max { if pn, perr := cwc.Conn.Write(p[:cs]); perr != nil { return pn, perr } time.Sleep(10 * time.Millisecond) if pn, perr := cwc.Conn.Write(p[cs:]); perr != nil { return pn, perr } return len(p), nil } return cwc.Conn.Write(p) } func TestMQTTPartial(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } mqtt { listen: "127.0.0.1:-1" } websocket { listen: "127.0.0.1:-1" no_tls: true } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) for _, test := range []struct { name string ws bool }{ {"standard", false}, {"websocket", true}, } { t.Run(test.name, func(t *testing.T) { ci := &mqttConnInfo{cleanSess: true, ws: test.ws} host, port := o.MQTT.Host, o.MQTT.Port if test.ws { host, port = o.Websocket.Host, o.Websocket.Port } c, r := testMQTTConnect(t, ci, host, port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) c = &chunkWriteConn{Conn: c} cp, rp := testMQTTConnect(t, ci, host, port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) cp = &chunkWriteConn{Conn: cp} subj := nuid.Next() testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: subj, qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) for i := 0; i < 10; i++ { testMQTTPublish(t, cp, rp, 1, false, false, subj, 1, []byte("msg")) testMQTTCheckPubMsg(t, c, r, subj, mqttPubQos1, []byte("msg")) } }) } } func TestMQTTWebsocketTLS(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } mqtt { listen: "127.0.0.1:-1" } websocket { listen: "127.0.0.1:-1" tls { cert_file: '../test/configs/certs/server-cert.pem' key_file: '../test/configs/certs/server-key.pem' ca_file: '../test/configs/certs/ca.pem' } } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) c, r := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", ws: true, tls: true}, o.Websocket.Host, o.Websocket.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c, nil, r) cp, rp := testMQTTConnect(t, &mqttConnInfo{clientID: "pub", ws: true, tls: true}, o.Websocket.Host, o.Websocket.Port) defer cp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, cp, rp, 1, false, false, "foo", 1, []byte("msg1")) testMQTTCheckPubMsg(t, c, r, "foo", mqttPubQos1, []byte("msg1")) } func TestMQTTTransferSessionStreamsToMuxed(t *testing.T) { cl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), "MQTT", 3) defer cl.shutdown() nc, js := jsClientConnect(t, cl.randomServer()) defer nc.Close() // Create 2 streams that start with "$MQTT_sess_" to check for transfer to new // mux'ed unique "$MQTT_sess" stream. One of this stream will not contain a // proper session record, and we will check that the stream does not get deleted. sessStreamName1 := mqttSessionsStreamNamePrefix + getHash("sub") if _, err := js.AddStream(&nats.StreamConfig{ Name: sessStreamName1, Subjects: []string{sessStreamName1}, Replicas: 3, MaxMsgs: 1, }); err != nil { t.Fatalf("Unable to add stream: %v", err) } // Then add the session record ps := mqttPersistedSession{ ID: "sub", Subs: map[string]byte{"foo": 1}, Cons: map[string]*ConsumerConfig{"foo": { Durable: "d6INCtp3_cK39H5WHEtOSU7sLy2oQv3", DeliverSubject: "$MQTT.sub.cK39H5WHEtOSU7sLy2oQrR", DeliverPolicy: DeliverNew, AckPolicy: AckExplicit, FilterSubject: "$MQTT.msgs.foo", MaxAckPending: 1024, }}, } b, _ := json.Marshal(&ps) if _, err := js.Publish(sessStreamName1, b); err != nil { t.Fatalf("Error on publish: %v", err) } // Create the stream that has "$MQTT_sess_" prefix, but that is not really a MQTT session stream sessStreamName2 := mqttSessionsStreamNamePrefix + "ivan" if _, err := js.AddStream(&nats.StreamConfig{ Name: sessStreamName2, Subjects: []string{sessStreamName2}, Replicas: 3, MaxMsgs: 1, }); err != nil { t.Fatalf("Unable to add stream: %v", err) } if _, err := js.Publish(sessStreamName2, []byte("some content")); err != nil { t.Fatalf("Error on publish: %v", err) } cl.waitOnStreamLeader(globalAccountName, sessStreamName1) cl.waitOnStreamLeader(globalAccountName, sessStreamName2) // Now create a real MQTT connection o := cl.opts[0] sc, sr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub"}, o.MQTT.Host, o.MQTT.Port, 10) defer sc.Close() testMQTTCheckConnAck(t, sr, mqttConnAckRCConnectionAccepted, true) // Check that old session stream is gone, but the non session stream is still present. var gotIt = false for info := range js.StreamsInfo() { if strings.HasPrefix(info.Config.Name, mqttSessionsStreamNamePrefix) { if strings.HasSuffix(info.Config.Name, "_ivan") { gotIt = true } else { t.Fatalf("The stream %q should have been deleted", info.Config.Name) } } } if !gotIt { t.Fatalf("The stream %q should not have been deleted", mqttSessionsStreamNamePrefix+"ivan") } // We want to check that the record was properly transferred. rmsg, err := js.GetMsg(mqttSessStreamName, 2) if err != nil { t.Fatalf("Unable to get session message: %v", err) } ps2 := &mqttPersistedSession{} if err := json.Unmarshal(rmsg.Data, ps2); err != nil { t.Fatalf("Error unpacking session record: %v", err) } if ps2.ID != "sub" { t.Fatalf("Unexpected session record, %+v vs %+v", ps2, ps) } if qos, ok := ps2.Subs["foo"]; !ok || qos != 1 { t.Fatalf("Unexpected session record, %+v vs %+v", ps2, ps) } if cons, ok := ps2.Cons["foo"]; !ok || !reflect.DeepEqual(cons, ps.Cons["foo"]) { t.Fatalf("Unexpected session record, %+v vs %+v", ps2, ps) } // Make sure we don't attempt to transfer again by creating a subscription // on the "stream names" API, which is used to get the list of streams to transfer sub := natsSubSync(t, nc, JSApiStreams) // Make sure to connect an MQTT client from a different node so that this node // gets a connection for the account for the first time and tries to create // all MQTT streams, etc.. o = cl.opts[1] sc, sr = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "sub2"}, o.MQTT.Host, o.MQTT.Port, 10) defer sc.Close() testMQTTCheckConnAck(t, sr, mqttConnAckRCConnectionAccepted, false) if _, err := sub.NextMsg(200 * time.Millisecond); err == nil { t.Fatal("Looks like attempt to transfer was done again") } } func TestMQTTConnectAndDisconnectEvent(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" http: "127.0.0.1:-1" server_name: "mqtt" jetstream { store_dir = %q } accounts { MQTT { jetstream: enabled users: [{user: "mqtt", password: "pwd"}] } SYS { users: [{user: "sys", password: "pwd"}] } } mqtt { listen: "127.0.0.1:-1" } system_account: "SYS" `, t.TempDir()))) defer os.Remove(conf) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL(), nats.UserInfo("sys", "pwd")) defer nc.Close() accConn := natsSubSync(t, nc, fmt.Sprintf(connectEventSubj, "MQTT")) accDisc := natsSubSync(t, nc, fmt.Sprintf(disconnectEventSubj, "MQTT")) accAuth := natsSubSync(t, nc, fmt.Sprintf(authErrorEventSubj, s.ID())) natsFlush(t, nc) checkConnEvent := func(data []byte, expected string) { t.Helper() var ce ConnectEventMsg json.Unmarshal(data, &ce) if ce.Client.MQTTClient != expected { t.Fatalf("Expected client ID %q, got this connect event: %+v", expected, ce) } } checkDiscEvent := func(data []byte, expected string) { t.Helper() var de DisconnectEventMsg json.Unmarshal(data, &de) if de.Client.MQTTClient != expected { t.Fatalf("Expected client ID %q, got this disconnect event: %+v", expected, de) } } c1, r1 := testMQTTConnect(t, &mqttConnInfo{user: "mqtt", pass: "pwd", clientID: "conn1", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) cm := natsNexMsg(t, accConn, time.Second) checkConnEvent(cm.Data, "conn1") c2, r2 := testMQTTConnect(t, &mqttConnInfo{user: "mqtt", pass: "pwd", clientID: "conn2", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) cm = natsNexMsg(t, accConn, time.Second) checkConnEvent(cm.Data, "conn2") c3, r3 := testMQTTConnect(t, &mqttConnInfo{user: "mqtt", pass: "pwd", clientID: "conn3", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c3.Close() testMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false) cm = natsNexMsg(t, accConn, time.Second) checkConnEvent(cm.Data, "conn3") testMQTTDisconnect(t, c3, nil) cm = natsNexMsg(t, accDisc, time.Second) checkDiscEvent(cm.Data, "conn3") // Now try a bad auth c4, r4 := testMQTTConnect(t, &mqttConnInfo{clientID: "conn4", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c4.Close() testMQTTCheckConnAck(t, r4, mqttConnAckRCNotAuthorized, false) // This will generate an auth error, which is a disconnect event cm = natsNexMsg(t, accAuth, time.Second) checkDiscEvent(cm.Data, "conn4") url := fmt.Sprintf("http://127.0.0.1:%d/", s.MonitorAddr().Port) for mode := 0; mode < 2; mode++ { c := pollConz(t, s, mode, url+"connz", nil) if c.Conns == nil || len(c.Conns) != 3 { t.Fatalf("Expected 3 connections in array, got %v", len(c.Conns)) } // Check that client ID is present for _, conn := range c.Conns { if conn.Type == clientTypeStringMap[MQTT] && conn.MQTTClient == _EMPTY_ { t.Fatalf("Expected a client ID to be set, got %+v", conn) } } // Check that we can select based on client ID: c = pollConz(t, s, mode, url+"connz?mqtt_client=conn2", &ConnzOptions{MQTTClient: "conn2"}) if c.Conns == nil || len(c.Conns) != 1 { t.Fatalf("Expected 1 connection in array, got %v", len(c.Conns)) } if c.Conns[0].MQTTClient != "conn2" { t.Fatalf("Unexpected client ID: %+v", c.Conns[0]) } // Check that we have the closed ones c = pollConz(t, s, mode, url+"connz?state=closed", &ConnzOptions{State: ConnClosed}) if c.Conns == nil || len(c.Conns) != 2 { t.Fatalf("Expected 2 connections in array, got %v", len(c.Conns)) } for _, conn := range c.Conns { if conn.MQTTClient == _EMPTY_ { t.Fatalf("Expected a client ID, got %+v", conn) } } // Check that we can select with client ID for closed state c = pollConz(t, s, mode, url+"connz?state=closed&mqtt_client=conn3", &ConnzOptions{State: ConnClosed, MQTTClient: "conn3"}) if c.Conns == nil || len(c.Conns) != 1 { t.Fatalf("Expected 1 connection in array, got %v", len(c.Conns)) } if c.Conns[0].MQTTClient != "conn3" { t.Fatalf("Unexpected client ID: %+v", c.Conns[0]) } // Check that we can select with client ID for closed state (but in this case not found) c = pollConz(t, s, mode, url+"connz?state=closed&mqtt_client=conn5", &ConnzOptions{State: ConnClosed, MQTTClient: "conn5"}) if len(c.Conns) != 0 { t.Fatalf("Expected 0 connection in array, got %v", len(c.Conns)) } } reply := nc.NewRespInbox() replySub := natsSubSync(t, nc, reply) // Test system events now for _, test := range []struct { opt any cid string }{ {&ConnzOptions{MQTTClient: "conn1"}, "conn1"}, {&ConnzOptions{MQTTClient: "conn3", State: ConnClosed}, "conn3"}, {&ConnzOptions{MQTTClient: "conn4", State: ConnClosed}, "conn4"}, {&ConnzOptions{MQTTClient: "conn5"}, _EMPTY_}, {json.RawMessage(`{"mqtt_client":"conn1"}`), "conn1"}, {json.RawMessage(fmt.Sprintf(`{"mqtt_client":"conn3", "state":%v}`, ConnClosed)), "conn3"}, {json.RawMessage(fmt.Sprintf(`{"mqtt_client":"conn4", "state":%v}`, ConnClosed)), "conn4"}, {json.RawMessage(`{"mqtt_client":"conn5"}`), _EMPTY_}, } { t.Run("sys connz", func(t *testing.T) { b, _ := json.Marshal(test.opt) // set a header to make sure request parsing knows to ignore them nc.PublishMsg(&nats.Msg{ Subject: fmt.Sprintf("%s.CONNZ", serverStatsPingReqSubj), Reply: reply, Data: b, }) msg := natsNexMsg(t, replySub, time.Second) var response ServerAPIResponse if err := json.Unmarshal(msg.Data, &response); err != nil { t.Fatalf("Error unmarshalling response json: %v", err) } tmp, _ := json.Marshal(response.Data) cz := &Connz{} if err := json.Unmarshal(tmp, cz); err != nil { t.Fatalf("Error unmarshalling connz: %v", err) } if test.cid == _EMPTY_ { if len(cz.Conns) != 0 { t.Fatalf("Expected no connections, got %v", len(cz.Conns)) } return } if len(cz.Conns) != 1 { t.Fatalf("Expected single connection, got %v", len(cz.Conns)) } conn := cz.Conns[0] if conn.MQTTClient != test.cid { t.Fatalf("Expected client ID %q, got %q", test.cid, conn.MQTTClient) } }) } } func TestMQTTClientIDInLogStatements(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) l := &captureDebugLogger{dbgCh: make(chan string, 10)} s.SetLogger(l, true, false) cisub := &mqttConnInfo{clientID: "my_client_id", cleanSess: false} c, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTDisconnect(t, c, nil) c.Close() tm := time.NewTimer(2 * time.Second) var connected bool var disconnected bool for { select { case dl := <-l.dbgCh: if strings.Contains(dl, "my_client_id") { if strings.Contains(dl, "Client connected") { connected = true } else if strings.Contains(dl, "Client connection closed") { disconnected = true } if connected && disconnected { // OK! return } } case <-tm.C: t.Fatal("Did not get the debug statements or client_id in them") } } } func TestMQTTStreamReplicasOverride(t *testing.T) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 stream_replicas: 3 } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(t, conf, "MQTT", 3) defer cl.shutdown() connectAndCheck := func(restarted bool) { t.Helper() o := cl.opts[0] mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, restarted) nc, js := jsClientConnect(t, cl.servers[2]) defer nc.Close() streams := []string{mqttStreamName, mqttRetainedMsgsStreamName, mqttSessStreamName} for _, sn := range streams { si, err := js.StreamInfo(sn) require_NoError(t, err) if n := len(si.Cluster.Replicas); n != 2 { t.Fatalf("Expected stream %q to have 2 replicas, got %v", sn, n) } } } connectAndCheck(false) cl.stopAll() for _, o := range cl.opts { o.MQTT.StreamReplicas = 2 } cl.restartAllSamePorts() cl.waitOnStreamLeader(globalAccountName, mqttStreamName) cl.waitOnStreamLeader(globalAccountName, mqttRetainedMsgsStreamName) cl.waitOnStreamLeader(globalAccountName, mqttSessStreamName) l := &captureWarnLogger{warn: make(chan string, 10)} cl.servers[0].SetLogger(l, false, false) connectAndCheck(true) select { case w := <-l.warn: if !strings.Contains(w, "current is 3 but configuration is 2") { t.Fatalf("Unexpected warning: %q", w) } case <-time.After(2 * time.Second): t.Fatal("Should have warned against replicas mismatch") } } func TestMQTTStreamReplicasConfigReload(t *testing.T) { tdir := t.TempDir() tmpl := ` jetstream { store_dir = %q } server_name: mqtt mqtt { port: -1 stream_replicas: %v } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, 3))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) l := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(l, false, false) _, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{clientID: "mqtt", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 0) if err == nil { t.Fatal("Expected to fail, did not") } select { case e := <-l.errCh: if !strings.Contains(e, NewJSStreamReplicasNotSupportedError().Description) { t.Fatalf("Expected error regarding replicas, got %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Did not get the error regarding replicas count") } reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, 1)) mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: "mqtt", cleanSess: false}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) } func TestMQTTStreamReplicasInsufficientResources(t *testing.T) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 stream_replicas: 5 } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(t, conf, "MQTT", 3) defer cl.shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} for _, s := range cl.servers { s.SetLogger(l, false, false) } o := cl.opts[1] _, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{clientID: "mqtt", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 0) if err == nil { t.Fatal("Expected to fail, did not") } select { case e := <-l.errCh: if !strings.Contains(e, fmt.Sprintf("%d", NewJSClusterNoPeersError(errors.New("")).ErrCode)) { t.Fatalf("Expected error regarding no peers error, got %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Did not get the error regarding replicas count") } } func TestMQTTConsumerReplicasValidate(t *testing.T) { o := testMQTTDefaultOptions() for _, test := range []struct { name string sr int cr int err bool }{ {"stream replicas neg", -1, 3, false}, {"stream replicas 0", 0, 3, false}, {"consumer replicas neg", 0, -1, false}, {"consumer replicas 0", -1, 0, false}, {"consumer replicas too high", 1, 2, true}, } { t.Run(test.name, func(t *testing.T) { o.MQTT.StreamReplicas = test.sr o.MQTT.ConsumerReplicas = test.cr err := validateMQTTOptions(o) if test.err { if err == nil { t.Fatal("Expected error, did not get one") } if !strings.Contains(err.Error(), "cannot be higher") { t.Fatalf("Unexpected error: %v", err) } // OK return } else if err != nil { t.Fatalf("Unexpected error: %v", err) } }) } } // This was ill-advised since the messages stream is currently interest policy. // Interest policy streams require consumers match the replica count. // Will leave her for now to make sure we do not override. func TestMQTTConsumerReplicasOverride(t *testing.T) { conf := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } mqtt { listen: 127.0.0.1:-1 stream_replicas: 5 consumer_replicas: 1 consumer_memory_storage: true } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` cl := createJetStreamClusterWithTemplate(t, conf, "MQTT", 5) defer cl.shutdown() connectAndCheck := func(subject string, restarted bool) { t.Helper() o := cl.opts[0] mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, restarted) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: subject, qos: 1}}, []byte{1}) nc, js := jsClientConnect(t, cl.servers[2]) defer nc.Close() for ci := range js.ConsumersInfo(mqttStreamName) { if ci.Config.FilterSubject == mqttStreamSubjectPrefix+subject { if rf := len(ci.Cluster.Replicas) + 1; rf != 5 { t.Fatalf("Expected consumer to be R5, got: %d", rf) } } } } connectAndCheck("foo", false) cl.stopAll() for _, o := range cl.opts { o.MQTT.ConsumerReplicas = 2 o.MQTT.ConsumerMemoryStorage = false } cl.restartAllSamePorts() cl.waitOnStreamLeader(globalAccountName, mqttStreamName) cl.waitOnStreamLeader(globalAccountName, mqttRetainedMsgsStreamName) cl.waitOnStreamLeader(globalAccountName, mqttSessStreamName) connectAndCheck("bar", true) } func TestMQTTConsumerMemStorageReload(t *testing.T) { tdir := t.TempDir() tmpl := ` jetstream { store_dir = %q } server_name: mqtt mqtt { port: -1 consumer_memory_storage: %s } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, "false"))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) l := &captureErrorLogger{errCh: make(chan string, 10)} s.SetLogger(l, false, false) c, r := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: false}, o.MQTT.Host, o.MQTT.Port) defer c.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, "true")) testMQTTSub(t, 1, c, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) mset, err := s.GlobalAccount().lookupStream(mqttStreamName) if err != nil { t.Fatalf("Error looking up stream: %v", err) } var cons *consumer mset.mu.RLock() for _, c := range mset.consumers { cons = c break } mset.mu.RUnlock() cons.mu.RLock() st := cons.store.Type() cons.mu.RUnlock() if st != MemoryStorage { t.Fatalf("Expected storage %v, got %v", MemoryStorage, st) } } // Test for auto-cleanup of consumers. func TestMQTTConsumerInactiveThreshold(t *testing.T) { tdir := t.TempDir() tmpl := ` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } mqtt { listen: 127.0.0.1:-1 consumer_inactive_threshold: %q } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, "0.2s"))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) nc, js := jsClientConnect(t, s) defer nc.Close() ci := <-js.ConsumersInfo("$MQTT_msgs") // Make sure we clean up this consumer. mc.Close() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { _, err := js.ConsumerInfo(ci.Stream, ci.Name) if err == nil { return fmt.Errorf("Consumer still present") } return nil }) // Check reload. // We will not redo existing consumers however. reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, "22s")) if opts := s.getOpts(); opts.MQTT.ConsumerInactiveThreshold != 22*time.Second { t.Fatalf("Expected reloaded value of %v but got %v", 22*time.Second, opts.MQTT.ConsumerInactiveThreshold) } } func TestMQTTSubjectMapping(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } mappings = { foo0: bar0 foo1: bar1 } mqtt { listen: 127.0.0.1:-1 } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) nc, js := jsClientConnect(t, s) defer nc.Close() for _, qos := range []byte{0, 1} { t.Run(fmt.Sprintf("qos%d", qos), func(t *testing.T) { mc, r := testMQTTConnect(t, &mqttConnInfo{clientID: "sub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() bar := fmt.Sprintf("bar%v", qos) foo := fmt.Sprintf("foo%v", qos) testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: bar, qos: qos}}, []byte{qos}) testMQTTFlush(t, mc, nil, r) mcp, rp := testMQTTConnect(t, &mqttConnInfo{clientID: "pub", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) natsPub(t, nc, foo, []byte("msg0")) testMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte("msg0")) testMQTTPublish(t, mcp, rp, 0, false, false, foo, 0, []byte("msg1")) testMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte("msg1")) testMQTTPublish(t, mcp, rp, 0, false, true, foo, 0, []byte("msg1_retained")) testMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte("msg1_retained")) testMQTTPublish(t, mcp, rp, 1, false, false, foo, 1, []byte("msg2")) // For the receiving side, the expected QoS is based on the subscription's QoS, // not the publisher. expected := byte(0) if qos == 1 { expected = mqttPubQos1 } testMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte("msg2")) testMQTTPublish(t, mcp, rp, 1, false, true, foo, 1, []byte("msg2_retained")) testMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte("msg2_retained")) testMQTTDisconnect(t, mcp, nil) // Try the with the "will" with QoS0 first mcp, rp = testMQTTConnect(t, &mqttConnInfo{ clientID: "pub", cleanSess: true, will: &mqttWill{topic: []byte(foo), qos: 0, message: []byte("willmsg1")}}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTFlush(t, mcp, nil, rp) // Close the connection without proper disconnect for will to be sent mcp.Close() testMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte("willmsg1")) // Try the with the "will" with QoS1 now mcp, rp = testMQTTConnect(t, &mqttConnInfo{ clientID: "pub", cleanSess: true, will: &mqttWill{topic: []byte(foo), qos: 1, message: []byte("willmsg2")}}, o.MQTT.Host, o.MQTT.Port) defer mcp.Close() testMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false) testMQTTFlush(t, mcp, nil, rp) // Close the connection without proper disconnect for will to be sent mcp.Close() testMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte("willmsg2")) si, err := js.StreamInfo("$MQTT_msgs") require_NoError(t, err) if qos == 0 { require_True(t, si.State.Msgs == 0) require_True(t, si.State.NumSubjects == 0) } else { // Number of QoS1 messages: 1 regular, 1 retained, 1 from will. require_True(t, si.State.Msgs == 3) require_True(t, si.State.NumSubjects == 1) } testMQTTDisconnect(t, mc, nil) }) } } func TestMQTTSubjectMappingWithImportExport(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } accounts { A { users = [{user: "a", password: "pwd"}] mappings = { bar: foo } exports = [{service: "bar"}] jetstream: enabled } B { users = [{user: "b", password: "pwd"}] mappings = { foo: bar } imports = [{service: {account: "A", subject: "bar"}}] jetstream: enabled } $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } mqtt { listen: 127.0.0.1:-1 } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) c1, r1 := testMQTTConnect(t, &mqttConnInfo{user: "a", pass: "pwd", clientID: "a1", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTFlush(t, c1, nil, r1) c2, r2 := testMQTTConnect(t, &mqttConnInfo{user: "a", pass: "pwd", clientID: "a2", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c2, r2, []*mqttFilter{{filter: "foo", qos: 1}}, []byte{1}) testMQTTFlush(t, c2, nil, r2) c3, r3 := testMQTTConnect(t, &mqttConnInfo{user: "a", pass: "pwd", clientID: "a3", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c3.Close() testMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c3, r3, []*mqttFilter{{filter: "bar", qos: 0}}, []byte{0}) testMQTTFlush(t, c3, nil, r3) c4, r4 := testMQTTConnect(t, &mqttConnInfo{user: "a", pass: "pwd", clientID: "a4", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c4.Close() testMQTTCheckConnAck(t, r4, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c4, r4, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, c4, nil, r4) bc, br := testMQTTConnect(t, &mqttConnInfo{user: "b", pass: "pwd", clientID: "b0", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer bc.Close() testMQTTCheckConnAck(t, br, mqttConnAckRCConnectionAccepted, false) bc1, br1 := testMQTTConnect(t, &mqttConnInfo{user: "b", pass: "pwd", clientID: "b1", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer bc1.Close() testMQTTCheckConnAck(t, br1, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, bc1, br1, []*mqttFilter{{filter: "foo", qos: 0}}, []byte{0}) testMQTTFlush(t, bc1, nil, br1) bc2, br2 := testMQTTConnect(t, &mqttConnInfo{user: "b", pass: "pwd", clientID: "b2", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer bc2.Close() testMQTTCheckConnAck(t, br2, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, bc2, br2, []*mqttFilter{{filter: "b", qos: 1}}, []byte{1}) testMQTTFlush(t, bc2, nil, br2) bc3, br3 := testMQTTConnect(t, &mqttConnInfo{user: "b", pass: "pwd", clientID: "b3", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer bc3.Close() testMQTTCheckConnAck(t, br3, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, bc3, br3, []*mqttFilter{{filter: "bar", qos: 0}}, []byte{0}) testMQTTFlush(t, bc3, nil, br3) bc4, br4 := testMQTTConnect(t, &mqttConnInfo{user: "b", pass: "pwd", clientID: "b4", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer bc4.Close() testMQTTCheckConnAck(t, br4, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, bc4, br4, []*mqttFilter{{filter: "bar", qos: 1}}, []byte{1}) testMQTTFlush(t, bc4, nil, br4) nc := natsConnect(t, fmt.Sprintf("nats://b:pwd@%s:%d", o.Host, o.Port)) defer nc.Close() natsPub(t, nc, "foo", []byte("msg0")) testMQTTCheckPubMsgNoAck(t, c1, r1, "foo", 0, []byte("msg0")) testMQTTCheckPubMsgNoAck(t, c2, r2, "foo", 0, []byte("msg0")) testMQTTExpectNothing(t, r3) testMQTTExpectNothing(t, r4) testMQTTExpectNothing(t, br1) testMQTTExpectNothing(t, br2) testMQTTCheckPubMsgNoAck(t, bc3, br3, "bar", 0, []byte("msg0")) testMQTTCheckPubMsgNoAck(t, bc4, br4, "bar", 0, []byte("msg0")) testMQTTPublish(t, bc, br, 0, false, false, "foo", 0, []byte("msg1")) testMQTTCheckPubMsgNoAck(t, c1, r1, "foo", 0, []byte("msg1")) testMQTTCheckPubMsgNoAck(t, c2, r2, "foo", 0, []byte("msg1")) testMQTTExpectNothing(t, r3) testMQTTExpectNothing(t, r4) testMQTTExpectNothing(t, br1) testMQTTExpectNothing(t, br2) testMQTTCheckPubMsgNoAck(t, bc3, br3, "bar", 0, []byte("msg1")) testMQTTCheckPubMsgNoAck(t, bc4, br4, "bar", 0, []byte("msg1")) testMQTTPublish(t, bc, br, 1, false, false, "foo", 1, []byte("msg2")) testMQTTCheckPubMsgNoAck(t, c1, r1, "foo", 0, []byte("msg2")) testMQTTCheckPubMsgNoAck(t, c2, r2, "foo", mqttPubQos1, []byte("msg2")) testMQTTExpectNothing(t, r3) testMQTTExpectNothing(t, r4) testMQTTExpectNothing(t, br1) testMQTTExpectNothing(t, br2) testMQTTCheckPubMsgNoAck(t, bc3, br3, "bar", 0, []byte("msg2")) testMQTTCheckPubMsgNoAck(t, bc4, br4, "bar", mqttPubQos1, []byte("msg2")) // Connection nc is for account B check := func(nc *nats.Conn, subj string) { t.Helper() req, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: ">"}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "$MQTT_msgs"), req, time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) require_True(t, si.State.Msgs == 1) require_True(t, si.State.NumSubjects == 1) require_True(t, si.State.Subjects[subj] == 1) } // Currently, nc is for account B check(nc, "$MQTT.msgs.bar") nc.Close() nc = natsConnect(t, fmt.Sprintf("nats://a:pwd@%s:%d", o.Host, o.Port)) defer nc.Close() check(nc, "$MQTT.msgs.foo") } func TestMQTTSubRetainedRace(t *testing.T) { o := testMQTTDefaultOptions() useCases := []struct { name string f func(t *testing.T, o *Options, subTopic, pubTopic string, QOS byte) }{ {"new top level", testMQTTNewSubRetainedRace}, {"existing top level", testMQTTNewSubWithExistingTopLevelRetainedRace}, } pubTopic := "/bar" subTopics := []string{"#", "/bar", "/+", "/#"} QOS := []byte{0, 1, 2} for _, tc := range useCases { t.Run(tc.name, func(t *testing.T) { for _, subTopic := range subTopics { t.Run(subTopic, func(t *testing.T) { for _, qos := range QOS { t.Run(fmt.Sprintf("QOS%d", qos), func(t *testing.T) { s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) tc.f(t, o, subTopic, pubTopic, qos) }) } }) } }) } } func testMQTTNewSubRetainedRace(t *testing.T, o *Options, subTopic, pubTopic string, QOS byte) { expectedFlags := (QOS << 1) | mqttPubFlagRetain payload := []byte("testmsg") pubID := nuid.Next() pubc, pubr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: pubID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 3) testMQTTCheckConnAck(t, pubr, mqttConnAckRCConnectionAccepted, false) defer testMQTTDisconnectEx(t, pubc, nil, true) defer pubc.Close() testMQTTPublish(t, pubc, pubr, QOS, false, true, pubTopic, 1, payload) subID := nuid.Next() subc, subr := testMQTTConnect(t, &mqttConnInfo{clientID: subID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(t, subr, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: subTopic, qos: QOS}}, []byte{QOS}) testMQTTCheckPubMsg(t, subc, subr, pubTopic, expectedFlags, payload) if QOS == 2 { testMQTTCheckPubAck(t, subr, mqttPacketPubRel) testMQTTSendPIPacket(mqttPacketPubComp, t, subc, 1) } testMQTTDisconnectEx(t, subc, nil, true) subc.Close() } func testMQTTNewSubWithExistingTopLevelRetainedRace(t *testing.T, o *Options, subTopic, pubTopic string, QOS byte) { expectedFlags := (QOS << 1) | mqttPubFlagRetain payload := []byte("testmsg") pubID := nuid.Next() pubc, pubr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: pubID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 3) testMQTTCheckConnAck(t, pubr, mqttConnAckRCConnectionAccepted, false) defer testMQTTDisconnectEx(t, pubc, nil, true) defer pubc.Close() subID := nuid.Next() subc, subr := testMQTTConnect(t, &mqttConnInfo{clientID: subID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(t, subr, mqttConnAckRCConnectionAccepted, false) // Clear retained messages from the prior run, and sleep a little to // ensure the change is propagated. testMQTTPublish(t, pubc, pubr, 0, false, true, pubTopic, 1, nil) time.Sleep(1 * time.Millisecond) // Subscribe to `#` first, make sure we can get get the retained message // there. It's a QOS0 sub, so expect a QOS0 message. testMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: `#`, qos: 0}}, []byte{0}) testMQTTExpectNothing(t, subr) // Publish the retained message with QOS2, see that the `#` subscriber gets it. testMQTTPublish(t, pubc, pubr, 2, false, true, pubTopic, 1, payload) testMQTTCheckPubMsg(t, subc, subr, pubTopic, 0, payload) testMQTTExpectNothing(t, subr) // Now subscribe to the topic we want to test. We should get the retained // message there. testMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: subTopic, qos: QOS}}, []byte{QOS}) testMQTTCheckPubMsg(t, subc, subr, pubTopic, expectedFlags, payload) if QOS == 2 { testMQTTCheckPubAck(t, subr, mqttPacketPubRel) testMQTTSendPIPacket(mqttPacketPubComp, t, subc, 1) } testMQTTDisconnectEx(t, subc, nil, true) subc.Close() } // Issue https://github.com/nats-io/nats-server/issues/3924 // The MQTT Server MUST NOT match Topic Filters starting with a wildcard character (# or +), // with Topic Names beginning with a $ character [MQTT-4.7.2-1] func TestMQTTSubjectWildcardStart(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } mqtt { listen: 127.0.0.1:-1 } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL()) defer nc.Close() mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) mc1, r1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) mc2, r2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) mc3, r3 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc3.Close() testMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false) // These will fail already with the bug due to messages already being queue up before the subAck. testMQTTSub(t, 1, mc1, r1, []*mqttFilter{{filter: "*", qos: 0}}, []byte{0}) testMQTTFlush(t, mc1, nil, r1) testMQTTSub(t, 1, mc2, r2, []*mqttFilter{{filter: "#", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, r2) testMQTTSub(t, 1, mc3, r3, []*mqttFilter{{filter: "*/foo", qos: 1}}, []byte{1}) testMQTTFlush(t, mc2, nil, r2) // Just as a barrier natsFlush(t, nc) // Now publish // NATS Publish msg := []byte("HELLO WORLD") natsPubReq(t, nc, "foo", _EMPTY_, msg) // Check messages received testMQTTCheckPubMsg(t, mc1, r1, "foo", 0, msg) testMQTTExpectNothing(t, r1) testMQTTCheckPubMsg(t, mc2, r2, "foo", 0, msg) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) // Anything that starts with $ is reserved against wildcard subjects like above. natsPubReq(t, nc, "$JS.foo", _EMPTY_, msg) testMQTTExpectNothing(t, r1) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) // Now do MQTT QoS-0 testMQTTPublish(t, mc, r, 0, false, false, "foo", 0, msg) testMQTTCheckPubMsg(t, mc1, r1, "foo", 0, msg) testMQTTExpectNothing(t, r1) testMQTTCheckPubMsg(t, mc2, r2, "foo", 0, msg) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) testMQTTPublish(t, mc, r, 0, false, false, "$JS/foo", 1, msg) testMQTTExpectNothing(t, r1) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) // Now do MQTT QoS-1 msg = []byte("HELLO WORLD - RETAINED") testMQTTPublish(t, mc, r, 1, false, false, "$JS/foo", 4, msg) testMQTTExpectNothing(t, r1) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) testMQTTPublish(t, mc, r, 1, false, false, "foo", 2, msg) testMQTTCheckPubMsg(t, mc1, r1, "foo", 0, msg) testMQTTExpectNothing(t, r1) testMQTTCheckPubMsg(t, mc2, r2, "foo", 2, msg) testMQTTExpectNothing(t, r2) testMQTTExpectNothing(t, r3) testMQTTPublish(t, mc, r, 1, false, false, "foo/foo", 3, msg) testMQTTExpectNothing(t, r1) testMQTTCheckPubMsg(t, mc2, r2, "foo/foo", 2, msg) testMQTTExpectNothing(t, r2) testMQTTCheckPubMsg(t, mc3, r3, "foo/foo", 2, msg) testMQTTExpectNothing(t, r3) // Make sure we did not retain the messages prefixed with $. nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo(mqttStreamName) require_NoError(t, err) require_True(t, si.State.Msgs == 0) } func TestMQTTTopicWithDot(t *testing.T) { o := testMQTTDefaultOptions() s := testMQTTRunServer(t, o) defer testMQTTShutdownServer(s) nc := natsConnect(t, s.ClientURL(), nats.UserInfo("mqtt", "pwd")) defer nc.Close() sub := natsSubSync(t, nc, "*.*") c1, r1 := testMQTTConnect(t, &mqttConnInfo{user: "mqtt", pass: "pwd", clientID: "conn1", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c1.Close() testMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: "spBv1.0/plant1", qos: 0}}, []byte{0}) testMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: "spBv1.0/plant2", qos: 1}}, []byte{1}) c2, r2 := testMQTTConnect(t, &mqttConnInfo{user: "mqtt", pass: "pwd", clientID: "conn2", cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer c2.Close() testMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, c2, r2, 0, false, false, "spBv1.0/plant1", 0, []byte("msg1")) testMQTTCheckPubMsg(t, c1, r1, "spBv1.0/plant1", 0, []byte("msg1")) msg := natsNexMsg(t, sub, time.Second) require_Equal(t, msg.Subject, "spBv1//0.plant1") testMQTTPublish(t, c2, r2, 1, false, false, "spBv1.0/plant2", 1, []byte("msg2")) testMQTTCheckPubMsg(t, c1, r1, "spBv1.0/plant2", mqttPubQos1, []byte("msg2")) msg = natsNexMsg(t, sub, time.Second) require_Equal(t, msg.Subject, "spBv1//0.plant2") natsPub(t, nc, "spBv1//0.plant1", []byte("msg3")) testMQTTCheckPubMsg(t, c1, r1, "spBv1.0/plant1", 0, []byte("msg3")) msg = natsNexMsg(t, sub, time.Second) require_Equal(t, msg.Subject, "spBv1//0.plant1") natsPub(t, nc, "spBv1//0.plant2", []byte("msg4")) testMQTTCheckPubMsg(t, c1, r1, "spBv1.0/plant2", 0, []byte("msg4")) msg = natsNexMsg(t, sub, time.Second) require_Equal(t, msg.Subject, "spBv1//0.plant2") } // Issue https://github.com/nats-io/nats-server/issues/4291 func TestMQTTJetStreamRepublishAndQoS0Subscribers(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } mqtt { listen: 127.0.0.1:-1 } `, t.TempDir()))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) nc, js := jsClientConnect(t, s) defer nc.Close() // Setup stream with republish on it. _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, RePublish: &nats.RePublish{ Source: "foo", Destination: "mqtt.foo", }, }) require_NoError(t, err) // Create QoS0 subscriber to catch re-publishes. mc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "mqtt/foo", qos: 0}}, []byte{0}) testMQTTFlush(t, mc, nil, r) msg := []byte("HELLO WORLD") _, err = js.Publish("foo", msg) require_NoError(t, err) testMQTTCheckPubMsg(t, mc, r, "mqtt/foo", 0, msg) testMQTTExpectNothing(t, r) } // Test for auto-cleanup of consumers. func TestMQTTDecodeRetainedMessage(t *testing.T) { tdir := t.TempDir() tmpl := ` listen: 127.0.0.1:-1 server_name: mqtt jetstream { store_dir = %q } mqtt { listen: 127.0.0.1:-1 consumer_inactive_threshold: %q } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, "0.2s"))) s, o := RunServerWithConfig(conf) defer testMQTTShutdownServer(s) // Connect and publish a retained message, this will be in the "newer" form, // with the metadata in the header. mc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte("msg1")) mc.Close() // Store a "legacy", JSON-encoded payload directly into the stream. nc, js := jsClientConnect(t, s) defer nc.Close() rm := mqttRetainedMsg{ Origin: "test", Subject: "foo.2", Topic: "foo/2", Flags: mqttPubFlagRetain, Msg: []byte("msg2"), } jsonData, _ := json.Marshal(rm) _, err := js.PublishMsg(&nats.Msg{ Subject: mqttRetainedMsgsStreamSubject + rm.Subject, Data: jsonData, }) if err != nil { t.Fatalf("Error publishing retained message to JS directly: %v", err) } // Restart the server to see that it picks up both retained messages on restart. s.Shutdown() s = RunServer(o) defer testMQTTShutdownServer(s) // Connect again, subscribe, and check that we get both messages. mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) for i := 0; i < 2; i++ { b, pl := testMQTTReadPacket(t, r) if pt := b & mqttPacketMask; pt != mqttPacketPub { t.Fatalf("Expected PUBLISH packet %x, got %x", mqttPacketPub, pt) } _, _, topic := testMQTTGetPubMsgExEx(t, mc, r, mqttPubFlagRetain, pl, "", nil) if string(topic) != "foo/1" && string(topic) != "foo/2" { t.Fatalf("Expected foo/1 or foo/2, got %q", topic) } } testMQTTExpectNothing(t, r) mc.Close() // Clear both retained messages. mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTPublish(t, mc, r, 0, false, true, "foo/1", 0, []byte{}) testMQTTPublish(t, mc, r, 0, false, true, "foo/2", 0, []byte{}) mc.Close() // Connect again, subscribe, and check that we get nothing. mc, r = testMQTTConnectRetry(t, &mqttConnInfo{clientID: "test", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5) defer mc.Close() testMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false) testMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: "foo/+", qos: 0}}, []byte{0}) testMQTTExpectNothing(t, r) } ////////////////////////////////////////////////////////////////////////// // // Benchmarks // ////////////////////////////////////////////////////////////////////////// const ( mqttPubSubj = "a" mqttBenchBufLen = 32768 ) func mqttBenchPubQoS0(b *testing.B, subject, payload string, numSubs int) { b.StopTimer() b.ReportAllocs() o := testMQTTDefaultOptions() s := RunServer(o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{clientID: "pub", cleanSess: true} c, br := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(b, br, mqttConnAckRCConnectionAccepted, false) w := newMQTTWriter(0) testMQTTWritePublishPacket(b, w, 0, false, false, subject, 0, []byte(payload)) sendOp := w.Bytes() dch := make(chan error, 1) totalSize := int64(len(sendOp)) cdch := 0 createSub := func(i int) { ci := &mqttConnInfo{clientID: fmt.Sprintf("sub%d", i), cleanSess: true} cs, brs := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(b, brs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(b, 1, cs, brs, []*mqttFilter{{filter: subject, qos: 0}}, []byte{0}) testMQTTFlush(b, cs, nil, brs) w := newMQTTWriter(0) varHeaderAndPayload := 2 + len(subject) + len(payload) w.WriteVarInt(varHeaderAndPayload) size := 1 + w.Len() + varHeaderAndPayload totalSize += int64(size) go func() { mqttBenchConsumeMsgQoS0(cs, int64(b.N)*int64(size), dch) cs.Close() }() } for i := 0; i < numSubs; i++ { createSub(i + 1) cdch++ } bw := bufio.NewWriterSize(c, mqttBenchBufLen) b.SetBytes(totalSize) b.StartTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } testMQTTFlush(b, c, bw, br) for i := 0; i < cdch; i++ { if e := <-dch; e != nil { b.Fatal(e.Error()) } } b.StopTimer() c.Close() s.Shutdown() } func mqttBenchConsumeMsgQoS0(c net.Conn, total int64, dch chan<- error) { var buf [mqttBenchBufLen]byte var err error var n int for size := int64(0); size < total; { n, err = c.Read(buf[:]) if err != nil { break } size += int64(n) } dch <- err } func mqttBenchPubQoS1(b *testing.B, subject, payload string, numSubs int) { b.StopTimer() o := testMQTTDefaultOptions() o.MQTT.MaxAckPending = 0xFFFF s := RunServer(o) defer testMQTTShutdownServer(s) ci := &mqttConnInfo{cleanSess: true} c, br := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(b, br, mqttConnAckRCConnectionAccepted, false) w := newMQTTWriter(0) testMQTTWritePublishPacket(b, w, 1, false, false, subject, 1, []byte(payload)) // For reported bytes we will count the PUBLISH + PUBACK (4 bytes) totalSize := int64(w.Len() + 4) w.Reset() pi := uint16(1) maxpi := uint16(60000) ppich := make(chan error, 10) dch := make(chan error, 1+numSubs) cdch := 1 // Start go routine to consume PUBACK for published QoS 1 messages. go mqttBenchConsumePubAck(c, b.N, dch, ppich) createSub := func(i int) { ci := &mqttConnInfo{clientID: fmt.Sprintf("sub%d", i), cleanSess: true} cs, brs := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port) testMQTTCheckConnAck(b, brs, mqttConnAckRCConnectionAccepted, false) testMQTTSub(b, 1, cs, brs, []*mqttFilter{{filter: subject, qos: 1}}, []byte{1}) testMQTTFlush(b, cs, nil, brs) w := newMQTTWriter(0) varHeaderAndPayload := 2 + len(subject) + 2 + len(payload) w.WriteVarInt(varHeaderAndPayload) size := 1 + w.Len() + varHeaderAndPayload // Add to the bytes reported the size of message sent to subscriber + PUBACK (4 bytes) totalSize += int64(size + 4) go func() { mqttBenchConsumeMsgQos1(cs, b.N, size, dch) cs.Close() }() } for i := 0; i < numSubs; i++ { createSub(i + 1) cdch++ } flush := func() { b.Helper() if _, err := c.Write(w.Bytes()); err != nil { b.Fatalf("Error on write: %v", err) } w.Reset() } b.SetBytes(totalSize) b.StartTimer() for i := 0; i < b.N; i++ { if pi <= maxpi { testMQTTWritePublishPacket(b, w, 1, false, false, subject, pi, []byte(payload)) pi++ if w.Len() >= mqttBenchBufLen { flush() } } else { if w.Len() > 0 { flush() } if pi > 60000 { pi = 1 maxpi = 0 } if e := <-ppich; e != nil { b.Fatal(e.Error()) } maxpi += 10000 i-- } } if w.Len() > 0 { flush() } for i := 0; i < cdch; i++ { if e := <-dch; e != nil { b.Fatal(e.Error()) } } b.StopTimer() c.Close() s.Shutdown() } func mqttBenchConsumeMsgQos1(c net.Conn, total, size int, dch chan<- error) { var buf [mqttBenchBufLen]byte pubAck := [4]byte{mqttPacketPubAck, 0x2, 0, 0} var err error var n int var pi uint16 var prev int for i := 0; i < total; { n, err = c.Read(buf[:]) if err != nil { break } n += prev for ; n >= size; n -= size { i++ pi++ pubAck[2] = byte(pi >> 8) pubAck[3] = byte(pi) if _, err = c.Write(pubAck[:4]); err != nil { dch <- err return } if pi == 60000 { pi = 0 } } prev = n } dch <- err } func mqttBenchConsumePubAck(c net.Conn, total int, dch, ppich chan<- error) { var buf [mqttBenchBufLen]byte var err error var n int var pi uint16 var prev int for i := 0; i < total; { n, err = c.Read(buf[:]) if err != nil { break } n += prev for ; n >= 4; n -= 4 { i++ pi++ if pi%10000 == 0 { ppich <- nil } if pi == 60001 { pi = 0 } } prev = n } ppich <- err dch <- err } func BenchmarkMQTT_QoS0_Pub_______0b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, "", 0) } func BenchmarkMQTT_QoS0_Pub_______8b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 0) } func BenchmarkMQTT_QoS0_Pub______32b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 0) } func BenchmarkMQTT_QoS0_Pub_____128b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 0) } func BenchmarkMQTT_QoS0_Pub_____256b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 0) } func BenchmarkMQTT_QoS0_Pub_______1K_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 0) } func BenchmarkMQTT_QoS0_PubSub1___0b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, "", 1) } func BenchmarkMQTT_QoS0_PubSub1___8b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 1) } func BenchmarkMQTT_QoS0_PubSub1__32b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 1) } func BenchmarkMQTT_QoS0_PubSub1_128b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 1) } func BenchmarkMQTT_QoS0_PubSub1_256b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 1) } func BenchmarkMQTT_QoS0_PubSub1___1K_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 1) } func BenchmarkMQTT_QoS0_PubSub2___0b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, "", 2) } func BenchmarkMQTT_QoS0_PubSub2___8b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 2) } func BenchmarkMQTT_QoS0_PubSub2__32b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 2) } func BenchmarkMQTT_QoS0_PubSub2_128b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 2) } func BenchmarkMQTT_QoS0_PubSub2_256b_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 2) } func BenchmarkMQTT_QoS0_PubSub2___1K_Payload(b *testing.B) { mqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 2) } func BenchmarkMQTT_QoS1_Pub_______0b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, "", 0) } func BenchmarkMQTT_QoS1_Pub_______8b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 0) } func BenchmarkMQTT_QoS1_Pub______32b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 0) } func BenchmarkMQTT_QoS1_Pub_____128b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 0) } func BenchmarkMQTT_QoS1_Pub_____256b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 0) } func BenchmarkMQTT_QoS1_Pub_______1K_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 0) } func BenchmarkMQTT_QoS1_PubSub1___0b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, "", 1) } func BenchmarkMQTT_QoS1_PubSub1___8b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 1) } func BenchmarkMQTT_QoS1_PubSub1__32b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 1) } func BenchmarkMQTT_QoS1_PubSub1_128b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 1) } func BenchmarkMQTT_QoS1_PubSub1_256b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 1) } func BenchmarkMQTT_QoS1_PubSub1___1K_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 1) } func BenchmarkMQTT_QoS1_PubSub2___0b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, "", 2) } func BenchmarkMQTT_QoS1_PubSub2___8b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 2) } func BenchmarkMQTT_QoS1_PubSub2__32b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 2) } func BenchmarkMQTT_QoS1_PubSub2_128b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 2) } func BenchmarkMQTT_QoS1_PubSub2_256b_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 2) } func BenchmarkMQTT_QoS1_PubSub2___1K_Payload(b *testing.B) { mqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 2) } nats-server-2.10.27/server/nkey.go000066400000000000000000000025401477524627100167570ustar00rootroot00000000000000// Copyright 2018-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( crand "crypto/rand" "encoding/base64" ) // Raw length of the nonce challenge const ( nonceRawLen = 11 nonceLen = 15 // base64.RawURLEncoding.EncodedLen(nonceRawLen) ) // NonceRequired tells us if we should send a nonce. func (s *Server) NonceRequired() bool { s.mu.Lock() defer s.mu.Unlock() return s.nonceRequired() } // nonceRequired tells us if we should send a nonce. // Lock should be held on entry. func (s *Server) nonceRequired() bool { return s.getOpts().AlwaysEnableNonce || len(s.nkeys) > 0 || s.trustedKeys != nil } // Generate a nonce for INFO challenge. // Assumes server lock is held func (s *Server) generateNonce(n []byte) { var raw [nonceRawLen]byte data := raw[:] crand.Read(data) base64.RawURLEncoding.Encode(n, data) } nats-server-2.10.27/server/nkey_test.go000066400000000000000000000202351477524627100200170ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" crand "crypto/rand" "encoding/base64" "encoding/json" "fmt" mrand "math/rand" "strings" "testing" "time" "github.com/nats-io/nkeys" ) // Nonce has to be a string since we used different encoding by default than json.Unmarshal. type nonceInfo struct { Id string `json:"server_id"` CID uint64 `json:"client_id,omitempty"` Nonce string `json:"nonce,omitempty"` } // This is a seed for a user. We can extract public and private keys from this for testing. var seed = []byte("SUAKYRHVIOREXV7EUZTBHUHL7NUMHPMAS7QMDU3GTIUWEI5LDNOXD43IZY") func nkeyBasicSetup() (*Server, *testAsyncClient, *bufio.Reader, string) { kp, _ := nkeys.FromSeed(seed) pub, _ := kp.PublicKey() opts := defaultServerOptions opts.Nkeys = []*NkeyUser{{Nkey: string(pub)}} return rawSetup(opts) } func mixedSetup() (*Server, *testAsyncClient, *bufio.Reader, string) { kp, _ := nkeys.FromSeed(seed) pub, _ := kp.PublicKey() opts := defaultServerOptions opts.Nkeys = []*NkeyUser{{Nkey: string(pub)}} opts.Users = []*User{{Username: "derek", Password: "foo"}} return rawSetup(opts) } func TestServerInfoNonceAlwaysEnabled(t *testing.T) { opts := defaultServerOptions opts.AlwaysEnableNonce = true s, c, _, l := rawSetup(opts) defer s.WaitForShutdown() defer s.Shutdown() defer c.close() if !strings.HasPrefix(l, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", l) } var info nonceInfo err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with AlwaysEnableNonce set") } } func TestServerInfoNonce(t *testing.T) { c, l := setUpClientWithResponse() defer c.close() if !strings.HasPrefix(l, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", l) } // Make sure payload is proper json var info nonceInfo err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce != "" { t.Fatalf("Expected an empty nonce with no nkeys defined") } // Now setup server with auth and nkeys to trigger nonce generation s, c, _, l := nkeyBasicSetup() defer c.close() if !strings.HasPrefix(l, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", l) } // Make sure payload is proper json err = json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } // Make sure new clients get new nonces oldNonce := info.Nonce c, _, l = newClientForServer(s) defer c.close() err = json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce") } if strings.Compare(oldNonce, info.Nonce) == 0 { t.Fatalf("Expected subsequent nonces to be different\n") } } func TestNkeyClientConnect(t *testing.T) { s, c, cr, _ := nkeyBasicSetup() defer c.close() // Send CONNECT with no signature or nkey, should fail. connectOp := "CONNECT {\"verbose\":true,\"pedantic\":true}\r\n" c.parseAsync(connectOp) l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } kp, _ := nkeys.FromSeed(seed) pubKey, _ := kp.PublicKey() // Send nkey but no signature c, cr, _ = newClientForServer(s) defer c.close() cs := fmt.Sprintf("CONNECT {\"nkey\":%q, \"verbose\":true,\"pedantic\":true}\r\n", pubKey) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } // Now improperly sign etc. c, cr, _ = newClientForServer(s) defer c.close() cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":%q,\"verbose\":true,\"pedantic\":true}\r\n", pubKey, "bad_sig") c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "-ERR ") { t.Fatalf("Expected an error") } // Now properly sign the nonce c, cr, l = newClientForServer(s) defer c.close() // Check for Nonce var info nonceInfo err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err := kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } } func TestMixedClientConnect(t *testing.T) { s, c, cr, _ := mixedSetup() defer c.close() // Normal user/pass c.parseAsync("CONNECT {\"user\":\"derek\",\"pass\":\"foo\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n") l, _ := cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } kp, _ := nkeys.FromSeed(seed) pubKey, _ := kp.PublicKey() c, cr, l = newClientForServer(s) defer c.close() // Check for Nonce var info nonceInfo err := json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err := kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs := fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } } func TestMixedClientConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` authorization { users = [ {nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV"} {user: alice, password: foo} ] }`)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error processing config file: %v", err) } if len(opts.Nkeys) != 1 { t.Fatalf("Expected 1 nkey, got %d", len(opts.Nkeys)) } if len(opts.Users) != 1 { t.Fatalf("Expected 1 user, got %d", len(opts.Users)) } } func BenchmarkCryptoRandGeneration(b *testing.B) { data := make([]byte, 16) for i := 0; i < b.N; i++ { crand.Read(data) } } func BenchmarkMathRandGeneration(b *testing.B) { data := make([]byte, 16) prng := mrand.New(mrand.NewSource(time.Now().UnixNano())) for i := 0; i < b.N; i++ { prng.Read(data) } } func BenchmarkNonceGeneration(b *testing.B) { data := make([]byte, nonceRawLen) b64 := make([]byte, nonceLen) prand := mrand.New(mrand.NewSource(time.Now().UnixNano())) for i := 0; i < b.N; i++ { prand.Read(data) base64.RawURLEncoding.Encode(b64, data) } } func BenchmarkPublicVerify(b *testing.B) { data := make([]byte, nonceRawLen) nonce := make([]byte, nonceLen) crand.Read(data) base64.RawURLEncoding.Encode(nonce, data) user, err := nkeys.CreateUser() if err != nil { b.Fatalf("Error creating User Nkey: %v", err) } sig, err := user.Sign(nonce) if err != nil { b.Fatalf("Error sigining nonce: %v", err) } pk, err := user.PublicKey() if err != nil { b.Fatalf("Could not extract public key from user: %v", err) } pub, err := nkeys.FromPublicKey(pk) if err != nil { b.Fatalf("Could not create public key pair from public key string: %v", err) } b.ResetTimer() for i := 0; i < b.N; i++ { if err := pub.Verify(nonce, sig); err != nil { b.Fatalf("Error verifying nonce: %v", err) } } } nats-server-2.10.27/server/norace_test.go000066400000000000000000011407331477524627100203270ustar00rootroot00000000000000// Copyright 2018-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !race && !skip_no_race_tests // +build !race,!skip_no_race_tests package server import ( "bufio" "bytes" "compress/gzip" "context" "encoding/binary" "encoding/json" "errors" "fmt" "io" "math" "math/rand" "net" "net/http" "net/url" "os" "path/filepath" "reflect" "runtime" "runtime/debug" "slices" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "crypto/hmac" crand "crypto/rand" "crypto/sha256" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server/avl" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) // IMPORTANT: Tests in this file are not executed when running with the -race flag. // The test name should be prefixed with TestNoRace so we can run only // those tests: go test -run=TestNoRace ... func TestNoRaceAvoidSlowConsumerBigMessages(t *testing.T) { opts := DefaultOptions() // Use defaults to make sure they avoid pending slow consumer. opts.NoSystemAccount = true s := RunServer(opts) defer s.Shutdown() nc1, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() data := make([]byte, 1024*1024) // 1MB payload crand.Read(data) expected := int32(500) received := int32(0) done := make(chan bool) // Create Subscription. nc1.Subscribe("slow.consumer", func(m *nats.Msg) { // Just eat it so that we are not measuring // code time, just delivery. atomic.AddInt32(&received, 1) if received >= expected { done <- true } }) // Create Error handler nc1.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) { t.Fatalf("Received an error on the subscription's connection: %v\n", err) }) nc1.Flush() for i := 0; i < int(expected); i++ { nc2.Publish("slow.consumer", data) } nc2.Flush() select { case <-done: return case <-time.After(10 * time.Second): r := atomic.LoadInt32(&received) if s.NumSlowConsumers() > 0 { t.Fatalf("Did not receive all large messages due to slow consumer status: %d of %d", r, expected) } t.Fatalf("Failed to receive all large messages: %d of %d\n", r, expected) } } func TestNoRaceRoutedQueueAutoUnsubscribe(t *testing.T) { optsA, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true optsA.NoSystemAccount = true srvA := RunServer(optsA) defer srvA.Shutdown() srvARouteURL := fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, srvA.ClusterAddr().Port) optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(srvARouteURL) srvB := RunServer(optsB) defer srvB.Shutdown() // Wait for these 2 to connect to each other checkClusterFormed(t, srvA, srvB) // Have a client connection to each server ncA, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncA.Close() ncB, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsB.Host, optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncB.Close() rbar := int32(0) barCb := func(m *nats.Msg) { atomic.AddInt32(&rbar, 1) } rbaz := int32(0) bazCb := func(m *nats.Msg) { atomic.AddInt32(&rbaz, 1) } // Create 125 queue subs with auto-unsubscribe to each server for // group bar and group baz. So 250 total per queue group. cons := []*nats.Conn{ncA, ncB} for _, c := range cons { for i := 0; i < 100; i++ { qsub, err := c.QueueSubscribe("foo", "bar", barCb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := qsub.AutoUnsubscribe(1); err != nil { t.Fatalf("Error on auto-unsubscribe: %v", err) } qsub, err = c.QueueSubscribe("foo", "baz", bazCb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := qsub.AutoUnsubscribe(1); err != nil { t.Fatalf("Error on auto-unsubscribe: %v", err) } } c.Subscribe("TEST.COMPLETE", func(m *nats.Msg) {}) } // We coelasce now so for each server we will have all local (200) plus // two from the remote side for each queue group. We also create one more // and will wait til each server has 204 subscriptions, that will make sure // that we have everything setup. checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { subsA := srvA.NumSubscriptions() subsB := srvB.NumSubscriptions() if subsA != 204 || subsB != 204 { return fmt.Errorf("Not all subs processed yet: %d and %d", subsA, subsB) } return nil }) expected := int32(200) // Now send messages from each server for i := int32(0); i < expected; i++ { c := cons[i%2] c.Publish("foo", []byte("Don't Drop Me!")) } for _, c := range cons { c.Flush() } checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { nbar := atomic.LoadInt32(&rbar) nbaz := atomic.LoadInt32(&rbaz) if nbar == expected && nbaz == expected { return nil } return fmt.Errorf("Did not receive all %d queue messages, received %d for 'bar' and %d for 'baz'", expected, atomic.LoadInt32(&rbar), atomic.LoadInt32(&rbaz)) }) } func TestNoRaceClosedSlowConsumerWriteDeadline(t *testing.T) { opts := DefaultOptions() opts.NoSystemAccount = true opts.WriteDeadline = 10 * time.Millisecond // Make very small to trip. opts.MaxPending = 500 * 1024 * 1024 // Set high so it will not trip here. s := RunServer(opts) defer s.Shutdown() c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port), 3*time.Second) if err != nil { t.Fatalf("Error on connect: %v", err) } defer c.Close() if _, err := c.Write([]byte("CONNECT {}\r\nPING\r\nSUB foo 1\r\n")); err != nil { t.Fatalf("Error sending protocols to server: %v", err) } // Reduce socket buffer to increase reliability of data backing up in the server destined // for our subscribed client. c.(*net.TCPConn).SetReadBuffer(128) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) sender, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer sender.Close() payload := make([]byte, 1024*1024) for i := 0; i < 100; i++ { if err := sender.Publish("foo", payload); err != nil { t.Fatalf("Error on publish: %v", err) } } // Flush sender connection to ensure that all data has been sent. if err := sender.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } // At this point server should have closed connection c. checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, SlowConsumerWriteDeadline) } func TestNoRaceClosedSlowConsumerPendingBytes(t *testing.T) { opts := DefaultOptions() opts.NoSystemAccount = true opts.WriteDeadline = 30 * time.Second // Wait for long time so write deadline does not trigger slow consumer. opts.MaxPending = 1 * 1024 * 1024 // Set to low value (1MB) to allow SC to trip. s := RunServer(opts) defer s.Shutdown() c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port), 3*time.Second) if err != nil { t.Fatalf("Error on connect: %v", err) } defer c.Close() if _, err := c.Write([]byte("CONNECT {}\r\nPING\r\nSUB foo 1\r\n")); err != nil { t.Fatalf("Error sending protocols to server: %v", err) } // Reduce socket buffer to increase reliability of data backing up in the server destined // for our subscribed client. c.(*net.TCPConn).SetReadBuffer(128) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) sender, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer sender.Close() payload := make([]byte, 1024*1024) for i := 0; i < 100; i++ { if err := sender.Publish("foo", payload); err != nil { t.Fatalf("Error on publish: %v", err) } } // Flush sender connection to ensure that all data has been sent. if err := sender.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } // At this point server should have closed connection c. checkClosedConns(t, s, 1, 2*time.Second) conns := s.closedClients() if lc := len(conns); lc != 1 { t.Fatalf("len(conns) expected to be %d, got %d\n", 1, lc) } checkReason(t, conns[0].Reason, SlowConsumerPendingBytes) } func TestNoRaceSlowConsumerPendingBytes(t *testing.T) { opts := DefaultOptions() opts.NoSystemAccount = true opts.WriteDeadline = 30 * time.Second // Wait for long time so write deadline does not trigger slow consumer. opts.MaxPending = 1 * 1024 * 1024 // Set to low value (1MB) to allow SC to trip. s := RunServer(opts) defer s.Shutdown() c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port), 3*time.Second) if err != nil { t.Fatalf("Error on connect: %v", err) } defer c.Close() if _, err := c.Write([]byte("CONNECT {}\r\nPING\r\nSUB foo 1\r\n")); err != nil { t.Fatalf("Error sending protocols to server: %v", err) } // Reduce socket buffer to increase reliability of data backing up in the server destined // for our subscribed client. c.(*net.TCPConn).SetReadBuffer(128) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) sender, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer sender.Close() payload := make([]byte, 1024*1024) for i := 0; i < 100; i++ { if err := sender.Publish("foo", payload); err != nil { t.Fatalf("Error on publish: %v", err) } } // Flush sender connection to ensure that all data has been sent. if err := sender.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } // At this point server should have closed connection c. // On certain platforms, it may take more than one call before // getting the error. for i := 0; i < 100; i++ { if _, err := c.Write([]byte("PUB bar 5\r\nhello\r\n")); err != nil { // ok return } } t.Fatal("Connection should have been closed") } func TestNoRaceGatewayNoMissingReplies(t *testing.T) { // This test will have following setup: // // responder1 requestor // | | // v v // [A1]<-------gw------------[B1] // | \ | // | \______gw__________ | route // | _\| | // [ ]--------gw----------->[ ] // [A2]<-------gw------------[B2] // [ ] [ ] // ^ // | // responder2 // // There is a possible race that when the requestor creates // a subscription on the reply subject, the subject interest // being sent from the inbound gateway, and B1 having none, // the SUB first goes to B2 before being sent to A1 from // B2's inbound GW. But the request can go from B1 to A1 // right away and the responder1 connecting to A1 may send // back the reply before the interest on the reply makes it // to A1 (from B2). // This test will also verify that if the responder is instead // connected to A2, the reply is properly received by requestor // on B1. // For this test we want to be in interestOnly mode, so // make it happen quickly gatewayMaxRUnsubBeforeSwitch = 1 defer func() { gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch }() // Start with setting up A2 and B2. ob2 := testDefaultOptionsForGateway("B") sb2 := runGatewayServer(ob2) defer sb2.Shutdown() oa2 := testGatewayOptionsFromToWithServers(t, "A", "B", sb2) sa2 := runGatewayServer(oa2) defer sa2.Shutdown() waitForOutboundGateways(t, sa2, 1, time.Second) waitForInboundGateways(t, sa2, 1, time.Second) waitForOutboundGateways(t, sb2, 1, time.Second) waitForInboundGateways(t, sb2, 1, time.Second) // Now start A1 which will connect to B2 oa1 := testGatewayOptionsFromToWithServers(t, "A", "B", sb2) oa1.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa2.Cluster.Host, oa2.Cluster.Port)) sa1 := runGatewayServer(oa1) defer sa1.Shutdown() waitForOutboundGateways(t, sa1, 1, time.Second) waitForInboundGateways(t, sb2, 2, time.Second) checkClusterFormed(t, sa1, sa2) // Finally, start B1 that will connect to A1. ob1 := testGatewayOptionsFromToWithServers(t, "B", "A", sa1) ob1.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", ob2.Cluster.Host, ob2.Cluster.Port)) sb1 := runGatewayServer(ob1) defer sb1.Shutdown() // Check that we have the outbound gateway from B1 to A1 checkFor(t, 3*time.Second, 15*time.Millisecond, func() error { c := sb1.getOutboundGatewayConnection("A") if c == nil { return fmt.Errorf("Outbound connection to A not created yet") } c.mu.Lock() name := c.opts.Name nc := c.nc c.mu.Unlock() if name != sa1.ID() { // Force a disconnect nc.Close() return fmt.Errorf("Was unable to have B1 connect to A1") } return nil }) waitForInboundGateways(t, sa1, 1, time.Second) checkClusterFormed(t, sb1, sb2) a1URL := fmt.Sprintf("nats://%s:%d", oa1.Host, oa1.Port) a2URL := fmt.Sprintf("nats://%s:%d", oa2.Host, oa2.Port) b1URL := fmt.Sprintf("nats://%s:%d", ob1.Host, ob1.Port) b2URL := fmt.Sprintf("nats://%s:%d", ob2.Host, ob2.Port) ncb1 := natsConnect(t, b1URL) defer ncb1.Close() ncb2 := natsConnect(t, b2URL) defer ncb2.Close() natsSubSync(t, ncb1, "just.a.sub") natsSubSync(t, ncb2, "just.a.sub") checkExpectedSubs(t, 2, sb1, sb2) // For this test, we want A to be checking B's interest in order // to send messages (which would cause replies to be dropped if // there is no interest registered on A). So from A servers, // send to various subjects and cause B's to switch to interestOnly // mode. nca1 := natsConnect(t, a1URL) defer nca1.Close() for i := 0; i < 10; i++ { natsPub(t, nca1, fmt.Sprintf("reject.%d", i), []byte("hello")) } nca2 := natsConnect(t, a2URL) defer nca2.Close() for i := 0; i < 10; i++ { natsPub(t, nca2, fmt.Sprintf("reject.%d", i), []byte("hello")) } checkSwitchedMode := func(t *testing.T, s *Server) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { var switchedMode bool c := s.getOutboundGatewayConnection("B") ei, _ := c.gw.outsim.Load(globalAccountName) if ei != nil { e := ei.(*outsie) e.RLock() switchedMode = e.ni == nil && e.mode == InterestOnly e.RUnlock() } if !switchedMode { return fmt.Errorf("Still not switched mode") } return nil }) } checkSwitchedMode(t, sa1) checkSwitchedMode(t, sa2) // Setup a subscriber on _INBOX.> on each of A's servers. total := 1000 expected := int32(total) rcvOnA := int32(0) qrcvOnA := int32(0) natsSub(t, nca1, "myreply.>", func(_ *nats.Msg) { atomic.AddInt32(&rcvOnA, 1) }) natsQueueSub(t, nca2, "myreply.>", "bar", func(_ *nats.Msg) { atomic.AddInt32(&qrcvOnA, 1) }) checkExpectedSubs(t, 2, sa1, sa2) // Ok.. so now we will run the actual test where we // create a responder on A1 and make sure that every // single request from B1 gets the reply. Will repeat // test with responder connected to A2. sendReqs := func(t *testing.T, subConn *nats.Conn) { t.Helper() responder := natsSub(t, subConn, "foo", func(m *nats.Msg) { m.Respond([]byte("reply")) }) natsFlush(t, subConn) checkExpectedSubs(t, 3, sa1, sa2) // We are not going to use Request() because this sets // a wildcard subscription on an INBOX and less likely // to produce the race. Instead we will explicitly set // the subscription on the reply subject and create one // per request. for i := 0; i < total/2; i++ { reply := fmt.Sprintf("myreply.%d", i) replySub := natsQueueSubSync(t, ncb1, reply, "bar") natsFlush(t, ncb1) // Let's make sure we have interest on B2. if r := sb2.globalAccount().sl.Match(reply); len(r.qsubs) == 0 { checkFor(t, time.Second, time.Millisecond, func() error { if r := sb2.globalAccount().sl.Match(reply); len(r.qsubs) == 0 { return fmt.Errorf("B still not registered interest on %s", reply) } return nil }) } natsPubReq(t, ncb1, "foo", reply, []byte("request")) if _, err := replySub.NextMsg(time.Second); err != nil { t.Fatalf("Did not receive reply: %v", err) } natsUnsub(t, replySub) } responder.Unsubscribe() natsFlush(t, subConn) checkExpectedSubs(t, 2, sa1, sa2) } sendReqs(t, nca1) sendReqs(t, nca2) checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := atomic.LoadInt32(&rcvOnA); n != expected { return fmt.Errorf("Subs on A expected to get %v replies, got %v", expected, n) } return nil }) // We should not have received a single message on the queue sub // on cluster A because messages will have been delivered to // the member on cluster B. if n := atomic.LoadInt32(&qrcvOnA); n != 0 { t.Fatalf("Queue sub on A should not have received message, got %v", n) } } func TestNoRaceRouteMemUsage(t *testing.T) { oa := DefaultOptions() sa := RunServer(oa) defer sa.Shutdown() ob := DefaultOptions() ob.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa.Cluster.Host, oa.Cluster.Port)) sb := RunServer(ob) defer sb.Shutdown() checkClusterFormed(t, sa, sb) responder := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer responder.Close() for i := 0; i < 10; i++ { natsSub(t, responder, "foo", func(m *nats.Msg) { m.Respond(m.Data) }) } natsFlush(t, responder) payload := make([]byte, 50*1024) bURL := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) // Capture mem usage mem := runtime.MemStats{} runtime.ReadMemStats(&mem) inUseBefore := mem.HeapInuse for i := 0; i < 100; i++ { requestor := natsConnect(t, bURL) // Don't use a defer here otherwise that will make the memory check fail! // We are closing the connection just after these few instructions that // are not calling t.Fatal() anyway. inbox := nats.NewInbox() sub := natsSubSync(t, requestor, inbox) natsPubReq(t, requestor, "foo", inbox, payload) for j := 0; j < 10; j++ { natsNexMsg(t, sub, time.Second) } requestor.Close() } runtime.GC() debug.FreeOSMemory() runtime.ReadMemStats(&mem) inUseNow := mem.HeapInuse if inUseNow > 3*inUseBefore { t.Fatalf("Heap in-use before was %v, now %v: too high", inUseBefore, inUseNow) } } func TestNoRaceRouteCache(t *testing.T) { maxPerAccountCacheSize = 20 prunePerAccountCacheSize = 5 closedSubsCheckInterval = 250 * time.Millisecond defer func() { maxPerAccountCacheSize = defaultMaxPerAccountCacheSize prunePerAccountCacheSize = defaultPrunePerAccountCacheSize closedSubsCheckInterval = defaultClosedSubsCheckInterval }() for _, test := range []struct { name string useQueue bool }{ {"plain_sub", false}, {"queue_sub", true}, } { t.Run(test.name, func(t *testing.T) { oa := DefaultOptions() oa.NoSystemAccount = true oa.Cluster.PoolSize = -1 sa := RunServer(oa) defer sa.Shutdown() ob := DefaultOptions() ob.NoSystemAccount = true ob.Cluster.PoolSize = -1 ob.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", oa.Cluster.Host, oa.Cluster.Port)) sb := RunServer(ob) defer sb.Shutdown() checkClusterFormed(t, sa, sb) responder := natsConnect(t, fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port)) defer responder.Close() natsSub(t, responder, "foo", func(m *nats.Msg) { m.Respond(m.Data) }) natsFlush(t, responder) checkExpectedSubs(t, 1, sa) checkExpectedSubs(t, 1, sb) bURL := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) requestor := natsConnect(t, bURL) defer requestor.Close() ch := make(chan struct{}, 1) cb := func(_ *nats.Msg) { select { case ch <- struct{}{}: default: } } sendReqs := func(t *testing.T, nc *nats.Conn, count int, unsub bool) { t.Helper() for i := 0; i < count; i++ { inbox := nats.NewInbox() var sub *nats.Subscription if test.useQueue { sub = natsQueueSub(t, nc, inbox, "queue", cb) } else { sub = natsSub(t, nc, inbox, cb) } natsPubReq(t, nc, "foo", inbox, []byte("hello")) select { case <-ch: case <-time.After(time.Second): t.Fatalf("Failed to get reply") } if unsub { natsUnsub(t, sub) } } } sendReqs(t, requestor, maxPerAccountCacheSize+1, true) var route *client sb.mu.Lock() route = getFirstRoute(sb) sb.mu.Unlock() checkExpected := func(t *testing.T, expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { route.mu.Lock() n := len(route.in.pacache) route.mu.Unlock() if n != expected { return fmt.Errorf("Expected %v subs in the cache, got %v", expected, n) } return nil }) } checkExpected(t, (maxPerAccountCacheSize+1)-(prunePerAccountCacheSize+1)) // Wait for more than the orphan check time.Sleep(2 * closedSubsCheckInterval) // Add a new subs up to point where new prune would occur sendReqs(t, requestor, prunePerAccountCacheSize+1, false) // Now closed subs should have been removed, so expected // subs in the cache should be the new ones. checkExpected(t, prunePerAccountCacheSize+1) // Now try wil implicit unsubscribe (due to connection close) sendReqs(t, requestor, maxPerAccountCacheSize+1, false) requestor.Close() checkExpected(t, maxPerAccountCacheSize-prunePerAccountCacheSize) // Wait for more than the orphan check time.Sleep(2 * closedSubsCheckInterval) // Now create new connection and send prunePerAccountCacheSize+1 // and that should cause all subs from previous connection to be // removed from cache requestor = natsConnect(t, bURL) defer requestor.Close() sendReqs(t, requestor, prunePerAccountCacheSize+1, false) checkExpected(t, prunePerAccountCacheSize+1) }) } } func TestNoRaceFetchAccountDoesNotRegisterAccountTwice(t *testing.T) { sa, oa, sb, ob, _ := runTrustedGateways(t) defer sa.Shutdown() defer sb.Shutdown() // Let's create a user account. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) jwt, _ := nac.Encode(okp) userAcc := pub // Replace B's account resolver with one that introduces // delay during the Fetch() sac := &slowAccResolver{AccountResolver: sb.AccountResolver()} sb.SetAccountResolver(sac) // Add the account in sa and sb addAccountToMemResolver(sa, userAcc, jwt) addAccountToMemResolver(sb, userAcc, jwt) // Tell the slow account resolver which account to slow down sac.Lock() sac.acc = userAcc sac.Unlock() urlA := fmt.Sprintf("nats://%s:%d", oa.Host, oa.Port) urlB := fmt.Sprintf("nats://%s:%d", ob.Host, ob.Port) nca, err := nats.Connect(urlA, createUserCreds(t, sa, akp)) if err != nil { t.Fatalf("Error connecting to A: %v", err) } defer nca.Close() // Since there is an optimistic send, this message will go to B // and on processing this message, B will lookup/fetch this // account, which can produce race with the fetch of this // account from A's system account that sent a notification // about this account, or with the client connect just after // that. nca.Publish("foo", []byte("hello")) // Now connect and create a subscription on B ncb, err := nats.Connect(urlB, createUserCreds(t, sb, akp)) if err != nil { t.Fatalf("Error connecting to A: %v", err) } defer ncb.Close() sub, err := ncb.SubscribeSync("foo") if err != nil { t.Fatalf("Error on subscribe: %v", err) } ncb.Flush() // Now send messages from A and B should ultimately start to receive // them (once the subscription has been correctly registered) ok := false for i := 0; i < 10; i++ { nca.Publish("foo", []byte("hello")) if _, err := sub.NextMsg(100 * time.Millisecond); err != nil { continue } ok = true break } if !ok { t.Fatalf("B should be able to receive messages") } checkTmpAccounts := func(t *testing.T, s *Server) { t.Helper() empty := true s.tmpAccounts.Range(func(_, _ any) bool { empty = false return false }) if !empty { t.Fatalf("tmpAccounts is not empty") } } checkTmpAccounts(t, sa) checkTmpAccounts(t, sb) } func TestNoRaceWriteDeadline(t *testing.T) { opts := DefaultOptions() opts.NoSystemAccount = true opts.WriteDeadline = 30 * time.Millisecond s := RunServer(opts) defer s.Shutdown() c, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port), 3*time.Second) if err != nil { t.Fatalf("Error on connect: %v", err) } defer c.Close() if _, err := c.Write([]byte("CONNECT {}\r\nPING\r\nSUB foo 1\r\n")); err != nil { t.Fatalf("Error sending protocols to server: %v", err) } // Reduce socket buffer to increase reliability of getting // write deadline errors. c.(*net.TCPConn).SetReadBuffer(4) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) sender, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer sender.Close() payload := make([]byte, 1000000) total := 1000 for i := 0; i < total; i++ { if err := sender.Publish("foo", payload); err != nil { t.Fatalf("Error on publish: %v", err) } } // Flush sender connection to ensure that all data has been sent. if err := sender.Flush(); err != nil { t.Fatalf("Error on flush: %v", err) } // At this point server should have closed connection c. // On certain platforms, it may take more than one call before // getting the error. for i := 0; i < 100; i++ { if _, err := c.Write([]byte("PUB bar 5\r\nhello\r\n")); err != nil { // ok return } } t.Fatal("Connection should have been closed") } func TestNoRaceLeafNodeClusterNameConflictDeadlock(t *testing.T) { o := DefaultOptions() o.LeafNode.Port = -1 s := RunServer(o) defer s.Shutdown() u, err := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", o.LeafNode.Port)) if err != nil { t.Fatalf("Error parsing url: %v", err) } o1 := DefaultOptions() o1.ServerName = "A1" o1.Cluster.Name = "clusterA" o1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} s1 := RunServer(o1) defer s1.Shutdown() checkLeafNodeConnected(t, s1) o2 := DefaultOptions() o2.ServerName = "A2" o2.Cluster.Name = "clusterA" o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} s2 := RunServer(o2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) checkClusterFormed(t, s1, s2) o3 := DefaultOptions() o3.ServerName = "A3" o3.Cluster.Name = "" // intentionally not set o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o3.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} s3 := RunServer(o3) defer s3.Shutdown() checkLeafNodeConnected(t, s3) checkClusterFormed(t, s1, s2, s3) } // This test is same than TestAccountAddServiceImportRace but running // without the -race flag, it would capture more easily the possible // duplicate sid, resulting in less than expected number of subscriptions // in the account's internal subscriptions map. func TestNoRaceAccountAddServiceImportRace(t *testing.T) { TestAccountAddServiceImportRace(t) } // Similar to the routed version. Make sure we receive all of the // messages with auto-unsubscribe enabled. func TestNoRaceQueueAutoUnsubscribe(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() rbar := int32(0) barCb := func(m *nats.Msg) { atomic.AddInt32(&rbar, 1) } rbaz := int32(0) bazCb := func(m *nats.Msg) { atomic.AddInt32(&rbaz, 1) } // Create 1000 subscriptions with auto-unsubscribe of 1. // Do two groups, one bar and one baz. total := 1000 for i := 0; i < total; i++ { qsub, err := nc.QueueSubscribe("foo", "bar", barCb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := qsub.AutoUnsubscribe(1); err != nil { t.Fatalf("Error on auto-unsubscribe: %v", err) } qsub, err = nc.QueueSubscribe("foo", "baz", bazCb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := qsub.AutoUnsubscribe(1); err != nil { t.Fatalf("Error on auto-unsubscribe: %v", err) } } nc.Flush() expected := int32(total) for i := int32(0); i < expected; i++ { nc.Publish("foo", []byte("Don't Drop Me!")) } nc.Flush() checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { nbar := atomic.LoadInt32(&rbar) nbaz := atomic.LoadInt32(&rbaz) if nbar == expected && nbaz == expected { return nil } return fmt.Errorf("Did not receive all %d queue messages, received %d for 'bar' and %d for 'baz'", expected, atomic.LoadInt32(&rbar), atomic.LoadInt32(&rbaz)) }) } func TestNoRaceAcceptLoopsDoNotLeaveOpenedConn(t *testing.T) { for _, test := range []struct { name string url func(o *Options) (string, int) }{ {"client", func(o *Options) (string, int) { return o.Host, o.Port }}, {"route", func(o *Options) (string, int) { return o.Cluster.Host, o.Cluster.Port }}, {"gateway", func(o *Options) (string, int) { return o.Gateway.Host, o.Gateway.Port }}, {"leafnode", func(o *Options) (string, int) { return o.LeafNode.Host, o.LeafNode.Port }}, {"websocket", func(o *Options) (string, int) { return o.Websocket.Host, o.Websocket.Port }}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() o.DisableShortFirstPing = true o.Accounts = []*Account{NewAccount("$SYS")} o.SystemAccount = "$SYS" o.Cluster.Name = "abc" o.Cluster.Host = "127.0.0.1" o.Cluster.Port = -1 o.Gateway.Name = "abc" o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 o.Websocket.Host = "127.0.0.1" o.Websocket.Port = -1 o.Websocket.HandshakeTimeout = 1 o.Websocket.NoTLS = true s := RunServer(o) defer s.Shutdown() host, port := test.url(o) url := fmt.Sprintf("%s:%d", host, port) var conns []net.Conn wg := sync.WaitGroup{} wg.Add(1) done := make(chan struct{}, 1) go func() { defer wg.Done() // Have an upper limit for i := 0; i < 200; i++ { c, err := net.Dial("tcp", url) if err != nil { return } conns = append(conns, c) select { case <-done: return default: } } }() time.Sleep(15 * time.Millisecond) s.Shutdown() close(done) wg.Wait() for _, c := range conns { c.SetReadDeadline(time.Now().Add(2 * time.Second)) br := bufio.NewReader(c) // Read INFO for connections that were accepted _, _, err := br.ReadLine() if err == nil { // After that, the connection should be closed, // so we should get an error here. _, _, err = br.ReadLine() } // We expect an io.EOF or any other error indicating the use of a closed // connection, but we should not get the timeout error. if ne, ok := err.(net.Error); ok && ne.Timeout() { err = nil } if err == nil { var buf [10]byte c.SetDeadline(time.Now().Add(2 * time.Second)) c.Write([]byte("C")) _, err = c.Read(buf[:]) if ne, ok := err.(net.Error); ok && ne.Timeout() { err = nil } } if err == nil { t.Fatalf("Connection should have been closed") } c.Close() } }) } } func TestNoRaceJetStreamDeleteStreamManyConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MYS" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } // This number needs to be higher than the internal sendq size to trigger what this test is testing. for i := 0; i < 2000; i++ { _, err := mset.addConsumer(&ConsumerConfig{ Durable: fmt.Sprintf("D-%d", i), DeliverSubject: fmt.Sprintf("deliver.%d", i), }) if err != nil { t.Fatalf("Error creating consumer: %v", err) } } // With bug this would not return and would hang. mset.delete() } // We used to swap accounts on an inbound message when processing service imports. // Until JetStream this was kinda ok, but with JetStream we can have pull consumers // trying to access the clients account in another Go routine now which causes issues. // This is not limited to the case above, its just the one that exposed it. // This test is to show that issue and that the fix works, meaning we no longer swap c.acc. func TestNoRaceJetStreamServiceImportAccountSwapIssue(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } beforeSubs := s.NumSubscriptions() // How long we want both sides to run. timeout := time.Now().Add(3 * time.Second) errs := make(chan error, 1) // Publishing side, which will signal the consumer that is waiting and which will access c.acc. If publish // operation runs concurrently we will catch c.acc being $SYS some of the time. go func() { time.Sleep(100 * time.Millisecond) for time.Now().Before(timeout) { // This will signal the delivery of the pull messages. js.Publish("foo", []byte("Hello")) // This will swap the account because of JetStream service import. // We can get an error here with the bug or not. if _, err := js.StreamInfo("TEST"); err != nil { errs <- err return } } errs <- nil }() // Pull messages flow. var received int for time.Now().Before(timeout.Add(2 * time.Second)) { if msgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond)); err == nil { for _, m := range msgs { received++ m.AckSync() } } else { break } } // Wait on publisher Go routine and check for errors. if err := <-errs; err != nil { t.Fatalf("Unexpected error: %v", err) } // Double check all received. si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } if int(si.State.Msgs) != received { t.Fatalf("Expected to receive %d msgs, only got %d", si.State.Msgs, received) } // Now check for leaked subs from the fetch call above. That is what we first saw from the bug. if afterSubs := s.NumSubscriptions(); afterSubs != beforeSubs { t.Fatalf("Leaked subscriptions: %d before, %d after", beforeSubs, afterSubs) } } func TestNoRaceJetStreamAPIStreamListPaging(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Create 2X limit streamsNum := 2 * JSApiNamesLimit for i := 1; i <= streamsNum; i++ { name := fmt.Sprintf("STREAM-%06d", i) cfg := StreamConfig{Name: name, Storage: MemoryStorage} _, err := s.GlobalAccount().addStream(&cfg) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } } // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() reqList := func(offset int) []byte { t.Helper() var req []byte if offset > 0 { req, _ = json.Marshal(&ApiPagedRequest{Offset: offset}) } resp, err := nc.Request(JSApiStreams, req, time.Second) if err != nil { t.Fatalf("Unexpected error getting stream list: %v", err) } return resp.Data } checkResp := func(resp []byte, expectedLen, expectedOffset int) { t.Helper() var listResponse JSApiStreamNamesResponse if err := json.Unmarshal(resp, &listResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(listResponse.Streams) != expectedLen { t.Fatalf("Expected only %d streams but got %d", expectedLen, len(listResponse.Streams)) } if listResponse.Total != streamsNum { t.Fatalf("Expected total to be %d but got %d", streamsNum, listResponse.Total) } if listResponse.Offset != expectedOffset { t.Fatalf("Expected offset to be %d but got %d", expectedOffset, listResponse.Offset) } if expectedLen < 1 { return } // Make sure we get the right stream. sname := fmt.Sprintf("STREAM-%06d", expectedOffset+1) if listResponse.Streams[0] != sname { t.Fatalf("Expected stream %q to be first, got %q", sname, listResponse.Streams[0]) } } checkResp(reqList(0), JSApiNamesLimit, 0) checkResp(reqList(JSApiNamesLimit), JSApiNamesLimit, JSApiNamesLimit) checkResp(reqList(streamsNum), 0, streamsNum) checkResp(reqList(streamsNum-22), 22, streamsNum-22) checkResp(reqList(streamsNum+22), 0, streamsNum) } func TestNoRaceJetStreamAPIConsumerListPaging(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() sname := "MYSTREAM" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sname}) if err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } // Client for API requests. nc := clientConnectToServer(t, s) defer nc.Close() consumersNum := JSApiNamesLimit for i := 1; i <= consumersNum; i++ { dsubj := fmt.Sprintf("d.%d", i) sub, _ := nc.SubscribeSync(dsubj) defer sub.Unsubscribe() nc.Flush() _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: dsubj}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } reqListSubject := fmt.Sprintf(JSApiConsumersT, sname) reqList := func(offset int) []byte { t.Helper() var req []byte if offset > 0 { req, _ = json.Marshal(&JSApiConsumersRequest{ApiPagedRequest: ApiPagedRequest{Offset: offset}}) } resp, err := nc.Request(reqListSubject, req, time.Second) if err != nil { t.Fatalf("Unexpected error getting stream list: %v", err) } return resp.Data } checkResp := func(resp []byte, expectedLen, expectedOffset int) { t.Helper() var listResponse JSApiConsumerNamesResponse if err := json.Unmarshal(resp, &listResponse); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(listResponse.Consumers) != expectedLen { t.Fatalf("Expected only %d streams but got %d", expectedLen, len(listResponse.Consumers)) } if listResponse.Total != consumersNum { t.Fatalf("Expected total to be %d but got %d", consumersNum, listResponse.Total) } if listResponse.Offset != expectedOffset { t.Fatalf("Expected offset to be %d but got %d", expectedOffset, listResponse.Offset) } } checkResp(reqList(0), JSApiNamesLimit, 0) checkResp(reqList(consumersNum-22), 22, consumersNum-22) checkResp(reqList(consumersNum+22), 0, consumersNum) } func TestNoRaceJetStreamWorkQueueLoadBalance(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() mname := "MY_MSG_SET" mset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Subjects: []string{"foo", "bar"}}) if err != nil { t.Fatalf("Unexpected error adding message set: %v", err) } defer mset.delete() // Create basic work queue mode consumer. oname := "WQ" o, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Expected no error with durable, got %v", err) } defer o.delete() // To send messages. nc := clientConnectToServer(t, s) defer nc.Close() // For normal work queue semantics, you send requests to the subject with stream and consumer name. reqMsgSubj := o.requestNextMsgSubject() numWorkers := 25 counts := make([]int32, numWorkers) var received int32 rwg := &sync.WaitGroup{} rwg.Add(numWorkers) wg := &sync.WaitGroup{} wg.Add(numWorkers) ch := make(chan bool) toSend := 1000 for i := 0; i < numWorkers; i++ { nc := clientConnectToServer(t, s) defer nc.Close() go func(index int32) { rwg.Done() defer wg.Done() <-ch for counter := &counts[index]; ; { m, err := nc.Request(reqMsgSubj, nil, 100*time.Millisecond) if err != nil { return } m.Respond(nil) atomic.AddInt32(counter, 1) if total := atomic.AddInt32(&received, 1); total >= int32(toSend) { return } } }(int32(i)) } // Wait for requestors to be ready rwg.Wait() close(ch) sendSubj := "bar" for i := 0; i < toSend; i++ { sendStreamMsg(t, nc, sendSubj, "Hello World!") } // Wait for test to complete. wg.Wait() target := toSend / numWorkers delta := target/2 + 5 low, high := int32(target-delta), int32(target+delta) for i := 0; i < numWorkers; i++ { if msgs := atomic.LoadInt32(&counts[i]); msgs < low || msgs > high { t.Fatalf("Messages received for worker [%d] too far off from target of %d, got %d", i, target, msgs) } } } func TestNoRaceJetStreamClusterLargeStreamInlineCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "LSS", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sr := c.randomNonStreamLeader("$G", "TEST") sr.Shutdown() // In case sr was meta leader. c.waitOnLeader() msg, toSend := []byte("Hello JS Clustering"), 5000 // Now fill up stream. for i := 0; i < toSend; i++ { if _, err = js.Publish("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } si, err := js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Check active state as well, shows that the owner answered. if si.State.Msgs != uint64(toSend) { t.Fatalf("Expected %d msgs, got bad state: %+v", toSend, si.State) } // Kill our current leader to make just 2. c.streamLeader("$G", "TEST").Shutdown() // Now restart the shutdown peer and wait for it to be current. sr = c.restartServer(sr) c.waitOnStreamCurrent(sr, "$G", "TEST") // Ask other servers to stepdown as leader so that sr becomes the leader. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader("$G", "TEST") if sl := c.streamLeader("$G", "TEST"); sl != sr { sl.JetStreamStepdownStream("$G", "TEST") return fmt.Errorf("Server %s is not leader yet", sr) } return nil }) si, err = js.StreamInfo("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Check that we have all of our messsages stored. // Wait for a bit for upper layers to process. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d msgs, got %d", toSend, si.State.Msgs) } return nil }) } func TestNoRaceJetStreamClusterStreamCreateAndLostQuorum(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sub, err := nc.SubscribeSync(JSAdvisoryStreamQuorumLostPre + ".*") if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{Name: "NO-LQ-START", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } c.waitOnStreamLeader("$G", "NO-LQ-START") checkSubsPending(t, sub, 0) c.stopAll() // Start up the one we were connected to first and wait for it to be connected. s = c.restartServer(s) nc, err = nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Failed to create client: %v", err) } defer nc.Close() sub, err = nc.SubscribeSync(JSAdvisoryStreamQuorumLostPre + ".*") if err != nil { t.Fatalf("Unexpected error: %v", err) } nc.Flush() c.restartAll() c.waitOnStreamLeader("$G", "NO-LQ-START") checkSubsPending(t, sub, 0) } func TestNoRaceJetStreamSuperClusterMirrors(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Client based API s := sc.clusterForName("C2").randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Create source stream. _, err := js.AddStream(&nats.StreamConfig{Name: "S1", Subjects: []string{"foo", "bar"}, Replicas: 3, Placement: &nats.Placement{Cluster: "C2"}}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Needed while Go client does not have mirror support. createStream := func(cfg *nats.StreamConfig) { t.Helper() if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %+v", err) } } // Send 100 messages. for i := 0; i < 100; i++ { if _, err := js.Publish("foo", []byte("MIRRORS!")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } createStream(&nats.StreamConfig{ Name: "M1", Mirror: &nats.StreamSource{Name: "S1"}, Placement: &nats.Placement{Cluster: "C1"}, }) checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("M1") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 100 { return fmt.Errorf("Expected 100 msgs, got state: %+v", si.State) } return nil }) // Purge the source stream. if err := js.PurgeStream("S1"); err != nil { t.Fatalf("Unexpected purge error: %v", err) } // Send 50 more msgs now. for i := 0; i < 50; i++ { if _, err := js.Publish("bar", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } createStream(&nats.StreamConfig{ Name: "M2", Mirror: &nats.StreamSource{Name: "S1"}, Replicas: 3, Placement: &nats.Placement{Cluster: "C3"}, }) checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("M2") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 101 { return fmt.Errorf("Expected start seq of 101, got state: %+v", si.State) } return nil }) sl := sc.clusterForName("C3").streamLeader("$G", "M2") doneCh := make(chan bool) // Now test that if the mirror get's interrupted that it picks up where it left off etc. go func() { // Send 100 more messages. for i := 0; i < 100; i++ { if _, err := js.Publish("foo", []byte("MIRRORS!")); err != nil { t.Errorf("Unexpected publish on %d error: %v", i, err) } time.Sleep(2 * time.Millisecond) } doneCh <- true }() time.Sleep(20 * time.Millisecond) sl.Shutdown() <-doneCh sc.clusterForName("C3").waitOnStreamLeader("$G", "M2") checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("M2") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 150 { return fmt.Errorf("Expected 150 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 101 { return fmt.Errorf("Expected start seq of 101, got state: %+v", si.State) } return nil }) } func TestNoRaceJetStreamSuperClusterMixedModeMirrors(t *testing.T) { // Unlike the similar sources test, this test is not reliably catching the bug // that would cause mirrors to not have the expected messages count. // Still, adding this test in case we have a regression and we are lucky in // getting the failure while running this. tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf: { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 7, 4, func(serverName, clusterName, storeDir, conf string) string { sname := serverName[strings.Index(serverName, "-")+1:] switch sname { case "S5", "S6", "S7": conf = strings.ReplaceAll(conf, "jetstream: { ", "#jetstream: { ") default: conf = strings.ReplaceAll(conf, "leaf: { ", "#leaf: { ") } return conf }, nil) defer sc.shutdown() // Connect our client to a non JS server c := sc.randomCluster() var s *Server for { if as := c.randomServer(); !as.JetStreamEnabled() { s = as break } } nc, js := jsClientConnect(t, s) defer nc.Close() numStreams := 10 toSend := 1000 errCh := make(chan error, numStreams) wg := sync.WaitGroup{} wg.Add(numStreams) // Create 10 origin streams for i := 0; i < 10; i++ { go func(idx int) { defer wg.Done() name := fmt.Sprintf("S%d", idx+1) if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { errCh <- fmt.Errorf("unexpected error: %v", err) return } c.waitOnStreamLeader(globalAccountName, name) // Load them up with a bunch of messages. for n := 0; n < toSend; n++ { m := nats.NewMsg(name) m.Header.Set("stream", name) m.Header.Set("idx", strconv.FormatInt(int64(n+1), 10)) if err := nc.PublishMsg(m); err != nil { errCh <- fmt.Errorf("unexpected publish error: %v", err) } } }(i) } wg.Wait() select { case err := <-errCh: t.Fatal(err) default: } for i := 0; i < 3; i++ { // Now create our mirrors wg := sync.WaitGroup{} mirrorsCount := 10 wg.Add(mirrorsCount) errCh := make(chan error, 1) for m := 0; m < mirrorsCount; m++ { sname := fmt.Sprintf("S%d", rand.Intn(10)+1) go func(sname string, mirrorIdx int) { defer wg.Done() if _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("M%d", mirrorIdx), Mirror: &nats.StreamSource{Name: sname}, Replicas: 3, }); err != nil { select { case errCh <- err: default: } } }(sname, m+1) } wg.Wait() select { case err := <-errCh: t.Fatalf("Error creating mirrors: %v", err) default: } // Now check the mirrors have all expected messages for m := 0; m < mirrorsCount; m++ { name := fmt.Sprintf("M%d", m+1) checkFor(t, 15*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo(name) if err != nil { t.Fatalf("Could not retrieve stream info") } if si.State.Msgs != uint64(toSend) { return fmt.Errorf("Expected %d msgs, got state: %+v", toSend, si.State) } return nil }) err := js.DeleteStream(name) require_NoError(t, err) } } } func TestNoRaceJetStreamSuperClusterSources(t *testing.T) { owt := srcConsumerWaitTime srcConsumerWaitTime = 2 * time.Second defer func() { srcConsumerWaitTime = owt }() sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Client based API s := sc.clusterForName("C1").randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // Create our source streams. for _, sname := range []string{"foo", "bar", "baz"} { if _, err := js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 1}); err != nil { t.Fatalf("Unexpected error: %v", err) } } sendBatch := func(subject string, n int) { for i := 0; i < n; i++ { msg := fmt.Sprintf("MSG-%d", i+1) if _, err := js.Publish(subject, []byte(msg)); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } // Populate each one. sendBatch("foo", 10) sendBatch("bar", 15) sendBatch("baz", 25) // Needed while Go client does not have mirror support for creating mirror or source streams. createStream := func(cfg *nats.StreamConfig) { t.Helper() if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %+v", err) } } cfg := &nats.StreamConfig{ Name: "MS", Sources: []*nats.StreamSource{ {Name: "foo"}, {Name: "bar"}, {Name: "baz"}, }, } createStream(cfg) time.Sleep(time.Second) // Faster timeout since we loop below checking for condition. js2, err := nc.JetStream(nats.MaxWait(50 * time.Millisecond)) if err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("MS") if err != nil { return err } if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } return nil }) // Purge the source streams. for _, sname := range []string{"foo", "bar", "baz"} { if err := js.PurgeStream(sname); err != nil { t.Fatalf("Unexpected purge error: %v", err) } } if err := js.DeleteStream("MS"); err != nil { t.Fatalf("Unexpected delete error: %v", err) } // Send more msgs now. sendBatch("foo", 10) sendBatch("bar", 15) sendBatch("baz", 25) cfg = &nats.StreamConfig{ Name: "MS2", Sources: []*nats.StreamSource{ {Name: "foo"}, {Name: "bar"}, {Name: "baz"}, }, Replicas: 3, Placement: &nats.Placement{Cluster: "C3"}, } createStream(cfg) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js2.StreamInfo("MS2") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 50 { return fmt.Errorf("Expected 50 msgs, got state: %+v", si.State) } if si.State.FirstSeq != 1 { return fmt.Errorf("Expected start seq of 1, got state: %+v", si.State) } return nil }) sl := sc.clusterForName("C3").streamLeader("$G", "MS2") doneCh := make(chan bool) if sl == sc.leader() { snc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() _, err := snc.Request(JSApiLeaderStepDown, nil, time.Second) require_NoError(t, err) sc.waitOnLeader() } // Now test that if the mirror get's interrupted that it picks up where it left off etc. go func() { // Send 50 more messages each. for i := 0; i < 50; i++ { msg := fmt.Sprintf("R-MSG-%d", i+1) for _, sname := range []string{"foo", "bar", "baz"} { m := nats.NewMsg(sname) m.Data = []byte(msg) if _, err := js.PublishMsg(m); err != nil { t.Errorf("Unexpected publish error: %v", err) } } time.Sleep(2 * time.Millisecond) } doneCh <- true }() time.Sleep(20 * time.Millisecond) sl.Shutdown() sc.clusterForName("C3").waitOnStreamLeader("$G", "MS2") <-doneCh checkFor(t, 20*time.Second, time.Second, func() error { si, err := js2.StreamInfo("MS2") if err != nil { return err } if si.State.Msgs != 200 { return fmt.Errorf("Expected 200 msgs, got state: %+v", si.State) } return nil }) } func TestNoRaceJetStreamClusterSourcesMuxd(t *testing.T) { c := createJetStreamClusterExplicit(t, "SMUX", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Send in 10000 messages. msg, toSend := make([]byte, 1024), 10000 crand.Read(msg) var sources []*nats.StreamSource // Create 10 origin streams. for i := 1; i <= 10; i++ { name := fmt.Sprintf("O-%d", i) if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we have a leader before publishing, especially since we use // non JS publisher, we would not know if the messages made it to those // streams or not. c.waitOnStreamLeader(globalAccountName, name) // Load them up with a bunch of messages. for n := 0; n < toSend; n++ { if err := nc.Publish(name, msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } sources = append(sources, &nats.StreamSource{Name: name}) } // Now create our downstream stream that sources from all of them. if _, err := js.AddStream(&nats.StreamConfig{Name: "S", Replicas: 2, Sources: sources}); err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { t.Fatalf("Could not retrieve stream info") } if si.State.Msgs != uint64(10*toSend) { return fmt.Errorf("Expected %d msgs, got state: %+v", toSend*10, si.State) } return nil }) } func TestNoRaceJetStreamSuperClusterMixedModeSources(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} leaf: { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 7, 2, func(serverName, clusterName, storeDir, conf string) string { sname := serverName[strings.Index(serverName, "-")+1:] switch sname { case "S5", "S6", "S7": conf = strings.ReplaceAll(conf, "jetstream: { ", "#jetstream: { ") default: conf = strings.ReplaceAll(conf, "leaf: { ", "#leaf: { ") } return conf }, nil) defer sc.shutdown() // Connect our client to a non JS server c := sc.randomCluster() var s *Server for { if as := c.randomServer(); !as.JetStreamEnabled() { s = as break } } nc, js := jsClientConnect(t, s) defer nc.Close() numStreams := 100 toSend := 1000 var sources []*nats.StreamSource errCh := make(chan error, numStreams) srcCh := make(chan *nats.StreamSource, numStreams) wg := sync.WaitGroup{} wg.Add(numStreams) // Create 100 origin streams. for i := 1; i <= numStreams; i++ { go func(idx int) { defer wg.Done() name := fmt.Sprintf("O-%d", idx) if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { errCh <- fmt.Errorf("unexpected error: %v", err) return } c.waitOnStreamLeader(globalAccountName, name) // Load them up with a bunch of messages. for n := 0; n < toSend; n++ { m := nats.NewMsg(name) m.Header.Set("stream", name) m.Header.Set("idx", strconv.FormatInt(int64(n+1), 10)) if err := nc.PublishMsg(m); err != nil { errCh <- fmt.Errorf("unexpected publish error: %v", err) return } } srcCh <- &nats.StreamSource{Name: name} }(i) } wg.Wait() select { case err := <-errCh: t.Fatal(err) default: } for i := 0; i < numStreams; i++ { sources = append(sources, <-srcCh) } for i := 0; i < 3; i++ { // Now create our downstream stream that sources from all of them. if _, err := js.AddStream(&nats.StreamConfig{Name: "S", Replicas: 3, Sources: sources}); err != nil { t.Fatalf("Unexpected error: %v", err) } checkFor(t, 15*time.Second, 1000*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { t.Fatalf("Could not retrieve stream info") } if si.State.Msgs != uint64(numStreams*toSend) { return fmt.Errorf("Expected %d msgs, got state: %+v", numStreams*toSend, si.State) } return nil }) err := js.DeleteStream("S") require_NoError(t, err) } } func TestNoRaceJetStreamClusterExtendedStreamPurgeStall(t *testing.T) { // Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm. skip(t) cerr := func(t *testing.T, err error) { t.Helper() if err != nil { t.Fatalf("unexepected err: %s", err) } } s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() si, err := js.AddStream(&nats.StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: nats.FileStorage, }) cerr(t, err) // 100kb messages spread over 1000 different subjects body := make([]byte, 100*1024) for i := 0; i < 50000; i++ { if _, err := js.PublishAsync(fmt.Sprintf("kv.%d", i%1000), body); err != nil { cerr(t, err) } } checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { if si, err = js.StreamInfo("KV"); err != nil { return err } if si.State.Msgs == 50000 { return nil } return fmt.Errorf("waiting for more") }) jp, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: "kv.20"}) start := time.Now() res, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "KV"), jp, time.Minute) elapsed := time.Since(start) cerr(t, err) pres := JSApiStreamPurgeResponse{} err = json.Unmarshal(res.Data, &pres) cerr(t, err) if !pres.Success { t.Fatalf("purge failed: %#v", pres) } if elapsed > time.Second { t.Fatalf("Purge took too long %s", elapsed) } v, _ := s.Varz(nil) if v.Mem > 100*1024*1024 { // 100MB limit but in practice < 100MB -> Was ~7GB when failing. t.Fatalf("Used too much memory: %v", friendlyBytes(v.Mem)) } } func TestNoRaceJetStreamClusterMirrorExpirationAndMissingSequences(t *testing.T) { c := createJetStreamClusterExplicit(t, "MMS", 9) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sendBatch := func(n int) { t.Helper() // Send a batch to a given subject. for i := 0; i < n; i++ { if _, err := js.Publish("TEST", []byte("OK")); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } checkStream := func(stream string, num uint64) { t.Helper() checkFor(t, 20*time.Second, 20*time.Millisecond, func() error { si, err := js.StreamInfo(stream) if err != nil { return err } if si.State.Msgs != num { return fmt.Errorf("Expected %d msgs, got %d", num, si.State.Msgs) } return nil }) } checkMirror := func(num uint64) { t.Helper(); checkStream("M", num) } checkTest := func(num uint64) { t.Helper(); checkStream("TEST", num) } // Origin _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", MaxAge: 500 * time.Millisecond, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } ts := c.streamLeader("$G", "TEST") ml := c.leader() // Create mirror now. for ms := ts; ms == ts || ms == ml; { _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "TEST"}, Replicas: 2, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } ms = c.streamLeader("$G", "M") if ts == ms || ms == ml { // Delete and retry. js.DeleteStream("M") } } sendBatch(10) checkMirror(10) // Now shutdown the server with the mirror. ms := c.streamLeader("$G", "M") ms.Shutdown() c.waitOnLeader() // Send more messages but let them expire. sendBatch(10) checkTest(0) c.restartServer(ms) c.checkClusterFormed() c.waitOnStreamLeader("$G", "M") sendBatch(10) checkMirror(20) } func TestNoRaceJetStreamClusterLargeActiveOnReplica(t *testing.T) { // Uncomment to run. skip(t) c := createJetStreamClusterExplicit(t, "LAG", 3) defer c.shutdown() // Client for API requests. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() timeout := time.Now().Add(60 * time.Second) for time.Now().Before(timeout) { si, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } for _, r := range si.Cluster.Replicas { if r.Active > 5*time.Second { t.Fatalf("Bad Active value: %+v", r) } } if err := js.DeleteStream("TEST"); err != nil { t.Fatalf("Unexpected delete error: %v", err) } } } func TestNoRaceJetStreamSuperClusterRIPStress(t *testing.T) { // Uncomment to run. Needs to be on a big machine. skip(t) sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() // Client based API s := sc.clusterForName("C2").randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() scm := make(map[string][]string) // Create 50 streams per cluster. for _, cn := range []string{"C1", "C2", "C3"} { var streams []string for i := 0; i < 50; i++ { sn := fmt.Sprintf("%s-S%d", cn, i+1) streams = append(streams, sn) _, err := js.AddStream(&nats.StreamConfig{ Name: sn, Replicas: 3, Placement: &nats.Placement{Cluster: cn}, MaxAge: 2 * time.Minute, MaxMsgs: 50_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } } scm[cn] = streams } sourceForCluster := func(cn string) []*nats.StreamSource { var sns []string switch cn { case "C1": sns = scm["C2"] case "C2": sns = scm["C3"] case "C3": sns = scm["C1"] default: t.Fatalf("Unknown cluster %q", cn) } var ss []*nats.StreamSource for _, sn := range sns { ss = append(ss, &nats.StreamSource{Name: sn}) } return ss } // Mux all 50 streams from one cluster to a single stream across a GW connection to another cluster. _, err := js.AddStream(&nats.StreamConfig{ Name: "C1-S-MUX", Replicas: 2, Placement: &nats.Placement{Cluster: "C1"}, Sources: sourceForCluster("C2"), MaxAge: time.Minute, MaxMsgs: 20_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "C2-S-MUX", Replicas: 2, Placement: &nats.Placement{Cluster: "C2"}, Sources: sourceForCluster("C3"), MaxAge: time.Minute, MaxMsgs: 20_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "C3-S-MUX", Replicas: 2, Placement: &nats.Placement{Cluster: "C3"}, Sources: sourceForCluster("C1"), MaxAge: time.Minute, MaxMsgs: 20_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now create mirrors for our mux'd streams. _, err = js.AddStream(&nats.StreamConfig{ Name: "C1-MIRROR", Replicas: 3, Placement: &nats.Placement{Cluster: "C1"}, Mirror: &nats.StreamSource{Name: "C3-S-MUX"}, MaxAge: 5 * time.Minute, MaxMsgs: 10_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "C2-MIRROR", Replicas: 3, Placement: &nats.Placement{Cluster: "C2"}, Mirror: &nats.StreamSource{Name: "C2-S-MUX"}, MaxAge: 5 * time.Minute, MaxMsgs: 10_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.AddStream(&nats.StreamConfig{ Name: "C3-MIRROR", Replicas: 3, Placement: &nats.Placement{Cluster: "C3"}, Mirror: &nats.StreamSource{Name: "C1-S-MUX"}, MaxAge: 5 * time.Minute, MaxMsgs: 10_000, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } var jsc []nats.JetStream // Create 64 clients. for i := 0; i < 64; i++ { s := sc.randomCluster().randomServer() nc, _ := jsClientConnect(t, s) defer nc.Close() js, err := nc.JetStream(nats.PublishAsyncMaxPending(8 * 1024)) if err != nil { t.Fatalf("Unexpected error: %v", err) } jsc = append(jsc, js) } msg := make([]byte, 1024) crand.Read(msg) // 10 minutes expires := time.Now().Add(480 * time.Second) for time.Now().Before(expires) { for _, sns := range scm { rand.Shuffle(len(sns), func(i, j int) { sns[i], sns[j] = sns[j], sns[i] }) for _, sn := range sns { js := jsc[rand.Intn(len(jsc))] if _, err = js.PublishAsync(sn, msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } } time.Sleep(10 * time.Millisecond) } } func TestNoRaceJetStreamSlowFilteredInititalPendingAndFirstMsg(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Create directly here to force multiple blocks, etc. a, err := s.LookupAccount("$G") if err != nil { t.Fatalf("Unexpected error: %v", err) } mset, err := a.addStreamWithStore( &StreamConfig{ Name: "S", Subjects: []string{"foo", "bar", "baz", "foo.bar.baz", "foo.*"}, }, &FileStoreConfig{ BlockSize: 4 * 1024 * 1024, AsyncFlush: true, }, ) if err != nil { t.Fatalf("Unexpected error: %v", err) } nc, js := jsClientConnect(t, s) defer nc.Close() toSend := 100_000 // 500k total though. // Messages will be 'foo' 'bar' 'baz' repeated 100k times. // Then 'foo.bar.baz' all contigous for 100k. // Then foo.N for 1-100000 for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("HELLO")) js.PublishAsync("bar", []byte("WORLD")) js.PublishAsync("baz", []byte("AGAIN")) } // Make contiguous block of same subject. for i := 0; i < toSend; i++ { js.PublishAsync("foo.bar.baz", []byte("ALL-TOGETHER")) } // Now add some more at the end. for i := 0; i < toSend; i++ { js.PublishAsync(fmt.Sprintf("foo.%d", i+1), []byte("LATER")) } checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { return err } if si.State.Msgs != uint64(5*toSend) { return fmt.Errorf("Expected %d msgs, got %d", 5*toSend, si.State.Msgs) } return nil }) // Threshold for taking too long. const thresh = 150 * time.Millisecond var dindex int testConsumerCreate := func(subj string, startSeq, expectedNumPending uint64) { t.Helper() dindex++ dname := fmt.Sprintf("dur-%d", dindex) cfg := ConsumerConfig{FilterSubject: subj, Durable: dname, AckPolicy: AckExplicit} if startSeq > 1 { cfg.OptStartSeq, cfg.DeliverPolicy = startSeq, DeliverByStartSequence } start := time.Now() o, err := mset.addConsumer(&cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } if delta := time.Since(start); delta > thresh { t.Fatalf("Creating consumer for %q and start: %d took too long: %v", subj, startSeq, delta) } if ci := o.info(); ci.NumPending != expectedNumPending { t.Fatalf("Expected NumPending of %d, got %d", expectedNumPending, ci.NumPending) } } testConsumerCreate("foo.100000", 1, 1) testConsumerCreate("foo.100000", 222_000, 1) testConsumerCreate("foo", 1, 100_000) testConsumerCreate("foo", 4, 100_000-1) testConsumerCreate("foo.bar.baz", 1, 100_000) testConsumerCreate("foo.bar.baz", 350_001, 50_000) testConsumerCreate("*", 1, 300_000) testConsumerCreate("*", 4, 300_000-3) testConsumerCreate(">", 1, 500_000) testConsumerCreate(">", 50_000, 500_000-50_000+1) testConsumerCreate("foo.10", 1, 1) // Also test that we do not take long if the start sequence is later in the stream. sub, err := js.PullSubscribe("foo.100000", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } start := time.Now() fetchMsgs(t, sub, 1, time.Second) if delta := time.Since(start); delta > thresh { t.Fatalf("Took too long for pull subscriber to fetch the message: %v", delta) } // Now do some deletes and make sure these are handled correctly. // Delete 3 foo messages. mset.removeMsg(1) mset.removeMsg(4) mset.removeMsg(7) testConsumerCreate("foo", 1, 100_000-3) // Make sure wider scoped subjects do the right thing from a pending perspective. o, err := mset.addConsumer(&ConsumerConfig{FilterSubject: ">", Durable: "cat", AckPolicy: AckExplicit}) if err != nil { t.Fatalf("Unexpected error: %v", err) } ci, expected := o.info(), uint64(500_000-3) if ci.NumPending != expected { t.Fatalf("Expected NumPending of %d, got %d", expected, ci.NumPending) } // Send another and make sure its captured by our wide scope consumer. js.Publish("foo", []byte("HELLO AGAIN")) if ci = o.info(); ci.NumPending != expected+1 { t.Fatalf("Expected the consumer to recognize the wide scoped consumer, wanted pending of %d, got %d", expected+1, ci.NumPending) } // Stop current server and test restart.. sd := s.JetStreamConfig().StoreDir s.Shutdown() // Restart. s = RunJetStreamServerOnPort(-1, sd) defer s.Shutdown() a, err = s.LookupAccount("$G") if err != nil { t.Fatalf("Unexpected error: %v", err) } mset, err = a.lookupStream("S") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Make sure we recovered our per subject state on restart. testConsumerCreate("foo.100000", 1, 1) testConsumerCreate("foo", 1, 100_000-2) } func TestNoRaceJetStreamFileStoreBufferReuse(t *testing.T) { // Uncomment to run. Needs to be on a big machine. skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() cfg := &StreamConfig{Name: "TEST", Subjects: []string{"foo", "bar", "baz"}, Storage: FileStorage} if _, err := s.GlobalAccount().addStreamWithStore(cfg, nil); err != nil { t.Fatalf("Unexpected error adding stream: %v", err) } // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() toSend := 200_000 m := nats.NewMsg("foo") m.Data = make([]byte, 8*1024) crand.Read(m.Data) start := time.Now() for i := 0; i < toSend; i++ { m.Reply = _EMPTY_ switch i % 3 { case 0: m.Subject = "foo" case 1: m.Subject = "bar" case 2: m.Subject = "baz" } m.Header.Set("X-ID2", fmt.Sprintf("XXXXX-%d", i)) if _, err := js.PublishMsgAsync(m); err != nil { t.Fatalf("Err on publish: %v", err) } } <-js.PublishAsyncComplete() fmt.Printf("TOOK %v to publish\n", time.Since(start)) v, err := s.Varz(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } fmt.Printf("MEM AFTER PUBLISH is %v\n", friendlyBytes(v.Mem)) si, _ := js.StreamInfo("TEST") fmt.Printf("si is %+v\n", si.State) received := 0 done := make(chan bool) cb := func(m *nats.Msg) { received++ if received >= toSend { done <- true } } start = time.Now() sub, err := js.Subscribe("*", cb, nats.EnableFlowControl(), nats.IdleHeartbeat(time.Second), nats.AckNone()) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() <-done fmt.Printf("TOOK %v to consume\n", time.Since(start)) v, err = s.Varz(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } fmt.Printf("MEM AFTER SUBSCRIBE is %v\n", friendlyBytes(v.Mem)) } // Report of slow restart for a server that has many messages that have expired while it was not running. func TestNoRaceJetStreamSlowRestartWithManyExpiredMsgs(t *testing.T) { opts := DefaultTestOptions opts.Port = -1 opts.JetStream = true s := RunServer(&opts) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() ttl := 2 * time.Second _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"orders.*"}, MaxAge: ttl, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Attach a consumer who is filtering on a wildcard subject as well. // This does not affect it like I thought originally but will keep it here. _, err = js.AddConsumer("ORDERS", &nats.ConsumerConfig{ Durable: "c22", FilterSubject: "orders.*", AckPolicy: nats.AckExplicitPolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Now fill up with messages. toSend := 100_000 for i := 1; i <= toSend; i++ { js.PublishAsync(fmt.Sprintf("orders.%d", i), []byte("OK")) } <-js.PublishAsyncComplete() sdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir) s.Shutdown() // Let them expire while not running. time.Sleep(ttl + 500*time.Millisecond) start := time.Now() opts.Port = -1 opts.StoreDir = sdir s = RunServer(&opts) elapsed := time.Since(start) defer s.Shutdown() if elapsed > 2*time.Second { t.Fatalf("Took %v for restart which is too long", elapsed) } // Check everything is correct. nc, js = jsClientConnect(t, s) defer nc.Close() si, err := js.StreamInfo("ORDERS") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { t.Fatalf("Expected no msgs after restart, got %d", si.State.Msgs) } } func TestNoRaceJetStreamStalledMirrorsAfterExpire(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Replicas: 1, MaxAge: 100 * time.Millisecond, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Error creating stream: %v", err) } if _, err := js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 2, Mirror: &nats.StreamSource{Name: "TEST"}, }); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch := func(batch int) { t.Helper() for i := 0; i < batch; i++ { js.PublishAsync("foo.bar", []byte("Hello")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } } numMsgs := 10_000 sendBatch(numMsgs) // Turn off expiration so we can test we did not stall. cfg.MaxAge = 0 if _, err := js.UpdateStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } sendBatch(numMsgs) // Wait for mirror to be caught up. checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo("M") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.LastSeq != uint64(2*numMsgs) { return fmt.Errorf("Expected %d as last sequence, got state: %+v", 2*numMsgs, si.State) } return nil }) } // We will use JetStream helpers to create supercluster but this test is about exposing the ability to access // account scoped connz with subject interest filtering. func TestNoRaceJetStreamSuperClusterAccountConnz(t *testing.T) { // This has 4 different account, 3 general and system. sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 3) defer sc.shutdown() // Create 20 connections on account one and two // Create JetStream assets for each as well to make sure by default we do not report them. num := 20 for i := 0; i < num; i++ { nc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo("one", "p"), nats.Name("one")) defer nc.Close() if i%2 == 0 { nc.SubscribeSync("foo") } else { nc.SubscribeSync("bar") } nc, js := jsClientConnect(t, sc.randomServer(), nats.UserInfo("two", "p"), nats.Name("two")) defer nc.Close() nc.SubscribeSync("baz") nc.SubscribeSync("foo.bar.*") nc.SubscribeSync(fmt.Sprintf("id.%d", i+1)) js.AddStream(&nats.StreamConfig{Name: fmt.Sprintf("TEST:%d", i+1)}) } type czapi struct { Server *ServerInfo Data *Connz Error *ApiError } parseConnz := func(buf []byte) *Connz { t.Helper() var cz czapi if err := json.Unmarshal(buf, &cz); err != nil { t.Fatalf("Unexpected error: %v", err) } if cz.Error != nil { t.Fatalf("Unexpected error: %+v", cz.Error) } return cz.Data } doRequest := func(reqSubj, acc, filter string, expected int) { t.Helper() nc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo(acc, "p"), nats.Name(acc)) defer nc.Close() mch := make(chan *nats.Msg, 9) sub, _ := nc.ChanSubscribe(nats.NewInbox(), mch) var req []byte if filter != _EMPTY_ { req, _ = json.Marshal(&ConnzOptions{FilterSubject: filter}) } if err := nc.PublishRequest(reqSubj, sub.Subject, req); err != nil { t.Fatalf("Unexpected error: %v", err) } // So we can igniore ourtselves. cid, _ := nc.GetClientID() sid := nc.ConnectedServerId() wt := time.NewTimer(200 * time.Millisecond) var conns []*ConnInfo LOOP: for { select { case m := <-mch: if len(m.Data) == 0 { t.Fatalf("No responders") } cr := parseConnz(m.Data) // For account scoped, NumConns and Total should be the same (sans limits and offsets). // It Total should not include other accounts since that would leak information about the system. if filter == _EMPTY_ && cr.NumConns != cr.Total { t.Fatalf("NumConns and Total should be same with account scoped connz, got %+v", cr) } for _, c := range cr.Conns { if c.Name != acc { t.Fatalf("Got wrong account: %q vs %q for %+v", acc, c.Account, c) } if !(c.Cid == cid && cr.ID == sid) { conns = append(conns, c) } } wt.Reset(200 * time.Millisecond) case <-wt.C: break LOOP } } if len(conns) != expected { t.Fatalf("Expected to see %d conns but got %d", expected, len(conns)) } } doSysRequest := func(acc string, expected int) { t.Helper() doRequest("$SYS.REQ.SERVER.PING.CONNZ", acc, _EMPTY_, expected) } doAccRequest := func(acc string, expected int) { t.Helper() doRequest("$SYS.REQ.ACCOUNT.PING.CONNZ", acc, _EMPTY_, expected) } doFiltered := func(acc, filter string, expected int) { t.Helper() doRequest("$SYS.REQ.SERVER.PING.CONNZ", acc, filter, expected) } doSysRequest("one", 20) doAccRequest("one", 20) doSysRequest("two", 20) doAccRequest("two", 20) // Now check filtering. doFiltered("one", _EMPTY_, 20) doFiltered("one", ">", 20) doFiltered("one", "bar", 10) doFiltered("two", "bar", 0) doFiltered("two", "id.1", 1) doFiltered("two", "id.*", 20) doFiltered("two", "foo.bar.*", 20) doFiltered("two", "foo.>", 20) } func TestNoRaceCompressedConnz(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() doRequest := func(compress string) { t.Helper() m := nats.NewMsg("$SYS.REQ.ACCOUNT.PING.CONNZ") m.Header.Add("Accept-Encoding", compress) resp, err := nc.RequestMsg(m, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } buf := resp.Data // Make sure we have an encoding header. ce := resp.Header.Get("Content-Encoding") switch strings.ToLower(ce) { case "gzip": zr, err := gzip.NewReader(bytes.NewReader(buf)) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer zr.Close() buf, err = io.ReadAll(zr) if err != nil && err != io.ErrUnexpectedEOF { t.Fatalf("Unexpected error: %v", err) } case "snappy", "s2": sr := s2.NewReader(bytes.NewReader(buf)) buf, err = io.ReadAll(sr) if err != nil && err != io.ErrUnexpectedEOF { t.Fatalf("Unexpected error: %v", err) } default: t.Fatalf("Unknown content-encoding of %q", ce) } var cz ServerAPIConnzResponse if err := json.Unmarshal(buf, &cz); err != nil { t.Fatalf("Unexpected error: %v", err) } if cz.Error != nil { t.Fatalf("Unexpected error: %+v", cz.Error) } } doRequest("gzip") doRequest("snappy") doRequest("s2") } func TestNoRaceJetStreamClusterExtendedStreamPurge(t *testing.T) { for _, st := range []StorageType{FileStorage, MemoryStorage} { t.Run(st.String(), func(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := StreamConfig{ Name: "KV", Subjects: []string{"kv.>"}, Storage: st, Replicas: 2, MaxMsgsPer: 100, } req, err := json.Marshal(cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) c.waitOnStreamLeader("$G", "KV") si, err := js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si == nil || si.Config.Name != "KV" { t.Fatalf("StreamInfo is not correct %+v", si) } for i := 0; i < 1000; i++ { js.PublishAsync("kv.foo", []byte("OK")) // 1 * i js.PublishAsync("kv.bar", []byte("OK")) // 2 * i js.PublishAsync("kv.baz", []byte("OK")) // 3 * i } // First is 2700, last is 3000 for i := 0; i < 700; i++ { js.PublishAsync(fmt.Sprintf("kv.%d", i+1), []byte("OK")) } // Now first is 2700, last is 3700 select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } si, err = js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 1000 { t.Fatalf("Expected %d msgs, got %d", 1000, si.State.Msgs) } shouldFail := func(preq *JSApiStreamPurgeRequest) { req, _ := json.Marshal(preq) resp, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "KV"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var pResp JSApiStreamPurgeResponse if err = json.Unmarshal(resp.Data, &pResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if pResp.Success || pResp.Error == nil { t.Fatalf("Expected an error response but got none") } } // Sequence and Keep should be mutually exclusive. shouldFail(&JSApiStreamPurgeRequest{Sequence: 10, Keep: 10}) purge := func(preq *JSApiStreamPurgeRequest, newTotal uint64) { t.Helper() req, _ := json.Marshal(preq) resp, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, "KV"), req, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var pResp JSApiStreamPurgeResponse if err = json.Unmarshal(resp.Data, &pResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if !pResp.Success || pResp.Error != nil { t.Fatalf("Got a bad response %+v", pResp) } si, err = js.StreamInfo("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != newTotal { t.Fatalf("Expected total after purge to be %d but got %d", newTotal, si.State.Msgs) } } expectLeft := func(subject string, expected uint64) { t.Helper() ci, err := js.AddConsumer("KV", &nats.ConsumerConfig{Durable: "dlc", FilterSubject: subject, AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer js.DeleteConsumer("KV", "dlc") if ci.NumPending != expected { t.Fatalf("Expected %d remaining but got %d", expected, ci.NumPending) } } purge(&JSApiStreamPurgeRequest{Subject: "kv.foo"}, 900) expectLeft("kv.foo", 0) purge(&JSApiStreamPurgeRequest{Subject: "kv.bar", Keep: 1}, 801) expectLeft("kv.bar", 1) purge(&JSApiStreamPurgeRequest{Subject: "kv.baz", Sequence: 2851}, 751) expectLeft("kv.baz", 50) purge(&JSApiStreamPurgeRequest{Subject: "kv.*"}, 0) // RESET js.DeleteStream("KV") // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) c.waitOnStreamLeader("$G", "KV") if _, err := js.StreamInfo("KV"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Put in 100. for i := 0; i < 100; i++ { js.PublishAsync("kv.foo", []byte("OK")) } select { case <-js.PublishAsyncComplete(): case <-time.After(time.Second): t.Fatalf("Did not receive completion signal") } purge(&JSApiStreamPurgeRequest{Subject: "kv.foo", Keep: 10}, 10) purge(&JSApiStreamPurgeRequest{Subject: "kv.foo", Keep: 10}, 10) expectLeft("kv.foo", 10) // RESET AGAIN js.DeleteStream("KV") // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) c.waitOnStreamLeader("$G", "KV") if _, err := js.StreamInfo("KV"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Put in 100. for i := 0; i < 100; i++ { js.Publish("kv.foo", []byte("OK")) } purge(&JSApiStreamPurgeRequest{Keep: 10}, 10) expectLeft(">", 10) // RESET AGAIN js.DeleteStream("KV") // Do manually for now. nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second) if _, err := js.StreamInfo("KV"); err != nil { t.Fatalf("Unexpected error: %v", err) } // Put in 100. for i := 0; i < 100; i++ { js.Publish("kv.foo", []byte("OK")) } purge(&JSApiStreamPurgeRequest{Sequence: 90}, 11) // Up to 90 so we keep that, hence the 11. expectLeft(">", 11) }) } } func TestNoRaceJetStreamFileStoreCompaction(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() cfg := &nats.StreamConfig{ Name: "KV", Subjects: []string{"KV.>"}, MaxMsgsPerSubject: 1, } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } toSend := 10_000 data := make([]byte, 4*1024) crand.Read(data) // First one. js.PublishAsync("KV.FM", data) for i := 0; i < toSend; i++ { js.PublishAsync(fmt.Sprintf("KV.%d", i+1), data) } // Do again and overwrite the previous batch. for i := 0; i < toSend; i++ { js.PublishAsync(fmt.Sprintf("KV.%d", i+1), data) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } // Now check by hand the utilization level. mset, err := s.GlobalAccount().lookupStream("KV") if err != nil { t.Fatalf("Unexpected error: %v", err) } total, used, _ := mset.Store().Utilization() if pu := 100.0 * float32(used) / float32(total); pu < 80.0 { t.Fatalf("Utilization is less than 80%%, got %.2f", pu) } } func TestNoRaceJetStreamEncryptionEnabledOnRestartWithExpire(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream { store_dir = %q } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() config := s.JetStreamConfig() if config == nil { t.Fatalf("Expected config but got none") } defer removeDir(t, config.StoreDir) nc, js := jsClientConnect(t, s) defer nc.Close() toSend := 10_000 cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, MaxMsgs: int64(toSend), } if _, err := js.AddStream(cfg); err != nil { t.Fatalf("Unexpected error: %v", err) } data := make([]byte, 4*1024) // 4K payload crand.Read(data) for i := 0; i < toSend; i++ { js.PublishAsync("foo", data) js.PublishAsync("bar", data) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dlc", AckPolicy: nats.AckExplicitPolicy}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Restart nc.Close() s.Shutdown() ncs := fmt.Sprintf("\nlisten: 127.0.0.1:-1\njetstream: {key: %q, store_dir: %q}\n", "s3cr3t!", config.StoreDir) conf = createConfFile(t, []byte(ncs)) // Try to drain entropy to see if effects startup time. drain := make([]byte, 32*1024*1024) // Pull 32Mb of crypto rand. crand.Read(drain) start := time.Now() s, _ = RunServerWithConfig(conf) defer s.Shutdown() dd := time.Since(start) if dd > 5*time.Second { t.Fatalf("Restart took longer than expected: %v", dd) } } // This test was from Ivan K. and showed a bug in the filestore implementation. // This is skipped by default since it takes >40s to run. func TestNoRaceJetStreamOrderedConsumerMissingMsg(t *testing.T) { // Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm. skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{ Name: "benchstream", Subjects: []string{"testsubject"}, Replicas: 1, }); err != nil { t.Fatalf("add stream failed: %s", err) } total := 1_000_000 numSubs := 10 ch := make(chan struct{}, numSubs) wg := sync.WaitGroup{} wg.Add(numSubs) errCh := make(chan error, 1) for i := 0; i < numSubs; i++ { nc, js := jsClientConnect(t, s) defer nc.Close() go func(nc *nats.Conn, js nats.JetStreamContext) { defer wg.Done() received := 0 _, err := js.Subscribe("testsubject", func(m *nats.Msg) { meta, _ := m.Metadata() if meta.Sequence.Consumer != meta.Sequence.Stream { nc.Close() errCh <- fmt.Errorf("Bad meta: %+v", meta) } received++ if received == total { ch <- struct{}{} } }, nats.OrderedConsumer()) if err != nil { select { case errCh <- fmt.Errorf("Error creating sub: %v", err): default: } } }(nc, js) } wg.Wait() select { case e := <-errCh: t.Fatal(e) default: } payload := make([]byte, 500) for i := 1; i <= total; i++ { js.PublishAsync("testsubject", payload) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not send all messages") } // Now wait for consumers to be done: for i := 0; i < numSubs; i++ { select { case <-ch: case <-time.After(10 * time.Second): t.Fatal("Did not receive all messages for all consumers in time") } } } // Issue #2488 - Bad accounting, can not reproduce the stalled consumers after last several PRs. // Issue did show bug in ack logic for no-ack and interest based retention. func TestNoRaceJetStreamClusterInterestPolicyAckNone(t *testing.T) { for _, test := range []struct { name string durable string }{ {"durable", "dlc"}, {"ephemeral", _EMPTY_}, } { t.Run(test.name, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "cluster", Subjects: []string{"cluster.*"}, Retention: nats.InterestPolicy, Discard: nats.DiscardOld, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } var received uint32 mh := func(m *nats.Msg) { atomic.AddUint32(&received, 1) } opts := []nats.SubOpt{nats.DeliverNew(), nats.AckNone()} if test.durable != _EMPTY_ { opts = append(opts, nats.Durable(test.durable)) } _, err = js.Subscribe("cluster.created", mh, opts...) if err != nil { t.Fatalf("Unexpected error: %v", err) } msg := []byte("ACK ME") const total = uint32(1_000) for i := 0; i < int(total); i++ { if _, err := js.Publish("cluster.created", msg); err != nil { t.Fatalf("Unexpected error: %v", err) } //time.Sleep(100 * time.Microsecond) } // Wait for all messages to be received. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { r := atomic.LoadUint32(&received) if r == total { return nil } return fmt.Errorf("Received only %d out of %d", r, total) }) checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("cluster") if err != nil { t.Fatalf("Error getting stream info: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected no messages, got %d", si.State.Msgs) } return nil }) }) } } // There was a bug in the filestore compact code that would cause a store // with JSExpectedLastSubjSeq to fail with "wrong last sequence: 0" func TestNoRaceJetStreamLastSubjSeqAndFilestoreCompact(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client based API nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "MQTT_sess", Subjects: []string{"MQTT.sess.>"}, Storage: nats.FileStorage, Retention: nats.LimitsPolicy, Replicas: 1, MaxMsgsPerSubject: 1, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } firstPayload := make([]byte, 40) secondPayload := make([]byte, 380) for iter := 0; iter < 2; iter++ { for i := 0; i < 4000; i++ { subj := "MQTT.sess." + getHash(fmt.Sprintf("client_%d", i)) pa, err := js.Publish(subj, firstPayload) if err != nil { t.Fatalf("Error on publish: %v", err) } m := nats.NewMsg(subj) m.Data = secondPayload eseq := strconv.FormatInt(int64(pa.Sequence), 10) m.Header.Set(JSExpectedLastSubjSeq, eseq) if _, err := js.PublishMsg(m); err != nil { t.Fatalf("Error on publish (iter=%v seq=%v): %v", iter+1, pa.Sequence, err) } } } } // Issue #2548 func TestNoRaceJetStreamClusterMemoryStreamConsumerRaftGrowth(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "memory-leak", Subjects: []string{"memory-leak"}, Retention: nats.LimitsPolicy, MaxMsgs: 1000, Discard: nats.DiscardOld, MaxAge: time.Minute, Storage: nats.MemoryStorage, Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = js.QueueSubscribe("memory-leak", "q1", func(msg *nats.Msg) { time.Sleep(1 * time.Second) msg.AckSync() }) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Send 10k (Must be > 8192 which is compactNumMin from monitorConsumer. msg := []byte("NATS is a connective technology that powers modern distributed systems.") for i := 0; i < 10_000; i++ { if _, err := js.Publish("memory-leak", msg); err != nil { t.Fatalf("Unexpected error: %v", err) } } // We will verify here that the underlying raft layer for the leader is not > 8192 cl := c.consumerLeader("$G", "memory-leak", "q1") mset, err := cl.GlobalAccount().lookupStream("memory-leak") if err != nil { t.Fatalf("Unexpected error: %v", err) } o := mset.lookupConsumer("q1") if o == nil { t.Fatalf("Error looking up consumer %q", "q1") } node := o.raftNode().(*raft) checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { if ms := node.wal.(*memStore); ms.State().Msgs > 8192 { return fmt.Errorf("Did not compact the raft memory WAL") } return nil }) } func TestNoRaceJetStreamClusterCorruptWAL(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Subjects: []string{"foo"}, Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } sub, err := js.PullSubscribe("foo", "dlc") if err != nil { t.Fatalf("Unexpected error: %v", err) } numMsgs := 1000 for i := 0; i < numMsgs; i++ { js.PublishAsync("foo", []byte("WAL")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } for i, m := range fetchMsgs(t, sub, 200, 5*time.Second) { // Ack first 50 and every other even on after that.. if i < 50 || i%2 == 1 { m.AckSync() } } // Make sure acks processed. time.Sleep(200 * time.Millisecond) nc.Close() // Check consumer consistency. checkConsumerWith := func(delivered, ackFloor uint64, ackPending int) { t.Helper() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { ci, err := js.ConsumerInfo("TEST", "dlc") if err != nil { return fmt.Errorf("Unexpected error: %v", err) } if ci.Delivered.Consumer != ci.Delivered.Stream || ci.Delivered.Consumer != delivered { return fmt.Errorf("Expected %d for delivered, got %+v", delivered, ci.Delivered) } if ci.AckFloor.Consumer != ci.AckFloor.Stream || ci.AckFloor.Consumer != ackFloor { return fmt.Errorf("Expected %d for ack floor, got %+v", ackFloor, ci.AckFloor) } nm := uint64(numMsgs) if ci.NumPending != nm-delivered { return fmt.Errorf("Expected num pending to be %d, got %d", nm-delivered, ci.NumPending) } if ci.NumAckPending != ackPending { return fmt.Errorf("Expected num ack pending to be %d, got %d", ackPending, ci.NumAckPending) } return nil }) } checkConsumer := func() { t.Helper() checkConsumerWith(200, 50, 75) } checkConsumer() // Grab the consumer leader. cl := c.consumerLeader("$G", "TEST", "dlc") mset, err := cl.GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } o := mset.lookupConsumer("dlc") if o == nil { t.Fatalf("Error looking up consumer %q", "dlc") } // Grab underlying raft node and the WAL (filestore) and we will attempt to "corrupt" it. node := o.raftNode().(*raft) // We are doing a stop here to prevent the internal consumer snapshot from happening on exit node.Stop() fs := node.wal.(*fileStore) fcfg, cfg := fs.fcfg, fs.cfg.StreamConfig // Stop all the servers. c.stopAll() // Manipulate directly with cluster down. fs, err = newFileStore(fcfg, cfg) if err != nil { t.Fatalf("Unexpected error: %v", err) } state := fs.State() sm, err := fs.LoadMsg(state.LastSeq, nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } ae, err := node.decodeAppendEntry(sm.msg, nil, _EMPTY_) if err != nil { t.Fatalf("Unexpected error: %v", err) } dentry := func(dseq, sseq, dc uint64, ts int64) []byte { b := make([]byte, 4*binary.MaxVarintLen64+1) b[0] = byte(updateDeliveredOp) n := 1 n += binary.PutUvarint(b[n:], dseq) n += binary.PutUvarint(b[n:], sseq) n += binary.PutUvarint(b[n:], dc) n += binary.PutVarint(b[n:], ts) return b[:n] } // Let's put a non-contigous AppendEntry into the system. ae.pindex += 10 // Add in delivered record. ae.entries = []*Entry{{EntryNormal, dentry(1000, 1000, 1, time.Now().UnixNano())}} encoded, err := ae.encode(nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } if _, _, err := fs.StoreMsg(_EMPTY_, nil, encoded); err != nil { t.Fatalf("Unexpected error: %v", err) } fs.Stop() c.restartAllSamePorts() c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "dlc") checkConsumer() // Now we will truncate out the WAL out from underneath the leader. // Grab the consumer leader. nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() cl = c.consumerLeader("$G", "TEST", "dlc") mset, err = cl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o = mset.lookupConsumer("dlc") require_NoError(t, err) // Grab underlying raft node and the WAL (filestore) and truncate it. // This will simulate the WAL losing state due to truncate and we want to make sure it recovers. fs = o.raftNode().(*raft).wal.(*fileStore) state = fs.State() err = fs.Truncate(state.FirstSeq) require_True(t, err == nil || err == ErrInvalidSequence) state = fs.State() sub, err = js.PullSubscribe("foo", "dlc") require_NoError(t, err) // This will cause us to stepdown and truncate our WAL. sub.Fetch(100) c.waitOnConsumerLeader("$G", "TEST", "dlc") // We can't trust the results sans that we have a leader back in place and the ackFloor. ci, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) if ci.AckFloor.Consumer != ci.AckFloor.Stream || ci.AckFloor.Consumer != 50 { t.Fatalf("Expected %d for ack floor, got %+v", 50, ci.AckFloor) } } func TestNoRaceJetStreamClusterInterestRetentionDeadlock(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // This can trigger deadlock with current architecture. // If stream is !limitsRetention and consumer is DIRECT and ack none we will try to place the msg seq // onto a chan for the stream to consider removing. All conditions above must hold to trigger. // We will attempt to trigger here with a stream mirror setup which uses and R=1 DIRECT consumer to replicate msgs. _, err := js.AddStream(&nats.StreamConfig{Name: "S", Retention: nats.InterestPolicy, Storage: nats.MemoryStorage}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Create a mirror which will create the consumer profile to trigger. _, err = js.AddStream(&nats.StreamConfig{Name: "M", Mirror: &nats.StreamSource{Name: "S"}}) if err != nil { t.Fatalf("Unexpected error: %v", err) } // Queue up alot of messages. numRequests := 20_000 for i := 0; i < numRequests; i++ { js.PublishAsync("S", []byte("Q")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("S") if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != 0 { return fmt.Errorf("Expected 0 msgs, got state: %+v", si.State) } return nil }) } func TestNoRaceJetStreamClusterMaxConsumersAndDirect(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() // We want to max sure max consumer limits do not affect mirrors or sources etc. _, err := js.AddStream(&nats.StreamConfig{Name: "S", Storage: nats.MemoryStorage, MaxConsumers: 1}) if err != nil { t.Fatalf("Unexpected error: %v", err) } var mirrors []string for i := 0; i < 10; i++ { // Create a mirror. mname := fmt.Sprintf("M-%d", i+1) mirrors = append(mirrors, mname) _, err = js.AddStream(&nats.StreamConfig{Name: mname, Mirror: &nats.StreamSource{Name: "S"}}) if err != nil { t.Fatalf("Unexpected error: %v", err) } } // Queue up messages. numRequests := 20 for i := 0; i < numRequests; i++ { js.Publish("S", []byte("Q")) } checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for _, mname := range mirrors { si, err := js.StreamInfo(mname) if err != nil { t.Fatalf("Unexpected error: %v", err) } if si.State.Msgs != uint64(numRequests) { return fmt.Errorf("Expected %d msgs for %q, got state: %+v", numRequests, mname, si.State) } } return nil }) } // Make sure when we try to hard reset a stream state in a cluster that we also re-create the consumers. func TestNoRaceJetStreamClusterStreamReset(t *testing.T) { // Speed up raft omin, omax, ohb := minElectionTimeout, maxElectionTimeout, hbInterval minElectionTimeout = 250 * time.Millisecond maxElectionTimeout = time.Second hbInterval = 50 * time.Millisecond defer func() { minElectionTimeout = omin maxElectionTimeout = omax hbInterval = ohb }() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Replicas: 2, Retention: nats.WorkQueuePolicy, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } numRequests := 20 for i := 0; i < numRequests; i++ { js.Publish("foo.created", []byte("REQ")) } // Durable. sub, err := js.SubscribeSync("foo.created", nats.Durable("d1")) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer sub.Unsubscribe() si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.Msgs == uint64(numRequests)) // Let settle a bit for Go routine checks. time.Sleep(500 * time.Millisecond) // Grab number go routines. base := runtime.NumGoroutine() // Make the consumer busy here by async sending a bunch of messages. for i := 0; i < numRequests*10; i++ { js.PublishAsync("foo.created", []byte("REQ")) } // Grab a server that is the consumer leader for the durable. cl := c.consumerLeader("$G", "TEST", "d1") mset, err := cl.GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } // Do a hard reset here by hand. mset.resetClusteredState(nil) // Wait til we have the consumer leader re-elected. c.waitOnConsumerLeader("$G", "TEST", "d1") // Make sure we can get the consumer info eventually. checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { _, err := js.ConsumerInfo("TEST", "d1", nats.MaxWait(250*time.Millisecond)) return err }) checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { if after := runtime.NumGoroutine(); base > after { return fmt.Errorf("Expected %d go routines, got %d", base, after) } return nil }) // Simulate a low level write error on our consumer and make sure we can recover etc. checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { if cl = c.consumerLeader("$G", "TEST", "d1"); cl != nil { return nil } return errors.New("waiting on consumer leader") }) mset, err = cl.GlobalAccount().lookupStream("TEST") if err != nil { t.Fatalf("Unexpected error: %v", err) } o := mset.lookupConsumer("d1") if o == nil { t.Fatalf("Did not retrieve consumer") } node := o.raftNode().(*raft) if node == nil { t.Fatalf("could not retrieve the raft node for consumer") } nc.Close() node.setWriteErr(io.ErrShortWrite) c.stopAll() c.restartAll() c.waitOnStreamLeader("$G", "TEST") c.waitOnConsumerLeader("$G", "TEST", "d1") } // Reports of high cpu on compaction for a KV store. func TestNoRaceJetStreamKeyValueCompaction(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "COMPACT", Replicas: 3, }) if err != nil { t.Fatalf("Unexpected error: %v", err) } value := strings.Repeat("A", 128*1024) for i := 0; i < 5_000; i++ { key := fmt.Sprintf("K-%d", rand.Intn(256)+1) if _, err := kv.PutString(key, value); err != nil { t.Fatalf("Unexpected error: %v", err) } } } // Trying to recreate an issue rip saw with KV and server restarts complaining about // mismatch for a few minutes and growing memory. func TestNoRaceJetStreamClusterStreamSeqMismatchIssue(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "MM", Replicas: 3, TTL: 500 * time.Millisecond, }) require_NoError(t, err) for i := 1; i <= 10; i++ { if _, err := kv.PutString("k", "1"); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Close in case we are connected here. Will recreate. nc.Close() // Shutdown a non-leader. s := c.randomNonStreamLeader("$G", "KV_MM") s.Shutdown() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err = js.KeyValue("MM") require_NoError(t, err) // Now change the state of the stream such that we have to do a compact upon restart // of the downed server. for i := 1; i <= 10; i++ { if _, err := kv.PutString("k", "2"); err != nil { t.Fatalf("Unexpected error: %v", err) } } // Raft could save us here so need to run a compact on the leader. snapshotLeader := func() { sl := c.streamLeader("$G", "KV_MM") if sl == nil { t.Fatalf("Did not get the leader") } mset, err := sl.GlobalAccount().lookupStream("KV_MM") require_NoError(t, err) node := mset.raftNode() if node == nil { t.Fatalf("Could not get stream group") } if err := node.InstallSnapshot(mset.stateSnapshot()); err != nil { t.Fatalf("Error installing snapshot: %v", err) } } // Now wait for expiration time.Sleep(time.Second) snapshotLeader() s = c.restartServer(s) c.waitOnServerCurrent(s) // We want to make sure we do not reset the raft state on a catchup due to no request yield. // Bug was if we did not actually request any help from snapshot we did not set mset.lseq properly. // So when we send next batch that would cause raft reset due to cluster reset for our stream. mset, err := s.GlobalAccount().lookupStream("KV_MM") require_NoError(t, err) for i := 1; i <= 10; i++ { if _, err := kv.PutString("k1", "X"); err != nil { t.Fatalf("Unexpected error: %v", err) } } c.waitOnStreamCurrent(s, "$G", "KV_MM") // Make sure we did not reset our stream. msetNew, err := s.GlobalAccount().lookupStream("KV_MM") require_NoError(t, err) if msetNew != mset { t.Fatalf("Stream was reset") } } func TestNoRaceJetStreamClusterStreamDropCLFS(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "CLFS", Replicas: 3, }) require_NoError(t, err) // Will work _, err = kv.Create("k.1", []byte("X")) require_NoError(t, err) // Drive up CLFS state on leader. for i := 0; i < 10; i++ { _, err = kv.Create("k.1", []byte("X")) require_Error(t, err) } // Bookend with new key success. _, err = kv.Create("k.2", []byte("Z")) require_NoError(t, err) // Close in case we are connected here. Will recreate. nc.Close() // Shutdown, which will also clear clfs. s := c.randomNonStreamLeader("$G", "KV_CLFS") s.Shutdown() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err = js.KeyValue("CLFS") require_NoError(t, err) // Drive up CLFS state on leader. for i := 0; i < 10; i++ { _, err = kv.Create("k.1", []byte("X")) require_Error(t, err) } sl := c.streamLeader("$G", "KV_CLFS") if sl == nil { t.Fatalf("Did not get the leader") } mset, err := sl.GlobalAccount().lookupStream("KV_CLFS") require_NoError(t, err) node := mset.raftNode() if node == nil { t.Fatalf("Could not get stream group") } if err := node.InstallSnapshot(mset.stateSnapshot()); err != nil { t.Fatalf("Error installing snapshot: %v", err) } _, err = kv.Create("k.3", []byte("ZZZ")) require_NoError(t, err) s = c.restartServer(s) c.waitOnServerCurrent(s) mset, err = s.GlobalAccount().lookupStream("KV_CLFS") require_NoError(t, err) _, err = kv.Create("k.4", []byte("YYY")) require_NoError(t, err) c.waitOnStreamCurrent(s, "$G", "KV_CLFS") // Make sure we did not reset our stream. msetNew, err := s.GlobalAccount().lookupStream("KV_CLFS") require_NoError(t, err) if msetNew != mset { t.Fatalf("Stream was reset") } } func TestNoRaceJetStreamMemstoreWithLargeInteriorDeletes(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage, }) require_NoError(t, err) acc, err := s.lookupAccount("$G") require_NoError(t, err) mset, err := acc.lookupStream("TEST") require_NoError(t, err) msg := []byte("Hello World!") if _, err := js.PublishAsync("foo", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } for i := 1; i <= 1_000_000; i++ { if _, err := js.PublishAsync("bar", msg); err != nil { t.Fatalf("Unexpected publish error: %v", err) } } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } now := time.Now() ss := mset.stateWithDetail(true) // Before the fix the snapshot for this test would be > 200ms on my setup. if elapsed := time.Since(now); elapsed > 100*time.Millisecond { t.Fatalf("Took too long to snapshot: %v", elapsed) } else if elapsed > 50*time.Millisecond { t.Logf("WRN: Took longer than usual to snapshot: %v", elapsed) } if ss.Msgs != 2 || ss.FirstSeq != 1 || ss.LastSeq != 1_000_001 || ss.NumDeleted != 999999 { // To not print out on error. ss.Deleted = nil t.Fatalf("Bad State: %+v", ss) } } // This is related to an issue reported where we were exhausting threads by trying to // cleanup too many consumers at the same time. // https://github.com/nats-io/nats-server/issues/2742 func TestNoRaceJetStreamConsumerFileStoreConcurrentDiskIO(t *testing.T) { storeDir := t.TempDir() // Artificially adjust our environment for this test. gmp := runtime.GOMAXPROCS(32) defer runtime.GOMAXPROCS(gmp) maxT := debug.SetMaxThreads(1050) // 1024 now defer debug.SetMaxThreads(maxT) fs, err := newFileStore(FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "MT", Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() startCh := make(chan bool) var wg sync.WaitGroup var swg sync.WaitGroup ts := time.Now().UnixNano() // Create 1000 consumerStores n := 1000 swg.Add(n) for i := 1; i <= n; i++ { name := fmt.Sprintf("o%d", i) o, err := fs.ConsumerStore(name, &ConsumerConfig{AckPolicy: AckExplicit}) require_NoError(t, err) wg.Add(1) swg.Done() go func() { defer wg.Done() // Will make everyone run concurrently. <-startCh o.UpdateDelivered(22, 22, 1, ts) buf, _ := o.(*consumerFileStore).encodeState() o.(*consumerFileStore).writeState(buf) o.Delete() }() } swg.Wait() close(startCh) wg.Wait() } func TestNoRaceJetStreamClusterHealthz(t *testing.T) { c := createJetStreamCluster(t, jsClusterAccountsTempl, "HZ", _EMPTY_, 3, 23033, true) defer c.shutdown() nc1, js1 := jsClientConnect(t, c.randomServer(), nats.UserInfo("one", "p")) defer nc1.Close() nc2, js2 := jsClientConnect(t, c.randomServer(), nats.UserInfo("two", "p")) defer nc2.Close() var err error for _, sname := range []string{"foo", "bar", "baz"} { _, err = js1.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3}) require_NoError(t, err) _, err = js2.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3}) require_NoError(t, err) } // R1 _, err = js1.AddStream(&nats.StreamConfig{Name: "r1", Replicas: 1}) require_NoError(t, err) // Now shutdown then send a bunch of data. s := c.servers[0] s.Shutdown() for i := 0; i < 5_000; i++ { _, err = js1.PublishAsync("foo", []byte("OK")) require_NoError(t, err) _, err = js2.PublishAsync("bar", []byte("OK")) require_NoError(t, err) } select { case <-js1.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } select { case <-js2.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } s = c.restartServer(s) opts := s.getOpts() opts.HTTPHost = "127.0.0.1" opts.HTTPPort = 11222 err = s.StartMonitoring() require_NoError(t, err) url := fmt.Sprintf("http://127.0.0.1:%d/healthz", opts.HTTPPort) getHealth := func() (int, *HealthStatus) { resp, err := http.Get(url) require_NoError(t, err) defer resp.Body.Close() body, err := io.ReadAll(resp.Body) require_NoError(t, err) var hs HealthStatus err = json.Unmarshal(body, &hs) require_NoError(t, err) return resp.StatusCode, &hs } errors := 0 checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { code, hs := getHealth() if code >= 200 && code < 300 { return nil } errors++ return fmt.Errorf("Got %d status with %+v", code, hs) }) if errors == 0 { t.Fatalf("Expected to have some errors until we became current, got none") } } // Test that we can receive larger messages with stream subject details. // Also test that we will fail at some point and the user can fall back to // an orderedconsumer like we do with watch for KV Keys() call. func TestNoRaceJetStreamStreamInfoSubjectDetailsLimits(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 jetstream { store_dir = %q } accounts: { default: { jetstream: true users: [ {user: me, password: pwd} ] limits { max_payload: 512 } } } `, t.TempDir()))) s, _ := RunServerWithConfig(conf) if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) } defer s.Shutdown() nc, js := jsClientConnect(t, s, nats.UserInfo("me", "pwd")) defer nc.Close() // Make sure to flush so we process the 2nd INFO. nc.Flush() // Make sure we cannot send larger than 512 bytes. // But we can receive larger. sub, err := nc.SubscribeSync("foo") require_NoError(t, err) err = nc.Publish("foo", []byte(strings.Repeat("A", 600))) require_Error(t, err, nats.ErrMaxPayload) sub.Unsubscribe() _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*", "X.*"}, }) require_NoError(t, err) n := JSMaxSubjectDetails for i := 0; i < n; i++ { _, err := js.PublishAsync(fmt.Sprintf("X.%d", i), []byte("OK")) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Need to grab StreamInfo by hand for now. req, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: "X.*"}) require_NoError(t, err) resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), req, 5*time.Second) require_NoError(t, err) var si StreamInfo err = json.Unmarshal(resp.Data, &si) require_NoError(t, err) if len(si.State.Subjects) != n { t.Fatalf("Expected to get %d subject details, got %d", n, len(si.State.Subjects)) } // Now add one more message to check pagination _, err = js.Publish("foo", []byte("TOO MUCH")) require_NoError(t, err) req, err = json.Marshal(&JSApiStreamInfoRequest{ApiPagedRequest: ApiPagedRequest{Offset: n}, SubjectsFilter: nats.AllKeys}) require_NoError(t, err) resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), req, 5*time.Second) require_NoError(t, err) var sir JSApiStreamInfoResponse err = json.Unmarshal(resp.Data, &sir) require_NoError(t, err) if len(sir.State.Subjects) != 1 { t.Fatalf("Expected to get 1 extra subject detail, got %d", len(sir.State.Subjects)) } } func TestNoRaceJetStreamSparseConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() msg := []byte("ok") cases := []struct { name string mconfig *nats.StreamConfig }{ {"MemoryStore", &nats.StreamConfig{Name: "TEST", Storage: nats.MemoryStorage, MaxMsgsPerSubject: 25_000_000, Subjects: []string{"*"}}}, {"FileStore", &nats.StreamConfig{Name: "TEST", Storage: nats.FileStorage, MaxMsgsPerSubject: 25_000_000, Subjects: []string{"*"}}}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { js.DeleteStream("TEST") _, err := js.AddStream(c.mconfig) require_NoError(t, err) // We will purposely place foo msgs near the beginning, then in middle, then at the end. for n := 0; n < 2; n++ { _, err = js.PublishAsync("foo", msg, nats.StallWait(800*time.Millisecond)) require_NoError(t, err) for i := 0; i < 1_000_000; i++ { _, err = js.PublishAsync("bar", msg, nats.StallWait(800*time.Millisecond)) require_NoError(t, err) } _, err = js.PublishAsync("foo", msg, nats.StallWait(800*time.Millisecond)) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Now create a consumer on foo. ci, err := js.AddConsumer("TEST", &nats.ConsumerConfig{DeliverSubject: "x.x", FilterSubject: "foo", AckPolicy: nats.AckNonePolicy}) require_NoError(t, err) done, received := make(chan bool), uint64(0) cb := func(m *nats.Msg) { received++ if received >= ci.NumPending { done <- true } } sub, err := nc.Subscribe("x.x", cb) require_NoError(t, err) defer sub.Unsubscribe() start := time.Now() var elapsed time.Duration select { case <-done: elapsed = time.Since(start) case <-time.After(10 * time.Second): t.Fatal("Did not receive all messages for all consumers in time") } if elapsed > 500*time.Millisecond { t.Fatalf("Getting all messages took longer than expected: %v", elapsed) } }) } } func TestNoRaceJetStreamConsumerFilterPerfDegradation(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() js, err := nc.JetStream(nats.PublishAsyncMaxPending(256)) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "test", Subjects: []string{"test.*.subj"}, Replicas: 1, }) require_NoError(t, err) toSend := 50_000 count := 0 ch := make(chan struct{}, 6) _, err = js.Subscribe("test.*.subj", func(m *nats.Msg) { m.Ack() if count++; count == toSend { ch <- struct{}{} } }, nats.DeliverNew(), nats.ManualAck()) require_NoError(t, err) msg := make([]byte, 1024) sent := int32(0) send := func() { defer func() { ch <- struct{}{} }() for i := 0; i < toSend/5; i++ { msgID := atomic.AddInt32(&sent, 1) _, err := js.Publish(fmt.Sprintf("test.%d.subj", msgID), msg) if err != nil { t.Error(err) return } } } for i := 0; i < 5; i++ { go send() } timeout := time.NewTimer(10 * time.Second) for i := 0; i < 6; i++ { select { case <-ch: case <-timeout.C: t.Fatal("Took too long") } } } func TestNoRaceJetStreamFileStoreKeyFileCleanup(t *testing.T) { storeDir := t.TempDir() prf := func(context []byte) ([]byte, error) { h := hmac.New(sha256.New, []byte("dlc22")) if _, err := h.Write(context); err != nil { return nil, err } return h.Sum(nil), nil } fs, err := newFileStoreWithCreated( FileStoreConfig{StoreDir: storeDir, BlockSize: 1024 * 1024}, StreamConfig{Name: "TEST", Storage: FileStorage}, time.Now(), prf, nil) require_NoError(t, err) defer fs.Stop() n, msg := 10_000, []byte(strings.Repeat("Z", 1024)) for i := 0; i < n; i++ { _, _, err := fs.StoreMsg(fmt.Sprintf("X.%d", i), nil, msg) require_NoError(t, err) } var seqs []uint64 for i := 1; i <= n; i++ { seqs = append(seqs, uint64(i)) } // Randomly delete msgs, make sure we cleanup as we empty the message blocks. rand.Shuffle(len(seqs), func(i, j int) { seqs[i], seqs[j] = seqs[j], seqs[i] }) for _, seq := range seqs { _, err := fs.RemoveMsg(seq) require_NoError(t, err) } // We will have cleanup the main .blk and .idx sans the lmb, but we should not have any *.fss files. kms, err := filepath.Glob(filepath.Join(storeDir, msgDir, keyScanAll)) require_NoError(t, err) if len(kms) > 1 { t.Fatalf("Expected to find only 1 key file, found %d", len(kms)) } } func TestNoRaceJetStreamMsgIdPerfDuringCatchup(t *testing.T) { // Uncomment to run. Needs to be on a bigger machine. Do not want as part of Travis tests atm. skip(t) c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.serverByName("S-1")) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, }) require_NoError(t, err) // This will be the one we restart. sl := c.streamLeader("$G", "TEST") // Now move leader. _, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, "TEST"), nil, time.Second) require_NoError(t, err) c.waitOnStreamLeader("$G", "TEST") // Connect to new leader. nc, _ = jsClientConnect(t, c.streamLeader("$G", "TEST")) defer nc.Close() js, err = nc.JetStream(nats.PublishAsyncMaxPending(1024)) require_NoError(t, err) n, ss, sr := 1_000_000, 250_000, 800_000 m := nats.NewMsg("TEST") m.Data = []byte(strings.Repeat("Z", 2048)) // Target rate 10k msgs/sec start := time.Now() for i := 0; i < n; i++ { m.Header.Set(JSMsgId, strconv.Itoa(i)) _, err := js.PublishMsgAsync(m) require_NoError(t, err) //time.Sleep(42 * time.Microsecond) if i == ss { fmt.Printf("SD") sl.Shutdown() } else if i == sr { nc.Flush() select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): } fmt.Printf("RS") sl = c.restartServer(sl) } if i%10_000 == 0 { fmt.Print("#") } } fmt.Println() // Wait to receive all messages. select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } tt := time.Since(start) si, err := js.StreamInfo("TEST") require_NoError(t, err) fmt.Printf("Took %v to send %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) c.waitOnStreamCurrent(sl, "$G", "TEST") for _, s := range c.servers { mset, _ := s.GlobalAccount().lookupStream("TEST") if state := mset.store.State(); state.Msgs != uint64(n) { t.Fatalf("Expected server %v to have correct number of msgs %d but got %d", s, n, state.Msgs) } } } func TestNoRaceJetStreamRebuildDeDupeAndMemoryPerf(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "DD"}) require_NoError(t, err) m := nats.NewMsg("DD") m.Data = []byte(strings.Repeat("Z", 2048)) start := time.Now() n := 1_000_000 for i := 0; i < n; i++ { m.Header.Set(JSMsgId, strconv.Itoa(i)) _, err := js.PublishMsgAsync(m) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } tt := time.Since(start) si, err := js.StreamInfo("DD") require_NoError(t, err) fmt.Printf("Took %v to send %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) v, _ := s.Varz(nil) fmt.Printf("Memory AFTER SEND: %v\n", friendlyBytes(v.Mem)) mset, err := s.GlobalAccount().lookupStream("DD") require_NoError(t, err) mset.mu.Lock() mset.ddloaded = false start = time.Now() mset.rebuildDedupe() fmt.Printf("TOOK %v to rebuild dd\n", time.Since(start)) mset.mu.Unlock() v, _ = s.Varz(nil) fmt.Printf("Memory: %v\n", friendlyBytes(v.Mem)) // Now do an ephemeral consumer and whip through every message. Doing same calculations. start = time.Now() received, done := 0, make(chan bool) sub, err := js.Subscribe("DD", func(m *nats.Msg) { received++ if received >= n { done <- true } }, nats.OrderedConsumer()) require_NoError(t, err) select { case <-done: case <-time.After(10 * time.Second): if s.NumSlowConsumers() > 0 { t.Fatalf("Did not receive all large messages due to slow consumer status: %d of %d", received, n) } t.Fatalf("Failed to receive all large messages: %d of %d\n", received, n) } fmt.Printf("TOOK %v to receive all %d msgs\n", time.Since(start), n) sub.Unsubscribe() v, _ = s.Varz(nil) fmt.Printf("Memory: %v\n", friendlyBytes(v.Mem)) } func TestNoRaceJetStreamMemoryUsageOnLimitedStreamWithMirror(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{Name: "DD", Subjects: []string{"ORDERS.*"}, MaxMsgs: 10_000}) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Mirror: &nats.StreamSource{Name: "DD"}, MaxMsgs: 10_000, }) require_NoError(t, err) m := nats.NewMsg("ORDERS.0") m.Data = []byte(strings.Repeat("Z", 2048)) start := time.Now() n := 1_000_000 for i := 0; i < n; i++ { m.Subject = fmt.Sprintf("ORDERS.%d", i) m.Header.Set(JSMsgId, strconv.Itoa(i)) _, err := js.PublishMsgAsync(m) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } tt := time.Since(start) si, err := js.StreamInfo("DD") require_NoError(t, err) fmt.Printf("Took %v to send %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) v, _ := s.Varz(nil) fmt.Printf("Memory AFTER SEND: %v\n", friendlyBytes(v.Mem)) } func TestNoRaceJetStreamOrderedConsumerLongRTTPerformance(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() js, err := nc.JetStream(nats.PublishAsyncMaxPending(1000)) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{Name: "OCP"}) require_NoError(t, err) n, msg := 100_000, []byte(strings.Repeat("D", 30_000)) for i := 0; i < n; i++ { _, err := js.PublishAsync("OCP", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Approximately 3GB si, err := js.StreamInfo("OCP") require_NoError(t, err) start := time.Now() received, done := 0, make(chan bool) sub, err := js.Subscribe("OCP", func(m *nats.Msg) { received++ if received >= n { done <- true } }, nats.OrderedConsumer()) require_NoError(t, err) defer sub.Unsubscribe() // Wait to receive all messages. select { case <-done: case <-time.After(30 * time.Second): t.Fatalf("Did not receive all of our messages") } tt := time.Since(start) fmt.Printf("Took %v to receive %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) sub.Unsubscribe() rtt := 10 * time.Millisecond bw := 10 * 1024 * 1024 * 1024 proxy := newNetProxy(rtt, bw, bw, s.ClientURL()) defer proxy.stop() nc, err = nats.Connect(proxy.clientURL()) require_NoError(t, err) defer nc.Close() js, err = nc.JetStream() require_NoError(t, err) start, received = time.Now(), 0 sub, err = js.Subscribe("OCP", func(m *nats.Msg) { received++ if received >= n { done <- true } }, nats.OrderedConsumer()) require_NoError(t, err) defer sub.Unsubscribe() // Wait to receive all messages. select { case <-done: case <-time.After(60 * time.Second): t.Fatalf("Did not receive all of our messages") } tt = time.Since(start) fmt.Printf("Proxy RTT: %v, UP: %d, DOWN: %d\n", rtt, bw, bw) fmt.Printf("Took %v to receive %d msgs\n", tt, n) fmt.Printf("%.0f msgs/s\n", float64(n)/tt.Seconds()) fmt.Printf("%.0f mb/s\n\n", float64(si.State.Bytes/(1024*1024))/tt.Seconds()) } var jsClusterStallCatchupTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 32GB, store_dir: '%s'} leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` // Test our global stall gate for outstanding catchup bytes. func TestNoRaceJetStreamClusterCatchupStallGate(t *testing.T) { skip(t) c := createJetStreamClusterWithTemplate(t, jsClusterStallCatchupTempl, "GSG", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // ~100k per message. msg := []byte(strings.Repeat("A", 99_960)) // Create 200 streams with 100MB. // Each server has ~2GB var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) go func(x int) { defer wg.Done() for n := 1; n <= 10; n++ { sn := fmt.Sprintf("S-%d", n+x) _, err := js.AddStream(&nats.StreamConfig{ Name: sn, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 100; i++ { _, err := js.Publish(sn, msg) require_NoError(t, err) } } }(i * 20) } wg.Wait() info, err := js.AccountInfo() require_NoError(t, err) require_True(t, info.Streams == 200) runtime.GC() debug.FreeOSMemory() // Now bring a server down and wipe its storage. s := c.servers[0] vz, err := s.Varz(nil) require_NoError(t, err) fmt.Printf("MEM BEFORE is %v\n", friendlyBytes(vz.Mem)) sd := s.JetStreamConfig().StoreDir s.Shutdown() removeDir(t, sd) s = c.restartServer(s) c.waitOnServerHealthz(s) runtime.GC() debug.FreeOSMemory() vz, err = s.Varz(nil) require_NoError(t, err) fmt.Printf("MEM AFTER is %v\n", friendlyBytes(vz.Mem)) } func TestNoRaceJetStreamClusterCatchupBailMidway(t *testing.T) { skip(t) c := createJetStreamClusterWithTemplate(t, jsClusterStallCatchupTempl, "GSG", 3) defer c.shutdown() ml := c.leader() nc, js := jsClientConnect(t, ml) defer nc.Close() msg := []byte(strings.Repeat("A", 480)) for i := 0; i < maxConcurrentSyncRequests*2; i++ { sn := fmt.Sprintf("CUP-%d", i+1) _, err := js.AddStream(&nats.StreamConfig{ Name: sn, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 10_000; i++ { _, err := js.PublishAsync(sn, msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } } jsz, _ := ml.Jsz(nil) expectedMsgs := jsz.Messages // Now select a server and shut it down, removing the storage directory. s := c.randomNonLeader() sd := s.JetStreamConfig().StoreDir s.Shutdown() removeDir(t, sd) // Now restart the server. s = c.restartServer(s) // We want to force the follower to bail before the catchup through the // upper level catchup logic completes. checkFor(t, 5*time.Second, 10*time.Millisecond, func() error { jsz, _ := s.Jsz(nil) if jsz.Messages > expectedMsgs/2 { s.Shutdown() return nil } return fmt.Errorf("Not enough yet") }) // Now restart the server. s = c.restartServer(s) checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { jsz, _ := s.Jsz(nil) if jsz.Messages == expectedMsgs { return nil } return fmt.Errorf("Not enough yet") }) } func TestNoRaceJetStreamAccountLimitsAndRestart(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterAccountLimitsTempl, "A3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() if _, err := js.AddStream(&nats.StreamConfig{Name: "TEST", Replicas: 3}); err != nil { t.Fatalf("Unexpected error: %v", err) } for i := 0; i < 20_000; i++ { if _, err := js.Publish("TEST", []byte("A")); err != nil { break } if i == 5_000 { snl := c.randomNonStreamLeader("$JS", "TEST") snl.Shutdown() } } c.stopAll() c.restartAll() c.waitOnLeader() c.waitOnStreamLeader("$JS", "TEST") for _, cs := range c.servers { c.waitOnStreamCurrent(cs, "$JS", "TEST") } } func TestNoRaceJetStreamPullConsumersAndInteriorDeletes(t *testing.T) { c := createJetStreamClusterExplicit(t, "ID", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "foo", Replicas: 3, MaxMsgs: 50000, Retention: nats.InterestPolicy, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "foo") _, err = js.AddConsumer("foo", &nats.ConsumerConfig{ Durable: "foo", FilterSubject: "foo", MaxAckPending: 20000, AckWait: time.Minute, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, "foo", "foo") rcv := int32(0) prods := 5 cons := 5 wg := sync.WaitGroup{} wg.Add(prods + cons) toSend := 100000 for i := 0; i < cons; i++ { go func() { defer wg.Done() sub, err := js.PullSubscribe("foo", "foo") if err != nil { return } for { msgs, err := sub.Fetch(200, nats.MaxWait(250*time.Millisecond)) if err != nil { if n := int(atomic.LoadInt32(&rcv)); n >= toSend { return } continue } for _, m := range msgs { m.Ack() atomic.AddInt32(&rcv, 1) } } }() } for i := 0; i < prods; i++ { go func() { defer wg.Done() for i := 0; i < toSend/prods; i++ { js.Publish("foo", []byte("hello")) } }() } time.Sleep(time.Second) resp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, "foo", "foo"), nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } var cdResp JSApiConsumerLeaderStepDownResponse if err := json.Unmarshal(resp.Data, &cdResp); err != nil { t.Fatalf("Unexpected error: %v", err) } if cdResp.Error != nil { t.Fatalf("Unexpected error: %+v", cdResp.Error) } ch := make(chan struct{}) go func() { wg.Wait() close(ch) }() select { case <-ch: // OK case <-time.After(30 * time.Second): t.Fatalf("Consumers took too long to consumer all messages") } } func TestNoRaceJetStreamClusterInterestPullConsumerStreamLimitBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() limit := uint64(1000) _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Retention: nats.InterestPolicy, MaxMsgs: int64(limit), Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{Durable: "dur", AckPolicy: nats.AckExplicitPolicy}) require_NoError(t, err) qch := make(chan bool) var wg sync.WaitGroup // Publisher wg.Add(1) go func() { defer wg.Done() for { pt := time.NewTimer(time.Duration(rand.Intn(2)) * time.Millisecond) select { case <-pt.C: _, err := js.Publish("foo", []byte("BUG!")) require_NoError(t, err) case <-qch: pt.Stop() return } } }() time.Sleep(time.Second) // Pull Consumers wg.Add(100) for i := 0; i < 100; i++ { go func() { defer wg.Done() nc := natsConnect(t, c.randomServer().ClientURL()) defer nc.Close() js, err := nc.JetStream(nats.MaxWait(time.Second)) require_NoError(t, err) var sub *nats.Subscription for j := 0; j < 5; j++ { sub, err = js.PullSubscribe("foo", "dur") if err == nil { break } } require_NoError(t, err) for { pt := time.NewTimer(time.Duration(rand.Intn(300)) * time.Millisecond) select { case <-pt.C: msgs, err := sub.Fetch(1) if err != nil { t.Logf("Got a Fetch error: %v", err) return } if len(msgs) > 0 { go func() { ackDelay := time.Duration(rand.Intn(375)+15) * time.Millisecond m := msgs[0] time.AfterFunc(ackDelay, func() { m.AckSync() }) }() } case <-qch: return } } }() } // Make sure we have hit the limit for the number of messages we expected. checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs < limit { return fmt.Errorf("Not hit limit yet") } return nil }) close(qch) wg.Wait() checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) ci, err := js.ConsumerInfo("TEST", "dur") require_NoError(t, err) np := ci.NumPending + uint64(ci.NumAckPending) if np != si.State.Msgs { return fmt.Errorf("Expected NumPending to be %d got %d", si.State.Msgs-uint64(ci.NumAckPending), ci.NumPending) } return nil }) } // Test that all peers have the direct access subs that participate in a queue group, // but only when they are current and ready. So we will start with R1, add in messages // then scale up while also still adding messages. func TestNoRaceJetStreamClusterDirectAccessAllPeersSubs(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Start as R1 cfg := &StreamConfig{ Name: "TEST", Subjects: []string{"kv.>"}, MaxMsgsPer: 10, AllowDirect: true, Replicas: 1, Storage: FileStorage, } addStream(t, nc, cfg) // Seed with enough messages to start then we will scale up while still adding more messages. num, msg := 1000, bytes.Repeat([]byte("XYZ"), 64) for i := 0; i < num; i++ { js.PublishAsync(fmt.Sprintf("kv.%d", i), msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } getSubj := fmt.Sprintf(JSDirectMsgGetT, "TEST") getMsg := func(key string) *nats.Msg { t.Helper() req := []byte(fmt.Sprintf(`{"last_by_subj":%q}`, key)) m, err := nc.Request(getSubj, req, time.Second) require_NoError(t, err) require_True(t, m.Header.Get(JSSubject) == key) return m } // Just make sure we can succeed here. getMsg("kv.22") // Now crank up a go routine to continue sending more messages. qch := make(chan bool) var wg sync.WaitGroup for i := 0; i < 5; i++ { wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for { select { case <-qch: select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): } return default: // Send as fast as we can. js.Publish(fmt.Sprintf("kv.%d", rand.Intn(1000)), msg) } } }() } time.Sleep(200 * time.Millisecond) // Now let's scale up to an R3. cfg.Replicas = 3 updateStream(t, nc, cfg) // Wait for the stream to register the new replicas and have a leader. checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") if err != nil { return err } if si.Cluster == nil { return fmt.Errorf("No cluster yet") } if si.Cluster.Leader == _EMPTY_ || len(si.Cluster.Replicas) != 2 { return fmt.Errorf("Cluster not ready yet") } return nil }) close(qch) wg.Wait() // Just make sure we can succeed here. getMsg("kv.22") // For each non-leader check that the direct sub fires up. // We just test all, the leader will already have a directSub. for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkFor(t, 20*time.Second, 500*time.Millisecond, func() error { mset.mu.RLock() ok := mset.directSub != nil mset.mu.RUnlock() if ok { return nil } return fmt.Errorf("No directSub yet") }) } si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == uint64(num) { t.Fatalf("Expected to see messages increase, got %d", si.State.Msgs) } checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { // Make sure they are all the same from a state perspective. // Leader will have the expected state. lmset, err := c.streamLeader("$G", "TEST").GlobalAccount().lookupStream("TEST") require_NoError(t, err) expected := lmset.state() for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) if state := mset.state(); !reflect.DeepEqual(expected, state) { return fmt.Errorf("Expected %+v, got %+v", expected, state) } } return nil }) } func TestNoRaceJetStreamClusterStreamNamesAndInfosMoreThanAPILimit(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() createStream := func(name string) { t.Helper() if _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil { t.Fatalf("Unexpected error: %v", err) } } max := JSApiListLimit if JSApiNamesLimit > max { max = JSApiNamesLimit } max += 10 for i := 0; i < max; i++ { name := fmt.Sprintf("foo_%d", i) createStream(name) } // Not using the JS API here because we want to make sure that the // server returns the proper Total count, but also that it does not // send more than when the API limit is in one go. check := func(subj string, limit int) { t.Helper() nreq := JSApiStreamNamesRequest{} b, _ := json.Marshal(nreq) msg, err := nc.Request(subj, b, 2*time.Second) require_NoError(t, err) switch subj { case JSApiStreams: nresp := JSApiStreamNamesResponse{} json.Unmarshal(msg.Data, &nresp) if n := nresp.ApiPaged.Total; n != max { t.Fatalf("Expected total to be %v, got %v", max, n) } if n := nresp.ApiPaged.Limit; n != limit { t.Fatalf("Expected limit to be %v, got %v", limit, n) } if n := len(nresp.Streams); n != limit { t.Fatalf("Expected number of streams to be %v, got %v", limit, n) } case JSApiStreamList: nresp := JSApiStreamListResponse{} json.Unmarshal(msg.Data, &nresp) if n := nresp.ApiPaged.Total; n != max { t.Fatalf("Expected total to be %v, got %v", max, n) } if n := nresp.ApiPaged.Limit; n != limit { t.Fatalf("Expected limit to be %v, got %v", limit, n) } if n := len(nresp.Streams); n != limit { t.Fatalf("Expected number of streams to be %v, got %v", limit, n) } } } check(JSApiStreams, JSApiNamesLimit) check(JSApiStreamList, JSApiListLimit) } func TestNoRaceJetStreamClusterConsumerListPaging(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomNonLeader() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") cfg := &nats.ConsumerConfig{ Replicas: 1, MemoryStorage: true, AckPolicy: nats.AckExplicitPolicy, } // create 3000 consumers. numConsumers := 3000 for i := 1; i <= numConsumers; i++ { cfg.Durable = fmt.Sprintf("d-%.4d", i) _, err := js.AddConsumer("TEST", cfg) require_NoError(t, err) } // Test both names and list operations. // Names reqSubj := fmt.Sprintf(JSApiConsumersT, "TEST") grabConsumerNames := func(offset int) []string { req := fmt.Sprintf(`{"offset":%d}`, offset) respMsg, err := nc.Request(reqSubj, []byte(req), time.Second) require_NoError(t, err) var resp JSApiConsumerNamesResponse err = json.Unmarshal(respMsg.Data, &resp) require_NoError(t, err) // Sanity check that we are actually paging properly around limits. if resp.Limit < len(resp.Consumers) { t.Fatalf("Expected total limited to %d but got %d", resp.Limit, len(resp.Consumers)) } if resp.Total != numConsumers { t.Fatalf("Invalid total response: expected %d got %d", numConsumers, resp.Total) } return resp.Consumers } results := make(map[string]bool) for offset := 0; len(results) < numConsumers; { consumers := grabConsumerNames(offset) offset += len(consumers) for _, name := range consumers { if results[name] { t.Fatalf("Found duplicate %q", name) } results[name] = true } } // List reqSubj = fmt.Sprintf(JSApiConsumerListT, "TEST") grabConsumerList := func(offset int) []*ConsumerInfo { req := fmt.Sprintf(`{"offset":%d}`, offset) respMsg, err := nc.Request(reqSubj, []byte(req), time.Second) require_NoError(t, err) var resp JSApiConsumerListResponse err = json.Unmarshal(respMsg.Data, &resp) require_NoError(t, err) // Sanity check that we are actually paging properly around limits. if resp.Limit < len(resp.Consumers) { t.Fatalf("Expected total limited to %d but got %d", resp.Limit, len(resp.Consumers)) } if resp.Total != numConsumers { t.Fatalf("Invalid total response: expected %d got %d", numConsumers, resp.Total) } return resp.Consumers } results = make(map[string]bool) for offset := 0; len(results) < numConsumers; { consumers := grabConsumerList(offset) offset += len(consumers) for _, ci := range consumers { name := ci.Config.Durable if results[name] { t.Fatalf("Found duplicate %q", name) } results[name] = true } } if len(results) != numConsumers { t.Fatalf("Received %d / %d consumers", len(results), numConsumers) } } func TestNoRaceJetStreamFileStoreLargeKVAccessTiming(t *testing.T) { storeDir := t.TempDir() blkSize := uint64(4 * 1024) // Compensate for slower IO on MacOSX if runtime.GOOS == "darwin" { blkSize *= 4 } fs, err := newFileStore( FileStoreConfig{StoreDir: storeDir, BlockSize: blkSize, CacheExpire: 30 * time.Second}, StreamConfig{Name: "zzz", Subjects: []string{"KV.STREAM_NAME.*"}, Storage: FileStorage, MaxMsgsPer: 1}, ) require_NoError(t, err) defer fs.Stop() tmpl := "KV.STREAM_NAME.%d" nkeys, val := 100_000, bytes.Repeat([]byte("Z"), 1024) for i := 1; i <= nkeys; i++ { subj := fmt.Sprintf(tmpl, i) _, _, err := fs.StoreMsg(subj, nil, val) require_NoError(t, err) } first := fmt.Sprintf(tmpl, 1) last := fmt.Sprintf(tmpl, nkeys) start := time.Now() sm, err := fs.LoadLastMsg(last, nil) require_NoError(t, err) base := time.Since(start) if !bytes.Equal(sm.msg, val) { t.Fatalf("Retrieved value did not match") } start = time.Now() _, err = fs.LoadLastMsg(first, nil) require_NoError(t, err) slow := time.Since(start) if base > 100*time.Microsecond || slow > 200*time.Microsecond { t.Fatalf("Took too long to look up first key vs last: %v vs %v", base, slow) } // time first seq lookup for both as well. // Base will be first in this case. fs.mu.Lock() start = time.Now() fs.firstSeqForSubj(first) base = time.Since(start) start = time.Now() fs.firstSeqForSubj(last) slow = time.Since(start) fs.mu.Unlock() if base > 100*time.Microsecond || slow > 200*time.Microsecond { t.Fatalf("Took too long to look up last key by subject vs first: %v vs %v", base, slow) } } func TestNoRaceJetStreamKVLock(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "LOCKS"}) require_NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() var wg sync.WaitGroup start := make(chan bool) var tracker int64 for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, s) defer nc.Close() kv, err := js.KeyValue("LOCKS") require_NoError(t, err) <-start for { last, err := kv.Create("MY_LOCK", []byte("Z")) if err != nil { select { case <-time.After(10 * time.Millisecond): continue case <-ctx.Done(): return } } if v := atomic.AddInt64(&tracker, 1); v != 1 { t.Logf("TRACKER NOT 1 -> %d\n", v) cancel() } time.Sleep(10 * time.Millisecond) if v := atomic.AddInt64(&tracker, -1); v != 0 { t.Logf("TRACKER NOT 0 AFTER RELEASE -> %d\n", v) cancel() } err = kv.Delete("MY_LOCK", nats.LastRevision(last)) if err != nil { t.Logf("Could not unlock for last %d: %v", last, err) } if ctx.Err() != nil { return } } }() } close(start) wg.Wait() } func TestNoRaceJetStreamSuperClusterStreamMoveLongRTT(t *testing.T) { // Make C2 far away. gwm := gwProxyMap{ "C2": &gwProxy{ rtt: 20 * time.Millisecond, up: 1 * 1024 * 1024 * 1024, // 1gbit down: 1 * 1024 * 1024 * 1024, // 1gbit }, } sc := createJetStreamTaggedSuperClusterWithGWProxy(t, gwm) defer sc.shutdown() nc, js := jsClientConnect(t, sc.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"chunk.*"}, Placement: &nats.Placement{Tags: []string{"cloud:aws", "country:us"}}, Replicas: 3, } // Place a stream in C1. _, err := js.AddStream(cfg, nats.MaxWait(10*time.Second)) require_NoError(t, err) chunk := bytes.Repeat([]byte("Z"), 1000*1024) // ~1MB // 256 MB for i := 0; i < 256; i++ { subj := fmt.Sprintf("chunk.%d", i) js.PublishAsync(subj, chunk) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } // C2, slow RTT. cfg.Placement = &nats.Placement{Tags: []string{"cloud:gcp", "country:uk"}} _, err = js.UpdateStream(cfg) require_NoError(t, err) checkFor(t, 20*time.Second, time.Second, func() error { si, err := js.StreamInfo("TEST", nats.MaxWait(time.Second)) if err != nil { return err } if si.Cluster.Name != "C2" { return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) } if si.Cluster.Leader == _EMPTY_ { return fmt.Errorf("No leader yet") } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) } // Now we want to see that we shrink back to original. if len(si.Cluster.Replicas) != cfg.Replicas-1 { return fmt.Errorf("Expected %d replicas, got %d", cfg.Replicas-1, len(si.Cluster.Replicas)) } return nil }) } // https://github.com/nats-io/nats-server/issues/3455 func TestNoRaceJetStreamConcurrentPullConsumerBatch(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ORDERS.*"}, Storage: nats.MemoryStorage, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) toSend := int32(100_000) for i := 0; i < 100_000; i++ { subj := fmt.Sprintf("ORDERS.%d", i+1) js.PublishAsync(subj, []byte("BUY")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "PROCESSOR", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 5000, }) require_NoError(t, err) nc, js = jsClientConnect(t, s) defer nc.Close() sub1, err := js.PullSubscribe(_EMPTY_, _EMPTY_, nats.Bind("TEST", "PROCESSOR")) require_NoError(t, err) nc, js = jsClientConnect(t, s) defer nc.Close() sub2, err := js.PullSubscribe(_EMPTY_, _EMPTY_, nats.Bind("TEST", "PROCESSOR")) require_NoError(t, err) startCh := make(chan bool) var received int32 wg := sync.WaitGroup{} fetchSize := 1000 fetch := func(sub *nats.Subscription) { <-startCh defer wg.Done() for { msgs, err := sub.Fetch(fetchSize, nats.MaxWait(time.Second)) if atomic.AddInt32(&received, int32(len(msgs))) >= toSend { break } // We should always receive a full batch here if not last competing fetch. if err != nil || len(msgs) != fetchSize { break } for _, m := range msgs { m.Ack() } } } wg.Add(2) go fetch(sub1) go fetch(sub2) close(startCh) wg.Wait() require_True(t, received == toSend) } func TestNoRaceJetStreamManyPullConsumersNeedAckOptimization(t *testing.T) { // Uncomment to run. Do not want as part of Travis tests atm. // Run with cpu and memory profiling to make sure we have improved. skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORDERS", Subjects: []string{"ORDERS.*"}, Storage: nats.MemoryStorage, Retention: nats.InterestPolicy, }) require_NoError(t, err) toSend := 100_000 numConsumers := 500 // Create 500 consumers for i := 1; i <= numConsumers; i++ { _, err := js.AddConsumer("ORDERS", &nats.ConsumerConfig{ Durable: fmt.Sprintf("ORDERS_%d", i), FilterSubject: fmt.Sprintf("ORDERS.%d", i), AckPolicy: nats.AckAllPolicy, }) require_NoError(t, err) } for i := 1; i <= toSend; i++ { subj := fmt.Sprintf("ORDERS.%d", i%numConsumers+1) js.PublishAsync(subj, []byte("HELLO")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sub, err := js.PullSubscribe("ORDERS.500", "ORDERS_500") require_NoError(t, err) fetchSize := toSend / numConsumers msgs, err := sub.Fetch(fetchSize, nats.MaxWait(time.Second)) require_NoError(t, err) last := msgs[len(msgs)-1] last.AckSync() } // https://github.com/nats-io/nats-server/issues/3499 func TestNoRaceJetStreamDeleteConsumerWithInterestStreamAndHighSeqs(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"log.>"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "c", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // Set baseline for time to delete so we can see linear increase as sequence numbers increase. start := time.Now() err = js.DeleteConsumer("TEST", "c") require_NoError(t, err) elapsed := time.Since(start) // Crank up sequence numbers. msg := []byte(strings.Repeat("ZZZ", 128)) for i := 0; i < 5_000_000; i++ { nc.Publish("log.Z", msg) } nc.Flush() _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "c", AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // We have a bug that spins unecessarily through all the sequences from this consumer's // ackfloor(0) and the last sequence for the stream. We will detect by looking for the time // to delete being 100x more. Should be the same since both times no messages exist in the stream. start = time.Now() err = js.DeleteConsumer("TEST", "c") require_NoError(t, err) if e := time.Since(start); e > 100*elapsed { t.Fatalf("Consumer delete took too long: %v vs baseline %v", e, elapsed) } } // Bug when we encode a timestamp that upon decode causes an error which causes server to panic. // This can happen on consumer redelivery since they adjusted timstamps can be in the future, and result // in a negative encoding. If that encoding was exactly -1 seconds, would cause decodeConsumerState to fail // and the server to panic. func TestNoRaceEncodeConsumerStateBug(t *testing.T) { for i := 0; i < 200_000; i++ { // Pretend we redelivered and updated the timestamp to reflect the new start time for expiration. // The bug will trip when time.Now() rounded to seconds in encode is 1 second below the truncated version // of pending. pending := Pending{Sequence: 1, Timestamp: time.Now().Add(time.Second).UnixNano()} state := ConsumerState{ Delivered: SequencePair{Consumer: 1, Stream: 1}, Pending: map[uint64]*Pending{1: &pending}, } buf := encodeConsumerState(&state) _, err := decodeConsumerState(buf) require_NoError(t, err) } } // Performance impact on stream ingress with large number of consumers. func TestNoRaceJetStreamLargeNumConsumersPerfImpact(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) // Baseline with no consumers. toSend := 1_000_000 start := time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() tt := time.Since(start) fmt.Printf("Base time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) err = js.PurgeStream("TEST") require_NoError(t, err) // Now add in 10 idle consumers. for i := 1; i <= 10; i++ { _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: fmt.Sprintf("d-%d", i), AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) } start = time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() tt = time.Since(start) fmt.Printf("\n10 consumers time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) err = js.PurgeStream("TEST") require_NoError(t, err) // Now add in 90 more idle consumers. for i := 11; i <= 100; i++ { _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: fmt.Sprintf("d-%d", i), AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) } start = time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() tt = time.Since(start) fmt.Printf("\n100 consumers time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) err = js.PurgeStream("TEST") require_NoError(t, err) // Now add in 900 more for i := 101; i <= 1000; i++ { _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: fmt.Sprintf("d-%d", i), AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) } start = time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("foo", []byte("OK")) } <-js.PublishAsyncComplete() tt = time.Since(start) fmt.Printf("\n1000 consumers time is %v\n", tt) fmt.Printf("%.0f msgs/sec\n", float64(toSend)/tt.Seconds()) } // Performance impact on large number of consumers but sparse delivery. func TestNoRaceJetStreamLargeNumConsumersSparseDelivery(t *testing.T) { skip(t) s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"ID.*"}, }) require_NoError(t, err) // Now add in ~10k consumers on different subjects. for i := 3; i <= 10_000; i++ { _, err := js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: fmt.Sprintf("d-%d", i), FilterSubject: fmt.Sprintf("ID.%d", i), AckPolicy: nats.AckNonePolicy, }) require_NoError(t, err) } toSend := 100_000 // Bind a consumer to ID.2. var received int done := make(chan bool) nc, js = jsClientConnect(t, s) defer nc.Close() mh := func(m *nats.Msg) { received++ if received >= toSend { close(done) } } _, err = js.Subscribe("ID.2", mh) require_NoError(t, err) last := make(chan bool) _, err = js.Subscribe("ID.1", func(_ *nats.Msg) { close(last) }) require_NoError(t, err) nc, _ = jsClientConnect(t, s) defer nc.Close() js, err = nc.JetStream(nats.PublishAsyncMaxPending(8 * 1024)) require_NoError(t, err) start := time.Now() for i := 0; i < toSend; i++ { js.PublishAsync("ID.2", []byte("ok")) } // Check latency for this one message. // This will show the issue better than throughput which can bypass signal processing. js.PublishAsync("ID.1", []byte("ok")) select { case <-done: break case <-time.After(10 * time.Second): t.Fatalf("Failed to receive all messages: %d of %d\n", received, toSend) } tt := time.Since(start) fmt.Printf("Took %v to receive %d msgs\n", tt, toSend) fmt.Printf("%.0f msgs/s\n", float64(toSend)/tt.Seconds()) select { case <-last: break case <-time.After(30 * time.Second): t.Fatalf("Failed to receive last message\n") } lt := time.Since(start) fmt.Printf("Took %v to receive last msg\n", lt) } func TestNoRaceJetStreamEndToEndLatency(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) nc, js = jsClientConnect(t, s) defer nc.Close() var sent time.Time var max time.Duration next := make(chan struct{}) mh := func(m *nats.Msg) { received := time.Now() tt := received.Sub(sent) if max == 0 || tt > max { max = tt } next <- struct{}{} } sub, err := js.Subscribe("foo", mh) require_NoError(t, err) nc, js = jsClientConnect(t, s) defer nc.Close() toSend := 50_000 for i := 0; i < toSend; i++ { sent = time.Now() js.Publish("foo", []byte("ok")) <-next } sub.Unsubscribe() if max > 250*time.Millisecond { t.Fatalf("Expected max latency to be < 250ms, got %v", max) } } func TestNoRaceJetStreamClusterEnsureWALCompact(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "dlc", DeliverSubject: "zz", Replicas: 3, }) require_NoError(t, err) // Force snapshot on stream leader. sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) node := mset.raftNode() require_True(t, node != nil) err = node.InstallSnapshot(mset.stateSnapshot()) require_NoError(t, err) // Now publish more than should be needed to cause an additional snapshot. ns := 75_000 for i := 0; i <= ns; i++ { _, err := js.Publish("foo", []byte("bar")) require_NoError(t, err) } // Grab progress and use that to look into WAL entries. _, _, applied := node.Progress() // If ne == ns that means snapshots and compacts were not happening when // they should have been. if ne, _ := node.Applied(applied); ne >= uint64(ns) { t.Fatalf("Did not snapshot and compact the raft WAL, entries == %d", ne) } // Now check consumer. // Force snapshot on consumerleader. cl := c.consumerLeader(globalAccountName, "TEST", "dlc") mset, err = cl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("dlc") require_True(t, o != nil) node = o.raftNode() require_True(t, node != nil) snap, err := o.store.EncodedState() require_NoError(t, err) err = node.InstallSnapshot(snap) require_NoError(t, err) received, done := 0, make(chan bool, 1) nc.Subscribe("zz", func(m *nats.Msg) { received++ if received >= ns { select { case done <- true: default: } } m.Ack() }) select { case <-done: return case <-time.After(10 * time.Second): t.Fatalf("Did not received all %d msgs, only %d", ns, received) } // Do same trick and check that WAL was compacted. // Grab progress and use that to look into WAL entries. _, _, applied = node.Progress() // If ne == ns that means snapshots and compacts were not happening when // they should have been. if ne, _ := node.Applied(applied); ne >= uint64(ns) { t.Fatalf("Did not snapshot and compact the raft WAL, entries == %d", ne) } } func TestNoRaceFileStoreStreamMaxAgePerformance(t *testing.T) { // Uncomment to run. skip(t) storeDir := t.TempDir() maxAge := 5 * time.Second fs, err := newFileStore( FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "MA", Subjects: []string{"foo.*"}, MaxAge: maxAge, Storage: FileStorage}, ) require_NoError(t, err) defer fs.Stop() // Simulate a callback similar to consumers decrementing. var mu sync.RWMutex var pending int64 fs.RegisterStorageUpdates(func(md, bd int64, seq uint64, subj string) { mu.Lock() defer mu.Unlock() pending += md }) start, num, subj := time.Now(), 0, "foo.foo" timeout := start.Add(maxAge) for time.Now().Before(timeout) { // We will store in blocks of 100. for i := 0; i < 100; i++ { _, _, err := fs.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) num++ } } elapsed := time.Since(start) fmt.Printf("Took %v to store %d\n", elapsed, num) fmt.Printf("%.0f msgs/sec\n", float64(num)/elapsed.Seconds()) // Now keep running for 2x longer knowing we are expiring messages in the background. // We want to see the effect on performance. start = time.Now() timeout = start.Add(maxAge * 2) for time.Now().Before(timeout) { // We will store in blocks of 100. for i := 0; i < 100; i++ { _, _, err := fs.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) num++ } } elapsed = time.Since(start) fmt.Printf("Took %v to store %d\n", elapsed, num) fmt.Printf("%.0f msgs/sec\n", float64(num)/elapsed.Seconds()) } // SequenceSet memory tests vs dmaps. func TestNoRaceSeqSetSizeComparison(t *testing.T) { // Create 5M random entries (dupes possible but ok for this test) out of 8M range. num := 5_000_000 max := 7_000_000 seqs := make([]uint64, 0, num) for i := 0; i < num; i++ { n := uint64(rand.Int63n(int64(max + 1))) seqs = append(seqs, n) } runtime.GC() // Disable to get stable results. gcp := debug.SetGCPercent(-1) defer debug.SetGCPercent(gcp) mem := runtime.MemStats{} runtime.ReadMemStats(&mem) inUseBefore := mem.HeapInuse dmap := make(map[uint64]struct{}, num) for _, n := range seqs { dmap[n] = struct{}{} } runtime.ReadMemStats(&mem) dmapUse := mem.HeapInuse - inUseBefore inUseBefore = mem.HeapInuse // Now do SequenceSet on same dataset. var sset avl.SequenceSet for _, n := range seqs { sset.Insert(n) } runtime.ReadMemStats(&mem) seqSetUse := mem.HeapInuse - inUseBefore if seqSetUse > 2*1024*1024 { t.Fatalf("Expected SequenceSet size to be < 2M, got %v", friendlyBytes(int64(seqSetUse))) } if seqSetUse*50 > dmapUse { t.Fatalf("Expected SequenceSet to be at least 50x better then dmap approach: %v vs %v", friendlyBytes(int64(seqSetUse)), friendlyBytes(int64(dmapUse)), ) } } // FilteredState for ">" with large interior deletes was very slow. func TestNoRaceFileStoreFilteredStateWithLargeDeletes(t *testing.T) { storeDir := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: storeDir, BlockSize: 4096}, StreamConfig{Name: "zzz", Subjects: []string{"foo"}, Storage: FileStorage}, ) require_NoError(t, err) defer fs.Stop() subj, msg := "foo", []byte("Hello World") toStore := 500_000 for i := 0; i < toStore; i++ { _, _, err := fs.StoreMsg(subj, nil, msg) require_NoError(t, err) } // Now delete every other one. for seq := 2; seq <= toStore; seq += 2 { _, err := fs.RemoveMsg(uint64(seq)) require_NoError(t, err) } runtime.GC() // Disable to get stable results. gcp := debug.SetGCPercent(-1) defer debug.SetGCPercent(gcp) start := time.Now() fss := fs.FilteredState(1, _EMPTY_) elapsed := time.Since(start) require_True(t, fss.Msgs == uint64(toStore/2)) require_True(t, elapsed < 500*time.Microsecond) } // ConsumerInfo seems to being called quite a bit more than we had anticipated. // Under certain circumstances, since we reset num pending, this can be very costly. // We will use the fast path to alleviate that performance bottleneck but also make // sure we are still being accurate. func TestNoRaceJetStreamClusterConsumerInfoSpeed(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() server := c.randomNonLeader() nc, js := jsClientConnect(t, server) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, Replicas: 3, }) require_NoError(t, err) // The issue is compounded when we have lots of different subjects captured // by a terminal fwc. The consumer will have a terminal pwc. // Here make all subjects unique. sub, err := js.PullSubscribe("events.*", "DLC") require_NoError(t, err) toSend := 250_000 for i := 0; i < toSend; i++ { subj := fmt.Sprintf("events.%d", i+1) js.PublishAsync(subj, []byte("ok")) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkNumPending := func(expected int) { t.Helper() start := time.Now() ci, err := js.ConsumerInfo("TEST", "DLC") require_NoError(t, err) // Make sure these are fast now. if elapsed := time.Since(start); elapsed > 5*time.Millisecond { t.Fatalf("ConsumerInfo took too long: %v", elapsed) } // Make sure pending == expected. if ci.NumPending != uint64(expected) { t.Fatalf("Expected %d NumPending, got %d", expected, ci.NumPending) } } // Make sure in simple case it is correct. checkNumPending(toSend) // Do a few acks. toAck := 25 for _, m := range fetchMsgs(t, sub, 25, time.Second) { err = m.AckSync() require_NoError(t, err) } checkNumPending(toSend - toAck) // Now do a purge such that we only keep so many. // We want to make sure we do the right thing here and have correct calculations. toKeep := 100_000 err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Keep: uint64(toKeep)}) require_NoError(t, err) checkNumPending(toKeep) } func TestNoRaceJetStreamKVAccountWithServerRestarts(t *testing.T) { // Uncomment to run. Needs fast machine to not time out on KeyValue lookup. skip(t) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, }) require_NoError(t, err) npubs := 10_000 par := 8 iter := 2 nsubjs := 250 wg := sync.WaitGroup{} putKeys := func() { wg.Add(1) go func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err := js.KeyValue("TEST") require_NoError(t, err) for i := 0; i < npubs; i++ { subj := fmt.Sprintf("KEY-%d", rand.Intn(nsubjs)) if _, err := kv.PutString(subj, "hello"); err != nil { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() kv, err = js.KeyValue("TEST") require_NoError(t, err) } } }() } restartServers := func() { time.Sleep(2 * time.Second) // Rotate through and restart the servers. for _, server := range c.servers { server.Shutdown() restarted := c.restartServer(server) checkFor(t, time.Second, 200*time.Millisecond, func() error { hs := restarted.healthz(&HealthzOptions{ JSEnabled: true, JSServerOnly: true, }) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) } c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "KV_TEST") } for n := 0; n < iter; n++ { for i := 0; i < par; i++ { putKeys() } restartServers() } wg.Wait() nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() si, err := js.StreamInfo("KV_TEST") require_NoError(t, err) require_True(t, si.State.NumSubjects == uint64(nsubjs)) } // Test for consumer create when the subject cardinality is high and the // consumer is filtered with a wildcard that forces linear scans. // We have an optimization to use in memory structures in filestore to speed up. // Only if asking to scan all (DeliverAll). func TestNoRaceJetStreamConsumerCreateTimeNumPending(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, }) require_NoError(t, err) n := 500_000 msg := bytes.Repeat([]byte("X"), 8*1024) for i := 0; i < n; i++ { subj := fmt.Sprintf("events.%d", rand.Intn(100_000)) js.PublishAsync(subj, msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): } // Should stay under 5ms now, but for Travis variability say 50ms. threshold := 50 * time.Millisecond start := time.Now() _, err = js.PullSubscribe("events.*", "dlc") require_NoError(t, err) if elapsed := time.Since(start); elapsed > threshold { t.Fatalf("Consumer create took longer than expected, %v vs %v", elapsed, threshold) } start = time.Now() _, err = js.PullSubscribe("events.99999", "xxx") require_NoError(t, err) if elapsed := time.Since(start); elapsed > threshold { t.Fatalf("Consumer create took longer than expected, %v vs %v", elapsed, threshold) } start = time.Now() _, err = js.PullSubscribe(">", "zzz") require_NoError(t, err) if elapsed := time.Since(start); elapsed > threshold { t.Fatalf("Consumer create took longer than expected, %v vs %v", elapsed, threshold) } } func TestNoRaceJetStreamClusterGhostConsumers(t *testing.T) { consumerNotActiveStartInterval = time.Second * 5 defer func() { consumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval }() c := createJetStreamClusterExplicit(t, "GHOST", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"events.>"}, Replicas: 3, }) require_NoError(t, err) for i := 0; i < 10; i++ { for j := 0; j < 10; j++ { require_NoError(t, nc.Publish(fmt.Sprintf("events.%d.%d", i, j), []byte(`test`))) } } fetch := func(id int) { subject := fmt.Sprintf("events.%d.*", id) subscription, err := js.PullSubscribe(subject, _EMPTY_, // ephemeral consumer nats.DeliverAll(), nats.ReplayInstant(), nats.BindStream("TEST"), nats.ConsumerReplicas(1), nats.ConsumerMemoryStorage(), ) if err != nil { return } defer subscription.Unsubscribe() info, err := subscription.ConsumerInfo() if err != nil { return } subscription.Fetch(int(info.NumPending)) } replay := func(ctx context.Context, id int) { for { select { case <-ctx.Done(): return default: fetch(id) } } } ctx, cancel := context.WithCancel(context.Background()) go replay(ctx, 0) go replay(ctx, 1) go replay(ctx, 2) go replay(ctx, 3) go replay(ctx, 4) go replay(ctx, 5) go replay(ctx, 6) go replay(ctx, 7) go replay(ctx, 8) go replay(ctx, 9) time.Sleep(5 * time.Second) for _, server := range c.servers { server.Shutdown() restarted := c.restartServer(server) checkFor(t, time.Second, 200*time.Millisecond, func() error { hs := restarted.healthz(&HealthzOptions{ JSEnabled: true, JSServerOnly: true, }) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) c.waitOnStreamLeader(globalAccountName, "TEST") time.Sleep(time.Second * 2) go replay(ctx, 5) go replay(ctx, 6) go replay(ctx, 7) go replay(ctx, 8) go replay(ctx, 9) } time.Sleep(5 * time.Second) cancel() // Check we don't report missing consumers. subj := fmt.Sprintf(JSApiConsumerListT, "TEST") checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { // Request will take at most 4 seconds if some consumers can't be found. m, err := nc.Request(subj, nil, 5*time.Second) if err != nil { return err } var resp JSApiConsumerListResponse require_NoError(t, json.Unmarshal(m.Data, &resp)) if len(resp.Missing) == 0 { return nil } return fmt.Errorf("Still have missing: %+v", resp.Missing) }) // Also check all servers agree on the available consumer assignments. // It could be the above check passes, i.e. our meta leader thinks all is okay, but other servers actually drifted. checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { var previousConsumers []string for _, s := range c.servers { sjs := s.getJetStream() sjs.mu.Lock() cc := sjs.cluster sa := cc.streams[globalAccountName]["TEST"] var consumers []string for cName := range sa.consumers { consumers = append(consumers, cName) } sjs.mu.Unlock() slices.Sort(consumers) if previousConsumers != nil && !slices.Equal(previousConsumers, consumers) { return fmt.Errorf("Consumer mismatch:\n- previous: %v\n- actual : %v\n", previousConsumers, consumers) } previousConsumers = consumers } return nil }) } // This is to test a publish slowdown and general instability experienced in a setup similar to this. // We have feeder streams that are all sourced to an aggregate stream. All streams are interest retention. // We want to monitor the avg publish time for the sync publishers to the feeder streams, the ingest rate to // the aggregate stream, and general health of the consumers on the aggregate stream. // Target publish rate is ~2k/s with publish time being ~40-60ms but remaining stable. // We can also simulate max redeliveries that create interior deletes in streams. func TestNoRaceJetStreamClusterF3Setup(t *testing.T) { // Uncomment to run. Needs to be on a pretty big machine. Do not want as part of Travis tests atm. skip(t) // These and the settings below achieve ~60ms pub time on avg and ~2k msgs per sec inbound to the aggregate stream. // On my machine though. np := clusterProxy{ rtt: 2 * time.Millisecond, up: 1 * 1024 * 1024 * 1024, // 1gbit down: 1 * 1024 * 1024 * 1024, // 1gbit } // Test params. numSourceStreams := 20 numConsumersPerSource := 1 numPullersPerConsumer := 50 numPublishers := 100 setHighStartSequence := false simulateMaxRedeliveries := false maxBadPubTimes := uint32(20) badPubThresh := 500 * time.Millisecond testTime := 5 * time.Minute // make sure to do --timeout=65m t.Logf("Starting Test: Total Test Time %v", testTime) c := createJetStreamClusterWithNetProxy(t, "R3S", 3, &np) defer c.shutdown() // Do some quick sanity checking for latency stuff. { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"foo"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) defer js.DeleteStream("TEST") sl := c.streamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, sl) defer nc.Close() start := time.Now() _, err = js.Publish("foo", []byte("hello")) require_NoError(t, err) // This is best case, and with client connection being close to free, this should be at least > rtt if elapsed := time.Since(start); elapsed < np.rtt { t.Fatalf("Expected publish time to be > %v, got %v", np.rtt, elapsed) } nl := c.randomNonStreamLeader(globalAccountName, "TEST") nc, js = jsClientConnect(t, nl) defer nc.Close() start = time.Now() _, err = js.Publish("foo", []byte("hello")) require_NoError(t, err) // This is worst case, meaning message has to travel to leader, then to fastest replica, then back. // So should be at 3x rtt, so check at least > 2x rtt. if elapsed := time.Since(start); elapsed < 2*np.rtt { t.Fatalf("Expected publish time to be > %v, got %v", 2*np.rtt, elapsed) } } // Setup source streams. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() t.Logf("Creating %d Source Streams", numSourceStreams) var sources []string wg := sync.WaitGroup{} for i := 0; i < numSourceStreams; i++ { sname := fmt.Sprintf("EVENT-%s", nuid.Next()) sources = append(sources, sname) wg.Add(1) go func(stream string) { defer wg.Done() t.Logf(" %q", stream) subj := fmt.Sprintf("%s.>", stream) _, err := js.AddStream(&nats.StreamConfig{ Name: stream, Subjects: []string{subj}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) for j := 0; j < numConsumersPerSource; j++ { consumer := fmt.Sprintf("C%d", j) _, err := js.Subscribe(_EMPTY_, func(msg *nats.Msg) { msg.Ack() }, nats.BindStream(stream), nats.Durable(consumer), nats.ManualAck()) require_NoError(t, err) } }(sname) } wg.Wait() var streamSources []*nats.StreamSource for _, src := range sources { streamSources = append(streamSources, &nats.StreamSource{Name: src}) } t.Log("Creating Aggregate Stream") // Now create the aggregate stream. _, err := js.AddStream(&nats.StreamConfig{ Name: "EVENTS", Replicas: 3, Retention: nats.InterestPolicy, Sources: streamSources, }) require_NoError(t, err) // Set first sequence to a high number. if setHighStartSequence { require_NoError(t, js.PurgeStream("EVENTS", &nats.StreamPurgeRequest{Sequence: 32_000_001})) } // Now create 2 pull consumers. _, err = js.PullSubscribe(_EMPTY_, "C1", nats.BindStream("EVENTS"), nats.MaxDeliver(1), nats.AckWait(10*time.Second), nats.ManualAck(), ) require_NoError(t, err) _, err = js.PullSubscribe(_EMPTY_, "C2", nats.BindStream("EVENTS"), nats.MaxDeliver(1), nats.AckWait(10*time.Second), nats.ManualAck(), ) require_NoError(t, err) t.Logf("Creating %d x 2 Pull Subscribers", numPullersPerConsumer) // Now create the pullers. for _, subName := range []string{"C1", "C2"} { for i := 0; i < numPullersPerConsumer; i++ { go func(subName string) { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() sub, err := js.PullSubscribe(_EMPTY_, subName, nats.BindStream("EVENTS"), nats.MaxDeliver(1), nats.AckWait(10*time.Second), nats.ManualAck(), ) require_NoError(t, err) for { msgs, err := sub.Fetch(25, nats.MaxWait(2*time.Second)) if err != nil && err != nats.ErrTimeout { t.Logf("Exiting pull subscriber %q: %v", subName, err) return } // Shuffle rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) // Wait for a random interval up to 100ms. time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) for _, m := range msgs { // If we want to simulate max redeliveries being hit, since not acking // once will cause it due to subscriber setup. // 100_000 == 0.01% if simulateMaxRedeliveries && rand.Intn(100_000) == 0 { md, err := m.Metadata() require_NoError(t, err) t.Logf("** Skipping Ack: %d **", md.Sequence.Stream) } else { m.Ack() } } } }(subName) } } // Now create feeder publishers. eventTypes := []string{"PAYMENT", "SUBMISSION", "CANCEL"} msg := make([]byte, 2*1024) // 2k payload crand.Read(msg) // For tracking pub times. var pubs int var totalPubTime time.Duration var pmu sync.Mutex last := time.Now() updatePubStats := func(elapsed time.Duration) { pmu.Lock() defer pmu.Unlock() // Reset every 5s if time.Since(last) > 5*time.Second { pubs = 0 totalPubTime = 0 last = time.Now() } pubs++ totalPubTime += elapsed } avgPubTime := func() time.Duration { pmu.Lock() np := pubs tpt := totalPubTime pmu.Unlock() return tpt / time.Duration(np) } t.Logf("Creating %d Publishers", numPublishers) var numLimitsExceeded atomic.Uint32 errCh := make(chan error, 100) for i := 0; i < numPublishers; i++ { go func() { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for { // Grab a random source stream stream := sources[rand.Intn(len(sources))] // Grab random event type. evt := eventTypes[rand.Intn(len(eventTypes))] subj := fmt.Sprintf("%s.%s", stream, evt) start := time.Now() _, err := js.Publish(subj, msg) if err != nil { t.Logf("Exiting publisher: %v", err) return } elapsed := time.Since(start) if elapsed > badPubThresh { t.Logf("Publish time took more than expected: %v", elapsed) numLimitsExceeded.Add(1) if ne := numLimitsExceeded.Load(); ne > maxBadPubTimes { errCh <- fmt.Errorf("Too many exceeded times on publish: %d", ne) return } } updatePubStats(elapsed) } }() } t.Log("Creating Monitoring Routine - Data in ~10s") // Create monitoring routine. go func() { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() fseq, lseq := uint64(0), uint64(0) for { // Grab consumers var minAckFloor uint64 = math.MaxUint64 for _, consumer := range []string{"C1", "C2"} { ci, err := js.ConsumerInfo("EVENTS", consumer) if err != nil { t.Logf("Exiting Monitor: %v", err) return } if lseq > 0 { t.Logf("%s:\n Delivered:\t%d\n AckFloor:\t%d\n AckPending:\t%d\n NumPending:\t%d", consumer, ci.Delivered.Stream, ci.AckFloor.Stream, ci.NumAckPending, ci.NumPending) } if ci.AckFloor.Stream < minAckFloor { minAckFloor = ci.AckFloor.Stream } } // Now grab aggregate stream state. si, err := js.StreamInfo("EVENTS") if err != nil { t.Logf("Exiting Monitor: %v", err) return } state := si.State if lseq != 0 { t.Logf("Stream:\n Msgs: \t%d\n First:\t%d\n Last: \t%d\n Deletes:\t%d\n", state.Msgs, state.FirstSeq, state.LastSeq, state.NumDeleted) t.Logf("Publish Stats:\n Msgs/s:\t%0.2f\n Avg Pub:\t%v\n\n", float64(si.State.LastSeq-lseq)/5.0, avgPubTime()) if si.State.FirstSeq < minAckFloor && si.State.FirstSeq == fseq { t.Log("Stream first seq < minimum ack floor") } } fseq, lseq = si.State.FirstSeq, si.State.LastSeq time.Sleep(5 * time.Second) } }() select { case e := <-errCh: t.Fatal(e) case <-time.After(testTime): t.Fatalf("Did not receive completion signal") } } // Unbalanced stretch cluster. // S2 (stream leader) will have a slow path to S1 (via proxy) and S3 (consumer leader) will have a fast path. // // Route Ports // "S1": 14622 // "S2": 15622 // "S3": 16622 func createStretchUnbalancedCluster(t testing.TB) (c *cluster, np *netProxy) { t.Helper() tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: "F3" listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` // Do these in order, S1, S2 (proxy) then S3. c = &cluster{t: t, servers: make([]*Server, 3), opts: make([]*Options, 3), name: "F3"} // S1 conf := fmt.Sprintf(tmpl, "S1", t.TempDir(), 14622, "route://127.0.0.1:15622, route://127.0.0.1:16622") c.servers[0], c.opts[0] = RunServerWithConfig(createConfFile(t, []byte(conf))) // S2 // Create the proxy first. Connect this to S1. Make it slow, e.g. 5ms RTT. np = createNetProxy(1*time.Millisecond, 1024*1024*1024, 1024*1024*1024, "route://127.0.0.1:14622", true) routes := fmt.Sprintf("%s, route://127.0.0.1:16622", np.routeURL()) conf = fmt.Sprintf(tmpl, "S2", t.TempDir(), 15622, routes) c.servers[1], c.opts[1] = RunServerWithConfig(createConfFile(t, []byte(conf))) // S3 conf = fmt.Sprintf(tmpl, "S3", t.TempDir(), 16622, "route://127.0.0.1:14622, route://127.0.0.1:15622") c.servers[2], c.opts[2] = RunServerWithConfig(createConfFile(t, []byte(conf))) c.checkClusterFormed() c.waitOnClusterReady() return c, np } // We test an interest based stream that has a cluster with a node with asymmetric paths from // the stream leader and the consumer leader such that the consumer leader path is fast and // replicated acks arrive sooner then the actual message. This path was considered, but also // categorized as very rare and was expensive as it tried to forward a new stream msg delete // proposal to the original stream leader. It now will deal with the issue locally and not // slow down the ingest rate to the stream's publishers. func TestNoRaceJetStreamClusterDifferentRTTInterestBasedStreamSetup(t *testing.T) { // Uncomment to run. Do not want as part of Travis tests atm. skip(t) c, np := createStretchUnbalancedCluster(t) defer c.shutdown() defer np.stop() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Now create the stream. _, err := js.AddStream(&nats.StreamConfig{ Name: "EVENTS", Subjects: []string{"EV.>"}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) // Make sure it's leader is on S2. sl := c.servers[1] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader(globalAccountName, "EVENTS") if s := c.streamLeader(globalAccountName, "EVENTS"); s != sl { s.JetStreamStepdownStream(globalAccountName, "EVENTS") return fmt.Errorf("Server %s is not stream leader yet", sl) } return nil }) // Now create the consumer. _, err = js.PullSubscribe(_EMPTY_, "C", nats.BindStream("EVENTS"), nats.ManualAck()) require_NoError(t, err) // Make sure the consumer leader is on S3. cl := c.servers[2] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "C") if s := c.consumerLeader(globalAccountName, "EVENTS", "C"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "C") return fmt.Errorf("Server %s is not consumer leader yet", cl) } return nil }) go func(js nats.JetStream) { sub, err := js.PullSubscribe(_EMPTY_, "C", nats.BindStream("EVENTS"), nats.ManualAck()) require_NoError(t, err) for { msgs, err := sub.Fetch(100, nats.MaxWait(2*time.Second)) if err != nil && err != nats.ErrTimeout { return } // Shuffle rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) for _, m := range msgs { m.Ack() } } }(js) numPublishers := 25 pubThresh := 2 * time.Second var maxExceeded atomic.Int64 errCh := make(chan error, numPublishers) wg := sync.WaitGroup{} msg := make([]byte, 2*1024) // 2k payload crand.Read(msg) // Publishers. for i := 0; i < numPublishers; i++ { wg.Add(1) go func(iter int) { defer wg.Done() // Connect to random, the slow ones will be connected to the slow node. // But if you connect them all there it will pass. s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() for i := 0; i < 1_000; i++ { start := time.Now() _, err := js.Publish("EV.PAID", msg) if err != nil { errCh <- fmt.Errorf("Publish error: %v", err) return } if elapsed := time.Since(start); elapsed > pubThresh { errCh <- fmt.Errorf("Publish time exceeded") if int64(elapsed) > maxExceeded.Load() { maxExceeded.Store(int64(elapsed)) } return } } }(i) } wg.Wait() select { case e := <-errCh: t.Fatalf("%v: threshold is %v, maximum seen: %v", e, pubThresh, time.Duration(maxExceeded.Load())) default: } } func TestNoRaceJetStreamInterestStreamCheckInterestRaceBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) numConsumers := 10 for i := 0; i < numConsumers; i++ { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err = js.Subscribe("foo", func(m *nats.Msg) { m.Ack() }, nats.Durable(fmt.Sprintf("C%d", i)), nats.ManualAck()) require_NoError(t, err) } numToSend := 10_000 for i := 0; i < numToSend; i++ { _, err := js.PublishAsync("foo", nil, nats.StallWait(800*time.Millisecond)) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } // Wait til ackfloor is correct for all consumers. checkFor(t, 20*time.Second, 100*time.Millisecond, func() error { for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() defer mset.mu.RUnlock() require_True(t, len(mset.consumers) == numConsumers) for _, o := range mset.consumers { state, err := o.store.State() require_NoError(t, err) if state.AckFloor.Stream != uint64(numToSend) { return fmt.Errorf("Ackfloor not correct yet") } } } return nil }) for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.RLock() defer mset.mu.RUnlock() state := mset.state() require_True(t, state.Msgs == 0) require_True(t, state.FirstSeq == uint64(numToSend+1)) } } func TestNoRaceJetStreamClusterInterestStreamConsistencyAfterRollingRestart(t *testing.T) { // Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm. skip(t) c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() numStreams := 200 numConsumersPer := 5 numPublishers := 10 nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() qch := make(chan bool) var mm sync.Mutex ackMap := make(map[string]map[uint64][]string) addAckTracking := func(seq uint64, stream, consumer string) { mm.Lock() defer mm.Unlock() sam := ackMap[stream] if sam == nil { sam = make(map[uint64][]string) ackMap[stream] = sam } sam[seq] = append(sam[seq], consumer) } doPullSubscriber := func(stream, consumer, filter string) { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() var err error var sub *nats.Subscription timeout := time.Now().Add(5 * time.Second) for time.Now().Before(timeout) { sub, err = js.PullSubscribe(filter, consumer, nats.BindStream(stream), nats.ManualAck()) if err == nil { break } } if err != nil { t.Logf("Error on pull subscriber: %v", err) return } for { select { case <-time.After(500 * time.Millisecond): msgs, err := sub.Fetch(100, nats.MaxWait(time.Second)) if err != nil { continue } // Shuffle rand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] }) for _, m := range msgs { meta, err := m.Metadata() require_NoError(t, err) m.Ack() addAckTracking(meta.Sequence.Stream, stream, consumer) if meta.NumDelivered > 1 { t.Logf("Got a msg redelivered %d for sequence %d on %q %q\n", meta.NumDelivered, meta.Sequence.Stream, stream, consumer) } } case <-qch: nc.Flush() return } } } // Setup wg := sync.WaitGroup{} for i := 0; i < numStreams; i++ { wg.Add(1) go func(stream string) { defer wg.Done() subj := fmt.Sprintf("%s.>", stream) _, err := js.AddStream(&nats.StreamConfig{ Name: stream, Subjects: []string{subj}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) for i := 0; i < numConsumersPer; i++ { consumer := fmt.Sprintf("C%d", i) filter := fmt.Sprintf("%s.%d", stream, i) _, err = js.AddConsumer(stream, &nats.ConsumerConfig{ Durable: consumer, FilterSubject: filter, AckPolicy: nats.AckExplicitPolicy, AckWait: 2 * time.Second, }) require_NoError(t, err) c.waitOnConsumerLeader(globalAccountName, stream, consumer) go doPullSubscriber(stream, consumer, filter) } }(fmt.Sprintf("A-%d", i)) } wg.Wait() msg := make([]byte, 2*1024) // 2k payload crand.Read(msg) // Controls if publishing is on or off. var pubActive atomic.Bool doPublish := func() { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for { select { case <-time.After(100 * time.Millisecond): if pubActive.Load() { for i := 0; i < numStreams; i++ { for j := 0; j < numConsumersPer; j++ { subj := fmt.Sprintf("A-%d.%d", i, j) // Don't care about errors here for this test. js.Publish(subj, msg) } } } case <-qch: return } } } pubActive.Store(true) for i := 0; i < numPublishers; i++ { go doPublish() } // Let run for a bit. time.Sleep(20 * time.Second) // Do a rolling restart. for _, s := range c.servers { t.Logf("Shutdown %v\n", s) s.Shutdown() s.WaitForShutdown() time.Sleep(20 * time.Second) t.Logf("Restarting %v\n", s) s = c.restartServer(s) c.waitOnServerHealthz(s) } // Let run for a bit longer. time.Sleep(10 * time.Second) // Stop pubs. pubActive.Store(false) // Let settle. time.Sleep(10 * time.Second) close(qch) time.Sleep(20 * time.Second) nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() minAckFloor := func(stream string) (uint64, string) { var maf uint64 = math.MaxUint64 var consumer string for i := 0; i < numConsumersPer; i++ { cname := fmt.Sprintf("C%d", i) ci, err := js.ConsumerInfo(stream, cname) require_NoError(t, err) if ci.AckFloor.Stream < maf { maf = ci.AckFloor.Stream consumer = cname } } return maf, consumer } checkStreamAcks := func(stream string) { mm.Lock() defer mm.Unlock() if sam := ackMap[stream]; sam != nil { for seq := 1; ; seq++ { acks := sam[uint64(seq)] if acks == nil { if sam[uint64(seq+1)] != nil { t.Logf("Missing an ack on stream %q for sequence %d\n", stream, seq) } else { break } } if len(acks) > 1 { t.Logf("Multiple acks for %d which is not expected: %+v", seq, acks) } } } } // Now check all streams such that their first sequence is equal to the minimum of all consumers. for i := 0; i < numStreams; i++ { stream := fmt.Sprintf("A-%d", i) si, err := js.StreamInfo(stream) require_NoError(t, err) if maf, consumer := minAckFloor(stream); maf > si.State.FirstSeq { t.Logf("\nBAD STATE DETECTED FOR %q, CHECKING OTHER SERVERS! ACK %d vs %+v LEADER %v, CL FOR %q %v\n", stream, maf, si.State, c.streamLeader(globalAccountName, stream), consumer, c.consumerLeader(globalAccountName, stream, consumer)) t.Logf("TEST ACKS %+v\n", ackMap) checkStreamAcks(stream) for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream(stream) require_NoError(t, err) state := mset.state() t.Logf("Server %v Stream STATE %+v\n", s, state) var smv StoreMsg if sm, err := mset.store.LoadMsg(state.FirstSeq, &smv); err == nil { t.Logf("Subject for msg %d is %q", state.FirstSeq, sm.subj) } else { t.Logf("Could not retrieve msg for %d: %v", state.FirstSeq, err) } if len(mset.preAcks) > 0 { t.Logf("%v preAcks %+v\n", s, mset.preAcks) } for _, o := range mset.consumers { ostate, err := o.store.State() require_NoError(t, err) t.Logf("Consumer STATE for %q is %+v\n", o.name, ostate) } } t.Fatalf("BAD STATE: ACKFLOOR > FIRST %d vs %d\n", maf, si.State.FirstSeq) } } } func TestNoRaceFileStoreNumPending(t *testing.T) { // No need for all permutations here. storeDir := t.TempDir() fcfg := FileStoreConfig{ StoreDir: storeDir, BlockSize: 2 * 1024, // Create many blocks on purpose. } fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"*.*.*.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() tokens := []string{"foo", "bar", "baz"} genSubj := func() string { return fmt.Sprintf("%s.%s.%s.%s", tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], tokens[rand.Intn(len(tokens))], ) } for i := 0; i < 50_000; i++ { subj := genSubj() _, _, err := fs.StoreMsg(subj, nil, []byte("Hello World")) require_NoError(t, err) } state := fs.State() // Scan one by one for sanity check against other calculations. sanityCheck := func(sseq uint64, filter string) SimpleState { t.Helper() var ss SimpleState var smv StoreMsg // For here we know 0 is invalid, set to 1. if sseq == 0 { sseq = 1 } for seq := sseq; seq <= state.LastSeq; seq++ { sm, err := fs.LoadMsg(seq, &smv) if err != nil { t.Logf("Encountered error %v loading sequence: %d", err, seq) continue } if subjectIsSubsetMatch(sm.subj, filter) { ss.Msgs++ ss.Last = seq if ss.First == 0 || seq < ss.First { ss.First = seq } } } return ss } check := func(sseq uint64, filter string) { t.Helper() np, lvs := fs.NumPending(sseq, filter, false) ss := fs.FilteredState(sseq, filter) sss := sanityCheck(sseq, filter) if lvs != state.LastSeq { t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) } if ss.Msgs != np { t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) } if ss != sss { t.Fatalf("Failed sanity check, expected %+v got %+v", sss, ss) } } sanityCheckLastOnly := func(sseq uint64, filter string) SimpleState { t.Helper() var ss SimpleState var smv StoreMsg // For here we know 0 is invalid, set to 1. if sseq == 0 { sseq = 1 } seen := make(map[string]bool) for seq := state.LastSeq; seq >= sseq; seq-- { sm, err := fs.LoadMsg(seq, &smv) if err != nil { t.Logf("Encountered error %v loading sequence: %d", err, seq) continue } if !seen[sm.subj] && subjectIsSubsetMatch(sm.subj, filter) { ss.Msgs++ if ss.Last == 0 { ss.Last = seq } if ss.First == 0 || seq < ss.First { ss.First = seq } seen[sm.subj] = true } } return ss } checkLastOnly := func(sseq uint64, filter string) { t.Helper() np, lvs := fs.NumPending(sseq, filter, true) ss := sanityCheckLastOnly(sseq, filter) if lvs != state.LastSeq { t.Fatalf("Expected NumPending to return valid through last of %d but got %d", state.LastSeq, lvs) } if ss.Msgs != np { t.Fatalf("NumPending of %d did not match ss.Msgs of %d", np, ss.Msgs) } } startSeqs := []uint64{0, 1, 2, 200, 444, 555, 2222, 8888, 12_345, 28_222, 33_456, 44_400, 49_999} checkSubs := []string{"foo.>", "*.bar.>", "foo.bar.*.baz", "*.bar.>", "*.foo.bar.*", "foo.foo.bar.baz"} for _, filter := range checkSubs { for _, start := range startSeqs { check(start, filter) checkLastOnly(start, filter) } } } func TestNoRaceJetStreamClusterUnbalancedInterestMultipleConsumers(t *testing.T) { c, np := createStretchUnbalancedCluster(t) defer c.shutdown() defer np.stop() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Now create the stream. _, err := js.AddStream(&nats.StreamConfig{ Name: "EVENTS", Subjects: []string{"EV.>"}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) // Make sure it's leader is on S2. sl := c.servers[1] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader(globalAccountName, "EVENTS") if s := c.streamLeader(globalAccountName, "EVENTS"); s != sl { s.JetStreamStepdownStream(globalAccountName, "EVENTS") return fmt.Errorf("Server %s is not stream leader yet", sl) } return nil }) // Create a fast ack consumer. _, err = js.Subscribe("EV.NEW", func(m *nats.Msg) { m.Ack() }, nats.Durable("C"), nats.ManualAck()) require_NoError(t, err) // Make sure the consumer leader is on S3. cl := c.servers[2] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "C") if s := c.consumerLeader(globalAccountName, "EVENTS", "C"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "C") return fmt.Errorf("Server %s is not consumer leader yet", cl) } return nil }) // Connect a client directly to the stream leader. nc, js = jsClientConnect(t, sl) defer nc.Close() // Now create a pull subscriber. sub, err := js.PullSubscribe("EV.NEW", "D", nats.ManualAck()) require_NoError(t, err) // Make sure this consumer leader is on S1. cl = c.servers[0] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "D") if s := c.consumerLeader(globalAccountName, "EVENTS", "D"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "D") return fmt.Errorf("Server %s is not consumer leader yet", cl) } return nil }) numToSend := 1000 for i := 0; i < numToSend; i++ { _, err := js.PublishAsync("EV.NEW", nil) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } // Now make sure we can pull messages since we have not acked. // The bug is that the acks arrive on S1 faster then the messages but we want to // make sure we do not remove prematurely. msgs, err := sub.Fetch(100, nats.MaxWait(time.Second)) require_NoError(t, err) require_Len(t, len(msgs), 100) for _, m := range msgs { m.AckSync() } ci, err := js.ConsumerInfo("EVENTS", "D") require_NoError(t, err) require_Equal(t, ci.NumPending, uint64(numToSend-100)) require_Equal(t, ci.NumAckPending, 0) require_Equal(t, ci.Delivered.Stream, 100) require_Equal(t, ci.AckFloor.Stream, 100) // Check stream state on all servers. // Since acks result in messages to be removed through proposals, // it could take some time to be reflected in the stream state. checkFor(t, 5*time.Second, 500*time.Millisecond, func() error { for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("EVENTS") if err != nil { return err } state := mset.state() if state.Msgs != 900 { return fmt.Errorf("expected state.Msgs=900, got %d", state.Msgs) } if state.FirstSeq != 101 { return fmt.Errorf("expected state.FirstSeq=101, got %d", state.FirstSeq) } if state.LastSeq != 1000 { return fmt.Errorf("expected state.LastSeq=1000, got %d", state.LastSeq) } if state.Consumers != 2 { return fmt.Errorf("expected state.Consumers=2, got %d", state.Consumers) } } return nil }) msgs, err = sub.Fetch(900, nats.MaxWait(time.Second)) require_NoError(t, err) require_Len(t, len(msgs), 900) for _, m := range msgs { m.AckSync() } // Let acks propagate. time.Sleep(250 * time.Millisecond) // Check final stream state on all servers. for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("EVENTS") require_NoError(t, err) state := mset.state() require_Equal(t, state.Msgs, 0) require_Equal(t, state.FirstSeq, 1001) require_Equal(t, state.LastSeq, 1000) require_Equal(t, state.Consumers, 2) // Now check preAcks mset.mu.RLock() numPreAcks := len(mset.preAcks) mset.mu.RUnlock() require_Len(t, numPreAcks, 0) } } func TestNoRaceJetStreamClusterUnbalancedInterestMultipleFilteredConsumers(t *testing.T) { c, np := createStretchUnbalancedCluster(t) defer c.shutdown() defer np.stop() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Now create the stream. _, err := js.AddStream(&nats.StreamConfig{ Name: "EVENTS", Subjects: []string{"EV.>"}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) // Make sure it's leader is on S2. sl := c.servers[1] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader(globalAccountName, "EVENTS") if s := c.streamLeader(globalAccountName, "EVENTS"); s != sl { s.JetStreamStepdownStream(globalAccountName, "EVENTS") return fmt.Errorf("Server %s is not stream leader yet", sl) } return nil }) // Create a fast ack consumer. _, err = js.Subscribe("EV.NEW", func(m *nats.Msg) { m.Ack() }, nats.Durable("C"), nats.ManualAck()) require_NoError(t, err) // Make sure the consumer leader is on S3. cl := c.servers[2] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "C") if s := c.consumerLeader(globalAccountName, "EVENTS", "C"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "C") return fmt.Errorf("Server %s is not consumer leader yet", cl) } return nil }) // Connect a client directly to the stream leader. nc, js = jsClientConnect(t, sl) defer nc.Close() // Now create another fast ack consumer. _, err = js.Subscribe("EV.UPDATED", func(m *nats.Msg) { m.Ack() }, nats.Durable("D"), nats.ManualAck()) require_NoError(t, err) // Make sure this consumer leader is on S1. cl = c.servers[0] checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "D") if s := c.consumerLeader(globalAccountName, "EVENTS", "D"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "D") return fmt.Errorf("Server %s is not consumer leader yet", cl) } return nil }) numToSend := 500 for i := 0; i < numToSend; i++ { _, err := js.PublishAsync("EV.NEW", nil) require_NoError(t, err) _, err = js.PublishAsync("EV.UPDATED", nil) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } // Let acks propagate. time.Sleep(250 * time.Millisecond) ci, err := js.ConsumerInfo("EVENTS", "D") require_NoError(t, err) require_Equal(t, ci.NumPending, 0) require_Equal(t, ci.NumAckPending, 0) require_Equal(t, ci.Delivered.Consumer, 500) require_Equal(t, ci.Delivered.Stream, 1000) require_Equal(t, ci.AckFloor.Consumer, 500) require_Equal(t, ci.AckFloor.Stream, 1000) // Check final stream state on all servers. for _, s := range c.servers { mset, err := s.GlobalAccount().lookupStream("EVENTS") require_NoError(t, err) state := mset.state() require_Equal(t, state.Msgs, 0) require_Equal(t, state.FirstSeq, 1001) require_Equal(t, state.LastSeq, 1000) require_Equal(t, state.Consumers, 2) // Now check preAcks mset.mu.RLock() numPreAcks := len(mset.preAcks) mset.mu.RUnlock() require_Len(t, numPreAcks, 0) } } func TestNoRaceParallelStreamAndConsumerCreation(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // stream config. scfg := &StreamConfig{ Name: "TEST", Subjects: []string{"foo", "bar"}, MaxMsgs: 10, Storage: FileStorage, Replicas: 1, } // Will do these direct against the low level API to really make // sure parallel creation ok. np := 1000 startCh := make(chan bool) errCh := make(chan error, np) wg := sync.WaitGroup{} wg.Add(np) var streams sync.Map for i := 0; i < np; i++ { go func() { defer wg.Done() // Make them all fire at once. <-startCh if mset, err := s.GlobalAccount().addStream(scfg); err != nil { t.Logf("Stream create got an error: %v", err) errCh <- err } else { streams.Store(mset, true) } }() } time.Sleep(100 * time.Millisecond) close(startCh) wg.Wait() // Check for no errors. if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } // Now make sure we really only created one stream. var numStreams int streams.Range(func(k, v any) bool { numStreams++ return true }) if numStreams > 1 { t.Fatalf("Expected only one stream to be really created, got %d out of %d attempts", numStreams, np) } // Also make sure we cleanup the inflight entries for streams. gacc := s.GlobalAccount() _, jsa, err := gacc.checkForJetStream() require_NoError(t, err) var numEntries int jsa.inflight.Range(func(k, v any) bool { numEntries++ return true }) if numEntries > 0 { t.Fatalf("Expected no inflight entries to be left over, got %d", numEntries) } // Now do consumers. mset, err := gacc.lookupStream("TEST") require_NoError(t, err) cfg := &ConsumerConfig{ DeliverSubject: "to", Name: "DLC", AckPolicy: AckExplicit, } startCh = make(chan bool) errCh = make(chan error, np) wg.Add(np) var consumers sync.Map for i := 0; i < np; i++ { go func() { defer wg.Done() // Make them all fire at once. <-startCh if _, err = mset.addConsumer(cfg); err != nil { t.Logf("Consumer create got an error: %v", err) errCh <- err } else { consumers.Store(mset, true) } }() } time.Sleep(100 * time.Millisecond) close(startCh) wg.Wait() // Check for no errors. if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } // Now make sure we really only created one stream. var numConsumers int consumers.Range(func(k, v any) bool { numConsumers++ return true }) if numConsumers > 1 { t.Fatalf("Expected only one consumer to be really created, got %d out of %d attempts", numConsumers, np) } } func TestNoRaceRoutePool(t *testing.T) { var dur1 time.Duration var dur2 time.Duration total := 1_000_000 for _, test := range []struct { name string poolSize int }{ {"no pooling", 0}, {"pooling", 5}, } { t.Run(test.name, func(t *testing.T) { tmpl := ` port: -1 accounts { A { users: [{user: "A", password: "A"}] } B { users: [{user: "B", password: "B"}] } C { users: [{user: "C", password: "C"}] } D { users: [{user: "D", password: "D"}] } E { users: [{user: "E", password: "E"}] } } cluster { port: -1 name: "local" %s pool_size: %d } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.poolSize))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), test.poolSize))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) wg := sync.WaitGroup{} wg.Add(5) sendAndRecv := func(acc string) (*nats.Conn, *nats.Conn) { t.Helper() s2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(acc, acc)) count := 0 natsSub(t, s2nc, "foo", func(_ *nats.Msg) { if count++; count == total { wg.Done() } }) natsFlush(t, s2nc) s1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(acc, acc)) checkSubInterest(t, s1, acc, "foo", time.Second) return s2nc, s1nc } var rcv = [5]*nats.Conn{} var snd = [5]*nats.Conn{} accs := []string{"A", "B", "C", "D", "E"} for i := 0; i < 5; i++ { rcv[i], snd[i] = sendAndRecv(accs[i]) defer rcv[i].Close() defer snd[i].Close() } payload := []byte("some message") start := time.Now() for i := 0; i < 5; i++ { go func(idx int) { for i := 0; i < total; i++ { snd[idx].Publish("foo", payload) } }(i) } wg.Wait() dur := time.Since(start) if test.poolSize == 0 { dur1 = dur } else { dur2 = dur } }) } perf1 := float64(total*5) / dur1.Seconds() t.Logf("No pooling: %.0f msgs/sec", perf1) perf2 := float64(total*5) / dur2.Seconds() t.Logf("Pooling : %.0f msgs/sec", perf2) t.Logf("Gain : %.2fx", perf2/perf1) } func testNoRaceRoutePerAccount(t *testing.T, useWildCard bool) { var dur1 time.Duration var dur2 time.Duration accounts := make([]string, 5) for i := 0; i < 5; i++ { akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() accounts[i] = pub } routeAccs := fmt.Sprintf("accounts: [\"%s\", \"%s\", \"%s\", \"%s\", \"%s\"]", accounts[0], accounts[1], accounts[2], accounts[3], accounts[4]) total := 1_000_000 for _, test := range []struct { name string dedicated bool }{ {"route for all accounts", false}, {"route per account", true}, } { t.Run(test.name, func(t *testing.T) { tmpl := ` server_name: "%s" port: -1 accounts { %s { users: [{user: "0", password: "0"}] } %s { users: [{user: "1", password: "1"}] } %s { users: [{user: "2", password: "2"}] } %s { users: [{user: "3", password: "3"}] } %s { users: [{user: "4", password: "4"}] } } cluster { port: -1 name: "local" %s %s } ` var racc string if test.dedicated { racc = routeAccs } else { racc = _EMPTY_ } conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", accounts[0], accounts[1], accounts[2], accounts[3], accounts[4], _EMPTY_, racc))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", accounts[0], accounts[1], accounts[2], accounts[3], accounts[4], fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), racc))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) wg := sync.WaitGroup{} wg.Add(5) sendAndRecv := func(acc string, user string) (*nats.Conn, *nats.Conn) { t.Helper() s2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, user)) count := 0 var subj string var checkSubj string if useWildCard { subj, checkSubj = "foo.*", "foo.0" } else { subj, checkSubj = "foo", "foo" } natsSub(t, s2nc, subj, func(_ *nats.Msg) { if count++; count == total { wg.Done() } }) natsFlush(t, s2nc) s1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, user)) checkSubInterest(t, s1, acc, checkSubj, time.Second) return s2nc, s1nc } var rcv = [5]*nats.Conn{} var snd = [5]*nats.Conn{} users := []string{"0", "1", "2", "3", "4"} for i := 0; i < 5; i++ { rcv[i], snd[i] = sendAndRecv(accounts[i], users[i]) defer rcv[i].Close() defer snd[i].Close() } payload := []byte("some message") start := time.Now() for i := 0; i < 5; i++ { go func(idx int) { for i := 0; i < total; i++ { var subj string if useWildCard { subj = fmt.Sprintf("foo.%d", i) } else { subj = "foo" } snd[idx].Publish(subj, payload) } }(i) } wg.Wait() dur := time.Since(start) if !test.dedicated { dur1 = dur } else { dur2 = dur } }) } perf1 := float64(total*5) / dur1.Seconds() t.Logf("Route for all accounts: %.0f msgs/sec", perf1) perf2 := float64(total*5) / dur2.Seconds() t.Logf("Route per account : %.0f msgs/sec", perf2) t.Logf("Gain : %.2fx", perf2/perf1) } func TestNoRaceRoutePerAccount(t *testing.T) { testNoRaceRoutePerAccount(t, false) } func TestNoRaceRoutePerAccountSubWithWildcard(t *testing.T) { testNoRaceRoutePerAccount(t, true) } // This test, which checks that messages are not duplicated when pooling or // per-account routes are reloaded, would cause a DATA RACE that is not // specific to the changes for pooling/per_account. For this reason, this // test is located in the norace_test.go file. func TestNoRaceRoutePoolAndPerAccountConfigReload(t *testing.T) { for _, test := range []struct { name string poolSizeBefore string poolSizeAfter string accountsBefore string accountsAfter string }{ {"from no pool to pool", _EMPTY_, "pool_size: 2", _EMPTY_, _EMPTY_}, {"increase pool size", "pool_size: 2", "pool_size: 5", _EMPTY_, _EMPTY_}, {"decrease pool size", "pool_size: 5", "pool_size: 2", _EMPTY_, _EMPTY_}, {"from pool to no pool", "pool_size: 5", _EMPTY_, _EMPTY_, _EMPTY_}, {"from no account to account", _EMPTY_, _EMPTY_, _EMPTY_, "accounts: [\"A\"]"}, {"add account", _EMPTY_, _EMPTY_, "accounts: [\"B\"]", "accounts: [\"A\",\"B\"]"}, {"remove account", _EMPTY_, _EMPTY_, "accounts: [\"A\",\"B\"]", "accounts: [\"B\"]"}, {"from account to no account", _EMPTY_, _EMPTY_, "accounts: [\"A\"]", _EMPTY_}, {"increase pool size and add account", "pool_size: 2", "pool_size: 3", "accounts: [\"B\"]", "accounts: [\"B\",\"A\"]"}, {"decrease pool size and remove account", "pool_size: 3", "pool_size: 2", "accounts: [\"A\",\"B\"]", "accounts: [\"B\"]"}, } { t.Run(test.name, func(t *testing.T) { tmplA := ` port: -1 server_name: "A" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } } cluster: { port: -1 name: "local" %s %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, test.poolSizeBefore, test.accountsBefore))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() tmplB := ` port: -1 server_name: "B" accounts { A { users: [{user: a, password: pwd}] } B { users: [{user: b, password: pwd}] } } cluster: { port: -1 name: "local" %s %s routes: ["nats://127.0.0.1:%d"] } ` confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, test.poolSizeBefore, test.accountsBefore, optsA.Cluster.Port))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) ncA := natsConnect(t, srva.ClientURL(), nats.UserInfo("a", "pwd")) defer ncA.Close() sub := natsSubSync(t, ncA, "foo") sub.SetPendingLimits(-1, -1) checkSubInterest(t, srvb, "A", "foo", time.Second) ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("a", "pwd")) defer ncB.Close() wg := sync.WaitGroup{} wg.Add(1) ch := make(chan struct{}) go func() { defer wg.Done() for i := 0; ; i++ { ncB.Publish("foo", []byte(fmt.Sprintf("%d", i))) select { case <-ch: return default: } if i%300 == 0 { time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond) } } }() var l *captureErrorLogger if test.accountsBefore != _EMPTY_ && test.accountsAfter == _EMPTY_ { l = &captureErrorLogger{errCh: make(chan string, 100)} srva.SetLogger(l, false, false) } time.Sleep(250 * time.Millisecond) reloadUpdateConfig(t, srva, confA, fmt.Sprintf(tmplA, test.poolSizeAfter, test.accountsAfter)) time.Sleep(125 * time.Millisecond) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(tmplB, test.poolSizeAfter, test.accountsAfter, optsA.Cluster.Port)) checkClusterFormed(t, srva, srvb) checkSubInterest(t, srvb, "A", "foo", time.Second) if l != nil { // Errors regarding "No route for account" should stop var ok bool for numErrs := 0; !ok && numErrs < 10; { select { case e := <-l.errCh: if strings.Contains(e, "No route for account") { numErrs++ } case <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond): ok = true } } if !ok { t.Fatalf("Still report of no route for account") } } close(ch) wg.Wait() for prev := -1; ; { msg, err := sub.NextMsg(50 * time.Millisecond) if err != nil { break } cur, _ := strconv.Atoi(string(msg.Data)) if cur <= prev { t.Fatalf("Previous was %d, got %d", prev, cur) } prev = cur } }) } } // This test ensures that outbound queues don't cause a run on // memory when sending something to lots of clients. func TestNoRaceClientOutboundQueueMemory(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() var before runtime.MemStats var after runtime.MemStats var err error clients := make([]*nats.Conn, 50000) wait := &sync.WaitGroup{} wait.Add(len(clients)) for i := 0; i < len(clients); i++ { clients[i], err = nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.InProcessServer(s)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer clients[i].Close() clients[i].Subscribe("test", func(m *nats.Msg) { wait.Done() }) } runtime.GC() runtime.ReadMemStats(&before) nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.InProcessServer(s)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() var m [48000]byte if err = nc.Publish("test", m[:]); err != nil { t.Fatal(err) } wait.Wait() runtime.GC() runtime.ReadMemStats(&after) hb, ha := float64(before.HeapAlloc), float64(after.HeapAlloc) ms := float64(len(m)) diff := float64(ha) - float64(hb) inc := (diff / float64(hb)) * 100 if inc > 10 { t.Logf("Message size: %.1fKB\n", ms/1024) t.Logf("Subscribed clients: %d\n", len(clients)) t.Logf("Heap allocs before: %.1fMB\n", hb/1024/1024) t.Logf("Heap allocs after: %.1fMB\n", ha/1024/1024) t.Logf("Heap allocs delta: %.1f%%\n", inc) t.Fatalf("memory increase was %.1f%% (should be <= 10%%)", inc) } } func TestNoRaceJetStreamClusterLeafnodeConnectPerf(t *testing.T) { // Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm. skip(t) tmpl := strings.Replace(jsClusterAccountsTempl, "store_dir:", "domain: cloud, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "CLOUD", _EMPTY_, 3, 18033, true) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "STATE", Subjects: []string{"STATE.GLOBAL.CELL1.*.>"}, Replicas: 3, }) require_NoError(t, err) tmpl = strings.Replace(jsClusterTemplWithSingleFleetLeafNode, "store_dir:", "domain: vehicle, store_dir:", 1) var vinSerial int genVIN := func() string { vinSerial++ return fmt.Sprintf("7PDSGAALXNN%06d", vinSerial) } numVehicles := 500 for i := 0; i < numVehicles; i++ { start := time.Now() vin := genVIN() ln := c.createLeafNodeWithTemplateNoSystemWithProto(vin, tmpl, "ws") nc, js := jsClientConnect(t, ln) _, err := js.AddStream(&nats.StreamConfig{ Name: "VEHICLE", Subjects: []string{"STATE.GLOBAL.LOCAL.>"}, Sources: []*nats.StreamSource{{ Name: "STATE", FilterSubject: fmt.Sprintf("STATE.GLOBAL.CELL1.%s.>", vin), External: &nats.ExternalStream{ APIPrefix: "$JS.cloud.API", DeliverPrefix: fmt.Sprintf("DELIVER.STATE.GLOBAL.CELL1.%s", vin), }, }}, }) require_NoError(t, err) // Create the sourced stream. checkLeafNodeConnectedCount(t, ln, 1) if elapsed := time.Since(start); elapsed > 2*time.Second { t.Fatalf("Took too long to create leafnode %d connection: %v", i+1, elapsed) } nc.Close() } } func TestNoRaceJetStreamClusterDifferentRTTInterestBasedStreamPreAck(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} cluster { name: "F3" listen: 127.0.0.1:%d routes = [%s] } accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` // Route Ports // "S1": 14622, // "S2": 15622, // "S3": 16622, // S2 (stream leader) will have a slow path to S1 (via proxy) and S3 (consumer leader) will have a fast path. // Do these in order, S1, S2 (proxy) then S3. c := &cluster{t: t, servers: make([]*Server, 3), opts: make([]*Options, 3), name: "F3"} // S1 // The route connection to S2 must be through a slow proxy np12 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, "route://127.0.0.1:15622", true) routes := fmt.Sprintf("%s, route://127.0.0.1:16622", np12.routeURL()) conf := fmt.Sprintf(tmpl, "S1", t.TempDir(), 14622, routes) c.servers[0], c.opts[0] = RunServerWithConfig(createConfFile(t, []byte(conf))) // S2 // The route connection to S1 must be through a slow proxy np21 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, "route://127.0.0.1:14622", true) routes = fmt.Sprintf("%s, route://127.0.0.1:16622", np21.routeURL()) conf = fmt.Sprintf(tmpl, "S2", t.TempDir(), 15622, routes) c.servers[1], c.opts[1] = RunServerWithConfig(createConfFile(t, []byte(conf))) // S3 conf = fmt.Sprintf(tmpl, "S3", t.TempDir(), 16622, "route://127.0.0.1:14622, route://127.0.0.1:15622") c.servers[2], c.opts[2] = RunServerWithConfig(createConfFile(t, []byte(conf))) c.checkClusterFormed() c.waitOnClusterReady() defer c.shutdown() defer np12.stop() defer np21.stop() slow := c.servers[0] // Expecting pre-acks here. sl := c.servers[1] // Stream leader, will publish here. cl := c.servers[2] // Consumer leader, will consume & ack here. snc, sjs := jsClientConnect(t, sl) defer snc.Close() cnc, cjs := jsClientConnect(t, cl) defer cnc.Close() // Now create the stream. _, err := sjs.AddStream(&nats.StreamConfig{ Name: "EVENTS", Subjects: []string{"EV.>"}, Replicas: 3, Retention: nats.InterestPolicy, }) require_NoError(t, err) // Make sure it's leader is on S2. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnStreamLeader(globalAccountName, "EVENTS") if s := c.streamLeader(globalAccountName, "EVENTS"); s != sl { s.JetStreamStepdownStream(globalAccountName, "EVENTS") return fmt.Errorf("Server %s is not stream leader yet", sl) } return nil }) // Now create the consumer. _, err = sjs.AddConsumer("EVENTS", &nats.ConsumerConfig{ Durable: "C", AckPolicy: nats.AckExplicitPolicy, DeliverSubject: "dx", }) require_NoError(t, err) // Make sure the consumer leader is on S3. checkFor(t, 20*time.Second, 200*time.Millisecond, func() error { c.waitOnConsumerLeader(globalAccountName, "EVENTS", "C") if s := c.consumerLeader(globalAccountName, "EVENTS", "C"); s != cl { s.JetStreamStepdownConsumer(globalAccountName, "EVENTS", "C") return fmt.Errorf("Server %s is not consumer leader yet", sl) } return nil }) _, err = cjs.Subscribe(_EMPTY_, func(msg *nats.Msg) { msg.Ack() }, nats.BindStream("EVENTS"), nats.Durable("C"), nats.ManualAck()) require_NoError(t, err) // Publish directly on the stream leader to make it efficient. for i := 0; i < 1_000; i++ { _, err := sjs.PublishAsync("EV.PAID", []byte("ok")) require_NoError(t, err) } select { case <-sjs.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } mset, err := slow.GlobalAccount().lookupStream("EVENTS") require_NoError(t, err) checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { state := mset.state() if state.LastSeq != 1000 { return fmt.Errorf("Haven't received all messages yet (last seq %d)", state.LastSeq) } mset.mu.RLock() preAcks := mset.preAcks mset.mu.RUnlock() if preAcks == nil { return fmt.Errorf("Expected to have preAcks by now") } if state.Msgs == 0 { mset.mu.RLock() lp := len(mset.preAcks) mset.mu.RUnlock() if lp == 0 { return nil } else { t.Fatalf("Expected no preAcks with no msgs, but got %d", lp) } } return fmt.Errorf("Still have %d msgs left", state.Msgs) }) } func TestNoRaceCheckAckFloorWithVeryLargeFirstSeqAndNewConsumers(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, _ := jsClientConnect(t, s) defer nc.Close() // Make sure to time bound here for the acksync call below. js, err := nc.JetStream(nats.MaxWait(200 * time.Millisecond)) require_NoError(t, err) _, err = js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"wq-req"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) largeFirstSeq := uint64(1_200_000_000) err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: largeFirstSeq}) require_NoError(t, err) si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == largeFirstSeq) // Add a simple request to the stream. sendStreamMsg(t, nc, "wq-req", "HELP") sub, err := js.PullSubscribe("wq-req", "dlc") require_NoError(t, err) msgs, err := sub.Fetch(1) require_NoError(t, err) require_True(t, len(msgs) == 1) // The bug is around the checkAckFloor walking the sequences from current ackfloor // to the first sequence of the stream. We time bound the max wait with the js context // to 200ms. Since checkAckFloor is spinning and holding up processing of acks this will fail. // We will short circuit new consumers to fix this one. require_NoError(t, msgs[0].AckSync()) // Now do again so we move past the new consumer with no ack floor situation. err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 2 * largeFirstSeq}) require_NoError(t, err) si, err = js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 2*largeFirstSeq) sendStreamMsg(t, nc, "wq-req", "MORE HELP") // We check this one directly for this use case. mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) o := mset.lookupConsumer("dlc") require_True(t, o != nil) // Purge will move the stream floor by default, so force into the situation where it is back to largeFirstSeq. // This will not trigger the new consumer logic, but will trigger a walk of the sequence space. // Fix will be to walk the lesser of the two linear spaces. o.mu.Lock() o.asflr = largeFirstSeq o.mu.Unlock() done := make(chan bool) go func() { o.checkAckFloor() done <- true }() select { case <-done: return case <-time.After(time.Second): t.Fatalf("Check ack floor taking too long!") } } func TestNoRaceReplicatedMirrorWithLargeStartingSequenceOverLeafnode(t *testing.T) { // Cluster B tmpl := strings.Replace(jsClusterTempl, "store_dir:", "domain: B, store_dir:", 1) c := createJetStreamCluster(t, tmpl, "B", _EMPTY_, 3, 22020, true) defer c.shutdown() // Cluster A // Domain is "A' lc := c.createLeafNodesWithStartPortAndDomain("A", 3, 22110, "A") defer lc.shutdown() lc.waitOnClusterReady() // Create a stream on B (HUB/CLOUD) and set its starting sequence very high. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 3, }) require_NoError(t, err) err = js.PurgeStream("TEST", &nats.StreamPurgeRequest{Sequence: 1_000_000_000}) require_NoError(t, err) // Send in a small amount of messages. for i := 0; i < 1000; i++ { sendStreamMsg(t, nc, "foo", "Hello") } si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 1_000_000_000) // Now try to create a replicated mirror on the leaf cluster. lnc, ljs := jsClientConnect(t, lc.randomServer()) defer lnc.Close() _, err = ljs.AddStream(&nats.StreamConfig{ Name: "TEST", Mirror: &nats.StreamSource{ Name: "TEST", Domain: "B", }, }) require_NoError(t, err) // Make sure we sync quickly. checkFor(t, time.Second, 200*time.Millisecond, func() error { si, err = ljs.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs == 1000 && si.State.FirstSeq == 1_000_000_000 { return nil } return fmt.Errorf("Mirror state not correct: %+v", si.State) }) } func TestNoRaceBinaryStreamSnapshotEncodingBasic(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, MaxMsgsPerSubject: 1, }) require_NoError(t, err) // Set first key sendStreamMsg(t, nc, "key:1", "hello") // Set Second key but keep updating it, causing a laggard pattern. value := bytes.Repeat([]byte("Z"), 8*1024) for i := 0; i <= 1000; i++ { _, err := js.PublishAsync("key:2", value) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Now do more of swiss cheese style. for i := 3; i <= 1000; i++ { key := fmt.Sprintf("key:%d", i) _, err := js.PublishAsync(key, value) require_NoError(t, err) // Send it twice to create hole right behind it, like swiss cheese. _, err = js.PublishAsync(key, value) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Make for round numbers for stream state. sendStreamMsg(t, nc, "key:2", "hello") sendStreamMsg(t, nc, "key:2", "world") si, err := js.StreamInfo("TEST") require_NoError(t, err) require_True(t, si.State.FirstSeq == 1) require_True(t, si.State.LastSeq == 3000) require_True(t, si.State.Msgs == 1000) require_True(t, si.State.NumDeleted == 2000) mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) snap, err := mset.store.EncodedStreamState(0) require_NoError(t, err) // Now decode the snapshot. ss, err := DecodeStreamState(snap) require_NoError(t, err) require_Equal(t, ss.FirstSeq, 1) require_Equal(t, ss.LastSeq, 3000) require_Equal(t, ss.Msgs, 1000) require_Equal(t, ss.Deleted.NumDeleted(), 2000) } func TestNoRaceFilestoreBinaryStreamSnapshotEncodingLargeGaps(t *testing.T) { storeDir := t.TempDir() fcfg := FileStoreConfig{ StoreDir: storeDir, BlockSize: 512, // Small on purpose to create alot of blks. } fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"zzz"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() subj, msg := "zzz", bytes.Repeat([]byte("X"), 128) numMsgs := 20_000 fs.StoreMsg(subj, nil, msg) for i := 2; i < numMsgs; i++ { seq, _, err := fs.StoreMsg(subj, nil, nil) require_NoError(t, err) fs.RemoveMsg(seq) } fs.StoreMsg(subj, nil, msg) snap, err := fs.EncodedStreamState(0) require_NoError(t, err) require_True(t, len(snap) < 512) // Now decode the snapshot. ss, err := DecodeStreamState(snap) require_NoError(t, err) require_True(t, ss.FirstSeq == 1) require_True(t, ss.LastSeq == 20_000) require_True(t, ss.Msgs == 2) require_True(t, len(ss.Deleted) <= 2) require_True(t, ss.Deleted.NumDeleted() == 19_998) } func TestNoRaceJetStreamClusterStreamSnapshotCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Client based API nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"*"}, MaxMsgsPerSubject: 1, Replicas: 3, }) require_NoError(t, err) msg := []byte("Hello World") _, err = js.Publish("foo", msg) require_NoError(t, err) for i := 1; i < 1000; i++ { _, err := js.PublishAsync("bar", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } sr := c.randomNonStreamLeader(globalAccountName, "TEST") sr.Shutdown() // In case we were connected to sr. nc, js = jsClientConnect(t, c.randomServer()) defer nc.Close() // Now create a large gap. for i := 0; i < 50_000; i++ { _, err := js.PublishAsync("bar", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(10 * time.Second): t.Fatalf("Did not receive completion signal") } sl := c.streamLeader(globalAccountName, "TEST") sl.JetStreamSnapshotStream(globalAccountName, "TEST") sr = c.restartServer(sr) c.checkClusterFormed() c.waitOnServerCurrent(sr) c.waitOnStreamCurrent(sr, globalAccountName, "TEST") mset, err := sr.GlobalAccount().lookupStream("TEST") require_NoError(t, err) // Make sure it's caught up var state StreamState mset.store.FastState(&state) require_Equal(t, state.Msgs, 2) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 51_000) require_Equal(t, state.NumDeleted, 51_000-2) sr.Shutdown() _, err = js.Publish("baz", msg) require_NoError(t, err) sl.JetStreamSnapshotStream(globalAccountName, "TEST") sr = c.restartServer(sr) c.checkClusterFormed() c.waitOnServerCurrent(sr) c.waitOnStreamCurrent(sr, globalAccountName, "TEST") mset, err = sr.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.store.FastState(&state) require_Equal(t, state.Msgs, 3) require_Equal(t, state.FirstSeq, 1) require_Equal(t, state.LastSeq, 51_001) require_Equal(t, state.NumDeleted, 51_001-3) } func TestNoRaceStoreStreamEncoderDecoder(t *testing.T) { cfg := &StreamConfig{ Name: "zzz", Subjects: []string{"*"}, MaxMsgsPer: 1, Storage: MemoryStorage, } ms, err := newMemStore(cfg) require_NoError(t, err) defer ms.Stop() fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"*"}, MaxMsgsPer: 1, Storage: FileStorage}, ) require_NoError(t, err) defer fs.Stop() const seed = 2222222 msg := bytes.Repeat([]byte("ABC"), 33) // ~100bytes maxEncodeTime := 2 * time.Second maxEncodeSize := 700 * 1024 test := func(t *testing.T, gs StreamStore) { t.Parallel() prand := rand.New(rand.NewSource(seed)) tick := time.NewTicker(time.Second) defer tick.Stop() done := time.NewTimer(10 * time.Second) for running := true; running; { select { case <-tick.C: var state StreamState gs.FastState(&state) if state.NumDeleted == 0 { continue } start := time.Now() snap, err := gs.EncodedStreamState(0) require_NoError(t, err) elapsed := time.Since(start) // Should take <1ms without race but if CI/CD is slow we will give it a bit of room. if elapsed > maxEncodeTime { t.Logf("Encode took longer then expected: %v", elapsed) } if len(snap) > maxEncodeSize { t.Fatalf("Expected snapshot size < %v got %v", friendlyBytes(maxEncodeSize), friendlyBytes(len(snap))) } ss, err := DecodeStreamState(snap) require_True(t, len(ss.Deleted) > 0) require_NoError(t, err) case <-done.C: running = false default: key := strconv.Itoa(prand.Intn(256_000)) gs.StoreMsg(key, nil, msg) } } } for _, gs := range []StreamStore{ms, fs} { switch gs.(type) { case *memStore: t.Run("MemStore", func(t *testing.T) { test(t, gs) }) case *fileStore: t.Run("FileStore", func(t *testing.T) { test(t, gs) }) } } } func TestNoRaceJetStreamClusterKVWithServerKill(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Setup the KV bucket and use for making assertions. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, History: 10, }) require_NoError(t, err) // Total number of keys to range over. numKeys := 50 // ID is the server id to explicitly connect to. work := func(ctx context.Context, wg *sync.WaitGroup, id int) { defer wg.Done() nc, js := jsClientConnectEx(t, c.servers[id], []nats.JSOpt{nats.Context(ctx)}) defer nc.Close() kv, err := js.KeyValue("TEST") require_NoError(t, err) // 100 messages a second for each single client. tk := time.NewTicker(10 * time.Millisecond) defer tk.Stop() for { select { case <-ctx.Done(): return case <-tk.C: // Pick a random key within the range. k := fmt.Sprintf("key.%d", rand.Intn(numKeys)) // Attempt to get a key. e, err := kv.Get(k) // If found, attempt to update or delete. if err == nil { if rand.Intn(10) < 3 { kv.Delete(k, nats.LastRevision(e.Revision())) } else { kv.Update(k, nil, e.Revision()) } } else if errors.Is(err, nats.ErrKeyNotFound) { kv.Create(k, nil) } } } } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var wg sync.WaitGroup wg.Add(3) go work(ctx, &wg, 0) go work(ctx, &wg, 1) go work(ctx, &wg, 2) time.Sleep(time.Second) // Simulate server stop and restart. for i := 0; i < 7; i++ { s := c.randomServer() s.Shutdown() c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "KV_TEST") // Wait for a bit and then start the server again. time.Sleep(time.Duration(rand.Intn(1250)) * time.Millisecond) s = c.restartServer(s) c.waitOnServerCurrent(s) c.waitOnLeader() c.waitOnStreamLeader(globalAccountName, "KV_TEST") c.waitOnPeerCount(3) } // Stop the workload. cancel() wg.Wait() type fullState struct { state StreamState lseq uint64 clfs uint64 } grabState := func(mset *stream) *fullState { mset.mu.RLock() defer mset.mu.RUnlock() var state StreamState mset.store.FastState(&state) return &fullState{state, mset.lseq, mset.clfs} } grabStore := func(mset *stream) map[string][]uint64 { mset.mu.RLock() store := mset.store mset.mu.RUnlock() var state StreamState store.FastState(&state) storeMap := make(map[string][]uint64) for seq := state.FirstSeq; seq <= state.LastSeq; seq++ { if sm, err := store.LoadMsg(seq, nil); err == nil { storeMap[sm.subj] = append(storeMap[sm.subj], sm.seq) } } return storeMap } checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { // Current stream leader. sl := c.streamLeader(globalAccountName, "KV_TEST") mset, err := sl.GlobalAccount().lookupStream("KV_TEST") require_NoError(t, err) lstate := grabState(mset) golden := grabStore(mset) // Report messages per server. for _, s := range c.servers { if s == sl { continue } mset, err := s.GlobalAccount().lookupStream("KV_TEST") require_NoError(t, err) state := grabState(mset) if !reflect.DeepEqual(state, lstate) { return fmt.Errorf("Expected follower state\n%+v\nto match leader's\n %+v", state, lstate) } sm := grabStore(mset) if !reflect.DeepEqual(sm, golden) { t.Fatalf("Expected follower store for %v\n%+v\nto match leader's %v\n %+v", s, sm, sl, golden) } } return nil }) } func TestNoRaceFileStoreLargeMsgsAndFirstMatching(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd, BlockSize: 8 * 1024 * 1024}, StreamConfig{Name: "zzz", Subjects: []string{">"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() for i := 0; i < 150_000; i++ { fs.StoreMsg(fmt.Sprintf("foo.bar.%d", i), nil, nil) } for i := 0; i < 150_000; i++ { fs.StoreMsg(fmt.Sprintf("foo.baz.%d", i), nil, nil) } require_Equal(t, fs.numMsgBlocks(), 2) fs.mu.RLock() mb := fs.blks[1] fs.mu.RUnlock() fseq := atomic.LoadUint64(&mb.first.seq) // The -40 leaves enough mb.fss entries to kick in linear scan. for seq := fseq; seq < 300_000-40; seq++ { fs.RemoveMsg(uint64(seq)) } start := time.Now() fs.LoadNextMsg("*.baz.*", true, fseq, nil) require_True(t, time.Since(start) < 200*time.Microsecond) // Now remove more to kick into non-linear logic. for seq := 300_000 - 40; seq < 300_000; seq++ { fs.RemoveMsg(uint64(seq)) } start = time.Now() fs.LoadNextMsg("*.baz.*", true, fseq, nil) require_True(t, time.Since(start) < 200*time.Microsecond) } func TestNoRaceWSNoCorruptionWithFrameSizeLimit(t *testing.T) { testWSNoCorruptionWithFrameSizeLimit(t, 50000) } func TestNoRaceJetStreamAPIDispatchQueuePending(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() // Setup the KV bucket and use for making assertions. nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*.*"}, }) require_NoError(t, err) // Queue up 500k messages all with different subjects. // We want to make num pending for a consumer expensive, so a large subject // space and wildcards for now does the trick. toks := []string{"foo", "bar", "baz"} // for second token. for i := 1; i <= 500_000; i++ { subj := fmt.Sprintf("foo.%s.%d", toks[rand.Intn(len(toks))], i) _, err := js.PublishAsync(subj, nil, nats.StallWait(time.Second)) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } // To back up our pending queue we will create lots of filtered, with wildcards, R1 consumers // from a different server then the one hosting the stream. // ok to share this connection here. sldr := c.streamLeader(globalAccountName, "TEST") for _, s := range c.servers { if s != sldr { nc, js = jsClientConnect(t, s) defer nc.Close() break } } ngr, ncons := 100, 10 startCh, errCh := make(chan bool), make(chan error, ngr) var wg, swg sync.WaitGroup wg.Add(ngr) swg.Add(ngr) // The wildcard in the filter subject is the key. cfg := &nats.ConsumerConfig{FilterSubject: "foo.*.22"} var tt atomic.Int64 for i := 0; i < ngr; i++ { go func() { defer wg.Done() swg.Done() // Make them all fire at once. <-startCh for i := 0; i < ncons; i++ { start := time.Now() if _, err := js.AddConsumer("TEST", cfg); err != nil { errCh <- err t.Logf("Got err creating consumer: %v", err) } elapsed := time.Since(start) tt.Add(int64(elapsed)) } }() } swg.Wait() close(startCh) time.Sleep(time.Millisecond) jsz, _ := sldr.Jsz(nil) // This could be 0 legit, so just log, don't fail. if jsz.JetStreamStats.API.Inflight == 0 { t.Log("Expected a non-zero inflight") } wg.Wait() if len(errCh) > 0 { t.Fatalf("Expected no errors, got %d", len(errCh)) } } func TestNoRaceJetStreamMirrorAndSourceConsumerFailBackoff(t *testing.T) { owt := srcConsumerWaitTime srcConsumerWaitTime = 2 * time.Second defer func() { srcConsumerWaitTime = owt }() // Check calculations first. for i := 1; i <= 20; i++ { backoff := calculateRetryBackoff(i) if i < 12 { require_Equal(t, backoff, time.Duration(i)*10*time.Second) } else { require_Equal(t, backoff, retryMaximum) } } c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*.*"}, }) require_NoError(t, err) sl := c.streamLeader(globalAccountName, "TEST") // Create a mirror. ml := sl // Make sure not on the same server. Should not happened in general but possible. for ml == sl { js.DeleteStream("MIRROR") _, err = js.AddStream(&nats.StreamConfig{ Name: "MIRROR", Mirror: &nats.StreamSource{Name: "TEST"}, }) require_NoError(t, err) ml = c.streamLeader(globalAccountName, "MIRROR") } // Create a source. srcl := sl for srcl == sl { js.DeleteStream("SOURCE") _, err = js.AddStream(&nats.StreamConfig{ Name: "SOURCE", Sources: []*nats.StreamSource{{Name: "TEST"}}, }) require_NoError(t, err) srcl = c.streamLeader(globalAccountName, "MIRROR") } // Create sub to watch for the consumer create requests. nc, _ = jsClientConnect(t, ml) defer nc.Close() sub := natsSubSync(t, nc, "$JS.API.CONSUMER.CREATE.>") // Kill the server where the source is.. sldr := c.streamLeader(globalAccountName, "TEST") sldr.Shutdown() // Wait for just greater than 5s. We should only see 1 request during this time. time.Sleep(6 * time.Second) // There should have been 2 requests, one for mirror, one for source n, _, _ := sub.Pending() require_Equal(t, n, 2) var mreq, sreq int for i := 0; i < 2; i++ { msg := natsNexMsg(t, sub, time.Second) if bytes.Contains(msg.Data, []byte("$JS.M.")) { mreq++ } else if bytes.Contains(msg.Data, []byte("$JS.S.")) { sreq++ } } if mreq != 1 || sreq != 1 { t.Fatalf("Consumer create captures invalid: mreq=%v sreq=%v", mreq, sreq) } // Now make sure that the fails is set properly. mset, err := c.streamLeader(globalAccountName, "MIRROR").GlobalAccount().lookupStream("MIRROR") require_NoError(t, err) mset.mu.RLock() fails := mset.mirror.fails mset.mu.RUnlock() require_Equal(t, fails, 1) mset, err = c.streamLeader(globalAccountName, "SOURCE").GlobalAccount().lookupStream("SOURCE") require_NoError(t, err) mset.mu.RLock() si := mset.sources["TEST > >"] mset.mu.RUnlock() require_True(t, si != nil) require_Equal(t, si.fails, 1) } func TestNoRaceJetStreamClusterStreamCatchupLargeInteriorDeletes(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, MaxMsgsPerSubject: 100, Replicas: 1, } _, err := js.AddStream(cfg) require_NoError(t, err) msg := bytes.Repeat([]byte("Z"), 2*1024) // We will create lots of interior deletes on our R1 then scale up. _, err = js.Publish("foo.0", msg) require_NoError(t, err) // Create 50k messages randomly from 1-100 for i := 0; i < 50_000; i++ { subj := fmt.Sprintf("foo.%d", rand.Intn(100)+1) js.PublishAsync(subj, msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Now create a large gap. for i := 0; i < 100_000; i++ { js.PublishAsync("foo.2", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // Do 50k random again at end. for i := 0; i < 50_000; i++ { subj := fmt.Sprintf("foo.%d", rand.Intn(100)+1) js.PublishAsync(subj, msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } si, err := js.StreamInfo("TEST") require_NoError(t, err) cfg.Replicas = 2 _, err = js.UpdateStream(cfg) require_NoError(t, err) // Let catchup start. c.waitOnStreamLeader(globalAccountName, "TEST") nl := c.randomNonStreamLeader(globalAccountName, "TEST") require_True(t, nl != nil) mset, err := nl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) checkFor(t, 10*time.Second, 500*time.Millisecond, func() error { state := mset.state() if state.Msgs == si.State.Msgs { return nil } return fmt.Errorf("Msgs not equal %d vs %d", state.Msgs, si.State.Msgs) }) } func TestNoRaceJetStreamClusterBadRestartsWithHealthzPolling(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.>"}, Replicas: 3, } _, err := js.AddStream(cfg) require_NoError(t, err) // We will poll healthz at a decent clip and make sure any restart logic works // correctly with assets coming and going. ch := make(chan struct{}) defer close(ch) go func() { for { select { case <-ch: return case <-time.After(50 * time.Millisecond): for _, s := range c.servers { s.healthz(nil) } } } }() numConsumers := 500 consumers := make([]string, 0, numConsumers) var wg sync.WaitGroup for i := 0; i < numConsumers; i++ { cname := fmt.Sprintf("CONS-%d", i+1) consumers = append(consumers, cname) wg.Add(1) go func() { defer wg.Done() _, err := js.PullSubscribe("foo.>", cname, nats.BindStream("TEST")) require_NoError(t, err) }() } wg.Wait() // Make sure all are reported. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for _, s := range c.servers { jsz, _ := s.Jsz(nil) if jsz.Consumers != numConsumers { return fmt.Errorf("%v wrong number of consumers: %d vs %d", s, jsz.Consumers, numConsumers) } } return nil }) // Now do same for streams. numStreams := 200 streams := make([]string, 0, numStreams) for i := 0; i < numStreams; i++ { sname := fmt.Sprintf("TEST-%d", i+1) streams = append(streams, sname) wg.Add(1) go func() { defer wg.Done() _, err := js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3}) require_NoError(t, err) }() } wg.Wait() // Make sure all are reported. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for _, s := range c.servers { jsz, _ := s.Jsz(nil) if jsz.Streams != numStreams+1 { return fmt.Errorf("%v wrong number of streams: %d vs %d", s, jsz.Streams, numStreams+1) } } return nil }) // Delete consumers. for _, cname := range consumers { err := js.DeleteConsumer("TEST", cname) require_NoError(t, err) } // Make sure reporting goes to zero. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for _, s := range c.servers { jsz, _ := s.Jsz(nil) if jsz.Consumers != 0 { return fmt.Errorf("%v still has %d consumers", s, jsz.Consumers) } } return nil }) // Delete streams for _, sname := range streams { err := js.DeleteStream(sname) require_NoError(t, err) } err = js.DeleteStream("TEST") require_NoError(t, err) // Make sure reporting goes to zero. checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for _, s := range c.servers { jsz, _ := s.Jsz(nil) if jsz.Streams != 0 { return fmt.Errorf("%v still has %d streams", s, jsz.Streams) } } return nil }) } func TestNoRaceJetStreamKVReplaceWithServerRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomServer()) defer nc.Close() // Shorten wait time for disconnects. js, err := nc.JetStream(nats.MaxWait(time.Second)) require_NoError(t, err) kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ Bucket: "TEST", Replicas: 3, }) require_NoError(t, err) createData := func(n int) []byte { const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" b := make([]byte, n) for i := range b { b[i] = letterBytes[rand.Intn(len(letterBytes))] } return b } _, err = kv.Create("foo", createData(160)) require_NoError(t, err) ch := make(chan struct{}) wg := sync.WaitGroup{} // For counting errors that should not happen. errCh := make(chan error, 1024) wg.Add(1) go func() { defer wg.Done() var lastData []byte var revision uint64 for { select { case <-ch: return default: k, err := kv.Get("foo") if err == nats.ErrKeyNotFound { errCh <- err } else if k != nil { if lastData != nil && k.Revision() == revision && !bytes.Equal(lastData, k.Value()) { errCh <- fmt.Errorf("data loss [%s][rev:%d] expected:[%q] is:[%q]\n", "foo", revision, lastData, k.Value()) } newData := createData(160) if revision, err = kv.Update("foo", newData, k.Revision()); err == nil { lastData = newData } } } } }() // Wait a short bit. time.Sleep(2 * time.Second) for _, s := range c.servers { s.Shutdown() // Need to leave servers down for awhile to trigger bug properly. time.Sleep(5 * time.Second) s = c.restartServer(s) c.waitOnServerHealthz(s) } // Shutdown the go routine above. close(ch) // Wait for it to finish. wg.Wait() if len(errCh) != 0 { for err := range errCh { t.Logf("Received err %v during test", err) } t.Fatalf("Encountered errors") } } func TestNoRaceMemStoreCompactPerformance(t *testing.T) { //Load MemStore so that it is full subj, msg := "foo", make([]byte, 1000) storedMsgSize := memStoreMsgSize(subj, nil, msg) toStore := uint64(10_000) toStoreOnTop := uint64(1_000) setSeqNo := uint64(10_000_000_000) expectedPurge := toStore - 1 maxBytes := storedMsgSize * toStore ms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)}) require_NoError(t, err) defer ms.Stop() for i := uint64(0); i < toStore; i++ { ms.StoreMsg(subj, nil, msg) } state := ms.State() require_Equal(t, toStore, state.Msgs) require_Equal(t, state.Bytes, storedMsgSize*toStore) //1st run: Load additional messages then compact for i := uint64(0); i < toStoreOnTop; i++ { ms.StoreMsg(subj, nil, msg) } startFirstRun := time.Now() purgedFirstRun, _ := ms.Compact(toStore + toStoreOnTop) elapsedFirstRun := time.Since(startFirstRun) require_Equal(t, expectedPurge, purgedFirstRun) //set the seq number to a very high value by compacting with a too high seq number purgedFull, _ := ms.Compact(setSeqNo) require_Equal(t, 1, purgedFull) //2nd run: Compact again for i := uint64(0); i < toStore; i++ { ms.StoreMsg(subj, nil, msg) } startSecondRun := time.Now() purgedSecondRun, _ := ms.Compact(setSeqNo + toStore - 1) elapsedSecondRun := time.Since(startSecondRun) require_Equal(t, expectedPurge, purgedSecondRun) //Calculate delta between runs and fail if it is too high require_LessThan(t, elapsedSecondRun-elapsedFirstRun, time.Duration(1)*time.Second) } func TestNoRaceJetStreamSnapshotsWithSlowAckDontSlowConsumer(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() ech := make(chan error) ecb := func(_ *nats.Conn, _ *nats.Subscription, err error) { if err != nil { ech <- err } } nc, js := jsClientConnect(t, s, nats.ErrorHandler(ecb)) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, }) require_NoError(t, err) // Put in over 64MB. msg, toSend := make([]byte, 1024*1024), 80 crand.Read(msg) for i := 0; i < toSend; i++ { _, err := js.Publish("foo", msg) require_NoError(t, err) } sreq := &JSApiStreamSnapshotRequest{ DeliverSubject: nats.NewInbox(), ChunkSize: 1024 * 1024, } req, _ := json.Marshal(sreq) rmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, "TEST"), req, time.Second) require_NoError(t, err) var resp JSApiStreamSnapshotResponse json.Unmarshal(rmsg.Data, &resp) require_True(t, resp.Error == nil) done := make(chan *nats.Msg) sub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) { // EOF if len(m.Data) == 0 { done <- m return } }) defer sub.Unsubscribe() // Check that we do not get disconnected due to slow consumer. select { case msg := <-done: require_Equal(t, msg.Header.Get("Status"), "408") require_Equal(t, msg.Header.Get("Description"), "No Flow Response") case <-ech: t.Fatalf("Got disconnected: %v", err) case <-time.After(5 * time.Second): t.Fatalf("Should have received EOF with error status") } } func TestNoRaceJetStreamWQSkippedMsgsOnScaleUp(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() const pre = "CORE_ENT_DR_OTP_22." wcSubj := pre + ">" _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{wcSubj}, Retention: nats.WorkQueuePolicy, AllowDirect: true, Replicas: 3, }) require_NoError(t, err) cfg := &nats.ConsumerConfig{ Durable: "dlc", FilterSubject: wcSubj, DeliverPolicy: nats.DeliverAllPolicy, AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 10_000, AckWait: 500 * time.Millisecond, MaxWaiting: 100, MaxRequestExpires: 1050 * time.Millisecond, } _, err = js.AddConsumer("TEST", cfg) require_NoError(t, err) pdone := make(chan bool) cdone := make(chan bool) // We will have 51 consumer apps and a producer app. Make sure to wait for // all go routines to end at the end of the test. wg := sync.WaitGroup{} wg.Add(52) // Publish routine go func() { defer wg.Done() publishSubjects := []string{ "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682066", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682067", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543211", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543212", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543213", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543214", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543215", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543216", "CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543217", } // ~1.7kb msg := bytes.Repeat([]byte("Z"), 1750) // 200 msgs/s st := time.NewTicker(5 * time.Millisecond) defer st.Stop() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for { select { case <-st.C: subj := publishSubjects[rand.Intn(len(publishSubjects))] _, err = js.Publish(subj, msg) require_NoError(t, err) case <-pdone: return } } }() consumerApp := func() { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.ConsumerInfo("TEST", "dlc") require_NoError(t, err) _, err = js.UpdateConsumer("TEST", cfg) require_NoError(t, err) sub, err := js.PullSubscribe(wcSubj, "dlc") require_NoError(t, err) st := time.NewTicker(100 * time.Millisecond) defer st.Stop() for { select { case <-st.C: msgs, err := sub.Fetch(1, nats.MaxWait(100*time.Millisecond)) if err != nil { continue } require_Equal(t, len(msgs), 1) m := msgs[0] if rand.Intn(10) == 1 { m.Nak() } else { // Wait up to 20ms to ack. time.Sleep(time.Duration(rand.Intn(20)) * time.Millisecond) // This could fail and that is ok, system should recover due to low ack wait. m.Ack() } case <-cdone: return } } } // Now consumer side single. go consumerApp() // Wait for 2s time.Sleep(2 * time.Second) // Now spin up 50 more. for i := 1; i <= 50; i++ { if i%5 == 0 { time.Sleep(200 * time.Millisecond) } go consumerApp() } timeout := time.Now().Add(8 * time.Second) for time.Now().Before(timeout) { time.Sleep(750 * time.Millisecond) if s := c.consumerLeader(globalAccountName, "TEST", "dlc"); s != nil { s.JetStreamStepdownConsumer(globalAccountName, "TEST", "dlc") } } // Close publishers and defer closing consumers. close(pdone) defer func() { close(cdone) wg.Wait() }() checkFor(t, 30*time.Second, 50*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.NumDeleted > 0 || si.State.Msgs > 0 { return fmt.Errorf("State not correct: %+v", si.State) } return nil }) } func TestNoRaceConnectionObjectReleased(t *testing.T) { ob1Conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" server_name: "B1" accounts { A { users: [{user: a, password: pwd}] } SYS { users: [{user: sys, password: pwd}] } } cluster { name: "B" listen: "127.0.0.1:-1" } gateway { name: "B" listen: "127.0.0.1:-1" } leaf { listen: "127.0.0.1:-1" } system_account: "SYS" `)) sb1, ob1 := RunServerWithConfig(ob1Conf) defer sb1.Shutdown() oaConf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "A" accounts { A { users: [{user: a, password: pwd}] } SYS { users: [{user: sys, password: pwd}] } } gateway { name: "A" listen: "127.0.0.1:-1" gateways [ { name: "B" url: "nats://a:pwd@127.0.0.1:%d" } ] } websocket { listen: "127.0.0.1:-1" no_tls: true } system_account: "SYS" `, ob1.Gateway.Port))) sa, oa := RunServerWithConfig(oaConf) defer sa.Shutdown() waitForOutboundGateways(t, sa, 1, 2*time.Second) waitForOutboundGateways(t, sb1, 1, 2*time.Second) ob2Conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "B2" accounts { A { users: [{user: a, password: pwd}] } SYS { users: [{user: sys, password: pwd}] } } cluster { name: "B" listen: "127.0.0.1:-1" routes: ["nats://127.0.0.1:%d"] } gateway { name: "B" listen: "127.0.0.1:-1" } system_account: "SYS" `, ob1.Cluster.Port))) sb2, _ := RunServerWithConfig(ob2Conf) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) waitForOutboundGateways(t, sb2, 1, 2*time.Second) waitForInboundGateways(t, sa, 2, 2*time.Second) leafConf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" server_name: "C" accounts { A { users: [{user: a, password: pwd}] } SYS { users: [{user: sys, password: pwd}] } } leafnodes { remotes [ { url: "nats://a:pwd@127.0.0.1:%d" } ] } system_account: "SYS" `, ob1.LeafNode.Port))) leaf, _ := RunServerWithConfig(leafConf) defer leaf.Shutdown() checkLeafNodeConnected(t, leaf) // Start an independent MQTT server to check MQTT client connection. mo := testMQTTDefaultOptions() mo.ServerName = "MQTTServer" sm := testMQTTRunServer(t, mo) defer testMQTTShutdownServer(sm) mc, mr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, mo.MQTT.Host, mo.MQTT.Port) defer mc.Close() testMQTTCheckConnAck(t, mr, mqttConnAckRCConnectionAccepted, false) nc := natsConnect(t, sb1.ClientURL(), nats.UserInfo("a", "pwd")) defer nc.Close() cid, err := nc.GetClientID() require_NoError(t, err) natsSubSync(t, nc, "foo") natsFlush(t, nc) ncWS := natsConnect(t, fmt.Sprintf("ws://a:pwd@127.0.0.1:%d", oa.Websocket.Port)) defer ncWS.Close() cidWS, err := ncWS.GetClientID() require_NoError(t, err) var conns []net.Conn var total int var ch chan string track := func(c *client) { total++ c.mu.Lock() conns = append(conns, c.nc) c.mu.Unlock() runtime.SetFinalizer(c, func(c *client) { ch <- fmt.Sprintf("Server=%s - Kind=%s - Conn=%v", c.srv, c.kindString(), c) }) } // Track the connection for the MQTT client sm.mu.RLock() for _, c := range sm.clients { track(c) } sm.mu.RUnlock() // Track the connection from the NATS client track(sb1.getClient(cid)) // The outbound connection to GW "A" track(sb1.getOutboundGatewayConnection("A")) // The inbound connection from GW "A" var inGW []*client sb1.getInboundGatewayConnections(&inGW) track(inGW[0]) // The routes from sb2 sb1.forEachRoute(func(r *client) { track(r) }) // The leaf form "LEAF" sb1.mu.RLock() for _, l := range sb1.leafs { track(l) } sb1.mu.RUnlock() // Now from sb2, the routes to sb1 sb2.forEachRoute(func(r *client) { track(r) }) // The outbound connection to GW "A" track(sb2.getOutboundGatewayConnection("A")) // From server "A", track the outbound GW track(sa.getOutboundGatewayConnection("B")) inGW = inGW[:0] // Track the inbound GW connections sa.getInboundGatewayConnections(&inGW) for _, ig := range inGW { track(ig) } // Track the websocket client track(sa.getClient(cidWS)) // From the LEAF server, the connection to sb1 leaf.mu.RLock() for _, l := range leaf.leafs { track(l) } leaf.mu.RUnlock() // Now close all connections and wait to see if all connections // with the finalizer set is invoked. ch = make(chan string, total) // Close the clients and then all other connections to create a disconnect. nc.Close() mc.Close() ncWS.Close() for _, conn := range conns { conn.Close() } // Wait and see if we get them all. tm := time.NewTimer(10 * time.Second) defer tm.Stop() tk := time.NewTicker(10 * time.Millisecond) for clients := make([]string, 0, total); len(clients) < total; { select { case <-tk.C: runtime.GC() case cs := <-ch: clients = append(clients, cs) case <-tm.C: // Don't fail the test since there is no guarantee that // finalizers are invoked. t.Logf("Got %v out of %v finalizers", len(clients), total) slices.Sort(clients) for _, cs := range clients { t.Logf(" => %s", cs) } return } } } func TestNoRaceFileStoreMsgLoadNextMsgMultiPerf(t *testing.T) { fs, err := newFileStore( FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}) require_NoError(t, err) defer fs.Stop() // Put 1k msgs in for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) fs.StoreMsg(subj, nil, []byte("ZZZ")) } var smv StoreMsg // Now do normal load next with no filter. // This is baseline. start := time.Now() for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsg(_EMPTY_, false, seq, &smv) require_NoError(t, err) require_True(t, sm.subj == fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } baseline := time.Since(start) t.Logf("Single - No filter %v", baseline) // Now do normal load next with wc filter. start = time.Now() for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsg("foo.>", true, seq, &smv) require_NoError(t, err) require_True(t, sm.subj == fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } elapsed := time.Since(start) require_True(t, elapsed < 2*baseline) t.Logf("Single - WC filter %v", elapsed) // Now do multi load next with 1 wc entry. sl := NewSublistWithCache() sl.Insert(&subscription{subject: []byte("foo.>")}) start = time.Now() for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) require_NoError(t, err) require_True(t, sm.subj == fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } elapsed = time.Since(start) require_True(t, elapsed < 2*baseline) t.Logf("Multi - Single WC filter %v", elapsed) // Now do multi load next with 1000 literal subjects. sl = NewSublistWithCache() for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) sl.Insert(&subscription{subject: []byte(subj)}) } start = time.Now() for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) require_NoError(t, err) require_True(t, sm.subj == fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } elapsed = time.Since(start) require_True(t, elapsed < 2*baseline) t.Logf("Multi - 1000 filters %v", elapsed) } func TestNoRaceWQAndMultiSubjectFilters(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"Z.>"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) stopPubs := make(chan bool) publish := func(subject string) { nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() for { select { case <-stopPubs: return default: _, _ = js.Publish(subject, []byte("hello")) } } } go publish("Z.foo") go publish("Z.bar") go publish("Z.baz") // Cancel pubs after 10s. time.AfterFunc(10*time.Second, func() { close(stopPubs) }) // Create a consumer _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "zzz", AckPolicy: nats.AckExplicitPolicy, AckWait: 5 * time.Second, FilterSubjects: []string{"Z.foo", "Z.bar", "Z.baz"}, }) require_NoError(t, err) sub, err := js.PullSubscribe(_EMPTY_, "zzz", nats.Bind("TEST", "zzz")) require_NoError(t, err) received := make([]uint64, 0, 256_000) batchSize := 10 for running := true; running; { msgs, err := sub.Fetch(batchSize, nats.MaxWait(2*time.Second)) if err == nats.ErrTimeout { running = false } for _, m := range msgs { meta, err := m.Metadata() require_NoError(t, err) received = append(received, meta.Sequence.Stream) m.Ack() } } slices.Sort(received) var pseq, gaps uint64 for _, seq := range received { if pseq != 0 && pseq != seq-1 { gaps += seq - pseq + 1 } pseq = seq } si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Msgs != 0 || gaps > 0 { t.Fatalf("Orphaned msgs %d with %d gaps detected", si.State.Msgs, gaps) } } // https://github.com/nats-io/nats-server/issues/4957 func TestNoRaceWQAndMultiSubjectFiltersRace(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"Z.>"}, Retention: nats.WorkQueuePolicy, Replicas: 1, }) require_NoError(t, err) // The bug would happen when the stream was on same server as meta-leader. // So make that so. // Make sure stream leader is on S-1 sl := c.streamLeader(globalAccountName, "TEST") checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if sl == c.leader() { return nil } // Move meta-leader since stream can be R1. snc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo("admin", "s3cr3t!")) defer snc.Close() if _, err := snc.Request(JSApiLeaderStepDown, nil, time.Second); err != nil { return err } return fmt.Errorf("stream leader on meta-leader") }) start := make(chan struct{}) var done, ready sync.WaitGroup // Create num go routines who will all race to create a consumer with the same filter subject but a different name. num := 10 ready.Add(num) done.Add(num) for i := 0; i < num; i++ { go func(n int) { // Connect directly to the meta leader but with our own connection. s := c.leader() nc, js := jsClientConnect(t, s) defer nc.Close() ready.Done() defer done.Done() <-start js.AddConsumer("TEST", &nats.ConsumerConfig{ Name: fmt.Sprintf("C-%d", n), FilterSubject: "Z.foo", AckPolicy: nats.AckExplicitPolicy, }) }(i) } // Wait for requestors to be ready ready.Wait() close(start) done.Wait() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { si, err := js.StreamInfo("TEST") require_NoError(t, err) if si.State.Consumers != 1 { return fmt.Errorf("Consumer count not correct: %d vs 1", si.State.Consumers) } return nil }) } func TestNoRaceFileStoreWriteFullStateUniqueSubjects(t *testing.T) { fcfg := FileStoreConfig{StoreDir: t.TempDir()} fs, err := newFileStore(fcfg, StreamConfig{Name: "zzz", Subjects: []string{"records.>"}, Storage: FileStorage, MaxMsgsPer: 1, MaxBytes: 15 * 1024 * 1024 * 1024}) require_NoError(t, err) defer fs.Stop() qch := make(chan struct{}) defer close(qch) go func() { const numThreshold = 1_000_000 tick := time.NewTicker(1 * time.Second) for { select { case <-qch: return case <-tick.C: err := fs.writeFullState() var state StreamState fs.FastState(&state) if state.Msgs > numThreshold && err != nil { require_Error(t, err, errStateTooBig) } } } }() labels := []string{"AAAAA", "BBBB", "CCCC", "DD", "EEEEE"} msg := []byte(strings.Repeat("Z", 128)) for i := 0; i < 100; i++ { partA := nuid.Next() for j := 0; j < 100; j++ { partB := nuid.Next() for k := 0; k < 500; k++ { partC := nuid.Next() partD := labels[rand.Intn(len(labels)-1)] subject := fmt.Sprintf("records.%s.%s.%s.%s.%s", partA, partB, partC, partD, nuid.Next()) start := time.Now() fs.StoreMsg(subject, nil, msg) elapsed := time.Since(start) if elapsed > 500*time.Millisecond { t.Fatalf("Slow store for %q: %v\n", subject, elapsed) } } } } // Make sure we do write the full state on stop. fs.Stop() fi, err := os.Stat(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)) require_NoError(t, err) // ~500MB, could change if we tweak encodings.. require_True(t, fi.Size() > 500*1024*1024) } // When a catchup takes a long time and the ingest rate is high enough to cause us // to drop append entries and move our first past out catchup window due to max bytes or max msgs, etc. func TestNoRaceLargeStreamCatchups(t *testing.T) { // This usually takes too long on Travis. t.Skip() var jsMaxOutTempl = ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_file_store: 22GB, store_dir: '%s', max_outstanding_catchup: 128KB} cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := createJetStreamClusterWithTemplate(t, jsMaxOutTempl, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomServer()) defer nc.Close() js, err := nc.JetStream(nats.PublishAsyncMaxPending(64 * 1024)) require_NoError(t, err) cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"Z.>"}, MaxBytes: 4 * 1024 * 1024 * 1024, Replicas: 1, // Start at R1 } _, err = js.AddStream(cfg) require_NoError(t, err) // Load up to a decent size first. num, msg := 25_000, bytes.Repeat([]byte("Z"), 256*1024) for i := 0; i < num; i++ { _, err := js.PublishAsync("Z.Z.Z", msg) require_NoError(t, err) } select { case <-js.PublishAsyncComplete(): case <-time.After(20 * time.Second): t.Fatalf("Did not receive completion signal") } var sl *Server time.AfterFunc(time.Second, func() { cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") sl = c.streamLeader(globalAccountName, "TEST") }) // Run for 60 seconds sending new messages at a high rate. timeout := time.Now().Add(60 * time.Second) for time.Now().Before(timeout) { for i := 0; i < 5_000; i++ { // Not worried about each message, so not checking err here. // Just generating high load of new traffic while trying to catch up. js.PublishAsync("Z.Z.Z", msg) } // This will gate us waiting on a response. js.Publish("Z.Z.Z", msg) } // Make sure the leader has not changed. require_Equal(t, sl, c.streamLeader(globalAccountName, "TEST")) // Grab the leader and its state. mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) expected := mset.state() checkFor(t, 45*time.Second, time.Second, func() error { for _, s := range c.servers { if s == sl { continue } mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) state := mset.state() if !reflect.DeepEqual(expected, state) { return fmt.Errorf("Follower %v state does not match: %+v vs %+v", s, state, expected) } } return nil }) } func TestNoRaceLargeNumDeletesStreamCatchups(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.randomServer()) defer nc.Close() js, err := nc.JetStream(nats.PublishAsyncMaxPending(16 * 1024)) require_NoError(t, err) cfg := &nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo"}, Replicas: 1, // Start at R1 } _, err = js.AddStream(cfg) require_NoError(t, err) // We will manipulate the stream at the lower level to achieve large number of interior deletes. // We will store only 2 msgs, but have 100M deletes in between. sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) mset.mu.Lock() mset.store.StoreMsg("foo", nil, []byte("ok")) mset.store.SkipMsgs(2, 1_000_000_000) mset.store.StoreMsg("foo", nil, []byte("ok")) mset.store.SkipMsgs(1_000_000_003, 1_000_000_000) var state StreamState mset.store.FastState(&state) mset.lseq = state.LastSeq mset.mu.Unlock() cfg.Replicas = 3 _, err = js.UpdateStream(cfg) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") mset, err = sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) expected := mset.state() // This should happen fast and not spin on all interior deletes. checkFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error { for _, s := range c.servers { if s == sl { continue } mset, err := s.GlobalAccount().lookupStream("TEST") require_NoError(t, err) state := mset.state() // Ignore LastTime for this test since we send delete range at end. state.LastTime = expected.LastTime if !reflect.DeepEqual(expected, state) { return fmt.Errorf("Follower %v state does not match: %+v vs %+v", s, state, expected) } } return nil }) } func TestNoRaceJetStreamClusterMemoryStreamLastSequenceResetAfterRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() numStreams := 250 var wg sync.WaitGroup wg.Add(numStreams) for i := 1; i <= numStreams; i++ { go func(n int) { defer wg.Done() _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST:%d", n), Storage: nats.MemoryStorage, Subjects: []string{fmt.Sprintf("foo.%d.*", n)}, Replicas: 3, }, nats.MaxWait(30*time.Second)) require_NoError(t, err) subj := fmt.Sprintf("foo.%d.bar", n) for i := 0; i < 222; i++ { js.Publish(subj, nil) } }(i) } wg.Wait() // Make sure all streams have a snapshot in place to stress the snapshot logic for memory based streams. for _, s := range c.servers { for i := 1; i <= numStreams; i++ { stream := fmt.Sprintf("TEST:%d", i) mset, err := s.GlobalAccount().lookupStream(stream) require_NoError(t, err) node := mset.raftNode() require_NotNil(t, node) node.InstallSnapshot(mset.stateSnapshot()) } } // Do 5 rolling restarts waiting on healthz in between. for i := 0; i < 5; i++ { // Walk the servers and shut each down, and wipe the storage directory. for _, s := range c.servers { s.Shutdown() s.WaitForShutdown() s = c.restartServer(s) checkFor(t, 30*time.Second, time.Second, func() error { hs := s.healthz(nil) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) // Make sure all streams are current after healthz returns ok. for i := 1; i <= numStreams; i++ { stream := fmt.Sprintf("TEST:%d", i) mset, err := s.GlobalAccount().lookupStream(stream) require_NoError(t, err) var state StreamState checkFor(t, 30*time.Second, time.Second, func() error { mset.store.FastState(&state) if state.LastSeq != 222 { return fmt.Errorf("%v Wrong last sequence %d for %q - State %+v", s, state.LastSeq, stream, state) } return nil }) } } } } func TestNoRaceJetStreamClusterMemoryWorkQueueLastSequenceResetAfterRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() numStreams := 50 var wg sync.WaitGroup wg.Add(numStreams) for i := 1; i <= numStreams; i++ { go func(n int) { defer wg.Done() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() checkFor(t, 5*time.Second, time.Second, func() error { _, err := js.AddStream(&nats.StreamConfig{ Name: fmt.Sprintf("TEST:%d", n), Storage: nats.MemoryStorage, Retention: nats.WorkQueuePolicy, Subjects: []string{fmt.Sprintf("foo.%d.*", n)}, Replicas: 3, }, nats.MaxWait(time.Second)) return err }) subj := fmt.Sprintf("foo.%d.bar", n) for i := 0; i < 22; i++ { checkFor(t, 5*time.Second, time.Second, func() error { _, err := js.Publish(subj, nil) return err }) } // Now consume them all as well. var err error var sub *nats.Subscription checkFor(t, 5*time.Second, time.Second, func() error { sub, err = js.PullSubscribe(subj, "wq") return err }) var msgs []*nats.Msg checkFor(t, 5*time.Second, time.Second, func() error { msgs, err = sub.Fetch(22, nats.MaxWait(time.Second)) return err }) require_Equal(t, len(msgs), 22) for _, m := range msgs { checkFor(t, 5*time.Second, time.Second, func() error { return m.AckSync() }) } }(i) } wg.Wait() // Do 2 rolling restarts waiting on healthz in between. for i := 0; i < 2; i++ { // Walk the servers and shut each down, and wipe the storage directory. for _, s := range c.servers { s.Shutdown() s.WaitForShutdown() s = c.restartServer(s) checkFor(t, 30*time.Second, time.Second, func() error { hs := s.healthz(nil) if hs.Error != _EMPTY_ { return errors.New(hs.Error) } return nil }) // Make sure all streams are current after healthz returns ok. for i := 1; i <= numStreams; i++ { stream := fmt.Sprintf("TEST:%d", i) mset, err := s.GlobalAccount().lookupStream(stream) require_NoError(t, err) var state StreamState checkFor(t, 20*time.Second, time.Second, func() error { mset.store.FastState(&state) if state.LastSeq != 22 { return fmt.Errorf("%v Wrong last sequence %d for %q - State %+v", s, state.LastSeq, stream, state) } if state.FirstSeq != 23 { return fmt.Errorf("%v Wrong first sequence %d for %q - State %+v", s, state.FirstSeq, stream, state) } return nil }) } } } } func TestNoRaceJetStreamClusterMirrorSkipSequencingBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R5S", 5) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "ORIGIN", Subjects: []string{"foo.*"}, MaxMsgsPerSubject: 1, Replicas: 1, Storage: nats.MemoryStorage, }) require_NoError(t, err) // Create a mirror with R5 such that it will be much slower than the ORIGIN. _, err = js.AddStream(&nats.StreamConfig{ Name: "M", Replicas: 5, Mirror: &nats.StreamSource{Name: "ORIGIN"}, }) require_NoError(t, err) // Connect new directly to ORIGIN s := c.streamLeader(globalAccountName, "ORIGIN") nc, js = jsClientConnect(t, s) defer nc.Close() // We are going to send at a high rate and also delete some along the way // via the max msgs per limit. for i := 0; i < 500_000; i++ { subj := fmt.Sprintf("foo.%d", i) js.PublishAsync(subj, nil) // Create sequence holes every 100k. if i%100_000 == 0 { js.PublishAsync(subj, nil) } } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } checkFor(t, 20*time.Second, time.Second, func() error { si, err := js.StreamInfo("M") require_NoError(t, err) if si.State.Msgs != 500_000 { return fmt.Errorf("Expected 1M msgs, got state: %+v", si.State) } return nil }) } func TestNoRaceJetStreamStandaloneDontReplyToAckBeforeProcessingIt(t *testing.T) { s := RunBasicJetStreamServer(t) defer s.Shutdown() // Client for API requests. nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "WQ", Discard: nats.DiscardNew, MaxMsgsPerSubject: 1, DiscardNewPerSubject: true, Retention: nats.WorkQueuePolicy, Subjects: []string{"queue.>"}, }) require_NoError(t, err) // Keep this low since we are going to run as many go routines // to consume, ack and republish the message. total := 10000 // Populate the queue, one message per subject. for i := 0; i < total; i++ { js.Publish(fmt.Sprintf("queue.%d", i), []byte("hello")) } _, err = js.AddConsumer("WQ", &nats.ConsumerConfig{ Durable: "cons", AckPolicy: nats.AckExplicitPolicy, MaxWaiting: 20000, MaxAckPending: -1, }) require_NoError(t, err) sub, err := js.PullSubscribe("queue.>", "cons", nats.BindStream("WQ")) require_NoError(t, err) errCh := make(chan error, total) var wg sync.WaitGroup for iter := 0; iter < 3; iter++ { wg.Add(total) for i := 0; i < total; i++ { go func() { defer wg.Done() msgs, err := sub.Fetch(1) if err != nil { errCh <- err return } msg := msgs[0] err = msg.AckSync() if err != nil { errCh <- err return } _, err = js.Publish(msg.Subject, []byte("hello")) if err != nil { errCh <- err return } }() } wg.Wait() select { case err := <-errCh: t.Fatalf("Test failed, first error was: %v", err) default: // OK! } } } // Under certain scenarios an old index.db with a stream that has msg limits set will not restore properly // due to an old index.db and compaction after the index.db took place which could lose per subject information. func TestNoRaceFileStoreMsgLimitsAndOldRecoverState(t *testing.T) { for _, test := range []struct { name string expectedFirstSeq uint64 expectedLastSeq uint64 expectedMsgs uint64 transform func(StreamConfig) StreamConfig }{ { name: "MaxMsgsPer", expectedFirstSeq: 10_001, expectedLastSeq: 1_010_001, expectedMsgs: 1_000_001, transform: func(config StreamConfig) StreamConfig { config.MaxMsgsPer = 1 return config }, }, { name: "MaxMsgs", expectedFirstSeq: 10_001, expectedLastSeq: 1_010_001, expectedMsgs: 1_000_001, transform: func(config StreamConfig) StreamConfig { config.MaxMsgs = 1_000_001 return config }, }, { name: "MaxBytes", expectedFirstSeq: 8_624, expectedLastSeq: 1_010_001, expectedMsgs: 1_001_378, transform: func(config StreamConfig) StreamConfig { config.MaxBytes = 1_065_353_216 return config }, }, } { t.Run(test.name, func(t *testing.T) { sd := t.TempDir() fs, err := newFileStore( FileStoreConfig{StoreDir: sd}, test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), ) require_NoError(t, err) defer fs.Stop() msg := make([]byte, 1024) for i := 0; i < 10_000; i++ { subj := fmt.Sprintf("foo.%d", i) fs.StoreMsg(subj, nil, msg) } // This will write the index.db file. We will capture this and use it to replace a new one. sfile := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile) fs.Stop() _, err = os.Stat(sfile) require_NoError(t, err) // Read it in and make sure len > 0. buf, err := os.ReadFile(sfile) require_NoError(t, err) require_True(t, len(buf) > 0) // Restart fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), ) require_NoError(t, err) defer fs.Stop() // Put in more messages with wider range. This will compact a bunch of the previous blocks. for i := 0; i < 1_000_001; i++ { subj := fmt.Sprintf("foo.%d", i) fs.StoreMsg(subj, nil, msg) } var ss StreamState fs.FastState(&ss) require_Equal(t, ss.FirstSeq, test.expectedFirstSeq) require_Equal(t, ss.LastSeq, test.expectedLastSeq) require_Equal(t, ss.Msgs, test.expectedMsgs) // Now stop again, but replace index.db with old one. fs.Stop() // Put back old stream state. require_NoError(t, os.WriteFile(sfile, buf, defaultFilePerms)) // Restart fs, err = newFileStore( FileStoreConfig{StoreDir: sd}, test.transform(StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage}), ) require_NoError(t, err) defer fs.Stop() fs.FastState(&ss) require_Equal(t, ss.FirstSeq, test.expectedFirstSeq) require_Equal(t, ss.LastSeq, test.expectedLastSeq) require_Equal(t, ss.Msgs, test.expectedMsgs) }) } } func TestNoRaceJetStreamClusterCheckInterestStatePerformanceWQ(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.WorkQueuePolicy, }) require_NoError(t, err) // Load up a bunch of messages for three different subjects. msg := bytes.Repeat([]byte("Z"), 4096) for i := 0; i < 100_000; i++ { js.PublishAsync("foo.foo", msg) } for i := 0; i < 5_000; i++ { js.PublishAsync("foo.bar", msg) js.PublishAsync("foo.baz", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // We will not process this one and leave it as "offline". _, err = js.PullSubscribe("foo.foo", "A") require_NoError(t, err) subB, err := js.PullSubscribe("foo.bar", "B") require_NoError(t, err) subC, err := js.PullSubscribe("foo.baz", "C") require_NoError(t, err) // Now catch up both B and C but let A simulate being offline of very behind. for i := 0; i < 5; i++ { for _, sub := range []*nats.Subscription{subB, subC} { msgs, err := sub.Fetch(1000) require_NoError(t, err) require_Equal(t, len(msgs), 1000) for _, m := range msgs { m.Ack() } } } // Let acks process. nc.Flush() time.Sleep(200 * time.Millisecond) // Now test the check checkInterestState() on the stream. sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) expireAllBlks := func() { mset.mu.RLock() fs := mset.store.(*fileStore) mset.mu.RUnlock() fs.mu.RLock() for _, mb := range fs.blks { mb.tryForceExpireCache() } fs.mu.RUnlock() } // First expire all the blocks. expireAllBlks() start := time.Now() mset.checkInterestState() elapsed := time.Since(start) // This is actually ~300 microseconds but due to travis and race flags etc. // Was > 30 ms before fix for comparison, M2 macbook air. require_LessThan(t, elapsed, 5*time.Millisecond) // Make sure we set the chkflr correctly. The chkflr should be equal to asflr+1. // Otherwise, if chkflr would be set higher a subsequent call to checkInterestState will be ineffective. requireFloorsEqual := func(o *consumer) { t.Helper() require_True(t, o != nil) o.mu.RLock() defer o.mu.RUnlock() require_Equal(t, o.chkflr, o.asflr+1) } requireFloorsEqual(mset.lookupConsumer("A")) requireFloorsEqual(mset.lookupConsumer("B")) requireFloorsEqual(mset.lookupConsumer("C")) // Expire all the blocks again. expireAllBlks() // This checks the chkflr state. start = time.Now() mset.checkInterestState() elapsed = time.Since(start) require_LessThan(t, elapsed, 5*time.Millisecond) } func TestNoRaceJetStreamClusterCheckInterestStatePerformanceInterest(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Subjects: []string{"foo.*"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) // We will not process this one and leave it as "offline". _, err = js.PullSubscribe("foo.foo", "A") require_NoError(t, err) _, err = js.PullSubscribe("foo.*", "B") require_NoError(t, err) // Make subC multi-subject. _, err = js.AddConsumer("TEST", &nats.ConsumerConfig{ Durable: "C", FilterSubjects: []string{"foo.foo", "foo.bar", "foo.baz"}, AckPolicy: nats.AckExplicitPolicy, }) require_NoError(t, err) // Load up a bunch of messages for three different subjects. msg := bytes.Repeat([]byte("Z"), 4096) for i := 0; i < 90_000; i++ { js.PublishAsync("foo.foo", msg) } for i := 0; i < 5_000; i++ { js.PublishAsync("foo.bar", msg) js.PublishAsync("foo.baz", msg) } select { case <-js.PublishAsyncComplete(): case <-time.After(5 * time.Second): t.Fatalf("Did not receive completion signal") } // This is so we do not asynchronously update our consumer state after we set the state due to notifications // from new messages for the stream. time.Sleep(250 * time.Millisecond) // Now catch up both B and C but let A simulate being offline of very behind. // Will do this manually here to speed up tests. sl := c.streamLeader(globalAccountName, "TEST") mset, err := sl.GlobalAccount().lookupStream("TEST") require_NoError(t, err) for _, cname := range []string{"B", "C"} { o := mset.lookupConsumer(cname) o.mu.Lock() o.setStoreState(&ConsumerState{ Delivered: SequencePair{100_000, 100_000}, AckFloor: SequencePair{100_000, 100_000}, }) o.mu.Unlock() } // Now test the check checkInterestState() on the stream. start := time.Now() mset.checkInterestState() elapsed := time.Since(start) // Make sure we set the chkflr correctly. checkFloor := func(o *consumer) uint64 { require_True(t, o != nil) o.mu.RLock() defer o.mu.RUnlock() return o.chkflr } require_Equal(t, checkFloor(mset.lookupConsumer("A")), 1) require_Equal(t, checkFloor(mset.lookupConsumer("B")), 90_001) require_Equal(t, checkFloor(mset.lookupConsumer("C")), 100_001) // This checks the chkflr state. For this test this should be much faster, // two orders of magnitude then the first time. start = time.Now() mset.checkInterestState() require_True(t, time.Since(start) < elapsed/100) } func TestNoRaceJetStreamClusterLargeMetaSnapshotTiming(t *testing.T) { // This test was to show improvements in speed for marshaling the meta layer with lots of assets. // Move to S2.Encode vs EncodeBetter which is 2x faster and actually better compression. // Also moved to goccy json which is faster then the default and in my tests now always matches // the default encoder byte for byte which last time I checked it did not. t.Skip() c := createJetStreamClusterExplicit(t, "R3F", 3) defer c.shutdown() // Create 200 streams, each with 500 consumers. numStreams := 200 numConsumers := 500 wg := sync.WaitGroup{} wg.Add(numStreams) for i := 0; i < numStreams; i++ { go func() { defer wg.Done() s := c.randomServer() nc, js := jsClientConnect(t, s) defer nc.Close() sname := fmt.Sprintf("TEST-SNAPSHOT-%d", i) subj := fmt.Sprintf("foo.%d", i) _, err := js.AddStream(&nats.StreamConfig{ Name: sname, Subjects: []string{subj}, Replicas: 3, }) require_NoError(t, err) // Now consumers. for c := 0; c < numConsumers; c++ { _, err = js.AddConsumer(sname, &nats.ConsumerConfig{ Durable: fmt.Sprintf("C-%d", c), FilterSubject: subj, AckPolicy: nats.AckExplicitPolicy, Replicas: 1, }) require_NoError(t, err) } }() } wg.Wait() s := c.leader() js := s.getJetStream() n := js.getMetaGroup() // Now let's see how long it takes to create a meta snapshot and how big it is. start := time.Now() snap, err := js.metaSnapshot() require_NoError(t, err) require_NoError(t, n.InstallSnapshot(snap)) t.Logf("Took %v to snap meta with size of %v\n", time.Since(start), friendlyBytes(len(snap))) } func TestNoRaceStoreReverseWalkWithDeletesPerf(t *testing.T) { cfg := StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}, Storage: FileStorage} fs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg) require_NoError(t, err) defer fs.Stop() cfg.Storage = MemoryStorage ms, err := newMemStore(&cfg) require_NoError(t, err) defer ms.Stop() msg := []byte("Hello") for _, store := range []StreamStore{fs, ms} { store.StoreMsg("foo.A", nil, msg) for i := 0; i < 1_000_000; i++ { store.StoreMsg("foo.B", nil, msg) } store.StoreMsg("foo.C", nil, msg) var ss StreamState store.FastState(&ss) require_Equal(t, ss.Msgs, 1_000_002) // Create a bunch of interior deletes. p, err := store.PurgeEx("foo.B", 1, 0) require_NoError(t, err) require_Equal(t, p, 1_000_000) // Now simulate a walk backwards as we currently do when searching for starting sequence numbers in sourced streams. start := time.Now() var smv StoreMsg for seq := ss.LastSeq; seq > 0; seq-- { _, err := store.LoadMsg(seq, &smv) if err == errDeletedMsg || err == ErrStoreMsgNotFound { continue } require_NoError(t, err) } elapsed := time.Since(start) // Now use the optimized load prev. seq, seen := ss.LastSeq, 0 start = time.Now() for { sm, err := store.LoadPrevMsg(seq, &smv) if err == ErrStoreEOF { break } require_NoError(t, err) seq = sm.seq - 1 seen++ } elapsedNew := time.Since(start) require_Equal(t, seen, 2) switch store.(type) { case *memStore: require_True(t, elapsedNew < elapsed) case *fileStore: // Bigger gains for filestore, 10x require_True(t, elapsedNew*10 < elapsed) } } } type fastProdLogger struct { DummyLogger gotIt chan struct{} } func (l *fastProdLogger) Debugf(format string, args ...any) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "fast producer") { select { case l.gotIt <- struct{}{}: default: } } } type slowWriteConn struct { sync.RWMutex net.Conn delay bool } func (c *slowWriteConn) Write(b []byte) (int, error) { c.RLock() delay := c.delay c.RUnlock() if delay { time.Sleep(100 * time.Millisecond) } return c.Conn.Write(b) } func TestNoRaceNoFastProducerStall(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" no_fast_producer_stall: %s ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "true"))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() l := &fastProdLogger{gotIt: make(chan struct{}, 1)} s.SetLogger(l, true, false) ncSlow := natsConnect(t, s.ClientURL(), nats.ReconnectWait(10*time.Millisecond)) defer ncSlow.Close() natsSub(t, ncSlow, "foo", func(_ *nats.Msg) {}) natsFlush(t, ncSlow) cid, err := ncSlow.GetClientID() require_NoError(t, err) c := s.GetClient(cid) require_True(t, c != nil) c.mu.Lock() swc := &slowWriteConn{Conn: c.nc, delay: true} c.nc = swc c.mu.Unlock() defer func() { swc.Lock() swc.delay = false swc.Unlock() }() // The producer could still overwhelm the "fast" consumer. // So we will send more than what it will use as a target // to consider the test done. total := 2_000_000 target := total / 2 ch := make(chan struct{}, 1) var count int ncFast := natsConnect(t, s.ClientURL(), nats.ReconnectWait(10*time.Millisecond)) defer ncFast.Close() fastSub := natsSub(t, ncFast, "foo", func(_ *nats.Msg) { if count++; count == target { ch <- struct{}{} } }) err = fastSub.SetPendingLimits(-1, -1) require_NoError(t, err) natsFlush(t, ncFast) ncProd := natsConnect(t, s.ClientURL(), nats.ReconnectWait(10*time.Millisecond)) defer ncProd.Close() cid, err = ncProd.GetClientID() require_NoError(t, err) pc := s.GetClient(cid) pc.mu.Lock() pcnc := pc.nc pc.mu.Unlock() payload := make([]byte, 256) for i := 0; i < total; i++ { natsPub(t, ncProd, "foo", payload) } select { case <-ch: // OK case <-time.After(10 * time.Second): t.Fatal("Test timed-out") } // Now wait a bit and make sure we did not get any fast producer debug statements. select { case <-l.gotIt: t.Fatal("Got debug logs about fast producer") case <-time.After(time.Second): // OK } // We don't need that one anymore ncFast.Close() // Now we will conf reload to enable fast producer stalling. reloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, "false")) // Since the producer can block, we will publish from a different routine, // and check for the debug trace from the main. wg := sync.WaitGroup{} wg.Add(1) doneCh := make(chan struct{}) go func() { defer wg.Done() for i := 0; ; i++ { ncProd.Publish("foo", payload) select { case <-doneCh: return default: } if i%1000 == 0 { time.Sleep(time.Millisecond) } } }() select { case <-l.gotIt: pcnc.Close() close(doneCh) case <-time.After(20 * time.Second): t.Fatal("Timed-out waiting for a warning") } wg.Wait() } func TestNoRaceProducerStallLimits(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" ` conf := createConfFile(t, []byte(tmpl)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() ncSlow := natsConnect(t, s.ClientURL()) defer ncSlow.Close() natsSub(t, ncSlow, "foo", func(m *nats.Msg) { m.Respond([]byte("42")) }) natsFlush(t, ncSlow) ncProd := natsConnect(t, s.ClientURL()) defer ncProd.Close() cid, err := ncSlow.GetClientID() require_NoError(t, err) c := s.GetClient(cid) require_True(t, c != nil) // Artificially set a stall channel on the subscriber. c.mu.Lock() c.out.stc = make(chan struct{}) c.mu.Unlock() start := time.Now() _, err = ncProd.Request("foo", []byte("HELLO"), time.Second) elapsed := time.Since(start) require_NoError(t, err) // This should have not cleared on its own but should have bettwen min and max pause. require_True(t, elapsed >= stallClientMinDuration) require_True(t, elapsed < stallClientMaxDuration) // Now test total maximum by loading up a bunch of requests and measuring the last one. // Artificially set a stall channel again on the subscriber. c.mu.Lock() c.out.stc = make(chan struct{}) // This will prevent us from clearing the stc. c.out.pb = c.out.mp/4*3 + 100 c.mu.Unlock() for i := 0; i < 10; i++ { err = ncProd.PublishRequest("foo", "bar", []byte("HELLO")) require_NoError(t, err) } start = time.Now() _, err = ncProd.Request("foo", []byte("HELLO"), time.Second) elapsed = time.Since(start) require_NoError(t, err) require_True(t, elapsed >= stallTotalAllowed) // Should always be close to totalAllowed (e.g. 10ms), but if you run alot of them in one go can bump up // just past 12ms, hence the Max setting below to avoid a flapper. require_True(t, elapsed < stallTotalAllowed+stallClientMaxDuration) } nats-server-2.10.27/server/ocsp.go000066400000000000000000000655111477524627100167640ustar00rootroot00000000000000// Copyright 2021-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/asn1" "encoding/base64" "encoding/pem" "errors" "fmt" "io" "net/http" "os" "path/filepath" "strings" "sync" "time" "golang.org/x/crypto/ocsp" "github.com/nats-io/nats-server/v2/server/certidp" "github.com/nats-io/nats-server/v2/server/certstore" ) const ( defaultOCSPStoreDir = "ocsp" defaultOCSPCheckInterval = 24 * time.Hour minOCSPCheckInterval = 2 * time.Minute ) type OCSPMode uint8 const ( // OCSPModeAuto staples a status, only if "status_request" is set in cert. OCSPModeAuto OCSPMode = iota // OCSPModeAlways enforces OCSP stapling for certs and shuts down the server in // case a server is revoked or cannot get OCSP staples. OCSPModeAlways // OCSPModeNever disables OCSP stapling even if cert has Must-Staple flag. OCSPModeNever // OCSPModeMust honors the Must-Staple flag from a certificate but also causing shutdown // in case the certificate has been revoked. OCSPModeMust ) // OCSPMonitor monitors the state of a staple per certificate. type OCSPMonitor struct { kind string mu sync.Mutex raw []byte srv *Server certFile string resp *ocsp.Response hc *http.Client stopCh chan struct{} Leaf *x509.Certificate Issuer *x509.Certificate shutdownOnRevoke bool } func (oc *OCSPMonitor) getNextRun() time.Duration { oc.mu.Lock() nextUpdate := oc.resp.NextUpdate oc.mu.Unlock() now := time.Now() if nextUpdate.IsZero() { // If response is missing NextUpdate, we check the day after. // Technically, if NextUpdate is missing, we can try whenever. // https://tools.ietf.org/html/rfc6960#section-4.2.2.1 return defaultOCSPCheckInterval } dur := nextUpdate.Sub(now) / 2 // If negative, then wait a couple of minutes before getting another staple. if dur < 0 { return minOCSPCheckInterval } return dur } func (oc *OCSPMonitor) getStatus() ([]byte, *ocsp.Response, error) { raw, resp := oc.getCacheStatus() if len(raw) > 0 && resp != nil { // Check if the OCSP is still valid. if err := validOCSPResponse(resp); err == nil { return raw, resp, nil } } var err error raw, resp, err = oc.getLocalStatus() if err == nil { return raw, resp, nil } return oc.getRemoteStatus() } func (oc *OCSPMonitor) getCacheStatus() ([]byte, *ocsp.Response) { oc.mu.Lock() defer oc.mu.Unlock() return oc.raw, oc.resp } func (oc *OCSPMonitor) getLocalStatus() ([]byte, *ocsp.Response, error) { opts := oc.srv.getOpts() storeDir := opts.StoreDir if storeDir == _EMPTY_ { return nil, nil, fmt.Errorf("store_dir not set") } // This key must be based upon the current full certificate, not the public key, // so MUST be on the full raw certificate and not an SPKI or other reduced form. key := fmt.Sprintf("%x", sha256.Sum256(oc.Leaf.Raw)) oc.mu.Lock() raw, err := os.ReadFile(filepath.Join(storeDir, defaultOCSPStoreDir, key)) oc.mu.Unlock() if err != nil { return nil, nil, err } resp, err := ocsp.ParseResponse(raw, oc.Issuer) if err != nil { return nil, nil, fmt.Errorf("failed to get local status: %w", err) } if err := validOCSPResponse(resp); err != nil { return nil, nil, err } // Cache the response. oc.mu.Lock() oc.raw = raw oc.resp = resp oc.mu.Unlock() return raw, resp, nil } func (oc *OCSPMonitor) getRemoteStatus() ([]byte, *ocsp.Response, error) { opts := oc.srv.getOpts() var overrideURLs []string if config := opts.OCSPConfig; config != nil { overrideURLs = config.OverrideURLs } getRequestBytes := func(u string, reqDER []byte, hc *http.Client) ([]byte, error) { reqEnc := base64.StdEncoding.EncodeToString(reqDER) u = fmt.Sprintf("%s/%s", u, reqEnc) start := time.Now() resp, err := hc.Get(u) if err != nil { return nil, err } defer resp.Body.Close() oc.srv.Debugf("Received OCSP response (method=GET, status=%v, url=%s, duration=%.3fs)", resp.StatusCode, u, time.Since(start).Seconds()) if resp.StatusCode > 299 { return nil, fmt.Errorf("non-ok http status on GET request (reqlen=%d): %d", len(reqEnc), resp.StatusCode) } return io.ReadAll(resp.Body) } postRequestBytes := func(u string, body []byte, hc *http.Client) ([]byte, error) { hreq, err := http.NewRequest("POST", u, bytes.NewReader(body)) if err != nil { return nil, err } hreq.Header.Add("Content-Type", "application/ocsp-request") hreq.Header.Add("Accept", "application/ocsp-response") start := time.Now() resp, err := hc.Do(hreq) if err != nil { return nil, err } defer resp.Body.Close() oc.srv.Debugf("Received OCSP response (method=POST, status=%v, url=%s, duration=%.3fs)", resp.StatusCode, u, time.Since(start).Seconds()) if resp.StatusCode > 299 { return nil, fmt.Errorf("non-ok http status on POST request (reqlen=%d): %d", len(body), resp.StatusCode) } return io.ReadAll(resp.Body) } // Request documentation: // https://tools.ietf.org/html/rfc6960#appendix-A.1 reqDER, err := ocsp.CreateRequest(oc.Leaf, oc.Issuer, nil) if err != nil { return nil, nil, err } responders := oc.Leaf.OCSPServer if len(overrideURLs) > 0 { responders = overrideURLs } if len(responders) == 0 { return nil, nil, fmt.Errorf("no available ocsp servers") } oc.mu.Lock() hc := oc.hc oc.mu.Unlock() var raw []byte for _, u := range responders { var postErr, getErr error u = strings.TrimSuffix(u, "/") // Prefer to make POST requests first. raw, postErr = postRequestBytes(u, reqDER, hc) if postErr == nil { err = nil break } else { // Fallback to use a GET request. raw, getErr = getRequestBytes(u, reqDER, hc) if getErr == nil { err = nil break } else { err = errors.Join(postErr, getErr) } } } if err != nil { return nil, nil, fmt.Errorf("exhausted ocsp servers: %w", err) } resp, err := ocsp.ParseResponse(raw, oc.Issuer) if err != nil { return nil, nil, fmt.Errorf("failed to get remote status: %w", err) } if err := validOCSPResponse(resp); err != nil { return nil, nil, err } if storeDir := opts.StoreDir; storeDir != _EMPTY_ { key := fmt.Sprintf("%x", sha256.Sum256(oc.Leaf.Raw)) if err := oc.writeOCSPStatus(storeDir, key, raw); err != nil { return nil, nil, fmt.Errorf("failed to write ocsp status: %w", err) } } oc.mu.Lock() oc.raw = raw oc.resp = resp oc.mu.Unlock() return raw, resp, nil } func (oc *OCSPMonitor) run() { s := oc.srv s.mu.Lock() quitCh := s.quitCh s.mu.Unlock() var doShutdown bool defer func() { // Need to decrement before shuting down, otherwise shutdown // would be stuck waiting on grWG to go down to 0. s.grWG.Done() if doShutdown { s.Shutdown() } }() oc.mu.Lock() shutdownOnRevoke := oc.shutdownOnRevoke certFile := oc.certFile stopCh := oc.stopCh kind := oc.kind oc.mu.Unlock() var nextRun time.Duration _, resp, err := oc.getStatus() if err == nil && resp.Status == ocsp.Good { nextRun = oc.getNextRun() t := resp.NextUpdate.Format(time.RFC3339Nano) s.Noticef( "Found OCSP status for %s certificate at '%s': good, next update %s, checking again in %s", kind, certFile, t, nextRun, ) } else if err == nil && shutdownOnRevoke { // If resp.Status is ocsp.Revoked, ocsp.Unknown, or any other value. s.Errorf("Found OCSP status for %s certificate at '%s': %s", kind, certFile, ocspStatusString(resp.Status)) doShutdown = true return } for { // On reload, if the certificate changes then need to stop this monitor. select { case <-time.After(nextRun): case <-stopCh: // In case of reload and have to restart the OCSP stapling monitoring. return case <-quitCh: // Server quit channel. return } _, resp, err := oc.getRemoteStatus() if err != nil { nextRun = oc.getNextRun() s.Errorf("Bad OCSP status update for certificate '%s': %s, trying again in %v", certFile, err, nextRun) continue } switch n := resp.Status; n { case ocsp.Good: nextRun = oc.getNextRun() t := resp.NextUpdate.Format(time.RFC3339Nano) s.Noticef( "Received OCSP status for %s certificate '%s': good, next update %s, checking again in %s", kind, certFile, t, nextRun, ) continue default: s.Errorf("Received OCSP status for %s certificate '%s': %s", kind, certFile, ocspStatusString(n)) if shutdownOnRevoke { doShutdown = true } return } } } func (oc *OCSPMonitor) stop() { oc.mu.Lock() stopCh := oc.stopCh oc.mu.Unlock() stopCh <- struct{}{} } // NewOCSPMonitor takes a TLS configuration then wraps it with the callbacks set for OCSP verification // along with a monitor that will periodically fetch OCSP staples. func (srv *Server) NewOCSPMonitor(config *tlsConfigKind) (*tls.Config, *OCSPMonitor, error) { kind := config.kind tc := config.tlsConfig tcOpts := config.tlsOpts opts := srv.getOpts() oc := opts.OCSPConfig // We need to track the CA certificate in case the CA is not present // in the chain to be able to verify the signature of the OCSP staple. var ( certFile string caFile string ) if kind == kindStringMap[CLIENT] { tcOpts = opts.tlsConfigOpts if opts.TLSCert != _EMPTY_ { certFile = opts.TLSCert } if opts.TLSCaCert != _EMPTY_ { caFile = opts.TLSCaCert } } if tcOpts != nil { certFile = tcOpts.CertFile caFile = tcOpts.CaFile } // NOTE: Currently OCSP Stapling is enabled only for the first certificate found. var mon *OCSPMonitor for _, currentCert := range tc.Certificates { // Create local copy since this will be used in the GetCertificate callback. cert := currentCert // This is normally non-nil, but can still be nil here when in tests // or in some embedded scenarios. if cert.Leaf == nil { if len(cert.Certificate) <= 0 { return nil, nil, fmt.Errorf("no certificate found") } var err error cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) if err != nil { return nil, nil, fmt.Errorf("error parsing certificate: %v", err) } } var shutdownOnRevoke bool mustStaple := hasOCSPStatusRequest(cert.Leaf) if oc != nil { switch { case oc.Mode == OCSPModeNever: if mustStaple { srv.Warnf("Certificate at '%s' has MustStaple but OCSP is disabled", certFile) } return tc, nil, nil case oc.Mode == OCSPModeAlways: // Start the monitor for this cert even if it does not have // the MustStaple flag and shutdown the server in case the // staple ever gets revoked. mustStaple = true shutdownOnRevoke = true case oc.Mode == OCSPModeMust && mustStaple: shutdownOnRevoke = true case oc.Mode == OCSPModeAuto && !mustStaple: // "status_request" MustStaple flag not set in certificate. No need to do anything. return tc, nil, nil } } if !mustStaple { // No explicit OCSP config and cert does not have MustStaple flag either. return tc, nil, nil } if err := srv.setupOCSPStapleStoreDir(); err != nil { return nil, nil, err } // TODO: Add OCSP 'responder_cert' option in case CA cert not available. issuer, err := getOCSPIssuer(caFile, cert.Certificate) if err != nil { return nil, nil, err } mon = &OCSPMonitor{ kind: kind, srv: srv, hc: &http.Client{Timeout: 30 * time.Second}, shutdownOnRevoke: shutdownOnRevoke, certFile: certFile, stopCh: make(chan struct{}, 1), Leaf: cert.Leaf, Issuer: issuer, } // Get the certificate status from the memory, then remote OCSP responder. if _, resp, err := mon.getStatus(); err != nil { return nil, nil, fmt.Errorf("bad OCSP status update for certificate at '%s': %s", certFile, err) } else if resp != nil && resp.Status != ocsp.Good && shutdownOnRevoke { return nil, nil, fmt.Errorf("found existing OCSP status for certificate at '%s': %s", certFile, ocspStatusString(resp.Status)) } // Callbacks below will be in charge of returning the certificate instead, // so this has to be nil. tc.Certificates = nil // GetCertificate returns a certificate that's presented to a client. tc.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { ccert := cert raw, _, err := mon.getStatus() if err != nil { return nil, err } return &tls.Certificate{ OCSPStaple: raw, Certificate: ccert.Certificate, PrivateKey: ccert.PrivateKey, SupportedSignatureAlgorithms: ccert.SupportedSignatureAlgorithms, SignedCertificateTimestamps: ccert.SignedCertificateTimestamps, Leaf: ccert.Leaf, }, nil } // Check whether need to verify staples from a peer router or gateway connection. switch kind { case kindStringMap[ROUTER], kindStringMap[GATEWAY]: tc.VerifyConnection = func(s tls.ConnectionState) error { oresp := s.OCSPResponse if oresp == nil { return fmt.Errorf("%s peer missing OCSP Staple", kind) } // Peer connections will verify the response of the staple. if len(s.VerifiedChains) == 0 { return fmt.Errorf("%s peer missing TLS verified chains", kind) } chain := s.VerifiedChains[0] peerLeaf := chain[0] peerIssuer := certidp.GetLeafIssuerCert(chain, 0) if peerIssuer == nil { return fmt.Errorf("failed to get issuer certificate for %s peer", kind) } // Response signature of issuer or issuer delegate is checked in the library parse resp, err := ocsp.ParseResponseForCert(oresp, peerLeaf, peerIssuer) if err != nil { return fmt.Errorf("failed to parse OCSP response from %s peer: %w", kind, err) } // If signer was issuer delegate double-check issuer delegate authorization if resp.Certificate != nil { ok := false for _, eku := range resp.Certificate.ExtKeyUsage { if eku == x509.ExtKeyUsageOCSPSigning { ok = true break } } if !ok { return fmt.Errorf("OCSP staple's signer missing authorization by CA to act as OCSP signer") } } // Check that the OCSP response is effective, take defaults for clockskew and default validity peerOpts := certidp.OCSPPeerConfig{ClockSkew: -1, TTLUnsetNextUpdate: -1} sLog := certidp.Log{Debugf: srv.Debugf} if !certidp.OCSPResponseCurrent(resp, &peerOpts, &sLog) { return fmt.Errorf("OCSP staple from %s peer not current", kind) } if resp.Status != ocsp.Good { return fmt.Errorf("bad status for OCSP Staple from %s peer: %s", kind, ocspStatusString(resp.Status)) } return nil } // When server makes a peer connection, need to also present an OCSP Staple. tc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { ccert := cert raw, _, err := mon.getStatus() if err != nil { return nil, err } // NOTE: crypto/tls.sendClientCertificate internally also calls getClientCertificate // so if for some reason these callbacks are triggered concurrently during a reconnect // there can be a race. To avoid that, the OCSP monitor lock is used to serialize access // to the staple which could also change inflight during an update. mon.mu.Lock() ccert.OCSPStaple = raw mon.mu.Unlock() return &ccert, nil } default: // GetClientCertificate returns a certificate that's presented to a server. tc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { return &cert, nil } } } return tc, mon, nil } func (s *Server) setupOCSPStapleStoreDir() error { opts := s.getOpts() storeDir := opts.StoreDir if storeDir == _EMPTY_ { return nil } storeDir = filepath.Join(storeDir, defaultOCSPStoreDir) if stat, err := os.Stat(storeDir); os.IsNotExist(err) { if err := os.MkdirAll(storeDir, defaultDirPerms); err != nil { return fmt.Errorf("could not create OCSP storage directory - %v", err) } } else if stat == nil || !stat.IsDir() { return fmt.Errorf("OCSP storage directory is not a directory") } return nil } type tlsConfigKind struct { tlsConfig *tls.Config tlsOpts *TLSConfigOpts kind string isLeafSpoke bool apply func(*tls.Config) } func (s *Server) configureOCSP() []*tlsConfigKind { sopts := s.getOpts() configs := make([]*tlsConfigKind, 0) if config := sopts.TLSConfig; config != nil { opts := sopts.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[CLIENT], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.TLSConfig = tc }, } configs = append(configs, o) } if config := sopts.Websocket.TLSConfig; config != nil { opts := sopts.Websocket.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[CLIENT], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.Websocket.TLSConfig = tc }, } configs = append(configs, o) } if config := sopts.MQTT.TLSConfig; config != nil { opts := sopts.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[CLIENT], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.MQTT.TLSConfig = tc }, } configs = append(configs, o) } if config := sopts.Cluster.TLSConfig; config != nil { opts := sopts.Cluster.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[ROUTER], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.Cluster.TLSConfig = tc }, } configs = append(configs, o) } if config := sopts.LeafNode.TLSConfig; config != nil { opts := sopts.LeafNode.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[LEAF], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.LeafNode.TLSConfig = tc }, } configs = append(configs, o) } for _, remote := range sopts.LeafNode.Remotes { if config := remote.TLSConfig; config != nil { // Use a copy of the remote here since will be used // in the apply func callback below. r, opts := remote, remote.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[LEAF], tlsConfig: config, tlsOpts: opts, isLeafSpoke: true, apply: func(tc *tls.Config) { r.TLSConfig = tc }, } configs = append(configs, o) } } if config := sopts.Gateway.TLSConfig; config != nil { opts := sopts.Gateway.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[GATEWAY], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { sopts.Gateway.TLSConfig = tc }, } configs = append(configs, o) } for _, remote := range sopts.Gateway.Gateways { if config := remote.TLSConfig; config != nil { gw, opts := remote, remote.tlsConfigOpts o := &tlsConfigKind{ kind: kindStringMap[GATEWAY], tlsConfig: config, tlsOpts: opts, apply: func(tc *tls.Config) { gw.TLSConfig = tc }, } configs = append(configs, o) } } return configs } func (s *Server) enableOCSP() error { configs := s.configureOCSP() for _, config := range configs { // We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer if config.kind != kindStringMap[LEAF] { // OCSP Stapling feature, will also enable tls server peer check for gateway and route peers tc, mon, err := s.NewOCSPMonitor(config) if err != nil { return err } // Check if an OCSP stapling monitor is required for this certificate. if mon != nil { s.ocsps = append(s.ocsps, mon) // Override the TLS config with one that follows OCSP stapling config.apply(tc) } } // OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS) if config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] { tc, plugged, err := s.plugTLSOCSPPeer(config) if err != nil { return err } if plugged && tc != nil { s.ocspPeerVerify = true config.apply(tc) } } } return nil } func (s *Server) startOCSPMonitoring() { s.mu.Lock() ocsps := s.ocsps s.mu.Unlock() if ocsps == nil { return } for _, mon := range ocsps { m := mon m.mu.Lock() kind := m.kind m.mu.Unlock() s.Noticef("OCSP Stapling enabled for %s connections", kind) s.startGoRoutine(func() { m.run() }) } } func (s *Server) reloadOCSP() error { if err := s.setupOCSPStapleStoreDir(); err != nil { return err } s.mu.Lock() ocsps := s.ocsps s.mu.Unlock() // Stop all OCSP Stapling monitors in case there were any running. for _, oc := range ocsps { oc.stop() } configs := s.configureOCSP() // Restart the monitors under the new configuration. ocspm := make([]*OCSPMonitor, 0) // Reset server's ocspPeerVerify flag to re-detect at least one plugged OCSP peer s.mu.Lock() s.ocspPeerVerify = false s.mu.Unlock() s.stopOCSPResponseCache() for _, config := range configs { // We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer if config.kind != kindStringMap[LEAF] { tc, mon, err := s.NewOCSPMonitor(config) if err != nil { return err } // Check if an OCSP stapling monitor is required for this certificate. if mon != nil { ocspm = append(ocspm, mon) // Apply latest TLS configuration after OCSP monitors have started. defer config.apply(tc) } } // OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS) if config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] { tc, plugged, err := s.plugTLSOCSPPeer(config) if err != nil { return err } if plugged && tc != nil { s.ocspPeerVerify = true defer config.apply(tc) } } } // Replace stopped monitors with the new ones. s.mu.Lock() s.ocsps = ocspm s.mu.Unlock() // Dispatch all goroutines once again. s.startOCSPMonitoring() // Init and restart OCSP responder cache s.stopOCSPResponseCache() s.initOCSPResponseCache() s.startOCSPResponseCache() return nil } func hasOCSPStatusRequest(cert *x509.Certificate) bool { // OID for id-pe-tlsfeature defined in RFC here: // https://datatracker.ietf.org/doc/html/rfc7633 tlsFeatures := asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 24} const statusRequestExt = 5 // Example values: // * [48 3 2 1 5] - seen when creating own certs locally // * [30 3 2 1 5] - seen in the wild // Documentation: // https://tools.ietf.org/html/rfc6066 for _, ext := range cert.Extensions { if !ext.Id.Equal(tlsFeatures) { continue } var val []int rest, err := asn1.Unmarshal(ext.Value, &val) if err != nil || len(rest) > 0 { return false } for _, n := range val { if n == statusRequestExt { return true } } break } return false } // writeOCSPStatus writes an OCSP status to a temporary file then moves it to a // new path, in an attempt to avoid corrupting existing data. func (oc *OCSPMonitor) writeOCSPStatus(storeDir, file string, data []byte) error { storeDir = filepath.Join(storeDir, defaultOCSPStoreDir) tmp, err := os.CreateTemp(storeDir, "tmp-cert-status") if err != nil { return err } if _, err := tmp.Write(data); err != nil { tmp.Close() os.Remove(tmp.Name()) return err } if err := tmp.Close(); err != nil { return err } oc.mu.Lock() err = os.Rename(tmp.Name(), filepath.Join(storeDir, file)) oc.mu.Unlock() if err != nil { os.Remove(tmp.Name()) return err } return nil } func parseCertPEM(name string) ([]*x509.Certificate, error) { data, err := os.ReadFile(name) if err != nil { return nil, err } var pemBytes []byte var block *pem.Block for len(data) != 0 { block, data = pem.Decode(data) if block == nil { break } if block.Type != "CERTIFICATE" { return nil, fmt.Errorf("unexpected PEM certificate type: %s", block.Type) } pemBytes = append(pemBytes, block.Bytes...) } return x509.ParseCertificates(pemBytes) } // getOCSPIssuerLocally determines a leaf's issuer from locally configured certificates func getOCSPIssuerLocally(trustedCAs []*x509.Certificate, certBundle []*x509.Certificate) (*x509.Certificate, error) { var vOpts x509.VerifyOptions var leaf *x509.Certificate trustedCAPool := x509.NewCertPool() // Require Leaf as first cert in bundle if len(certBundle) > 0 { leaf = certBundle[0] } else { return nil, fmt.Errorf("invalid ocsp ca configuration") } // Allow Issuer to be configured as second cert in bundle if len(certBundle) > 1 { // The operator may have misconfigured the cert bundle issuerCandidate := certBundle[1] err := issuerCandidate.CheckSignature(leaf.SignatureAlgorithm, leaf.RawTBSCertificate, leaf.Signature) if err != nil { return nil, fmt.Errorf("invalid issuer configuration: %w", err) } else { return issuerCandidate, nil } } // Operator did not provide the Leaf Issuer in cert bundle second position // so we will attempt to create at least one ordered verified chain from the // trusted CA pool. // Specify CA trust store to validator; if unset, system trust store used if len(trustedCAs) > 0 { for _, ca := range trustedCAs { trustedCAPool.AddCert(ca) } vOpts.Roots = trustedCAPool } return certstore.GetLeafIssuer(leaf, vOpts), nil } // getOCSPIssuer determines an issuer certificate from the cert (bundle) or the file-based CA trust store func getOCSPIssuer(caFile string, chain [][]byte) (*x509.Certificate, error) { var issuer *x509.Certificate var trustedCAs []*x509.Certificate var certBundle []*x509.Certificate var err error // FIXME(tgb): extend if pluggable CA store provider added to NATS (i.e. other than PEM file) // Non-system default CA trust store passed if caFile != _EMPTY_ { trustedCAs, err = parseCertPEM(caFile) if err != nil { return nil, fmt.Errorf("failed to parse ca_file: %v", err) } } // Specify bundled intermediate CA store for _, certBytes := range chain { cert, err := x509.ParseCertificate(certBytes) if err != nil { return nil, fmt.Errorf("failed to parse cert: %v", err) } certBundle = append(certBundle, cert) } issuer, err = getOCSPIssuerLocally(trustedCAs, certBundle) if err != nil || issuer == nil { return nil, fmt.Errorf("no issuers found") } if !issuer.IsCA { return nil, fmt.Errorf("%s invalid ca basic constraints: is not ca", issuer.Subject) } return issuer, nil } func ocspStatusString(n int) string { switch n { case ocsp.Good: return "good" case ocsp.Revoked: return "revoked" default: return "unknown" } } func validOCSPResponse(r *ocsp.Response) error { // Time validation not handled by ParseResponse. // https://tools.ietf.org/html/rfc6960#section-4.2.2.1 if !r.NextUpdate.IsZero() && r.NextUpdate.Before(time.Now()) { t := r.NextUpdate.Format(time.RFC3339Nano) return fmt.Errorf("invalid ocsp NextUpdate, is past time: %s", t) } if r.ThisUpdate.After(time.Now()) { t := r.ThisUpdate.Format(time.RFC3339Nano) return fmt.Errorf("invalid ocsp ThisUpdate, is future time: %s", t) } return nil } nats-server-2.10.27/server/ocsp_peer.go000066400000000000000000000346041477524627100177760ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "crypto/tls" "crypto/x509" "errors" "fmt" "strings" "time" "golang.org/x/crypto/ocsp" "github.com/nats-io/nats-server/v2/server/certidp" ) func parseOCSPPeer(v any) (pcfg *certidp.OCSPPeerConfig, retError error) { var lt token defer convertPanicToError(<, &retError) tk, v := unwrapValue(v, <) cm, ok := v.(map[string]any) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalPeerOptsConfig, v)} } pcfg = certidp.NewOCSPPeerConfig() retError = nil for mk, mv := range cm { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "verify": verify, ok := mv.(bool) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} } pcfg.Verify = verify case "allowed_clockskew": at := float64(0) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv case string: d, err := time.ParseDuration(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} } at = d.Seconds() default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} } if at >= 0 { pcfg.ClockSkew = at } case "ca_timeout": at := float64(0) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv case string: d, err := time.ParseDuration(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} } at = d.Seconds() default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} } if at >= 0 { pcfg.Timeout = at } case "cache_ttl_when_next_update_unset": at := float64(0) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv case string: d, err := time.ParseDuration(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} } at = d.Seconds() default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, "unexpected type")} } if at >= 0 { pcfg.TTLUnsetNextUpdate = at } case "warn_only": warnOnly, ok := mv.(bool) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} } pcfg.WarnOnly = warnOnly case "unknown_is_good": unknownIsGood, ok := mv.(bool) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} } pcfg.UnknownIsGood = unknownIsGood case "allow_when_ca_unreachable": allowWhenCAUnreachable, ok := mv.(bool) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} } pcfg.AllowWhenCAUnreachable = allowWhenCAUnreachable default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)} } } return pcfg, nil } func peerFromVerifiedChains(chains [][]*x509.Certificate) *x509.Certificate { if len(chains) == 0 || len(chains[0]) == 0 { return nil } return chains[0][0] } // plugTLSOCSPPeer will plug the TLS handshake lifecycle for client mTLS connections and Leaf connections func (s *Server) plugTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { if config == nil || config.tlsConfig == nil { return nil, false, errors.New(certidp.ErrUnableToPlugTLSEmptyConfig) } kind := config.kind isSpoke := config.isLeafSpoke tcOpts := config.tlsOpts if tcOpts == nil || tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { return nil, false, nil } s.Debugf(certidp.DbgPlugTLSForKind, config.kind) // peer is a tls client if kind == kindStringMap[CLIENT] || (kind == kindStringMap[LEAF] && !isSpoke) { if !tcOpts.Verify { return nil, false, errors.New(certidp.ErrMTLSRequired) } return s.plugClientTLSOCSPPeer(config) } // peer is a tls server if kind == kindStringMap[LEAF] && isSpoke { return s.plugServerTLSOCSPPeer(config) } return nil, false, nil } func (s *Server) plugClientTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { if config == nil || config.tlsConfig == nil || config.tlsOpts == nil { return nil, false, errors.New(certidp.ErrUnableToPlugTLSClient) } tc := config.tlsConfig tcOpts := config.tlsOpts kind := config.kind if tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { return tc, false, nil } tc.VerifyConnection = func(cs tls.ConnectionState) error { if !s.tlsClientOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) { s.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSClientRejectConnection) return errors.New(certidp.MsgTLSClientRejectConnection) } return nil } return tc, true, nil } func (s *Server) plugServerTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) { if config == nil || config.tlsConfig == nil || config.tlsOpts == nil { return nil, false, errors.New(certidp.ErrUnableToPlugTLSServer) } tc := config.tlsConfig tcOpts := config.tlsOpts kind := config.kind if tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify { return tc, false, nil } tc.VerifyConnection = func(cs tls.ConnectionState) error { if !s.tlsServerOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) { s.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSServerRejectConnection) return errors.New(certidp.MsgTLSServerRejectConnection) } return nil } return tc, true, nil } // tlsServerOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP // eligibility. A verified chain is considered OCSP Valid if either none of the links are // OCSP eligible, or current "good" responses from the CA can be obtained for each eligible link. // Upon first OCSP Valid chain found, the Server is deemed OCSP Valid. If none of the chains are // OCSP Valid, the Server is deemed OCSP Invalid. A verified self-signed certificate (chain length 1) // is also considered OCSP Valid. func (s *Server) tlsServerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { s.Debugf(certidp.DbgNumServerChains, len(chains)) return s.peerOCSPValid(chains, opts) } // tlsClientOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP // eligibility. A verified chain is considered OCSP Valid if either none of the links are // OCSP eligible, or current "good" responses from the CA can be obtained for each eligible link. // Upon first OCSP Valid chain found, the Client is deemed OCSP Valid. If none of the chains are // OCSP Valid, the Client is deemed OCSP Invalid. A verified self-signed certificate (chain length 1) // is also considered OCSP Valid. func (s *Server) tlsClientOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { s.Debugf(certidp.DbgNumClientChains, len(chains)) return s.peerOCSPValid(chains, opts) } func (s *Server) peerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool { peer := peerFromVerifiedChains(chains) if peer == nil { s.Errorf(certidp.ErrPeerEmptyAutoReject) return false } for ci, chain := range chains { s.Debugf(certidp.DbgLinksInChain, ci, len(chain)) // Self-signed certificate is Client OCSP Valid (no CA) if len(chain) == 1 { s.Debugf(certidp.DbgSelfSignedValid, ci) return true } // Check if any of the links in the chain are OCSP eligible chainEligible := false var eligibleLinks []*certidp.ChainLink // Iterate over links skipping the root cert which is not OCSP eligible (self == issuer) for linkPos := 0; linkPos < len(chain)-1; linkPos++ { cert := chain[linkPos] link := &certidp.ChainLink{ Leaf: cert, } if certidp.CertOCSPEligible(link) { chainEligible = true issuerCert := certidp.GetLeafIssuerCert(chain, linkPos) if issuerCert == nil { // unexpected chain condition, reject Client as OCSP Invalid return false } link.Issuer = issuerCert eligibleLinks = append(eligibleLinks, link) } } // A trust-store verified chain that is not OCSP eligible is always OCSP Valid if !chainEligible { s.Debugf(certidp.DbgValidNonOCSPChain, ci) return true } s.Debugf(certidp.DbgChainIsOCSPEligible, ci, len(eligibleLinks)) // Chain has at least one OCSP eligible link, so check each eligible link; // any link with a !good OCSP response chain OCSP Invalid chainValid := true for _, link := range eligibleLinks { // if option selected, good could reflect either ocsp.Good or ocsp.Unknown if badReason, good := s.certOCSPGood(link, opts); !good { s.Debugf(badReason) s.sendOCSPPeerChainlinkInvalidEvent(peer, link.Leaf, badReason) chainValid = false break } } if chainValid { s.Debugf(certidp.DbgChainIsOCSPValid, ci) return true } } // If we are here, all chains had OCSP eligible links, but none of the chains achieved OCSP valid s.Debugf(certidp.DbgNoOCSPValidChains) return false } func (s *Server) certOCSPGood(link *certidp.ChainLink, opts *certidp.OCSPPeerConfig) (string, bool) { if link == nil || link.Leaf == nil || link.Issuer == nil || link.OCSPWebEndpoints == nil || len(*link.OCSPWebEndpoints) < 1 { return "Empty chainlink found", false } var err error sLogs := &certidp.Log{ Debugf: s.Debugf, Noticef: s.Noticef, Warnf: s.Warnf, Errorf: s.Errorf, Tracef: s.Tracef, } fingerprint := certidp.GenerateFingerprint(link.Leaf) // Used for debug/operator only, not match subj := certidp.GetSubjectDNForm(link.Leaf) var rawResp []byte var ocspr *ocsp.Response var useCachedResp bool var rc = s.ocsprc var cachedRevocation bool // Check our cache before calling out to the CA OCSP responder s.Debugf(certidp.DbgCheckingCacheForCert, subj, fingerprint) if rawResp = rc.Get(fingerprint, sLogs); len(rawResp) > 0 { // Signature validation of CA's OCSP response occurs in ParseResponse ocspr, err = ocsp.ParseResponse(rawResp, link.Issuer) if err == nil && ocspr != nil { // Check if OCSP Response delegation present and if so is valid if !certidp.ValidDelegationCheck(link.Issuer, ocspr) { // Invalid delegation was already in cache, purge it and don't use it s.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj) rc.Delete(fingerprint, true, sLogs) goto AFTERCACHE } if certidp.OCSPResponseCurrent(ocspr, opts, sLogs) { s.Debugf(certidp.DbgCurrentResponseCached, certidp.GetStatusAssertionStr(ocspr.Status)) useCachedResp = true } else { // Cached response is not current, delete it and tidy runtime stats to reflect a miss; // if preserve_revoked is enabled, the cache will not delete the cached response s.Debugf(certidp.DbgExpiredResponseCached, certidp.GetStatusAssertionStr(ocspr.Status)) rc.Delete(fingerprint, true, sLogs) } // Regardless of currency, record a cached revocation found in case AllowWhenCAUnreachable is set if ocspr.Status == ocsp.Revoked { cachedRevocation = true } } else { // Bogus cached assertion, purge it and don't use it s.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj, fingerprint) rc.Delete(fingerprint, true, sLogs) goto AFTERCACHE } } AFTERCACHE: if !useCachedResp { // CA OCSP responder callout needed rawResp, err = certidp.FetchOCSPResponse(link, opts, sLogs) if err != nil || rawResp == nil || len(rawResp) == 0 { s.Warnf(certidp.ErrCAResponderCalloutFail, subj, err) if opts.WarnOnly { s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) return _EMPTY_, true } if opts.AllowWhenCAUnreachable && !cachedRevocation { // Link has no cached history of revocation, so allow it to pass s.Warnf(certidp.MsgAllowWhenCAUnreachableOccurred, subj) return _EMPTY_, true } else if opts.AllowWhenCAUnreachable { // Link has cached but expired revocation so reject when CA is unreachable s.Warnf(certidp.MsgAllowWhenCAUnreachableOccurredCachedRevoke, subj) } return certidp.MsgFailedOCSPResponseFetch, false } // Signature validation of CA's OCSP response occurs in ParseResponse ocspr, err = ocsp.ParseResponse(rawResp, link.Issuer) if err == nil && ocspr != nil { // Check if OCSP Response delegation present and if so is valid if !certidp.ValidDelegationCheck(link.Issuer, ocspr) { s.Warnf(certidp.MsgOCSPResponseDelegationInvalid, subj) if opts.WarnOnly { // Can't use bogus assertion, but warn-only set so allow link to pass s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) return _EMPTY_, true } return fmt.Sprintf(certidp.MsgOCSPResponseDelegationInvalid, subj), false } if !certidp.OCSPResponseCurrent(ocspr, opts, sLogs) { s.Warnf(certidp.ErrNewCAResponseNotCurrent, subj) if opts.WarnOnly { // Can't use non-effective assertion, but warn-only set so allow link to pass s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) return _EMPTY_, true } return certidp.MsgOCSPResponseNotEffective, false } } else { s.Errorf(certidp.ErrCAResponseParseFailed, subj, err) if opts.WarnOnly { // Can't use bogus assertion, but warn-only set so allow link to pass s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) return _EMPTY_, true } return certidp.MsgFailedOCSPResponseParse, false } // cache the valid fetched CA OCSP Response rc.Put(fingerprint, ocspr, subj, sLogs) } // Whether through valid cache response available or newly fetched valid response, now check the status if ocspr.Status == ocsp.Revoked || (ocspr.Status == ocsp.Unknown && !opts.UnknownIsGood) { s.Warnf(certidp.ErrOCSPInvalidPeerLink, subj, certidp.GetStatusAssertionStr(ocspr.Status)) if opts.WarnOnly { s.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj) return _EMPTY_, true } return fmt.Sprintf(certidp.MsgOCSPResponseInvalidStatus, certidp.GetStatusAssertionStr(ocspr.Status)), false } s.Debugf(certidp.DbgOCSPValidPeerLink, subj) return _EMPTY_, true } nats-server-2.10.27/server/ocsp_responsecache.go000066400000000000000000000374101477524627100216630ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "encoding/json" "errors" "fmt" "io" "os" "path" "path/filepath" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "golang.org/x/crypto/ocsp" "github.com/nats-io/nats-server/v2/server/certidp" ) const ( OCSPResponseCacheDefaultDir = "_rc_" OCSPResponseCacheDefaultFilename = "cache.json" OCSPResponseCacheDefaultTempFilePrefix = "ocsprc-*" OCSPResponseCacheMinimumSaveInterval = 1 * time.Second OCSPResponseCacheDefaultSaveInterval = 5 * time.Minute ) type OCSPResponseCacheType int const ( NONE OCSPResponseCacheType = iota + 1 LOCAL ) var OCSPResponseCacheTypeMap = map[string]OCSPResponseCacheType{ "none": NONE, "local": LOCAL, } type OCSPResponseCacheConfig struct { Type OCSPResponseCacheType LocalStore string PreserveRevoked bool SaveInterval float64 } func NewOCSPResponseCacheConfig() *OCSPResponseCacheConfig { return &OCSPResponseCacheConfig{ Type: LOCAL, LocalStore: OCSPResponseCacheDefaultDir, PreserveRevoked: false, SaveInterval: OCSPResponseCacheDefaultSaveInterval.Seconds(), } } type OCSPResponseCacheStats struct { Responses int64 `json:"size"` Hits int64 `json:"hits"` Misses int64 `json:"misses"` Revokes int64 `json:"revokes"` Goods int64 `json:"goods"` Unknowns int64 `json:"unknowns"` } type OCSPResponseCacheItem struct { Subject string `json:"subject,omitempty"` CachedAt time.Time `json:"cached_at"` RespStatus certidp.StatusAssertion `json:"resp_status"` RespExpires time.Time `json:"resp_expires,omitempty"` Resp []byte `json:"resp"` } type OCSPResponseCache interface { Put(key string, resp *ocsp.Response, subj string, log *certidp.Log) Get(key string, log *certidp.Log) []byte Delete(key string, miss bool, log *certidp.Log) Type() string Start(s *Server) Stop(s *Server) Online() bool Config() *OCSPResponseCacheConfig Stats() *OCSPResponseCacheStats } // NoOpCache is a no-op implementation of OCSPResponseCache type NoOpCache struct { config *OCSPResponseCacheConfig stats *OCSPResponseCacheStats online bool mu *sync.RWMutex } func (c *NoOpCache) Put(_ string, _ *ocsp.Response, _ string, _ *certidp.Log) {} func (c *NoOpCache) Get(_ string, _ *certidp.Log) []byte { return nil } func (c *NoOpCache) Delete(_ string, _ bool, _ *certidp.Log) {} func (c *NoOpCache) Start(_ *Server) { c.mu.Lock() defer c.mu.Unlock() c.stats = &OCSPResponseCacheStats{} c.online = true } func (c *NoOpCache) Stop(_ *Server) { c.mu.Lock() defer c.mu.Unlock() c.online = false } func (c *NoOpCache) Online() bool { c.mu.RLock() defer c.mu.RUnlock() return c.online } func (c *NoOpCache) Type() string { c.mu.RLock() defer c.mu.RUnlock() return "none" } func (c *NoOpCache) Config() *OCSPResponseCacheConfig { c.mu.RLock() defer c.mu.RUnlock() return c.config } func (c *NoOpCache) Stats() *OCSPResponseCacheStats { c.mu.RLock() defer c.mu.RUnlock() return c.stats } // LocalCache is a local file implementation of OCSPResponseCache type LocalCache struct { config *OCSPResponseCacheConfig stats *OCSPResponseCacheStats online bool cache map[string]OCSPResponseCacheItem mu *sync.RWMutex saveInterval time.Duration dirty bool timer *time.Timer } // Put captures a CA OCSP response to the OCSP peer cache indexed by response fingerprint (a hash) func (c *LocalCache) Put(key string, caResp *ocsp.Response, subj string, log *certidp.Log) { c.mu.RLock() if !c.online || caResp == nil || key == "" { c.mu.RUnlock() return } c.mu.RUnlock() log.Debugf(certidp.DbgCachingResponse, subj, key) rawC, err := c.Compress(caResp.Raw) if err != nil { log.Errorf(certidp.ErrResponseCompressFail, key, err) return } log.Debugf(certidp.DbgAchievedCompression, float64(len(rawC))/float64(len(caResp.Raw))) c.mu.Lock() defer c.mu.Unlock() // check if we are replacing and do stats item, ok := c.cache[key] if ok { c.adjustStats(-1, item.RespStatus) } item = OCSPResponseCacheItem{ Subject: subj, CachedAt: time.Now().UTC().Round(time.Second), RespStatus: certidp.StatusAssertionIntToVal[caResp.Status], RespExpires: caResp.NextUpdate, Resp: rawC, } c.cache[key] = item c.adjustStats(1, item.RespStatus) c.dirty = true } // Get returns a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash) func (c *LocalCache) Get(key string, log *certidp.Log) []byte { c.mu.RLock() defer c.mu.RUnlock() if !c.online || key == "" { return nil } val, ok := c.cache[key] if ok { atomic.AddInt64(&c.stats.Hits, 1) log.Debugf(certidp.DbgCacheHit, key) } else { atomic.AddInt64(&c.stats.Misses, 1) log.Debugf(certidp.DbgCacheMiss, key) return nil } resp, err := c.Decompress(val.Resp) if err != nil { log.Errorf(certidp.ErrResponseDecompressFail, key, err) return nil } return resp } func (c *LocalCache) adjustStatsHitToMiss() { atomic.AddInt64(&c.stats.Misses, 1) atomic.AddInt64(&c.stats.Hits, -1) } func (c *LocalCache) adjustStats(delta int64, rs certidp.StatusAssertion) { if delta == 0 { return } atomic.AddInt64(&c.stats.Responses, delta) switch rs { case ocsp.Good: atomic.AddInt64(&c.stats.Goods, delta) case ocsp.Revoked: atomic.AddInt64(&c.stats.Revokes, delta) case ocsp.Unknown: atomic.AddInt64(&c.stats.Unknowns, delta) } } // Delete removes a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash) func (c *LocalCache) Delete(key string, wasMiss bool, log *certidp.Log) { c.mu.Lock() defer c.mu.Unlock() if !c.online || key == "" || c.config == nil { return } item, ok := c.cache[key] if !ok { return } if item.RespStatus == ocsp.Revoked && c.config.PreserveRevoked { log.Debugf(certidp.DbgPreservedRevocation, key) if wasMiss { c.adjustStatsHitToMiss() } return } log.Debugf(certidp.DbgDeletingCacheResponse, key) delete(c.cache, key) c.adjustStats(-1, item.RespStatus) if wasMiss { c.adjustStatsHitToMiss() } c.dirty = true } // Start initializes the configured OCSP peer cache, loads a saved cache from disk (if present), and initializes runtime statistics func (c *LocalCache) Start(s *Server) { s.Debugf(certidp.DbgStartingCache) c.loadCache(s) c.initStats() c.mu.Lock() c.online = true c.mu.Unlock() } func (c *LocalCache) Stop(s *Server) { c.mu.Lock() s.Debugf(certidp.DbgStoppingCache) c.online = false c.timer.Stop() c.mu.Unlock() c.saveCache(s) } func (c *LocalCache) Online() bool { c.mu.RLock() defer c.mu.RUnlock() return c.online } func (c *LocalCache) Type() string { c.mu.RLock() defer c.mu.RUnlock() return "local" } func (c *LocalCache) Config() *OCSPResponseCacheConfig { c.mu.RLock() defer c.mu.RUnlock() return c.config } func (c *LocalCache) Stats() *OCSPResponseCacheStats { c.mu.RLock() defer c.mu.RUnlock() if c.stats == nil { return nil } stats := OCSPResponseCacheStats{ Responses: c.stats.Responses, Hits: c.stats.Hits, Misses: c.stats.Misses, Revokes: c.stats.Revokes, Goods: c.stats.Goods, Unknowns: c.stats.Unknowns, } return &stats } func (c *LocalCache) initStats() { c.mu.Lock() defer c.mu.Unlock() c.stats = &OCSPResponseCacheStats{} c.stats.Hits = 0 c.stats.Misses = 0 c.stats.Responses = int64(len(c.cache)) for _, resp := range c.cache { switch resp.RespStatus { case ocsp.Good: c.stats.Goods++ case ocsp.Revoked: c.stats.Revokes++ case ocsp.Unknown: c.stats.Unknowns++ } } } func (c *LocalCache) Compress(buf []byte) ([]byte, error) { bodyLen := int64(len(buf)) var output bytes.Buffer writer := s2.NewWriter(&output) input := bytes.NewReader(buf[:bodyLen]) if n, err := io.CopyN(writer, input, bodyLen); err != nil { return nil, fmt.Errorf(certidp.ErrCannotWriteCompressed, err) } else if n != bodyLen { return nil, fmt.Errorf(certidp.ErrTruncatedWrite, n, bodyLen) } if err := writer.Close(); err != nil { return nil, fmt.Errorf(certidp.ErrCannotCloseWriter, err) } return output.Bytes(), nil } func (c *LocalCache) Decompress(buf []byte) ([]byte, error) { bodyLen := int64(len(buf)) input := bytes.NewReader(buf[:bodyLen]) reader := io.NopCloser(s2.NewReader(input)) output, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf(certidp.ErrCannotReadCompressed, err) } return output, reader.Close() } func (c *LocalCache) loadCache(s *Server) { d := s.opts.OCSPCacheConfig.LocalStore if d == _EMPTY_ { d = OCSPResponseCacheDefaultDir } f := OCSPResponseCacheDefaultFilename store, err := filepath.Abs(path.Join(d, f)) if err != nil { s.Errorf(certidp.ErrLoadCacheFail, err) return } s.Debugf(certidp.DbgLoadingCache, store) c.mu.Lock() defer c.mu.Unlock() c.cache = make(map[string]OCSPResponseCacheItem) dat, err := os.ReadFile(store) if err != nil { if errors.Is(err, os.ErrNotExist) { s.Debugf(certidp.DbgNoCacheFound) } else { s.Warnf(certidp.ErrLoadCacheFail, err) } return } err = json.Unmarshal(dat, &c.cache) if err != nil { // make sure clean cache c.cache = make(map[string]OCSPResponseCacheItem) s.Warnf(certidp.ErrLoadCacheFail, err) c.dirty = true return } c.dirty = false } func (c *LocalCache) saveCache(s *Server) { c.mu.RLock() dirty := c.dirty c.mu.RUnlock() if !dirty { return } s.Debugf(certidp.DbgCacheDirtySave) var d string if c.config.LocalStore != _EMPTY_ { d = c.config.LocalStore } else { d = OCSPResponseCacheDefaultDir } f := OCSPResponseCacheDefaultFilename store, err := filepath.Abs(path.Join(d, f)) if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } s.Debugf(certidp.DbgSavingCache, store) if _, err := os.Stat(d); os.IsNotExist(err) { err = os.Mkdir(d, defaultDirPerms) if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } } tmp, err := os.CreateTemp(d, OCSPResponseCacheDefaultTempFilePrefix) if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } defer func() { tmp.Close() os.Remove(tmp.Name()) }() // clean up any temp files // RW lock here because we're going to snapshot the cache to disk and mark as clean if successful c.mu.Lock() defer c.mu.Unlock() dat, err := json.MarshalIndent(c.cache, "", " ") if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } cacheSize, err := tmp.Write(dat) if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } err = tmp.Sync() if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } err = tmp.Close() if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } // do the final swap and overwrite any old saved peer cache err = os.Rename(tmp.Name(), store) if err != nil { s.Errorf(certidp.ErrSaveCacheFail, err) return } c.dirty = false s.Debugf(certidp.DbgCacheSaved, cacheSize) } var OCSPResponseCacheUsage = ` You may enable OCSP peer response cacheing at server configuration root level: (If no TLS blocks are configured with OCSP peer verification, ocsp_cache is ignored.) ... # short form enables with defaults ocsp_cache: true # if false or undefined and one or more TLS blocks are configured with OCSP peer verification, "none" is implied # long form includes settable options ocsp_cache { # Cache type (default local) type: local # Cache file directory for local-type cache (default _rc_ in current working directory) local_store: "_rc_" # Ignore cache deletes if cached OCSP response is Revoked status (default false) preserve_revoked: false # For local store, interval to save in-memory cache to disk in seconds (default 300 seconds, minimum 1 second) save_interval: 300 } ... Note: Cache of server's own OCSP response (staple) is enabled using the 'ocsp' configuration option. ` func (s *Server) initOCSPResponseCache() { // No mTLS OCSP or Leaf OCSP enablements, so no need to init cache s.mu.RLock() if !s.ocspPeerVerify { s.mu.RUnlock() return } s.mu.RUnlock() so := s.getOpts() if so.OCSPCacheConfig == nil { so.OCSPCacheConfig = NewOCSPResponseCacheConfig() } var cc = so.OCSPCacheConfig s.mu.Lock() defer s.mu.Unlock() switch cc.Type { case NONE: s.ocsprc = &NoOpCache{config: cc, online: true, mu: &sync.RWMutex{}} case LOCAL: c := &LocalCache{ config: cc, online: false, cache: make(map[string]OCSPResponseCacheItem), mu: &sync.RWMutex{}, dirty: false, } c.saveInterval = time.Duration(cc.SaveInterval) * time.Second c.timer = time.AfterFunc(c.saveInterval, func() { s.Debugf(certidp.DbgCacheSaveTimerExpired) c.saveCache(s) c.timer.Reset(c.saveInterval) }) s.ocsprc = c default: s.Fatalf(certidp.ErrBadCacheTypeConfig, cc.Type) } } func (s *Server) startOCSPResponseCache() { // No mTLS OCSP or Leaf OCSP enablements, so no need to start cache s.mu.RLock() if !s.ocspPeerVerify || s.ocsprc == nil { s.mu.RUnlock() return } s.mu.RUnlock() // Could be heavier operation depending on cache implementation s.ocsprc.Start(s) if s.ocsprc.Online() { s.Noticef(certidp.MsgCacheOnline, s.ocsprc.Type()) } else { s.Noticef(certidp.MsgCacheOffline, s.ocsprc.Type()) } } func (s *Server) stopOCSPResponseCache() { s.mu.RLock() if s.ocsprc == nil { s.mu.RUnlock() return } s.mu.RUnlock() s.ocsprc.Stop(s) } func parseOCSPResponseCache(v any) (pcfg *OCSPResponseCacheConfig, retError error) { var lt token defer convertPanicToError(<, &retError) tk, v := unwrapValue(v, <) cm, ok := v.(map[string]any) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalCacheOptsConfig, v)} } pcfg = NewOCSPResponseCacheConfig() retError = nil for mk, mv := range cm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "type": cache, ok := mv.(string) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} } cacheType, exists := OCSPResponseCacheTypeMap[strings.ToLower(cache)] if !exists { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrUnknownCacheType, cache)} } pcfg.Type = cacheType case "local_store": store, ok := mv.(string) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} } pcfg.LocalStore = store case "preserve_revoked": preserve, ok := mv.(bool) if !ok { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} } pcfg.PreserveRevoked = preserve case "save_interval": at := float64(0) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv case string: d, err := time.ParseDuration(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)} } at = d.Seconds() default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldTypeConversion, "unexpected type")} } si := time.Duration(at) * time.Second if si < OCSPResponseCacheMinimumSaveInterval { si = OCSPResponseCacheMinimumSaveInterval } pcfg.SaveInterval = si.Seconds() default: return nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)} } } return pcfg, nil } nats-server-2.10.27/server/opts.go000066400000000000000000005236201477524627100170050ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "crypto/tls" "crypto/x509" "errors" "flag" "fmt" "math" "net" "net/url" "os" "path" "path/filepath" "regexp" "runtime" "strconv" "strings" "sync/atomic" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/conf" "github.com/nats-io/nats-server/v2/server/certidp" "github.com/nats-io/nats-server/v2/server/certstore" "github.com/nats-io/nkeys" ) var allowUnknownTopLevelField = int32(0) // NoErrOnUnknownFields can be used to change the behavior the processing // of a configuration file. By default, an error is reported if unknown // fields are found. If `noError` is set to true, no error will be reported // if top-level unknown fields are found. func NoErrOnUnknownFields(noError bool) { var val int32 if noError { val = int32(1) } atomic.StoreInt32(&allowUnknownTopLevelField, val) } // PinnedCertSet is a set of lower case hex-encoded sha256 of DER encoded SubjectPublicKeyInfo type PinnedCertSet map[string]struct{} // ClusterOpts are options for clusters. // NOTE: This structure is no longer used for monitoring endpoints // and json tags are deprecated and may be removed in the future. type ClusterOpts struct { Name string `json:"-"` Host string `json:"addr,omitempty"` Port int `json:"cluster_port,omitempty"` Username string `json:"-"` Password string `json:"-"` AuthTimeout float64 `json:"auth_timeout,omitempty"` Permissions *RoutePermissions `json:"-"` TLSTimeout float64 `json:"-"` TLSConfig *tls.Config `json:"-"` TLSMap bool `json:"-"` TLSCheckKnownURLs bool `json:"-"` TLSPinnedCerts PinnedCertSet `json:"-"` ListenStr string `json:"-"` Advertise string `json:"-"` NoAdvertise bool `json:"-"` ConnectRetries int `json:"-"` PoolSize int `json:"-"` PinnedAccounts []string `json:"-"` Compression CompressionOpts `json:"-"` PingInterval time.Duration `json:"-"` MaxPingsOut int `json:"-"` // Not exported (used in tests) resolver netResolver // Snapshot of configured TLS options. tlsConfigOpts *TLSConfigOpts } // CompressionOpts defines the compression mode and optional configuration. type CompressionOpts struct { Mode string // If `Mode` is set to CompressionS2Auto, RTTThresholds provides the // thresholds at which the compression level will go from // CompressionS2Uncompressed to CompressionS2Fast, CompressionS2Better // or CompressionS2Best. If a given level is not desired, specify 0 // for this slot. For instance, the slice []{0, 10ms, 20ms} means that // for any RTT up to 10ms included the compression level will be // CompressionS2Fast, then from ]10ms..20ms], the level will be selected // as CompressionS2Better. Anything above 20ms will result in picking // the CompressionS2Best compression level. RTTThresholds []time.Duration } // GatewayOpts are options for gateways. // NOTE: This structure is no longer used for monitoring endpoints // and json tags are deprecated and may be removed in the future. type GatewayOpts struct { Name string `json:"name"` Host string `json:"addr,omitempty"` Port int `json:"port,omitempty"` Username string `json:"-"` Password string `json:"-"` AuthTimeout float64 `json:"auth_timeout,omitempty"` TLSConfig *tls.Config `json:"-"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSMap bool `json:"-"` TLSCheckKnownURLs bool `json:"-"` TLSPinnedCerts PinnedCertSet `json:"-"` Advertise string `json:"advertise,omitempty"` ConnectRetries int `json:"connect_retries,omitempty"` Gateways []*RemoteGatewayOpts `json:"gateways,omitempty"` RejectUnknown bool `json:"reject_unknown,omitempty"` // config got renamed to reject_unknown_cluster // Not exported, for tests. resolver netResolver sendQSubsBufSize int // Snapshot of configured TLS options. tlsConfigOpts *TLSConfigOpts } // RemoteGatewayOpts are options for connecting to a remote gateway // NOTE: This structure is no longer used for monitoring endpoints // and json tags are deprecated and may be removed in the future. type RemoteGatewayOpts struct { Name string `json:"name"` TLSConfig *tls.Config `json:"-"` TLSTimeout float64 `json:"tls_timeout,omitempty"` URLs []*url.URL `json:"urls,omitempty"` tlsConfigOpts *TLSConfigOpts } // LeafNodeOpts are options for a given server to accept leaf node connections and/or connect to a remote cluster. type LeafNodeOpts struct { Host string `json:"addr,omitempty"` Port int `json:"port,omitempty"` Username string `json:"-"` Password string `json:"-"` Nkey string `json:"-"` Account string `json:"-"` Users []*User `json:"-"` AuthTimeout float64 `json:"auth_timeout,omitempty"` TLSConfig *tls.Config `json:"-"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSMap bool `json:"-"` TLSPinnedCerts PinnedCertSet `json:"-"` TLSHandshakeFirst bool `json:"-"` Advertise string `json:"-"` NoAdvertise bool `json:"-"` ReconnectInterval time.Duration `json:"-"` // Compression options Compression CompressionOpts `json:"-"` // For solicited connections to other clusters/superclusters. Remotes []*RemoteLeafOpts `json:"remotes,omitempty"` // This is the minimum version that is accepted for remote connections. // Note that since the server version in the CONNECT protocol was added // only starting at v2.8.0, any version below that will be rejected // (since empty version string in CONNECT would fail the "version at // least" test). MinVersion string // Not exported, for tests. resolver netResolver dialTimeout time.Duration connDelay time.Duration // Snapshot of configured TLS options. tlsConfigOpts *TLSConfigOpts } // SignatureHandler is used to sign a nonce from the server while // authenticating with Nkeys. The callback should sign the nonce and // return the JWT and the raw signature. type SignatureHandler func([]byte) (string, []byte, error) // RemoteLeafOpts are options for connecting to a remote server as a leaf node. type RemoteLeafOpts struct { LocalAccount string `json:"local_account,omitempty"` NoRandomize bool `json:"-"` URLs []*url.URL `json:"urls,omitempty"` Credentials string `json:"-"` Nkey string `json:"-"` SignatureCB SignatureHandler `json:"-"` TLS bool `json:"-"` TLSConfig *tls.Config `json:"-"` TLSTimeout float64 `json:"tls_timeout,omitempty"` TLSHandshakeFirst bool `json:"-"` Hub bool `json:"hub,omitempty"` DenyImports []string `json:"-"` DenyExports []string `json:"-"` // FirstInfoTimeout is the amount of time the server will wait for the // initial INFO protocol from the remote server before closing the // connection. FirstInfoTimeout time.Duration `json:"-"` // Compression options for this remote. Each remote could have a different // setting and also be different from the LeafNode options. Compression CompressionOpts `json:"-"` // When an URL has the "ws" (or "wss") scheme, then the server will initiate the // connection as a websocket connection. By default, the websocket frames will be // masked (as if this server was a websocket client to the remote server). The // NoMasking option will change this behavior and will send umasked frames. Websocket struct { Compression bool `json:"-"` NoMasking bool `json:"-"` } tlsConfigOpts *TLSConfigOpts // If we are clustered and our local account has JetStream, if apps are accessing // a stream or consumer leader through this LN and it gets dropped, the apps will // not be able to work. This tells the system to migrate the leaders away from this server. // This only changes leader for R>1 assets. JetStreamClusterMigrate bool `json:"jetstream_cluster_migrate,omitempty"` } type JSLimitOpts struct { MaxRequestBatch int MaxAckPending int MaxHAAssets int Duplicates time.Duration } // AuthCallout option used to map external AuthN to NATS based AuthZ. type AuthCallout struct { // Must be a public account Nkey. Issuer string // Account to be used for sending requests. Account string // Users that will bypass auth_callout and be used for the auth service itself. AuthUsers []string // XKey is a public xkey for the authorization service. // This will enable encryption for server requests and the authorization service responses. XKey string } // Options block for nats-server. // NOTE: This structure is no longer used for monitoring endpoints // and json tags are deprecated and may be removed in the future. type Options struct { ConfigFile string `json:"-"` ServerName string `json:"server_name"` Host string `json:"addr"` Port int `json:"port"` DontListen bool `json:"dont_listen"` ClientAdvertise string `json:"-"` Trace bool `json:"-"` Debug bool `json:"-"` TraceVerbose bool `json:"-"` NoLog bool `json:"-"` NoSigs bool `json:"-"` NoSublistCache bool `json:"-"` NoHeaderSupport bool `json:"-"` DisableShortFirstPing bool `json:"-"` Logtime bool `json:"-"` LogtimeUTC bool `json:"-"` MaxConn int `json:"max_connections"` MaxSubs int `json:"max_subscriptions,omitempty"` MaxSubTokens uint8 `json:"-"` Nkeys []*NkeyUser `json:"-"` Users []*User `json:"-"` Accounts []*Account `json:"-"` NoAuthUser string `json:"-"` SystemAccount string `json:"-"` NoSystemAccount bool `json:"-"` Username string `json:"-"` Password string `json:"-"` Authorization string `json:"-"` AuthCallout *AuthCallout `json:"-"` PingInterval time.Duration `json:"ping_interval"` MaxPingsOut int `json:"ping_max"` HTTPHost string `json:"http_host"` HTTPPort int `json:"http_port"` HTTPBasePath string `json:"http_base_path"` HTTPSPort int `json:"https_port"` AuthTimeout float64 `json:"auth_timeout"` MaxControlLine int32 `json:"max_control_line"` MaxPayload int32 `json:"max_payload"` MaxPending int64 `json:"max_pending"` NoFastProducerStall bool `json:"-"` Cluster ClusterOpts `json:"cluster,omitempty"` Gateway GatewayOpts `json:"gateway,omitempty"` LeafNode LeafNodeOpts `json:"leaf,omitempty"` JetStream bool `json:"jetstream"` JetStreamMaxMemory int64 `json:"-"` JetStreamMaxStore int64 `json:"-"` JetStreamDomain string `json:"-"` JetStreamExtHint string `json:"-"` JetStreamKey string `json:"-"` JetStreamOldKey string `json:"-"` JetStreamCipher StoreCipher `json:"-"` JetStreamUniqueTag string JetStreamLimits JSLimitOpts JetStreamMaxCatchup int64 JetStreamRequestQueueLimit int64 StoreDir string `json:"-"` SyncInterval time.Duration `json:"-"` SyncAlways bool `json:"-"` JsAccDefaultDomain map[string]string `json:"-"` // account to domain name mapping Websocket WebsocketOpts `json:"-"` MQTT MQTTOpts `json:"-"` ProfPort int `json:"-"` ProfBlockRate int `json:"-"` PidFile string `json:"-"` PortsFileDir string `json:"-"` LogFile string `json:"-"` LogSizeLimit int64 `json:"-"` LogMaxFiles int64 `json:"-"` Syslog bool `json:"-"` RemoteSyslog string `json:"-"` Routes []*url.URL `json:"-"` RoutesStr string `json:"-"` TLSTimeout float64 `json:"tls_timeout"` TLS bool `json:"-"` TLSVerify bool `json:"-"` TLSMap bool `json:"-"` TLSCert string `json:"-"` TLSKey string `json:"-"` TLSCaCert string `json:"-"` TLSConfig *tls.Config `json:"-"` TLSPinnedCerts PinnedCertSet `json:"-"` TLSRateLimit int64 `json:"-"` // When set to true, the server will perform the TLS handshake before // sending the INFO protocol. For clients that are not configured // with a similar option, their connection will fail with some sort // of timeout or EOF error since they are expecting to receive an // INFO protocol first. TLSHandshakeFirst bool `json:"-"` // If TLSHandshakeFirst is true and this value is strictly positive, // the server will wait for that amount of time for the TLS handshake // to start before falling back to previous behavior of sending the // INFO protocol first. It allows for a mix of newer clients that can // require a TLS handshake first, and older clients that can't. TLSHandshakeFirstFallback time.Duration `json:"-"` AllowNonTLS bool `json:"-"` WriteDeadline time.Duration `json:"-"` MaxClosedClients int `json:"-"` LameDuckDuration time.Duration `json:"-"` LameDuckGracePeriod time.Duration `json:"-"` // MaxTracedMsgLen is the maximum printable length for traced messages. MaxTracedMsgLen int `json:"-"` // Operating a trusted NATS server TrustedKeys []string `json:"-"` TrustedOperators []*jwt.OperatorClaims `json:"-"` AccountResolver AccountResolver `json:"-"` AccountResolverTLSConfig *tls.Config `json:"-"` // AlwaysEnableNonce will always present a nonce to new connections // typically used by custom Authentication implementations who embeds // the server and so not presented as a configuration option AlwaysEnableNonce bool CustomClientAuthentication Authentication `json:"-"` CustomRouterAuthentication Authentication `json:"-"` // CheckConfig configuration file syntax test was successful and exit. CheckConfig bool `json:"-"` // DisableJetStreamBanner will not print the ascii art on startup for JetStream enabled servers DisableJetStreamBanner bool `json:"-"` // ConnectErrorReports specifies the number of failed attempts // at which point server should report the failure of an initial // connection to a route, gateway or leaf node. // See DEFAULT_CONNECT_ERROR_REPORTS for default value. ConnectErrorReports int // ReconnectErrorReports is similar to ConnectErrorReports except // that this applies to reconnect events. ReconnectErrorReports int // Tags describing the server. They will be included in varz // and used as a filter criteria for some system requests. Tags jwt.TagList `json:"-"` // OCSPConfig enables OCSP Stapling in the server. OCSPConfig *OCSPConfig tlsConfigOpts *TLSConfigOpts // private fields, used to know if bool options are explicitly // defined in config and/or command line params. inConfig map[string]bool inCmdLine map[string]bool // private fields for operator mode operatorJWT []string resolverPreloads map[string]string resolverPinnedAccounts map[string]struct{} // private fields, used for testing gatewaysSolicitDelay time.Duration routeProto int // JetStream maxMemSet bool maxStoreSet bool syncSet bool // OCSP Cache config enables next-gen cache for OCSP features OCSPCacheConfig *OCSPResponseCacheConfig // Used to mark that we had a top level authorization block. authBlockDefined bool } // WebsocketOpts are options for websocket type WebsocketOpts struct { // The server will accept websocket client connections on this hostname/IP. Host string // The server will accept websocket client connections on this port. Port int // The host:port to advertise to websocket clients in the cluster. Advertise string // If no user name is provided when a client connects, will default to the // matching user from the global list of users in `Options.Users`. NoAuthUser string // Name of the cookie, which if present in WebSocket upgrade headers, // will be treated as JWT during CONNECT phase as long as // "jwt" specified in the CONNECT options is missing or empty. JWTCookie string // Authentication section. If anything is configured in this section, // it will override the authorization configuration of regular clients. Username string Password string Token string // Timeout for the authentication process. AuthTimeout float64 // By default the server will enforce the use of TLS. If no TLS configuration // is provided, you need to explicitly set NoTLS to true to allow the server // to start without TLS configuration. Note that if a TLS configuration is // present, this boolean is ignored and the server will run the Websocket // server with that TLS configuration. // Running without TLS is less secure since Websocket clients that use bearer // tokens will send them in clear. So this should not be used in production. NoTLS bool // TLS configuration is required. TLSConfig *tls.Config // If true, map certificate values for authentication purposes. TLSMap bool // When present, accepted client certificates (verify/verify_and_map) must be in this list TLSPinnedCerts PinnedCertSet // If true, the Origin header must match the request's host. SameOrigin bool // Only origins in this list will be accepted. If empty and // SameOrigin is false, any origin is accepted. AllowedOrigins []string // If set to true, the server will negotiate with clients // if compression can be used. If this is false, no compression // will be used (both in server and clients) since it has to // be negotiated between both endpoints Compression bool // Total time allowed for the server to read the client request // and write the response back to the client. This include the // time needed for the TLS Handshake. HandshakeTimeout time.Duration // Snapshot of configured TLS options. tlsConfigOpts *TLSConfigOpts } // MQTTOpts are options for MQTT type MQTTOpts struct { // The server will accept MQTT client connections on this hostname/IP. Host string // The server will accept MQTT client connections on this port. Port int // If no user name is provided when a client connects, will default to the // matching user from the global list of users in `Options.Users`. NoAuthUser string // Authentication section. If anything is configured in this section, // it will override the authorization configuration of regular clients. Username string Password string Token string // JetStream domain mqtt is supposed to pick up JsDomain string // Number of replicas for MQTT streams. // Negative or 0 value means that the server(s) will pick a replica // number based on the known size of the cluster (but capped at 3). // Note that if an account was already connected, the stream's replica // count is not modified. Use the NATS CLI to update the count if desired. StreamReplicas int // Number of replicas for MQTT consumers. // Negative or 0 value means that there is no override and the consumer // will have the same replica factor that the stream it belongs to. // If a value is specified, it will require to be lower than the stream // replicas count (lower than StreamReplicas if specified, but also lower // than the automatic value determined by cluster size). // Note that existing consumers are not modified. // // UPDATE: This is no longer used while messages stream has interest policy retention // which requires consumer replica count to match the parent stream. ConsumerReplicas int // Indicate if the consumers should be created with memory storage. // Note that existing consumers are not modified. ConsumerMemoryStorage bool // If specified will have the system auto-cleanup the consumers after being // inactive for the specified amount of time. ConsumerInactiveThreshold time.Duration // Timeout for the authentication process. AuthTimeout float64 // TLS configuration is required. TLSConfig *tls.Config // If true, map certificate values for authentication purposes. TLSMap bool // Timeout for the TLS handshake TLSTimeout float64 // Set of allowable certificates TLSPinnedCerts PinnedCertSet // AckWait is the amount of time after which a QoS 1 or 2 message sent to a // client is redelivered as a DUPLICATE if the server has not received the // PUBACK on the original Packet Identifier. The same value applies to // PubRel redelivery. The value has to be positive. Zero will cause the // server to use the default value (30 seconds). Note that changes to this // option is applied only to new MQTT subscriptions (or sessions for // PubRels). AckWait time.Duration // MaxAckPending is the amount of QoS 1 and 2 messages (combined) the server // can send to a subscription without receiving any PUBACK for those // messages. The valid range is [0..65535]. // // The total of subscriptions' MaxAckPending on a given session cannot // exceed 65535. Attempting to create a subscription that would bring the // total above the limit would result in the server returning 0x80 in the // SUBACK for this subscription. // // Due to how the NATS Server handles the MQTT "#" wildcard, each // subscription ending with "#" will use 2 times the MaxAckPending value. // Note that changes to this option is applied only to new subscriptions. MaxAckPending uint16 // Snapshot of configured TLS options. tlsConfigOpts *TLSConfigOpts // rejectQoS2Pub tells the MQTT client to not accept QoS2 PUBLISH, instead // error and terminate the connection. rejectQoS2Pub bool // downgradeQOS2Sub tells the MQTT client to downgrade QoS2 SUBSCRIBE // requests to QoS1. downgradeQoS2Sub bool } type netResolver interface { LookupHost(ctx context.Context, host string) ([]string, error) } // Clone performs a deep copy of the Options struct, returning a new clone // with all values copied. func (o *Options) Clone() *Options { if o == nil { return nil } clone := &Options{} *clone = *o if o.Users != nil { clone.Users = make([]*User, len(o.Users)) for i, user := range o.Users { clone.Users[i] = user.clone() } } if o.Nkeys != nil { clone.Nkeys = make([]*NkeyUser, len(o.Nkeys)) for i, nkey := range o.Nkeys { clone.Nkeys[i] = nkey.clone() } } if o.Routes != nil { clone.Routes = deepCopyURLs(o.Routes) } if o.TLSConfig != nil { clone.TLSConfig = o.TLSConfig.Clone() } if o.Cluster.TLSConfig != nil { clone.Cluster.TLSConfig = o.Cluster.TLSConfig.Clone() } if o.Gateway.TLSConfig != nil { clone.Gateway.TLSConfig = o.Gateway.TLSConfig.Clone() } if len(o.Gateway.Gateways) > 0 { clone.Gateway.Gateways = make([]*RemoteGatewayOpts, len(o.Gateway.Gateways)) for i, g := range o.Gateway.Gateways { clone.Gateway.Gateways[i] = g.clone() } } // FIXME(dlc) - clone leaf node stuff. return clone } func deepCopyURLs(urls []*url.URL) []*url.URL { if urls == nil { return nil } curls := make([]*url.URL, len(urls)) for i, u := range urls { cu := &url.URL{} *cu = *u curls[i] = cu } return curls } // Configuration file authorization section. type authorization struct { // Singles user string pass string token string nkey string acc string // Multiple Nkeys/Users nkeys []*NkeyUser users []*User timeout float64 defaultPermissions *Permissions // Auth Callouts callout *AuthCallout } // TLSConfigOpts holds the parsed tls config information, // used with flag parsing type TLSConfigOpts struct { CertFile string KeyFile string CaFile string Verify bool Insecure bool Map bool TLSCheckKnownURLs bool HandshakeFirst bool // Indicate that the TLS handshake should occur first, before sending the INFO protocol. FallbackDelay time.Duration // Where supported, indicates how long to wait for the handshake before falling back to sending the INFO protocol first. Timeout float64 RateLimit int64 Ciphers []uint16 CurvePreferences []tls.CurveID PinnedCerts PinnedCertSet CertStore certstore.StoreType CertMatchBy certstore.MatchByType CertMatch string CertMatchSkipInvalid bool CaCertsMatch []string OCSPPeerConfig *certidp.OCSPPeerConfig Certificates []*TLSCertPairOpt MinVersion uint16 } // TLSCertPairOpt are the paths to a certificate and private key. type TLSCertPairOpt struct { CertFile string KeyFile string } // OCSPConfig represents the options of OCSP stapling options. type OCSPConfig struct { // Mode defines the policy for OCSP stapling. Mode OCSPMode // OverrideURLs is the http URL endpoint used to get OCSP staples. OverrideURLs []string } var tlsUsage = ` TLS configuration is specified in the tls section of a configuration file: e.g. tls { cert_file: "./certs/server-cert.pem" key_file: "./certs/server-key.pem" ca_file: "./certs/ca.pem" verify: true verify_and_map: true cipher_suites: [ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" ] curve_preferences: [ "CurveP256", "CurveP384", "CurveP521" ] } Available cipher suites include: ` // ProcessConfigFile processes a configuration file. // FIXME(dlc): A bit hacky func ProcessConfigFile(configFile string) (*Options, error) { opts := &Options{} if err := opts.ProcessConfigFile(configFile); err != nil { // If only warnings then continue and return the options. if cerr, ok := err.(*processConfigErr); ok && len(cerr.Errors()) == 0 { return opts, nil } return nil, err } return opts, nil } // token is an item parsed from the configuration. type token interface { Value() any Line() int IsUsedVariable() bool SourceFile() string Position() int } // unwrapValue can be used to get the token and value from an item // to be able to report the line number in case of an incorrect // configuration. // also stores the token in lastToken for use in convertPanicToError func unwrapValue(v any, lastToken *token) (token, any) { switch tk := v.(type) { case token: if lastToken != nil { *lastToken = tk } return tk, tk.Value() default: return nil, v } } // use in defer to recover from panic and turn it into an error associated with last token func convertPanicToErrorList(lastToken *token, errors *[]error) { // only recover if an error can be stored if errors == nil { return } else if err := recover(); err == nil { return } else if lastToken != nil && *lastToken != nil { *errors = append(*errors, &configErr{*lastToken, fmt.Sprint(err)}) } else { *errors = append(*errors, fmt.Errorf("encountered panic without a token %v", err)) } } // use in defer to recover from panic and turn it into an error associated with last token func convertPanicToError(lastToken *token, e *error) { // only recover if an error can be stored if e == nil || *e != nil { return } else if err := recover(); err == nil { return } else if lastToken != nil && *lastToken != nil { *e = &configErr{*lastToken, fmt.Sprint(err)} } else { *e = fmt.Errorf("%v", err) } } // configureSystemAccount configures a system account // if present in the configuration. func configureSystemAccount(o *Options, m map[string]any) (retErr error) { var lt token defer convertPanicToError(<, &retErr) configure := func(v any) error { tk, v := unwrapValue(v, <) sa, ok := v.(string) if !ok { return &configErr{tk, "system account name must be a string"} } o.SystemAccount = sa return nil } if v, ok := m["system_account"]; ok { return configure(v) } else if v, ok := m["system"]; ok { return configure(v) } return nil } // ProcessConfigFile updates the Options structure with options // present in the given configuration file. // This version is convenient if one wants to set some default // options and then override them with what is in the config file. // For instance, this version allows you to do something such as: // // opts := &Options{Debug: true} // opts.ProcessConfigFile(myConfigFile) // // If the config file contains "debug: false", after this call, // opts.Debug would really be false. It would be impossible to // achieve that with the non receiver ProcessConfigFile() version, // since one would not know after the call if "debug" was not present // or was present but set to false. func (o *Options) ProcessConfigFile(configFile string) error { o.ConfigFile = configFile if configFile == _EMPTY_ { return nil } m, err := conf.ParseFileWithChecks(configFile) if err != nil { return err } // Collect all errors and warnings and report them all together. errors := make([]error, 0) warnings := make([]error, 0) if len(m) == 0 { warnings = append(warnings, fmt.Errorf("%s: config has no values or is empty", configFile)) } // First check whether a system account has been defined, // as that is a condition for other features to be enabled. if err := configureSystemAccount(o, m); err != nil { errors = append(errors, err) } for k, v := range m { o.processConfigFileLine(k, v, &errors, &warnings) } if len(errors) > 0 || len(warnings) > 0 { return &processConfigErr{ errors: errors, warnings: warnings, } } return nil } func (o *Options) processConfigFileLine(k string, v any, errors *[]error, warnings *[]error) { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) switch strings.ToLower(k) { case "listen": hp, err := parseListen(v) if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) return } o.Host = hp.host o.Port = hp.port case "client_advertise": o.ClientAdvertise = v.(string) case "port": o.Port = int(v.(int64)) case "server_name": o.ServerName = v.(string) case "host", "net": o.Host = v.(string) case "debug": o.Debug = v.(bool) trackExplicitVal(&o.inConfig, "Debug", o.Debug) case "trace": o.Trace = v.(bool) trackExplicitVal(&o.inConfig, "Trace", o.Trace) case "trace_verbose": o.TraceVerbose = v.(bool) o.Trace = v.(bool) trackExplicitVal(&o.inConfig, "TraceVerbose", o.TraceVerbose) trackExplicitVal(&o.inConfig, "Trace", o.Trace) case "logtime": o.Logtime = v.(bool) trackExplicitVal(&o.inConfig, "Logtime", o.Logtime) case "logtime_utc": o.LogtimeUTC = v.(bool) trackExplicitVal(&o.inConfig, "LogtimeUTC", o.LogtimeUTC) case "mappings", "maps": gacc := NewAccount(globalAccountName) o.Accounts = append(o.Accounts, gacc) err := parseAccountMappings(tk, gacc, errors) if err != nil { *errors = append(*errors, err) return } case "disable_sublist_cache", "no_sublist_cache": o.NoSublistCache = v.(bool) case "accounts": err := parseAccounts(tk, o, errors, warnings) if err != nil { *errors = append(*errors, err) return } case "authorization": auth, err := parseAuthorization(tk, errors) if err != nil { *errors = append(*errors, err) return } o.authBlockDefined = true o.Username = auth.user o.Password = auth.pass o.Authorization = auth.token o.AuthTimeout = auth.timeout o.AuthCallout = auth.callout if (auth.user != _EMPTY_ || auth.pass != _EMPTY_) && auth.token != _EMPTY_ { err := &configErr{tk, "Cannot have a user/pass and token"} *errors = append(*errors, err) return } // In case parseAccounts() was done first, we need to check for duplicates. unames := setupUsersAndNKeysDuplicateCheckMap(o) // Check for multiple users defined. // Note: auth.users will be != nil as long as `users: []` is present // in the authorization block, even if empty, and will also account for // nkey users. We also check for users/nkeys that may have been already // added in parseAccounts() (which means they will be in unames) if auth.users != nil || len(unames) > 0 { if auth.user != _EMPTY_ { err := &configErr{tk, "Can not have a single user/pass and a users array"} *errors = append(*errors, err) return } if auth.token != _EMPTY_ { err := &configErr{tk, "Can not have a token and a users array"} *errors = append(*errors, err) return } // Now check that if we have users, there is no duplicate, including // users that may have been configured in parseAccounts(). if len(auth.users) > 0 { for _, u := range auth.users { if _, ok := unames[u.Username]; ok { err := &configErr{tk, fmt.Sprintf("Duplicate user %q detected", u.Username)} *errors = append(*errors, err) return } unames[u.Username] = struct{}{} } // Users may have been added from Accounts parsing, so do an append here o.Users = append(o.Users, auth.users...) } } // Check for nkeys if len(auth.nkeys) > 0 { for _, u := range auth.nkeys { if _, ok := unames[u.Nkey]; ok { err := &configErr{tk, fmt.Sprintf("Duplicate nkey %q detected", u.Nkey)} *errors = append(*errors, err) return } unames[u.Nkey] = struct{}{} } // NKeys may have been added from Accounts parsing, so do an append here o.Nkeys = append(o.Nkeys, auth.nkeys...) } case "http": hp, err := parseListen(v) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) return } o.HTTPHost = hp.host o.HTTPPort = hp.port case "https": hp, err := parseListen(v) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) return } o.HTTPHost = hp.host o.HTTPSPort = hp.port case "http_port", "monitor_port": o.HTTPPort = int(v.(int64)) case "https_port": o.HTTPSPort = int(v.(int64)) case "http_base_path": o.HTTPBasePath = v.(string) case "cluster": err := parseCluster(tk, o, errors, warnings) if err != nil { *errors = append(*errors, err) return } case "gateway": if err := parseGateway(tk, o, errors, warnings); err != nil { *errors = append(*errors, err) return } case "leaf", "leafnodes": err := parseLeafNodes(tk, o, errors, warnings) if err != nil { *errors = append(*errors, err) return } case "store_dir", "storedir": // Check if JetStream configuration is also setting the storage directory. if o.StoreDir != _EMPTY_ { *errors = append(*errors, &configErr{tk, "Duplicate 'store_dir' configuration"}) return } o.StoreDir = v.(string) case "jetstream": err := parseJetStream(tk, o, errors, warnings) if err != nil { *errors = append(*errors, err) return } case "logfile", "log_file": o.LogFile = v.(string) case "logfile_size_limit", "log_size_limit": o.LogSizeLimit = v.(int64) case "logfile_max_num", "log_max_num": o.LogMaxFiles = v.(int64) case "syslog": o.Syslog = v.(bool) trackExplicitVal(&o.inConfig, "Syslog", o.Syslog) case "remote_syslog": o.RemoteSyslog = v.(string) case "pidfile", "pid_file": o.PidFile = v.(string) case "ports_file_dir": o.PortsFileDir = v.(string) case "prof_port": o.ProfPort = int(v.(int64)) case "prof_block_rate": o.ProfBlockRate = int(v.(int64)) case "max_control_line": if v.(int64) > 1<<31-1 { err := &configErr{tk, fmt.Sprintf("%s value is too big", k)} *errors = append(*errors, err) return } o.MaxControlLine = int32(v.(int64)) case "max_payload": if v.(int64) > 1<<31-1 { err := &configErr{tk, fmt.Sprintf("%s value is too big", k)} *errors = append(*errors, err) return } o.MaxPayload = int32(v.(int64)) case "max_pending": o.MaxPending = v.(int64) case "max_connections", "max_conn": o.MaxConn = int(v.(int64)) case "max_traced_msg_len": o.MaxTracedMsgLen = int(v.(int64)) case "max_subscriptions", "max_subs": o.MaxSubs = int(v.(int64)) case "max_sub_tokens", "max_subscription_tokens": if n := v.(int64); n > math.MaxUint8 { err := &configErr{tk, fmt.Sprintf("%s value is too big", k)} *errors = append(*errors, err) return } else if n <= 0 { err := &configErr{tk, fmt.Sprintf("%s value can not be negative", k)} *errors = append(*errors, err) return } else { o.MaxSubTokens = uint8(n) } case "ping_interval": o.PingInterval = parseDuration("ping_interval", tk, v, errors, warnings) case "ping_max": o.MaxPingsOut = int(v.(int64)) case "tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) return } if o.TLSConfig, err = GenTLSConfig(tc); err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) return } o.TLSTimeout = tc.Timeout o.TLSMap = tc.Map o.TLSPinnedCerts = tc.PinnedCerts o.TLSRateLimit = tc.RateLimit o.TLSHandshakeFirst = tc.HandshakeFirst o.TLSHandshakeFirstFallback = tc.FallbackDelay // Need to keep track of path of the original TLS config // and certs path for OCSP Stapling monitoring. o.tlsConfigOpts = tc case "ocsp": switch vv := v.(type) { case bool: if vv { // Default is Auto which honors Must Staple status request // but does not shutdown the server in case it is revoked, // letting the client choose whether to trust or not the server. o.OCSPConfig = &OCSPConfig{Mode: OCSPModeAuto} } else { o.OCSPConfig = &OCSPConfig{Mode: OCSPModeNever} } case map[string]any: ocsp := &OCSPConfig{Mode: OCSPModeAuto} for kk, kv := range vv { _, v = unwrapValue(kv, &tk) switch kk { case "mode": mode := v.(string) switch { case strings.EqualFold(mode, "always"): ocsp.Mode = OCSPModeAlways case strings.EqualFold(mode, "must"): ocsp.Mode = OCSPModeMust case strings.EqualFold(mode, "never"): ocsp.Mode = OCSPModeNever case strings.EqualFold(mode, "auto"): ocsp.Mode = OCSPModeAuto default: *errors = append(*errors, &configErr{tk, fmt.Sprintf("error parsing ocsp config: unsupported ocsp mode %T", mode)}) } case "urls": urls := v.([]string) ocsp.OverrideURLs = urls case "url": url := v.(string) ocsp.OverrideURLs = []string{url} default: *errors = append(*errors, &configErr{tk, fmt.Sprintf("error parsing ocsp config: unsupported field %T", kk)}) return } } o.OCSPConfig = ocsp default: *errors = append(*errors, &configErr{tk, fmt.Sprintf("error parsing ocsp config: unsupported type %T", v)}) return } case "allow_non_tls": o.AllowNonTLS = v.(bool) case "write_deadline": o.WriteDeadline = parseDuration("write_deadline", tk, v, errors, warnings) case "lame_duck_duration": dur, err := time.ParseDuration(v.(string)) if err != nil { err := &configErr{tk, fmt.Sprintf("error parsing lame_duck_duration: %v", err)} *errors = append(*errors, err) return } if dur < 30*time.Second { err := &configErr{tk, fmt.Sprintf("invalid lame_duck_duration of %v, minimum is 30 seconds", dur)} *errors = append(*errors, err) return } o.LameDuckDuration = dur case "lame_duck_grace_period": dur, err := time.ParseDuration(v.(string)) if err != nil { err := &configErr{tk, fmt.Sprintf("error parsing lame_duck_grace_period: %v", err)} *errors = append(*errors, err) return } if dur < 0 { err := &configErr{tk, "invalid lame_duck_grace_period, needs to be positive"} *errors = append(*errors, err) return } o.LameDuckGracePeriod = dur case "operator", "operators", "roots", "root", "root_operators", "root_operator": opFiles := []string{} switch v := v.(type) { case string: opFiles = append(opFiles, v) case []string: opFiles = append(opFiles, v...) case []any: for _, t := range v { if token, ok := t.(token); ok { if v, ok := token.Value().(string); ok { opFiles = append(opFiles, v) } else { err := &configErr{tk, fmt.Sprintf("error parsing operators: unsupported type %T where string is expected", token)} *errors = append(*errors, err) break } } else { err := &configErr{tk, fmt.Sprintf("error parsing operators: unsupported type %T", t)} *errors = append(*errors, err) break } } default: err := &configErr{tk, fmt.Sprintf("error parsing operators: unsupported type %T", v)} *errors = append(*errors, err) } // Assume for now these are file names, but they can also be the JWT itself inline. o.TrustedOperators = make([]*jwt.OperatorClaims, 0, len(opFiles)) for _, fname := range opFiles { theJWT, opc, err := readOperatorJWT(fname) if err != nil { err := &configErr{tk, fmt.Sprintf("error parsing operator JWT: %v", err)} *errors = append(*errors, err) continue } o.operatorJWT = append(o.operatorJWT, theJWT) o.TrustedOperators = append(o.TrustedOperators, opc) } if len(o.TrustedOperators) == 1 { // In case "resolver" is defined as well, it takes precedence if o.AccountResolver == nil { if accUrl, err := parseURL(o.TrustedOperators[0].AccountServerURL, "account resolver"); err == nil { // nsc automatically appends "/accounts" during nsc push o.AccountResolver, _ = NewURLAccResolver(accUrl.String() + "/accounts") } } // In case "system_account" is defined as well, it takes precedence if o.SystemAccount == _EMPTY_ { o.SystemAccount = o.TrustedOperators[0].SystemAccount } } case "resolver", "account_resolver", "accounts_resolver": switch v := v.(type) { case string: // "resolver" takes precedence over value obtained from "operator". // Clear so that parsing errors are not silently ignored. o.AccountResolver = nil memResolverRe := regexp.MustCompile(`(?i)(MEM|MEMORY)\s*`) resolverRe := regexp.MustCompile(`(?i)(?:URL){1}(?:\({1}\s*"?([^\s"]*)"?\s*\){1})?\s*`) if memResolverRe.MatchString(v) { o.AccountResolver = &MemAccResolver{} } else if items := resolverRe.FindStringSubmatch(v); len(items) == 2 { url := items[1] _, err := parseURL(url, "account resolver") if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) return } if ur, err := NewURLAccResolver(url); err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) return } else { o.AccountResolver = ur } } case map[string]any: del := false hdel := false hdel_set := false dir := _EMPTY_ dirType := _EMPTY_ limit := int64(0) ttl := time.Duration(0) sync := time.Duration(0) opts := []DirResOption{} var err error if v, ok := v["dir"]; ok { _, v := unwrapValue(v, <) dir = v.(string) } if v, ok := v["type"]; ok { _, v := unwrapValue(v, <) dirType = v.(string) } if v, ok := v["allow_delete"]; ok { _, v := unwrapValue(v, <) del = v.(bool) } if v, ok := v["hard_delete"]; ok { _, v := unwrapValue(v, <) hdel_set = true hdel = v.(bool) } if v, ok := v["limit"]; ok { _, v := unwrapValue(v, <) limit = v.(int64) } if v, ok := v["ttl"]; ok { _, v := unwrapValue(v, <) ttl, err = time.ParseDuration(v.(string)) } if v, ok := v["interval"]; err == nil && ok { _, v := unwrapValue(v, <) sync, err = time.ParseDuration(v.(string)) } if v, ok := v["timeout"]; err == nil && ok { _, v := unwrapValue(v, <) var to time.Duration if to, err = time.ParseDuration(v.(string)); err == nil { opts = append(opts, FetchTimeout(to)) } } if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) return } checkDir := func() { if dir == _EMPTY_ { *errors = append(*errors, &configErr{tk, "dir has no value and needs to point to a directory"}) return } if info, _ := os.Stat(dir); info != nil && (!info.IsDir() || info.Mode().Perm()&(1<<(uint(7))) == 0) { *errors = append(*errors, &configErr{tk, "dir needs to point to an accessible directory"}) return } } var res AccountResolver switch strings.ToUpper(dirType) { case "CACHE": checkDir() if sync != 0 { *errors = append(*errors, &configErr{tk, "CACHE does not accept sync"}) } if del { *errors = append(*errors, &configErr{tk, "CACHE does not accept allow_delete"}) } if hdel_set { *errors = append(*errors, &configErr{tk, "CACHE does not accept hard_delete"}) } res, err = NewCacheDirAccResolver(dir, limit, ttl, opts...) case "FULL": checkDir() if ttl != 0 { *errors = append(*errors, &configErr{tk, "FULL does not accept ttl"}) } if hdel_set && !del { *errors = append(*errors, &configErr{tk, "hard_delete has no effect without delete"}) } delete := NoDelete if del { if hdel { delete = HardDelete } else { delete = RenameDeleted } } res, err = NewDirAccResolver(dir, limit, sync, delete, opts...) case "MEM", "MEMORY": res = &MemAccResolver{} } if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) return } o.AccountResolver = res default: err := &configErr{tk, fmt.Sprintf("error parsing operator resolver, wrong type %T", v)} *errors = append(*errors, err) return } if o.AccountResolver == nil { err := &configErr{tk, "error parsing account resolver, should be MEM or " + " URL(\"url\") or a map containing dir and type state=[FULL|CACHE])"} *errors = append(*errors, err) } case "resolver_tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) return } tlsConfig, err := GenTLSConfig(tc) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) return } o.AccountResolverTLSConfig = tlsConfig // GenTLSConfig loads the CA file into ClientCAs, but since this will // be used as a client connection, we need to set RootCAs. o.AccountResolverTLSConfig.RootCAs = tlsConfig.ClientCAs case "resolver_preload": mp, ok := v.(map[string]any) if !ok { err := &configErr{tk, "preload should be a map of account_public_key:account_jwt"} *errors = append(*errors, err) return } o.resolverPreloads = make(map[string]string) for key, val := range mp { tk, val = unwrapValue(val, <) if jwtstr, ok := val.(string); !ok { *errors = append(*errors, &configErr{tk, "preload map value should be a string JWT"}) continue } else { // Make sure this is a valid account JWT, that is a config error. // We will warn of expirations, etc later. if _, err := jwt.DecodeAccountClaims(jwtstr); err != nil { err := &configErr{tk, "invalid account JWT"} *errors = append(*errors, err) continue } o.resolverPreloads[key] = jwtstr } } case "resolver_pinned_accounts": switch v := v.(type) { case string: o.resolverPinnedAccounts = map[string]struct{}{v: {}} case []string: o.resolverPinnedAccounts = make(map[string]struct{}) for _, mv := range v { o.resolverPinnedAccounts[mv] = struct{}{} } case []any: o.resolverPinnedAccounts = make(map[string]struct{}) for _, mv := range v { tk, mv = unwrapValue(mv, <) if key, ok := mv.(string); ok { o.resolverPinnedAccounts[key] = struct{}{} } else { err := &configErr{tk, fmt.Sprintf("error parsing resolver_pinned_accounts: unsupported type in array %T", mv)} *errors = append(*errors, err) continue } } default: err := &configErr{tk, fmt.Sprintf("error parsing resolver_pinned_accounts: unsupported type %T", v)} *errors = append(*errors, err) return } case "no_auth_user": o.NoAuthUser = v.(string) case "system_account", "system": // Already processed at the beginning so we just skip them // to not treat them as unknown values. return case "no_system_account", "no_system", "no_sys_acc": o.NoSystemAccount = v.(bool) case "no_header_support": o.NoHeaderSupport = v.(bool) case "trusted", "trusted_keys": switch v := v.(type) { case string: o.TrustedKeys = []string{v} case []string: o.TrustedKeys = v case []any: keys := make([]string, 0, len(v)) for _, mv := range v { tk, mv = unwrapValue(mv, <) if key, ok := mv.(string); ok { keys = append(keys, key) } else { err := &configErr{tk, fmt.Sprintf("error parsing trusted: unsupported type in array %T", mv)} *errors = append(*errors, err) continue } } o.TrustedKeys = keys default: err := &configErr{tk, fmt.Sprintf("error parsing trusted: unsupported type %T", v)} *errors = append(*errors, err) } // Do a quick sanity check on keys for _, key := range o.TrustedKeys { if !nkeys.IsValidPublicOperatorKey(key) { err := &configErr{tk, fmt.Sprintf("trust key %q required to be a valid public operator nkey", key)} *errors = append(*errors, err) } } case "connect_error_reports": o.ConnectErrorReports = int(v.(int64)) case "reconnect_error_reports": o.ReconnectErrorReports = int(v.(int64)) case "websocket", "ws": if err := parseWebsocket(tk, o, errors); err != nil { *errors = append(*errors, err) return } case "mqtt": if err := parseMQTT(tk, o, errors, warnings); err != nil { *errors = append(*errors, err) return } case "server_tags": var err error switch v := v.(type) { case string: o.Tags.Add(v) case []string: o.Tags.Add(v...) case []any: for _, t := range v { if token, ok := t.(token); ok { if ts, ok := token.Value().(string); ok { o.Tags.Add(ts) continue } else { err = &configErr{tk, fmt.Sprintf("error parsing tags: unsupported type %T where string is expected", token)} } } else { err = &configErr{tk, fmt.Sprintf("error parsing tags: unsupported type %T", t)} } break } default: err = &configErr{tk, fmt.Sprintf("error parsing tags: unsupported type %T", v)} } if err != nil { *errors = append(*errors, err) return } case "default_js_domain": vv, ok := v.(map[string]any) if !ok { *errors = append(*errors, &configErr{tk, fmt.Sprintf("error default_js_domain config: unsupported type %T", v)}) return } m := make(map[string]string) for kk, kv := range vv { _, v = unwrapValue(kv, &tk) m[kk] = v.(string) } o.JsAccDefaultDomain = m case "ocsp_cache": var err error switch vv := v.(type) { case bool: pc := NewOCSPResponseCacheConfig() if vv { // Set enabled pc.Type = LOCAL o.OCSPCacheConfig = pc } else { // Set disabled (none cache) pc.Type = NONE o.OCSPCacheConfig = pc } case map[string]any: pc, err := parseOCSPResponseCache(v) if err != nil { *errors = append(*errors, err) return } o.OCSPCacheConfig = pc default: err = &configErr{tk, fmt.Sprintf("error parsing tags: unsupported type %T", v)} } if err != nil { *errors = append(*errors, err) return } case "no_fast_producer_stall": o.NoFastProducerStall = v.(bool) case "max_closed_clients": o.MaxClosedClients = int(v.(int64)) default: if au := atomic.LoadInt32(&allowUnknownTopLevelField); au == 0 && !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } } } func setupUsersAndNKeysDuplicateCheckMap(o *Options) map[string]struct{} { unames := make(map[string]struct{}, len(o.Users)+len(o.Nkeys)) for _, u := range o.Users { unames[u.Username] = struct{}{} } for _, u := range o.Nkeys { unames[u.Nkey] = struct{}{} } return unames } func parseDuration(field string, tk token, v any, errors *[]error, warnings *[]error) time.Duration { if wd, ok := v.(string); ok { if dur, err := time.ParseDuration(wd); err != nil { err := &configErr{tk, fmt.Sprintf("error parsing %s: %v", field, err)} *errors = append(*errors, err) return 0 } else { return dur } } else { // Backward compatible with old type, assume this is the // number of seconds. err := &configWarningErr{ field: field, configErr: configErr{ token: tk, reason: field + " should be converted to a duration", }, } *warnings = append(*warnings, err) return time.Duration(v.(int64)) * time.Second } } func trackExplicitVal(pm *map[string]bool, name string, val bool) { m := *pm if m == nil { m = make(map[string]bool) *pm = m } m[name] = val } // hostPort is simple struct to hold parsed listen/addr strings. type hostPort struct { host string port int } // parseListen will parse listen option which is replacing host/net and port func parseListen(v any) (*hostPort, error) { hp := &hostPort{} switch vv := v.(type) { // Only a port case int64: hp.port = int(vv) case string: host, port, err := net.SplitHostPort(vv) if err != nil { return nil, fmt.Errorf("could not parse address string %q", vv) } hp.port, err = strconv.Atoi(port) if err != nil { return nil, fmt.Errorf("could not parse port %q", port) } hp.host = host default: return nil, fmt.Errorf("expected port or host:port, got %T", vv) } return hp, nil } // parseCluster will parse the cluster config. func parseCluster(v any, opts *Options, errors *[]error, warnings *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) cm, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected map to define cluster, got %T", v)} } for mk, mv := range cm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "name": opts.Cluster.Name = mv.(string) case "listen": hp, err := parseListen(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } opts.Cluster.Host = hp.host opts.Cluster.Port = hp.port case "port": opts.Cluster.Port = int(mv.(int64)) case "host", "net": opts.Cluster.Host = mv.(string) case "authorization": auth, err := parseAuthorization(tk, errors) if err != nil { *errors = append(*errors, err) continue } if auth.users != nil { err := &configErr{tk, "Cluster authorization does not allow multiple users"} *errors = append(*errors, err) continue } if auth.token != _EMPTY_ { err := &configErr{tk, "Cluster authorization does not support tokens"} *errors = append(*errors, err) continue } if auth.callout != nil { err := &configErr{tk, "Cluster authorization does not support callouts"} *errors = append(*errors, err) continue } opts.Cluster.Username = auth.user opts.Cluster.Password = auth.pass opts.Cluster.AuthTimeout = auth.timeout if auth.defaultPermissions != nil { err := &configWarningErr{ field: mk, configErr: configErr{ token: tk, reason: `setting "permissions" within cluster authorization block is deprecated`, }, } *warnings = append(*warnings, err) // Do not set permissions if they were specified in top-level cluster block. if opts.Cluster.Permissions == nil { setClusterPermissions(&opts.Cluster, auth.defaultPermissions) } } case "routes": ra := mv.([]any) routes, errs := parseURLs(ra, "route", warnings) if errs != nil { *errors = append(*errors, errs...) continue } opts.Routes = routes case "tls": config, tlsopts, err := getTLSConfig(tk) if err != nil { *errors = append(*errors, err) continue } opts.Cluster.TLSConfig = config opts.Cluster.TLSTimeout = tlsopts.Timeout opts.Cluster.TLSMap = tlsopts.Map opts.Cluster.TLSPinnedCerts = tlsopts.PinnedCerts opts.Cluster.TLSCheckKnownURLs = tlsopts.TLSCheckKnownURLs opts.Cluster.tlsConfigOpts = tlsopts case "cluster_advertise", "advertise": opts.Cluster.Advertise = mv.(string) case "no_advertise": opts.Cluster.NoAdvertise = mv.(bool) trackExplicitVal(&opts.inConfig, "Cluster.NoAdvertise", opts.Cluster.NoAdvertise) case "connect_retries": opts.Cluster.ConnectRetries = int(mv.(int64)) case "permissions": perms, err := parseUserPermissions(mv, errors) if err != nil { *errors = append(*errors, err) continue } // Dynamic response permissions do not make sense here. if perms.Response != nil { err := &configErr{tk, "Cluster permissions do not support dynamic responses"} *errors = append(*errors, err) continue } // This will possibly override permissions that were define in auth block setClusterPermissions(&opts.Cluster, perms) case "pool_size": opts.Cluster.PoolSize = int(mv.(int64)) case "accounts": opts.Cluster.PinnedAccounts, _ = parseStringArray("accounts", tk, <, mv, errors) case "compression": if err := parseCompression(&opts.Cluster.Compression, CompressionS2Fast, tk, mk, mv); err != nil { *errors = append(*errors, err) continue } case "ping_interval": opts.Cluster.PingInterval = parseDuration("ping_interval", tk, mv, errors, warnings) if opts.Cluster.PingInterval > routeMaxPingInterval { *warnings = append(*warnings, &configErr{tk, fmt.Sprintf("Cluster 'ping_interval' will reset to %v which is the max for routes", routeMaxPingInterval)}) } case "ping_max": opts.Cluster.MaxPingsOut = int(mv.(int64)) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } return nil } // The parameter `chosenModeForOn` indicates which compression mode to use // when the user selects "on" (or enabled, true, etc..). This is because // we may have different defaults depending on where the compression is used. func parseCompression(c *CompressionOpts, chosenModeForOn string, tk token, mk string, mv any) (retErr error) { var lt token defer convertPanicToError(<, &retErr) switch mv := mv.(type) { case string: // Do not validate here, it will be done in NewServer. c.Mode = mv case bool: if mv { c.Mode = chosenModeForOn } else { c.Mode = CompressionOff } case map[string]any: for mk, mv := range mv { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "mode": c.Mode = mv.(string) case "rtt_thresholds", "thresholds", "rtts", "rtt": for _, iv := range mv.([]any) { _, mv := unwrapValue(iv, <) dur, err := time.ParseDuration(mv.(string)) if err != nil { return &configErr{tk, err.Error()} } c.RTTThresholds = append(c.RTTThresholds, dur) } default: if !tk.IsUsedVariable() { return &configErr{tk, fmt.Sprintf("unknown field %q", mk)} } } } default: return &configErr{tk, fmt.Sprintf("field %q should be a boolean or a structure, got %T", mk, mv)} } return nil } func parseURLs(a []any, typ string, warnings *[]error) (urls []*url.URL, errors []error) { urls = make([]*url.URL, 0, len(a)) var lt token defer convertPanicToErrorList(<, &errors) dd := make(map[string]bool) for _, u := range a { tk, u := unwrapValue(u, <) sURL := u.(string) if dd[sURL] { err := &configWarningErr{ field: sURL, configErr: configErr{ token: tk, reason: fmt.Sprintf("Duplicate %s entry detected", typ), }, } *warnings = append(*warnings, err) continue } dd[sURL] = true url, err := parseURL(sURL, typ) if err != nil { err := &configErr{tk, err.Error()} errors = append(errors, err) continue } urls = append(urls, url) } return urls, errors } func parseURL(u string, typ string) (*url.URL, error) { urlStr := strings.TrimSpace(u) url, err := url.Parse(urlStr) if err != nil { // Security note: if it's not well-formed but still reached us, then we're going to log as-is which might include password information here. // If the URL parses, we don't log the credentials ever, but if it doesn't even parse we don't have a sane way to redact. return nil, fmt.Errorf("error parsing %s url [%q]", typ, urlStr) } return url, nil } func parseGateway(v any, o *Options, errors *[]error, warnings *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) gm, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected gateway to be a map, got %T", v)} } for mk, mv := range gm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "name": o.Gateway.Name = mv.(string) case "listen": hp, err := parseListen(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } o.Gateway.Host = hp.host o.Gateway.Port = hp.port case "port": o.Gateway.Port = int(mv.(int64)) case "host", "net": o.Gateway.Host = mv.(string) case "authorization": auth, err := parseAuthorization(tk, errors) if err != nil { *errors = append(*errors, err) continue } if auth.users != nil { *errors = append(*errors, &configErr{tk, "Gateway authorization does not allow multiple users"}) continue } if auth.token != _EMPTY_ { err := &configErr{tk, "Gateway authorization does not support tokens"} *errors = append(*errors, err) continue } if auth.callout != nil { err := &configErr{tk, "Gateway authorization does not support callouts"} *errors = append(*errors, err) continue } o.Gateway.Username = auth.user o.Gateway.Password = auth.pass o.Gateway.AuthTimeout = auth.timeout case "tls": config, tlsopts, err := getTLSConfig(tk) if err != nil { *errors = append(*errors, err) continue } o.Gateway.TLSConfig = config o.Gateway.TLSTimeout = tlsopts.Timeout o.Gateway.TLSMap = tlsopts.Map o.Gateway.TLSCheckKnownURLs = tlsopts.TLSCheckKnownURLs o.Gateway.TLSPinnedCerts = tlsopts.PinnedCerts o.Gateway.tlsConfigOpts = tlsopts case "advertise": o.Gateway.Advertise = mv.(string) case "connect_retries": o.Gateway.ConnectRetries = int(mv.(int64)) case "gateways": gateways, err := parseGateways(mv, errors, warnings) if err != nil { return err } o.Gateway.Gateways = gateways case "reject_unknown", "reject_unknown_cluster": o.Gateway.RejectUnknown = mv.(bool) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } return nil } var dynamicJSAccountLimits = JetStreamAccountLimits{-1, -1, -1, -1, -1, -1, -1, false} var defaultJSAccountTiers = map[string]JetStreamAccountLimits{_EMPTY_: dynamicJSAccountLimits} // Parses jetstream account limits for an account. Simple setup with boolen is allowed, and we will // use dynamic account limits. func parseJetStreamForAccount(v any, acc *Account, errors *[]error) error { var lt token tk, v := unwrapValue(v, <) // Value here can be bool, or string "enabled" or a map. switch vv := v.(type) { case bool: if vv { acc.jsLimits = defaultJSAccountTiers } case string: switch strings.ToLower(vv) { case "enabled", "enable": acc.jsLimits = defaultJSAccountTiers case "disabled", "disable": acc.jsLimits = nil default: return &configErr{tk, fmt.Sprintf("Expected 'enabled' or 'disabled' for string value, got '%s'", vv)} } case map[string]any: jsLimits := JetStreamAccountLimits{-1, -1, -1, -1, -1, -1, -1, false} for mk, mv := range vv { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "max_memory", "max_mem", "mem", "memory": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MaxMemory = vv case "max_store", "max_file", "max_disk", "store", "disk": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MaxStore = vv case "max_streams", "streams": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MaxStreams = int(vv) case "max_consumers", "consumers": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MaxConsumers = int(vv) case "max_bytes_required", "max_stream_bytes", "max_bytes": vv, ok := mv.(bool) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable bool for %q, got %v", mk, mv)} } jsLimits.MaxBytesRequired = vv case "mem_max_stream_bytes", "memory_max_stream_bytes": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MemoryMaxStreamBytes = vv case "disk_max_stream_bytes", "store_max_stream_bytes": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.StoreMaxStreamBytes = vv case "max_ack_pending": vv, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } jsLimits.MaxAckPending = int(vv) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } acc.jsLimits = map[string]JetStreamAccountLimits{_EMPTY_: jsLimits} default: return &configErr{tk, fmt.Sprintf("Expected map, bool or string to define JetStream, got %T", v)} } return nil } // takes in a storage size as either an int or a string and returns an int64 value based on the input. func getStorageSize(v any) (int64, error) { _, ok := v.(int64) if ok { return v.(int64), nil } s, ok := v.(string) if !ok { return 0, fmt.Errorf("must be int64 or string") } if s == _EMPTY_ { return 0, nil } suffix := s[len(s)-1:] prefix := s[:len(s)-1] num, err := strconv.ParseInt(prefix, 10, 64) if err != nil { return 0, err } suffixMap := map[string]int64{"K": 10, "M": 20, "G": 30, "T": 40} mult, ok := suffixMap[suffix] if !ok { return 0, fmt.Errorf("sizes defined as strings must end in K, M, G, T") } num *= 1 << mult return num, nil } // Parse enablement of jetstream for a server. func parseJetStreamLimits(v any, opts *Options, errors *[]error) error { var lt token tk, v := unwrapValue(v, <) lim := JSLimitOpts{} vv, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected a map to define JetStreamLimits, got %T", v)} } for mk, mv := range vv { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "max_ack_pending": lim.MaxAckPending = int(mv.(int64)) case "max_ha_assets": lim.MaxHAAssets = int(mv.(int64)) case "max_request_batch": lim.MaxRequestBatch = int(mv.(int64)) case "duplicate_window": var err error lim.Duplicates, err = time.ParseDuration(mv.(string)) if err != nil { *errors = append(*errors, err) } default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } opts.JetStreamLimits = lim return nil } // Parse enablement of jetstream for a server. func parseJetStream(v any, opts *Options, errors *[]error, warnings *[]error) error { var lt token tk, v := unwrapValue(v, <) // Value here can be bool, or string "enabled" or a map. switch vv := v.(type) { case bool: opts.JetStream = v.(bool) case string: switch strings.ToLower(vv) { case "enabled", "enable": opts.JetStream = true case "disabled", "disable": opts.JetStream = false default: return &configErr{tk, fmt.Sprintf("Expected 'enabled' or 'disabled' for string value, got '%s'", vv)} } case map[string]any: doEnable := true for mk, mv := range vv { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "store", "store_dir", "storedir": // StoreDir can be set at the top level as well so have to prevent ambiguous declarations. if opts.StoreDir != _EMPTY_ { return &configErr{tk, "Duplicate 'store_dir' configuration"} } opts.StoreDir = mv.(string) case "sync", "sync_interval": if v, ok := mv.(string); ok && strings.ToLower(v) == "always" { opts.SyncInterval = defaultSyncInterval opts.SyncAlways = true } else { opts.SyncInterval = parseDuration(mk, tk, mv, errors, warnings) } opts.syncSet = true case "max_memory_store", "max_mem_store", "max_mem": s, err := getStorageSize(mv) if err != nil { return &configErr{tk, fmt.Sprintf("max_mem_store %s", err)} } opts.JetStreamMaxMemory = s opts.maxMemSet = true case "max_file_store", "max_file": s, err := getStorageSize(mv) if err != nil { return &configErr{tk, fmt.Sprintf("max_file_store %s", err)} } opts.JetStreamMaxStore = s opts.maxStoreSet = true case "domain": opts.JetStreamDomain = mv.(string) case "enable", "enabled": doEnable = mv.(bool) case "key", "ek", "encryption_key": opts.JetStreamKey = mv.(string) case "prev_key", "prev_ek", "prev_encryption_key": opts.JetStreamOldKey = mv.(string) case "cipher": switch strings.ToLower(mv.(string)) { case "chacha", "chachapoly": opts.JetStreamCipher = ChaCha case "aes": opts.JetStreamCipher = AES default: return &configErr{tk, fmt.Sprintf("Unknown cipher type: %q", mv)} } case "extension_hint": opts.JetStreamExtHint = mv.(string) case "limits": if err := parseJetStreamLimits(tk, opts, errors); err != nil { return err } case "unique_tag": opts.JetStreamUniqueTag = strings.ToLower(strings.TrimSpace(mv.(string))) case "max_outstanding_catchup": s, err := getStorageSize(mv) if err != nil { return &configErr{tk, fmt.Sprintf("%s %s", strings.ToLower(mk), err)} } opts.JetStreamMaxCatchup = s case "request_queue_limit": lim, ok := mv.(int64) if !ok { return &configErr{tk, fmt.Sprintf("Expected a parseable size for %q, got %v", mk, mv)} } opts.JetStreamRequestQueueLimit = lim default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } opts.JetStream = doEnable default: return &configErr{tk, fmt.Sprintf("Expected map, bool or string to define JetStream, got %T", v)} } return nil } // parseLeafNodes will parse the leaf node config. func parseLeafNodes(v any, opts *Options, errors *[]error, warnings *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) cm, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected map to define a leafnode, got %T", v)} } for mk, mv := range cm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "listen": hp, err := parseListen(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } opts.LeafNode.Host = hp.host opts.LeafNode.Port = hp.port case "port": opts.LeafNode.Port = int(mv.(int64)) case "host", "net": opts.LeafNode.Host = mv.(string) case "authorization": auth, err := parseLeafAuthorization(tk, errors) if err != nil { *errors = append(*errors, err) continue } opts.LeafNode.Username = auth.user opts.LeafNode.Password = auth.pass opts.LeafNode.AuthTimeout = auth.timeout opts.LeafNode.Account = auth.acc opts.LeafNode.Users = auth.users opts.LeafNode.Nkey = auth.nkey // Validate user info config for leafnode authorization if err := validateLeafNodeAuthOptions(opts); err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) continue } case "remotes": // Parse the remote options here. remotes, err := parseRemoteLeafNodes(tk, errors, warnings) if err != nil { *errors = append(*errors, err) continue } opts.LeafNode.Remotes = remotes case "reconnect", "reconnect_delay", "reconnect_interval": opts.LeafNode.ReconnectInterval = parseDuration("reconnect", tk, mv, errors, warnings) case "tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) continue } if opts.LeafNode.TLSConfig, err = GenTLSConfig(tc); err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } opts.LeafNode.TLSTimeout = tc.Timeout opts.LeafNode.TLSMap = tc.Map opts.LeafNode.TLSPinnedCerts = tc.PinnedCerts opts.LeafNode.TLSHandshakeFirst = tc.HandshakeFirst opts.LeafNode.tlsConfigOpts = tc case "leafnode_advertise", "advertise": opts.LeafNode.Advertise = mv.(string) case "no_advertise": opts.LeafNode.NoAdvertise = mv.(bool) trackExplicitVal(&opts.inConfig, "LeafNode.NoAdvertise", opts.LeafNode.NoAdvertise) case "min_version", "minimum_version": version := mv.(string) if err := checkLeafMinVersionConfig(version); err != nil { err = &configErr{tk, err.Error()} *errors = append(*errors, err) continue } opts.LeafNode.MinVersion = version case "compression": if err := parseCompression(&opts.LeafNode.Compression, CompressionS2Auto, tk, mk, mv); err != nil { *errors = append(*errors, err) continue } default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } return nil } // This is the authorization parser adapter for the leafnode's // authorization config. func parseLeafAuthorization(v any, errors *[]error) (*authorization, error) { var ( am map[string]any tk token lt token auth = &authorization{} ) defer convertPanicToErrorList(<, errors) _, v = unwrapValue(v, <) am = v.(map[string]any) for mk, mv := range am { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "user", "username": auth.user = mv.(string) case "pass", "password": auth.pass = mv.(string) case "nkey": nk := mv.(string) if !nkeys.IsValidPublicUserKey(nk) { *errors = append(*errors, &configErr{tk, "Not a valid public nkey for leafnode authorization"}) } auth.nkey = nk case "timeout": at := float64(1) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv } auth.timeout = at case "users": users, err := parseLeafUsers(tk, errors) if err != nil { *errors = append(*errors, err) continue } auth.users = users case "account": auth.acc = mv.(string) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } continue } } return auth, nil } // This is a trimmed down version of parseUsers that is adapted // for the users possibly defined in the authorization{} section // of leafnodes {}. func parseLeafUsers(mv any, errors *[]error) ([]*User, error) { var ( tk token lt token users = []*User{} ) defer convertPanicToErrorList(<, errors) tk, mv = unwrapValue(mv, <) // Make sure we have an array uv, ok := mv.([]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected users field to be an array, got %v", mv)} } for _, u := range uv { tk, u = unwrapValue(u, <) // Check its a map/struct um, ok := u.(map[string]any) if !ok { err := &configErr{tk, fmt.Sprintf("Expected user entry to be a map/struct, got %v", u)} *errors = append(*errors, err) continue } user := &User{} for k, v := range um { tk, v = unwrapValue(v, <) switch strings.ToLower(k) { case "user", "username": user.Username = v.(string) case "pass", "password": user.Password = v.(string) case "account": // We really want to save just the account name here, but // the User object is *Account. So we create an account object // but it won't be registered anywhere. The server will just // use opts.LeafNode.Users[].Account.Name. Alternatively // we need to create internal objects to store u/p and account // name and have a server structure to hold that. user.Account = NewAccount(v.(string)) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } users = append(users, user) } return users, nil } func parseRemoteLeafNodes(v any, errors *[]error, warnings *[]error) ([]*RemoteLeafOpts, error) { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) ra, ok := v.([]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected remotes field to be an array, got %T", v)} } remotes := make([]*RemoteLeafOpts, 0, len(ra)) for _, r := range ra { tk, r = unwrapValue(r, <) // Check its a map/struct rm, ok := r.(map[string]any) if !ok { *errors = append(*errors, &configErr{tk, fmt.Sprintf("Expected remote leafnode entry to be a map/struct, got %v", r)}) continue } remote := &RemoteLeafOpts{} for k, v := range rm { tk, v = unwrapValue(v, <) switch strings.ToLower(k) { case "no_randomize", "dont_randomize": remote.NoRandomize = v.(bool) case "url", "urls": switch v := v.(type) { case []any, []string: urls, errs := parseURLs(v.([]any), "leafnode", warnings) if errs != nil { *errors = append(*errors, errs...) continue } remote.URLs = urls case string: url, err := parseURL(v, "leafnode") if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) continue } remote.URLs = append(remote.URLs, url) default: *errors = append(*errors, &configErr{tk, fmt.Sprintf("Expected remote leafnode url to be an array or string, got %v", v)}) continue } case "account", "local": remote.LocalAccount = v.(string) case "creds", "credentials": p, err := expandPath(v.(string)) if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) continue } // Can't have both creds and nkey if remote.Nkey != _EMPTY_ { *errors = append(*errors, &configErr{tk, "Remote leafnode can not have both creds and nkey defined"}) continue } remote.Credentials = p case "nkey", "seed": nk := v.(string) if pb, _, err := nkeys.DecodeSeed([]byte(nk)); err != nil || pb != nkeys.PrefixByteUser { err := &configErr{tk, fmt.Sprintf("Remote leafnode nkey is not a valid seed: %q", v)} *errors = append(*errors, err) continue } if remote.Credentials != _EMPTY_ { *errors = append(*errors, &configErr{tk, "Remote leafnode can not have both creds and nkey defined"}) continue } remote.Nkey = nk case "tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) continue } if remote.TLSConfig, err = GenTLSConfig(tc); err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) continue } // If ca_file is defined, GenTLSConfig() sets TLSConfig.ClientCAs. // Set RootCAs since this tls.Config is used when soliciting // a connection (therefore behaves as a client). remote.TLSConfig.RootCAs = remote.TLSConfig.ClientCAs if tc.Timeout > 0 { remote.TLSTimeout = tc.Timeout } else { remote.TLSTimeout = float64(DEFAULT_LEAF_TLS_TIMEOUT) / float64(time.Second) } remote.TLSHandshakeFirst = tc.HandshakeFirst remote.tlsConfigOpts = tc case "hub": remote.Hub = v.(bool) case "deny_imports", "deny_import": subjects, err := parsePermSubjects(tk, errors) if err != nil { *errors = append(*errors, err) continue } remote.DenyImports = subjects case "deny_exports", "deny_export": subjects, err := parsePermSubjects(tk, errors) if err != nil { *errors = append(*errors, err) continue } remote.DenyExports = subjects case "ws_compress", "ws_compression", "websocket_compress", "websocket_compression": remote.Websocket.Compression = v.(bool) case "ws_no_masking", "websocket_no_masking": remote.Websocket.NoMasking = v.(bool) case "jetstream_cluster_migrate", "js_cluster_migrate": remote.JetStreamClusterMigrate = true case "compression": if err := parseCompression(&remote.Compression, CompressionS2Auto, tk, k, v); err != nil { *errors = append(*errors, err) continue } case "first_info_timeout": remote.FirstInfoTimeout = parseDuration(k, tk, v, errors, warnings) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } remotes = append(remotes, remote) } return remotes, nil } // Parse TLS and returns a TLSConfig and TLSTimeout. // Used by cluster and gateway parsing. func getTLSConfig(tk token) (*tls.Config, *TLSConfigOpts, error) { tc, err := parseTLS(tk, false) if err != nil { return nil, nil, err } config, err := GenTLSConfig(tc) if err != nil { err := &configErr{tk, err.Error()} return nil, nil, err } // For clusters/gateways, we will force strict verification. We also act // as both client and server, so will mirror the rootCA to the // clientCA pool. config.ClientAuth = tls.RequireAndVerifyClientCert config.RootCAs = config.ClientCAs return config, tc, nil } func parseGateways(v any, errors *[]error, warnings *[]error) ([]*RemoteGatewayOpts, error) { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) // Make sure we have an array ga, ok := v.([]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected gateways field to be an array, got %T", v)} } gateways := []*RemoteGatewayOpts{} for _, g := range ga { tk, g = unwrapValue(g, <) // Check its a map/struct gm, ok := g.(map[string]any) if !ok { *errors = append(*errors, &configErr{tk, fmt.Sprintf("Expected gateway entry to be a map/struct, got %v", g)}) continue } gateway := &RemoteGatewayOpts{} for k, v := range gm { tk, v = unwrapValue(v, <) switch strings.ToLower(k) { case "name": gateway.Name = v.(string) case "tls": tls, tlsopts, err := getTLSConfig(tk) if err != nil { *errors = append(*errors, err) continue } gateway.TLSConfig = tls gateway.TLSTimeout = tlsopts.Timeout gateway.tlsConfigOpts = tlsopts case "url": url, err := parseURL(v.(string), "gateway") if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) continue } gateway.URLs = append(gateway.URLs, url) case "urls": urls, errs := parseURLs(v.([]any), "gateway", warnings) if errs != nil { *errors = append(*errors, errs...) continue } gateway.URLs = urls default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } gateways = append(gateways, gateway) } return gateways, nil } // Sets cluster's permissions based on given pub/sub permissions, // doing the appropriate translation. func setClusterPermissions(opts *ClusterOpts, perms *Permissions) { // Import is whether or not we will send a SUB for interest to the other side. // Export is whether or not we will accept a SUB from the remote for a given subject. // Both only effect interest registration. // The parsing sets Import into Publish and Export into Subscribe, convert // accordingly. opts.Permissions = &RoutePermissions{ Import: perms.Publish, Export: perms.Subscribe, } } // Temp structures to hold account import and export defintions since they need // to be processed after being parsed. type export struct { acc *Account sub string accs []string rt ServiceRespType lat *serviceLatency rthr time.Duration tPos uint } type importStream struct { acc *Account an string sub string to string pre string } type importService struct { acc *Account an string sub string to string share bool } // Checks if an account name is reserved. func isReservedAccount(name string) bool { return name == globalAccountName } func parseAccountMapDest(v any, tk token, errors *[]error) (*MapDest, *configErr) { // These should be maps. mv, ok := v.(map[string]any) if !ok { err := &configErr{tk, "Expected an entry for the mapping destination"} *errors = append(*errors, err) return nil, err } mdest := &MapDest{} var lt token var sw bool for k, v := range mv { tk, dmv := unwrapValue(v, <) switch strings.ToLower(k) { case "dest", "destination": mdest.Subject = dmv.(string) case "weight": switch vv := dmv.(type) { case string: ws := vv ws = strings.TrimSuffix(ws, "%") weight, err := strconv.Atoi(ws) if err != nil { err := &configErr{tk, fmt.Sprintf("Invalid weight %q for mapping destination", ws)} *errors = append(*errors, err) return nil, err } if weight > 100 || weight < 0 { err := &configErr{tk, fmt.Sprintf("Invalid weight %d for mapping destination", weight)} *errors = append(*errors, err) return nil, err } mdest.Weight = uint8(weight) sw = true case int64: weight := vv if weight > 100 || weight < 0 { err := &configErr{tk, fmt.Sprintf("Invalid weight %d for mapping destination", weight)} *errors = append(*errors, err) return nil, err } mdest.Weight = uint8(weight) sw = true default: err := &configErr{tk, fmt.Sprintf("Unknown entry type for weight of %v\n", vv)} *errors = append(*errors, err) return nil, err } case "cluster": mdest.Cluster = dmv.(string) default: err := &configErr{tk, fmt.Sprintf("Unknown field %q for mapping destination", k)} *errors = append(*errors, err) return nil, err } } if !sw { err := &configErr{tk, fmt.Sprintf("Missing weight for mapping destination %q", mdest.Subject)} *errors = append(*errors, err) return nil, err } return mdest, nil } // parseAccountMappings is called to parse account mappings. func parseAccountMappings(v any, acc *Account, errors *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) am := v.(map[string]any) for subj, mv := range am { if !IsValidSubject(subj) { err := &configErr{tk, fmt.Sprintf("Subject %q is not a valid subject", subj)} *errors = append(*errors, err) continue } tk, v := unwrapValue(mv, <) switch vv := v.(type) { case string: if err := acc.AddMapping(subj, v.(string)); err != nil { err := &configErr{tk, fmt.Sprintf("Error adding mapping for %q to %q : %v", subj, v.(string), err)} *errors = append(*errors, err) continue } case []any: var mappings []*MapDest for _, mv := range v.([]any) { tk, amv := unwrapValue(mv, <) mdest, err := parseAccountMapDest(amv, tk, errors) if err != nil { continue } mappings = append(mappings, mdest) } // Now add them in.. if err := acc.AddWeightedMappings(subj, mappings...); err != nil { err := &configErr{tk, fmt.Sprintf("Error adding mapping for %q : %v", subj, err)} *errors = append(*errors, err) continue } case any: tk, amv := unwrapValue(mv, <) mdest, err := parseAccountMapDest(amv, tk, errors) if err != nil { continue } // Now add it in.. if err := acc.AddWeightedMappings(subj, mdest); err != nil { err := &configErr{tk, fmt.Sprintf("Error adding mapping for %q : %v", subj, err)} *errors = append(*errors, err) continue } default: err := &configErr{tk, fmt.Sprintf("Unknown type %T for mapping destination", vv)} *errors = append(*errors, err) continue } } return nil } // parseAccountLimits is called to parse account limits in a server config. func parseAccountLimits(mv any, acc *Account, errors *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(mv, <) am, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected account limits to be a map/struct, got %+v", v)} } for k, v := range am { tk, mv = unwrapValue(v, <) switch strings.ToLower(k) { case "max_connections", "max_conn": acc.mconns = int32(mv.(int64)) case "max_subscriptions", "max_subs": acc.msubs = int32(mv.(int64)) case "max_payload", "max_pay": acc.mpay = int32(mv.(int64)) case "max_leafnodes", "max_leafs": acc.mleafs = int32(mv.(int64)) default: if !tk.IsUsedVariable() { err := &configErr{tk, fmt.Sprintf("Unknown field %q parsing account limits", k)} *errors = append(*errors, err) } } } return nil } // parseAccounts will parse the different accounts syntax. func parseAccounts(v any, opts *Options, errors *[]error, warnings *[]error) error { var ( importStreams []*importStream importServices []*importService exportStreams []*export exportServices []*export lt token ) defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) switch vv := v.(type) { // Simple array of account names. case []any, []string: m := make(map[string]struct{}, len(v.([]any))) for _, n := range v.([]any) { tk, name := unwrapValue(n, <) ns := name.(string) // Check for reserved names. if isReservedAccount(ns) { err := &configErr{tk, fmt.Sprintf("%q is a Reserved Account", ns)} *errors = append(*errors, err) continue } if _, ok := m[ns]; ok { err := &configErr{tk, fmt.Sprintf("Duplicate Account Entry: %s", ns)} *errors = append(*errors, err) continue } opts.Accounts = append(opts.Accounts, NewAccount(ns)) m[ns] = struct{}{} } // More common map entry case map[string]any: // Track users across accounts, must be unique across // accounts and nkeys vs users. // We also want to check for users that may have been added in // parseAuthorization{} if that happened first. uorn := setupUsersAndNKeysDuplicateCheckMap(opts) for aname, mv := range vv { tk, amv := unwrapValue(mv, <) // Skip referenced config vars within the account block. if tk.IsUsedVariable() { continue } // These should be maps. mv, ok := amv.(map[string]any) if !ok { err := &configErr{tk, "Expected map entries for accounts"} *errors = append(*errors, err) continue } if isReservedAccount(aname) { err := &configErr{tk, fmt.Sprintf("%q is a Reserved Account", aname)} *errors = append(*errors, err) continue } var ( users []*User nkeyUsr []*NkeyUser usersTk token ) acc := NewAccount(aname) opts.Accounts = append(opts.Accounts, acc) for k, v := range mv { tk, mv := unwrapValue(v, <) switch strings.ToLower(k) { case "nkey": nk, ok := mv.(string) if !ok || !nkeys.IsValidPublicAccountKey(nk) { err := &configErr{tk, fmt.Sprintf("Not a valid public nkey for an account: %q", mv)} *errors = append(*errors, err) continue } acc.Nkey = nk case "imports": streams, services, err := parseAccountImports(tk, acc, errors) if err != nil { *errors = append(*errors, err) continue } importStreams = append(importStreams, streams...) importServices = append(importServices, services...) case "exports": streams, services, err := parseAccountExports(tk, acc, errors) if err != nil { *errors = append(*errors, err) continue } exportStreams = append(exportStreams, streams...) exportServices = append(exportServices, services...) case "jetstream": err := parseJetStreamForAccount(mv, acc, errors) if err != nil { *errors = append(*errors, err) continue } case "users": var err error usersTk = tk nkeyUsr, users, err = parseUsers(mv, errors) if err != nil { *errors = append(*errors, err) continue } case "default_permissions": permissions, err := parseUserPermissions(tk, errors) if err != nil { *errors = append(*errors, err) continue } acc.defaultPerms = permissions case "mappings", "maps": err := parseAccountMappings(tk, acc, errors) if err != nil { *errors = append(*errors, err) continue } case "limits": err := parseAccountLimits(tk, acc, errors) if err != nil { *errors = append(*errors, err) continue } default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } } } // Report error if there is an authorization{} block // with u/p or token and any user defined in accounts{} if len(nkeyUsr) > 0 || len(users) > 0 { if opts.Username != _EMPTY_ { err := &configErr{usersTk, "Can not have a single user/pass and accounts"} *errors = append(*errors, err) continue } if opts.Authorization != _EMPTY_ { err := &configErr{usersTk, "Can not have a token and accounts"} *errors = append(*errors, err) continue } } applyDefaultPermissions(users, nkeyUsr, acc.defaultPerms) for _, u := range nkeyUsr { if _, ok := uorn[u.Nkey]; ok { err := &configErr{usersTk, fmt.Sprintf("Duplicate nkey %q detected", u.Nkey)} *errors = append(*errors, err) continue } uorn[u.Nkey] = struct{}{} u.Account = acc } opts.Nkeys = append(opts.Nkeys, nkeyUsr...) for _, u := range users { if _, ok := uorn[u.Username]; ok { err := &configErr{usersTk, fmt.Sprintf("Duplicate user %q detected", u.Username)} *errors = append(*errors, err) continue } uorn[u.Username] = struct{}{} u.Account = acc } opts.Users = append(opts.Users, users...) } } lt = tk // Bail already if there are previous errors. if len(*errors) > 0 { return nil } // Parse Imports and Exports here after all accounts defined. // Do exports first since they need to be defined for imports to succeed // since we do permissions checks. // Create a lookup map for accounts lookups. am := make(map[string]*Account, len(opts.Accounts)) for _, a := range opts.Accounts { am[a.Name] = a } // Do stream exports for _, stream := range exportStreams { // Make array of accounts if applicable. var accounts []*Account for _, an := range stream.accs { ta := am[an] if ta == nil { msg := fmt.Sprintf("%q account not defined for stream export", an) *errors = append(*errors, &configErr{tk, msg}) continue } accounts = append(accounts, ta) } if err := stream.acc.addStreamExportWithAccountPos(stream.sub, accounts, stream.tPos); err != nil { msg := fmt.Sprintf("Error adding stream export %q: %v", stream.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } } for _, service := range exportServices { // Make array of accounts if applicable. var accounts []*Account for _, an := range service.accs { ta := am[an] if ta == nil { msg := fmt.Sprintf("%q account not defined for service export", an) *errors = append(*errors, &configErr{tk, msg}) continue } accounts = append(accounts, ta) } if err := service.acc.addServiceExportWithResponseAndAccountPos(service.sub, service.rt, accounts, service.tPos); err != nil { msg := fmt.Sprintf("Error adding service export %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } if service.rthr != 0 { // Response threshold was set in options. if err := service.acc.SetServiceExportResponseThreshold(service.sub, service.rthr); err != nil { msg := fmt.Sprintf("Error adding service export response threshold for %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } } if service.lat != nil { // System accounts are on be default so just make sure we have not opted out.. if opts.NoSystemAccount { msg := fmt.Sprintf("Error adding service latency sampling for %q: %v", service.sub, ErrNoSysAccount.Error()) *errors = append(*errors, &configErr{tk, msg}) continue } if err := service.acc.TrackServiceExportWithSampling(service.sub, service.lat.subject, int(service.lat.sampling)); err != nil { msg := fmt.Sprintf("Error adding service latency sampling for %q on subject %q: %v", service.sub, service.lat.subject, err) *errors = append(*errors, &configErr{tk, msg}) continue } } } for _, stream := range importStreams { ta := am[stream.an] if ta == nil { msg := fmt.Sprintf("%q account not defined for stream import", stream.an) *errors = append(*errors, &configErr{tk, msg}) continue } if stream.pre != _EMPTY_ { if err := stream.acc.AddStreamImport(ta, stream.sub, stream.pre); err != nil { msg := fmt.Sprintf("Error adding stream import %q: %v", stream.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } } else { if err := stream.acc.AddMappedStreamImport(ta, stream.sub, stream.to); err != nil { msg := fmt.Sprintf("Error adding stream import %q: %v", stream.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } } } for _, service := range importServices { ta := am[service.an] if ta == nil { msg := fmt.Sprintf("%q account not defined for service import", service.an) *errors = append(*errors, &configErr{tk, msg}) continue } if service.to == _EMPTY_ { service.to = service.sub } if err := service.acc.AddServiceImport(ta, service.to, service.sub); err != nil { msg := fmt.Sprintf("Error adding service import %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } if err := service.acc.SetServiceImportSharing(ta, service.sub, service.share); err != nil { msg := fmt.Sprintf("Error setting service import sharing %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue } } return nil } // Parse the account exports func parseAccountExports(v any, acc *Account, errors *[]error) ([]*export, []*export, error) { var lt token defer convertPanicToErrorList(<, errors) // This should be an array of objects/maps. tk, v := unwrapValue(v, <) ims, ok := v.([]any) if !ok { return nil, nil, &configErr{tk, fmt.Sprintf("Exports should be an array, got %T", v)} } var services []*export var streams []*export for _, v := range ims { // Should have stream or service stream, service, err := parseExportStreamOrService(v, errors) if err != nil { *errors = append(*errors, err) continue } if service != nil { service.acc = acc services = append(services, service) } if stream != nil { stream.acc = acc streams = append(streams, stream) } } return streams, services, nil } // Parse the account imports func parseAccountImports(v any, acc *Account, errors *[]error) ([]*importStream, []*importService, error) { var lt token defer convertPanicToErrorList(<, errors) // This should be an array of objects/maps. tk, v := unwrapValue(v, <) ims, ok := v.([]any) if !ok { return nil, nil, &configErr{tk, fmt.Sprintf("Imports should be an array, got %T", v)} } var services []*importService var streams []*importStream svcSubjects := map[string]*importService{} for _, v := range ims { // Should have stream or service stream, service, err := parseImportStreamOrService(v, errors) if err != nil { *errors = append(*errors, err) continue } if service != nil { if dup := svcSubjects[service.to]; dup != nil { tk, _ := unwrapValue(v, <) err := &configErr{tk, fmt.Sprintf("Duplicate service import subject %q, previously used in import for account %q, subject %q", service.to, dup.an, dup.sub)} *errors = append(*errors, err) continue } svcSubjects[service.to] = service service.acc = acc services = append(services, service) } if stream != nil { stream.acc = acc streams = append(streams, stream) } } return streams, services, nil } // Helper to parse an embedded account description for imported services or streams. func parseAccount(v map[string]any, errors *[]error) (string, string, error) { var lt token defer convertPanicToErrorList(<, errors) var accountName, subject string for mk, mv := range v { tk, mv := unwrapValue(mv, <) switch strings.ToLower(mk) { case "account": accountName = mv.(string) case "subject": subject = mv.(string) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } } } return accountName, subject, nil } // Parse an export stream or service. // e.g. // {stream: "public.>"} # No accounts means public. // {stream: "synadia.private.>", accounts: [cncf, natsio]} // {service: "pub.request"} # No accounts means public. // {service: "pub.special.request", accounts: [nats.io]} func parseExportStreamOrService(v any, errors *[]error) (*export, *export, error) { var ( curStream *export curService *export accounts []string rt ServiceRespType rtSeen bool rtToken token lat *serviceLatency threshSeen bool thresh time.Duration latToken token lt token accTokPos uint ) defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) vv, ok := v.(map[string]any) if !ok { return nil, nil, &configErr{tk, fmt.Sprintf("Export Items should be a map with type entry, got %T", v)} } for mk, mv := range vv { tk, mv := unwrapValue(mv, <) switch strings.ToLower(mk) { case "stream": if curService != nil { err := &configErr{tk, fmt.Sprintf("Detected stream %q but already saw a service", mv)} *errors = append(*errors, err) continue } if rtToken != nil { err := &configErr{rtToken, "Detected response directive on non-service"} *errors = append(*errors, err) continue } if latToken != nil { err := &configErr{latToken, "Detected latency directive on non-service"} *errors = append(*errors, err) continue } mvs, ok := mv.(string) if !ok { err := &configErr{tk, fmt.Sprintf("Expected stream name to be string, got %T", mv)} *errors = append(*errors, err) continue } curStream = &export{sub: mvs} if accounts != nil { curStream.accs = accounts } case "service": if curStream != nil { err := &configErr{tk, fmt.Sprintf("Detected service %q but already saw a stream", mv)} *errors = append(*errors, err) continue } mvs, ok := mv.(string) if !ok { err := &configErr{tk, fmt.Sprintf("Expected service name to be string, got %T", mv)} *errors = append(*errors, err) continue } curService = &export{sub: mvs} if accounts != nil { curService.accs = accounts } if rtSeen { curService.rt = rt } if lat != nil { curService.lat = lat } if threshSeen { curService.rthr = thresh } case "response", "response_type": if rtSeen { err := &configErr{tk, "Duplicate response type definition"} *errors = append(*errors, err) continue } rtSeen = true rtToken = tk mvs, ok := mv.(string) if !ok { err := &configErr{tk, fmt.Sprintf("Expected response type to be string, got %T", mv)} *errors = append(*errors, err) continue } switch strings.ToLower(mvs) { case "single", "singleton": rt = Singleton case "stream": rt = Streamed case "chunk", "chunked": rt = Chunked default: err := &configErr{tk, fmt.Sprintf("Unknown response type: %q", mvs)} *errors = append(*errors, err) continue } if curService != nil { curService.rt = rt } if curStream != nil { err := &configErr{tk, "Detected response directive on non-service"} *errors = append(*errors, err) } case "threshold", "response_threshold", "response_max_time", "response_time": if threshSeen { err := &configErr{tk, "Duplicate response threshold detected"} *errors = append(*errors, err) continue } threshSeen = true mvs, ok := mv.(string) if !ok { err := &configErr{tk, fmt.Sprintf("Expected response threshold to be a parseable time duration, got %T", mv)} *errors = append(*errors, err) continue } var err error thresh, err = time.ParseDuration(mvs) if err != nil { err := &configErr{tk, fmt.Sprintf("Expected response threshold to be a parseable time duration, got %q", mvs)} *errors = append(*errors, err) continue } if curService != nil { curService.rthr = thresh } if curStream != nil { err := &configErr{tk, "Detected response directive on non-service"} *errors = append(*errors, err) } case "accounts": for _, iv := range mv.([]any) { _, mv := unwrapValue(iv, <) accounts = append(accounts, mv.(string)) } if curStream != nil { curStream.accs = accounts } else if curService != nil { curService.accs = accounts } case "latency": latToken = tk var err error lat, err = parseServiceLatency(tk, mv) if err != nil { *errors = append(*errors, err) continue } if curStream != nil { err = &configErr{tk, "Detected latency directive on non-service"} *errors = append(*errors, err) continue } if curService != nil { curService.lat = lat } case "account_token_position": accTokPos = uint(mv.(int64)) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } } } if curStream != nil { curStream.tPos = accTokPos } if curService != nil { curService.tPos = accTokPos } return curStream, curService, nil } // parseServiceLatency returns a latency config block. func parseServiceLatency(root token, v any) (l *serviceLatency, retErr error) { var lt token defer convertPanicToError(<, &retErr) if subject, ok := v.(string); ok { return &serviceLatency{ subject: subject, sampling: DEFAULT_SERVICE_LATENCY_SAMPLING, }, nil } latency, ok := v.(map[string]any) if !ok { return nil, &configErr{token: root, reason: fmt.Sprintf("Expected latency entry to be a map/struct or string, got %T", v)} } sl := serviceLatency{ sampling: DEFAULT_SERVICE_LATENCY_SAMPLING, } // Read sampling value. if v, ok := latency["sampling"]; ok { tk, v := unwrapValue(v, <) header := false var sample int64 switch vv := v.(type) { case int64: // Sample is an int, like 50. sample = vv case string: // Sample is a string, like "50%". if strings.ToLower(strings.TrimSpace(vv)) == "headers" { header = true sample = 0 break } s := strings.TrimSuffix(vv, "%") n, err := strconv.Atoi(s) if err != nil { return nil, &configErr{token: tk, reason: fmt.Sprintf("Failed to parse latency sample: %v", err)} } sample = int64(n) default: return nil, &configErr{token: tk, reason: fmt.Sprintf("Expected latency sample to be a string or map/struct, got %T", v)} } if !header { if sample < 1 || sample > 100 { return nil, &configErr{token: tk, reason: ErrBadSampling.Error()} } } sl.sampling = int8(sample) } // Read subject value. v, ok = latency["subject"] if !ok { return nil, &configErr{token: root, reason: "Latency subject required, but missing"} } tk, v := unwrapValue(v, <) subject, ok := v.(string) if !ok { return nil, &configErr{token: tk, reason: fmt.Sprintf("Expected latency subject to be a string, got %T", subject)} } sl.subject = subject return &sl, nil } // Parse an import stream or service. // e.g. // {stream: {account: "synadia", subject:"public.synadia"}, prefix: "imports.synadia"} // {stream: {account: "synadia", subject:"synadia.private.*"}} // {service: {account: "synadia", subject: "pub.special.request"}, to: "synadia.request"} func parseImportStreamOrService(v any, errors *[]error) (*importStream, *importService, error) { var ( curStream *importStream curService *importService pre, to string share bool lt token ) defer convertPanicToErrorList(<, errors) tk, mv := unwrapValue(v, <) vv, ok := mv.(map[string]any) if !ok { return nil, nil, &configErr{tk, fmt.Sprintf("Import Items should be a map with type entry, got %T", mv)} } for mk, mv := range vv { tk, mv := unwrapValue(mv, <) switch strings.ToLower(mk) { case "stream": if curService != nil { err := &configErr{tk, "Detected stream but already saw a service"} *errors = append(*errors, err) continue } ac, ok := mv.(map[string]any) if !ok { err := &configErr{tk, fmt.Sprintf("Stream entry should be an account map, got %T", mv)} *errors = append(*errors, err) continue } // Make sure this is a map with account and subject accountName, subject, err := parseAccount(ac, errors) if err != nil { *errors = append(*errors, err) continue } if accountName == _EMPTY_ || subject == _EMPTY_ { err := &configErr{tk, "Expect an account name and a subject"} *errors = append(*errors, err) continue } curStream = &importStream{an: accountName, sub: subject} if to != _EMPTY_ { curStream.to = to } if pre != _EMPTY_ { curStream.pre = pre } case "service": if curStream != nil { err := &configErr{tk, "Detected service but already saw a stream"} *errors = append(*errors, err) continue } ac, ok := mv.(map[string]interface{}) if !ok { err := &configErr{tk, fmt.Sprintf("Service entry should be an account map, got %T", mv)} *errors = append(*errors, err) continue } // Make sure this is a map with account and subject accountName, subject, err := parseAccount(ac, errors) if err != nil { *errors = append(*errors, err) continue } if accountName == _EMPTY_ || subject == _EMPTY_ { err := &configErr{tk, "Expect an account name and a subject"} *errors = append(*errors, err) continue } curService = &importService{an: accountName, sub: subject} if to != _EMPTY_ { curService.to = to } else { curService.to = subject } curService.share = share case "prefix": pre = mv.(string) if curStream != nil { curStream.pre = pre } case "to": to = mv.(string) if curService != nil { curService.to = to } if curStream != nil { curStream.to = to if curStream.pre != _EMPTY_ { err := &configErr{tk, "Stream import can not have a 'prefix' and a 'to' property"} *errors = append(*errors, err) continue } } case "share": share = mv.(bool) if curService != nil { curService.share = share } default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } } } return curStream, curService, nil } // Apply permission defaults to users/nkeyuser that don't have their own. func applyDefaultPermissions(users []*User, nkeys []*NkeyUser, defaultP *Permissions) { if defaultP == nil { return } for _, user := range users { if user.Permissions == nil { user.Permissions = defaultP } } for _, user := range nkeys { if user.Permissions == nil { user.Permissions = defaultP } } } // Helper function to parse Authorization configs. func parseAuthorization(v any, errors *[]error) (*authorization, error) { var ( am map[string]any tk token lt token auth = &authorization{} ) defer convertPanicToErrorList(<, errors) _, v = unwrapValue(v, <) am = v.(map[string]any) for mk, mv := range am { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "user", "username": auth.user = mv.(string) case "pass", "password": auth.pass = mv.(string) case "token": auth.token = mv.(string) case "timeout": at := float64(1) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv } auth.timeout = at case "users": nkeys, users, err := parseUsers(tk, errors) if err != nil { *errors = append(*errors, err) continue } auth.users = users auth.nkeys = nkeys case "default_permission", "default_permissions", "permissions": permissions, err := parseUserPermissions(tk, errors) if err != nil { *errors = append(*errors, err) continue } auth.defaultPermissions = permissions case "auth_callout", "auth_hook": ac, err := parseAuthCallout(tk, errors) if err != nil { *errors = append(*errors, err) continue } auth.callout = ac default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } continue } applyDefaultPermissions(auth.users, auth.nkeys, auth.defaultPermissions) } return auth, nil } // Helper function to parse multiple users array with optional permissions. func parseUsers(mv any, errors *[]error) ([]*NkeyUser, []*User, error) { var ( tk token lt token keys []*NkeyUser users = []*User{} ) defer convertPanicToErrorList(<, errors) tk, mv = unwrapValue(mv, <) // Make sure we have an array uv, ok := mv.([]any) if !ok { return nil, nil, &configErr{tk, fmt.Sprintf("Expected users field to be an array, got %v", mv)} } for _, u := range uv { tk, u = unwrapValue(u, <) // Check its a map/struct um, ok := u.(map[string]any) if !ok { err := &configErr{tk, fmt.Sprintf("Expected user entry to be a map/struct, got %v", u)} *errors = append(*errors, err) continue } var ( user = &User{} nkey = &NkeyUser{} perms *Permissions err error ) for k, v := range um { // Also needs to unwrap first tk, v = unwrapValue(v, <) switch strings.ToLower(k) { case "nkey": nkey.Nkey = v.(string) case "user", "username": user.Username = v.(string) case "pass", "password": user.Password = v.(string) case "permission", "permissions", "authorization": perms, err = parseUserPermissions(tk, errors) if err != nil { *errors = append(*errors, err) continue } case "allowed_connection_types", "connection_types", "clients": cts := parseAllowedConnectionTypes(tk, <, v, errors) nkey.AllowedConnectionTypes = cts user.AllowedConnectionTypes = cts default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: k, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } // Place perms if we have them. if perms != nil { // nkey takes precedent. if nkey.Nkey != _EMPTY_ { nkey.Permissions = perms } else { user.Permissions = perms } } // Check to make sure we have at least an nkey or username defined. if nkey.Nkey == _EMPTY_ && user.Username == _EMPTY_ { return nil, nil, &configErr{tk, "User entry requires a user"} } else if nkey.Nkey != _EMPTY_ { // Make sure the nkey a proper public nkey for a user.. if !nkeys.IsValidPublicUserKey(nkey.Nkey) { return nil, nil, &configErr{tk, "Not a valid public nkey for a user"} } // If we have user or password defined here that is an error. if user.Username != _EMPTY_ || user.Password != _EMPTY_ { return nil, nil, &configErr{tk, "Nkey users do not take usernames or passwords"} } keys = append(keys, nkey) } else { users = append(users, user) } } return keys, users, nil } func parseAllowedConnectionTypes(tk token, lt *token, mv any, errors *[]error) map[string]struct{} { cts, err := parseStringArray("allowed connection types", tk, lt, mv, errors) // If error, it has already been added to the `errors` array, simply return if err != nil { return nil } m, err := convertAllowedConnectionTypes(cts) if err != nil { *errors = append(*errors, &configErr{tk, err.Error()}) } return m } // Helper function to parse auth callouts. func parseAuthCallout(mv any, errors *[]error) (*AuthCallout, error) { var ( tk token lt token ac = &AuthCallout{} ) defer convertPanicToErrorList(<, errors) tk, mv = unwrapValue(mv, <) pm, ok := mv.(map[string]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected authorization callout to be a map/struct, got %+v", mv)} } for k, v := range pm { tk, mv = unwrapValue(v, <) switch strings.ToLower(k) { case "issuer": ac.Issuer = mv.(string) if !nkeys.IsValidPublicAccountKey(ac.Issuer) { return nil, &configErr{tk, fmt.Sprintf("Expected callout user to be a valid public account nkey, got %q", ac.Issuer)} } case "account", "acc": ac.Account = mv.(string) case "auth_users", "users": aua, ok := mv.([]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected auth_users field to be an array, got %T", v)} } for _, uv := range aua { _, uv = unwrapValue(uv, <) ac.AuthUsers = append(ac.AuthUsers, uv.(string)) } case "xkey", "key": ac.XKey = mv.(string) if !nkeys.IsValidPublicCurveKey(ac.XKey) { return nil, &configErr{tk, fmt.Sprintf("Expected callout xkey to be a valid public xkey, got %q", ac.XKey)} } default: if !tk.IsUsedVariable() { err := &configErr{tk, fmt.Sprintf("Unknown field %q parsing authorization callout", k)} *errors = append(*errors, err) } } } // Make sure we have all defined. All fields are required. // If no account specified, selet $G. if ac.Account == _EMPTY_ { ac.Account = globalAccountName } if ac.Issuer == _EMPTY_ { return nil, &configErr{tk, "Authorization callouts require an issuer to be specified"} } if len(ac.AuthUsers) == 0 { return nil, &configErr{tk, "Authorization callouts require authorized users to be specified"} } return ac, nil } // Helper function to parse user/account permissions func parseUserPermissions(mv any, errors *[]error) (*Permissions, error) { var ( tk token lt token p = &Permissions{} ) defer convertPanicToErrorList(<, errors) tk, mv = unwrapValue(mv, <) pm, ok := mv.(map[string]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("Expected permissions to be a map/struct, got %+v", mv)} } for k, v := range pm { tk, mv = unwrapValue(v, <) switch strings.ToLower(k) { // For routes: // Import is Publish // Export is Subscribe case "pub", "publish", "import": perms, err := parseVariablePermissions(mv, errors) if err != nil { *errors = append(*errors, err) continue } p.Publish = perms case "sub", "subscribe", "export": perms, err := parseVariablePermissions(mv, errors) if err != nil { *errors = append(*errors, err) continue } p.Subscribe = perms case "publish_allow_responses", "allow_responses": rp := &ResponsePermission{ MaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS, Expires: DEFAULT_ALLOW_RESPONSE_EXPIRATION, } // Try boolean first responses, ok := mv.(bool) if ok { if responses { p.Response = rp } } else { p.Response = parseAllowResponses(v, errors) } if p.Response != nil { if p.Publish == nil { p.Publish = &SubjectPermission{} } if p.Publish.Allow == nil { // We turn off the blanket allow statement. p.Publish.Allow = []string{} } } default: if !tk.IsUsedVariable() { err := &configErr{tk, fmt.Sprintf("Unknown field %q parsing permissions", k)} *errors = append(*errors, err) } } } return p, nil } // Top level parser for authorization configurations. func parseVariablePermissions(v any, errors *[]error) (*SubjectPermission, error) { switch vv := v.(type) { case map[string]any: // New style with allow and/or deny properties. return parseSubjectPermission(vv, errors) default: // Old style return parseOldPermissionStyle(v, errors) } } // Helper function to parse subject singletons and/or arrays func parsePermSubjects(v any, errors *[]error) ([]string, error) { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) var subjects []string switch vv := v.(type) { case string: subjects = append(subjects, vv) case []string: subjects = vv case []any: for _, i := range vv { tk, i := unwrapValue(i, <) subject, ok := i.(string) if !ok { return nil, &configErr{tk, "Subject in permissions array cannot be cast to string"} } subjects = append(subjects, subject) } default: return nil, &configErr{tk, fmt.Sprintf("Expected subject permissions to be a subject, or array of subjects, got %T", v)} } if err := checkPermSubjectArray(subjects); err != nil { return nil, &configErr{tk, err.Error()} } return subjects, nil } // Helper function to parse a ResponsePermission. func parseAllowResponses(v any, errors *[]error) *ResponsePermission { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) // Check if this is a map. pm, ok := v.(map[string]any) if !ok { err := &configErr{tk, "error parsing response permissions, expected a boolean or a map"} *errors = append(*errors, err) return nil } rp := &ResponsePermission{ MaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS, Expires: DEFAULT_ALLOW_RESPONSE_EXPIRATION, } for k, v := range pm { tk, v = unwrapValue(v, <) switch strings.ToLower(k) { case "max", "max_msgs", "max_messages", "max_responses": max := int(v.(int64)) // Negative values are accepted (mean infinite), and 0 // means default value (set above). if max != 0 { rp.MaxMsgs = max } case "expires", "expiration", "ttl": wd, ok := v.(string) if ok { ttl, err := time.ParseDuration(wd) if err != nil { err := &configErr{tk, fmt.Sprintf("error parsing expires: %v", err)} *errors = append(*errors, err) return nil } // Negative values are accepted (mean infinite), and 0 // means default value (set above). if ttl != 0 { rp.Expires = ttl } } else { err := &configErr{tk, "error parsing expires, not a duration string"} *errors = append(*errors, err) return nil } default: if !tk.IsUsedVariable() { err := &configErr{tk, fmt.Sprintf("Unknown field %q parsing permissions", k)} *errors = append(*errors, err) } } } return rp } // Helper function to parse old style authorization configs. func parseOldPermissionStyle(v any, errors *[]error) (*SubjectPermission, error) { subjects, err := parsePermSubjects(v, errors) if err != nil { return nil, err } return &SubjectPermission{Allow: subjects}, nil } // Helper function to parse new style authorization into a SubjectPermission with Allow and Deny. func parseSubjectPermission(v any, errors *[]error) (*SubjectPermission, error) { var lt token defer convertPanicToErrorList(<, errors) m := v.(map[string]any) if len(m) == 0 { return nil, nil } p := &SubjectPermission{} for k, v := range m { tk, _ := unwrapValue(v, <) switch strings.ToLower(k) { case "allow": subjects, err := parsePermSubjects(tk, errors) if err != nil { *errors = append(*errors, err) continue } p.Allow = subjects case "deny": subjects, err := parsePermSubjects(tk, errors) if err != nil { *errors = append(*errors, err) continue } p.Deny = subjects default: if !tk.IsUsedVariable() { err := &configErr{tk, fmt.Sprintf("Unknown field name %q parsing subject permissions, only 'allow' or 'deny' are permitted", k)} *errors = append(*errors, err) } } } return p, nil } // Helper function to validate permissions subjects. func checkPermSubjectArray(sa []string) error { for _, s := range sa { if !IsValidSubject(s) { // Check here if this is a queue group qualified subject. elements := strings.Fields(s) if len(elements) != 2 { return fmt.Errorf("subject %q is not a valid subject", s) } else if !IsValidSubject(elements[0]) { return fmt.Errorf("subject %q is not a valid subject", elements[0]) } } } return nil } // PrintTLSHelpAndDie prints TLS usage and exits. func PrintTLSHelpAndDie() { fmt.Printf("%s", tlsUsage) for k := range cipherMap { fmt.Printf(" %s\n", k) } fmt.Printf("\nAvailable curve preferences include:\n") for k := range curvePreferenceMap { fmt.Printf(" %s\n", k) } if runtime.GOOS == "windows" { fmt.Printf("%s\n", certstore.Usage) } fmt.Printf("%s", certidp.OCSPPeerUsage) fmt.Printf("%s", OCSPResponseCacheUsage) os.Exit(0) } func parseCipher(cipherName string) (uint16, error) { cipher, exists := cipherMap[cipherName] if !exists { return 0, fmt.Errorf("unrecognized cipher %s", cipherName) } return cipher, nil } func parseCurvePreferences(curveName string) (tls.CurveID, error) { curve, exists := curvePreferenceMap[curveName] if !exists { return 0, fmt.Errorf("unrecognized curve preference %s", curveName) } return curve, nil } func parseTLSVersion(v any) (uint16, error) { var tlsVersionNumber uint16 switch v := v.(type) { case string: n, err := tlsVersionFromString(v) if err != nil { return 0, err } tlsVersionNumber = n default: return 0, fmt.Errorf("'min_version' wrong type: %v", v) } if tlsVersionNumber < tls.VersionTLS12 { return 0, fmt.Errorf("unsupported TLS version: %s", tls.VersionName(tlsVersionNumber)) } return tlsVersionNumber, nil } // Helper function to parse TLS configs. func parseTLS(v any, isClientCtx bool) (t *TLSConfigOpts, retErr error) { var ( tlsm map[string]any tc = TLSConfigOpts{} lt token ) defer convertPanicToError(<, &retErr) tk, v := unwrapValue(v, <) tlsm = v.(map[string]any) for mk, mv := range tlsm { tk, mv := unwrapValue(mv, <) switch strings.ToLower(mk) { case "cert_file": certFile, ok := mv.(string) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'cert_file' to be filename"} } tc.CertFile = certFile case "key_file": keyFile, ok := mv.(string) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'key_file' to be filename"} } tc.KeyFile = keyFile case "ca_file": caFile, ok := mv.(string) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'ca_file' to be filename"} } tc.CaFile = caFile case "insecure": insecure, ok := mv.(bool) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'insecure' to be a boolean"} } tc.Insecure = insecure case "verify": verify, ok := mv.(bool) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'verify' to be a boolean"} } tc.Verify = verify case "verify_and_map": verify, ok := mv.(bool) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'verify_and_map' to be a boolean"} } if verify { tc.Verify = verify } tc.Map = verify case "verify_cert_and_check_known_urls": verify, ok := mv.(bool) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'verify_cert_and_check_known_urls' to be a boolean"} } if verify && isClientCtx { return nil, &configErr{tk, "verify_cert_and_check_known_urls not supported in this context"} } if verify { tc.Verify = verify } tc.TLSCheckKnownURLs = verify case "cipher_suites": ra := mv.([]any) if len(ra) == 0 { return nil, &configErr{tk, "error parsing tls config, 'cipher_suites' cannot be empty"} } tc.Ciphers = make([]uint16, 0, len(ra)) for _, r := range ra { tk, r := unwrapValue(r, <) cipher, err := parseCipher(r.(string)) if err != nil { return nil, &configErr{tk, err.Error()} } tc.Ciphers = append(tc.Ciphers, cipher) } case "curve_preferences": ra := mv.([]any) if len(ra) == 0 { return nil, &configErr{tk, "error parsing tls config, 'curve_preferences' cannot be empty"} } tc.CurvePreferences = make([]tls.CurveID, 0, len(ra)) for _, r := range ra { tk, r := unwrapValue(r, <) cps, err := parseCurvePreferences(r.(string)) if err != nil { return nil, &configErr{tk, err.Error()} } tc.CurvePreferences = append(tc.CurvePreferences, cps) } case "timeout": at := float64(0) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv case string: d, err := time.ParseDuration(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf("error parsing tls config, 'timeout' %s", err)} } at = d.Seconds() default: return nil, &configErr{tk, "error parsing tls config, 'timeout' wrong type"} } tc.Timeout = at case "connection_rate_limit": at := int64(0) switch mv := mv.(type) { case int64: at = mv default: return nil, &configErr{tk, "error parsing tls config, 'connection_rate_limit' wrong type"} } tc.RateLimit = at case "pinned_certs": ra, ok := mv.([]any) if !ok { return nil, &configErr{tk, "error parsing tls config, expected 'pinned_certs' to be a list of hex-encoded sha256 of DER encoded SubjectPublicKeyInfo"} } if len(ra) != 0 { wl := PinnedCertSet{} re := regexp.MustCompile("^[A-Fa-f0-9]{64}$") for _, r := range ra { tk, r := unwrapValue(r, <) entry := strings.ToLower(r.(string)) if !re.MatchString(entry) { return nil, &configErr{tk, fmt.Sprintf("error parsing tls config, 'pinned_certs' key %s does not look like hex-encoded sha256 of DER encoded SubjectPublicKeyInfo", entry)} } wl[entry] = struct{}{} } tc.PinnedCerts = wl } case "cert_store": certStore, ok := mv.(string) if !ok || certStore == _EMPTY_ { return nil, &configErr{tk, certstore.ErrBadCertStoreField.Error()} } certStoreType, err := certstore.ParseCertStore(certStore) if err != nil { return nil, &configErr{tk, err.Error()} } tc.CertStore = certStoreType case "cert_match_by": certMatchBy, ok := mv.(string) if !ok || certMatchBy == _EMPTY_ { return nil, &configErr{tk, certstore.ErrBadCertMatchByField.Error()} } certMatchByType, err := certstore.ParseCertMatchBy(certMatchBy) if err != nil { return nil, &configErr{tk, err.Error()} } tc.CertMatchBy = certMatchByType case "cert_match": certMatch, ok := mv.(string) if !ok || certMatch == _EMPTY_ { return nil, &configErr{tk, certstore.ErrBadCertMatchField.Error()} } tc.CertMatch = certMatch case "ca_certs_match": rv := []string{} switch mv := mv.(type) { case string: rv = append(rv, mv) case []string: rv = append(rv, mv...) case []interface{}: for _, t := range mv { if token, ok := t.(token); ok { if ts, ok := token.Value().(string); ok { rv = append(rv, ts) continue } else { return nil, &configErr{tk, fmt.Sprintf("error parsing ca_cert_match: unsupported type %T where string is expected", token)} } } else { return nil, &configErr{tk, fmt.Sprintf("error parsing ca_cert_match: unsupported type %T", t)} } } } tc.CaCertsMatch = rv case "handshake_first", "first", "immediate": switch mv := mv.(type) { case bool: tc.HandshakeFirst = mv case string: switch strings.ToLower(mv) { case "true", "on": tc.HandshakeFirst = true case "false", "off": tc.HandshakeFirst = false case "auto", "auto_fallback": tc.HandshakeFirst = true tc.FallbackDelay = DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY default: // Check to see if this is a duration. if dur, err := time.ParseDuration(mv); err == nil { tc.HandshakeFirst = true tc.FallbackDelay = dur break } return nil, &configErr{tk, fmt.Sprintf("field %q's value %q is invalid", mk, mv)} } default: return nil, &configErr{tk, fmt.Sprintf("field %q should be a boolean or a string, got %T", mk, mv)} } case "cert_match_skip_invalid": certMatchSkipInvalid, ok := mv.(bool) if !ok { return nil, &configErr{tk, certstore.ErrBadCertMatchSkipInvalidField.Error()} } tc.CertMatchSkipInvalid = certMatchSkipInvalid case "ocsp_peer": switch vv := mv.(type) { case bool: pc := certidp.NewOCSPPeerConfig() if vv { // Set enabled pc.Verify = true tc.OCSPPeerConfig = pc } else { // Set disabled pc.Verify = false tc.OCSPPeerConfig = pc } case map[string]any: pc, err := parseOCSPPeer(mv) if err != nil { return nil, &configErr{tk, err.Error()} } tc.OCSPPeerConfig = pc default: return nil, &configErr{tk, fmt.Sprintf("error parsing ocsp peer config: unsupported type %T", v)} } case "certs", "certificates": certs, ok := mv.([]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", v)} } tc.Certificates = make([]*TLSCertPairOpt, len(certs)) for i, v := range certs { tk, vv := unwrapValue(v, <) pair, ok := vv.(map[string]any) if !ok { return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", vv)} } certPair := &TLSCertPairOpt{} for k, v := range pair { tk, vv = unwrapValue(v, <) file, ok := vv.(string) if !ok { return nil, &configErr{tk, fmt.Sprintf("error parsing certificates config: unsupported type %T", vv)} } switch k { case "cert_file": certPair.CertFile = file case "key_file": certPair.KeyFile = file default: return nil, &configErr{tk, fmt.Sprintf("error parsing tls certs config, unknown field %q", k)} } } if certPair.CertFile == _EMPTY_ || certPair.KeyFile == _EMPTY_ { return nil, &configErr{tk, "error parsing certificates config: both 'cert_file' and 'cert_key' options are required"} } tc.Certificates[i] = certPair } case "min_version": minVersion, err := parseTLSVersion(mv) if err != nil { return nil, &configErr{tk, fmt.Sprintf("error parsing tls config: %v", err)} } tc.MinVersion = minVersion default: return nil, &configErr{tk, fmt.Sprintf("error parsing tls config, unknown field %q", mk)} } } if len(tc.Certificates) > 0 && tc.CertFile != _EMPTY_ { return nil, &configErr{tk, "error parsing tls config, cannot combine 'cert_file' option with 'certs' option"} } // If cipher suites were not specified then use the defaults if tc.Ciphers == nil { tc.Ciphers = defaultCipherSuites() } // If curve preferences were not specified, then use the defaults if tc.CurvePreferences == nil { tc.CurvePreferences = defaultCurvePreferences() } return &tc, nil } func parseSimpleAuth(v any, errors *[]error) *authorization { var ( am map[string]any tk token lt token auth = &authorization{} ) defer convertPanicToErrorList(<, errors) _, v = unwrapValue(v, <) am = v.(map[string]any) for mk, mv := range am { tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "user", "username": auth.user = mv.(string) case "pass", "password": auth.pass = mv.(string) case "token": auth.token = mv.(string) case "timeout": at := float64(1) switch mv := mv.(type) { case int64: at = float64(mv) case float64: at = mv } auth.timeout = at default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) } continue } } return auth } func parseStringArray(fieldName string, tk token, lt *token, mv any, errors *[]error) ([]string, error) { switch mv := mv.(type) { case string: return []string{mv}, nil case []any: strs := make([]string, 0, len(mv)) for _, val := range mv { tk, val = unwrapValue(val, lt) if str, ok := val.(string); ok { strs = append(strs, str) } else { err := &configErr{tk, fmt.Sprintf("error parsing %s: unsupported type in array %T", fieldName, val)} *errors = append(*errors, err) continue } } return strs, nil default: err := &configErr{tk, fmt.Sprintf("error parsing %s: unsupported type %T", fieldName, mv)} *errors = append(*errors, err) return nil, err } } func parseWebsocket(v any, o *Options, errors *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) gm, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected websocket to be a map, got %T", v)} } for mk, mv := range gm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "listen": hp, err := parseListen(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } o.Websocket.Host = hp.host o.Websocket.Port = hp.port case "port": o.Websocket.Port = int(mv.(int64)) case "host", "net": o.Websocket.Host = mv.(string) case "advertise": o.Websocket.Advertise = mv.(string) case "no_tls": o.Websocket.NoTLS = mv.(bool) case "tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) continue } if o.Websocket.TLSConfig, err = GenTLSConfig(tc); err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } o.Websocket.TLSMap = tc.Map o.Websocket.TLSPinnedCerts = tc.PinnedCerts o.Websocket.tlsConfigOpts = tc case "same_origin": o.Websocket.SameOrigin = mv.(bool) case "allowed_origins", "allowed_origin", "allow_origins", "allow_origin", "origins", "origin": o.Websocket.AllowedOrigins, _ = parseStringArray("allowed origins", tk, <, mv, errors) case "handshake_timeout": ht := time.Duration(0) switch mv := mv.(type) { case int64: ht = time.Duration(mv) * time.Second case string: var err error ht, err = time.ParseDuration(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } default: err := &configErr{tk, fmt.Sprintf("error parsing handshake timeout: unsupported type %T", mv)} *errors = append(*errors, err) } o.Websocket.HandshakeTimeout = ht case "compress", "compression": o.Websocket.Compression = mv.(bool) case "authorization", "authentication": auth := parseSimpleAuth(tk, errors) o.Websocket.Username = auth.user o.Websocket.Password = auth.pass o.Websocket.Token = auth.token o.Websocket.AuthTimeout = auth.timeout case "jwt_cookie": o.Websocket.JWTCookie = mv.(string) case "no_auth_user": o.Websocket.NoAuthUser = mv.(string) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } return nil } func parseMQTT(v any, o *Options, errors *[]error, warnings *[]error) error { var lt token defer convertPanicToErrorList(<, errors) tk, v := unwrapValue(v, <) gm, ok := v.(map[string]any) if !ok { return &configErr{tk, fmt.Sprintf("Expected mqtt to be a map, got %T", v)} } for mk, mv := range gm { // Again, unwrap token value if line check is required. tk, mv = unwrapValue(mv, <) switch strings.ToLower(mk) { case "listen": hp, err := parseListen(mv) if err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } o.MQTT.Host = hp.host o.MQTT.Port = hp.port case "port": o.MQTT.Port = int(mv.(int64)) case "host", "net": o.MQTT.Host = mv.(string) case "tls": tc, err := parseTLS(tk, true) if err != nil { *errors = append(*errors, err) continue } if o.MQTT.TLSConfig, err = GenTLSConfig(tc); err != nil { err := &configErr{tk, err.Error()} *errors = append(*errors, err) continue } o.MQTT.TLSTimeout = tc.Timeout o.MQTT.TLSMap = tc.Map o.MQTT.TLSPinnedCerts = tc.PinnedCerts o.MQTT.tlsConfigOpts = tc case "authorization", "authentication": auth := parseSimpleAuth(tk, errors) o.MQTT.Username = auth.user o.MQTT.Password = auth.pass o.MQTT.Token = auth.token o.MQTT.AuthTimeout = auth.timeout case "no_auth_user": o.MQTT.NoAuthUser = mv.(string) case "ack_wait", "ackwait": o.MQTT.AckWait = parseDuration("ack_wait", tk, mv, errors, warnings) case "max_ack_pending", "max_pending", "max_inflight": tmp := int(mv.(int64)) if tmp < 0 || tmp > 0xFFFF { err := &configErr{tk, fmt.Sprintf("invalid value %v, should in [0..%d] range", tmp, 0xFFFF)} *errors = append(*errors, err) } else { o.MQTT.MaxAckPending = uint16(tmp) } case "js_domain": o.MQTT.JsDomain = mv.(string) case "stream_replicas": o.MQTT.StreamReplicas = int(mv.(int64)) case "consumer_replicas": err := &configWarningErr{ field: mk, configErr: configErr{ token: tk, reason: `consumer replicas setting ignored in this server version`, }, } *warnings = append(*warnings, err) case "consumer_memory_storage": o.MQTT.ConsumerMemoryStorage = mv.(bool) case "consumer_inactive_threshold", "consumer_auto_cleanup": o.MQTT.ConsumerInactiveThreshold = parseDuration("consumer_inactive_threshold", tk, mv, errors, warnings) case "reject_qos2_publish": o.MQTT.rejectQoS2Pub = mv.(bool) case "downgrade_qos2_subscribe": o.MQTT.downgradeQoS2Sub = mv.(bool) default: if !tk.IsUsedVariable() { err := &unknownConfigFieldErr{ field: mk, configErr: configErr{ token: tk, }, } *errors = append(*errors, err) continue } } } return nil } // GenTLSConfig loads TLS related configuration parameters. func GenTLSConfig(tc *TLSConfigOpts) (*tls.Config, error) { // Create the tls.Config from our options before including the certs. // It will determine the cipher suites that we prefer. // FIXME(dlc) change if ARM based. config := tls.Config{ MinVersion: tls.VersionTLS12, CipherSuites: tc.Ciphers, PreferServerCipherSuites: true, CurvePreferences: tc.CurvePreferences, InsecureSkipVerify: tc.Insecure, } switch { case tc.CertFile != _EMPTY_ && tc.CertStore != certstore.STOREEMPTY: return nil, certstore.ErrConflictCertFileAndStore case tc.CertFile != _EMPTY_ && tc.KeyFile == _EMPTY_: return nil, fmt.Errorf("missing 'key_file' in TLS configuration") case tc.CertFile == _EMPTY_ && tc.KeyFile != _EMPTY_: return nil, fmt.Errorf("missing 'cert_file' in TLS configuration") case tc.CertFile != _EMPTY_ && tc.KeyFile != _EMPTY_: // Now load in cert and private key cert, err := tls.LoadX509KeyPair(tc.CertFile, tc.KeyFile) if err != nil { return nil, fmt.Errorf("error parsing X509 certificate/key pair: %v", err) } cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) if err != nil { return nil, fmt.Errorf("error parsing certificate: %v", err) } config.Certificates = []tls.Certificate{cert} case tc.CertStore != certstore.STOREEMPTY: err := certstore.TLSConfig(tc.CertStore, tc.CertMatchBy, tc.CertMatch, tc.CaCertsMatch, tc.CertMatchSkipInvalid, &config) if err != nil { return nil, err } case tc.Certificates != nil: // Multiple certificate support. config.Certificates = make([]tls.Certificate, len(tc.Certificates)) for i, certPair := range tc.Certificates { cert, err := tls.LoadX509KeyPair(certPair.CertFile, certPair.KeyFile) if err != nil { return nil, fmt.Errorf("error parsing X509 certificate/key pair %d/%d: %v", i+1, len(tc.Certificates), err) } cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) if err != nil { return nil, fmt.Errorf("error parsing certificate %d/%d: %v", i+1, len(tc.Certificates), err) } config.Certificates[i] = cert } } // Require client certificates as needed if tc.Verify { config.ClientAuth = tls.RequireAndVerifyClientCert } // Add in CAs if applicable. if tc.CaFile != _EMPTY_ { rootPEM, err := os.ReadFile(tc.CaFile) if err != nil || rootPEM == nil { return nil, err } pool := x509.NewCertPool() ok := pool.AppendCertsFromPEM(rootPEM) if !ok { return nil, fmt.Errorf("failed to parse root ca certificate") } config.ClientCAs = pool } // Allow setting TLS minimum version. if tc.MinVersion > 0 { if tc.MinVersion < tls.VersionTLS12 { return nil, fmt.Errorf("unsupported minimum TLS version: %s", tls.VersionName(tc.MinVersion)) } config.MinVersion = tc.MinVersion } return &config, nil } // MergeOptions will merge two options giving preference to the flagOpts // if the item is present. func MergeOptions(fileOpts, flagOpts *Options) *Options { if fileOpts == nil { return flagOpts } if flagOpts == nil { return fileOpts } // Merge the two, flagOpts override opts := *fileOpts if flagOpts.Port != 0 { opts.Port = flagOpts.Port } if flagOpts.Host != _EMPTY_ { opts.Host = flagOpts.Host } if flagOpts.DontListen { opts.DontListen = flagOpts.DontListen } if flagOpts.ClientAdvertise != _EMPTY_ { opts.ClientAdvertise = flagOpts.ClientAdvertise } if flagOpts.Username != _EMPTY_ { opts.Username = flagOpts.Username } if flagOpts.Password != _EMPTY_ { opts.Password = flagOpts.Password } if flagOpts.Authorization != _EMPTY_ { opts.Authorization = flagOpts.Authorization } if flagOpts.HTTPPort != 0 { opts.HTTPPort = flagOpts.HTTPPort } if flagOpts.HTTPBasePath != _EMPTY_ { opts.HTTPBasePath = flagOpts.HTTPBasePath } if flagOpts.Debug { opts.Debug = true } if flagOpts.Trace { opts.Trace = true } if flagOpts.Logtime { opts.Logtime = true } if flagOpts.LogFile != _EMPTY_ { opts.LogFile = flagOpts.LogFile } if flagOpts.PidFile != _EMPTY_ { opts.PidFile = flagOpts.PidFile } if flagOpts.PortsFileDir != _EMPTY_ { opts.PortsFileDir = flagOpts.PortsFileDir } if flagOpts.ProfPort != 0 { opts.ProfPort = flagOpts.ProfPort } if flagOpts.Cluster.ListenStr != _EMPTY_ { opts.Cluster.ListenStr = flagOpts.Cluster.ListenStr } if flagOpts.Cluster.NoAdvertise { opts.Cluster.NoAdvertise = true } if flagOpts.Cluster.ConnectRetries != 0 { opts.Cluster.ConnectRetries = flagOpts.Cluster.ConnectRetries } if flagOpts.Cluster.Advertise != _EMPTY_ { opts.Cluster.Advertise = flagOpts.Cluster.Advertise } if flagOpts.RoutesStr != _EMPTY_ { mergeRoutes(&opts, flagOpts) } if flagOpts.JetStream { fileOpts.JetStream = flagOpts.JetStream } return &opts } // RoutesFromStr parses route URLs from a string func RoutesFromStr(routesStr string) []*url.URL { routes := strings.Split(routesStr, ",") if len(routes) == 0 { return nil } routeUrls := []*url.URL{} for _, r := range routes { r = strings.TrimSpace(r) u, _ := url.Parse(r) routeUrls = append(routeUrls, u) } return routeUrls } // This will merge the flag routes and override anything that was present. func mergeRoutes(opts, flagOpts *Options) { routeUrls := RoutesFromStr(flagOpts.RoutesStr) if routeUrls == nil { return } opts.Routes = routeUrls opts.RoutesStr = flagOpts.RoutesStr } // RemoveSelfReference removes this server from an array of routes func RemoveSelfReference(clusterPort int, routes []*url.URL) ([]*url.URL, error) { var cleanRoutes []*url.URL cport := strconv.Itoa(clusterPort) selfIPs, err := getInterfaceIPs() if err != nil { return nil, err } for _, r := range routes { host, port, err := net.SplitHostPort(r.Host) if err != nil { return nil, err } ipList, err := getURLIP(host) if err != nil { return nil, err } if cport == port && isIPInList(selfIPs, ipList) { continue } cleanRoutes = append(cleanRoutes, r) } return cleanRoutes, nil } func isIPInList(list1 []net.IP, list2 []net.IP) bool { for _, ip1 := range list1 { for _, ip2 := range list2 { if ip1.Equal(ip2) { return true } } } return false } func getURLIP(ipStr string) ([]net.IP, error) { ipList := []net.IP{} ip := net.ParseIP(ipStr) if ip != nil { ipList = append(ipList, ip) return ipList, nil } hostAddr, err := net.LookupHost(ipStr) if err != nil { return nil, fmt.Errorf("Error looking up host with route hostname: %v", err) } for _, addr := range hostAddr { ip = net.ParseIP(addr) if ip != nil { ipList = append(ipList, ip) } } return ipList, nil } func getInterfaceIPs() ([]net.IP, error) { var localIPs []net.IP interfaceAddr, err := net.InterfaceAddrs() if err != nil { return nil, fmt.Errorf("Error getting self referencing address: %v", err) } for i := 0; i < len(interfaceAddr); i++ { interfaceIP, _, _ := net.ParseCIDR(interfaceAddr[i].String()) if net.ParseIP(interfaceIP.String()) != nil { localIPs = append(localIPs, interfaceIP) } else { return nil, fmt.Errorf("Error parsing self referencing address: %v", err) } } return localIPs, nil } func setBaselineOptions(opts *Options) { // Setup non-standard Go defaults if opts.Host == _EMPTY_ { opts.Host = DEFAULT_HOST } if opts.HTTPHost == _EMPTY_ { // Default to same bind from server if left undefined opts.HTTPHost = opts.Host } if opts.Port == 0 { opts.Port = DEFAULT_PORT } else if opts.Port == RANDOM_PORT { // Choose randomly inside of net.Listen opts.Port = 0 } if opts.MaxConn == 0 { opts.MaxConn = DEFAULT_MAX_CONNECTIONS } if opts.PingInterval == 0 { opts.PingInterval = DEFAULT_PING_INTERVAL } if opts.MaxPingsOut == 0 { opts.MaxPingsOut = DEFAULT_PING_MAX_OUT } if opts.TLSTimeout == 0 { opts.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second) } if opts.AuthTimeout == 0 { opts.AuthTimeout = getDefaultAuthTimeout(opts.TLSConfig, opts.TLSTimeout) } if opts.Cluster.Port != 0 || opts.Cluster.ListenStr != _EMPTY_ { if opts.Cluster.Host == _EMPTY_ { opts.Cluster.Host = DEFAULT_HOST } if opts.Cluster.TLSTimeout == 0 { opts.Cluster.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second) } if opts.Cluster.AuthTimeout == 0 { opts.Cluster.AuthTimeout = getDefaultAuthTimeout(opts.Cluster.TLSConfig, opts.Cluster.TLSTimeout) } if opts.Cluster.PoolSize == 0 { opts.Cluster.PoolSize = DEFAULT_ROUTE_POOL_SIZE } // Unless pooling/accounts are disabled (by PoolSize being set to -1), // check for Cluster.Accounts. Add the system account if not present and // unless we have a configuration that disabled it. if opts.Cluster.PoolSize > 0 { sysAccName := opts.SystemAccount if sysAccName == _EMPTY_ && !opts.NoSystemAccount { sysAccName = DEFAULT_SYSTEM_ACCOUNT } if sysAccName != _EMPTY_ { var found bool for _, acc := range opts.Cluster.PinnedAccounts { if acc == sysAccName { found = true break } } if !found { opts.Cluster.PinnedAccounts = append(opts.Cluster.PinnedAccounts, sysAccName) } } } // Default to compression "accept", which means that compression is not // initiated, but if the remote selects compression, this server will // use the same. if c := &opts.Cluster.Compression; c.Mode == _EMPTY_ { if testDefaultClusterCompression != _EMPTY_ { c.Mode = testDefaultClusterCompression } else { c.Mode = CompressionAccept } } } if opts.LeafNode.Port != 0 { if opts.LeafNode.Host == _EMPTY_ { opts.LeafNode.Host = DEFAULT_HOST } if opts.LeafNode.TLSTimeout == 0 { opts.LeafNode.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second) } if opts.LeafNode.AuthTimeout == 0 { opts.LeafNode.AuthTimeout = getDefaultAuthTimeout(opts.LeafNode.TLSConfig, opts.LeafNode.TLSTimeout) } // Default to compression "s2_auto". if c := &opts.LeafNode.Compression; c.Mode == _EMPTY_ { if testDefaultLeafNodeCompression != _EMPTY_ { c.Mode = testDefaultLeafNodeCompression } else { c.Mode = CompressionS2Auto } } } // Set baseline connect port for remotes. for _, r := range opts.LeafNode.Remotes { if r != nil { for _, u := range r.URLs { if u.Port() == _EMPTY_ { u.Host = net.JoinHostPort(u.Host, strconv.Itoa(DEFAULT_LEAFNODE_PORT)) } } // Default to compression "s2_auto". if c := &r.Compression; c.Mode == _EMPTY_ { if testDefaultLeafNodeCompression != _EMPTY_ { c.Mode = testDefaultLeafNodeCompression } else { c.Mode = CompressionS2Auto } } // Set default first info timeout value if not set. if r.FirstInfoTimeout <= 0 { r.FirstInfoTimeout = DEFAULT_LEAFNODE_INFO_WAIT } } } // Set this regardless of opts.LeafNode.Port if opts.LeafNode.ReconnectInterval == 0 { opts.LeafNode.ReconnectInterval = DEFAULT_LEAF_NODE_RECONNECT } if opts.MaxControlLine == 0 { opts.MaxControlLine = MAX_CONTROL_LINE_SIZE } if opts.MaxPayload == 0 { opts.MaxPayload = MAX_PAYLOAD_SIZE } if opts.MaxPending == 0 { opts.MaxPending = MAX_PENDING_SIZE } if opts.WriteDeadline == time.Duration(0) { opts.WriteDeadline = DEFAULT_FLUSH_DEADLINE } if opts.MaxClosedClients == 0 { opts.MaxClosedClients = DEFAULT_MAX_CLOSED_CLIENTS } if opts.LameDuckDuration == 0 { opts.LameDuckDuration = DEFAULT_LAME_DUCK_DURATION } if opts.LameDuckGracePeriod == 0 { opts.LameDuckGracePeriod = DEFAULT_LAME_DUCK_GRACE_PERIOD } if opts.Gateway.Port != 0 { if opts.Gateway.Host == _EMPTY_ { opts.Gateway.Host = DEFAULT_HOST } if opts.Gateway.TLSTimeout == 0 { opts.Gateway.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second) } if opts.Gateway.AuthTimeout == 0 { opts.Gateway.AuthTimeout = getDefaultAuthTimeout(opts.Gateway.TLSConfig, opts.Gateway.TLSTimeout) } } if opts.ConnectErrorReports == 0 { opts.ConnectErrorReports = DEFAULT_CONNECT_ERROR_REPORTS } if opts.ReconnectErrorReports == 0 { opts.ReconnectErrorReports = DEFAULT_RECONNECT_ERROR_REPORTS } if opts.Websocket.Port != 0 { if opts.Websocket.Host == _EMPTY_ { opts.Websocket.Host = DEFAULT_HOST } } if opts.MQTT.Port != 0 { if opts.MQTT.Host == _EMPTY_ { opts.MQTT.Host = DEFAULT_HOST } if opts.MQTT.TLSTimeout == 0 { opts.MQTT.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second) } } // JetStream if opts.JetStreamMaxMemory == 0 && !opts.maxMemSet { opts.JetStreamMaxMemory = -1 } if opts.JetStreamMaxStore == 0 && !opts.maxStoreSet { opts.JetStreamMaxStore = -1 } if opts.SyncInterval == 0 && !opts.syncSet { opts.SyncInterval = defaultSyncInterval } if opts.JetStreamRequestQueueLimit <= 0 { opts.JetStreamRequestQueueLimit = JSDefaultRequestQueueLimit } } func getDefaultAuthTimeout(tls *tls.Config, tlsTimeout float64) float64 { var authTimeout float64 if tls != nil { authTimeout = tlsTimeout + 1.0 } else { authTimeout = float64(AUTH_TIMEOUT / time.Second) } return authTimeout } // ConfigureOptions accepts a flag set and augments it with NATS Server // specific flags. On success, an options structure is returned configured // based on the selected flags and/or configuration file. // The command line options take precedence to the ones in the configuration file. func ConfigureOptions(fs *flag.FlagSet, args []string, printVersion, printHelp, printTLSHelp func()) (*Options, error) { opts := &Options{} var ( showVersion bool showHelp bool showTLSHelp bool signal string configFile string dbgAndTrace bool trcAndVerboseTrc bool dbgAndTrcAndVerboseTrc bool err error ) fs.BoolVar(&showHelp, "h", false, "Show this message.") fs.BoolVar(&showHelp, "help", false, "Show this message.") fs.IntVar(&opts.Port, "port", 0, "Port to listen on.") fs.IntVar(&opts.Port, "p", 0, "Port to listen on.") fs.StringVar(&opts.ServerName, "n", _EMPTY_, "Server name.") fs.StringVar(&opts.ServerName, "name", _EMPTY_, "Server name.") fs.StringVar(&opts.ServerName, "server_name", _EMPTY_, "Server name.") fs.StringVar(&opts.Host, "addr", _EMPTY_, "Network host to listen on.") fs.StringVar(&opts.Host, "a", _EMPTY_, "Network host to listen on.") fs.StringVar(&opts.Host, "net", _EMPTY_, "Network host to listen on.") fs.StringVar(&opts.ClientAdvertise, "client_advertise", _EMPTY_, "Client URL to advertise to other servers.") fs.BoolVar(&opts.Debug, "D", false, "Enable Debug logging.") fs.BoolVar(&opts.Debug, "debug", false, "Enable Debug logging.") fs.BoolVar(&opts.Trace, "V", false, "Enable Trace logging.") fs.BoolVar(&trcAndVerboseTrc, "VV", false, "Enable Verbose Trace logging. (Traces system account as well)") fs.BoolVar(&opts.Trace, "trace", false, "Enable Trace logging.") fs.BoolVar(&dbgAndTrace, "DV", false, "Enable Debug and Trace logging.") fs.BoolVar(&dbgAndTrcAndVerboseTrc, "DVV", false, "Enable Debug and Verbose Trace logging. (Traces system account as well)") fs.BoolVar(&opts.Logtime, "T", true, "Timestamp log entries.") fs.BoolVar(&opts.Logtime, "logtime", true, "Timestamp log entries.") fs.BoolVar(&opts.LogtimeUTC, "logtime_utc", false, "Timestamps in UTC instead of local timezone.") fs.StringVar(&opts.Username, "user", _EMPTY_, "Username required for connection.") fs.StringVar(&opts.Password, "pass", _EMPTY_, "Password required for connection.") fs.StringVar(&opts.Authorization, "auth", _EMPTY_, "Authorization token required for connection.") fs.IntVar(&opts.HTTPPort, "m", 0, "HTTP Port for /varz, /connz endpoints.") fs.IntVar(&opts.HTTPPort, "http_port", 0, "HTTP Port for /varz, /connz endpoints.") fs.IntVar(&opts.HTTPSPort, "ms", 0, "HTTPS Port for /varz, /connz endpoints.") fs.IntVar(&opts.HTTPSPort, "https_port", 0, "HTTPS Port for /varz, /connz endpoints.") fs.StringVar(&configFile, "c", _EMPTY_, "Configuration file.") fs.StringVar(&configFile, "config", _EMPTY_, "Configuration file.") fs.BoolVar(&opts.CheckConfig, "t", false, "Check configuration and exit.") fs.StringVar(&signal, "sl", "", "Send signal to nats-server process (ldm, stop, quit, term, reopen, reload).") fs.StringVar(&signal, "signal", "", "Send signal to nats-server process (ldm, stop, quit, term, reopen, reload).") fs.StringVar(&opts.PidFile, "P", "", "File to store process pid.") fs.StringVar(&opts.PidFile, "pid", "", "File to store process pid.") fs.StringVar(&opts.PortsFileDir, "ports_file_dir", "", "Creates a ports file in the specified directory (_.ports).") fs.StringVar(&opts.LogFile, "l", "", "File to store logging output.") fs.StringVar(&opts.LogFile, "log", "", "File to store logging output.") fs.Int64Var(&opts.LogSizeLimit, "log_size_limit", 0, "Logfile size limit being auto-rotated") fs.BoolVar(&opts.Syslog, "s", false, "Enable syslog as log method.") fs.BoolVar(&opts.Syslog, "syslog", false, "Enable syslog as log method.") fs.StringVar(&opts.RemoteSyslog, "r", _EMPTY_, "Syslog server addr (udp://127.0.0.1:514).") fs.StringVar(&opts.RemoteSyslog, "remote_syslog", _EMPTY_, "Syslog server addr (udp://127.0.0.1:514).") fs.BoolVar(&showVersion, "version", false, "Print version information.") fs.BoolVar(&showVersion, "v", false, "Print version information.") fs.IntVar(&opts.ProfPort, "profile", 0, "Profiling HTTP port.") fs.StringVar(&opts.RoutesStr, "routes", _EMPTY_, "Routes to actively solicit a connection.") fs.StringVar(&opts.Cluster.ListenStr, "cluster", _EMPTY_, "Cluster url from which members can solicit routes.") fs.StringVar(&opts.Cluster.ListenStr, "cluster_listen", _EMPTY_, "Cluster url from which members can solicit routes.") fs.StringVar(&opts.Cluster.Advertise, "cluster_advertise", _EMPTY_, "Cluster URL to advertise to other servers.") fs.BoolVar(&opts.Cluster.NoAdvertise, "no_advertise", false, "Advertise known cluster IPs to clients.") fs.IntVar(&opts.Cluster.ConnectRetries, "connect_retries", 0, "For implicit routes, number of connect retries.") fs.StringVar(&opts.Cluster.Name, "cluster_name", _EMPTY_, "Cluster Name, if not set one will be dynamically generated.") fs.BoolVar(&showTLSHelp, "help_tls", false, "TLS help.") fs.BoolVar(&opts.TLS, "tls", false, "Enable TLS.") fs.BoolVar(&opts.TLSVerify, "tlsverify", false, "Enable TLS with client verification.") fs.StringVar(&opts.TLSCert, "tlscert", _EMPTY_, "Server certificate file.") fs.StringVar(&opts.TLSKey, "tlskey", _EMPTY_, "Private key for server certificate.") fs.StringVar(&opts.TLSCaCert, "tlscacert", _EMPTY_, "Client certificate CA for verification.") fs.IntVar(&opts.MaxTracedMsgLen, "max_traced_msg_len", 0, "Maximum printable length for traced messages. 0 for unlimited.") fs.BoolVar(&opts.JetStream, "js", false, "Enable JetStream.") fs.BoolVar(&opts.JetStream, "jetstream", false, "Enable JetStream.") fs.StringVar(&opts.StoreDir, "sd", _EMPTY_, "Storage directory.") fs.StringVar(&opts.StoreDir, "store_dir", _EMPTY_, "Storage directory.") // The flags definition above set "default" values to some of the options. // Calling Parse() here will override the default options with any value // specified from the command line. This is ok. We will then update the // options with the content of the configuration file (if present), and then, // call Parse() again to override the default+config with command line values. // Calling Parse() before processing config file is necessary since configFile // itself is a command line argument, and also Parse() is required in order // to know if user wants simply to show "help" or "version", etc... if err := fs.Parse(args); err != nil { return nil, err } if showVersion { printVersion() return nil, nil } if showHelp { printHelp() return nil, nil } if showTLSHelp { printTLSHelp() return nil, nil } // Process args looking for non-flag options, // 'version' and 'help' only for now showVersion, showHelp, err = ProcessCommandLineArgs(fs) if err != nil { return nil, err } else if showVersion { printVersion() return nil, nil } else if showHelp { printHelp() return nil, nil } // Snapshot flag options. FlagSnapshot = opts.Clone() // Keep track of the boolean flags that were explicitly set with their value. fs.Visit(func(f *flag.Flag) { switch f.Name { case "DVV": trackExplicitVal(&FlagSnapshot.inCmdLine, "Debug", dbgAndTrcAndVerboseTrc) trackExplicitVal(&FlagSnapshot.inCmdLine, "Trace", dbgAndTrcAndVerboseTrc) trackExplicitVal(&FlagSnapshot.inCmdLine, "TraceVerbose", dbgAndTrcAndVerboseTrc) case "DV": trackExplicitVal(&FlagSnapshot.inCmdLine, "Debug", dbgAndTrace) trackExplicitVal(&FlagSnapshot.inCmdLine, "Trace", dbgAndTrace) case "D": fallthrough case "debug": trackExplicitVal(&FlagSnapshot.inCmdLine, "Debug", FlagSnapshot.Debug) case "VV": trackExplicitVal(&FlagSnapshot.inCmdLine, "Trace", trcAndVerboseTrc) trackExplicitVal(&FlagSnapshot.inCmdLine, "TraceVerbose", trcAndVerboseTrc) case "V": fallthrough case "trace": trackExplicitVal(&FlagSnapshot.inCmdLine, "Trace", FlagSnapshot.Trace) case "T": fallthrough case "logtime": trackExplicitVal(&FlagSnapshot.inCmdLine, "Logtime", FlagSnapshot.Logtime) case "s": fallthrough case "syslog": trackExplicitVal(&FlagSnapshot.inCmdLine, "Syslog", FlagSnapshot.Syslog) case "no_advertise": trackExplicitVal(&FlagSnapshot.inCmdLine, "Cluster.NoAdvertise", FlagSnapshot.Cluster.NoAdvertise) } }) // Process signal control. if signal != _EMPTY_ { if err := processSignal(signal); err != nil { return nil, err } } // Parse config if given if configFile != _EMPTY_ { // This will update the options with values from the config file. err := opts.ProcessConfigFile(configFile) if err != nil { if opts.CheckConfig { return nil, err } if cerr, ok := err.(*processConfigErr); !ok || len(cerr.Errors()) != 0 { return nil, err } // If we get here we only have warnings and can still continue fmt.Fprint(os.Stderr, err) } else if opts.CheckConfig { // Report configuration file syntax test was successful and exit. return opts, nil } // Call this again to override config file options with options from command line. // Note: We don't need to check error here since if there was an error, it would // have been caught the first time this function was called (after setting up the // flags). fs.Parse(args) } else if opts.CheckConfig { return nil, fmt.Errorf("must specify [-c, --config] option to check configuration file syntax") } // Special handling of some flags var ( flagErr error tlsDisabled bool tlsOverride bool ) fs.Visit(func(f *flag.Flag) { // short-circuit if an error was encountered if flagErr != nil { return } if strings.HasPrefix(f.Name, "tls") { if f.Name == "tls" { if !opts.TLS { // User has specified "-tls=false", we need to disable TLS opts.TLSConfig = nil tlsDisabled = true tlsOverride = false return } tlsOverride = true } else if !tlsDisabled { tlsOverride = true } } else { switch f.Name { case "VV": opts.Trace, opts.TraceVerbose = trcAndVerboseTrc, trcAndVerboseTrc case "DVV": opts.Trace, opts.Debug, opts.TraceVerbose = dbgAndTrcAndVerboseTrc, dbgAndTrcAndVerboseTrc, dbgAndTrcAndVerboseTrc case "DV": // Check value to support -DV=false opts.Trace, opts.Debug = dbgAndTrace, dbgAndTrace case "cluster", "cluster_listen": // Override cluster config if explicitly set via flags. flagErr = overrideCluster(opts) case "routes": // Keep in mind that the flag has updated opts.RoutesStr at this point. if opts.RoutesStr == _EMPTY_ { // Set routes array to nil since routes string is empty opts.Routes = nil return } routeUrls := RoutesFromStr(opts.RoutesStr) opts.Routes = routeUrls } } }) if flagErr != nil { return nil, flagErr } // This will be true if some of the `-tls` params have been set and // `-tls=false` has not been set. if tlsOverride { if err := overrideTLS(opts); err != nil { return nil, err } } // If we don't have cluster defined in the configuration // file and no cluster listen string override, but we do // have a routes override, we need to report misconfiguration. if opts.RoutesStr != _EMPTY_ && opts.Cluster.ListenStr == _EMPTY_ && opts.Cluster.Host == _EMPTY_ && opts.Cluster.Port == 0 { return nil, errors.New("solicited routes require cluster capabilities, e.g. --cluster") } return opts, nil } func normalizeBasePath(p string) string { if len(p) == 0 { return "/" } // add leading slash if p[0] != '/' { p = "/" + p } return path.Clean(p) } // overrideTLS is called when at least "-tls=true" has been set. func overrideTLS(opts *Options) error { if opts.TLSCert == _EMPTY_ { return errors.New("TLS Server certificate must be present and valid") } if opts.TLSKey == _EMPTY_ { return errors.New("TLS Server private key must be present and valid") } tc := TLSConfigOpts{} tc.CertFile = opts.TLSCert tc.KeyFile = opts.TLSKey tc.CaFile = opts.TLSCaCert tc.Verify = opts.TLSVerify tc.Ciphers = defaultCipherSuites() var err error opts.TLSConfig, err = GenTLSConfig(&tc) return err } // overrideCluster updates Options.Cluster if that flag "cluster" (or "cluster_listen") // has explicitly be set in the command line. If it is set to empty string, it will // clear the Cluster options. func overrideCluster(opts *Options) error { if opts.Cluster.ListenStr == _EMPTY_ { // This one is enough to disable clustering. opts.Cluster.Port = 0 return nil } // -1 will fail url.Parse, so if we have -1, change it to // 0, and then after parse, replace the port with -1 so we get // automatic port allocation wantsRandom := false if strings.HasSuffix(opts.Cluster.ListenStr, ":-1") { wantsRandom = true cls := fmt.Sprintf("%s:0", opts.Cluster.ListenStr[0:len(opts.Cluster.ListenStr)-3]) opts.Cluster.ListenStr = cls } clusterURL, err := url.Parse(opts.Cluster.ListenStr) if err != nil { return err } h, p, err := net.SplitHostPort(clusterURL.Host) if err != nil { return err } if wantsRandom { p = "-1" } opts.Cluster.Host = h _, err = fmt.Sscan(p, &opts.Cluster.Port) if err != nil { return err } if clusterURL.User != nil { pass, hasPassword := clusterURL.User.Password() if !hasPassword { return errors.New("expected cluster password to be set") } opts.Cluster.Password = pass user := clusterURL.User.Username() opts.Cluster.Username = user } else { // Since we override from flag and there is no user/pwd, make // sure we clear what we may have gotten from config file. opts.Cluster.Username = _EMPTY_ opts.Cluster.Password = _EMPTY_ } return nil } func processSignal(signal string) error { var ( pid string commandAndPid = strings.Split(signal, "=") ) if l := len(commandAndPid); l == 2 { pid = maybeReadPidFile(commandAndPid[1]) } else if l > 2 { return fmt.Errorf("invalid signal parameters: %v", commandAndPid[2:]) } if err := ProcessSignal(Command(commandAndPid[0]), pid); err != nil { return err } os.Exit(0) return nil } // maybeReadPidFile returns a PID or Windows service name obtained via the following method: // 1. Try to open a file with path "pidStr" (absolute or relative). // 2. If such a file exists and can be read, return its contents. // 3. Otherwise, return the original "pidStr" string. func maybeReadPidFile(pidStr string) string { if b, err := os.ReadFile(pidStr); err == nil { return string(b) } return pidStr } func homeDir() (string, error) { if runtime.GOOS == "windows" { homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH") userProfile := os.Getenv("USERPROFILE") home := filepath.Join(homeDrive, homePath) if homeDrive == _EMPTY_ || homePath == _EMPTY_ { if userProfile == _EMPTY_ { return _EMPTY_, errors.New("nats: failed to get home dir, require %HOMEDRIVE% and %HOMEPATH% or %USERPROFILE%") } home = userProfile } return home, nil } home := os.Getenv("HOME") if home == _EMPTY_ { return _EMPTY_, errors.New("failed to get home dir, require $HOME") } return home, nil } func expandPath(p string) (string, error) { p = os.ExpandEnv(p) if !strings.HasPrefix(p, "~") { return p, nil } home, err := homeDir() if err != nil { return _EMPTY_, err } return filepath.Join(home, p[1:]), nil } nats-server-2.10.27/server/opts_test.go000066400000000000000000002735371477524627100200550ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "encoding/json" "flag" "fmt" "net/url" "os" "reflect" "runtime" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func checkOptionsEqual(t *testing.T, golden, opts *Options) { t.Helper() // Clone them so we can remove private fields that we don't // want to be compared. goldenClone := golden.Clone() goldenClone.inConfig, goldenClone.inCmdLine = nil, nil optsClone := opts.Clone() optsClone.inConfig, optsClone.inCmdLine = nil, nil if !reflect.DeepEqual(goldenClone, optsClone) { t.Fatalf("Options are incorrect.\nexpected: %+v\ngot: %+v", goldenClone, optsClone) } } func TestDefaultOptions(t *testing.T) { golden := &Options{ Host: DEFAULT_HOST, Port: DEFAULT_PORT, MaxConn: DEFAULT_MAX_CONNECTIONS, HTTPHost: DEFAULT_HOST, PingInterval: DEFAULT_PING_INTERVAL, MaxPingsOut: DEFAULT_PING_MAX_OUT, TLSTimeout: float64(TLS_TIMEOUT) / float64(time.Second), AuthTimeout: float64(AUTH_TIMEOUT) / float64(time.Second), MaxControlLine: MAX_CONTROL_LINE_SIZE, MaxPayload: MAX_PAYLOAD_SIZE, MaxPending: MAX_PENDING_SIZE, WriteDeadline: DEFAULT_FLUSH_DEADLINE, MaxClosedClients: DEFAULT_MAX_CLOSED_CLIENTS, LameDuckDuration: DEFAULT_LAME_DUCK_DURATION, LameDuckGracePeriod: DEFAULT_LAME_DUCK_GRACE_PERIOD, LeafNode: LeafNodeOpts{ ReconnectInterval: DEFAULT_LEAF_NODE_RECONNECT, }, ConnectErrorReports: DEFAULT_CONNECT_ERROR_REPORTS, ReconnectErrorReports: DEFAULT_RECONNECT_ERROR_REPORTS, MaxTracedMsgLen: 0, JetStreamMaxMemory: -1, JetStreamMaxStore: -1, SyncInterval: 2 * time.Minute, JetStreamRequestQueueLimit: JSDefaultRequestQueueLimit, } opts := &Options{} setBaselineOptions(opts) checkOptionsEqual(t, golden, opts) } func TestOptions_RandomPort(t *testing.T) { opts := &Options{Port: RANDOM_PORT} setBaselineOptions(opts) if opts.Port != 0 { t.Fatalf("Process of options should have resolved random port to "+ "zero.\nexpected: %d\ngot: %d", 0, opts.Port) } } func TestConfigFile(t *testing.T) { golden := &Options{ ConfigFile: "./configs/test.conf", ServerName: "testing_server", Host: "127.0.0.1", Port: 4242, Username: "derek", Password: "porkchop", AuthTimeout: 1.0, Debug: false, Trace: true, Logtime: false, HTTPPort: 8222, HTTPBasePath: "/nats", PidFile: "/tmp/nats-server/nats-server.pid", ProfPort: 6543, Syslog: true, RemoteSyslog: "udp://foo.com:33", MaxControlLine: 2048, MaxPayload: 65536, MaxConn: 100, MaxSubs: 1000, MaxPending: 10000000, PingInterval: 60 * time.Second, MaxPingsOut: 3, WriteDeadline: 3 * time.Second, LameDuckDuration: 4 * time.Minute, ConnectErrorReports: 86400, ReconnectErrorReports: 5, authBlockDefined: true, } opts, err := ProcessConfigFile("./configs/test.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } checkOptionsEqual(t, golden, opts) } func TestTLSConfigFile(t *testing.T) { golden := &Options{ ConfigFile: "./configs/tls.conf", Host: "127.0.0.1", Port: 4443, Username: "derek", Password: "foo", AuthTimeout: 1.0, TLSTimeout: 2.0, authBlockDefined: true, } opts, err := ProcessConfigFile("./configs/tls.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } tlsConfig := opts.TLSConfig if tlsConfig == nil { t.Fatal("Expected opts.TLSConfig to be non-nil") } opts.TLSConfig = nil opts.tlsConfigOpts = nil checkOptionsEqual(t, golden, opts) // Now check TLSConfig a bit more closely // CipherSuites ciphers := defaultCipherSuites() if !reflect.DeepEqual(tlsConfig.CipherSuites, ciphers) { t.Fatalf("Got incorrect cipher suite list: [%+v]", tlsConfig.CipherSuites) } if tlsConfig.MinVersion != tls.VersionTLS12 { t.Fatalf("Expected MinVersion of 1.2 [%v], got [%v]", tls.VersionTLS12, tlsConfig.MinVersion) } //lint:ignore SA1019 We want to retry on a bunch of errors here. if !tlsConfig.PreferServerCipherSuites { // nolint:staticcheck t.Fatal("Expected PreferServerCipherSuites to be true") } // Verify hostname is correct in certificate if len(tlsConfig.Certificates) != 1 { t.Fatal("Expected 1 certificate") } cert := tlsConfig.Certificates[0].Leaf if err := cert.VerifyHostname("127.0.0.1"); err != nil { t.Fatalf("Could not verify hostname in certificate: %v", err) } // Now test adding cipher suites. opts, err = ProcessConfigFile("./configs/tls_ciphers.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } tlsConfig = opts.TLSConfig if tlsConfig == nil { t.Fatal("Expected opts.TLSConfig to be non-nil") } // CipherSuites listed in the config - test all of them. ciphers = []uint16{ tls.TLS_RSA_WITH_RC4_128_SHA, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, } if !reflect.DeepEqual(tlsConfig.CipherSuites, ciphers) { t.Fatalf("Got incorrect cipher suite list: [%+v]", tlsConfig.CipherSuites) } // Test an unrecognized/bad cipher if _, err := ProcessConfigFile("./configs/tls_bad_cipher.conf"); err == nil { t.Fatal("Did not receive an error from a unrecognized cipher") } // Test an empty cipher entry in a config file. if _, err := ProcessConfigFile("./configs/tls_empty_cipher.conf"); err == nil { t.Fatal("Did not receive an error from empty cipher_suites") } // Test a curve preference from the config. curves := []tls.CurveID{ tls.CurveP256, } // test on a file that will load the curve preference defaults opts, err = ProcessConfigFile("./configs/tls_ciphers.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } if !reflect.DeepEqual(opts.TLSConfig.CurvePreferences, defaultCurvePreferences()) { t.Fatalf("Got incorrect curve preference list: [%+v]", tlsConfig.CurvePreferences) } // Test specifying a single curve preference opts, err = ProcessConfigFile("./configs/tls_curve_prefs.conf") if err != nil { t.Fatal("Did not receive an error from a unrecognized cipher.") } if !reflect.DeepEqual(opts.TLSConfig.CurvePreferences, curves) { t.Fatalf("Got incorrect cipher suite list: [%+v]", tlsConfig.CurvePreferences) } // Test an unrecognized/bad curve preference if _, err := ProcessConfigFile("./configs/tls_bad_curve_prefs.conf"); err == nil { t.Fatal("Did not receive an error from a unrecognized curve preference") } // Test an empty curve preference if _, err := ProcessConfigFile("./configs/tls_empty_curve_prefs.conf"); err == nil { t.Fatal("Did not receive an error from empty curve preferences") } } func TestMergeOverrides(t *testing.T) { golden := &Options{ ConfigFile: "./configs/test.conf", ServerName: "testing_server", Host: "127.0.0.1", Port: 2222, Username: "derek", Password: "porkchop", AuthTimeout: 1.0, Debug: true, Trace: true, Logtime: false, HTTPPort: DEFAULT_HTTP_PORT, HTTPBasePath: DEFAULT_HTTP_BASE_PATH, PidFile: "/tmp/nats-server/nats-server.pid", ProfPort: 6789, Syslog: true, RemoteSyslog: "udp://foo.com:33", MaxControlLine: 2048, MaxPayload: 65536, MaxConn: 100, MaxSubs: 1000, MaxPending: 10000000, PingInterval: 60 * time.Second, MaxPingsOut: 3, Cluster: ClusterOpts{ NoAdvertise: true, ConnectRetries: 2, }, WriteDeadline: 3 * time.Second, LameDuckDuration: 4 * time.Minute, ConnectErrorReports: 86400, ReconnectErrorReports: 5, authBlockDefined: true, } fopts, err := ProcessConfigFile("./configs/test.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } // Overrides via flags opts := &Options{ Port: 2222, Password: "porkchop", Debug: true, HTTPPort: DEFAULT_HTTP_PORT, HTTPBasePath: DEFAULT_HTTP_BASE_PATH, ProfPort: 6789, Cluster: ClusterOpts{ NoAdvertise: true, ConnectRetries: 2, }, } merged := MergeOptions(fopts, opts) checkOptionsEqual(t, golden, merged) } func TestRemoveSelfReference(t *testing.T) { url1, _ := url.Parse("nats-route://user:password@10.4.5.6:4223") url2, _ := url.Parse("nats-route://user:password@127.0.0.1:4223") url3, _ := url.Parse("nats-route://user:password@127.0.0.1:4223") routes := []*url.URL{url1, url2, url3} newroutes, err := RemoveSelfReference(4223, routes) if err != nil { t.Fatalf("Error during RemoveSelfReference: %v", err) } if len(newroutes) != 1 { t.Fatalf("Wrong number of routes: %d", len(newroutes)) } if newroutes[0] != routes[0] { t.Fatalf("Self reference IP address %s in Routes", routes[0]) } } func TestAllowRouteWithDifferentPort(t *testing.T) { url1, _ := url.Parse("nats-route://user:password@127.0.0.1:4224") routes := []*url.URL{url1} newroutes, err := RemoveSelfReference(4223, routes) if err != nil { t.Fatalf("Error during RemoveSelfReference: %v", err) } if len(newroutes) != 1 { t.Fatalf("Wrong number of routes: %d", len(newroutes)) } } func TestRouteFlagOverride(t *testing.T) { routeFlag := "nats-route://ruser:top_secret@127.0.0.1:8246" rurl, _ := url.Parse(routeFlag) golden := &Options{ ConfigFile: "./configs/srv_a.conf", Host: "127.0.0.1", Port: 7222, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: 7244, Username: "ruser", Password: "top_secret", AuthTimeout: 0.5, }, Routes: []*url.URL{rurl}, RoutesStr: routeFlag, } fopts, err := ProcessConfigFile("./configs/srv_a.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } // Overrides via flags opts := &Options{ RoutesStr: routeFlag, } merged := MergeOptions(fopts, opts) checkOptionsEqual(t, golden, merged) } func TestClusterFlagsOverride(t *testing.T) { routeFlag := "nats-route://ruser:top_secret@127.0.0.1:7246" rurl, _ := url.Parse(routeFlag) // In this test, we override the cluster listen string. Note that in // the golden options, the cluster other infos correspond to what // is recovered from the configuration file, this explains the // discrepency between ClusterListenStr and the rest. // The server would then process the ClusterListenStr override and // correctly override ClusterHost/ClustherPort/etc.. golden := &Options{ ConfigFile: "./configs/srv_a.conf", Host: "127.0.0.1", Port: 7222, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: 7244, ListenStr: "nats://127.0.0.1:8224", Username: "ruser", Password: "top_secret", AuthTimeout: 0.5, }, Routes: []*url.URL{rurl}, } fopts, err := ProcessConfigFile("./configs/srv_a.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } // Overrides via flags opts := &Options{ Cluster: ClusterOpts{ ListenStr: "nats://127.0.0.1:8224", }, } merged := MergeOptions(fopts, opts) checkOptionsEqual(t, golden, merged) } func TestRouteFlagOverrideWithMultiple(t *testing.T) { routeFlag := "nats-route://ruser:top_secret@127.0.0.1:8246, nats-route://ruser:top_secret@127.0.0.1:8266" rurls := RoutesFromStr(routeFlag) golden := &Options{ ConfigFile: "./configs/srv_a.conf", Host: "127.0.0.1", Port: 7222, Cluster: ClusterOpts{ Host: "127.0.0.1", Name: "abc", Port: 7244, Username: "ruser", Password: "top_secret", AuthTimeout: 0.5, }, Routes: rurls, RoutesStr: routeFlag, } fopts, err := ProcessConfigFile("./configs/srv_a.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } // Overrides via flags opts := &Options{ RoutesStr: routeFlag, } merged := MergeOptions(fopts, opts) checkOptionsEqual(t, golden, merged) } func TestDynamicPortOnListen(t *testing.T) { opts, err := ProcessConfigFile("./configs/listen-1.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } if opts.Port != -1 { t.Fatalf("Received incorrect port %v, expected -1", opts.Port) } if opts.HTTPPort != -1 { t.Fatalf("Received incorrect monitoring port %v, expected -1", opts.HTTPPort) } if opts.HTTPSPort != -1 { t.Fatalf("Received incorrect secure monitoring port %v, expected -1", opts.HTTPSPort) } } func TestListenConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/listen.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) // Normal clients host := "10.0.1.22" port := 4422 monHost := "127.0.0.1" if opts.Host != host { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, host) } if opts.HTTPHost != monHost { t.Fatalf("Received incorrect host %q, expected %q", opts.HTTPHost, monHost) } if opts.Port != port { t.Fatalf("Received incorrect port %v, expected %v", opts.Port, port) } // Clustering clusterHost := "127.0.0.1" clusterPort := 4244 if opts.Cluster.Host != clusterHost { t.Fatalf("Received incorrect cluster host %q, expected %q", opts.Cluster.Host, clusterHost) } if opts.Cluster.Port != clusterPort { t.Fatalf("Received incorrect cluster port %v, expected %v", opts.Cluster.Port, clusterPort) } // HTTP httpHost := "127.0.0.1" httpPort := 8422 if opts.HTTPHost != httpHost { t.Fatalf("Received incorrect http host %q, expected %q", opts.HTTPHost, httpHost) } if opts.HTTPPort != httpPort { t.Fatalf("Received incorrect http port %v, expected %v", opts.HTTPPort, httpPort) } // HTTPS httpsPort := 9443 if opts.HTTPSPort != httpsPort { t.Fatalf("Received incorrect https port %v, expected %v", opts.HTTPSPort, httpsPort) } } func TestListenPortOnlyConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/listen_port.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) port := 8922 if opts.Host != DEFAULT_HOST { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, DEFAULT_HOST) } if opts.HTTPHost != DEFAULT_HOST { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, DEFAULT_HOST) } if opts.Port != port { t.Fatalf("Received incorrect port %v, expected %v", opts.Port, port) } } func TestListenPortWithColonConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/listen_port_with_colon.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) port := 8922 if opts.Host != DEFAULT_HOST { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, DEFAULT_HOST) } if opts.HTTPHost != DEFAULT_HOST { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, DEFAULT_HOST) } if opts.Port != port { t.Fatalf("Received incorrect port %v, expected %v", opts.Port, port) } } func TestListenMonitoringDefault(t *testing.T) { opts := &Options{ Host: "10.0.1.22", } setBaselineOptions(opts) host := "10.0.1.22" if opts.Host != host { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, host) } if opts.HTTPHost != host { t.Fatalf("Received incorrect host %q, expected %q", opts.Host, host) } if opts.Port != DEFAULT_PORT { t.Fatalf("Received incorrect port %v, expected %v", opts.Port, DEFAULT_PORT) } } func TestMultipleUsersConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/multiple_users.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) } // Test highly depends on contents of the config file listed below. Any changes to that file // may very well break this test. func TestAuthorizationConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/authorization.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) lu := len(opts.Users) if lu != 5 { t.Fatalf("Expected 5 users, got %d", lu) } // Build a map mu := make(map[string]*User) for _, u := range opts.Users { mu[u.Username] = u } // Alice alice, ok := mu["alice"] if !ok { t.Fatalf("Expected to see user Alice") } // Check for permissions details if alice.Permissions == nil { t.Fatalf("Expected Alice's permissions to be non-nil") } if alice.Permissions.Publish == nil { t.Fatalf("Expected Alice's publish permissions to be non-nil") } if len(alice.Permissions.Publish.Allow) != 1 { t.Fatalf("Expected Alice's publish permissions to have 1 element, got %d", len(alice.Permissions.Publish.Allow)) } pubPerm := alice.Permissions.Publish.Allow[0] if pubPerm != "*" { t.Fatalf("Expected Alice's publish permissions to be '*', got %q", pubPerm) } if alice.Permissions.Subscribe == nil { t.Fatalf("Expected Alice's subscribe permissions to be non-nil") } if len(alice.Permissions.Subscribe.Allow) != 1 { t.Fatalf("Expected Alice's subscribe permissions to have 1 element, got %d", len(alice.Permissions.Subscribe.Allow)) } subPerm := alice.Permissions.Subscribe.Allow[0] if subPerm != ">" { t.Fatalf("Expected Alice's subscribe permissions to be '>', got %q", subPerm) } // Bob bob, ok := mu["bob"] if !ok { t.Fatalf("Expected to see user Bob") } if bob.Permissions == nil { t.Fatalf("Expected Bob's permissions to be non-nil") } // Susan susan, ok := mu["susan"] if !ok { t.Fatalf("Expected to see user Susan") } if susan.Permissions == nil { t.Fatalf("Expected Susan's permissions to be non-nil") } // Check susan closely since she inherited the default permissions. if susan.Permissions == nil { t.Fatalf("Expected Susan's permissions to be non-nil") } if susan.Permissions.Publish != nil { t.Fatalf("Expected Susan's publish permissions to be nil") } if susan.Permissions.Subscribe == nil { t.Fatalf("Expected Susan's subscribe permissions to be non-nil") } if len(susan.Permissions.Subscribe.Allow) != 1 { t.Fatalf("Expected Susan's subscribe permissions to have 1 element, got %d", len(susan.Permissions.Subscribe.Allow)) } subPerm = susan.Permissions.Subscribe.Allow[0] if subPerm != "PUBLIC.>" { t.Fatalf("Expected Susan's subscribe permissions to be 'PUBLIC.>', got %q", subPerm) } // Service A svca, ok := mu["svca"] if !ok { t.Fatalf("Expected to see user Service A") } if svca.Permissions == nil { t.Fatalf("Expected Service A's permissions to be non-nil") } if svca.Permissions.Subscribe == nil { t.Fatalf("Expected Service A's subscribe permissions to be non-nil") } if len(svca.Permissions.Subscribe.Allow) != 1 { t.Fatalf("Expected Service A's subscribe permissions to have 1 element, got %d", len(svca.Permissions.Subscribe.Allow)) } subPerm = svca.Permissions.Subscribe.Allow[0] if subPerm != "my.service.req" { t.Fatalf("Expected Service A's subscribe permissions to be 'my.service.req', got %q", subPerm) } // We want allow_responses to essentially set deny all, or allow none in this case. if svca.Permissions.Publish == nil { t.Fatalf("Expected Service A's publish permissions to be non-nil") } if len(svca.Permissions.Publish.Allow) != 0 { t.Fatalf("Expected Service A's publish permissions to have no elements, got %d", len(svca.Permissions.Publish.Allow)) } // We should have a ResponsePermission present with default values. if svca.Permissions.Response == nil { t.Fatalf("Expected Service A's response permissions to be non-nil") } if svca.Permissions.Response.MaxMsgs != DEFAULT_ALLOW_RESPONSE_MAX_MSGS { t.Fatalf("Expected Service A's response permissions of max msgs to be %d, got %d", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, svca.Permissions.Response.MaxMsgs, ) } if svca.Permissions.Response.Expires != DEFAULT_ALLOW_RESPONSE_EXPIRATION { t.Fatalf("Expected Service A's response permissions of expiration to be %v, got %v", DEFAULT_ALLOW_RESPONSE_EXPIRATION, svca.Permissions.Response.Expires, ) } // Service B svcb, ok := mu["svcb"] if !ok { t.Fatalf("Expected to see user Service B") } if svcb.Permissions == nil { t.Fatalf("Expected Service B's permissions to be non-nil") } if svcb.Permissions.Subscribe == nil { t.Fatalf("Expected Service B's subscribe permissions to be non-nil") } if len(svcb.Permissions.Subscribe.Allow) != 1 { t.Fatalf("Expected Service B's subscribe permissions to have 1 element, got %d", len(svcb.Permissions.Subscribe.Allow)) } subPerm = svcb.Permissions.Subscribe.Allow[0] if subPerm != "my.service.req" { t.Fatalf("Expected Service B's subscribe permissions to be 'my.service.req', got %q", subPerm) } // We want allow_responses to essentially set deny all, or allow none in this case. if svcb.Permissions.Publish == nil { t.Fatalf("Expected Service B's publish permissions to be non-nil") } if len(svcb.Permissions.Publish.Allow) != 0 { t.Fatalf("Expected Service B's publish permissions to have no elements, got %d", len(svcb.Permissions.Publish.Allow)) } // We should have a ResponsePermission present with default values. if svcb.Permissions.Response == nil { t.Fatalf("Expected Service B's response permissions to be non-nil") } if svcb.Permissions.Response.MaxMsgs != 10 { t.Fatalf("Expected Service B's response permissions of max msgs to be %d, got %d", 10, svcb.Permissions.Response.MaxMsgs, ) } if svcb.Permissions.Response.Expires != time.Minute { t.Fatalf("Expected Service B's response permissions of expiration to be %v, got %v", time.Minute, svcb.Permissions.Response.Expires, ) } } // Test highly depends on contents of the config file listed below. Any changes to that file // may very well break this test. func TestNewStyleAuthorizationConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/new_style_authorization.conf") if err != nil { t.Fatalf("Received an error reading config file: %v", err) } setBaselineOptions(opts) lu := len(opts.Users) if lu != 2 { t.Fatalf("Expected 2 users, got %d", lu) } // Build a map mu := make(map[string]*User) for _, u := range opts.Users { mu[u.Username] = u } // Alice alice, ok := mu["alice"] if !ok { t.Fatalf("Expected to see user Alice") } if alice.Permissions == nil { t.Fatalf("Expected Alice's permissions to be non-nil") } if alice.Permissions.Publish == nil { t.Fatalf("Expected Alice's publish permissions to be non-nil") } if len(alice.Permissions.Publish.Allow) != 3 { t.Fatalf("Expected Alice's allowed publish permissions to have 3 elements, got %d", len(alice.Permissions.Publish.Allow)) } pubPerm := alice.Permissions.Publish.Allow[0] if pubPerm != "foo" { t.Fatalf("Expected Alice's first allowed publish permission to be 'foo', got %q", pubPerm) } pubPerm = alice.Permissions.Publish.Allow[1] if pubPerm != "bar" { t.Fatalf("Expected Alice's second allowed publish permission to be 'bar', got %q", pubPerm) } pubPerm = alice.Permissions.Publish.Allow[2] if pubPerm != "baz" { t.Fatalf("Expected Alice's third allowed publish permission to be 'baz', got %q", pubPerm) } if len(alice.Permissions.Publish.Deny) != 0 { t.Fatalf("Expected Alice's denied publish permissions to have 0 elements, got %d", len(alice.Permissions.Publish.Deny)) } if alice.Permissions.Subscribe == nil { t.Fatalf("Expected Alice's subscribe permissions to be non-nil") } if len(alice.Permissions.Subscribe.Allow) != 0 { t.Fatalf("Expected Alice's allowed subscribe permissions to have 0 elements, got %d", len(alice.Permissions.Subscribe.Allow)) } if len(alice.Permissions.Subscribe.Deny) != 1 { t.Fatalf("Expected Alice's denied subscribe permissions to have 1 element, got %d", len(alice.Permissions.Subscribe.Deny)) } subPerm := alice.Permissions.Subscribe.Deny[0] if subPerm != "$SYS.>" { t.Fatalf("Expected Alice's only denied subscribe permission to be '$SYS.>', got %q", subPerm) } // Bob bob, ok := mu["bob"] if !ok { t.Fatalf("Expected to see user Bob") } if bob.Permissions == nil { t.Fatalf("Expected Bob's permissions to be non-nil") } if bob.Permissions.Publish == nil { t.Fatalf("Expected Bobs's publish permissions to be non-nil") } if len(bob.Permissions.Publish.Allow) != 1 { t.Fatalf("Expected Bob's allowed publish permissions to have 1 element, got %d", len(bob.Permissions.Publish.Allow)) } pubPerm = bob.Permissions.Publish.Allow[0] if pubPerm != "$SYS.>" { t.Fatalf("Expected Bob's first allowed publish permission to be '$SYS.>', got %q", pubPerm) } if len(bob.Permissions.Publish.Deny) != 0 { t.Fatalf("Expected Bob's denied publish permissions to have 0 elements, got %d", len(bob.Permissions.Publish.Deny)) } if bob.Permissions.Subscribe == nil { t.Fatalf("Expected Bob's subscribe permissions to be non-nil") } if len(bob.Permissions.Subscribe.Allow) != 0 { t.Fatalf("Expected Bob's allowed subscribe permissions to have 0 elements, got %d", len(bob.Permissions.Subscribe.Allow)) } if len(bob.Permissions.Subscribe.Deny) != 3 { t.Fatalf("Expected Bobs's denied subscribe permissions to have 3 elements, got %d", len(bob.Permissions.Subscribe.Deny)) } subPerm = bob.Permissions.Subscribe.Deny[0] if subPerm != "foo" { t.Fatalf("Expected Bobs's first denied subscribe permission to be 'foo', got %q", subPerm) } subPerm = bob.Permissions.Subscribe.Deny[1] if subPerm != "bar" { t.Fatalf("Expected Bobs's second denied subscribe permission to be 'bar', got %q", subPerm) } subPerm = bob.Permissions.Subscribe.Deny[2] if subPerm != "baz" { t.Fatalf("Expected Bobs's third denied subscribe permission to be 'baz', got %q", subPerm) } } // Test new nkey users func TestNkeyUsersConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` authorization { users = [ {nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV"} {nkey: "UA3C5TBZYK5GJQJRWPMU6NFY5JNAEVQB2V2TUZFZDHFJFUYVKTTUOFKZ"} ] }`)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error reading config file: %v", err) } lu := len(opts.Nkeys) if lu != 2 { t.Fatalf("Expected 2 nkey users, got %d", lu) } } // Test pinned certificates func TestTlsPinnedCertificates(t *testing.T) { confFileName := createConfFile(t, []byte(` tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } cluster { port -1 name cluster-hub tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } } leafnodes { port -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } } gateway { name: "A" port -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } } websocket { port -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } } mqtt { port -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069", "a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32"] } }`)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error reading config file: %v", err) } check := func(set PinnedCertSet) { t.Helper() if l := len(set); l != 2 { t.Fatalf("Expected 2 pinned certificates, got got %d", l) } } check(opts.TLSPinnedCerts) check(opts.LeafNode.TLSPinnedCerts) check(opts.Cluster.TLSPinnedCerts) check(opts.MQTT.TLSPinnedCerts) check(opts.Gateway.TLSPinnedCerts) check(opts.Websocket.TLSPinnedCerts) } func TestNkeyUsersDefaultPermissionsConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` authorization { default_permissions = { publish = "foo" } users = [ { user: "user", password: "pwd"} { user: "other", password: "pwd", permissions = { subscribe = "bar" } } { nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV" } { nkey: "UA3C5TBZYK5GJQJRWPMU6NFY5JNAEVQB2V2TUZFZDHFJFUYVKTTUOFKZ", permissions = { subscribe = "bar" } } ] } accounts { A { default_permissions = { publish = "foo" } users = [ { user: "accuser", password: "pwd"} { user: "accother", password: "pwd", permissions = { subscribe = "bar" } } { nkey: "UC4YEYJHYKTU4LHROX7UEKEIO5RP5OUWDYXELHWXZOQHZYXHUD44LCRS" } { nkey: "UDLSDF4UY3YW7JJQCYE6T2D4KFDCH6RGF3R65KHK247G3POJPI27VMQ3", permissions = { subscribe = "bar" } } ] } } `)) checkPerms := func(permsDef *Permissions, permsNonDef *Permissions) { if permsDef.Publish.Allow[0] != "foo" { t.Fatal("Publish allow foo missing") } else if permsDef.Subscribe != nil { t.Fatal("Has unexpected Subscribe permission") } else if permsNonDef.Subscribe.Allow[0] != "bar" { t.Fatal("Subscribe allow bar missing") } else if permsNonDef.Publish != nil { t.Fatal("Has unexpected Publish permission") } } opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error reading config file: %v", err) } findUsers := func(u1, u2 string) (found []*User) { find := []string{u1, u2} for _, f := range find { for _, u := range opts.Users { if u.Username == f { found = append(found, u) break } } } return } findNkeyUsers := func(nk1, nk2 string) (found []*NkeyUser) { find := []string{nk1, nk2} for _, f := range find { for _, u := range opts.Nkeys { if strings.HasPrefix(u.Nkey, f) { found = append(found, u) break } } } return } if lu := len(opts.Users); lu != 4 { t.Fatalf("Expected 4 nkey users, got %d", lu) } foundU := findUsers("user", "other") checkPerms(foundU[0].Permissions, foundU[1].Permissions) foundU = findUsers("accuser", "accother") checkPerms(foundU[0].Permissions, foundU[1].Permissions) if lu := len(opts.Nkeys); lu != 4 { t.Fatalf("Expected 4 nkey users, got %d", lu) } foundNk := findNkeyUsers("UDK", "UA3") checkPerms(foundNk[0].Permissions, foundNk[1].Permissions) foundNk = findNkeyUsers("UC4", "UDL") checkPerms(foundNk[0].Permissions, foundNk[1].Permissions) } func TestNkeyUsersWithPermsConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` authorization { users = [ {nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV", permissions = { publish = "$SYS.>" subscribe = { deny = ["foo", "bar", "baz"] } } } ] }`)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error reading config file: %v", err) } lu := len(opts.Nkeys) if lu != 1 { t.Fatalf("Expected 1 nkey user, got %d", lu) } nk := opts.Nkeys[0] if nk.Permissions == nil { t.Fatal("Expected to have permissions") } if nk.Permissions.Publish == nil { t.Fatal("Expected to have publish permissions") } if nk.Permissions.Publish.Allow[0] != "$SYS.>" { t.Fatalf("Expected publish to allow \"$SYS.>\", but got %v", nk.Permissions.Publish.Allow[0]) } if nk.Permissions.Subscribe == nil { t.Fatal("Expected to have subscribe permissions") } if nk.Permissions.Subscribe.Allow != nil { t.Fatal("Expected to have no subscribe allow permissions") } deny := nk.Permissions.Subscribe.Deny if deny == nil || len(deny) != 3 || deny[0] != "foo" || deny[1] != "bar" || deny[2] != "baz" { t.Fatalf("Expected to have subscribe deny permissions, got %v", deny) } } func TestBadNkeyConfig(t *testing.T) { confFileName := "nkeys_bad.conf" content := ` authorization { users = [ {nkey: "Ufoo"}] }` if err := os.WriteFile(confFileName, []byte(content), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, confFileName) if _, err := ProcessConfigFile(confFileName); err == nil { t.Fatalf("Expected an error from nkey entry with password") } } func TestNkeyWithPassConfig(t *testing.T) { confFileName := "nkeys_pass.conf" content := ` authorization { users = [ {nkey: "UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV", pass: "foo"} ] }` if err := os.WriteFile(confFileName, []byte(content), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, confFileName) if _, err := ProcessConfigFile(confFileName); err == nil { t.Fatalf("Expected an error from bad nkey entry") } } func TestTokenWithUserPass(t *testing.T) { confFileName := "test.conf" content := ` authorization={ user: user pass: password token: $2a$11$whatever }` if err := os.WriteFile(confFileName, []byte(content), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, confFileName) _, err := ProcessConfigFile(confFileName) if err == nil { t.Fatal("Expected error, got none") } if !strings.Contains(err.Error(), "token") { t.Fatalf("Expected error related to token, got %v", err) } } func TestTokenWithUsers(t *testing.T) { confFileName := "test.conf" content := ` authorization={ token: $2a$11$whatever users: [ {user: test, password: test} ] }` if err := os.WriteFile(confFileName, []byte(content), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, confFileName) _, err := ProcessConfigFile(confFileName) if err == nil { t.Fatal("Expected error, got none") } if !strings.Contains(err.Error(), "token") { t.Fatalf("Expected error related to token, got %v", err) } } func TestParseWriteDeadline(t *testing.T) { confFile := createConfFile(t, []byte("write_deadline: \"1x\"")) _, err := ProcessConfigFile(confFile) if err == nil { t.Fatal("Expected error, got none") } if !strings.Contains(err.Error(), "parsing") { t.Fatalf("Expected error related to parsing, got %v", err) } confFile = createConfFile(t, []byte("write_deadline: \"1s\"")) opts, err := ProcessConfigFile(confFile) if err != nil { t.Fatalf("Unexpected error: %v", err) } if opts.WriteDeadline != time.Second { t.Fatalf("Expected write_deadline to be 1s, got %v", opts.WriteDeadline) } oldStdout := os.Stdout _, w, _ := os.Pipe() defer func() { w.Close() os.Stdout = oldStdout }() os.Stdout = w confFile = createConfFile(t, []byte("write_deadline: 2")) opts, err = ProcessConfigFile(confFile) if err != nil { t.Fatalf("Unexpected error: %v", err) } if opts.WriteDeadline != 2*time.Second { t.Fatalf("Expected write_deadline to be 2s, got %v", opts.WriteDeadline) } } func TestOptionsClone(t *testing.T) { opts := &Options{ ConfigFile: "./configs/test.conf", Host: "127.0.0.1", Port: 2222, Username: "derek", Password: "porkchop", AuthTimeout: 1.0, Debug: true, Trace: true, Logtime: false, HTTPPort: DEFAULT_HTTP_PORT, HTTPBasePath: DEFAULT_HTTP_BASE_PATH, PidFile: "/tmp/nats-server/nats-server.pid", ProfPort: 6789, Syslog: true, RemoteSyslog: "udp://foo.com:33", MaxControlLine: 2048, MaxPayload: 65536, MaxConn: 100, PingInterval: 60 * time.Second, MaxPingsOut: 3, Cluster: ClusterOpts{ NoAdvertise: true, ConnectRetries: 2, }, Gateway: GatewayOpts{ Name: "A", Gateways: []*RemoteGatewayOpts{ {Name: "B", URLs: []*url.URL{{Scheme: "nats", Host: "host:5222"}}}, {Name: "C"}, }, }, WriteDeadline: 3 * time.Second, Routes: []*url.URL{{}}, Users: []*User{{Username: "foo", Password: "bar"}}, } clone := opts.Clone() if !reflect.DeepEqual(opts, clone) { t.Fatalf("Cloned Options are incorrect.\nexpected: %+v\ngot: %+v", clone, opts) } clone.Users[0].Password = "baz" if reflect.DeepEqual(opts, clone) { t.Fatal("Expected Options to be different") } opts.Gateway.Gateways[0].URLs[0] = nil if reflect.DeepEqual(opts.Gateway.Gateways[0], clone.Gateway.Gateways[0]) { t.Fatal("Expected Options to be different") } if clone.Gateway.Gateways[0].URLs[0].Host != "host:5222" { t.Fatalf("Unexpected URL: %v", clone.Gateway.Gateways[0].URLs[0]) } } func TestOptionsCloneNilLists(t *testing.T) { opts := &Options{} clone := opts.Clone() if clone.Routes != nil { t.Fatalf("Expected Routes to be nil, got: %v", clone.Routes) } if clone.Users != nil { t.Fatalf("Expected Users to be nil, got: %v", clone.Users) } } func TestOptionsCloneNil(t *testing.T) { opts := (*Options)(nil) clone := opts.Clone() if clone != nil { t.Fatalf("Expected nil, got: %+v", clone) } } func TestEmptyConfig(t *testing.T) { opts, err := ProcessConfigFile("") if err != nil { t.Fatalf("Expected no error from empty config, got: %+v", err) } if opts.ConfigFile != "" { t.Fatalf("Expected empty config, got: %+v", opts) } } func TestMalformedListenAddress(t *testing.T) { opts, err := ProcessConfigFile("./configs/malformed_listen_address.conf") if err == nil { t.Fatalf("Expected an error reading config file: got %+v", opts) } } func TestMalformedClusterAddress(t *testing.T) { opts, err := ProcessConfigFile("./configs/malformed_cluster_address.conf") if err == nil { t.Fatalf("Expected an error reading config file: got %+v", opts) } } func TestPanic(t *testing.T) { conf := createConfFile(t, []byte(`port: "this_string_trips_a_panic"`)) opts, err := ProcessConfigFile(conf) if err == nil { t.Fatalf("Expected an error reading config file: got %+v", opts) } else { if !strings.Contains(err.Error(), ":1:0: interface conversion:") { t.Fatalf("This was supposed to trip a panic on interface conversion right at the beginning") } } } func TestMaxClosedClients(t *testing.T) { conf := createConfFile(t, []byte(`max_closed_clients: 5`)) opts, err := ProcessConfigFile(conf) require_NoError(t, err) require_Equal(t, opts.MaxClosedClients, 5) } func TestPingIntervalOld(t *testing.T) { conf := createConfFile(t, []byte(`ping_interval: 5`)) opts := &Options{} err := opts.ProcessConfigFile(conf) if err == nil { t.Fatalf("expected an error") } errTyped, ok := err.(*processConfigErr) if !ok { t.Fatalf("expected an error of type processConfigErr") } if len(errTyped.warnings) != 1 { t.Fatalf("expected processConfigErr to have one warning") } if len(errTyped.errors) != 0 { t.Fatalf("expected processConfigErr to have no error") } if opts.PingInterval != 5*time.Second { t.Fatalf("expected ping interval to be 5 seconds") } } func TestPingIntervalNew(t *testing.T) { conf := createConfFile(t, []byte(`ping_interval: "5m"`)) opts := &Options{} if err := opts.ProcessConfigFile(conf); err != nil { t.Fatalf("expected no error") } if opts.PingInterval != 5*time.Minute { t.Fatalf("expected ping interval to be 5 minutes") } } func TestOptionsProcessConfigFile(t *testing.T) { // Create options with default values of Debug and Trace // that are the opposite of what is in the config file. // Set another option that is not present in the config file. logFileName := "test.log" opts := &Options{ Debug: true, Trace: false, LogFile: logFileName, } configFileName := "./configs/test.conf" if err := opts.ProcessConfigFile(configFileName); err != nil { t.Fatalf("Error processing config file: %v", err) } // Verify that values are as expected if opts.ConfigFile != configFileName { t.Fatalf("Expected ConfigFile to be set to %q, got %v", configFileName, opts.ConfigFile) } if opts.Debug { t.Fatal("Debug option should have been set to false from config file") } if !opts.Trace { t.Fatal("Trace option should have been set to true from config file") } if opts.LogFile != logFileName { t.Fatalf("Expected LogFile to be %q, got %q", logFileName, opts.LogFile) } } func TestConfigureOptions(t *testing.T) { // Options.Configure() will snapshot the flags. This is used by the reload code. // We need to set it back to nil otherwise it will impact reload tests. defer func() { FlagSnapshot = nil }() ch := make(chan bool, 1) checkPrintInvoked := func() { ch <- true } usage := func() { panic("should not get there") } var fs *flag.FlagSet type testPrint struct { args []string version, help, tlsHelp func() } testFuncs := []testPrint{ {[]string{"-v"}, checkPrintInvoked, usage, PrintTLSHelpAndDie}, {[]string{"version"}, checkPrintInvoked, usage, PrintTLSHelpAndDie}, {[]string{"-h"}, PrintServerAndExit, checkPrintInvoked, PrintTLSHelpAndDie}, {[]string{"help"}, PrintServerAndExit, checkPrintInvoked, PrintTLSHelpAndDie}, {[]string{"-help_tls"}, PrintServerAndExit, usage, checkPrintInvoked}, } for _, tf := range testFuncs { fs = flag.NewFlagSet("test", flag.ContinueOnError) opts, err := ConfigureOptions(fs, tf.args, tf.version, tf.help, tf.tlsHelp) if err != nil { t.Fatalf("Error on configure: %v", err) } if opts != nil { t.Fatalf("Expected options to be nil, got %v", opts) } select { case <-ch: case <-time.After(time.Second): t.Fatalf("Should have invoked print function for args=%v", tf.args) } } // Helper function that expect parsing with given args to not produce an error. mustNotFail := func(args []string) *Options { fs := flag.NewFlagSet("test", flag.ContinueOnError) opts, err := ConfigureOptions(fs, args, PrintServerAndExit, fs.Usage, PrintTLSHelpAndDie) if err != nil { stackFatalf(t, "Error on configure: %v", err) } return opts } // Helper function that expect configuration to fail. expectToFail := func(args []string, errContent ...string) { fs := flag.NewFlagSet("test", flag.ContinueOnError) // Silence the flagSet so that on failure nothing is printed. // (flagSet would print error message about unknown flags, etc..) silenceOuput := &bytes.Buffer{} fs.SetOutput(silenceOuput) opts, err := ConfigureOptions(fs, args, PrintServerAndExit, fs.Usage, PrintTLSHelpAndDie) if opts != nil || err == nil { stackFatalf(t, "Expected no option and an error, got opts=%v and err=%v", opts, err) } for _, testErr := range errContent { if strings.Contains(err.Error(), testErr) { // We got the error we wanted. return } } stackFatalf(t, "Expected errors containing any of those %v, got %v", errContent, err) } // Basic test with port number opts := mustNotFail([]string{"-p", "1234"}) if opts.Port != 1234 { t.Fatalf("Expected port to be 1234, got %v", opts.Port) } // Should fail because of unknown parameter expectToFail([]string{"foo"}, "command") // Should fail because unknown flag expectToFail([]string{"-xxx", "foo"}, "flag") // Should fail because of config file missing expectToFail([]string{"-c", "xxx.cfg"}, "file") // Should fail because of too many args for signal command expectToFail([]string{"-sl", "quit=pid=foo"}, "signal") // Should fail because of invalid pid // On windows, if not running with admin privileges, you would get access denied. expectToFail([]string{"-sl", "quit=pid"}, "pid", "denied") // The config file set Trace to true. opts = mustNotFail([]string{"-c", "./configs/test.conf"}) if !opts.Trace { t.Fatal("Trace should have been set to true") } // The config file set Trace to true, but was overridden by param -V=false opts = mustNotFail([]string{"-c", "./configs/test.conf", "-V=false"}) if opts.Trace { t.Fatal("Trace should have been set to false") } // The config file set Trace to true, but was overridden by param -DV=false opts = mustNotFail([]string{"-c", "./configs/test.conf", "-DV=false"}) if opts.Debug || opts.Trace { t.Fatal("Debug and Trace should have been set to false") } // The config file set Trace to true, but was overridden by param -DV opts = mustNotFail([]string{"-c", "./configs/test.conf", "-DV"}) if !opts.Debug || !opts.Trace { t.Fatal("Debug and Trace should have been set to true") } // This should fail since -cluster is missing expectedURL, _ := url.Parse("nats://127.0.0.1:6223") expectToFail([]string{"-routes", expectedURL.String()}, "solicited routes") // Ensure that we can set cluster and routes from command line opts = mustNotFail([]string{"-cluster", "nats://127.0.0.1:6222", "-routes", expectedURL.String()}) if opts.Cluster.ListenStr != "nats://127.0.0.1:6222" { t.Fatalf("Unexpected Cluster.ListenStr=%q", opts.Cluster.ListenStr) } if opts.RoutesStr != "nats://127.0.0.1:6223" || len(opts.Routes) != 1 || opts.Routes[0].String() != expectedURL.String() { t.Fatalf("Unexpected RoutesStr: %q and Routes: %v", opts.RoutesStr, opts.Routes) } // Use a config with cluster configuration and explicit route defined. // Override with empty routes string. opts = mustNotFail([]string{"-c", "./configs/srv_a.conf", "-routes", ""}) if opts.RoutesStr != "" || len(opts.Routes) != 0 { t.Fatalf("Unexpected RoutesStr: %q and Routes: %v", opts.RoutesStr, opts.Routes) } // Use a config with cluster configuration and override cluster listen string expectedURL, _ = url.Parse("nats-route://ruser:top_secret@127.0.0.1:7246") opts = mustNotFail([]string{"-c", "./configs/srv_a.conf", "-cluster", "nats://ivan:pwd@127.0.0.1:6222"}) if opts.Cluster.Username != "ivan" || opts.Cluster.Password != "pwd" || opts.Cluster.Port != 6222 || len(opts.Routes) != 1 || opts.Routes[0].String() != expectedURL.String() { t.Fatalf("Unexpected Cluster and/or Routes: %#v - %v", opts.Cluster, opts.Routes) } // Disable clustering from command line opts = mustNotFail([]string{"-c", "./configs/srv_a.conf", "-cluster", ""}) if opts.Cluster.Port != 0 { t.Fatalf("Unexpected Cluster: %v", opts.Cluster) } // Various erros due to malformed cluster listen string. // (adding -routes to have more than 1 set flag to check // that Visit() stops when an error is found). expectToFail([]string{"-cluster", ":", "-routes", ""}, "protocol") expectToFail([]string{"-cluster", "nats://127.0.0.1", "-routes", ""}, "port") expectToFail([]string{"-cluster", "nats://127.0.0.1:xxx", "-routes", ""}, "invalid port") expectToFail([]string{"-cluster", "nats://ivan:127.0.0.1:6222", "-routes", ""}, "colons") expectToFail([]string{"-cluster", "nats://ivan@127.0.0.1:6222", "-routes", ""}, "password") // Override config file's TLS configuration from command line, and completely disable TLS opts = mustNotFail([]string{"-c", "./configs/tls.conf", "-tls=false"}) if opts.TLSConfig != nil || opts.TLS { t.Fatal("Expected TLS to be disabled") } // Override config file's TLS configuration from command line, and force TLS verification. // However, since TLS config has to be regenerated, user need to provide -tlscert and -tlskey too. // So this should fail. expectToFail([]string{"-c", "./configs/tls.conf", "-tlsverify"}, "valid") // Now same than above, but with all valid params. opts = mustNotFail([]string{"-c", "./configs/tls.conf", "-tlsverify", "-tlscert", "./configs/certs/server.pem", "-tlskey", "./configs/certs/key.pem"}) if opts.TLSConfig == nil || !opts.TLSVerify { t.Fatal("Expected TLS to be configured and force verification") } // Configure TLS, but some TLS params missing expectToFail([]string{"-tls"}, "valid") expectToFail([]string{"-tls", "-tlscert", "./configs/certs/server.pem"}, "valid") // One of the file does not exist expectToFail([]string{"-tls", "-tlscert", "./configs/certs/server.pem", "-tlskey", "./configs/certs/notfound.pem"}, "file") // Configure TLS and check that this results in a TLSConfig option. opts = mustNotFail([]string{"-tls", "-tlscert", "./configs/certs/server.pem", "-tlskey", "./configs/certs/key.pem"}) if opts.TLSConfig == nil || !opts.TLS { t.Fatal("Expected TLSConfig to be set") } // Check that we use default TLS ciphers if !reflect.DeepEqual(opts.TLSConfig.CipherSuites, defaultCipherSuites()) { t.Fatalf("Default ciphers not set, expected %v, got %v", defaultCipherSuites(), opts.TLSConfig.CipherSuites) } } func TestClusterPermissionsConfig(t *testing.T) { template := ` cluster { port: 1234 %s authorization { user: ivan password: pwd permissions { import { allow: "foo" } export { allow: "bar" } } } } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, ""))) opts, err := ProcessConfigFile(conf) if err != nil { if cerr, ok := err.(*processConfigErr); ok && len(cerr.Errors()) > 0 { t.Fatalf("Error processing config file: %v", err) } } if opts.Cluster.Permissions == nil { t.Fatal("Expected cluster permissions to be set") } if opts.Cluster.Permissions.Import == nil { t.Fatal("Expected cluster import permissions to be set") } if len(opts.Cluster.Permissions.Import.Allow) != 1 || opts.Cluster.Permissions.Import.Allow[0] != "foo" { t.Fatalf("Expected cluster import permissions to have %q, got %v", "foo", opts.Cluster.Permissions.Import.Allow) } if opts.Cluster.Permissions.Export == nil { t.Fatal("Expected cluster export permissions to be set") } if len(opts.Cluster.Permissions.Export.Allow) != 1 || opts.Cluster.Permissions.Export.Allow[0] != "bar" { t.Fatalf("Expected cluster export permissions to have %q, got %v", "bar", opts.Cluster.Permissions.Export.Allow) } // Now add permissions in top level cluster and check // that this is the one that is being used. conf = createConfFile(t, []byte(fmt.Sprintf(template, ` permissions { import { allow: "baz" } export { allow: "bat" } } `))) opts, err = ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } if opts.Cluster.Permissions == nil { t.Fatal("Expected cluster permissions to be set") } if opts.Cluster.Permissions.Import == nil { t.Fatal("Expected cluster import permissions to be set") } if len(opts.Cluster.Permissions.Import.Allow) != 1 || opts.Cluster.Permissions.Import.Allow[0] != "baz" { t.Fatalf("Expected cluster import permissions to have %q, got %v", "baz", opts.Cluster.Permissions.Import.Allow) } if opts.Cluster.Permissions.Export == nil { t.Fatal("Expected cluster export permissions to be set") } if len(opts.Cluster.Permissions.Export.Allow) != 1 || opts.Cluster.Permissions.Export.Allow[0] != "bat" { t.Fatalf("Expected cluster export permissions to have %q, got %v", "bat", opts.Cluster.Permissions.Export.Allow) } // Tests with invalid permissions invalidPerms := []string{ `permissions: foo`, `permissions { unknown_field: "foo" }`, `permissions { import: [1, 2, 3] }`, `permissions { import { unknown_field: "foo" } }`, `permissions { import { allow { x: y } } }`, `permissions { import { deny { x: y } } }`, `permissions { export: [1, 2, 3] }`, `permissions { export { unknown_field: "foo" } }`, `permissions { export { allow { x: y } } }`, `permissions { export { deny { x: y } } }`, } for _, perms := range invalidPerms { conf = createConfFile(t, []byte(fmt.Sprintf(` cluster { port: 1234 %s } `, perms))) _, err := ProcessConfigFile(conf) if err == nil { t.Fatalf("Expected failure for permissions %s", perms) } } for _, perms := range invalidPerms { conf = createConfFile(t, []byte(fmt.Sprintf(` cluster { port: 1234 authorization { user: ivan password: pwd %s } } `, perms))) _, err := ProcessConfigFile(conf) if err == nil { t.Fatalf("Expected failure for permissions %s", perms) } } } func TestParseServiceLatency(t *testing.T) { cases := []struct { name string conf string want *serviceLatency wantErr bool }{ { name: "block with percent sample default value", conf: `system_account = nats.io accounts { nats.io { exports [{ service: nats.add latency: { sampling: 100% subject: latency.tracking.add } }] } }`, want: &serviceLatency{ subject: "latency.tracking.add", sampling: 100, }, }, { name: "block with percent sample nondefault value", conf: `system_account = nats.io accounts { nats.io { exports [{ service: nats.add latency: { sampling: 33% subject: latency.tracking.add } }] } }`, want: &serviceLatency{ subject: "latency.tracking.add", sampling: 33, }, }, { name: "block with number sample nondefault value", conf: `system_account = nats.io accounts { nats.io { exports [{ service: nats.add latency: { sampling: 87 subject: latency.tracking.add } }] } }`, want: &serviceLatency{ subject: "latency.tracking.add", sampling: 87, }, }, { name: "field with subject", conf: `system_account = nats.io accounts { nats.io { exports [{ service: nats.add latency: latency.tracking.add }] } }`, want: &serviceLatency{ subject: "latency.tracking.add", sampling: 100, }, }, { name: "block with missing subject", conf: `system_account = nats.io accounts { nats.io { exports [{ service: nats.add latency: { sampling: 87 } }] } }`, wantErr: true, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { f := createConfFile(t, []byte(c.conf)) opts, err := ProcessConfigFile(f) switch { case c.wantErr && err == nil: t.Fatalf("Expected ProcessConfigFile to fail, but didn't") case c.wantErr && err != nil: // We wanted an error and got one, test passed. return case !c.wantErr && err == nil: // We didn't want an error and didn't get one, keep going. break case !c.wantErr && err != nil: t.Fatalf("Failed to process config: %v", err) } if len(opts.Accounts) != 1 { t.Fatalf("Expected accounts to have len %d, got %d", 1, len(opts.Accounts)) } if len(opts.Accounts[0].exports.services) != 1 { t.Fatalf("Expected export services to have len %d, got %d", 1, len(opts.Accounts[0].exports.services)) } s, ok := opts.Accounts[0].exports.services["nats.add"] if !ok { t.Fatalf("Expected export service nats.add, missing") } if !reflect.DeepEqual(s.latency, c.want) { t.Fatalf("Expected latency to be %#v, got %#v", c.want, s.latency) } }) } } func TestParseExport(t *testing.T) { conf := ` port: -1 system_account: sys accounts { sys { exports [{ stream "$SYS.SERVER.ACCOUNT.*.CONNS" account_token_position 4 }] } accE { exports [{ service foo.* account_token_position 2 }] users [{ user ue password pwd }], } accI1 { imports [{ service { account accE subject foo.accI1 } to foo },{ stream { account sys subject "$SYS.SERVER.ACCOUNT.accI1.CONNS" } }], users [{ user u1 password pwd }], } accI2 { imports [{ service { account accE subject foo.accI2 } to foo },{ stream { account sys subject "$SYS.SERVER.ACCOUNT.accI2.CONNS" } }], users [{ user u2 password pwd }], } }` f := createConfFile(t, []byte(conf)) s, o := RunServerWithConfig(f) if s == nil { t.Fatal("Failed startup") } defer s.Shutdown() connect := func(user string) *nats.Conn { nc, err := nats.Connect(fmt.Sprintf("nats://%s:pwd@%s:%d", user, o.Host, o.Port)) require_NoError(t, err) return nc } nc1 := connect("u1") defer nc1.Close() nc2 := connect("u2") defer nc2.Close() // Due to the fact that above CONNECT events are generated and sent from // a system go routine, it is possible that by the time we create the // subscriptions below, the interest would exist and messages be sent, // which was causing issues since wg.Done() was called too many times. // Add a little delay to minimize risk, but also use counter to decide // when to call wg.Done() to avoid panic due to negative number. time.Sleep(100 * time.Millisecond) wg := sync.WaitGroup{} wg.Add(1) count := int32(0) // We expect a total of 6 messages expected := int32(6) subscribe := func(nc *nats.Conn, subj string) { t.Helper() _, err := nc.Subscribe(subj, func(msg *nats.Msg) { if msg.Reply != _EMPTY_ { msg.Respond(msg.Data) } if atomic.AddInt32(&count, 1) == expected { wg.Done() } }) require_NoError(t, err) nc.Flush() } //Subscribe to CONNS events subscribe(nc1, "$SYS.SERVER.ACCOUNT.accI1.CONNS") subscribe(nc2, "$SYS.SERVER.ACCOUNT.accI2.CONNS") // Trigger 2 CONNS event nc3 := connect("u1") nc3.Close() nc4 := connect("u2") nc4.Close() // test service ncE := connect("ue") defer ncE.Close() subscribe(ncE, "foo.*") request := func(nc *nats.Conn, msg string) { if m, err := nc.Request("foo", []byte(msg), time.Second); err != nil { t.Fatal("Failed request ", msg, err) } else if m == nil { t.Fatal("No response msg") } else if string(m.Data) != msg { t.Fatal("Wrong response msg", string(m.Data)) } } request(nc1, "1") request(nc2, "1") wg.Wait() } func TestAccountUsersLoadedProperly(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" authorization { users [ {user: ivan, password: bar} {nkey : UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } accounts { synadia { users [ {user: derek, password: foo} {nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E} ] } } `)) check := func(t *testing.T) { t.Helper() s, _ := RunServerWithConfig(conf) defer s.Shutdown() opts := s.getOpts() if n := len(opts.Users); n != 2 { t.Fatalf("Should have 2 users, got %v", n) } if n := len(opts.Nkeys); n != 2 { t.Fatalf("Should have 2 nkeys, got %v", n) } } // Repeat test since issue was with ordering of processing // of authorization vs accounts that depends on range of a map (after actual parsing) for i := 0; i < 20; i++ { check(t) } } func TestParsingGateways(t *testing.T) { content := ` gateway { name: "A" listen: "127.0.0.1:4444" host: "127.0.0.1" port: 4444 reject_unknown_cluster: true authorization { user: "ivan" password: "pwd" timeout: 2.0 } tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 3.0 } advertise: "me:1" connect_retries: 10 gateways: [ { name: "B" urls: ["nats://user1:pwd1@host2:5222", "nats://user1:pwd1@host3:6222"] } { name: "C" url: "nats://host4:7222" } ] } ` file := "server_config_gateways.conf" if err := os.WriteFile(file, []byte(content), 0600); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, file) opts, err := ProcessConfigFile(file) if err != nil { t.Fatalf("Error processing file: %v", err) } expected := &GatewayOpts{ Name: "A", Host: "127.0.0.1", Port: 4444, Username: "ivan", Password: "pwd", AuthTimeout: 2.0, Advertise: "me:1", ConnectRetries: 10, TLSTimeout: 3.0, RejectUnknown: true, } u1, _ := url.Parse("nats://user1:pwd1@host2:5222") u2, _ := url.Parse("nats://user1:pwd1@host3:6222") urls := []*url.URL{u1, u2} gw := &RemoteGatewayOpts{ Name: "B", URLs: urls, } expected.Gateways = append(expected.Gateways, gw) u1, _ = url.Parse("nats://host4:7222") urls = []*url.URL{u1} gw = &RemoteGatewayOpts{ Name: "C", URLs: urls, } expected.Gateways = append(expected.Gateways, gw) // Just make sure that TLSConfig is set.. we have aother test // to check proper generating TLSConfig from config file... if opts.Gateway.TLSConfig == nil { t.Fatalf("Expected TLSConfig, got none") } opts.Gateway.TLSConfig = nil opts.Gateway.tlsConfigOpts = nil if !reflect.DeepEqual(&opts.Gateway, expected) { t.Fatalf("Expected %v, got %v", expected, opts.Gateway) } } func TestParsingGatewaysErrors(t *testing.T) { for _, test := range []struct { name string content string expectedErr string }{ { "bad_type", `gateway: "bad_type"`, "Expected gateway to be a map", }, { "bad_listen", `gateway { name: "A" port: -1 listen: "bad::address" }`, "parse address", }, { "bad_auth", `gateway { name: "A" port: -1 authorization { users { } } }`, "be an array", }, { "unknown_field", `gateway { name: "A" port: -1 reject_unknown_cluster: true unknown_field: 1 }`, "unknown field", }, { "users_not_supported", `gateway { name: "A" port: -1 authorization { users [ {user: alice, password: foo} {user: bob, password: bar} ] } }`, "does not allow multiple users", }, { "tls_error", `gateway { name: "A" port: -1 tls { cert_file: 123 } }`, "to be filename", }, { "tls_gen_error_cert_file_not_found", `gateway { name: "A" port: -1 tls { cert_file: "./configs/certs/missing.pem" key_file: "./configs/certs/server-key.pem" } }`, "certificate/key pair", }, { "tls_gen_error_key_file_not_found", `gateway { name: "A" port: -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/missing.pem" } }`, "certificate/key pair", }, { "tls_gen_error_key_file_missing", `gateway { name: "A" port: -1 tls { cert_file: "./configs/certs/server.pem" } }`, `missing 'key_file' in TLS configuration`, }, { "tls_gen_error_cert_file_missing", `gateway { name: "A" port: -1 tls { key_file: "./configs/certs/server-key.pem" } }`, `missing 'cert_file' in TLS configuration`, }, { "tls_gen_error_key_file_not_found", `gateway { name: "A" port: -1 tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/missing.pem" } }`, "certificate/key pair", }, { "gateways_needs_to_be_an_array", `gateway { name: "A" gateways { name: "B" } }`, "Expected gateways field to be an array", }, { "gateways_entry_needs_to_be_a_map", `gateway { name: "A" gateways [ "g1", "g2" ] }`, "Expected gateway entry to be a map", }, { "bad_url", `gateway { name: "A" gateways [ { name: "B" url: "nats://wrong url" } ] }`, "error parsing gateway url", }, { "bad_urls", `gateway { name: "A" gateways [ { name: "B" urls: ["nats://wrong url", "nats://host:5222"] } ] }`, "error parsing gateway url", }, { "gateway_tls_error", `gateway { name: "A" port: -1 gateways [ { name: "B" tls { cert_file: 123 } } ] }`, "to be filename", }, { "gateway_unknown_field", `gateway { name: "A" port: -1 gateways [ { name: "B" unknown_field: 1 } ] }`, "unknown field", }, } { t.Run(test.name, func(t *testing.T) { file := fmt.Sprintf("server_config_gateways_%s.conf", test.name) if err := os.WriteFile(file, []byte(test.content), 0600); err != nil { t.Fatalf("Error writing config file: %v", err) } defer removeFile(t, file) _, err := ProcessConfigFile(file) if err == nil { t.Fatalf("Expected to fail, did not. Content:\n%s", test.content) } else if !strings.Contains(err.Error(), test.expectedErr) { t.Fatalf("Expected error containing %q, got %q, for content:\n%s", test.expectedErr, err, test.content) } }) } } func TestParsingLeafNodesListener(t *testing.T) { content := ` leafnodes { listen: "127.0.0.1:3333" host: "127.0.0.1" port: 3333 advertise: "me:22" authorization { user: "derek" password: "s3cr3t!" timeout: 2.2 } tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" timeout: 3.3 } } ` conf := createConfFile(t, []byte(content)) opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing file: %v", err) } expected := &LeafNodeOpts{ Host: "127.0.0.1", Port: 3333, Username: "derek", Password: "s3cr3t!", AuthTimeout: 2.2, Advertise: "me:22", TLSTimeout: 3.3, } if opts.LeafNode.TLSConfig == nil { t.Fatalf("Expected TLSConfig, got none") } if opts.LeafNode.tlsConfigOpts == nil { t.Fatalf("Expected TLSConfig snapshot, got none") } opts.LeafNode.TLSConfig = nil opts.LeafNode.tlsConfigOpts = nil if !reflect.DeepEqual(&opts.LeafNode, expected) { t.Fatalf("Expected %v, got %v", expected, opts.LeafNode) } } func TestParsingLeafNodeRemotes(t *testing.T) { t.Run("parse config file with relative path", func(t *testing.T) { content := ` leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:2222 account: foobar // Local Account to bind to.. credentials: "./my.creds" } ] } ` conf := createConfFile(t, []byte(content)) opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing file: %v", err) } if len(opts.LeafNode.Remotes) != 1 { t.Fatalf("Expected 1 remote, got %d", len(opts.LeafNode.Remotes)) } expected := &RemoteLeafOpts{ LocalAccount: "foobar", Credentials: "./my.creds", } u, _ := url.Parse("nats-leaf://127.0.0.1:2222") expected.URLs = append(expected.URLs, u) if !reflect.DeepEqual(opts.LeafNode.Remotes[0], expected) { t.Fatalf("Expected %v, got %v", expected, opts.LeafNode.Remotes[0]) } }) t.Run("parse config file with tilde path", func(t *testing.T) { if runtime.GOOS == "windows" { t.SkipNow() } origHome := os.Getenv("HOME") defer os.Setenv("HOME", origHome) os.Setenv("HOME", "/home/foo") content := ` leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:2222 account: foobar // Local Account to bind to.. credentials: "~/my.creds" } ] } ` conf := createConfFile(t, []byte(content)) opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing file: %v", err) } expected := &RemoteLeafOpts{ LocalAccount: "foobar", Credentials: "/home/foo/my.creds", } u, _ := url.Parse("nats-leaf://127.0.0.1:2222") expected.URLs = append(expected.URLs, u) if !reflect.DeepEqual(opts.LeafNode.Remotes[0], expected) { t.Fatalf("Expected %v, got %v", expected, opts.LeafNode.Remotes[0]) } }) t.Run("url ordering", func(t *testing.T) { // 16! possible permutations. orderedURLs := make([]string, 0, 16) for i := 0; i < cap(orderedURLs); i++ { orderedURLs = append(orderedURLs, fmt.Sprintf("nats-leaf://host%d:7422", i)) } confURLs, err := json.Marshal(orderedURLs) if err != nil { t.Fatal(err) } content := ` port: -1 leafnodes { remotes = [ { dont_randomize: true urls: %[1]s } { urls: %[1]s } ] } ` conf := createConfFile(t, []byte(fmt.Sprintf(content, confURLs))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() s.mu.Lock() r1 := s.leafRemoteCfgs[0] r2 := s.leafRemoteCfgs[1] s.mu.Unlock() r1.RLock() gotOrdered := r1.urls r1.RUnlock() if got, want := len(gotOrdered), len(orderedURLs); got != want { t.Fatalf("Unexpected rem0 len URLs, got %d, want %d", got, want) } // These should be IN order. for i := range orderedURLs { if got, want := gotOrdered[i].String(), orderedURLs[i]; got != want { t.Fatalf("Unexpected ordered url, got %s, want %s", got, want) } } r2.RLock() gotRandom := r2.urls r2.RUnlock() if got, want := len(gotRandom), len(orderedURLs); got != want { t.Fatalf("Unexpected rem1 len URLs, got %d, want %d", got, want) } // These should be OUT of order. var random bool for i := range orderedURLs { if gotRandom[i].String() != orderedURLs[i] { random = true break } } if !random { t.Fatal("Expected urls to be random") } }) } func TestLargeMaxControlLine(t *testing.T) { confFileName := createConfFile(t, []byte(` max_control_line = 3000000000 `)) if _, err := ProcessConfigFile(confFileName); err == nil { t.Fatalf("Expected an error from too large of a max_control_line entry") } } func TestLargeMaxPayload(t *testing.T) { confFileName := createConfFile(t, []byte(` max_payload = 3000000000 `)) if _, err := ProcessConfigFile(confFileName); err == nil { t.Fatalf("Expected an error from too large of a max_payload entry") } confFileName = createConfFile(t, []byte(` max_payload = 100000 max_pending = 50000 `)) o := LoadConfig(confFileName) s, err := NewServer(o) if err == nil || !strings.Contains(err.Error(), "cannot be higher") { if s != nil { s.Shutdown() } t.Fatalf("Unexpected error: %v", err) } } func TestHandleUnknownTopLevelConfigurationField(t *testing.T) { conf := createConfFile(t, []byte(` port: 1234 streaming { id: "me" } `)) // Verify that we get an error because of unknown "streaming" field. opts := &Options{} if err := opts.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), "streaming") { t.Fatal("Expected error, got none") } // Verify that if that is set, we get no error NoErrOnUnknownFields(true) defer NoErrOnUnknownFields(false) if err := opts.ProcessConfigFile(conf); err != nil { t.Fatalf("Unexpected error: %v", err) } if opts.Port != 1234 { t.Fatalf("Port was not parsed correctly: %v", opts.Port) } // Verify that ignore works only on top level fields. changeCurrentConfigContentWithNewContent(t, conf, []byte(` port: 1234 cluster { non_top_level_unknown_field: 123 } streaming { id: "me" } `)) if err := opts.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), "non_top_level") { t.Fatal("Expected error, got none") } } func TestSublistNoCacheConfig(t *testing.T) { confFileName := createConfFile(t, []byte(` disable_sublist_cache: true `)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received an error reading config file: %v", err) } if !opts.NoSublistCache { t.Fatalf("Expected sublist cache to be disabled") } } func TestSublistNoCacheConfigOnAccounts(t *testing.T) { confFileName := createConfFile(t, []byte(` listen: "127.0.0.1:-1" disable_sublist_cache: true accounts { synadia { users [ {nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E} ] } nats.io { users [ {nkey : UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } } no_sys_acc = true `)) s, _ := RunServerWithConfig(confFileName) defer s.Shutdown() // Check that all account sublists do not have caching enabled. ta := s.numReservedAccounts() + 2 if la := s.numAccounts(); la != ta { t.Fatalf("Expected to have a server with %d active accounts, got %v", ta, la) } s.accounts.Range(func(k, v any) bool { acc := v.(*Account) if acc == nil { t.Fatalf("Expected non-nil sublist for account") } if acc.sl.CacheEnabled() { t.Fatalf("Expected the account sublist to not have caching enabled") } return true }) } func TestParsingResponsePermissions(t *testing.T) { template := ` listen: "127.0.0.1:-1" authorization { users [ { user: ivan password: pwd permissions { allow_responses { %s %s } } } ] } ` check := func(t *testing.T, conf string, expectedError string, expectedMaxMsgs int, expectedTTL time.Duration) { t.Helper() opts, err := ProcessConfigFile(conf) if expectedError != "" { if err == nil || !strings.Contains(err.Error(), expectedError) { t.Fatalf("Expected error about %q, got %q", expectedError, err) } // OK! return } if err != nil { t.Fatalf("Error on process: %v", err) } u := opts.Users[0] p := u.Permissions.Response if p == nil { t.Fatalf("Expected response permissions to be set, it was not") } if n := p.MaxMsgs; n != expectedMaxMsgs { t.Fatalf("Expected response max msgs to be %v, got %v", expectedMaxMsgs, n) } if ttl := p.Expires; ttl != expectedTTL { t.Fatalf("Expected response ttl to be %v, got %v", expectedTTL, ttl) } } // Check defaults conf := createConfFile(t, []byte(fmt.Sprintf(template, "", ""))) check(t, conf, "", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 10", ""))) check(t, conf, "", 10, DEFAULT_ALLOW_RESPONSE_EXPIRATION) conf = createConfFile(t, []byte(fmt.Sprintf(template, "", "ttl: 5s"))) check(t, conf, "", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, 5*time.Second) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 0", ""))) check(t, conf, "", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION) conf = createConfFile(t, []byte(fmt.Sprintf(template, "", `ttl: "0s"`))) check(t, conf, "", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION) // Check normal values conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 10", `ttl: "5s"`))) check(t, conf, "", 10, 5*time.Second) // Check negative values ok conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: -1", `ttl: "5s"`))) check(t, conf, "", -1, 5*time.Second) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 10", `ttl: "-1s"`))) check(t, conf, "", 10, -1*time.Second) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: -1", `ttl: "-1s"`))) check(t, conf, "", -1, -1*time.Second) // Check parsing errors conf = createConfFile(t, []byte(fmt.Sprintf(template, "unknown_field: 123", ""))) check(t, conf, "Unknown field", 0, 0) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 10", "ttl: 123"))) check(t, conf, "not a duration string", 0, 0) conf = createConfFile(t, []byte(fmt.Sprintf(template, "max: 10", "ttl: xyz"))) check(t, conf, "error parsing expires", 0, 0) } func TestExpandPath(t *testing.T) { if runtime.GOOS == "windows" { origUserProfile := os.Getenv("USERPROFILE") origHomeDrive, origHomePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH") defer func() { os.Setenv("USERPROFILE", origUserProfile) os.Setenv("HOMEDRIVE", origHomeDrive) os.Setenv("HOMEPATH", origHomePath) }() cases := []struct { path string userProfile string homeDrive string homePath string wantPath string wantErr bool }{ // Missing HOMEDRIVE and HOMEPATH. {path: "/Foo/Bar", userProfile: `C:\Foo\Bar`, wantPath: "/Foo/Bar"}, {path: "Foo/Bar", userProfile: `C:\Foo\Bar`, wantPath: "Foo/Bar"}, {path: "~/Fizz", userProfile: `C:\Foo\Bar`, wantPath: `C:\Foo\Bar\Fizz`}, {path: `${HOMEDRIVE}${HOMEPATH}\Fizz`, homeDrive: `C:`, homePath: `\Foo\Bar`, wantPath: `C:\Foo\Bar\Fizz`}, // Missing USERPROFILE. {path: "~/Fizz", homeDrive: "X:", homePath: `\Foo\Bar`, wantPath: `X:\Foo\Bar\Fizz`}, // Set all environment variables. HOMEDRIVE and HOMEPATH take // precedence. {path: "~/Fizz", userProfile: `C:\Foo\Bar`, homeDrive: "X:", homePath: `\Foo\Bar`, wantPath: `X:\Foo\Bar\Fizz`}, // Missing all environment variables. {path: "~/Fizz", wantErr: true}, } for i, c := range cases { t.Run(fmt.Sprintf("windows case %d", i), func(t *testing.T) { os.Setenv("USERPROFILE", c.userProfile) os.Setenv("HOMEDRIVE", c.homeDrive) os.Setenv("HOMEPATH", c.homePath) gotPath, err := expandPath(c.path) if !c.wantErr && err != nil { t.Fatalf("unexpected error: got=%v; want=%v", err, nil) } else if c.wantErr && err == nil { t.Fatalf("unexpected success: got=%v; want=%v", nil, "err") } if gotPath != c.wantPath { t.Fatalf("unexpected path: got=%v; want=%v", gotPath, c.wantPath) } }) } return } // Unix tests origHome := os.Getenv("HOME") defer os.Setenv("HOME", origHome) cases := []struct { path string home string wantPath string wantErr bool }{ {path: "/foo/bar", home: "/fizz/buzz", wantPath: "/foo/bar"}, {path: "foo/bar", home: "/fizz/buzz", wantPath: "foo/bar"}, {path: "~/fizz", home: "/foo/bar", wantPath: "/foo/bar/fizz"}, {path: "$HOME/fizz", home: "/foo/bar", wantPath: "/foo/bar/fizz"}, // missing HOME env var {path: "~/fizz", wantErr: true}, } for i, c := range cases { t.Run(fmt.Sprintf("unix case %d", i), func(t *testing.T) { os.Setenv("HOME", c.home) gotPath, err := expandPath(c.path) if !c.wantErr && err != nil { t.Fatalf("unexpected error: got=%v; want=%v", err, nil) } else if c.wantErr && err == nil { t.Fatalf("unexpected success: got=%v; want=%v", nil, "err") } if gotPath != c.wantPath { t.Fatalf("unexpected path: got=%v; want=%v", gotPath, c.wantPath) } }) } } func TestNoAuthUserCode(t *testing.T) { confFileName := createConfFile(t, []byte(` listen: "127.0.0.1:-1" no_auth_user: $NO_AUTH_USER accounts { synadia { users [ {user: "a", password: "a"}, {nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E}, ] } acc { users [ {user: "c", password: "c"} ] } } # config for $G authorization { users [ {user: "b", password: "b"} ] } `)) defer os.Unsetenv("NO_AUTH_USER") for _, user := range []string{"a", "b", "b"} { t.Run(user, func(t *testing.T) { os.Setenv("NO_AUTH_USER", user) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } else { opts.NoLog = true srv := RunServer(opts) nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", opts.Port)) if err != nil { t.Fatalf("couldn't connect %s", err) } nc.Close() srv.Shutdown() } }) } for _, badUser := range []string{"notthere", "UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2F"} { t.Run(badUser, func(t *testing.T) { os.Setenv("NO_AUTH_USER", badUser) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } s, err := NewServer(opts) if err != nil { if !strings.HasPrefix(err.Error(), "no_auth_user") { t.Fatalf("Received unexpected error %s", err) } return // error looks as expected } s.Shutdown() t.Fatalf("Received no error, where no_auth_user error was expected") }) } } const operatorJwtWithSysAccAndUrlResolver = ` listen: "127.0.0.1:-1" operator: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJJVEdJNjNCUUszM1VNN1pBSzZWT1RXNUZEU01ESlNQU1pRQ0RMNUlLUzZQTVhBU0ROQ01RIiwiaWF0IjoxNTg5ODM5MjA1LCJpc3MiOiJPQ1k2REUyRVRTTjNVT0RGVFlFWEJaTFFMSTdYNEdTWFI1NE5aQzRCQkxJNlFDVFpVVDY1T0lWTiIsIm5hbWUiOiJPUCIsInN1YiI6Ik9DWTZERTJFVFNOM1VPREZUWUVYQlpMUUxJN1g0R1NYUjU0TlpDNEJCTEk2UUNUWlVUNjVPSVZOIiwidHlwZSI6Im9wZXJhdG9yIiwibmF0cyI6eyJhY2NvdW50X3NlcnZlcl91cmwiOiJodHRwOi8vbG9jYWxob3N0OjgwMDAvand0L3YxIiwib3BlcmF0b3Jfc2VydmljZV91cmxzIjpbIm5hdHM6Ly9sb2NhbGhvc3Q6NDIyMiJdLCJzeXN0ZW1fYWNjb3VudCI6IkFEWjU0N0IyNFdIUExXT0s3VE1MTkJTQTdGUUZYUjZVTTJOWjRISE5JQjdSREZWWlFGT1o0R1FRIn19.3u710KqMLwgXwsMvhxfEp9xzK84XyAZ-4dd6QY0T6hGj8Bw9mS-HcQ7HbvDDNU01S61tNFfpma_JR6LtB3ixBg ` func TestReadOperatorJWT(t *testing.T) { confFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndUrlResolver)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } if opts.SystemAccount != "ADZ547B24WHPLWOK7TMLNBSA7FQFXR6UM2NZ4HHNIB7RDFVZQFOZ4GQQ" { t.Fatalf("Expected different SystemAccount: %s", opts.SystemAccount) } if r, ok := opts.AccountResolver.(*URLAccResolver); !ok { t.Fatalf("Expected different SystemAccount: %s", opts.SystemAccount) } else if r.url != "http://localhost:8000/jwt/v1/accounts/" { t.Fatalf("Expected different SystemAccount: %s", r.url) } } const operatorJwtList = ` listen: "127.0.0.1:-1" system_account = SYSACC operator: [ eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJJVEdJNjNCUUszM1VNN1pBSzZWT1RXNUZEU01ESlNQU1pRQ0RMNUlLUzZQTVhBU0ROQ01RIiwiaWF0IjoxNTg5ODM5MjA1LCJpc3MiOiJPQ1k2REUyRVRTTjNVT0RGVFlFWEJaTFFMSTdYNEdTWFI1NE5aQzRCQkxJNlFDVFpVVDY1T0lWTiIsIm5hbWUiOiJPUCIsInN1YiI6Ik9DWTZERTJFVFNOM1VPREZUWUVYQlpMUUxJN1g0R1NYUjU0TlpDNEJCTEk2UUNUWlVUNjVPSVZOIiwidHlwZSI6Im9wZXJhdG9yIiwibmF0cyI6eyJhY2NvdW50X3NlcnZlcl91cmwiOiJodHRwOi8vbG9jYWxob3N0OjgwMDAvand0L3YxIiwib3BlcmF0b3Jfc2VydmljZV91cmxzIjpbIm5hdHM6Ly9sb2NhbGhvc3Q6NDIyMiJdLCJzeXN0ZW1fYWNjb3VudCI6IkFEWjU0N0IyNFdIUExXT0s3VE1MTkJTQTdGUUZYUjZVTTJOWjRISE5JQjdSREZWWlFGT1o0R1FRIn19.3u710KqMLwgXwsMvhxfEp9xzK84XyAZ-4dd6QY0T6hGj8Bw9mS-HcQ7HbvDDNU01S61tNFfpma_JR6LtB3ixBg, eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIzTVJCS1BRTU1IUjdOQVFQU080NUlWTlkyMzVMRlQyTEs0WkZFVU1KWU9EWUJXU0RXWlRBIiwiaWF0IjoxNzI2NTYwMjAwLCJpc3MiOiJPQkxPR1VCSVVQSkhGVE00RjRaTE9CR1BMSlBJRjRTR0JDWUVERUtFUVNNWVVaTVFTMkRGTUUyWCIsIm5hbWUiOiJvcDIiLCJzdWIiOiJPQkxPR1VCSVVQSkhGVE00RjRaTE9CR1BMSlBJRjRTR0JDWUVERUtFUVNNWVVaTVFTMkRGTUUyWCIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9ES0xMSTZWWldWNk03V1RaV0I3MjVITE9MVFFRVERLNE5RR1ZFR0Q0Q083SjJMMlVJWk81U0dXIl0sInN5c3RlbV9hY2NvdW50IjoiQUNRVFdWR1NHSFlWWTNSNkQyV01PM1Y2TFYyTUdLNUI3RzQ3RTQzQkhKQjZGUVZZN0VITlRNTUciLCJ0eXBlIjoib3BlcmF0b3IiLCJ2ZXJzaW9uIjoyfX0.8kUmC6CwGLTJSs1zj_blsMpP5b6n2jZhZFNvMPXvJlRyyR5ZbCsxJ442BimaxaiosS8T-IFcZAIphtiOcqhRCg ] ` func TestReadMultipleOperatorJWT(t *testing.T) { confFileName := createConfFile(t, []byte(operatorJwtList)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } require_Equal(t, len(opts.TrustedOperators), 2) require_Equal(t, opts.TrustedOperators[0].Name, "OP") require_Equal(t, opts.TrustedOperators[1].Name, "op2") require_Equal(t, opts.TrustedOperators[0].SystemAccount, "ADZ547B24WHPLWOK7TMLNBSA7FQFXR6UM2NZ4HHNIB7RDFVZQFOZ4GQQ") require_Equal(t, opts.TrustedOperators[1].SystemAccount, "ACQTWVGSGHYVY3R6D2WMO3V6LV2MGK5B7G47E43BHJB6FQVY7EHNTMMG") // check if system account precedence is correct require_Equal(t, opts.SystemAccount, "SYSACC") } // using memory resolver so this test does not have to start the memory resolver const operatorJwtWithSysAccAndMemResolver = ` listen: "127.0.0.1:-1" // Operator "TESTOP" operator: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJLRTZRU0tWTU1VWFFKNFZCTDNSNDdGRFlIWElaTDRZSE1INjVIT0k1UjZCNUpPUkxVQlZBIiwiaWF0IjoxNTg5OTE2MzgyLCJpc3MiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsIm5hbWUiOiJURVNUT1AiLCJzdWIiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic3lzdGVtX2FjY291bnQiOiJBRFNQT1lNSFhKTjZKVllRQ0xSWjVYUTVJVU42QTNTMzNYQTROVjRWSDc0NDIzVTdVN1lSNFlWVyJ9fQ.HiyUtlk8kectKHeQHtuqFcjFt0RbYZE_WAqPCcoWlV2IFVdXuOTzShYEMgDmtgvsFG_zxNQOj08Gr6a06ovwBA resolver: MEMORY resolver_preload: { // Account "TESTSYS" ADSPOYMHXJN6JVYQCLRZ5XQ5IUN6A3S33XA4NV4VH74423U7U7YR4YVW: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiI2WEtYUFZNTjdEVFlBSUE0R1JDWUxXUElSM1ZEM1Q2UVk2RFg3NURHTVFVWkdVWTJSRFNRIiwiaWF0IjoxNTg5OTE2MzIzLCJpc3MiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsIm5hbWUiOiJURVNUU1lTIiwic3ViIjoiQURTUE9ZTUhYSk42SlZZUUNMUlo1WFE1SVVONkEzUzMzWEE0TlY0Vkg3NDQyM1U3VTdZUjRZVlciLCJ0eXBlIjoiYWNjb3VudCIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJjb25uIjotMSwibGVhZiI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ3aWxkY2FyZHMiOnRydWV9fX0.vhtWanIrOncdNfg-yO-7L61ccc-yRacvVtEsaIgWBEmW4czlEPhsiF1MkUKG91rtgcbwUf73ZIFEfja5MgFBAQ } ` func TestReadOperatorJWTSystemAccountMatch(t *testing.T) { confFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndMemResolver+` system_account: ADSPOYMHXJN6JVYQCLRZ5XQ5IUN6A3S33XA4NV4VH74423U7U7YR4YVW `)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } s, err := NewServer(opts) if err != nil { t.Fatalf("Received unexpected error %s", err) } s.Shutdown() } func TestReadOperatorJWTSystemAccountMismatch(t *testing.T) { confFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndMemResolver+` system_account: ADXJJCDCSRSMCOV25FXQW7R4QOG7R763TVEXBNWJHLBMBGWOJYG5XZBG `)) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } s, err := NewServer(opts) if err == nil { s.Shutdown() t.Fatalf("Received no error") } else if !strings.Contains(err.Error(), "system_account in config and operator JWT must be identical") { t.Fatalf("Received unexpected error %s", err) } } func TestReadOperatorAssertVersion(t *testing.T) { kp, _ := nkeys.CreateOperator() pk, _ := kp.PublicKey() op := jwt.NewOperatorClaims(pk) op.AssertServerVersion = "1.2.3" jwt, err := op.Encode(kp) if err != nil { t.Fatalf("Received unexpected error %s", err) } confFileName := createConfFile(t, []byte(fmt.Sprintf(` operator: %s resolver: MEM `, jwt))) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } s, err := NewServer(opts) if err != nil { t.Fatalf("Received unexpected error %s", err) } s.Shutdown() } func TestReadOperatorAssertVersionFail(t *testing.T) { kp, _ := nkeys.CreateOperator() pk, _ := kp.PublicKey() op := jwt.NewOperatorClaims(pk) op.AssertServerVersion = "10.20.30" jwt, err := op.Encode(kp) if err != nil { t.Fatalf("Received unexpected error %s", err) } confFileName := createConfFile(t, []byte(fmt.Sprintf(` operator: %s resolver: MEM `, jwt))) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } s, err := NewServer(opts) if err == nil { s.Shutdown() t.Fatalf("Received no error") } else if !strings.Contains(err.Error(), "expected major version 10 > server major version") { t.Fatal("expected different error got: ", err) } } func TestClusterNameAndGatewayNameConflict(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { name: A listen: 127.0.0.1:-1 } gateway { name: B listen: 127.0.0.1:-1 } `)) opts, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Unexpected error: %v", err) } if err := validateOptions(opts); err != ErrClusterNameConfigConflict { t.Fatalf("Expected ErrClusterNameConfigConflict got %v", err) } } func TestDefaultAuthTimeout(t *testing.T) { opts := DefaultOptions() opts.AuthTimeout = 0 s := RunServer(opts) defer s.Shutdown() sopts := s.getOpts() if at := time.Duration(sopts.AuthTimeout * float64(time.Second)); at != AUTH_TIMEOUT { t.Fatalf("Expected auth timeout to be %v, got %v", AUTH_TIMEOUT, at) } s.Shutdown() opts = DefaultOptions() tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", Timeout: 4.0, } tlsConfig, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } opts.TLSConfig = tlsConfig opts.TLSTimeout = tc.Timeout s = RunServer(opts) defer s.Shutdown() sopts = s.getOpts() if sopts.AuthTimeout != 5 { t.Fatalf("Expected auth timeout to be %v, got %v", 5, sopts.AuthTimeout) } } func TestQueuePermissions(t *testing.T) { cfgFmt := ` listen: 127.0.0.1:-1 no_auth_user: u authorization { users [{ user: u, password: pwd, permissions: { sub: { %s: ["foo.> *.dev"] } } }] }` errChan := make(chan error, 1) defer close(errChan) for _, test := range []struct { permType string queue string errExpected bool }{ {"allow", "queue.dev", false}, {"allow", "", true}, {"allow", "bad", true}, {"deny", "", false}, {"deny", "queue.dev", true}, } { t.Run(test.permType+test.queue, func(t *testing.T) { confFileName := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, test.permType))) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Received unexpected error %s", err) } opts.NoLog, opts.NoSigs = true, true s := RunServer(opts) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", opts.Port), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { errChan <- err })) if err != nil { t.Fatalf("No error expected: %v", err) } defer nc.Close() if test.queue == "" { if _, err := nc.Subscribe("foo.bar", func(msg *nats.Msg) {}); err != nil { t.Fatalf("no error expected: %v", err) } } else { if _, err := nc.QueueSubscribe("foo.bar", test.queue, func(msg *nats.Msg) {}); err != nil { t.Fatalf("no error expected: %v", err) } } nc.Flush() select { case err := <-errChan: if !test.errExpected { t.Fatalf("Expected no error, got %v", err) } if !strings.Contains(err.Error(), `Permissions Violation for Subscription to "foo.bar"`) { t.Fatalf("error %v", err) } case <-time.After(150 * time.Millisecond): if test.errExpected { t.Fatal("Expected an error") } } }) } } func TestResolverPinnedAccountsFail(t *testing.T) { cfgFmt := ` operator: %s resolver: URL(foo.bar) resolver_pinned_accounts: [%s] ` conf := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, ojwt, "f"))) srv, err := NewServer(LoadConfig(conf)) defer srv.Shutdown() require_Error(t, err) require_Contains(t, err.Error(), " is not a valid public account nkey") conf = createConfFile(t, []byte(fmt.Sprintf(cfgFmt, ojwt, "1, x"))) _, err = ProcessConfigFile(conf) require_Error(t, err) require_Contains(t, "parsing resolver_pinned_accounts: unsupported type") } func TestMaxSubTokens(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 max_sub_tokens: 4 `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() errs := make(chan error, 1) nc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { errs <- err }) bad := "a.b.c.d.e" _, err = nc.SubscribeSync(bad) require_NoError(t, err) select { case e := <-errs: if !strings.Contains(e.Error(), "too many tokens") { t.Fatalf("Got wrong error: %v", err) } case <-time.After(2 * time.Second): t.Fatal("Did not get the permissions error") } } func TestGetStorageSize(t *testing.T) { tt := []struct { input string want int64 err bool }{ {"1K", 1024, false}, {"1M", 1048576, false}, {"1G", 1073741824, false}, {"1T", 1099511627776, false}, {"1L", 0, true}, {"TT", 0, true}, {"", 0, false}, } for _, v := range tt { var testErr bool got, err := getStorageSize(v.input) if err != nil { testErr = true } if got != v.want || v.err != testErr { t.Errorf("Got: %v, want %v with error: %v", got, v.want, testErr) } } } func TestAuthorizationAndAccountsMisconfigurations(t *testing.T) { // There is a test called TestConfigCheck but we can't use it // because the place where the error will be reported will depend // if the error is found while parsing authorization{} or // accounts{}, but we can't control the internal parsing of those // (due to lexer giving back a map and iteration over map is random) // regardless of the ordering in the configuration file. // The test is also repeated for _, test := range []struct { name string config string err string }{ { "duplicate users", ` authorization = {users = [ {user: "user1", pass: "pwd"} ] } accounts { ACC { users = [ {user: "user1"} ] } } `, fmt.Sprintf("Duplicate user %q detected", "user1"), }, { "duplicate nkey", ` authorization = {users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } accounts { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } } `, fmt.Sprintf("Duplicate nkey %q detected", "UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX"), }, { "auth single user and password and accounts users", ` authorization = {user: "user1", password: "pwd"} accounts = { ACC { users = [ {user: "user2", pass: "pwd"} ] } } `, "Can not have a single user/pass", }, { "auth single user and password and accounts nkeys", ` authorization = {user: "user1", password: "pwd"} accounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } } `, "Can not have a single user/pass", }, { "auth token and accounts users", ` authorization = {token: "my_token"} accounts = { ACC { users = [ {user: "user2", pass: "pwd"} ] } } `, "Can not have a token", }, { "auth token and accounts nkeys", ` authorization = {token: "my_token"} accounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } } `, "Can not have a token", }, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(test.config)) if _, err := ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error %q, got %q", test.err, err.Error()) } }) } } nats-server-2.10.27/server/parser.go000066400000000000000000000620321477524627100173070ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "fmt" "net/http" "net/textproto" ) type parserState int type parseState struct { state parserState op byte as int drop int pa pubArg argBuf []byte msgBuf []byte header http.Header // access via getHeader scratch [MAX_CONTROL_LINE_SIZE]byte } type pubArg struct { arg []byte pacache []byte origin []byte account []byte subject []byte deliver []byte mapped []byte reply []byte szb []byte hdb []byte queues [][]byte size int hdr int psi []*serviceImport delivered bool // Only used for service imports } // Parser constants const ( OP_START parserState = iota OP_PLUS OP_PLUS_O OP_PLUS_OK OP_MINUS OP_MINUS_E OP_MINUS_ER OP_MINUS_ERR OP_MINUS_ERR_SPC MINUS_ERR_ARG OP_C OP_CO OP_CON OP_CONN OP_CONNE OP_CONNEC OP_CONNECT CONNECT_ARG OP_H OP_HP OP_HPU OP_HPUB OP_HPUB_SPC HPUB_ARG OP_HM OP_HMS OP_HMSG OP_HMSG_SPC HMSG_ARG OP_P OP_PU OP_PUB OP_PUB_SPC PUB_ARG OP_PI OP_PIN OP_PING OP_PO OP_PON OP_PONG MSG_PAYLOAD MSG_END_R MSG_END_N OP_S OP_SU OP_SUB OP_SUB_SPC SUB_ARG OP_A OP_ASUB OP_ASUB_SPC ASUB_ARG OP_AUSUB OP_AUSUB_SPC AUSUB_ARG OP_L OP_LS OP_R OP_RS OP_U OP_UN OP_UNS OP_UNSU OP_UNSUB OP_UNSUB_SPC UNSUB_ARG OP_M OP_MS OP_MSG OP_MSG_SPC MSG_ARG OP_I OP_IN OP_INF OP_INFO INFO_ARG ) func (c *client) parse(buf []byte) error { // Branch out to mqtt clients. c.mqtt is immutable, but should it become // an issue (say data race detection), we could branch outside in readLoop if c.isMqtt() { return c.mqttParse(buf) } var i int var b byte var lmsg bool // Snapshots c.mu.Lock() // Snapshot and then reset when we receive a // proper CONNECT if needed. authSet := c.awaitingAuth() // Snapshot max control line as well. s, mcl, trace := c.srv, c.mcl, c.trace c.mu.Unlock() // Move to loop instead of range syntax to allow jumping of i for i = 0; i < len(buf); i++ { b = buf[i] switch c.state { case OP_START: c.op = b if b != 'C' && b != 'c' { if authSet { if s == nil { goto authErr } var ok bool // Check here for NoAuthUser. If this is set allow non CONNECT protos as our first. // E.g. telnet proto demos. if noAuthUser := s.getOpts().NoAuthUser; noAuthUser != _EMPTY_ { s.mu.Lock() user, exists := s.users[noAuthUser] s.mu.Unlock() if exists { c.RegisterUser(user) c.mu.Lock() c.clearAuthTimer() c.flags.set(connectReceived) c.mu.Unlock() authSet, ok = false, true } } if !ok { goto authErr } } // If the connection is a gateway connection, make sure that // if this is an inbound, it starts with a CONNECT. if c.kind == GATEWAY && !c.gw.outbound && !c.gw.connected { // Use auth violation since no CONNECT was sent. // It could be a parseErr too. goto authErr } } switch b { case 'P', 'p': c.state = OP_P case 'H', 'h': c.state = OP_H case 'S', 's': c.state = OP_S case 'U', 'u': c.state = OP_U case 'R', 'r': if c.kind == CLIENT { goto parseErr } else { c.state = OP_R } case 'L', 'l': if c.kind != LEAF && c.kind != ROUTER { goto parseErr } else { c.state = OP_L } case 'A', 'a': if c.kind == CLIENT { goto parseErr } else { c.state = OP_A } case 'C', 'c': c.state = OP_C case 'I', 'i': c.state = OP_I case '+': c.state = OP_PLUS case '-': c.state = OP_MINUS default: goto parseErr } case OP_H: switch b { case 'P', 'p': c.state = OP_HP case 'M', 'm': c.state = OP_HM default: goto parseErr } case OP_HP: switch b { case 'U', 'u': c.state = OP_HPU default: goto parseErr } case OP_HPU: switch b { case 'B', 'b': c.state = OP_HPUB default: goto parseErr } case OP_HPUB: switch b { case ' ', '\t': c.state = OP_HPUB_SPC default: goto parseErr } case OP_HPUB_SPC: switch b { case ' ', '\t': continue default: c.pa.hdr = 0 c.state = HPUB_ARG c.as = i } case HPUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if trace { c.traceInOp("HPUB", arg) } if err := c.processHeaderPub(arg); err != nil { return err } c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD // If we don't have a saved buffer then jump ahead with // the index. If this overruns what is left we fall out // and process split buffer. if c.msgBuf == nil { i = c.as + c.pa.size - LEN_CR_LF } default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_HM: switch b { case 'S', 's': c.state = OP_HMS default: goto parseErr } case OP_HMS: switch b { case 'G', 'g': c.state = OP_HMSG default: goto parseErr } case OP_HMSG: switch b { case ' ', '\t': c.state = OP_HMSG_SPC default: goto parseErr } case OP_HMSG_SPC: switch b { case ' ', '\t': continue default: c.pa.hdr = 0 c.state = HMSG_ARG c.as = i } case HMSG_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } var err error if c.kind == ROUTER || c.kind == GATEWAY { if trace { c.traceInOp("HMSG", arg) } err = c.processRoutedHeaderMsgArgs(arg) } else if c.kind == LEAF { if trace { c.traceInOp("HMSG", arg) } err = c.processLeafHeaderMsgArgs(arg) } if err != nil { return err } c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD // jump ahead with the index. If this overruns // what is left we fall out and process split // buffer. i = c.as + c.pa.size - LEN_CR_LF default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_P: switch b { case 'U', 'u': c.state = OP_PU case 'I', 'i': c.state = OP_PI case 'O', 'o': c.state = OP_PO default: goto parseErr } case OP_PU: switch b { case 'B', 'b': c.state = OP_PUB default: goto parseErr } case OP_PUB: switch b { case ' ', '\t': c.state = OP_PUB_SPC default: goto parseErr } case OP_PUB_SPC: switch b { case ' ', '\t': continue default: c.pa.hdr = -1 c.state = PUB_ARG c.as = i } case PUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if trace { c.traceInOp("PUB", arg) } if err := c.processPub(arg); err != nil { return err } c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD // If we don't have a saved buffer then jump ahead with // the index. If this overruns what is left we fall out // and process split buffer. if c.msgBuf == nil { i = c.as + c.pa.size - LEN_CR_LF } default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case MSG_PAYLOAD: if c.msgBuf != nil { // copy as much as we can to the buffer and skip ahead. toCopy := c.pa.size - len(c.msgBuf) avail := len(buf) - i if avail < toCopy { toCopy = avail } if toCopy > 0 { start := len(c.msgBuf) // This is needed for copy to work. c.msgBuf = c.msgBuf[:start+toCopy] copy(c.msgBuf[start:], buf[i:i+toCopy]) // Update our index i = (i + toCopy) - 1 } else { // Fall back to append if needed. c.msgBuf = append(c.msgBuf, b) } if len(c.msgBuf) >= c.pa.size { c.state = MSG_END_R } } else if i-c.as+1 >= c.pa.size { c.state = MSG_END_R } case MSG_END_R: if b != '\r' { goto parseErr } if c.msgBuf != nil { c.msgBuf = append(c.msgBuf, b) } c.state = MSG_END_N case MSG_END_N: if b != '\n' { goto parseErr } if c.msgBuf != nil { c.msgBuf = append(c.msgBuf, b) } else { c.msgBuf = buf[c.as : i+1] } // Check for mappings. if (c.kind == CLIENT || c.kind == LEAF) && c.in.flags.isSet(hasMappings) { changed := c.selectMappedSubject() if trace && changed { c.traceInOp("MAPPING", []byte(fmt.Sprintf("%s -> %s", c.pa.mapped, c.pa.subject))) } } if trace { c.traceMsg(c.msgBuf) } c.processInboundMsg(c.msgBuf) c.argBuf, c.msgBuf, c.header = nil, nil, nil c.drop, c.as, c.state = 0, i+1, OP_START // Drop all pub args c.pa.arg, c.pa.pacache, c.pa.origin, c.pa.account, c.pa.subject, c.pa.mapped = nil, nil, nil, nil, nil, nil c.pa.reply, c.pa.hdr, c.pa.size, c.pa.szb, c.pa.hdb, c.pa.queues = nil, -1, 0, nil, nil, nil c.pa.delivered = false lmsg = false case OP_A: switch b { case '+': c.state = OP_ASUB case '-', 'u': c.state = OP_AUSUB default: goto parseErr } case OP_ASUB: switch b { case ' ', '\t': c.state = OP_ASUB_SPC default: goto parseErr } case OP_ASUB_SPC: switch b { case ' ', '\t': continue default: c.state = ASUB_ARG c.as = i } case ASUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if trace { c.traceInOp("A+", arg) } if err := c.processAccountSub(arg); err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_AUSUB: switch b { case ' ', '\t': c.state = OP_AUSUB_SPC default: goto parseErr } case OP_AUSUB_SPC: switch b { case ' ', '\t': continue default: c.state = AUSUB_ARG c.as = i } case AUSUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if trace { c.traceInOp("A-", arg) } c.processAccountUnsub(arg) c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_S: switch b { case 'U', 'u': c.state = OP_SU default: goto parseErr } case OP_SU: switch b { case 'B', 'b': c.state = OP_SUB default: goto parseErr } case OP_SUB: switch b { case ' ', '\t': c.state = OP_SUB_SPC default: goto parseErr } case OP_SUB_SPC: switch b { case ' ', '\t': continue default: c.state = SUB_ARG c.as = i } case SUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } var err error switch c.kind { case CLIENT: if trace { c.traceInOp("SUB", arg) } err = c.parseSub(arg, false) case ROUTER: switch c.op { case 'R', 'r': if trace { c.traceInOp("RS+", arg) } err = c.processRemoteSub(arg, false) case 'L', 'l': if trace { c.traceInOp("LS+", arg) } err = c.processRemoteSub(arg, true) } case GATEWAY: if trace { c.traceInOp("RS+", arg) } err = c.processGatewayRSub(arg) case LEAF: if trace { c.traceInOp("LS+", arg) } err = c.processLeafSub(arg) } if err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_L: switch b { case 'S', 's': c.state = OP_LS case 'M', 'm': c.state = OP_M default: goto parseErr } case OP_LS: switch b { case '+': c.state = OP_SUB case '-': c.state = OP_UNSUB default: goto parseErr } case OP_R: switch b { case 'S', 's': c.state = OP_RS case 'M', 'm': c.state = OP_M default: goto parseErr } case OP_RS: switch b { case '+': c.state = OP_SUB case '-': c.state = OP_UNSUB default: goto parseErr } case OP_U: switch b { case 'N', 'n': c.state = OP_UN default: goto parseErr } case OP_UN: switch b { case 'S', 's': c.state = OP_UNS default: goto parseErr } case OP_UNS: switch b { case 'U', 'u': c.state = OP_UNSU default: goto parseErr } case OP_UNSU: switch b { case 'B', 'b': c.state = OP_UNSUB default: goto parseErr } case OP_UNSUB: switch b { case ' ', '\t': c.state = OP_UNSUB_SPC default: goto parseErr } case OP_UNSUB_SPC: switch b { case ' ', '\t': continue default: c.state = UNSUB_ARG c.as = i } case UNSUB_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } var err error switch c.kind { case CLIENT: if trace { c.traceInOp("UNSUB", arg) } err = c.processUnsub(arg) case ROUTER: if trace && c.srv != nil { switch c.op { case 'R', 'r': c.traceInOp("RS-", arg) case 'L', 'l': c.traceInOp("LS-", arg) } } leafUnsub := c.op == 'L' || c.op == 'l' err = c.processRemoteUnsub(arg, leafUnsub) case GATEWAY: if trace { c.traceInOp("RS-", arg) } err = c.processGatewayRUnsub(arg) case LEAF: if trace { c.traceInOp("LS-", arg) } err = c.processLeafUnsub(arg) } if err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_PI: switch b { case 'N', 'n': c.state = OP_PIN default: goto parseErr } case OP_PIN: switch b { case 'G', 'g': c.state = OP_PING default: goto parseErr } case OP_PING: switch b { case '\n': if trace { c.traceInOp("PING", nil) } c.processPing() c.drop, c.state = 0, OP_START } case OP_PO: switch b { case 'N', 'n': c.state = OP_PON default: goto parseErr } case OP_PON: switch b { case 'G', 'g': c.state = OP_PONG default: goto parseErr } case OP_PONG: switch b { case '\n': if trace { c.traceInOp("PONG", nil) } c.processPong() c.drop, c.state = 0, OP_START } case OP_C: switch b { case 'O', 'o': c.state = OP_CO default: goto parseErr } case OP_CO: switch b { case 'N', 'n': c.state = OP_CON default: goto parseErr } case OP_CON: switch b { case 'N', 'n': c.state = OP_CONN default: goto parseErr } case OP_CONN: switch b { case 'E', 'e': c.state = OP_CONNE default: goto parseErr } case OP_CONNE: switch b { case 'C', 'c': c.state = OP_CONNEC default: goto parseErr } case OP_CONNEC: switch b { case 'T', 't': c.state = OP_CONNECT default: goto parseErr } case OP_CONNECT: switch b { case ' ', '\t': continue default: c.state = CONNECT_ARG c.as = i } case CONNECT_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if trace { c.traceInOp("CONNECT", removePassFromTrace(arg)) } if err := c.processConnect(arg); err != nil { return err } c.drop, c.state = 0, OP_START // Reset notion on authSet c.mu.Lock() authSet = c.awaitingAuth() c.mu.Unlock() default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_M: switch b { case 'S', 's': c.state = OP_MS default: goto parseErr } case OP_MS: switch b { case 'G', 'g': c.state = OP_MSG default: goto parseErr } case OP_MSG: switch b { case ' ', '\t': c.state = OP_MSG_SPC default: goto parseErr } case OP_MSG_SPC: switch b { case ' ', '\t': continue default: c.pa.hdr = -1 c.state = MSG_ARG c.as = i } case MSG_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } var err error if c.kind == ROUTER || c.kind == GATEWAY { switch c.op { case 'R', 'r': if trace { c.traceInOp("RMSG", arg) } err = c.processRoutedMsgArgs(arg) case 'L', 'l': if trace { c.traceInOp("LMSG", arg) } lmsg = true err = c.processRoutedOriginClusterMsgArgs(arg) } } else if c.kind == LEAF { if trace { c.traceInOp("LMSG", arg) } err = c.processLeafMsgArgs(arg) } if err != nil { return err } c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD // jump ahead with the index. If this overruns // what is left we fall out and process split // buffer. i = c.as + c.pa.size - LEN_CR_LF default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_I: switch b { case 'N', 'n': c.state = OP_IN default: goto parseErr } case OP_IN: switch b { case 'F', 'f': c.state = OP_INF default: goto parseErr } case OP_INF: switch b { case 'O', 'o': c.state = OP_INFO default: goto parseErr } case OP_INFO: switch b { case ' ', '\t': continue default: c.state = INFO_ARG c.as = i } case INFO_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } if err := c.processInfo(arg); err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } case OP_PLUS: switch b { case 'O', 'o': c.state = OP_PLUS_O default: goto parseErr } case OP_PLUS_O: switch b { case 'K', 'k': c.state = OP_PLUS_OK default: goto parseErr } case OP_PLUS_OK: switch b { case '\n': c.drop, c.state = 0, OP_START } case OP_MINUS: switch b { case 'E', 'e': c.state = OP_MINUS_E default: goto parseErr } case OP_MINUS_E: switch b { case 'R', 'r': c.state = OP_MINUS_ER default: goto parseErr } case OP_MINUS_ER: switch b { case 'R', 'r': c.state = OP_MINUS_ERR default: goto parseErr } case OP_MINUS_ERR: switch b { case ' ', '\t': c.state = OP_MINUS_ERR_SPC default: goto parseErr } case OP_MINUS_ERR_SPC: switch b { case ' ', '\t': continue default: c.state = MINUS_ERR_ARG c.as = i } case MINUS_ERR_ARG: switch b { case '\r': c.drop = 1 case '\n': var arg []byte if c.argBuf != nil { arg = c.argBuf c.argBuf = nil } else { arg = buf[c.as : i-c.drop] } if err := c.overMaxControlLineLimit(arg, mcl); err != nil { return err } c.processErr(string(arg)) c.drop, c.as, c.state = 0, i+1, OP_START default: if c.argBuf != nil { c.argBuf = append(c.argBuf, b) } } default: goto parseErr } } // Check for split buffer scenarios for any ARG state. if c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG || c.state == HPUB_ARG || c.state == ASUB_ARG || c.state == AUSUB_ARG || c.state == MSG_ARG || c.state == HMSG_ARG || c.state == MINUS_ERR_ARG || c.state == CONNECT_ARG || c.state == INFO_ARG { // Setup a holder buffer to deal with split buffer scenario. if c.argBuf == nil { c.argBuf = c.scratch[:0] c.argBuf = append(c.argBuf, buf[c.as:i-c.drop]...) } // Check for violations of control line length here. Note that this is not // exact at all but the performance hit is too great to be precise, and // catching here should prevent memory exhaustion attacks. if err := c.overMaxControlLineLimit(c.argBuf, mcl); err != nil { return err } } // Check for split msg if (c.state == MSG_PAYLOAD || c.state == MSG_END_R || c.state == MSG_END_N) && c.msgBuf == nil { // We need to clone the pubArg if it is still referencing the // read buffer and we are not able to process the msg. if c.argBuf == nil { // Works also for MSG_ARG, when message comes from ROUTE or GATEWAY. if err := c.clonePubArg(lmsg); err != nil { goto parseErr } } // If we will overflow the scratch buffer, just create a // new buffer to hold the split message. if c.pa.size > cap(c.scratch)-len(c.argBuf) { lrem := len(buf[c.as:]) // Consider it a protocol error when the remaining payload // is larger than the reported size for PUB. It can happen // when processing incomplete messages from rogue clients. if lrem > c.pa.size+LEN_CR_LF { goto parseErr } c.msgBuf = make([]byte, lrem, c.pa.size+LEN_CR_LF) copy(c.msgBuf, buf[c.as:]) } else { c.msgBuf = c.scratch[len(c.argBuf):len(c.argBuf)] c.msgBuf = append(c.msgBuf, (buf[c.as:])...) } } return nil authErr: c.authViolation() return ErrAuthentication parseErr: c.sendErr("Unknown Protocol Operation") snip := protoSnippet(i, PROTO_SNIPPET_SIZE, buf) err := fmt.Errorf("%s parser ERROR, state=%d, i=%d: proto='%s...'", c.kindString(), c.state, i, snip) return err } func protoSnippet(start, max int, buf []byte) string { stop := start + max bufSize := len(buf) if start >= bufSize { return `""` } if stop > bufSize { stop = bufSize - 1 } return fmt.Sprintf("%q", buf[start:stop]) } // Check if the length of buffer `arg` is over the max control line limit `mcl`. // If so, an error is sent to the client and the connection is closed. // The error ErrMaxControlLine is returned. func (c *client) overMaxControlLineLimit(arg []byte, mcl int32) error { if c.kind != CLIENT { return nil } if len(arg) > int(mcl) { err := NewErrorCtx(ErrMaxControlLine, "State %d, max_control_line %d, Buffer len %d (snip: %s...)", c.state, int(mcl), len(c.argBuf), protoSnippet(0, MAX_CONTROL_LINE_SNIPPET_SIZE, arg)) c.sendErr(err.Error()) c.closeConnection(MaxControlLineExceeded) return err } return nil } // clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but // we need to hold onto it into the next read. func (c *client) clonePubArg(lmsg bool) error { // Just copy and re-process original arg buffer. c.argBuf = c.scratch[:0] c.argBuf = append(c.argBuf, c.pa.arg...) switch c.kind { case ROUTER, GATEWAY: if lmsg { return c.processRoutedOriginClusterMsgArgs(c.argBuf) } if c.pa.hdr < 0 { return c.processRoutedMsgArgs(c.argBuf) } else { return c.processRoutedHeaderMsgArgs(c.argBuf) } case LEAF: if c.pa.hdr < 0 { return c.processLeafMsgArgs(c.argBuf) } else { return c.processLeafHeaderMsgArgs(c.argBuf) } default: if c.pa.hdr < 0 { return c.processPub(c.argBuf) } else { return c.processHeaderPub(c.argBuf) } } } func (ps *parseState) getHeader() http.Header { if ps.header == nil { if hdr := ps.pa.hdr; hdr > 0 { reader := bufio.NewReader(bytes.NewReader(ps.msgBuf[0:hdr])) tp := textproto.NewReader(reader) tp.ReadLine() // skip over first line, contains version if mimeHeader, err := tp.ReadMIMEHeader(); err == nil { ps.header = http.Header(mimeHeader) } } } return ps.header } nats-server-2.10.27/server/parser_test.go000066400000000000000000000716761477524627100203640ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "testing" ) func dummyClient() *client { return &client{srv: New(&defaultServerOptions), msubs: -1, mpay: -1, mcl: MAX_CONTROL_LINE_SIZE} } func dummyRouteClient() *client { return &client{srv: New(&defaultServerOptions), kind: ROUTER} } func TestParsePing(t *testing.T) { c := dummyClient() if c.state != OP_START { t.Fatalf("Expected OP_START vs %d\n", c.state) } ping := []byte("PING\r\n") err := c.parse(ping[:1]) if err != nil || c.state != OP_P { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping[1:2]) if err != nil || c.state != OP_PI { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping[2:3]) if err != nil || c.state != OP_PIN { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping[3:4]) if err != nil || c.state != OP_PING { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping[4:5]) if err != nil || c.state != OP_PING { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping[5:6]) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(ping) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } // Should tolerate spaces ping = []byte("PING \r") err = c.parse(ping) if err != nil || c.state != OP_PING { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } c.state = OP_START ping = []byte("PING \r \n") err = c.parse(ping) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } } func TestParsePong(t *testing.T) { c := dummyClient() if c.state != OP_START { t.Fatalf("Expected OP_START vs %d\n", c.state) } pong := []byte("PONG\r\n") err := c.parse(pong[:1]) if err != nil || c.state != OP_P { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(pong[1:2]) if err != nil || c.state != OP_PO { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(pong[2:3]) if err != nil || c.state != OP_PON { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(pong[3:4]) if err != nil || c.state != OP_PONG { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(pong[4:5]) if err != nil || c.state != OP_PONG { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(pong[5:6]) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if c.ping.out != 0 { t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) } err = c.parse(pong) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if c.ping.out != 0 { t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) } // Should tolerate spaces pong = []byte("PONG \r") err = c.parse(pong) if err != nil || c.state != OP_PONG { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } c.state = OP_START pong = []byte("PONG \r \n") err = c.parse(pong) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if c.ping.out != 0 { t.Fatalf("Unexpected ping.out value: %d vs 0\n", c.ping.out) } // Should be adjusting c.pout (Pings Outstanding): reset to 0 c.state = OP_START c.ping.out = 10 pong = []byte("PONG\r\n") err = c.parse(pong) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if c.ping.out != 0 { t.Fatalf("Unexpected ping.out: %d vs 0\n", c.ping.out) } } func TestParseConnect(t *testing.T) { c := dummyClient() connect := []byte("CONNECT {\"verbose\":false,\"pedantic\":true,\"tls_required\":false}\r\n") err := c.parse(connect) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } // Check saved state if c.as != 8 { t.Fatalf("ArgStart state incorrect: 8 vs %d\n", c.as) } } func TestParseSub(t *testing.T) { c := dummyClient() sub := []byte("SUB foo 1\r") err := c.parse(sub) if err != nil || c.state != SUB_ARG { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } // Check saved state if c.as != 4 { t.Fatalf("ArgStart state incorrect: 4 vs %d\n", c.as) } if c.drop != 1 { t.Fatalf("Drop state incorrect: 1 vs %d\n", c.as) } if !bytes.Equal(sub[c.as:], []byte("foo 1\r")) { t.Fatalf("Arg state incorrect: %s\n", sub[c.as:]) } } func TestParsePub(t *testing.T) { c := dummyClient() pub := []byte("PUB foo 5\r\nhello\r") err := c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", string(c.pa.subject)) } if c.pa.reply != nil { t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", string(c.pa.reply)) } if c.pa.size != 5 { t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("PUB foo.bar INBOX.22 11\r\nhello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", string(c.pa.subject)) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", string(c.pa.reply)) } if c.pa.size != 11 { t.Fatalf("Did not parse msg size correctly: 11 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START // This is the case when data has more bytes than expected by size. pub = []byte("PUB foo.bar 11\r\nhello world hello world\r") err = c.parse(pub) if err == nil { t.Fatalf("Expected an error parsing longer than expected message body") } if c.msgBuf != nil { t.Fatalf("Did not expect a c.msgBuf to be non-nil") } } // https://www.twistlock.com/labs-blog/finding-dos-vulnerability-nats-go-fuzz-cve-2019-13126/ func TestParsePubSizeOverflow(t *testing.T) { c := dummyClient() pub := []byte("PUB foo 3333333333333333333333333333333333333333333333333333333333333333\r\n") if err := c.parse(pub); err == nil { t.Fatalf("Expected an error") } } func TestParsePubArg(t *testing.T) { c := dummyClient() for _, test := range []struct { arg string subject string reply string size int szb string }{ {arg: "a 2", subject: "a", reply: "", size: 2, szb: "2"}, {arg: "a 222", subject: "a", reply: "", size: 222, szb: "222"}, {arg: "foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: " foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo 22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: " foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: " foo 22 ", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: " foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "foo bar 22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: " foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: " foo bar 22 ", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: " foo bar 2222 ", subject: "foo", reply: "bar", size: 2222, szb: "2222"}, {arg: " foo 2222 ", subject: "foo", reply: "", size: 2222, szb: "2222"}, {arg: "a\t2", subject: "a", reply: "", size: 2, szb: "2"}, {arg: "a\t222", subject: "a", reply: "", size: 222, szb: "222"}, {arg: "foo\t22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "\tfoo\t22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo\t\t\t22", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "\tfoo\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "\tfoo\t\t\t22\t", subject: "foo", reply: "", size: 22, szb: "22"}, {arg: "foo\tbar\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "\tfoo\tbar\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "foo\tbar\t22\t", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "foo\t\tbar\t\t22", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "\tfoo\tbar\t22\t", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "\t \tfoo\t \t \tbar\t \t22\t \t", subject: "foo", reply: "bar", size: 22, szb: "22"}, {arg: "\t\tfoo\t\t\tbar\t\t2222\t\t", subject: "foo", reply: "bar", size: 2222, szb: "2222"}, {arg: "\t \tfoo\t \t \t\t\t2222\t \t", subject: "foo", reply: "", size: 2222, szb: "2222"}, } { t.Run(test.arg, func(t *testing.T) { if err := c.processPub([]byte(test.arg)); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if !bytes.Equal(c.pa.subject, []byte(test.subject)) { t.Fatalf("Mismatched subject: '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte(test.reply)) { t.Fatalf("Mismatched reply subject: '%s'\n", c.pa.reply) } if !bytes.Equal(c.pa.szb, []byte(test.szb)) { t.Fatalf("Bad size buf: '%s'\n", c.pa.szb) } if c.pa.size != test.size { t.Fatalf("Bad size: %d\n", c.pa.size) } }) } } func TestParsePubBadSize(t *testing.T) { c := dummyClient() // Setup localized max payload c.mpay = 32768 if err := c.processPub([]byte("foo 2222222222222222")); err == nil { t.Fatalf("Expected parse error for size too large") } } func TestParseHeaderPub(t *testing.T) { c := dummyClient() c.headers = true hpub := []byte("HPUB foo 12 17\r\nname:derek\r\nHELLO\r") if err := c.parse(hpub); err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'", c.pa.subject) } if c.pa.reply != nil { t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'", c.pa.reply) } if c.pa.hdr != 12 { t.Fatalf("Did not parse msg header size correctly: 12 vs %d", c.pa.hdr) } if !bytes.Equal(c.pa.hdb, []byte("12")) { t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) } if c.pa.size != 17 { t.Fatalf("Did not parse msg size correctly: 17 vs %d", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START hpub = []byte("HPUB foo INBOX.22 12 17\r\nname:derek\r\nHELLO\r") if err := c.parse(hpub); err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'", c.pa.reply) } if c.pa.hdr != 12 { t.Fatalf("Did not parse msg header size correctly: 12 vs %d", c.pa.hdr) } if !bytes.Equal(c.pa.hdb, []byte("12")) { t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) } if c.pa.size != 17 { t.Fatalf("Did not parse msg size correctly: 17 vs %d", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START hpub = []byte("HPUB foo INBOX.22 0 5\r\nHELLO\r") if err := c.parse(hpub); err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) } if c.pa.hdr != 0 { t.Fatalf("Did not parse msg header size correctly: 0 vs %d\n", c.pa.hdr) } if !bytes.Equal(c.pa.hdb, []byte("0")) { t.Fatalf("Did not parse or capture the header size as bytes correctly: %q", c.pa.hdb) } if c.pa.size != 5 { t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) } } func TestParseHeaderPubArg(t *testing.T) { c := dummyClient() c.headers = true for _, test := range []struct { arg string subject string reply string hdr int size int szb string }{ {arg: "a 2 4", subject: "a", reply: "", hdr: 2, size: 4, szb: "4"}, {arg: "a 22 222", subject: "a", reply: "", hdr: 22, size: 222, szb: "222"}, {arg: "foo 3 22", subject: "foo", reply: "", hdr: 3, size: 22, szb: "22"}, {arg: " foo 1 22", subject: "foo", reply: "", hdr: 1, size: 22, szb: "22"}, {arg: "foo 0 22 ", subject: "foo", reply: "", hdr: 0, size: 22, szb: "22"}, {arg: "foo 0 22", subject: "foo", reply: "", hdr: 0, size: 22, szb: "22"}, {arg: " foo 1 22 ", subject: "foo", reply: "", hdr: 1, size: 22, szb: "22"}, {arg: " foo 3 22 ", subject: "foo", reply: "", hdr: 3, size: 22, szb: "22"}, {arg: "foo bar 1 22", subject: "foo", reply: "bar", hdr: 1, size: 22, szb: "22"}, {arg: " foo bar 11 22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "foo bar 11 22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: " foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: " foo bar 11 22 ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: " foo bar 22 2222 ", subject: "foo", reply: "bar", hdr: 22, size: 2222, szb: "2222"}, {arg: " foo 1 2222 ", subject: "foo", reply: "", hdr: 1, size: 2222, szb: "2222"}, {arg: "a\t2\t22", subject: "a", reply: "", hdr: 2, size: 22, szb: "22"}, {arg: "a\t2\t\t222", subject: "a", reply: "", hdr: 2, size: 222, szb: "222"}, {arg: "foo\t2 22", subject: "foo", reply: "", hdr: 2, size: 22, szb: "22"}, {arg: "\tfoo\t11\t 22", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, {arg: "foo\t11\t22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, {arg: "foo\t\t\t11 22", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, {arg: "\tfoo\t11\t \t 22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, {arg: "\tfoo\t\t\t11 22\t", subject: "foo", reply: "", hdr: 11, size: 22, szb: "22"}, {arg: "foo\tbar\t2 22", subject: "foo", reply: "bar", hdr: 2, size: 22, szb: "22"}, {arg: "\tfoo\tbar\t11\t22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "foo\tbar\t11\t\t22\t ", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "foo\t\tbar\t\t11\t\t\t22", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "\tfoo\tbar\t11\t22\t", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "\t \tfoo\t \t \tbar\t \t11\t 22\t \t", subject: "foo", reply: "bar", hdr: 11, size: 22, szb: "22"}, {arg: "\t\tfoo\t\t\tbar\t\t22\t\t\t2222\t\t", subject: "foo", reply: "bar", hdr: 22, size: 2222, szb: "2222"}, {arg: "\t \tfoo\t \t \t\t\t11\t\t 2222\t \t", subject: "foo", reply: "", hdr: 11, size: 2222, szb: "2222"}, } { t.Run(test.arg, func(t *testing.T) { if err := c.processHeaderPub([]byte(test.arg)); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if !bytes.Equal(c.pa.subject, []byte(test.subject)) { t.Fatalf("Mismatched subject: '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte(test.reply)) { t.Fatalf("Mismatched reply subject: '%s'\n", c.pa.reply) } if !bytes.Equal(c.pa.szb, []byte(test.szb)) { t.Fatalf("Bad size buf: '%s'\n", c.pa.szb) } if c.pa.hdr != test.hdr { t.Fatalf("Bad header size: %d\n", c.pa.hdr) } if c.pa.size != test.size { t.Fatalf("Bad size: %d\n", c.pa.size) } }) } } func TestParseRoutedHeaderMsg(t *testing.T) { c := dummyRouteClient() c.route = &route{} pub := []byte("HMSG $foo foo 10 8\r\nXXXhello\r") if err := c.parse(pub); err == nil { t.Fatalf("Expected an error") } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("HMSG $foo foo 3 8\r\nXXXhello\r") err := c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$foo")) { t.Fatalf("Did not parse account correctly: '$foo' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if c.pa.reply != nil { t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", c.pa.reply) } if c.pa.hdr != 3 { t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) } if c.pa.size != 8 { t.Fatalf("Did not parse msg size correctly: 8 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("HMSG $G foo.bar INBOX.22 3 14\r\nOK:hello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) } if c.pa.hdr != 3 { t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) } if c.pa.size != 14 { t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("HMSG $G foo.bar + reply baz 3 14\r\nOK:hello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("reply")) { t.Fatalf("Did not parse reply correctly: 'reply' vs '%s'\n", c.pa.reply) } if len(c.pa.queues) != 1 { t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) } if !bytes.Equal(c.pa.queues[0], []byte("baz")) { t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) } if c.pa.hdr != 3 { t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) } if c.pa.size != 14 { t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("HMSG $G foo.bar | baz 3 14\r\nOK:hello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("")) { t.Fatalf("Did not parse reply correctly: '' vs '%s'\n", c.pa.reply) } if len(c.pa.queues) != 1 { t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) } if !bytes.Equal(c.pa.queues[0], []byte("baz")) { t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) } if c.pa.hdr != 3 { t.Fatalf("Did not parse header size correctly: 3 vs %d\n", c.pa.hdr) } if c.pa.size != 14 { t.Fatalf("Did not parse msg size correctly: 14 vs %d\n", c.pa.size) } } func TestParseRouteMsg(t *testing.T) { c := dummyRouteClient() c.route = &route{} pub := []byte("MSG $foo foo 5\r\nhello\r") err := c.parse(pub) if err == nil { t.Fatalf("Expected an error, got none") } pub = []byte("RMSG $foo foo 5\r\nhello\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$foo")) { t.Fatalf("Did not parse account correctly: '$foo' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if c.pa.reply != nil { t.Fatalf("Did not parse reply correctly: 'nil' vs '%s'\n", c.pa.reply) } if c.pa.size != 5 { t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("RMSG $G foo.bar INBOX.22 11\r\nhello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("Did not parse reply correctly: 'INBOX.22' vs '%s'\n", c.pa.reply) } if c.pa.size != 11 { t.Fatalf("Did not parse msg size correctly: 11 vs %d\n", c.pa.size) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("RMSG $G foo.bar + reply baz 11\r\nhello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("reply")) { t.Fatalf("Did not parse reply correctly: 'reply' vs '%s'\n", c.pa.reply) } if len(c.pa.queues) != 1 { t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) } if !bytes.Equal(c.pa.queues[0], []byte("baz")) { t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START pub = []byte("RMSG $G foo.bar | baz 11\r\nhello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END_N { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } if !bytes.Equal(c.pa.account, []byte("$G")) { t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("")) { t.Fatalf("Did not parse reply correctly: '' vs '%s'\n", c.pa.reply) } if len(c.pa.queues) != 1 { t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) } if !bytes.Equal(c.pa.queues[0], []byte("baz")) { t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) } } func TestParseMsgSpace(t *testing.T) { c := dummyRouteClient() // Ivan bug he found if err := c.parse([]byte("MSG \r\n")); err == nil { t.Fatalf("Expected parse error for MSG ") } c = dummyClient() // Anything with an M from a client should parse error if err := c.parse([]byte("M")); err == nil { t.Fatalf("Expected parse error for M* from a client") } } func TestShouldFail(t *testing.T) { wrongProtos := []string{ "xxx", "Px", "PIx", "PINx", " PING", "POx", "PONx", "+x", "+Ox", "-x", "-Ex", "-ERx", "-ERRx", "Cx", "COx", "CONx", "CONNx", "CONNEx", "CONNECx", "CONNECx", "CONNECT \r\n", "PUx", "PUB foo\r\n", "PUB \r\n", "PUB foo bar \r\n", "PUB foo 2\r\nok \r\n", "PUB foo 2\r\nok\r \n", "Sx", "SUx", "SUB\r\n", "SUB \r\n", "SUB foo\r\n", "SUB foo bar baz 22\r\n", "Ux", "UNx", "UNSx", "UNSUx", "UNSUBx", "UNSUBUNSUB 1\r\n", "UNSUB_2\r\n", "UNSUB_UNSUB_UNSUB 2\r\n", "UNSUB_\t2\r\n", "UNSUB\r\n", "UNSUB \r\n", "UNSUB \t \r\n", "Ix", "INx", "INFx", "INFO \r\n", } for _, proto := range wrongProtos { c := dummyClient() if err := c.parse([]byte(proto)); err == nil { t.Fatalf("Should have received a parse error for: %v", proto) } } // Special case for MSG, type needs to not be client. wrongProtos = []string{"Mx", "MSx", "MSGx", "MSG \r\n"} for _, proto := range wrongProtos { c := dummyClient() c.kind = ROUTER if err := c.parse([]byte(proto)); err == nil { t.Fatalf("Should have received a parse error for: %v", proto) } } } func TestProtoSnippet(t *testing.T) { sample := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") tests := []struct { input int expected string }{ {0, `"abcdefghijklmnopqrstuvwxyzABCDEF"`}, {1, `"bcdefghijklmnopqrstuvwxyzABCDEFG"`}, {2, `"cdefghijklmnopqrstuvwxyzABCDEFGH"`}, {3, `"defghijklmnopqrstuvwxyzABCDEFGHI"`}, {4, `"efghijklmnopqrstuvwxyzABCDEFGHIJ"`}, {5, `"fghijklmnopqrstuvwxyzABCDEFGHIJK"`}, {6, `"ghijklmnopqrstuvwxyzABCDEFGHIJKL"`}, {7, `"hijklmnopqrstuvwxyzABCDEFGHIJKLM"`}, {8, `"ijklmnopqrstuvwxyzABCDEFGHIJKLMN"`}, {9, `"jklmnopqrstuvwxyzABCDEFGHIJKLMNO"`}, {10, `"klmnopqrstuvwxyzABCDEFGHIJKLMNOP"`}, {11, `"lmnopqrstuvwxyzABCDEFGHIJKLMNOPQ"`}, {12, `"mnopqrstuvwxyzABCDEFGHIJKLMNOPQR"`}, {13, `"nopqrstuvwxyzABCDEFGHIJKLMNOPQRS"`}, {14, `"opqrstuvwxyzABCDEFGHIJKLMNOPQRST"`}, {15, `"pqrstuvwxyzABCDEFGHIJKLMNOPQRSTU"`}, {16, `"qrstuvwxyzABCDEFGHIJKLMNOPQRSTUV"`}, {17, `"rstuvwxyzABCDEFGHIJKLMNOPQRSTUVW"`}, {18, `"stuvwxyzABCDEFGHIJKLMNOPQRSTUVWX"`}, {19, `"tuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, {20, `"uvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"`}, {21, `"vwxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, {22, `"wxyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, {23, `"xyzABCDEFGHIJKLMNOPQRSTUVWXY"`}, {24, `"yzABCDEFGHIJKLMNOPQRSTUVWXY"`}, {25, `"zABCDEFGHIJKLMNOPQRSTUVWXY"`}, {26, `"ABCDEFGHIJKLMNOPQRSTUVWXY"`}, {27, `"BCDEFGHIJKLMNOPQRSTUVWXY"`}, {28, `"CDEFGHIJKLMNOPQRSTUVWXY"`}, {29, `"DEFGHIJKLMNOPQRSTUVWXY"`}, {30, `"EFGHIJKLMNOPQRSTUVWXY"`}, {31, `"FGHIJKLMNOPQRSTUVWXY"`}, {32, `"GHIJKLMNOPQRSTUVWXY"`}, {33, `"HIJKLMNOPQRSTUVWXY"`}, {34, `"IJKLMNOPQRSTUVWXY"`}, {35, `"JKLMNOPQRSTUVWXY"`}, {36, `"KLMNOPQRSTUVWXY"`}, {37, `"LMNOPQRSTUVWXY"`}, {38, `"MNOPQRSTUVWXY"`}, {39, `"NOPQRSTUVWXY"`}, {40, `"OPQRSTUVWXY"`}, {41, `"PQRSTUVWXY"`}, {42, `"QRSTUVWXY"`}, {43, `"RSTUVWXY"`}, {44, `"STUVWXY"`}, {45, `"TUVWXY"`}, {46, `"UVWXY"`}, {47, `"VWXY"`}, {48, `"WXY"`}, {49, `"XY"`}, {50, `"Y"`}, {51, `""`}, {52, `""`}, {53, `""`}, {54, `""`}, } for _, tt := range tests { got := protoSnippet(tt.input, PROTO_SNIPPET_SIZE, sample) if tt.expected != got { t.Errorf("Expected protocol snippet to be %s when start=%d but got %s\n", tt.expected, tt.input, got) } } } func TestParseOK(t *testing.T) { c := dummyClient() if c.state != OP_START { t.Fatalf("Expected OP_START vs %d\n", c.state) } okProto := []byte("+OK\r\n") err := c.parse(okProto[:1]) if err != nil || c.state != OP_PLUS { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(okProto[1:2]) if err != nil || c.state != OP_PLUS_O { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(okProto[2:3]) if err != nil || c.state != OP_PLUS_OK { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(okProto[3:4]) if err != nil || c.state != OP_PLUS_OK { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } err = c.parse(okProto[4:5]) if err != nil || c.state != OP_START { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } } func TestMaxControlLine(t *testing.T) { for _, test := range []struct { name string kind int shouldFail bool }{ {"client", CLIENT, true}, {"leaf", LEAF, false}, {"route", ROUTER, false}, {"gateway", GATEWAY, false}, } { t.Run(test.name, func(t *testing.T) { pub := []byte("PUB foo.bar.baz 2\r\nok\r\n") setupClient := func() *client { c := dummyClient() c.setNoReconnect() c.flags.set(connectReceived) c.kind = test.kind switch test.kind { case ROUTER: c.route = &route{} case GATEWAY: c.gw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie)} } c.mcl = 8 return c } c := setupClient() // First try with a partial: // PUB foo.bar.baz 2\r\nok\r\n // .............^ err := c.parse(pub[:14]) switch test.shouldFail { case true: if !ErrorIs(err, ErrMaxControlLine) { t.Fatalf("Expected an error parsing longer than expected control line") } case false: if err != nil { t.Fatalf("Should not have failed, got %v", err) } } // Now with full protocol (no split) and we should still enforce. c = setupClient() err = c.parse(pub) switch test.shouldFail { case true: if !ErrorIs(err, ErrMaxControlLine) { t.Fatalf("Expected an error parsing longer than expected control line") } case false: if err != nil { t.Fatalf("Should not have failed, got %v", err) } } }) } } nats-server-2.10.27/server/ping_test.go000066400000000000000000000033051477524627100200050ustar00rootroot00000000000000// Copyright 2015-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "fmt" "net" "testing" "time" ) const PING_CLIENT_PORT = 11228 var DefaultPingOptions = Options{ Host: "127.0.0.1", Port: PING_CLIENT_PORT, NoLog: true, NoSigs: true, PingInterval: 50 * time.Millisecond, } func TestPing(t *testing.T) { o := DefaultPingOptions o.DisableShortFirstPing = true s := RunServer(&o) defer s.Shutdown() c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", PING_CLIENT_PORT)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer c.Close() br := bufio.NewReader(c) // Wait for INFO br.ReadLine() // Send CONNECT c.Write([]byte("CONNECT {\"verbose\":false}\r\nPING\r\n")) // Wait for first PONG br.ReadLine() // Wait for PING start := time.Now() for i := 0; i < 3; i++ { l, _, err := br.ReadLine() if err != nil { t.Fatalf("Error: %v", err) } if string(l) != "PING" { t.Fatalf("Expected PING, got %q", l) } if dur := time.Since(start); dur < 25*time.Millisecond || dur > 75*time.Millisecond { t.Fatalf("Pings duration off: %v", dur) } c.Write([]byte(pongProto)) start = time.Now() } } nats-server-2.10.27/server/pse/000077500000000000000000000000001477524627100162505ustar00rootroot00000000000000nats-server-2.10.27/server/pse/freebsd.txt000066400000000000000000000017301477524627100204240ustar00rootroot00000000000000/* * Compile and run this as a C program to get the kinfo_proc offsets * for your architecture. * While FreeBSD works hard at binary-compatibility within an ABI, various * we can't say for sure that these are right for _all_ use on a hardware * platform. The LP64 ifdef affects the offsets considerably. * * We use these offsets in hardware-specific files for FreeBSD, to avoid a cgo * compilation-time dependency, allowing us to cross-compile for FreeBSD from * other hardware platforms, letting us distribute binaries for FreeBSD. */ #include #include #include #include #include #define SHOW_OFFSET(FIELD) printf(" KIP_OFF_%s = %zu\n", #FIELD, offsetof(struct kinfo_proc, ki_ ## FIELD)) int main(int argc, char *argv[]) { /* Uncomment these if you want some extra debugging aids: SHOW_OFFSET(pid); SHOW_OFFSET(ppid); SHOW_OFFSET(uid); */ SHOW_OFFSET(size); SHOW_OFFSET(rssize); SHOW_OFFSET(pctcpu); } nats-server-2.10.27/server/pse/pse_darwin.go000066400000000000000000000052161477524627100207360ustar00rootroot00000000000000// Copyright 2015-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pse // On macs after some studying it seems that typical tools like ps and activity monitor report MaxRss and not // current RSS. I wrote some C code to pull the real RSS and although it does not go down very often, when it does // that is not reflected in the typical tooling one might compare us to, so we can skip cgo and just use rusage imo. // We also do not use virtual memory in the upper layers at all, so ok to skip since rusage does not report vss. import ( "math" "sync" "syscall" "time" ) type lastUsage struct { sync.Mutex last time.Time cpu time.Duration rss int64 pcpu float64 } // To hold the last usage and call time. var lu lastUsage func init() { updateUsage() periodic() } // Get our usage. func getUsage() (now time.Time, cpu time.Duration, rss int64) { var ru syscall.Rusage syscall.Getrusage(syscall.RUSAGE_SELF, &ru) now = time.Now() cpu = time.Duration(ru.Utime.Sec)*time.Second + time.Duration(ru.Utime.Usec)*time.Microsecond cpu += time.Duration(ru.Stime.Sec)*time.Second + time.Duration(ru.Stime.Usec)*time.Microsecond return now, cpu, ru.Maxrss } // Update last usage. // We need to have a prior sample to compute pcpu. func updateUsage() (pcpu float64, rss int64) { lu.Lock() defer lu.Unlock() now, cpu, rss := getUsage() // Don't skew pcpu by sampling too close to last sample. if elapsed := now.Sub(lu.last); elapsed < 500*time.Millisecond { // Always update rss. lu.rss = rss } else { tcpu := float64(cpu - lu.cpu) lu.last, lu.cpu, lu.rss = now, cpu, rss // Want to make this one decimal place and not count on upper layers. // Cores already taken into account via cpu time measurements. lu.pcpu = math.Round(tcpu/float64(elapsed)*1000) / 10 } return lu.pcpu, lu.rss } // Sampling function to keep pcpu relevant. func periodic() { updateUsage() time.AfterFunc(time.Second, periodic) } // ProcUsage returns CPU and memory usage. // Note upper layers do not use virtual memory size, so ok that it is not filled in here. func ProcUsage(pcpu *float64, rss, vss *int64) error { *pcpu, *rss = updateUsage() return nil } nats-server-2.10.27/server/pse/pse_dragonfly.go000066400000000000000000000021341477524627100214330ustar00rootroot00000000000000// Copyright 2015-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // Copied from pse_openbsd.go package pse import ( "fmt" "os" "os/exec" ) // ProcUsage returns CPU usage func ProcUsage(pcpu *float64, rss, vss *int64) error { pidStr := fmt.Sprintf("%d", os.Getpid()) out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() if err != nil { *rss, *vss = -1, -1 return fmt.Errorf("ps call failed:%v", err) } fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) *rss *= 1024 // 1k blocks, want bytes. *vss *= 1024 // 1k blocks, want bytes. return nil } nats-server-2.10.27/server/pse/pse_freebsd.go000066400000000000000000000033111477524627100210560ustar00rootroot00000000000000// Copyright 2015-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !amd64 // +build !amd64 package pse /* #include #include #include #include #include long pagetok(long size) { int pageshift, pagesize; pagesize = getpagesize(); pageshift = 0; while (pagesize > 1) { pageshift++; pagesize >>= 1; } return (size << pageshift); } int getusage(double *pcpu, unsigned int *rss, unsigned int *vss) { int mib[4], ret; size_t len; struct kinfo_proc kp; len = 4; sysctlnametomib("kern.proc.pid", mib, &len); mib[3] = getpid(); len = sizeof(kp); ret = sysctl(mib, 4, &kp, &len, NULL, 0); if (ret != 0) { return (errno); } *rss = pagetok(kp.ki_rssize); *vss = kp.ki_size; *pcpu = (double)kp.ki_pctcpu / FSCALE; return 0; } */ import "C" import ( "syscall" ) // This is a placeholder for now. func ProcUsage(pcpu *float64, rss, vss *int64) error { var r, v C.uint var c C.double if ret := C.getusage(&c, &r, &v); ret != 0 { return syscall.Errno(ret) } *pcpu = float64(c) *rss = int64(r) *vss = int64(v) return nil } nats-server-2.10.27/server/pse/pse_freebsd_amd64.go000066400000000000000000000060311477524627100220530ustar00rootroot00000000000000// Copyright 2015-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // This is the amd64-specific FreeBSD implementation, with hard-coded offset // constants derived by running freebsd.txt; having this implementation allows // us to compile without CGO, which lets us cross-compile for FreeBSD from our // CI system and so supply binaries for FreeBSD amd64. // // To generate for other architectures: // 1. Update pse_freebsd.go, change the build exclusion to exclude your arch // 2. Copy this file to be built for your arch // 3. Update `nativeEndian` below // 4. Link `freebsd.txt` to have a .c filename and compile and run, then // paste the outputs into the const section below. package pse import ( "encoding/binary" "syscall" "golang.org/x/sys/unix" ) // On FreeBSD, to get proc information we read it out of the kernel using a // binary sysctl. The endianness of integers is thus explicitly "host", rather // than little or big endian. var nativeEndian = binary.LittleEndian const ( KIP_OFF_size = 256 KIP_OFF_rssize = 264 KIP_OFF_pctcpu = 308 ) var pageshift int func init() { // To get the physical page size, the C library checks two places: // process ELF auxiliary info, AT_PAGESZ // as a fallback, the hw.pagesize sysctl // In looking closely, I found that the Go runtime support is handling // this for us, and exposing that as syscall.Getpagesize, having checked // both in the same ways, at process start, so a call to that should return // a memory value without even a syscall bounce. pagesize := syscall.Getpagesize() pageshift = 0 for pagesize > 1 { pageshift += 1 pagesize >>= 1 } } func ProcUsage(pcpu *float64, rss, vss *int64) error { rawdata, err := unix.SysctlRaw("kern.proc.pid", unix.Getpid()) if err != nil { return err } r_vss_bytes := nativeEndian.Uint32(rawdata[KIP_OFF_size:]) r_rss_pages := nativeEndian.Uint32(rawdata[KIP_OFF_rssize:]) rss_bytes := r_rss_pages << pageshift // In C: fixpt_t ki_pctcpu // Doc: %cpu for process during ki_swtime // fixpt_t is __uint32_t // usr.bin/top uses pctdouble to convert to a double (float64) // define pctdouble(p) ((double)(p) / FIXED_PCTCPU) // FIXED_PCTCPU is _usually_ FSCALE (some architectures are special) // has: // #define FSHIFT 11 /* bits to right of fixed binary point */ // #define FSCALE (1< 0 { atomic.StoreInt64(&ipcpu, (total*1000/ticks)/seconds) } time.AfterFunc(1*time.Second, periodic) } // ProcUsage returns CPU usage func ProcUsage(pcpu *float64, rss, vss *int64) error { contents, err := os.ReadFile(procStatFile) if err != nil { return err } fields := bytes.Fields(contents) // Memory *rss = (parseInt64(fields[rssPos])) << 12 *vss = parseInt64(fields[vssPos]) // PCPU // We track this with periodic sampling, so just load and go. *pcpu = float64(atomic.LoadInt64(&ipcpu)) / 10.0 return nil } // Ascii numbers 0-9 const ( asciiZero = 48 asciiNine = 57 ) // parseInt64 expects decimal positive numbers. We // return -1 to signal error func parseInt64(d []byte) (n int64) { if len(d) == 0 { return -1 } for _, dec := range d { if dec < asciiZero || dec > asciiNine { return -1 } n = n*10 + (int64(dec) - asciiZero) } return n } nats-server-2.10.27/server/pse/pse_netbsd.go000066400000000000000000000021271477524627100207270ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // Copied from pse_openbsd.go package pse import ( "fmt" "os" "os/exec" ) // ProcUsage returns CPU usage func ProcUsage(pcpu *float64, rss, vss *int64) error { pidStr := fmt.Sprintf("%d", os.Getpid()) out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() if err != nil { *rss, *vss = -1, -1 return fmt.Errorf("ps call failed:%v", err) } fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) *rss *= 1024 // 1k blocks, want bytes. *vss *= 1024 // 1k blocks, want bytes. return nil } nats-server-2.10.27/server/pse/pse_openbsd.go000066400000000000000000000021331477524627100210770ustar00rootroot00000000000000// Copyright 2015-2018 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // Copied from pse_darwin.go package pse import ( "fmt" "os" "os/exec" ) // ProcUsage returns CPU usage func ProcUsage(pcpu *float64, rss, vss *int64) error { pidStr := fmt.Sprintf("%d", os.Getpid()) out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() if err != nil { *rss, *vss = -1, -1 return fmt.Errorf("ps call failed:%v", err) } fmt.Sscanf(string(out), "%f %d %d", pcpu, rss, vss) *rss *= 1024 // 1k blocks, want bytes. *vss *= 1024 // 1k blocks, want bytes. return nil } nats-server-2.10.27/server/pse/pse_rumprun.go000066400000000000000000000014141477524627100211560ustar00rootroot00000000000000// Copyright 2015-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build rumprun // +build rumprun package pse // This is a placeholder for now. func ProcUsage(pcpu *float64, rss, vss *int64) error { *pcpu = 0.0 *rss = 0 *vss = 0 return nil } nats-server-2.10.27/server/pse/pse_solaris.go000066400000000000000000000013461477524627100211260ustar00rootroot00000000000000// Copyright 2015-2018 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pse // This is a placeholder for now. func ProcUsage(pcpu *float64, rss, vss *int64) error { *pcpu = 0.0 *rss = 0 *vss = 0 return nil } nats-server-2.10.27/server/pse/pse_test.go000066400000000000000000000032121477524627100204230ustar00rootroot00000000000000// Copyright 2015-2018 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pse import ( "fmt" "os" "os/exec" "runtime" "testing" ) func TestPSEmulation(t *testing.T) { if runtime.GOOS == "windows" { t.Skipf("Skipping this test on Windows") } var rss, vss, psRss, psVss int64 var pcpu, psPcpu float64 runtime.GC() // PS version first pidStr := fmt.Sprintf("%d", os.Getpid()) out, err := exec.Command("ps", "o", "pcpu=,rss=,vsz=", "-p", pidStr).Output() if err != nil { t.Fatalf("Failed to execute ps command: %v\n", err) } fmt.Sscanf(string(out), "%f %d %d", &psPcpu, &psRss, &psVss) psRss *= 1024 // 1k blocks, want bytes. psVss *= 1024 // 1k blocks, want bytes. runtime.GC() // Our internal version ProcUsage(&pcpu, &rss, &vss) if pcpu != psPcpu { delta := int64(pcpu - psPcpu) if delta < 0 { delta = -delta } if delta > 30 { // 30%? t.Fatalf("CPUs did not match close enough: %f vs %f", pcpu, psPcpu) } } if rss != psRss { delta := rss - psRss if delta < 0 { delta = -delta } if delta > 1024*1024 { // 1MB t.Fatalf("RSSs did not match close enough: %d vs %d", rss, psRss) } } } nats-server-2.10.27/server/pse/pse_wasm.go000066400000000000000000000014011477524627100204110ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build wasm // +build wasm package pse // This is a placeholder for now. func ProcUsage(pcpu *float64, rss, vss *int64) error { *pcpu = 0.0 *rss = 0 *vss = 0 return nil } nats-server-2.10.27/server/pse/pse_windows.go000066400000000000000000000204671477524627100211510ustar00rootroot00000000000000// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows // +build windows package pse import ( "fmt" "os" "path/filepath" "strings" "sync" "syscall" "time" "unsafe" "golang.org/x/sys/windows" ) var ( pdh = windows.NewLazySystemDLL("pdh.dll") winPdhOpenQuery = pdh.NewProc("PdhOpenQuery") winPdhAddCounter = pdh.NewProc("PdhAddCounterW") winPdhCollectQueryData = pdh.NewProc("PdhCollectQueryData") winPdhGetFormattedCounterValue = pdh.NewProc("PdhGetFormattedCounterValue") winPdhGetFormattedCounterArray = pdh.NewProc("PdhGetFormattedCounterArrayW") ) func init() { if err := pdh.Load(); err != nil { panic(err) } for _, p := range []*windows.LazyProc{ winPdhOpenQuery, winPdhAddCounter, winPdhCollectQueryData, winPdhGetFormattedCounterValue, winPdhGetFormattedCounterArray, } { if err := p.Find(); err != nil { panic(err) } } } // global performance counter query handle and counters var ( pcHandle PDH_HQUERY pidCounter, cpuCounter, rssCounter, vssCounter PDH_HCOUNTER prevCPU float64 prevRss int64 prevVss int64 lastSampleTime time.Time processPid int pcQueryLock sync.Mutex initialSample = true ) // maxQuerySize is the number of values to return from a query. // It represents the maximum # of servers that can be queried // simultaneously running on a machine. const maxQuerySize = 512 // Keep static memory around to reuse; this works best for passing // into the pdh API. var counterResults [maxQuerySize]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // PDH Types type ( PDH_HQUERY syscall.Handle PDH_HCOUNTER syscall.Handle ) // PDH constants used here const ( PDH_FMT_DOUBLE = 0x00000200 PDH_INVALID_DATA = 0xC0000BC6 PDH_MORE_DATA = 0x800007D2 ) // PDH_FMT_COUNTERVALUE_DOUBLE - double value type PDH_FMT_COUNTERVALUE_DOUBLE struct { CStatus uint32 DoubleValue float64 } // PDH_FMT_COUNTERVALUE_ITEM_DOUBLE is an array // element of a double value type PDH_FMT_COUNTERVALUE_ITEM_DOUBLE struct { SzName *uint16 // pointer to a string FmtValue PDH_FMT_COUNTERVALUE_DOUBLE } func pdhAddCounter(hQuery PDH_HQUERY, szFullCounterPath string, dwUserData uintptr, phCounter *PDH_HCOUNTER) error { ptxt, _ := syscall.UTF16PtrFromString(szFullCounterPath) r0, _, _ := winPdhAddCounter.Call( uintptr(hQuery), uintptr(unsafe.Pointer(ptxt)), dwUserData, uintptr(unsafe.Pointer(phCounter))) if r0 != 0 { return fmt.Errorf("pdhAddCounter failed. %d", r0) } return nil } func pdhOpenQuery(datasrc *uint16, userdata uint32, query *PDH_HQUERY) error { r0, _, _ := syscall.Syscall(winPdhOpenQuery.Addr(), 3, 0, uintptr(userdata), uintptr(unsafe.Pointer(query))) if r0 != 0 { return fmt.Errorf("pdhOpenQuery failed - %d", r0) } return nil } func pdhCollectQueryData(hQuery PDH_HQUERY) error { r0, _, _ := winPdhCollectQueryData.Call(uintptr(hQuery)) if r0 != 0 { return fmt.Errorf("pdhCollectQueryData failed - %d", r0) } return nil } // pdhGetFormattedCounterArrayDouble returns the value of return code // rather than error, to easily check return codes func pdhGetFormattedCounterArrayDouble(hCounter PDH_HCOUNTER, lpdwBufferSize *uint32, lpdwBufferCount *uint32, itemBuffer *PDH_FMT_COUNTERVALUE_ITEM_DOUBLE) uint32 { ret, _, _ := winPdhGetFormattedCounterArray.Call( uintptr(hCounter), uintptr(PDH_FMT_DOUBLE), uintptr(unsafe.Pointer(lpdwBufferSize)), uintptr(unsafe.Pointer(lpdwBufferCount)), uintptr(unsafe.Pointer(itemBuffer))) return uint32(ret) } func getCounterArrayData(counter PDH_HCOUNTER) ([]float64, error) { var bufSize uint32 var bufCount uint32 // Retrieving array data requires two calls, the first which // requires an addressable empty buffer, and sets size fields. // The second call returns the data. initialBuf := make([]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, 1) ret := pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &initialBuf[0]) if ret == PDH_MORE_DATA { // we'll likely never get here, but be safe. if bufCount > maxQuerySize { bufCount = maxQuerySize } ret = pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &counterResults[0]) if ret == 0 { rv := make([]float64, bufCount) for i := 0; i < int(bufCount); i++ { rv[i] = counterResults[i].FmtValue.DoubleValue } return rv, nil } } if ret != 0 { return nil, fmt.Errorf("getCounterArrayData failed - %d", ret) } return nil, nil } // getProcessImageName returns the name of the process image, as expected by // the performance counter API. func getProcessImageName() (name string) { name = filepath.Base(os.Args[0]) name = strings.TrimSuffix(name, ".exe") return } // initialize our counters func initCounters() (err error) { processPid = os.Getpid() // require an addressible nil pointer var source uint16 if err := pdhOpenQuery(&source, 0, &pcHandle); err != nil { return err } // setup the performance counters, search for all server instances name := fmt.Sprintf("%s*", getProcessImageName()) pidQuery := fmt.Sprintf("\\Process(%s)\\ID Process", name) cpuQuery := fmt.Sprintf("\\Process(%s)\\%% Processor Time", name) rssQuery := fmt.Sprintf("\\Process(%s)\\Working Set - Private", name) vssQuery := fmt.Sprintf("\\Process(%s)\\Virtual Bytes", name) if err = pdhAddCounter(pcHandle, pidQuery, 0, &pidCounter); err != nil { return err } if err = pdhAddCounter(pcHandle, cpuQuery, 0, &cpuCounter); err != nil { return err } if err = pdhAddCounter(pcHandle, rssQuery, 0, &rssCounter); err != nil { return err } if err = pdhAddCounter(pcHandle, vssQuery, 0, &vssCounter); err != nil { return err } // prime the counters by collecting once, and sleep to get somewhat // useful information the first request. Counters for the CPU require // at least two collect calls. if err = pdhCollectQueryData(pcHandle); err != nil { return err } time.Sleep(50) return nil } // ProcUsage returns process CPU and memory statistics func ProcUsage(pcpu *float64, rss, vss *int64) error { var err error // For simplicity, protect the entire call. // Most simultaneous requests will immediately return // with cached values. pcQueryLock.Lock() defer pcQueryLock.Unlock() // First time through, initialize counters. if initialSample { if err = initCounters(); err != nil { return err } initialSample = false } else if time.Since(lastSampleTime) < (2 * time.Second) { // only refresh every two seconds as to minimize impact // on the server. *pcpu = prevCPU *rss = prevRss *vss = prevVss return nil } // always save the sample time, even on errors. defer func() { lastSampleTime = time.Now() }() // refresh the performance counter data if err = pdhCollectQueryData(pcHandle); err != nil { return err } // retrieve the data var pidAry, cpuAry, rssAry, vssAry []float64 if pidAry, err = getCounterArrayData(pidCounter); err != nil { return err } if cpuAry, err = getCounterArrayData(cpuCounter); err != nil { return err } if rssAry, err = getCounterArrayData(rssCounter); err != nil { return err } if vssAry, err = getCounterArrayData(vssCounter); err != nil { return err } // find the index of the entry for this process idx := int(-1) for i := range pidAry { if int(pidAry[i]) == processPid { idx = i break } } // no pid found... if idx < 0 { return fmt.Errorf("could not find pid in performance counter results") } // assign values from the performance counters *pcpu = cpuAry[idx] *rss = int64(rssAry[idx]) *vss = int64(vssAry[idx]) // save off cache values prevCPU = *pcpu prevRss = *rss prevVss = *vss return nil } nats-server-2.10.27/server/pse/pse_windows_test.go000066400000000000000000000046731477524627100222110ustar00rootroot00000000000000// Copyright 2015-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows // +build windows package pse import ( "fmt" "os/exec" "runtime" "strconv" "strings" "testing" ) func checkValues(t *testing.T, pcpu, tPcpu float64, rss, tRss int64) { if pcpu != tPcpu { delta := int64(pcpu - tPcpu) if delta < 0 { delta = -delta } if delta > 30 { // 30%? t.Fatalf("CPUs did not match close enough: %f vs %f", pcpu, tPcpu) } } if rss != tRss { delta := rss - tRss if delta < 0 { delta = -delta } if delta > 1024*1024 { // 1MB t.Fatalf("RSSs did not match close enough: %d vs %d", rss, tRss) } } } func TestPSEmulationWin(t *testing.T) { var pcpu, tPcpu float64 var rss, vss, tRss int64 runtime.GC() if err := ProcUsage(&pcpu, &rss, &vss); err != nil { t.Fatalf("Error: %v", err) } runtime.GC() imageName := getProcessImageName() // query the counters using typeperf out, err := exec.Command("typeperf.exe", fmt.Sprintf("\\Process(%s)\\%% Processor Time", imageName), fmt.Sprintf("\\Process(%s)\\Working Set - Private", imageName), fmt.Sprintf("\\Process(%s)\\Virtual Bytes", imageName), "-sc", "1").Output() if err != nil { t.Fatal("unable to run command", err) } // parse out results - refer to comments in procUsage for detail results := strings.Split(string(out), "\r\n") values := strings.Split(results[2], ",") // parse pcpu tPcpu, err = strconv.ParseFloat(strings.Trim(values[1], "\""), 64) if err != nil { t.Fatalf("Unable to parse percent cpu: %s", values[1]) } // parse private bytes (rss) fval, err := strconv.ParseFloat(strings.Trim(values[2], "\""), 64) if err != nil { t.Fatalf("Unable to parse private bytes: %s", values[2]) } tRss = int64(fval) checkValues(t, pcpu, tPcpu, rss, tRss) runtime.GC() // Again to test caching if err = ProcUsage(&pcpu, &rss, &vss); err != nil { t.Fatalf("Error: %v", err) } checkValues(t, pcpu, tPcpu, rss, tRss) } nats-server-2.10.27/server/pse/pse_zos.go000066400000000000000000000013771477524627100202710ustar00rootroot00000000000000// Copyright 2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build zos // +build zos package pse // This is a placeholder for now. func ProcUsage(pcpu *float64, rss, vss *int64) error { *pcpu = 0.0 *rss = 0 *vss = 0 return nil } nats-server-2.10.27/server/raft.go000066400000000000000000003342131477524627100167520ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/sha256" "encoding/binary" "errors" "fmt" "hash" "math" "math/rand" "net" "os" "path/filepath" "runtime" "strings" "sync" "sync/atomic" "time" "github.com/nats-io/nats-server/v2/internal/fastrand" "github.com/minio/highwayhash" ) type RaftNode interface { Propose(entry []byte) error ProposeMulti(entries []*Entry) error ForwardProposal(entry []byte) error InstallSnapshot(snap []byte) error SendSnapshot(snap []byte) error NeedSnapshot() bool Applied(index uint64) (entries uint64, bytes uint64) State() RaftState Size() (entries, bytes uint64) Progress() (index, commit, applied uint64) Leader() bool Quorum() bool Current() bool Healthy() bool Term() uint64 Leaderless() bool GroupLeader() string HadPreviousLeader() bool StepDown(preferred ...string) error SetObserver(isObserver bool) IsObserver() bool Campaign() error ID() string Group() string Peers() []*Peer ProposeKnownPeers(knownPeers []string) UpdateKnownPeers(knownPeers []string) ProposeAddPeer(peer string) error ProposeRemovePeer(peer string) error AdjustClusterSize(csz int) error AdjustBootClusterSize(csz int) error ClusterSize() int ApplyQ() *ipQueue[*CommittedEntry] PauseApply() error ResumeApply() LeadChangeC() <-chan bool QuitC() <-chan struct{} Created() time.Time Stop() WaitForStop() Delete() IsSystemAccount() bool } type WAL interface { Type() StorageType StoreMsg(subj string, hdr, msg []byte) (uint64, int64, error) LoadMsg(index uint64, sm *StoreMsg) (*StoreMsg, error) RemoveMsg(index uint64) (bool, error) Compact(index uint64) (uint64, error) Purge() (uint64, error) PurgeEx(subject string, seq, keep uint64) (uint64, error) Truncate(seq uint64) error State() StreamState FastState(*StreamState) Stop() error Delete() error } type Peer struct { ID string Current bool Last time.Time Lag uint64 } type RaftState uint8 // Allowable states for a NATS Consensus Group. const ( Follower RaftState = iota Leader Candidate Closed ) func (state RaftState) String() string { switch state { case Follower: return "FOLLOWER" case Candidate: return "CANDIDATE" case Leader: return "LEADER" case Closed: return "CLOSED" } return "UNKNOWN" } type raft struct { sync.RWMutex created time.Time // Time that the group was created accName string // Account name of the asset this raft group is for group string // Raft group sd string // Store directory id string // Node ID wg sync.WaitGroup // Wait for running goroutines to exit on shutdown wal WAL // WAL store (filestore or memstore) wtype StorageType // WAL type, e.g. FileStorage or MemoryStorage track bool // werr error // Last write error state atomic.Int32 // RaftState hh hash.Hash64 // Highwayhash, used for snapshots snapfile string // Snapshot filename csz int // Cluster size qn int // Number of nodes needed to establish quorum peers map[string]*lps // Other peers in the Raft group removed map[string]struct{} // Peers that were removed from the group acks map[uint64]map[string]struct{} // Append entry responses/acks, map of entry index -> peer ID pae map[uint64]*appendEntry // Pending append entries elect *time.Timer // Election timer, normally accessed via electTimer etlr time.Time // Election timer last reset time, for unit tests only active time.Time // Last activity time, i.e. for heartbeats llqrt time.Time // Last quorum lost time lsut time.Time // Last scale-up time term uint64 // The current vote term pterm uint64 // Previous term from the last snapshot pindex uint64 // Previous index from the last snapshot commit uint64 // Index of the most recent commit applied uint64 // Index of the most recently applied commit leader string // The ID of the leader vote string // Our current vote state lxfer bool // Are we doing a leadership transfer? hcbehind bool // Were we falling behind at the last health check? (see: isCurrent) s *Server // Reference to top-level server c *client // Internal client for subscriptions js *jetStream // JetStream, if running, to see if we are out of resources dflag bool // Debug flag hasleader atomic.Bool // Is there a group leader right now? pleader atomic.Bool // Has the group ever had a leader? observer bool // The node is observing, i.e. not participating in voting extSt extensionState // Extension state psubj string // Proposals subject rpsubj string // Remove peers subject vsubj string // Vote requests subject vreply string // Vote responses subject asubj string // Append entries subject areply string // Append entries responses subject sq *sendq // Send queue for outbound RPC messages aesub *subscription // Subscription for handleAppendEntry callbacks wtv []byte // Term and vote to be written wps []byte // Peer state to be written catchup *catchupState // For when we need to catch up as a follower. progress map[string]*ipQueue[uint64] // For leader or server catching up a follower. paused bool // Whether or not applies are paused hcommit uint64 // The commit at the time that applies were paused pobserver bool // Whether we were an observer at the time that applies were paused prop *ipQueue[*proposedEntry] // Proposals entry *ipQueue[*appendEntry] // Append entries resp *ipQueue[*appendEntryResponse] // Append entries responses apply *ipQueue[*CommittedEntry] // Apply queue (committed entries to be passed to upper layer) reqs *ipQueue[*voteRequest] // Vote requests votes *ipQueue[*voteResponse] // Vote responses leadc chan bool // Leader changes quit chan struct{} // Raft group shutdown } type proposedEntry struct { *Entry reply string // Optional, to respond once proposal handled } // cacthupState structure that holds our subscription, and catchup term and index // as well as starting term and index and how many updates we have seen. type catchupState struct { sub *subscription // Subscription that catchup messages will arrive on cterm uint64 // Catchup term cindex uint64 // Catchup index pterm uint64 // Starting term pindex uint64 // Starting index active time.Time // Last time we received a message for this catchup } // lps holds peer state of last time and last index replicated. type lps struct { ts int64 // Last timestamp li uint64 // Last index replicated kp bool // Known peer } const ( minElectionTimeoutDefault = 4 * time.Second maxElectionTimeoutDefault = 9 * time.Second minCampaignTimeoutDefault = 100 * time.Millisecond maxCampaignTimeoutDefault = 8 * minCampaignTimeoutDefault hbIntervalDefault = 1 * time.Second lostQuorumIntervalDefault = hbIntervalDefault * 10 // 10 seconds lostQuorumCheckIntervalDefault = hbIntervalDefault * 10 // 10 seconds observerModeIntervalDefault = 48 * time.Hour ) var ( minElectionTimeout = minElectionTimeoutDefault maxElectionTimeout = maxElectionTimeoutDefault minCampaignTimeout = minCampaignTimeoutDefault maxCampaignTimeout = maxCampaignTimeoutDefault hbInterval = hbIntervalDefault lostQuorumInterval = lostQuorumIntervalDefault lostQuorumCheck = lostQuorumCheckIntervalDefault observerModeInterval = observerModeIntervalDefault ) type RaftConfig struct { Name string Store string Log WAL Track bool Observer bool } var ( errNotLeader = errors.New("raft: not leader") errAlreadyLeader = errors.New("raft: already leader") errNilCfg = errors.New("raft: no config given") errCorruptPeers = errors.New("raft: corrupt peer state") errEntryLoadFailed = errors.New("raft: could not load entry from WAL") errEntryStoreFailed = errors.New("raft: could not store entry to WAL") errNodeClosed = errors.New("raft: node is closed") errBadSnapName = errors.New("raft: snapshot name could not be parsed") errNoSnapAvailable = errors.New("raft: no snapshot available") errCatchupsRunning = errors.New("raft: snapshot can not be installed while catchups running") errSnapshotCorrupt = errors.New("raft: snapshot corrupt") errTooManyPrefs = errors.New("raft: stepdown requires at most one preferred new leader") errNoPeerState = errors.New("raft: no peerstate") errAdjustBootCluster = errors.New("raft: can not adjust boot peer size on established group") errLeaderLen = fmt.Errorf("raft: leader should be exactly %d bytes", idLen) errTooManyEntries = errors.New("raft: append entry can contain a max of 64k entries") errBadAppendEntry = errors.New("raft: append entry corrupt") errNoInternalClient = errors.New("raft: no internal client") ) // This will bootstrap a raftNode by writing its config into the store directory. func (s *Server) bootstrapRaftNode(cfg *RaftConfig, knownPeers []string, allPeersKnown bool) error { if cfg == nil { return errNilCfg } // Check validity of peers if presented. for _, p := range knownPeers { if len(p) != idLen { return fmt.Errorf("raft: illegal peer: %q", p) } } expected := len(knownPeers) // We need to adjust this is all peers are not known. if !allPeersKnown { s.Debugf("Determining expected peer size for JetStream meta group") if expected < 2 { expected = 2 } opts := s.getOpts() nrs := len(opts.Routes) cn := s.ClusterName() ngwps := 0 for _, gw := range opts.Gateway.Gateways { // Ignore our own cluster if specified. if gw.Name == cn { continue } for _, u := range gw.URLs { host := u.Hostname() // If this is an IP just add one. if net.ParseIP(host) != nil { ngwps++ } else { addrs, _ := net.LookupHost(host) ngwps += len(addrs) } } } if expected < nrs+ngwps { expected = nrs + ngwps s.Debugf("Adjusting expected peer set size to %d with %d known", expected, len(knownPeers)) } } // Check the store directory. If we have a memory based WAL we need to make sure the directory is setup. if stat, err := os.Stat(cfg.Store); os.IsNotExist(err) { if err := os.MkdirAll(cfg.Store, defaultDirPerms); err != nil { return fmt.Errorf("raft: could not create storage directory - %v", err) } } else if stat == nil || !stat.IsDir() { return fmt.Errorf("raft: storage directory is not a directory") } tmpfile, err := os.CreateTemp(cfg.Store, "_test_") if err != nil { return fmt.Errorf("raft: storage directory is not writable") } tmpfile.Close() os.Remove(tmpfile.Name()) return writePeerState(cfg.Store, &peerState{knownPeers, expected, extUndetermined}) } // initRaftNode will initialize the raft node, to be used by startRaftNode or when testing to not run the Go routine. func (s *Server) initRaftNode(accName string, cfg *RaftConfig, labels pprofLabels) (*raft, error) { if cfg == nil { return nil, errNilCfg } s.mu.RLock() if s.sys == nil { s.mu.RUnlock() return nil, ErrNoSysAccount } sq := s.sys.sq sacc := s.sys.account hash := s.sys.shash s.mu.RUnlock() // Do this here to process error quicker. ps, err := readPeerState(cfg.Store) if err != nil { return nil, err } if ps == nil { return nil, errNoPeerState } qpfx := fmt.Sprintf("[ACC:%s] RAFT '%s' ", accName, cfg.Name) n := &raft{ created: time.Now(), id: hash[:idLen], group: cfg.Name, sd: cfg.Store, wal: cfg.Log, wtype: cfg.Log.Type(), track: cfg.Track, csz: ps.clusterSize, qn: ps.clusterSize/2 + 1, peers: make(map[string]*lps), acks: make(map[uint64]map[string]struct{}), pae: make(map[uint64]*appendEntry), s: s, c: s.createInternalSystemClient(), js: s.getJetStream(), sq: sq, quit: make(chan struct{}), reqs: newIPQueue[*voteRequest](s, qpfx+"vreq"), votes: newIPQueue[*voteResponse](s, qpfx+"vresp"), prop: newIPQueue[*proposedEntry](s, qpfx+"entry"), entry: newIPQueue[*appendEntry](s, qpfx+"appendEntry"), resp: newIPQueue[*appendEntryResponse](s, qpfx+"appendEntryResponse"), apply: newIPQueue[*CommittedEntry](s, qpfx+"committedEntry"), accName: accName, leadc: make(chan bool, 32), observer: cfg.Observer, extSt: ps.domainExt, } n.c.registerWithAccount(sacc) if atomic.LoadInt32(&s.logging.debug) > 0 { n.dflag = true } // Set up the highwayhash for the snapshots. key := sha256.Sum256([]byte(n.group)) n.hh, _ = highwayhash.New64(key[:]) // If we have a term and vote file (tav.idx on the filesystem) then read in // what we think the term and vote was. It's possible these are out of date // so a catch-up may be required. if term, vote, err := n.readTermVote(); err == nil && term > 0 { n.term = term n.vote = vote } // Can't recover snapshots if memory based since wal will be reset. // We will inherit from the current leader. if _, ok := n.wal.(*memStore); ok { _ = os.RemoveAll(filepath.Join(n.sd, snapshotsDir)) } else { // See if we have any snapshots and if so load and process on startup. n.setupLastSnapshot() } // Make sure that the snapshots directory exists. if err := os.MkdirAll(filepath.Join(n.sd, snapshotsDir), defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create snapshots directory - %v", err) } truncateAndErr := func(index uint64) { if err := n.wal.Truncate(index); err != nil { n.setWriteErr(err) } } // Retrieve the stream state from the WAL. If there are pending append // entries that were committed but not applied before we last shut down, // we will try to replay them and process them here. var state StreamState n.wal.FastState(&state) if state.Msgs > 0 { n.debug("Replaying state of %d entries", state.Msgs) if first, err := n.loadFirstEntry(); err == nil { n.pterm, n.pindex = first.pterm, first.pindex if first.commit > 0 && first.commit > n.commit { n.commit = first.commit } } // This process will queue up entries on our applied queue but prior to the upper // state machine running. So we will monitor how much we have queued and if we // reach a limit will pause the apply queue and resume inside of run() go routine. const maxQsz = 32 * 1024 * 1024 // 32MB max // It looks like there are entries we have committed but not applied // yet. Replay them. for index, qsz := state.FirstSeq, 0; index <= state.LastSeq; index++ { ae, err := n.loadEntry(index) if err != nil { n.warn("Could not load %d from WAL [%+v]: %v", index, state, err) truncateAndErr(index) break } if ae.pindex != index-1 { n.warn("Corrupt WAL, will truncate") truncateAndErr(index) break } n.processAppendEntry(ae, nil) // Check how much we have queued up so far to determine if we should pause. for _, e := range ae.entries { qsz += len(e.Data) if qsz > maxQsz && !n.paused { n.PauseApply() } } } } // Make sure to track ourselves. n.peers[n.id] = &lps{time.Now().UnixNano(), 0, true} // Track known peers for _, peer := range ps.knownPeers { if peer != n.id { // Set these to 0 to start but mark as known peer. n.peers[peer] = &lps{0, 0, true} } } // Setup our internal subscriptions for proposals, votes and append entries. // If we fail to do this for some reason then this is fatal — we cannot // continue setting up or the Raft node may be partially/totally isolated. if err := n.createInternalSubs(); err != nil { n.shutdown() return nil, err } n.debug("Started") // Check if we need to start in observer mode due to lame duck status. // This will stop us from taking on the leader role when we're about to // shutdown anyway. if s.isLameDuckMode() { n.debug("Will start in observer mode due to lame duck status") n.SetObserver(true) } // Set the election timer and lost quorum timers to now, so that we // won't accidentally trigger either state without knowing the real state // of the other nodes. n.Lock() n.resetElectionTimeout() n.llqrt = time.Now() n.Unlock() // Register the Raft group. labels["group"] = n.group s.registerRaftNode(n.group, n) return n, nil } // startRaftNode will start the raft node. func (s *Server) startRaftNode(accName string, cfg *RaftConfig, labels pprofLabels) (RaftNode, error) { n, err := s.initRaftNode(accName, cfg, labels) if err != nil { return nil, err } // Start the run goroutine for the Raft state machine. n.wg.Add(1) s.startGoRoutine(n.run, labels) return n, nil } // Whether we are using the system account or not. // In 2.10.x this is always true as there is no account NRG like in 2.11.x. func (n *raft) IsSystemAccount() bool { return true } // outOfResources checks to see if we are out of resources. func (n *raft) outOfResources() bool { js := n.js if !n.track || js == nil { return false } return js.limitsExceeded(n.wtype) } // Maps node names back to server names. func (s *Server) serverNameForNode(node string) string { if si, ok := s.nodeToInfo.Load(node); ok && si != nil { return si.(nodeInfo).name } return _EMPTY_ } // Maps node names back to cluster names. func (s *Server) clusterNameForNode(node string) string { if si, ok := s.nodeToInfo.Load(node); ok && si != nil { return si.(nodeInfo).cluster } return _EMPTY_ } // Registers the Raft node with the server, as it will track all of the Raft // nodes. func (s *Server) registerRaftNode(group string, n RaftNode) { s.rnMu.Lock() defer s.rnMu.Unlock() if s.raftNodes == nil { s.raftNodes = make(map[string]RaftNode) } s.raftNodes[group] = n } // Unregisters the Raft node from the server, i.e. at shutdown. func (s *Server) unregisterRaftNode(group string) { s.rnMu.Lock() defer s.rnMu.Unlock() if s.raftNodes != nil { delete(s.raftNodes, group) } } // Returns how many Raft nodes are running in this server instance. func (s *Server) numRaftNodes() int { s.rnMu.RLock() defer s.rnMu.RUnlock() return len(s.raftNodes) } // Finds the Raft node for a given Raft group, if any. If there is no Raft node // running for this group then it can return nil. func (s *Server) lookupRaftNode(group string) RaftNode { s.rnMu.RLock() defer s.rnMu.RUnlock() var n RaftNode if s.raftNodes != nil { n = s.raftNodes[group] } return n } // Reloads the debug state for all running Raft nodes. This is necessary when // the configuration has been reloaded and the debug log level has changed. func (s *Server) reloadDebugRaftNodes(debug bool) { if s == nil { return } s.rnMu.RLock() for _, ni := range s.raftNodes { n := ni.(*raft) n.Lock() n.dflag = debug n.Unlock() } s.rnMu.RUnlock() } // Requests that all Raft nodes on this server step down and place them into // observer mode. This is called when the server is shutting down. func (s *Server) stepdownRaftNodes() { if s == nil { return } s.rnMu.RLock() if len(s.raftNodes) == 0 { s.rnMu.RUnlock() return } s.Debugf("Stepping down all leader raft nodes") nodes := make([]RaftNode, 0, len(s.raftNodes)) for _, n := range s.raftNodes { nodes = append(nodes, n) } s.rnMu.RUnlock() for _, node := range nodes { if node.Leader() { node.StepDown() } node.SetObserver(true) } } // Shuts down all Raft nodes on this server. This is called either when the // server is either entering lame duck mode, shutting down or when JetStream // has been disabled. func (s *Server) shutdownRaftNodes() { if s == nil { return } s.rnMu.RLock() if len(s.raftNodes) == 0 { s.rnMu.RUnlock() return } nodes := make([]RaftNode, 0, len(s.raftNodes)) s.Debugf("Shutting down all raft nodes") for _, n := range s.raftNodes { nodes = append(nodes, n) } s.rnMu.RUnlock() for _, node := range nodes { node.Stop() } } // Used in lameduck mode to move off the leaders. // We also put all nodes in observer mode so new leaders // can not be placed on this server. func (s *Server) transferRaftLeaders() bool { if s == nil { return false } s.rnMu.RLock() if len(s.raftNodes) == 0 { s.rnMu.RUnlock() return false } nodes := make([]RaftNode, 0, len(s.raftNodes)) for _, n := range s.raftNodes { nodes = append(nodes, n) } s.rnMu.RUnlock() var didTransfer bool for _, node := range nodes { if node.Leader() { node.StepDown() didTransfer = true } node.SetObserver(true) } return didTransfer } // Formal API // Propose will propose a new entry to the group. // This should only be called on the leader. func (n *raft) Propose(data []byte) error { if state := n.State(); state != Leader { n.debug("Proposal ignored, not leader (state: %v)", state) return errNotLeader } n.Lock() defer n.Unlock() // Error if we had a previous write error. if werr := n.werr; werr != nil { return werr } n.prop.push(newProposedEntry(newEntry(EntryNormal, data), _EMPTY_)) return nil } // ProposeDirect will propose multiple entries at once. // This should only be called on the leader. func (n *raft) ProposeMulti(entries []*Entry) error { if state := n.State(); state != Leader { n.debug("Direct proposal ignored, not leader (state: %v)", state) return errNotLeader } n.Lock() defer n.Unlock() // Error if we had a previous write error. if werr := n.werr; werr != nil { return werr } for _, e := range entries { n.prop.push(newProposedEntry(e, _EMPTY_)) } return nil } // ForwardProposal will forward the proposal to the leader if known. // If we are the leader this is the same as calling propose. func (n *raft) ForwardProposal(entry []byte) error { if n.Leader() { return n.Propose(entry) } // TODO: Currently we do not set a reply subject, even though we are // now capable of responding. Do this once enough time has passed, // i.e. maybe in 2.12. n.sendRPC(n.psubj, _EMPTY_, entry) return nil } // ProposeAddPeer is called to add a peer to the group. func (n *raft) ProposeAddPeer(peer string) error { if n.State() != Leader { return errNotLeader } n.RLock() // Error if we had a previous write error. if werr := n.werr; werr != nil { n.RUnlock() return werr } prop := n.prop n.RUnlock() prop.push(newProposedEntry(newEntry(EntryAddPeer, []byte(peer)), _EMPTY_)) return nil } // As a leader if we are proposing to remove a peer assume its already gone. func (n *raft) doRemovePeerAsLeader(peer string) { n.Lock() if n.removed == nil { n.removed = map[string]struct{}{} } n.removed[peer] = struct{}{} if _, ok := n.peers[peer]; ok { delete(n.peers, peer) // We should decrease our cluster size since we are tracking this peer and the peer is most likely already gone. n.adjustClusterSizeAndQuorum() } n.Unlock() } // ProposeRemovePeer is called to remove a peer from the group. func (n *raft) ProposeRemovePeer(peer string) error { n.RLock() prop, subj := n.prop, n.rpsubj isLeader := n.State() == Leader werr := n.werr n.RUnlock() // Error if we had a previous write error. if werr != nil { return werr } // If we are the leader then we are responsible for processing the // peer remove and then notifying the rest of the group that the // peer was removed. if isLeader { prop.push(newProposedEntry(newEntry(EntryRemovePeer, []byte(peer)), _EMPTY_)) n.doRemovePeerAsLeader(peer) return nil } // Otherwise we need to forward the proposal to the leader. n.sendRPC(subj, _EMPTY_, []byte(peer)) return nil } // ClusterSize reports back the total cluster size. // This effects quorum etc. func (n *raft) ClusterSize() int { n.Lock() defer n.Unlock() return n.csz } // AdjustBootClusterSize can be called to adjust the boot cluster size. // Will error if called on a group with a leader or a previous leader. // This can be helpful in mixed mode. func (n *raft) AdjustBootClusterSize(csz int) error { n.Lock() defer n.Unlock() if n.leader != noLeader || n.pleader.Load() { return errAdjustBootCluster } // Same floor as bootstrap. if csz < 2 { csz = 2 } // Adjust the cluster size and the number of nodes needed to establish // a quorum. n.csz = csz n.qn = n.csz/2 + 1 return nil } // AdjustClusterSize will change the cluster set size. // Must be the leader. func (n *raft) AdjustClusterSize(csz int) error { if n.State() != Leader { return errNotLeader } n.Lock() // Same floor as bootstrap. if csz < 2 { csz = 2 } // Adjust the cluster size and the number of nodes needed to establish // a quorum. n.csz = csz n.qn = n.csz/2 + 1 n.Unlock() n.sendPeerState() return nil } // PauseApply will allow us to pause processing of append entries onto our // external apply queue. In effect this means that the upper layer will no longer // receive any new entries from the Raft group. func (n *raft) PauseApply() error { if n.State() == Leader { return errAlreadyLeader } n.Lock() defer n.Unlock() // If we are currently a candidate make sure we step down. if n.State() == Candidate { n.stepdownLocked(noLeader) } n.debug("Pausing our apply channel") n.paused = true n.hcommit = n.commit // Also prevent us from trying to become a leader while paused and catching up. n.pobserver, n.observer = n.observer, true n.resetElect(observerModeInterval) return nil } // ResumeApply will resume sending applies to the external apply queue. This // means that we will start sending new entries to the upper layer. func (n *raft) ResumeApply() { n.Lock() defer n.Unlock() if !n.paused { return } n.debug("Resuming our apply channel") // Reset before we start. n.resetElectionTimeout() // Run catchup.. if n.hcommit > n.commit { n.debug("Resuming %d replays", n.hcommit+1-n.commit) for index := n.commit + 1; index <= n.hcommit; index++ { if err := n.applyCommit(index); err != nil { n.warn("Got error on apply commit during replay: %v", err) break } // We want to unlock here to allow the upper layers to call Applied() without blocking. n.Unlock() // Give hint to let other Go routines run. // Might not be necessary but seems to make it more fine grained interleaving. runtime.Gosched() // Simply re-acquire n.Lock() // Need to check if we got closed. if n.State() == Closed { return } } } // Clear our observer and paused state after we apply. n.observer, n.pobserver = n.pobserver, false n.paused = false n.hcommit = 0 // If we had been selected to be the next leader campaign here now that we have resumed. if n.lxfer { n.xferCampaign() } else { n.resetElectionTimeout() } } // Applied is a callback that must be be called by the upper layer when it // has successfully applied the committed entries that it received from the // apply queue. It will return the number of entries and an estimation of the // byte size that could be removed with a snapshot/compact. func (n *raft) Applied(index uint64) (entries uint64, bytes uint64) { n.Lock() defer n.Unlock() // Ignore if not applicable. This can happen during a reset. if index > n.commit { return 0, 0 } // Ignore if already applied. if index > n.applied { n.applied = index } // Calculate the number of entries and estimate the byte size that // we can now remove with a compaction/snapshot. var state StreamState n.wal.FastState(&state) if n.applied > state.FirstSeq { entries = n.applied - state.FirstSeq } if state.Msgs > 0 { bytes = entries * state.Bytes / state.Msgs } return entries, bytes } // For capturing data needed by snapshot. type snapshot struct { lastTerm uint64 lastIndex uint64 peerstate []byte data []byte } const minSnapshotLen = 28 // Encodes a snapshot into a buffer for storage. // Lock should be held. func (n *raft) encodeSnapshot(snap *snapshot) []byte { if snap == nil { return nil } var le = binary.LittleEndian buf := make([]byte, minSnapshotLen+len(snap.peerstate)+len(snap.data)) le.PutUint64(buf[0:], snap.lastTerm) le.PutUint64(buf[8:], snap.lastIndex) // Peer state le.PutUint32(buf[16:], uint32(len(snap.peerstate))) wi := 20 copy(buf[wi:], snap.peerstate) wi += len(snap.peerstate) // data itself. copy(buf[wi:], snap.data) wi += len(snap.data) // Now do the hash for the end. n.hh.Reset() n.hh.Write(buf[:wi]) checksum := n.hh.Sum(nil) copy(buf[wi:], checksum) wi += len(checksum) return buf[:wi] } // SendSnapshot will send the latest snapshot as a normal AE. // Should only be used when the upper layers know this is most recent. // Used when restoring streams, moving a stream from R1 to R>1, etc. func (n *raft) SendSnapshot(data []byte) error { n.sendAppendEntry([]*Entry{{EntrySnapshot, data}}) return nil } // Used to install a snapshot for the given term and applied index. This will release // all of the log entries up to and including index. This should not be called with // entries that have been applied to the FSM but have not been applied to the raft state. func (n *raft) InstallSnapshot(data []byte) error { if n.State() == Closed { return errNodeClosed } n.Lock() defer n.Unlock() // If a write error has occurred already then stop here. if werr := n.werr; werr != nil { return werr } // Check that a catchup isn't already taking place. If it is then we won't // allow installing snapshots until it is done. if len(n.progress) > 0 || n.paused { return errCatchupsRunning } if n.applied == 0 { n.debug("Not snapshotting as there are no applied entries") return errNoSnapAvailable } term := n.pterm if ae, _ := n.loadEntry(n.applied); ae != nil { term = ae.term } n.debug("Installing snapshot of %d bytes", len(data)) return n.installSnapshot(&snapshot{ lastTerm: term, lastIndex: n.applied, peerstate: encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt}), data: data, }) } // Install the snapshot. // Lock should be held. func (n *raft) installSnapshot(snap *snapshot) error { snapDir := filepath.Join(n.sd, snapshotsDir) sn := fmt.Sprintf(snapFileT, snap.lastTerm, snap.lastIndex) sfile := filepath.Join(snapDir, sn) if err := writeFileWithSync(sfile, n.encodeSnapshot(snap), defaultFilePerms); err != nil { // We could set write err here, but if this is a temporary situation, too many open files etc. // we want to retry and snapshots are not fatal. return err } // Delete our previous snapshot file if it exists. if n.snapfile != _EMPTY_ && n.snapfile != sfile { os.Remove(n.snapfile) } // Remember our latest snapshot file. n.snapfile = sfile if _, err := n.wal.Compact(snap.lastIndex + 1); err != nil { n.setWriteErrLocked(err) return err } return nil } // NeedSnapshot returns true if it is necessary to try to install a snapshot, i.e. // after we have finished recovering/replaying at startup, on a regular interval or // as a part of cleaning up when shutting down. func (n *raft) NeedSnapshot() bool { n.RLock() defer n.RUnlock() return n.snapfile == _EMPTY_ && n.applied > 1 } const ( snapshotsDir = "snapshots" snapFileT = "snap.%d.%d" ) // termAndIndexFromSnapfile tries to load the snapshot file and returns the term // and index from that snapshot. func termAndIndexFromSnapFile(sn string) (term, index uint64, err error) { if sn == _EMPTY_ { return 0, 0, errBadSnapName } fn := filepath.Base(sn) if n, err := fmt.Sscanf(fn, snapFileT, &term, &index); err != nil || n != 2 { return 0, 0, errBadSnapName } return term, index, nil } // setupLastSnapshot is called at startup to try and recover the last snapshot from // the disk if possible. We will try to recover the term, index and commit/applied // indices and then notify the upper layer what we found. Compacts the WAL if needed. func (n *raft) setupLastSnapshot() { snapDir := filepath.Join(n.sd, snapshotsDir) psnaps, err := os.ReadDir(snapDir) if err != nil { return } var lterm, lindex uint64 var latest string for _, sf := range psnaps { sfile := filepath.Join(snapDir, sf.Name()) var term, index uint64 term, index, err := termAndIndexFromSnapFile(sf.Name()) if err == nil { if term > lterm { lterm, lindex = term, index latest = sfile } else if term == lterm && index > lindex { lindex = index latest = sfile } } else { // Clean this up, can't parse the name. // TODO(dlc) - We could read in and check actual contents. n.debug("Removing snapshot, can't parse name: %q", sf.Name()) os.Remove(sfile) } } // Now cleanup any old entries for _, sf := range psnaps { sfile := filepath.Join(snapDir, sf.Name()) if sfile != latest { n.debug("Removing old snapshot: %q", sfile) os.Remove(sfile) } } if latest == _EMPTY_ { return } // Set latest snapshot we have. n.Lock() defer n.Unlock() n.snapfile = latest snap, err := n.loadLastSnapshot() if err != nil { // We failed to recover the last snapshot for some reason, so we will // assume it has been corrupted and will try to delete it. if n.snapfile != _EMPTY_ { os.Remove(n.snapfile) n.snapfile = _EMPTY_ } return } // We successfully recovered the last snapshot from the disk. // Recover state from the snapshot and then notify the upper layer. // Compact the WAL when we're done if needed. n.pindex = snap.lastIndex n.pterm = snap.lastTerm n.commit = snap.lastIndex n.applied = snap.lastIndex n.apply.push(newCommittedEntry(n.commit, []*Entry{{EntrySnapshot, snap.data}})) if _, err := n.wal.Compact(snap.lastIndex + 1); err != nil { n.setWriteErrLocked(err) } } // loadLastSnapshot will load and return our last snapshot. // Lock should be held. func (n *raft) loadLastSnapshot() (*snapshot, error) { if n.snapfile == _EMPTY_ { return nil, errNoSnapAvailable } <-dios buf, err := os.ReadFile(n.snapfile) dios <- struct{}{} if err != nil { n.warn("Error reading snapshot: %v", err) os.Remove(n.snapfile) n.snapfile = _EMPTY_ return nil, err } if len(buf) < minSnapshotLen { n.warn("Snapshot corrupt, too short") os.Remove(n.snapfile) n.snapfile = _EMPTY_ return nil, errSnapshotCorrupt } // Check to make sure hash is consistent. hoff := len(buf) - 8 lchk := buf[hoff:] n.hh.Reset() n.hh.Write(buf[:hoff]) if !bytes.Equal(lchk[:], n.hh.Sum(nil)) { n.warn("Snapshot corrupt, checksums did not match") os.Remove(n.snapfile) n.snapfile = _EMPTY_ return nil, errSnapshotCorrupt } var le = binary.LittleEndian lps := le.Uint32(buf[16:]) snap := &snapshot{ lastTerm: le.Uint64(buf[0:]), lastIndex: le.Uint64(buf[8:]), peerstate: buf[20 : 20+lps], data: buf[20+lps : hoff], } // We had a bug in 2.9.12 that would allow snapshots on last index of 0. // Detect that here and return err. if snap.lastIndex == 0 { n.warn("Snapshot with last index 0 is invalid, cleaning up") os.Remove(n.snapfile) n.snapfile = _EMPTY_ return nil, errSnapshotCorrupt } return snap, nil } // Leader returns if we are the leader for our group. // We use an atomic here now vs acquiring the read lock. func (n *raft) Leader() bool { if n == nil { return false } return n.State() == Leader } // stepdown immediately steps down the Raft node to the // follower state. This will take the lock itself. func (n *raft) stepdown(newLeader string) { n.Lock() defer n.Unlock() n.stepdownLocked(newLeader) } // stepdownLocked immediately steps down the Raft node to the // follower state. This requires the lock is already held. func (n *raft) stepdownLocked(newLeader string) { n.debug("Stepping down") n.switchToFollowerLocked(newLeader) } // isCatchingUp returns true if a catchup is currently taking place. func (n *raft) isCatchingUp() bool { n.RLock() defer n.RUnlock() return n.catchup != nil } // isCurrent is called from the healthchecks and returns true if we believe // that the upper layer is current with the Raft layer, i.e. that it has applied // all of the commits that we have given it. // Optionally we can also check whether or not we're making forward progress if we // aren't current, in which case this function may block for up to ~10ms to find out. // Lock should be held. func (n *raft) isCurrent(includeForwardProgress bool) bool { // Check if we are closed. if n.State() == Closed { n.debug("Not current, node is closed") return false } // Check whether we've made progress on any state, 0 is invalid so not healthy. if n.commit == 0 { n.debug("Not current, no commits") return false } // If we were previously logging about falling behind, also log when the problem // was cleared. clearBehindState := func() { if n.hcbehind { n.warn("Health check OK, no longer falling behind") n.hcbehind = false } } // Make sure we are the leader or we know we have heard from the leader recently. if n.State() == Leader { clearBehindState() return true } // Check to see that we have heard from the current leader lately. if n.leader != noLeader && n.leader != n.id && n.catchup == nil { okInterval := int64(hbInterval) * 2 ts := time.Now().UnixNano() if ps := n.peers[n.leader]; ps == nil || ps.ts == 0 && (ts-ps.ts) > okInterval { n.debug("Not current, no recent leader contact") return false } } if cs := n.catchup; cs != nil { // We're actively catching up, can't mark current even if commit==applied. n.debug("Not current, still catching up pindex=%d, cindex=%d", n.pindex, cs.cindex) return false } if n.paused && n.hcommit > n.commit { // We're currently paused, waiting to be resumed to apply pending commits. n.debug("Not current, waiting to resume applies commit=%d, hcommit=%d", n.commit, n.hcommit) return false } if n.commit == n.applied { // At this point if we are current, we can return saying so. clearBehindState() return true } else if !includeForwardProgress { // Otherwise, if we aren't allowed to include forward progress // (i.e. we are checking "current" instead of "healthy") then // give up now. return false } // Otherwise, wait for a short period of time and see if we are making any // forward progress. if startDelta := n.commit - n.applied; startDelta > 0 { for i := 0; i < 10; i++ { // 10ms, in 1ms increments n.Unlock() time.Sleep(time.Millisecond) n.Lock() if n.commit-n.applied < startDelta { // The gap is getting smaller, so we're making forward progress. clearBehindState() return true } } } n.hcbehind = true n.warn("Falling behind in health check, commit %d != applied %d", n.commit, n.applied) return false } // Current returns if we are the leader for our group or an up to date follower. func (n *raft) Current() bool { if n == nil { return false } n.Lock() defer n.Unlock() return n.isCurrent(false) } // Healthy returns if we are the leader for our group and nearly up-to-date. func (n *raft) Healthy() bool { if n == nil { return false } n.Lock() defer n.Unlock() return n.isCurrent(true) } // HadPreviousLeader indicates if this group ever had a leader. func (n *raft) HadPreviousLeader() bool { return n.pleader.Load() } // GroupLeader returns the current leader of the group. func (n *raft) GroupLeader() string { if n == nil { return noLeader } n.RLock() defer n.RUnlock() return n.leader } // Leaderless is a lockless way of finding out if the group has a // leader or not. Use instead of GroupLeader in hot paths. func (n *raft) Leaderless() bool { if n == nil { return true } // Negated because we want the default state of hasLeader to be // false until the first setLeader() call. return !n.hasleader.Load() } // Guess the best next leader. Stepdown will check more thoroughly. // Lock should be held. func (n *raft) selectNextLeader() string { nextLeader, hli := noLeader, uint64(0) for peer, ps := range n.peers { if peer == n.id || ps.li <= hli { continue } hli = ps.li nextLeader = peer } return nextLeader } // StepDown will have a leader stepdown and optionally do a leader transfer. func (n *raft) StepDown(preferred ...string) error { n.Lock() if len(preferred) > 1 { n.Unlock() return errTooManyPrefs } if n.State() != Leader { n.Unlock() return errNotLeader } n.debug("Being asked to stepdown") // See if we have up to date followers. maybeLeader := noLeader if len(preferred) > 0 { if preferred[0] != _EMPTY_ { maybeLeader = preferred[0] } else { preferred = nil } } // Can't pick ourselves. if maybeLeader == n.id { maybeLeader = noLeader preferred = nil } nowts := time.Now().UnixNano() // If we have a preferred check it first. if maybeLeader != noLeader { var isHealthy bool if ps, ok := n.peers[maybeLeader]; ok { si, ok := n.s.nodeToInfo.Load(maybeLeader) isHealthy = ok && !si.(nodeInfo).offline && (nowts-ps.ts) < int64(hbInterval*3) } if !isHealthy { maybeLeader = noLeader } } // If we do not have a preferred at this point pick the first healthy one. // Make sure not ourselves. if maybeLeader == noLeader { for peer, ps := range n.peers { if peer == n.id { continue } si, ok := n.s.nodeToInfo.Load(peer) isHealthy := ok && !si.(nodeInfo).offline && (nowts-ps.ts) < int64(hbInterval*3) if isHealthy { maybeLeader = peer break } } } // Clear our vote state. n.vote = noVote n.writeTermVote() n.Unlock() if len(preferred) > 0 && maybeLeader == noLeader { n.debug("Can not transfer to preferred peer %q", preferred[0]) } // If we have a new leader selected, transfer over to them. // Send the append entry directly rather than via the proposals queue, // as we will switch to follower state immediately and will blow away // the contents of the proposal queue in the process. if maybeLeader != noLeader { n.debug("Selected %q for new leader, stepping down due to leadership transfer", maybeLeader) ae := newEntry(EntryLeaderTransfer, []byte(maybeLeader)) n.sendAppendEntry([]*Entry{ae}) } // Force us to stepdown here. n.stepdown(noLeader) return nil } // Campaign will have our node start a leadership vote. func (n *raft) Campaign() error { n.Lock() defer n.Unlock() return n.campaign() } func randCampaignTimeout() time.Duration { delta := rand.Int63n(int64(maxCampaignTimeout - minCampaignTimeout)) return (minCampaignTimeout + time.Duration(delta)) } // Campaign will have our node start a leadership vote. // Lock should be held. func (n *raft) campaign() error { n.debug("Starting campaign") if n.State() == Leader { return errAlreadyLeader } n.resetElect(randCampaignTimeout()) return nil } // xferCampaign will have our node start an immediate leadership vote. // Lock should be held. func (n *raft) xferCampaign() error { n.debug("Starting transfer campaign") if n.State() == Leader { n.lxfer = false return errAlreadyLeader } n.resetElect(10 * time.Millisecond) return nil } // State returns the current state for this node. func (n *raft) State() RaftState { return RaftState(n.state.Load()) } // Progress returns the current index, commit and applied values. func (n *raft) Progress() (index, commit, applied uint64) { n.RLock() defer n.RUnlock() return n.pindex + 1, n.commit, n.applied } // Size returns number of entries and total bytes for our WAL. func (n *raft) Size() (uint64, uint64) { n.RLock() var state StreamState n.wal.FastState(&state) n.RUnlock() return state.Msgs, state.Bytes } func (n *raft) ID() string { if n == nil { return _EMPTY_ } // Lock not needed as n.id is never changed after creation. return n.id } func (n *raft) Group() string { // Lock not needed as n.group is never changed after creation. return n.group } func (n *raft) Peers() []*Peer { n.RLock() defer n.RUnlock() var peers []*Peer for id, ps := range n.peers { var lag uint64 if n.commit > ps.li { lag = n.commit - ps.li } p := &Peer{ ID: id, Current: id == n.leader || ps.li >= n.applied, Last: time.Unix(0, ps.ts), Lag: lag, } peers = append(peers, p) } return peers } // Update and propose our known set of peers. func (n *raft) ProposeKnownPeers(knownPeers []string) { // If we are the leader update and send this update out. if n.State() != Leader { return } n.UpdateKnownPeers(knownPeers) n.sendPeerState() } // Update our known set of peers. func (n *raft) UpdateKnownPeers(knownPeers []string) { n.Lock() // Process like peer state update. ps := &peerState{knownPeers, len(knownPeers), n.extSt} n.processPeerState(ps) n.Unlock() } // ApplyQ returns the apply queue that new commits will be sent to for the // upper layer to apply. func (n *raft) ApplyQ() *ipQueue[*CommittedEntry] { return n.apply } // LeadChangeC returns the leader change channel, notifying when the Raft // leader role has moved. func (n *raft) LeadChangeC() <-chan bool { return n.leadc } // QuitC returns the quit channel, notifying when the Raft group has shut down. func (n *raft) QuitC() <-chan struct{} { return n.quit } func (n *raft) Created() time.Time { // Lock not needed as n.created is never changed after creation. return n.created } func (n *raft) Stop() { n.shutdown() } func (n *raft) WaitForStop() { if n.state.Load() == int32(Closed) { n.wg.Wait() } } func (n *raft) Delete() { n.shutdown() n.wg.Wait() n.Lock() defer n.Unlock() if wal := n.wal; wal != nil { wal.Delete() } os.RemoveAll(n.sd) n.debug("Deleted") } func (n *raft) shutdown() { // First call to Stop or Delete should close the quit chan // to notify the runAs goroutines to stop what they're doing. if n.state.Swap(int32(Closed)) != int32(Closed) { close(n.quit) } } const ( raftAllSubj = "$NRG.>" raftVoteSubj = "$NRG.V.%s" raftAppendSubj = "$NRG.AE.%s" raftPropSubj = "$NRG.P.%s" raftRemovePeerSubj = "$NRG.RP.%s" raftReply = "$NRG.R.%s" raftCatchupReply = "$NRG.CR.%s" ) // Lock should be held (due to use of random generator) func (n *raft) newCatchupInbox() string { var b [replySuffixLen]byte rn := fastrand.Uint64() for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } return fmt.Sprintf(raftCatchupReply, b[:]) } func (n *raft) newInbox() string { var b [replySuffixLen]byte rn := fastrand.Uint64() for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } return fmt.Sprintf(raftReply, b[:]) } // Our internal subscribe. // Lock should be held. func (n *raft) subscribe(subject string, cb msgHandler) (*subscription, error) { if n.c == nil { return nil, errNoInternalClient } return n.s.systemSubscribe(subject, _EMPTY_, false, n.c, cb) } // Lock should be held. func (n *raft) unsubscribe(sub *subscription) { if n.c != nil && sub != nil { n.c.processUnsub(sub.sid) } } func (n *raft) createInternalSubs() error { n.Lock() defer n.Unlock() n.vsubj, n.vreply = fmt.Sprintf(raftVoteSubj, n.group), n.newInbox() n.asubj, n.areply = fmt.Sprintf(raftAppendSubj, n.group), n.newInbox() n.psubj = fmt.Sprintf(raftPropSubj, n.group) n.rpsubj = fmt.Sprintf(raftRemovePeerSubj, n.group) // Votes if _, err := n.subscribe(n.vreply, n.handleVoteResponse); err != nil { return err } if _, err := n.subscribe(n.vsubj, n.handleVoteRequest); err != nil { return err } // AppendEntry if _, err := n.subscribe(n.areply, n.handleAppendEntryResponse); err != nil { return err } if sub, err := n.subscribe(n.asubj, n.handleAppendEntry); err != nil { return err } else { n.aesub = sub } return nil } func randElectionTimeout() time.Duration { delta := rand.Int63n(int64(maxElectionTimeout - minElectionTimeout)) return (minElectionTimeout + time.Duration(delta)) } // Lock should be held. func (n *raft) resetElectionTimeout() { n.resetElect(randElectionTimeout()) } func (n *raft) resetElectionTimeoutWithLock() { n.resetElectWithLock(randElectionTimeout()) } // Lock should be held. func (n *raft) resetElect(et time.Duration) { n.etlr = time.Now() if n.elect == nil { n.elect = time.NewTimer(et) } else { if !n.elect.Stop() { select { case <-n.elect.C: default: } } n.elect.Reset(et) } } func (n *raft) resetElectWithLock(et time.Duration) { n.Lock() n.resetElect(et) n.Unlock() } // run is the top-level runner for the Raft state machine. Depending on the // state of the node (leader, follower, candidate, observer), this will call // through to other functions. It is expected that this function will run for // the entire life of the Raft node once started. func (n *raft) run() { s := n.s defer s.grWG.Done() defer n.wg.Done() // We want to wait for some routing to be enabled, so we will wait for // at least a route, leaf or gateway connection to be established before // starting the run loop. for gw := s.gateway; ; { s.mu.RLock() ready, gwEnabled := s.numRemotes()+len(s.leafs) > 0, gw.enabled s.mu.RUnlock() if !ready && gwEnabled { gw.RLock() ready = len(gw.out)+len(gw.in) > 0 gw.RUnlock() } if !ready { select { case <-s.quitCh: return case <-time.After(100 * time.Millisecond): s.RateLimitWarnf("Waiting for routing to be established...") } } else { break } } // We may have paused adding entries to apply queue, resume here. // No-op if not paused. n.ResumeApply() // Send nil entry to signal the upper layers we are done doing replay/restore. n.apply.push(nil) runner: for s.isRunning() { switch n.State() { case Follower: n.runAsFollower() case Candidate: n.runAsCandidate() case Leader: n.runAsLeader() case Closed: break runner } } // If we've reached this point then we're shutting down, either because // the server is stopping or because the Raft group is closing/closed. n.Lock() defer n.Unlock() if c := n.c; c != nil { var subs []*subscription c.mu.Lock() for _, sub := range c.subs { subs = append(subs, sub) } c.mu.Unlock() for _, sub := range subs { n.unsubscribe(sub) } c.closeConnection(InternalClient) n.c = nil } // Unregistering ipQueues do not prevent them from push/pop // just will remove them from the central monitoring map queues := []interface { unregister() drain() int }{n.reqs, n.votes, n.prop, n.entry, n.resp, n.apply} for _, q := range queues { q.drain() q.unregister() } n.s.unregisterRaftNode(n.group) if wal := n.wal; wal != nil { wal.Stop() } n.debug("Shutdown") } func (n *raft) debug(format string, args ...any) { if n.dflag { nf := fmt.Sprintf("RAFT [%s - %s] %s", n.id, n.group, format) n.s.Debugf(nf, args...) } } func (n *raft) warn(format string, args ...any) { nf := fmt.Sprintf("RAFT [%s - %s] %s", n.id, n.group, format) n.s.RateLimitWarnf(nf, args...) } func (n *raft) error(format string, args ...any) { nf := fmt.Sprintf("RAFT [%s - %s] %s", n.id, n.group, format) n.s.Errorf(nf, args...) } func (n *raft) electTimer() *time.Timer { n.RLock() defer n.RUnlock() return n.elect } func (n *raft) IsObserver() bool { n.RLock() defer n.RUnlock() return n.observer } // Sets the state to observer only. func (n *raft) SetObserver(isObserver bool) { n.setObserver(isObserver, extUndetermined) } func (n *raft) setObserver(isObserver bool, extSt extensionState) { n.Lock() defer n.Unlock() if n.paused { // Applies are paused so we're already in observer state. // Resuming the applies will set the state back to whatever // is in "pobserver", so update that instead. n.pobserver = isObserver return } wasObserver := n.observer n.observer = isObserver n.extSt = extSt // If we're leaving observer state then reset the election timer or // we might end up waiting for up to the observerModeInterval. if wasObserver && !isObserver { n.resetElect(randCampaignTimeout()) } } // processAppendEntries is called by the Raft state machine when there are // new append entries to be committed and sent to the upper state machine. func (n *raft) processAppendEntries() { canProcess := true if n.isClosed() { n.debug("AppendEntry not processing inbound, closed") canProcess = false } if n.outOfResources() { n.debug("AppendEntry not processing inbound, no resources") canProcess = false } // Always pop the entries, but check if we can process them. If we can't // then the entries are effectively dropped. aes := n.entry.pop() if canProcess { for _, ae := range aes { n.processAppendEntry(ae, ae.sub) } } n.entry.recycle(&aes) } // runAsFollower is called by run and will block for as long as the node is // running in the follower state. func (n *raft) runAsFollower() { for n.State() == Follower { elect := n.electTimer() select { case <-n.entry.ch: // New append entries have arrived over the network. n.processAppendEntries() case <-n.s.quitCh: // The server is shutting down. return case <-n.quit: // The Raft node is shutting down. return case <-elect.C: // The election timer has fired so we think it's time to call an election. // If we are out of resources we just want to stay in this state for the moment. if n.outOfResources() { n.resetElectionTimeoutWithLock() n.debug("Not switching to candidate, no resources") } else if n.IsObserver() { n.resetElectWithLock(observerModeInterval) n.debug("Not switching to candidate, observer only") } else if n.isCatchingUp() { n.debug("Not switching to candidate, catching up") // Check to see if our catchup has stalled. n.Lock() if n.catchupStalled() { n.cancelCatchup() } n.resetElectionTimeout() n.Unlock() } else { n.switchToCandidate() return } case <-n.votes.ch: // We're receiving votes from the network, probably because we have only // just stepped down and they were already in flight. Ignore them. n.debug("Ignoring old vote response, we have stepped down") n.votes.popOne() case <-n.resp.ch: // Ignore append entry responses received from before the state change. n.resp.drain() case <-n.prop.ch: // Ignore proposals received from before the state change. n.prop.drain() case <-n.reqs.ch: // We've just received a vote request from the network. // Because of drain() it is possible that we get nil from popOne(). if voteReq, ok := n.reqs.popOne(); ok { n.processVoteRequest(voteReq) } } } } // Pool for CommittedEntry re-use. var cePool = sync.Pool{ New: func() any { return &CommittedEntry{} }, } // CommittedEntry is handed back to the user to apply a commit to their upper layer. type CommittedEntry struct { Index uint64 Entries []*Entry } // Create a new CommittedEntry. When the returned entry is no longer needed, it // should be returned to the pool by calling ReturnToPool. func newCommittedEntry(index uint64, entries []*Entry) *CommittedEntry { ce := cePool.Get().(*CommittedEntry) ce.Index, ce.Entries = index, entries return ce } // ReturnToPool returns the CommittedEntry to the pool, after which point it is // no longer safe to reuse. func (ce *CommittedEntry) ReturnToPool() { if ce == nil { return } if len(ce.Entries) > 0 { for _, e := range ce.Entries { entryPool.Put(e) } } ce.Index, ce.Entries = 0, nil cePool.Put(ce) } // Pool for Entry re-use. var entryPool = sync.Pool{ New: func() any { return &Entry{} }, } // Helper to create new entries. When the returned entry is no longer needed, it // should be returned to the entryPool pool. func newEntry(t EntryType, data []byte) *Entry { entry := entryPool.Get().(*Entry) entry.Type, entry.Data = t, data return entry } // Pool for appendEntry re-use. var aePool = sync.Pool{ New: func() any { return &appendEntry{} }, } // appendEntry is the main struct that is used to sync raft peers. type appendEntry struct { leader string // The leader that this append entry came from. term uint64 // The current term, as the leader understands it. commit uint64 // The commit index, as the leader understands it. pterm uint64 // The previous term, for checking consistency. pindex uint64 // The previous commit index, for checking consistency. entries []*Entry // Entries to process. // Below fields are for internal use only: reply string // Reply subject to respond to once committed. sub *subscription // The subscription that the append entry came in on. buf []byte } // Create a new appendEntry. func newAppendEntry(leader string, term, commit, pterm, pindex uint64, entries []*Entry) *appendEntry { ae := aePool.Get().(*appendEntry) ae.leader, ae.term, ae.commit, ae.pterm, ae.pindex, ae.entries = leader, term, commit, pterm, pindex, entries ae.reply, ae.sub, ae.buf = _EMPTY_, nil, nil return ae } // Will return this append entry, and its interior entries to their respective pools. func (ae *appendEntry) returnToPool() { ae.entries, ae.buf, ae.sub, ae.reply = nil, nil, nil, _EMPTY_ aePool.Put(ae) } // Pool for proposedEntry re-use. var pePool = sync.Pool{ New: func() any { return &proposedEntry{} }, } // Create a new proposedEntry. func newProposedEntry(entry *Entry, reply string) *proposedEntry { pe := pePool.Get().(*proposedEntry) pe.Entry, pe.reply = entry, reply return pe } // Will return this proosed entry. func (pe *proposedEntry) returnToPool() { pe.Entry, pe.reply = nil, _EMPTY_ pePool.Put(pe) } type EntryType uint8 const ( EntryNormal EntryType = iota EntryOldSnapshot EntryPeerState EntryAddPeer EntryRemovePeer EntryLeaderTransfer EntrySnapshot ) func (t EntryType) String() string { switch t { case EntryNormal: return "Normal" case EntryOldSnapshot: return "OldSnapshot" case EntryPeerState: return "PeerState" case EntryAddPeer: return "AddPeer" case EntryRemovePeer: return "RemovePeer" case EntryLeaderTransfer: return "LeaderTransfer" case EntrySnapshot: return "Snapshot" } return fmt.Sprintf("Unknown [%d]", uint8(t)) } type Entry struct { Type EntryType Data []byte } func (ae *appendEntry) String() string { return fmt.Sprintf("&{leader:%s term:%d commit:%d pterm:%d pindex:%d entries: %d}", ae.leader, ae.term, ae.commit, ae.pterm, ae.pindex, len(ae.entries)) } const appendEntryBaseLen = idLen + 4*8 + 2 func (ae *appendEntry) encode(b []byte) ([]byte, error) { if ll := len(ae.leader); ll != idLen && ll != 0 { return nil, errLeaderLen } if len(ae.entries) > math.MaxUint16 { return nil, errTooManyEntries } var elen int for _, e := range ae.entries { elen += len(e.Data) + 1 + 4 // 1 is type, 4 is for size. } tlen := appendEntryBaseLen + elen + 1 var buf []byte if cap(b) >= tlen { buf = b[:tlen] } else { buf = make([]byte, tlen) } var le = binary.LittleEndian copy(buf[:idLen], ae.leader) le.PutUint64(buf[8:], ae.term) le.PutUint64(buf[16:], ae.commit) le.PutUint64(buf[24:], ae.pterm) le.PutUint64(buf[32:], ae.pindex) le.PutUint16(buf[40:], uint16(len(ae.entries))) wi := 42 for _, e := range ae.entries { le.PutUint32(buf[wi:], uint32(len(e.Data)+1)) wi += 4 buf[wi] = byte(e.Type) wi++ copy(buf[wi:], e.Data) wi += len(e.Data) } return buf[:wi], nil } // This can not be used post the wire level callback since we do not copy. func (n *raft) decodeAppendEntry(msg []byte, sub *subscription, reply string) (*appendEntry, error) { if len(msg) < appendEntryBaseLen { return nil, errBadAppendEntry } var le = binary.LittleEndian ae := newAppendEntry(string(msg[:idLen]), le.Uint64(msg[8:]), le.Uint64(msg[16:]), le.Uint64(msg[24:]), le.Uint64(msg[32:]), nil) ae.reply, ae.sub = reply, sub // Decode Entries. ne, ri := int(le.Uint16(msg[40:])), 42 for i, max := 0, len(msg); i < ne; i++ { if ri >= max-1 { return nil, errBadAppendEntry } le := int(le.Uint32(msg[ri:])) ri += 4 if le <= 0 || ri+le > max { return nil, errBadAppendEntry } entry := newEntry(EntryType(msg[ri]), msg[ri+1:ri+le]) ae.entries = append(ae.entries, entry) ri += le } ae.buf = msg return ae, nil } // Pool for appendEntryResponse re-use. var arPool = sync.Pool{ New: func() any { return &appendEntryResponse{} }, } // We want to make sure this does not change from system changing length of syshash. const idLen = 8 const appendEntryResponseLen = 24 + 1 // appendEntryResponse is our response to a received appendEntry. type appendEntryResponse struct { term uint64 index uint64 peer string reply string // internal usage. success bool } // Create a new appendEntryResponse. func newAppendEntryResponse(term, index uint64, peer string, success bool) *appendEntryResponse { ar := arPool.Get().(*appendEntryResponse) ar.term, ar.index, ar.peer, ar.success = term, index, peer, success // Always empty out. ar.reply = _EMPTY_ return ar } func (ar *appendEntryResponse) encode(b []byte) []byte { var buf []byte if cap(b) >= appendEntryResponseLen { buf = b[:appendEntryResponseLen] } else { buf = make([]byte, appendEntryResponseLen) } var le = binary.LittleEndian le.PutUint64(buf[0:], ar.term) le.PutUint64(buf[8:], ar.index) copy(buf[16:16+idLen], ar.peer) if ar.success { buf[24] = 1 } else { buf[24] = 0 } return buf[:appendEntryResponseLen] } // Track all peers we may have ever seen to use an string interns for appendEntryResponse decoding. var peers sync.Map func (n *raft) decodeAppendEntryResponse(msg []byte) *appendEntryResponse { if len(msg) != appendEntryResponseLen { return nil } var le = binary.LittleEndian ar := arPool.Get().(*appendEntryResponse) ar.term = le.Uint64(msg[0:]) ar.index = le.Uint64(msg[8:]) peer, ok := peers.Load(string(msg[16 : 16+idLen])) if !ok { // We missed so store inline here. peer = string(msg[16 : 16+idLen]) peers.Store(peer, peer) } ar.peer = peer.(string) ar.success = msg[24] == 1 return ar } // Called when a remove peer proposal has been forwarded func (n *raft) handleForwardedRemovePeerProposal(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) { n.debug("Received forwarded remove peer proposal: %q", msg) if !n.Leader() { n.debug("Ignoring forwarded peer removal proposal, not leader") return } if len(msg) != idLen { n.warn("Received invalid peer name for remove proposal: %q", msg) return } n.RLock() prop, werr := n.prop, n.werr n.RUnlock() // Ignore if we have had a write error previous. if werr != nil { return } // Need to copy since this is underlying client/route buffer. peer := copyBytes(msg) prop.push(newProposedEntry(newEntry(EntryRemovePeer, peer), reply)) } // Called when a peer has forwarded a proposal. func (n *raft) handleForwardedProposal(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) { if !n.Leader() { n.debug("Ignoring forwarded proposal, not leader") return } // Need to copy since this is underlying client/route buffer. msg = copyBytes(msg) n.RLock() prop, werr := n.prop, n.werr n.RUnlock() // Ignore if we have had a write error previous. if werr != nil { return } prop.push(newProposedEntry(newEntry(EntryNormal, msg), reply)) } func (n *raft) runAsLeader() { if n.State() == Closed { return } n.Lock() psubj, rpsubj := n.psubj, n.rpsubj // For forwarded proposals, both normal and remove peer proposals. fsub, err := n.subscribe(psubj, n.handleForwardedProposal) if err != nil { n.warn("Error subscribing to forwarded proposals: %v", err) n.stepdownLocked(noLeader) n.Unlock() return } rpsub, err := n.subscribe(rpsubj, n.handleForwardedRemovePeerProposal) if err != nil { n.warn("Error subscribing to forwarded remove peer proposals: %v", err) n.unsubscribe(fsub) n.stepdownLocked(noLeader) n.Unlock() return } n.Unlock() // Cleanup our subscription when we leave. defer func() { n.Lock() n.unsubscribe(fsub) n.unsubscribe(rpsub) n.Unlock() }() // To send out our initial peer state. n.sendPeerState() hb := time.NewTicker(hbInterval) defer hb.Stop() lq := time.NewTicker(lostQuorumCheck) defer lq.Stop() for n.State() == Leader { select { case <-n.s.quitCh: return case <-n.quit: return case <-n.resp.ch: ars := n.resp.pop() for _, ar := range ars { n.processAppendEntryResponse(ar) } n.resp.recycle(&ars) case <-n.prop.ch: const maxBatch = 256 * 1024 const maxEntries = 512 var entries []*Entry es, sz := n.prop.pop(), 0 for _, b := range es { if b.Type == EntryRemovePeer { n.doRemovePeerAsLeader(string(b.Data)) } entries = append(entries, b.Entry) // Increment size. sz += len(b.Data) + 1 // If below thresholds go ahead and send. if sz < maxBatch && len(entries) < maxEntries { continue } n.sendAppendEntry(entries) // Reset our sz and entries. // We need to re-create `entries` because there is a reference // to it in the node's pae map. sz, entries = 0, nil } if len(entries) > 0 { n.sendAppendEntry(entries) } // Respond to any proposals waiting for a confirmation. for _, pe := range es { if pe.reply != _EMPTY_ { n.sendReply(pe.reply, nil) } pe.returnToPool() } n.prop.recycle(&es) case <-hb.C: if n.notActive() { n.sendHeartbeat() } case <-lq.C: if n.lostQuorum() { n.stepdown(noLeader) return } case <-n.votes.ch: // Because of drain() it is possible that we get nil from popOne(). vresp, ok := n.votes.popOne() if !ok { continue } if vresp.term > n.Term() { n.stepdown(noLeader) return } n.trackPeer(vresp.peer) case <-n.reqs.ch: // Because of drain() it is possible that we get nil from popOne(). if voteReq, ok := n.reqs.popOne(); ok { n.processVoteRequest(voteReq) } case <-n.entry.ch: n.processAppendEntries() } } } // Quorum reports the quorum status. Will be called on former leaders. func (n *raft) Quorum() bool { n.RLock() defer n.RUnlock() now, nc := time.Now().UnixNano(), 0 for id, peer := range n.peers { if id == n.id || time.Duration(now-peer.ts) < lostQuorumInterval { nc++ if nc >= n.qn { return true } } } return false } func (n *raft) lostQuorum() bool { n.RLock() defer n.RUnlock() return n.lostQuorumLocked() } func (n *raft) lostQuorumLocked() bool { // In order to avoid false positives that can happen in heavily loaded systems // make sure nothing is queued up that we have not processed yet. // Also make sure we let any scale up actions settle before deciding. if n.resp.len() != 0 || (!n.lsut.IsZero() && time.Since(n.lsut) < lostQuorumInterval) { return false } now, nc := time.Now().UnixNano(), 0 for id, peer := range n.peers { if id == n.id || time.Duration(now-peer.ts) < lostQuorumInterval { nc++ if nc >= n.qn { return false } } } return true } // Check for being not active in terms of sending entries. // Used in determining if we need to send a heartbeat. func (n *raft) notActive() bool { n.RLock() defer n.RUnlock() return time.Since(n.active) > hbInterval } // Return our current term. func (n *raft) Term() uint64 { n.RLock() defer n.RUnlock() return n.term } // Lock should be held. func (n *raft) loadFirstEntry() (ae *appendEntry, err error) { var state StreamState n.wal.FastState(&state) return n.loadEntry(state.FirstSeq) } func (n *raft) runCatchup(ar *appendEntryResponse, indexUpdatesQ *ipQueue[uint64]) { n.RLock() s, reply := n.s, n.areply peer, subj, last := ar.peer, ar.reply, n.pindex n.RUnlock() defer s.grWG.Done() defer arPool.Put(ar) defer func() { n.Lock() delete(n.progress, peer) if len(n.progress) == 0 { n.progress = nil } // Check if this is a new peer and if so go ahead and propose adding them. _, exists := n.peers[peer] n.Unlock() if !exists { n.debug("Catchup done for %q, will add into peers", peer) n.ProposeAddPeer(peer) } indexUpdatesQ.unregister() }() n.debug("Running catchup for %q", peer) const maxOutstanding = 2 * 1024 * 1024 // 2MB for now. next, total, om := uint64(0), 0, make(map[uint64]int) sendNext := func() bool { for total <= maxOutstanding { next++ if next > last { return true } ae, err := n.loadEntry(next) if err != nil { if err != ErrStoreEOF { n.warn("Got an error loading %d index: %v", next, err) } return true } // Update our tracking total. om[next] = len(ae.buf) total += len(ae.buf) n.sendRPC(subj, reply, ae.buf) } return false } const activityInterval = 2 * time.Second timeout := time.NewTimer(activityInterval) defer timeout.Stop() stepCheck := time.NewTicker(100 * time.Millisecond) defer stepCheck.Stop() // Run as long as we are leader and still not caught up. for n.Leader() { select { case <-n.s.quitCh: return case <-n.quit: return case <-stepCheck.C: if !n.Leader() { n.debug("Catching up canceled, no longer leader") return } case <-timeout.C: n.debug("Catching up for %q stalled", peer) return case <-indexUpdatesQ.ch: if index, ok := indexUpdatesQ.popOne(); ok { // Update our activity timer. timeout.Reset(activityInterval) // Update outstanding total. total -= om[index] delete(om, index) if next == 0 { next = index } // Check if we are done. if index > last || sendNext() { n.debug("Finished catching up") return } } } } } // Lock should be held. func (n *raft) sendSnapshotToFollower(subject string) (uint64, error) { snap, err := n.loadLastSnapshot() if err != nil { // We need to stepdown here when this happens. n.stepdownLocked(noLeader) // We need to reset our state here as well. n.resetWAL() return 0, err } // Go ahead and send the snapshot and peerstate here as first append entry to the catchup follower. ae := n.buildAppendEntry([]*Entry{{EntrySnapshot, snap.data}, {EntryPeerState, snap.peerstate}}) ae.pterm, ae.pindex = snap.lastTerm, snap.lastIndex var state StreamState n.wal.FastState(&state) fpIndex := state.FirstSeq - 1 if snap.lastIndex < fpIndex && state.FirstSeq != 0 { snap.lastIndex = fpIndex ae.pindex = fpIndex } encoding, err := ae.encode(nil) if err != nil { return 0, err } n.sendRPC(subject, n.areply, encoding) return snap.lastIndex, nil } func (n *raft) catchupFollower(ar *appendEntryResponse) { n.debug("Being asked to catch up follower: %q", ar.peer) n.Lock() if n.progress == nil { n.progress = make(map[string]*ipQueue[uint64]) } else if q, ok := n.progress[ar.peer]; ok { n.debug("Will cancel existing entry for catching up %q", ar.peer) delete(n.progress, ar.peer) q.push(n.pindex) } // Check to make sure we have this entry. start := ar.index + 1 var state StreamState n.wal.FastState(&state) if start < state.FirstSeq || (state.Msgs == 0 && start <= state.LastSeq) { n.debug("Need to send snapshot to follower") if lastIndex, err := n.sendSnapshotToFollower(ar.reply); err != nil { n.error("Error sending snapshot to follower [%s]: %v", ar.peer, err) n.Unlock() arPool.Put(ar) return } else { start = lastIndex + 1 // If no other entries, we can just return here. if state.Msgs == 0 || start > state.LastSeq { n.debug("Finished catching up") n.Unlock() arPool.Put(ar) return } n.debug("Snapshot sent, reset first catchup entry to %d", lastIndex) } } ae, err := n.loadEntry(start) if err != nil { n.warn("Request from follower for entry at index [%d] errored for state %+v - %v", start, state, err) if err == ErrStoreEOF { // If we are here we are seeing a request for an item beyond our state, meaning we should stepdown. n.stepdownLocked(noLeader) n.Unlock() arPool.Put(ar) return } ae, err = n.loadFirstEntry() } if err != nil || ae == nil { n.warn("Could not find a starting entry for catchup request: %v", err) // If we are here we are seeing a request for an item we do not have, meaning we should stepdown. // This is possible on a reset of our WAL but the other side has a snapshot already. // If we do not stepdown this can cycle. n.stepdownLocked(noLeader) n.Unlock() arPool.Put(ar) return } if ae.pindex != ar.index || ae.pterm != ar.term { n.debug("Our first entry [%d:%d] does not match request from follower [%d:%d]", ae.pterm, ae.pindex, ar.term, ar.index) } // Create a queue for delivering updates from responses. indexUpdates := newIPQueue[uint64](n.s, fmt.Sprintf("[ACC:%s] RAFT '%s' indexUpdates", n.accName, n.group)) indexUpdates.push(ae.pindex) n.progress[ar.peer] = indexUpdates n.Unlock() n.wg.Add(1) n.s.startGoRoutine(func() { defer n.wg.Done() n.runCatchup(ar, indexUpdates) }) } func (n *raft) loadEntry(index uint64) (*appendEntry, error) { var smp StoreMsg sm, err := n.wal.LoadMsg(index, &smp) if err != nil { return nil, err } return n.decodeAppendEntry(sm.msg, nil, _EMPTY_) } // applyCommit will update our commit index and apply the entry to the apply queue. // lock should be held. func (n *raft) applyCommit(index uint64) error { if n.State() == Closed { return errNodeClosed } if index <= n.commit { n.debug("Ignoring apply commit for %d, already processed", index) return nil } if n.State() == Leader { delete(n.acks, index) } ae := n.pae[index] if ae == nil { var state StreamState n.wal.FastState(&state) if index < state.FirstSeq { return nil } var err error if ae, err = n.loadEntry(index); err != nil { if err != ErrStoreClosed && err != ErrStoreEOF { n.warn("Got an error loading %d index: %v - will reset", index, err) if n.State() == Leader { n.stepdownLocked(n.selectNextLeader()) } // Reset and cancel any catchup. n.resetWAL() n.cancelCatchup() } return errEntryLoadFailed } } else { defer delete(n.pae, index) } n.commit = index ae.buf = nil var committed []*Entry for _, e := range ae.entries { switch e.Type { case EntryNormal: committed = append(committed, e) case EntryOldSnapshot: // For old snapshots in our WAL. committed = append(committed, newEntry(EntrySnapshot, e.Data)) case EntrySnapshot: committed = append(committed, e) case EntryPeerState: if n.State() != Leader { if ps, err := decodePeerState(e.Data); err == nil { n.processPeerState(ps) } } case EntryAddPeer: newPeer := string(e.Data) n.debug("Added peer %q", newPeer) // Store our peer in our global peer map for all peers. peers.LoadOrStore(newPeer, newPeer) // If we were on the removed list reverse that here. if n.removed != nil { delete(n.removed, newPeer) } if lp, ok := n.peers[newPeer]; !ok { // We are not tracking this one automatically so we need to bump cluster size. n.peers[newPeer] = &lps{time.Now().UnixNano(), 0, true} } else { // Mark as added. lp.kp = true } // Adjust cluster size and quorum if needed. n.adjustClusterSizeAndQuorum() // Write out our new state. n.writePeerState(&peerState{n.peerNames(), n.csz, n.extSt}) // We pass these up as well. committed = append(committed, e) case EntryRemovePeer: peer := string(e.Data) n.debug("Removing peer %q", peer) // Make sure we have our removed map. if n.removed == nil { n.removed = make(map[string]struct{}) } n.removed[peer] = struct{}{} if _, ok := n.peers[peer]; ok { delete(n.peers, peer) // We should decrease our cluster size since we are tracking this peer. n.adjustClusterSizeAndQuorum() // Write out our new state. n.writePeerState(&peerState{n.peerNames(), n.csz, n.extSt}) } // If this is us and we are the leader we should attempt to stepdown. if peer == n.id && n.State() == Leader { n.stepdownLocked(n.selectNextLeader()) } // Remove from string intern map. peers.Delete(peer) // We pass these up as well. committed = append(committed, e) } } // Pass to the upper layers if we have normal entries. It is // entirely possible that 'committed' might be an empty slice here, // which will happen if we've processed updates inline (like peer // states). In which case the upper layer will just call down with // Applied() with no further action. n.apply.push(newCommittedEntry(index, committed)) // Place back in the pool. ae.returnToPool() return nil } // Used to track a success response and apply entries. func (n *raft) trackResponse(ar *appendEntryResponse) { if n.State() == Closed { return } n.Lock() // Update peer's last index. if ps := n.peers[ar.peer]; ps != nil && ar.index > ps.li { ps.li = ar.index } // If we are tracking this peer as a catchup follower, update that here. if indexUpdateQ := n.progress[ar.peer]; indexUpdateQ != nil { indexUpdateQ.push(ar.index) } // Ignore items already committed. if ar.index <= n.commit { n.Unlock() return } // See if we have items to apply. var sendHB bool if results := n.acks[ar.index]; results != nil { results[ar.peer] = struct{}{} if nr := len(results); nr >= n.qn { // We have a quorum. for index := n.commit + 1; index <= ar.index; index++ { if err := n.applyCommit(index); err != nil && err != errNodeClosed { n.error("Got an error applying commit for %d: %v", index, err) break } } sendHB = n.prop.len() == 0 } } n.Unlock() if sendHB { n.sendHeartbeat() } } // Used to adjust cluster size and peer count based on added official peers. // lock should be held. func (n *raft) adjustClusterSizeAndQuorum() { pcsz, ncsz := n.csz, 0 for _, peer := range n.peers { if peer.kp { ncsz++ } } n.csz = ncsz n.qn = n.csz/2 + 1 if ncsz > pcsz { n.debug("Expanding our clustersize: %d -> %d", pcsz, ncsz) n.lsut = time.Now() } else if ncsz < pcsz { n.debug("Decreasing our clustersize: %d -> %d", pcsz, ncsz) if n.State() == Leader { go n.sendHeartbeat() } } } // Track interactions with this peer. func (n *raft) trackPeer(peer string) error { n.Lock() var needPeerAdd, isRemoved bool if n.removed != nil { _, isRemoved = n.removed[peer] } if n.State() == Leader { if lp, ok := n.peers[peer]; !ok || !lp.kp { // Check if this peer had been removed previously. needPeerAdd = !isRemoved } } if ps := n.peers[peer]; ps != nil { ps.ts = time.Now().UnixNano() } else if !isRemoved { n.peers[peer] = &lps{time.Now().UnixNano(), 0, false} } n.Unlock() if needPeerAdd { n.ProposeAddPeer(peer) } return nil } func (n *raft) runAsCandidate() { n.Lock() // Drain old responses. n.votes.drain() n.Unlock() // Send out our request for votes. n.requestVote() // We vote for ourselves. votes := map[string]struct{}{ n.ID(): {}, } for n.State() == Candidate { elect := n.electTimer() select { case <-n.entry.ch: n.processAppendEntries() case <-n.resp.ch: // Ignore append entry responses received from before the state change. n.resp.drain() case <-n.prop.ch: // Ignore proposals received from before the state change. n.prop.drain() case <-n.s.quitCh: return case <-n.quit: return case <-elect.C: n.switchToCandidate() return case <-n.votes.ch: // Because of drain() it is possible that we get nil from popOne(). vresp, ok := n.votes.popOne() if !ok { continue } n.RLock() nterm := n.term n.RUnlock() if vresp.granted && nterm == vresp.term { // only track peers that would be our followers n.trackPeer(vresp.peer) votes[vresp.peer] = struct{}{} if n.wonElection(len(votes)) { // Become LEADER if we have won and gotten a quorum with everyone we should hear from. n.switchToLeader() return } } else if vresp.term > nterm { // if we observe a bigger term, we should start over again or risk forming a quorum fully knowing // someone with a better term exists. This is even the right thing to do if won == true. n.Lock() n.debug("Stepping down from candidate, detected higher term: %d vs %d", vresp.term, n.term) n.term = vresp.term n.vote = noVote n.writeTermVote() n.lxfer = false n.stepdownLocked(noLeader) n.Unlock() } case <-n.reqs.ch: // Because of drain() it is possible that we get nil from popOne(). if voteReq, ok := n.reqs.popOne(); ok { n.processVoteRequest(voteReq) } } } } // handleAppendEntry handles an append entry from the wire. This function // is an internal callback from the "asubj" append entry subscription. func (n *raft) handleAppendEntry(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { msg = copyBytes(msg) if ae, err := n.decodeAppendEntry(msg, sub, reply); err == nil { // Push to the new entry channel. From here one of the worker // goroutines (runAsLeader, runAsFollower, runAsCandidate) will // pick it up. n.entry.push(ae) } else { n.warn("AppendEntry failed to be placed on internal channel: corrupt entry") } } // cancelCatchup will stop an in-flight catchup by unsubscribing from the // catchup subscription. // Lock should be held. func (n *raft) cancelCatchup() { n.debug("Canceling catchup subscription since we are now up to date") if n.catchup != nil && n.catchup.sub != nil { n.unsubscribe(n.catchup.sub) } n.catchup = nil } // catchupStalled will try to determine if we are stalled. This is called // on a new entry from our leader. // Lock should be held. func (n *raft) catchupStalled() bool { if n.catchup == nil { return false } if n.catchup.pindex == n.pindex { return time.Since(n.catchup.active) > 2*time.Second } n.catchup.pindex = n.pindex n.catchup.active = time.Now() return false } // createCatchup will create the state needed to track a catchup as it // runs. It then creates a unique inbox for this catchup and subscribes // to it. The remote side will stream entries to that subject. // Lock should be held. func (n *raft) createCatchup(ae *appendEntry) string { // Cleanup any old ones. if n.catchup != nil && n.catchup.sub != nil { n.unsubscribe(n.catchup.sub) } // Snapshot term and index. n.catchup = &catchupState{ cterm: ae.pterm, cindex: ae.pindex, pterm: n.pterm, pindex: n.pindex, active: time.Now(), } inbox := n.newCatchupInbox() sub, _ := n.subscribe(inbox, n.handleAppendEntry) n.catchup.sub = sub return inbox } // Truncate our WAL and reset. // Lock should be held. func (n *raft) truncateWAL(term, index uint64) { n.debug("Truncating and repairing WAL to Term %d Index %d", term, index) if term == 0 && index == 0 { n.warn("Resetting WAL state") } defer func() { // Check to see if we invalidated any snapshots that might have held state // from the entries we are truncating. if snap, _ := n.loadLastSnapshot(); snap != nil && snap.lastIndex > index { os.Remove(n.snapfile) n.snapfile = _EMPTY_ } // Make sure to reset commit and applied if above if n.commit > n.pindex { n.commit = n.pindex } if n.applied > n.commit { n.applied = n.commit } }() if err := n.wal.Truncate(index); err != nil { // If we get an invalid sequence, reset our wal all together. // We will not have holes, so this means we do not have this message stored anymore. if err == ErrInvalidSequence { n.debug("Resetting WAL") n.wal.Truncate(0) // If our index is non-zero use PurgeEx to set us to the correct next index. if index > 0 { n.wal.PurgeEx(fwcs, index+1, 0) } } else { n.warn("Error truncating WAL: %v", err) n.setWriteErrLocked(err) return } } // Set after we know we have truncated properly. n.pterm, n.pindex = term, index } // Reset our WAL. This is equivalent to truncating all data from the log. // Lock should be held. func (n *raft) resetWAL() { n.truncateWAL(0, 0) } // Lock should be held func (n *raft) updateLeader(newLeader string) { n.leader = newLeader n.hasleader.Store(newLeader != _EMPTY_) if !n.pleader.Load() && newLeader != noLeader { n.pleader.Store(true) } } // processAppendEntry will process an appendEntry. This is called either // during recovery or from processAppendEntries when there are new entries // to be committed. func (n *raft) processAppendEntry(ae *appendEntry, sub *subscription) { n.Lock() // Don't reset here if we have been asked to assume leader position. if !n.lxfer { n.resetElectionTimeout() } // Just return if closed or we had previous write error. if n.State() == Closed || n.werr != nil { n.Unlock() return } // Scratch buffer for responses. var scratch [appendEntryResponseLen]byte arbuf := scratch[:] // Are we receiving from another leader. if n.State() == Leader { // If we are the same we should step down to break the tie. if ae.term >= n.term { // If the append entry term is newer than the current term, erase our // vote. if ae.term > n.term { n.term = ae.term n.vote = noVote n.writeTermVote() } n.debug("Received append entry from another leader, stepping down to %q", ae.leader) n.stepdownLocked(ae.leader) } else { // Let them know we are the leader. ar := newAppendEntryResponse(n.term, n.pindex, n.id, false) n.debug("AppendEntry ignoring old term from another leader") n.sendRPC(ae.reply, _EMPTY_, ar.encode(arbuf)) arPool.Put(ar) } // Always return here from processing. n.Unlock() return } // If we received an append entry as a candidate then it would appear that // another node has taken on the leader role already, so we should convert // to a follower of that node instead. if n.State() == Candidate { // If we have a leader in the current term or higher, we should stepdown, // write the term and vote if the term of the request is higher. if ae.term >= n.term { // If the append entry term is newer than the current term, erase our // vote. if ae.term > n.term { n.term = ae.term n.vote = noVote n.writeTermVote() } n.debug("Received append entry in candidate state from %q, converting to follower", ae.leader) n.stepdownLocked(ae.leader) } } // Catching up state. catchingUp := n.catchup != nil // Is this a new entry? New entries will be delivered on the append entry // sub, rather than a catch-up sub. isNew := sub != nil && sub == n.aesub // Track leader directly if isNew && ae.leader != noLeader { if ps := n.peers[ae.leader]; ps != nil { ps.ts = time.Now().UnixNano() } else { n.peers[ae.leader] = &lps{time.Now().UnixNano(), 0, true} } } // If we are catching up ignore old catchup subs. // This could happen when we stall or cancel a catchup. if !isNew && catchingUp && sub != n.catchup.sub { n.Unlock() n.debug("AppendEntry ignoring old entry from previous catchup") return } // Check state if we are catching up. if catchingUp { if cs := n.catchup; cs != nil && n.pterm >= cs.cterm && n.pindex >= cs.cindex { // If we are here we are good, so if we have a catchup pending we can cancel. n.cancelCatchup() // Reset our notion of catching up. catchingUp = false } else if isNew { var ar *appendEntryResponse var inbox string // Check to see if we are stalled. If so recreate our catchup state and resend response. if n.catchupStalled() { n.debug("Catchup may be stalled, will request again") inbox = n.createCatchup(ae) ar = newAppendEntryResponse(n.pterm, n.pindex, n.id, false) } n.Unlock() if ar != nil { n.sendRPC(ae.reply, inbox, ar.encode(arbuf)) arPool.Put(ar) } // Ignore new while catching up or replaying. return } } // If this term is greater than ours. if ae.term > n.term { n.term = ae.term n.vote = noVote if isNew { n.writeTermVote() } if n.State() != Follower { n.debug("Term higher than ours and we are not a follower: %v, stepping down to %q", n.State(), ae.leader) n.stepdownLocked(ae.leader) } } else if ae.term < n.term && !catchingUp && isNew { n.debug("Rejected AppendEntry from a leader (%s) with term %d which is less than ours", ae.leader, ae.term) ar := newAppendEntryResponse(n.term, n.pindex, n.id, false) n.Unlock() n.sendRPC(ae.reply, _EMPTY_, ar.encode(arbuf)) arPool.Put(ar) return } if isNew && n.leader != ae.leader && n.State() == Follower { n.debug("AppendEntry updating leader to %q", ae.leader) n.updateLeader(ae.leader) n.writeTermVote() n.resetElectionTimeout() n.updateLeadChange(false) } if ae.pterm != n.pterm || ae.pindex != n.pindex { // Check if this is a lower or equal index than what we were expecting. if ae.pindex <= n.pindex { n.debug("AppendEntry detected pindex less than/equal to ours: %d:%d vs %d:%d", ae.pterm, ae.pindex, n.pterm, n.pindex) var ar *appendEntryResponse var success bool if ae.pindex < n.commit { // If we have already committed this entry, just mark success. success = true } else if eae, _ := n.loadEntry(ae.pindex); eae == nil { // If terms are equal, and we are not catching up, we have simply already processed this message. // So we will ACK back to the leader. This can happen on server restarts based on timings of snapshots. if ae.pterm == n.pterm && !catchingUp { success = true } else if ae.pindex == n.pindex { // Check if only our terms do not match here. // Make sure pterms match and we take on the leader's. // This prevents constant spinning. n.truncateWAL(ae.pterm, ae.pindex) } else { n.resetWAL() } } else if eae.term == ae.pterm { // If terms match we can delete all entries past this one, and then continue storing the current entry. n.truncateWAL(ae.pterm, ae.pindex) // Only continue if truncation was successful, and we ended up such that we can safely continue. if ae.pterm == n.pterm && ae.pindex == n.pindex { goto CONTINUE } } else { // If terms mismatched, delete that entry and all others past it. // Make sure to cancel any catchups in progress. // Truncate will reset our pterm and pindex. Only do so if we have an entry. n.truncateWAL(eae.pterm, eae.pindex) } // Cancel regardless if unsuccessful. if !success { n.cancelCatchup() } // Create response. ar = newAppendEntryResponse(ae.pterm, ae.pindex, n.id, success) n.Unlock() n.sendRPC(ae.reply, _EMPTY_, ar.encode(arbuf)) arPool.Put(ar) return } // Check if we are catching up. If we are here we know the leader did not have all of the entries // so make sure this is a snapshot entry. If it is not start the catchup process again since it // means we may have missed additional messages. if catchingUp { // This means we already entered into a catchup state but what the leader sent us did not match what we expected. // Snapshots and peerstate will always be together when a leader is catching us up in this fashion. if len(ae.entries) != 2 || ae.entries[0].Type != EntrySnapshot || ae.entries[1].Type != EntryPeerState { n.warn("Expected first catchup entry to be a snapshot and peerstate, will retry") n.cancelCatchup() n.Unlock() return } if ps, err := decodePeerState(ae.entries[1].Data); err == nil { n.processPeerState(ps) // Also need to copy from client's buffer. ae.entries[0].Data = copyBytes(ae.entries[0].Data) } else { n.warn("Could not parse snapshot peerstate correctly") n.cancelCatchup() n.Unlock() return } // Inherit state from appendEntry with the leader's snapshot. n.pindex = ae.pindex n.pterm = ae.pterm n.commit = ae.pindex if _, err := n.wal.Compact(n.pindex + 1); err != nil { n.setWriteErrLocked(err) n.Unlock() return } snap := &snapshot{ lastTerm: n.pterm, lastIndex: n.pindex, peerstate: encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt}), data: ae.entries[0].Data, } // Install the leader's snapshot as our own. if err := n.installSnapshot(snap); err != nil { n.setWriteErrLocked(err) n.Unlock() return } // Now send snapshot to upper levels. Only send the snapshot, not the peerstate entry. n.apply.push(newCommittedEntry(n.commit, ae.entries[:1])) n.Unlock() return } // Setup our state for catching up. n.debug("AppendEntry did not match %d %d with %d %d", ae.pterm, ae.pindex, n.pterm, n.pindex) inbox := n.createCatchup(ae) ar := newAppendEntryResponse(n.pterm, n.pindex, n.id, false) n.Unlock() n.sendRPC(ae.reply, inbox, ar.encode(arbuf)) arPool.Put(ar) return } CONTINUE: // Save to our WAL if we have entries. if ae.shouldStore() { // Only store if an original which will have sub != nil if sub != nil { if err := n.storeToWAL(ae); err != nil { if err != ErrStoreClosed { n.warn("Error storing entry to WAL: %v", err) } n.Unlock() return } // Save in memory for faster processing during applyCommit. // Only save so many however to avoid memory bloat. if l := len(n.pae); l <= paeDropThreshold { n.pae[n.pindex], l = ae, l+1 if l > paeWarnThreshold && l%paeWarnModulo == 0 { n.warn("%d append entries pending", len(n.pae)) } } else { // Invalidate cache entry at this index, we might have // stored it previously with a different value. delete(n.pae, n.pindex) if l%paeWarnModulo == 0 { n.debug("Not saving to append entries pending") } } } else { // This is a replay on startup so just take the appendEntry version. n.pterm = ae.term n.pindex = ae.pindex + 1 } } // Check to see if we have any related entries to process here. for _, e := range ae.entries { switch e.Type { case EntryLeaderTransfer: // Only process these if they are new, so no replays or catchups. if isNew { maybeLeader := string(e.Data) // This is us. We need to check if we can become the leader. if maybeLeader == n.id { // If not an observer and not paused we are good to go. if !n.observer && !n.paused { n.lxfer = true n.xferCampaign() } else if n.paused && !n.pobserver { // Here we can become a leader but need to wait for resume of the apply queue. n.lxfer = true } } else if n.vote != noVote { // Since we are here we are not the chosen one but we should clear any vote preference. n.vote = noVote n.writeTermVote() } } case EntryAddPeer: if newPeer := string(e.Data); len(newPeer) == idLen { // Track directly, but wait for commit to be official if ps := n.peers[newPeer]; ps != nil { ps.ts = time.Now().UnixNano() } else { n.peers[newPeer] = &lps{time.Now().UnixNano(), 0, false} } // Store our peer in our global peer map for all peers. peers.LoadOrStore(newPeer, newPeer) } } } // Make a copy of these values, as the AppendEntry might be cached and returned to the pool in applyCommit. aeCommit := ae.commit aeReply := ae.reply // Apply anything we need here. if aeCommit > n.commit { if n.paused { n.hcommit = aeCommit n.debug("Paused, not applying %d", aeCommit) } else { for index := n.commit + 1; index <= aeCommit; index++ { if err := n.applyCommit(index); err != nil { break } } } } var ar *appendEntryResponse if sub != nil { ar = newAppendEntryResponse(n.pterm, n.pindex, n.id, true) } n.Unlock() // Success. Send our response. if ar != nil { n.sendRPC(aeReply, _EMPTY_, ar.encode(arbuf)) arPool.Put(ar) } } // processPeerState is called when a peer state entry is received // over the wire or when we're updating known peers. // Lock should be held. func (n *raft) processPeerState(ps *peerState) { // Update our version of peers to that of the leader. Calculate // the number of nodes needed to establish a quorum. n.csz = ps.clusterSize n.qn = n.csz/2 + 1 old := n.peers n.peers = make(map[string]*lps) for _, peer := range ps.knownPeers { if lp := old[peer]; lp != nil { lp.kp = true n.peers[peer] = lp } else { n.peers[peer] = &lps{0, 0, true} } } n.debug("Update peers from leader to %+v", n.peers) n.writePeerState(ps) } // processAppendEntryResponse is called when we receive an append entry // response from another node. They will send a confirmation to tell us // whether they successfully committed the entry or not. func (n *raft) processAppendEntryResponse(ar *appendEntryResponse) { n.trackPeer(ar.peer) if ar.success { // The remote node successfully committed the append entry. n.trackResponse(ar) arPool.Put(ar) } else if ar.term > n.term { // The remote node didn't commit the append entry, it looks like // they are on a newer term than we are. Step down. n.Lock() n.term = ar.term n.vote = noVote n.writeTermVote() n.warn("Detected another leader with higher term, will stepdown") n.stepdownLocked(noLeader) n.Unlock() arPool.Put(ar) } else if ar.reply != _EMPTY_ { // The remote node didn't commit the append entry and they are // still on the same term, so let's try to catch them up. n.catchupFollower(ar) } } // handleAppendEntryResponse processes responses to append entries. func (n *raft) handleAppendEntryResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { ar := n.decodeAppendEntryResponse(msg) ar.reply = reply n.resp.push(ar) } func (n *raft) buildAppendEntry(entries []*Entry) *appendEntry { return newAppendEntry(n.id, n.term, n.commit, n.pterm, n.pindex, entries) } // Determine if we should store an entry. This stops us from storing // heartbeat messages. func (ae *appendEntry) shouldStore() bool { return ae != nil && len(ae.entries) > 0 } // Store our append entry to our WAL. // lock should be held. func (n *raft) storeToWAL(ae *appendEntry) error { if ae == nil { return fmt.Errorf("raft: Missing append entry for storage") } if n.werr != nil { return n.werr } seq, _, err := n.wal.StoreMsg(_EMPTY_, nil, ae.buf) if err != nil { n.setWriteErrLocked(err) return err } // Sanity checking for now. if index := ae.pindex + 1; index != seq { n.warn("Wrong index, ae is %+v, index stored was %d, n.pindex is %d, will reset", ae, seq, n.pindex) if n.State() == Leader { n.stepdownLocked(n.selectNextLeader()) } // Reset and cancel any catchup. n.resetWAL() n.cancelCatchup() return errEntryStoreFailed } n.pterm = ae.term n.pindex = seq return nil } const ( paeDropThreshold = 20_000 paeWarnThreshold = 10_000 paeWarnModulo = 5_000 ) func (n *raft) sendAppendEntry(entries []*Entry) { n.Lock() defer n.Unlock() ae := n.buildAppendEntry(entries) var err error var scratch [1024]byte ae.buf, err = ae.encode(scratch[:]) if err != nil { return } // If we have entries store this in our wal. shouldStore := ae.shouldStore() if shouldStore { if err := n.storeToWAL(ae); err != nil { return } // We count ourselves. n.acks[n.pindex] = map[string]struct{}{n.id: {}} n.active = time.Now() // Save in memory for faster processing during applyCommit. n.pae[n.pindex] = ae if l := len(n.pae); l > paeWarnThreshold && l%paeWarnModulo == 0 { n.warn("%d append entries pending", len(n.pae)) } } n.sendRPC(n.asubj, n.areply, ae.buf) if !shouldStore { ae.returnToPool() } } type extensionState uint16 const ( extUndetermined = extensionState(iota) extExtended extNotExtended ) type peerState struct { knownPeers []string clusterSize int domainExt extensionState } func peerStateBufSize(ps *peerState) int { return 4 + 4 + (idLen * len(ps.knownPeers)) + 2 } func encodePeerState(ps *peerState) []byte { var le = binary.LittleEndian buf := make([]byte, peerStateBufSize(ps)) le.PutUint32(buf[0:], uint32(ps.clusterSize)) le.PutUint32(buf[4:], uint32(len(ps.knownPeers))) wi := 8 for _, peer := range ps.knownPeers { copy(buf[wi:], peer) wi += idLen } le.PutUint16(buf[wi:], uint16(ps.domainExt)) return buf } func decodePeerState(buf []byte) (*peerState, error) { if len(buf) < 8 { return nil, errCorruptPeers } var le = binary.LittleEndian ps := &peerState{clusterSize: int(le.Uint32(buf[0:]))} expectedPeers := int(le.Uint32(buf[4:])) buf = buf[8:] ri := 0 for i, n := 0, expectedPeers; i < n && ri < len(buf); i++ { ps.knownPeers = append(ps.knownPeers, string(buf[ri:ri+idLen])) ri += idLen } if len(ps.knownPeers) != expectedPeers { return nil, errCorruptPeers } if len(buf[ri:]) >= 2 { ps.domainExt = extensionState(le.Uint16(buf[ri:])) } return ps, nil } // Lock should be held. func (n *raft) peerNames() []string { var peers []string for name, peer := range n.peers { if peer.kp { peers = append(peers, name) } } return peers } func (n *raft) currentPeerState() *peerState { n.RLock() ps := &peerState{n.peerNames(), n.csz, n.extSt} n.RUnlock() return ps } // sendPeerState will send our current peer state to the cluster. func (n *raft) sendPeerState() { n.sendAppendEntry([]*Entry{{EntryPeerState, encodePeerState(n.currentPeerState())}}) } // Send a heartbeat. func (n *raft) sendHeartbeat() { n.sendAppendEntry(nil) } type voteRequest struct { term uint64 lastTerm uint64 lastIndex uint64 candidate string // internal only. reply string } const voteRequestLen = 24 + idLen func (vr *voteRequest) encode() []byte { var buf [voteRequestLen]byte var le = binary.LittleEndian le.PutUint64(buf[0:], vr.term) le.PutUint64(buf[8:], vr.lastTerm) le.PutUint64(buf[16:], vr.lastIndex) copy(buf[24:24+idLen], vr.candidate) return buf[:voteRequestLen] } func decodeVoteRequest(msg []byte, reply string) *voteRequest { if len(msg) != voteRequestLen { return nil } var le = binary.LittleEndian return &voteRequest{ term: le.Uint64(msg[0:]), lastTerm: le.Uint64(msg[8:]), lastIndex: le.Uint64(msg[16:]), candidate: string(copyBytes(msg[24 : 24+idLen])), reply: reply, } } const peerStateFile = "peers.idx" // Lock should be held. func (n *raft) writePeerState(ps *peerState) { pse := encodePeerState(ps) if bytes.Equal(n.wps, pse) { return } // Stamp latest and write the peer state file. n.wps = pse if err := writePeerState(n.sd, ps); err != nil && !n.isClosed() { n.setWriteErrLocked(err) n.warn("Error writing peer state file for %q: %v", n.group, err) } } // Writes out our peer state outside of a specific raft context. func writePeerState(sd string, ps *peerState) error { psf := filepath.Join(sd, peerStateFile) if _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) { return err } return writeFileWithSync(psf, encodePeerState(ps), defaultFilePerms) } func readPeerState(sd string) (ps *peerState, err error) { <-dios buf, err := os.ReadFile(filepath.Join(sd, peerStateFile)) dios <- struct{}{} if err != nil { return nil, err } return decodePeerState(buf) } const termVoteFile = "tav.idx" const termLen = 8 // uint64 const termVoteLen = idLen + termLen // Writes out our term & vote outside of a specific raft context. func writeTermVote(sd string, wtv []byte) error { psf := filepath.Join(sd, termVoteFile) if _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) { return err } return writeFileWithSync(psf, wtv, defaultFilePerms) } // readTermVote will read the largest term and who we voted from to stable storage. // Lock should be held. func (n *raft) readTermVote() (term uint64, voted string, err error) { <-dios buf, err := os.ReadFile(filepath.Join(n.sd, termVoteFile)) dios <- struct{}{} if err != nil { return 0, noVote, err } if len(buf) < termLen { // Not enough bytes for the uint64 below, so avoid a panic. return 0, noVote, nil } var le = binary.LittleEndian term = le.Uint64(buf[0:]) if len(buf) < termVoteLen { return term, noVote, nil } voted = string(buf[8:]) return term, voted, nil } // Lock should be held. func (n *raft) setWriteErrLocked(err error) { // Check if we are closed already. if n.State() == Closed { return } // Ignore if already set. if n.werr == err || err == nil { return } // Ignore non-write errors. if err == ErrStoreClosed || err == ErrStoreEOF || err == ErrInvalidSequence || err == ErrStoreMsgNotFound || err == errNoPending || err == errPartialCache { return } // If this is a not found report but do not disable. if os.IsNotExist(err) { n.error("Resource not found: %v", err) return } n.error("Critical write error: %v", err) n.werr = err if isPermissionError(err) { go n.s.handleWritePermissionError() } if isOutOfSpaceErr(err) { // For now since this can be happening all under the covers, we will call up and disable JetStream. go n.s.handleOutOfSpace(nil) } } // Helper to check if we are closed when we do not hold a lock already. func (n *raft) isClosed() bool { return n.State() == Closed } // Capture our write error if any and hold. func (n *raft) setWriteErr(err error) { n.Lock() defer n.Unlock() n.setWriteErrLocked(err) } // writeTermVote will record the largest term and who we voted for to stable storage. // Lock should be held. func (n *raft) writeTermVote() { var buf [termVoteLen]byte var le = binary.LittleEndian le.PutUint64(buf[0:], n.term) copy(buf[8:], n.vote) b := buf[:8+len(n.vote)] // If the term and vote hasn't changed then don't rewrite to disk. if bytes.Equal(n.wtv, b) { return } // Stamp latest and write the term & vote file. n.wtv = b if err := writeTermVote(n.sd, n.wtv); err != nil && !n.isClosed() { // Clear wtv since we failed. n.wtv = nil n.setWriteErrLocked(err) n.warn("Error writing term and vote file for %q: %v", n.group, err) } } // voteResponse is a response to a vote request. type voteResponse struct { term uint64 peer string granted bool } const voteResponseLen = 8 + 8 + 1 func (vr *voteResponse) encode() []byte { var buf [voteResponseLen]byte var le = binary.LittleEndian le.PutUint64(buf[0:], vr.term) copy(buf[8:], vr.peer) if vr.granted { buf[16] = 1 } else { buf[16] = 0 } return buf[:voteResponseLen] } func decodeVoteResponse(msg []byte) *voteResponse { if len(msg) != voteResponseLen { return nil } var le = binary.LittleEndian vr := &voteResponse{term: le.Uint64(msg[0:]), peer: string(msg[8:16])} vr.granted = msg[16] == 1 return vr } func (n *raft) handleVoteResponse(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) { vr := decodeVoteResponse(msg) n.debug("Received a voteResponse %+v", vr) if vr == nil { n.error("Received malformed vote response for %q", n.group) return } if state := n.State(); state != Candidate && state != Leader { n.debug("Ignoring old vote response, we have stepped down") return } n.votes.push(vr) } func (n *raft) processVoteRequest(vr *voteRequest) error { // To simplify calling code, we can possibly pass `nil` to this function. // If that is the case, does not consider it an error. if vr == nil { return nil } n.debug("Received a voteRequest %+v", vr) if err := n.trackPeer(vr.candidate); err != nil { return err } n.Lock() vresp := &voteResponse{n.term, n.id, false} defer n.debug("Sending a voteResponse %+v -> %q", vresp, vr.reply) // Ignore if we are newer. This is important so that we don't accidentally process // votes from a previous term if they were still in flight somewhere. if vr.term < n.term { n.Unlock() n.sendReply(vr.reply, vresp.encode()) return nil } // If this is a higher term go ahead and stepdown. if vr.term > n.term { if n.State() != Follower { n.debug("Stepping down from %s, detected higher term: %d vs %d", strings.ToLower(n.State().String()), vr.term, n.term) n.stepdownLocked(noLeader) } n.cancelCatchup() n.term = vr.term n.vote = noVote n.writeTermVote() } // Only way we get to yes is through here. voteOk := n.vote == noVote || n.vote == vr.candidate if voteOk && (vr.lastTerm > n.pterm || vr.lastTerm == n.pterm && vr.lastIndex >= n.pindex) { vresp.granted = true n.term = vr.term n.vote = vr.candidate n.writeTermVote() n.resetElectionTimeout() } else if n.vote == noVote && n.State() != Candidate { // We have a more up-to-date log, and haven't voted yet. // Start campaigning earlier, but only if not candidate already, as that would short-circuit us. n.resetElect(randCampaignTimeout()) } // Term might have changed, make sure response has the most current vresp.term = n.term n.Unlock() n.sendReply(vr.reply, vresp.encode()) return nil } func (n *raft) handleVoteRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) { vr := decodeVoteRequest(msg, reply) if vr == nil { n.error("Received malformed vote request for %q", n.group) return } n.reqs.push(vr) } func (n *raft) requestVote() { n.Lock() if n.State() != Candidate { n.Unlock() return } n.vote = n.id n.writeTermVote() vr := voteRequest{n.term, n.pterm, n.pindex, n.id, _EMPTY_} subj, reply := n.vsubj, n.vreply n.Unlock() n.debug("Sending out voteRequest %+v", vr) // Now send it out. n.sendRPC(subj, reply, vr.encode()) } func (n *raft) sendRPC(subject, reply string, msg []byte) { if n.sq != nil { n.sq.send(subject, reply, nil, msg) } } func (n *raft) sendReply(subject string, msg []byte) { if n.sq != nil { n.sq.send(subject, _EMPTY_, nil, msg) } } func (n *raft) wonElection(votes int) bool { return votes >= n.quorumNeeded() } // Return the quorum size for a given cluster config. func (n *raft) quorumNeeded() int { n.RLock() qn := n.qn n.RUnlock() return qn } // Lock should be held. func (n *raft) updateLeadChange(isLeader bool) { // We don't care about values that have not been consumed (transitory states), // so we dequeue any state that is pending and push the new one. for { select { case n.leadc <- isLeader: return default: select { case <-n.leadc: default: // May have been consumed by the "reader" go routine, so go back // to the top of the loop and try to send again. } } } } // Lock should be held. func (n *raft) switchState(state RaftState) { retry: pstate := n.State() if pstate == Closed { return } // Set our state. If something else has changed our state // then retry, this will either be a Stop or Delete call. if !n.state.CompareAndSwap(int32(pstate), int32(state)) { goto retry } // Reset the election timer. n.resetElectionTimeout() if pstate == Leader && state != Leader { n.updateLeadChange(false) // Drain the append entry response and proposal queues. n.resp.drain() n.prop.drain() } else if state == Leader && pstate != Leader { if len(n.pae) > 0 { n.pae = make(map[uint64]*appendEntry) } n.updateLeadChange(true) } n.writeTermVote() } const ( noLeader = _EMPTY_ noVote = _EMPTY_ ) func (n *raft) switchToFollower(leader string) { n.Lock() defer n.Unlock() n.switchToFollowerLocked(leader) } func (n *raft) switchToFollowerLocked(leader string) { if n.State() == Closed { return } n.debug("Switching to follower") n.lxfer = false n.updateLeader(leader) n.switchState(Follower) } func (n *raft) switchToCandidate() { if n.State() == Closed { return } n.Lock() defer n.Unlock() // If we are catching up or are in observer mode we can not switch. // Avoid petitioning to become leader if we're behind on applies. if n.observer || n.paused || n.applied < n.commit { n.resetElect(minElectionTimeout / 4) return } if n.State() != Candidate { n.debug("Switching to candidate") } else { if n.lostQuorumLocked() && time.Since(n.llqrt) > 20*time.Second { // We signal to the upper layers such that can alert on quorum lost. n.updateLeadChange(false) n.llqrt = time.Now() } } // Increment the term. n.term++ // Clear current Leader. n.updateLeader(noLeader) n.switchState(Candidate) } func (n *raft) switchToLeader() { if n.State() == Closed { return } n.Lock() n.debug("Switching to leader") var state StreamState n.wal.FastState(&state) // Check if we have items pending as we are taking over. sendHB := state.LastSeq > n.commit n.lxfer = false n.updateLeader(n.id) n.switchState(Leader) n.Unlock() if sendHB { n.sendHeartbeat() } } nats-server-2.10.27/server/raft_helpers_test.go000066400000000000000000000215261477524627100215330ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Do not exlude this file with the !skip_js_tests since those helpers // are also used by MQTT. package server import ( "encoding/binary" "fmt" "math/rand" "sync" "testing" "time" ) type stateMachine interface { server() *Server node() RaftNode waitGroup() *sync.WaitGroup // This will call forward as needed so can be called on any node. propose(data []byte) // When entries have been committed and can be applied. applyEntry(ce *CommittedEntry) // When a leader change happens. leaderChange(isLeader bool) // Stop the raft group. stop() // Restart restart() } // Factory function needed for constructor. type smFactory func(s *Server, cfg *RaftConfig, node RaftNode) stateMachine type smGroup []stateMachine // Leader of the group. func (sg smGroup) leader() stateMachine { for _, sm := range sg { if sm.node().Leader() { return sm } } return nil } // Wait on a leader to be elected. func (sg smGroup) waitOnLeader() { expires := time.Now().Add(10 * time.Second) for time.Now().Before(expires) { for _, sm := range sg { if sm.node().Leader() { return } } time.Sleep(100 * time.Millisecond) } } // Pick a random member. func (sg smGroup) randomMember() stateMachine { return sg[rand.Intn(len(sg))] } // Return a non-leader func (sg smGroup) nonLeader() stateMachine { for _, sm := range sg { if !sm.node().Leader() { return sm } } return nil } // Take out the lock on all nodes. func (sg smGroup) lockAll() { for _, sm := range sg { sm.node().(*raft).Lock() } } // Release the lock on all nodes. func (sg smGroup) unlockAll() { for _, sm := range sg { sm.node().(*raft).Unlock() } } // Create a raft group and place on numMembers servers at random. // Filestore based. func (c *cluster) createRaftGroup(name string, numMembers int, smf smFactory) smGroup { return c.createRaftGroupEx(name, numMembers, smf, FileStorage) } func (c *cluster) createMemRaftGroup(name string, numMembers int, smf smFactory) smGroup { return c.createRaftGroupEx(name, numMembers, smf, MemoryStorage) } func (c *cluster) createRaftGroupEx(name string, numMembers int, smf smFactory, st StorageType) smGroup { c.t.Helper() if numMembers > len(c.servers) { c.t.Fatalf("Members > Peers: %d vs %d", numMembers, len(c.servers)) } servers := append([]*Server{}, c.servers...) rand.Shuffle(len(servers), func(i, j int) { servers[i], servers[j] = servers[j], servers[i] }) return c.createRaftGroupWithPeers(name, servers[:numMembers], smf, st) } func (c *cluster) createRaftGroupWithPeers(name string, servers []*Server, smf smFactory, st StorageType) smGroup { c.t.Helper() var sg smGroup var peers []string for _, s := range servers { // generate peer names. s.mu.RLock() peers = append(peers, s.sys.shash) s.mu.RUnlock() } for _, s := range servers { var cfg *RaftConfig if st == FileStorage { fs, err := newFileStore( FileStoreConfig{StoreDir: c.t.TempDir(), BlockSize: defaultMediumBlockSize, AsyncFlush: false, SyncInterval: 5 * time.Minute}, StreamConfig{Name: name, Storage: FileStorage}, ) require_NoError(c.t, err) cfg = &RaftConfig{Name: name, Store: c.t.TempDir(), Log: fs} } else { ms, err := newMemStore(&StreamConfig{Name: name, Storage: MemoryStorage}) require_NoError(c.t, err) cfg = &RaftConfig{Name: name, Store: c.t.TempDir(), Log: ms} } s.bootstrapRaftNode(cfg, peers, true) n, err := s.startRaftNode(globalAccountName, cfg, pprofLabels{}) require_NoError(c.t, err) sm := smf(s, cfg, n) sg = append(sg, sm) go smLoop(sm) } return sg } // Driver program for the state machine. // Should be run in its own go routine. func smLoop(sm stateMachine) { s, n, wg := sm.server(), sm.node(), sm.waitGroup() qch, lch, aq := n.QuitC(), n.LeadChangeC(), n.ApplyQ() // Wait group used to allow waiting until we exit from here. wg.Add(1) defer wg.Done() for { select { case <-s.quitCh: return case <-qch: return case <-aq.ch: ces := aq.pop() for _, ce := range ces { sm.applyEntry(ce) } aq.recycle(&ces) case isLeader := <-lch: sm.leaderChange(isLeader) } } } // Simple implementation of a replicated state. // The adder state just sums up int64 values. type stateAdder struct { sync.Mutex s *Server n RaftNode wg sync.WaitGroup cfg *RaftConfig sum int64 lch chan bool } // Simple getters for server and the raft node. func (a *stateAdder) server() *Server { a.Lock() defer a.Unlock() return a.s } func (a *stateAdder) node() RaftNode { a.Lock() defer a.Unlock() return a.n } func (a *stateAdder) waitGroup() *sync.WaitGroup { a.Lock() defer a.Unlock() return &a.wg } func (a *stateAdder) propose(data []byte) { a.Lock() defer a.Unlock() a.n.ForwardProposal(data) } func (a *stateAdder) applyEntry(ce *CommittedEntry) { a.Lock() defer a.Unlock() if ce == nil { // This means initial state is done/replayed. return } for _, e := range ce.Entries { if e.Type == EntryNormal { delta, _ := binary.Varint(e.Data) a.sum += delta } else if e.Type == EntrySnapshot { a.sum, _ = binary.Varint(e.Data) } } // Update applied. a.n.Applied(ce.Index) } func (a *stateAdder) leaderChange(isLeader bool) { select { case a.lch <- isLeader: default: } } // Adder specific to change the total. func (a *stateAdder) proposeDelta(delta int64) { data := make([]byte, binary.MaxVarintLen64) n := binary.PutVarint(data, int64(delta)) a.propose(data[:n]) } // Stop the group. func (a *stateAdder) stop() { n, wg := a.node(), a.waitGroup() n.Stop() n.WaitForStop() wg.Wait() } // Restart the group func (a *stateAdder) restart() { a.Lock() defer a.Unlock() if a.n.State() != Closed { return } // The filestore is stopped as well, so need to extract the parts to recreate it. rn := a.n.(*raft) var err error switch rn.wal.(type) { case *fileStore: fs := rn.wal.(*fileStore) a.cfg.Log, err = newFileStore(fs.fcfg, fs.cfg.StreamConfig) case *memStore: ms := rn.wal.(*memStore) a.cfg.Log, err = newMemStore(&ms.cfg) } if err != nil { panic(err) } // Must reset in-memory state. // A real restart would not preserve it, but more importantly we have no way to detect if we // already applied an entry. So, the sum must only be updated based on append entries or snapshots. a.sum = 0 a.n, err = a.s.startRaftNode(globalAccountName, a.cfg, pprofLabels{}) if err != nil { panic(err) } // Finally restart the driver. go smLoop(a) } // Total for the adder state machine. func (a *stateAdder) total() int64 { a.Lock() defer a.Unlock() return a.sum } // Install a snapshot. func (a *stateAdder) snapshot(t *testing.T) { a.Lock() defer a.Unlock() data := make([]byte, binary.MaxVarintLen64) n := binary.PutVarint(data, a.sum) snap := data[:n] require_NoError(t, a.n.InstallSnapshot(snap)) } // Helper to wait for a certain state. func (rg smGroup) waitOnTotal(t *testing.T, expected int64) { t.Helper() checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { for _, sm := range rg { asm := sm.(*stateAdder) if total := asm.total(); total != expected { return fmt.Errorf("Adder on %v has wrong total: %d vs %d", asm.server(), total, expected) } } return nil }) } // Factory function. func newStateAdder(s *Server, cfg *RaftConfig, n RaftNode) stateMachine { return &stateAdder{s: s, n: n, cfg: cfg, lch: make(chan bool, 1)} } func initSingleMemRaftNode(t *testing.T) (*raft, func()) { t.Helper() n, c := initSingleMemRaftNodeWithCluster(t) cleanup := func() { c.shutdown() } return n, cleanup } func initSingleMemRaftNodeWithCluster(t *testing.T) (*raft, *cluster) { t.Helper() c := createJetStreamClusterExplicit(t, "R3S", 3) s := c.servers[0] // RunBasicJetStreamServer not available ms, err := newMemStore(&StreamConfig{Name: "TEST", Storage: MemoryStorage}) require_NoError(t, err) cfg := &RaftConfig{Name: "TEST", Store: t.TempDir(), Log: ms} err = s.bootstrapRaftNode(cfg, nil, false) require_NoError(t, err) n, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{}) require_NoError(t, err) return n, c } // Encode an AppendEntry. // An AppendEntry is encoded into a buffer and that's stored into the WAL. // This is a helper function to generate that buffer. func encode(t *testing.T, ae *appendEntry) *appendEntry { t.Helper() buf, err := ae.encode(nil) require_NoError(t, err) ae.buf = buf return ae } nats-server-2.10.27/server/raft_test.go000066400000000000000000002022731477524627100200110ustar00rootroot00000000000000// Copyright 2021-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "encoding/binary" "errors" "fmt" "math" "math/rand" "os" "path" "path/filepath" "testing" "time" "github.com/nats-io/nats.go" ) func TestNRGSimple(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Do several state transitions. rg.randomMember().(*stateAdder).proposeDelta(22) rg.randomMember().(*stateAdder).proposeDelta(-11) rg.randomMember().(*stateAdder).proposeDelta(-10) // Wait for all members to have the correct state. rg.waitOnTotal(t, 1) } func TestNRGSnapshotAndRestart(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() var expectedTotal int64 leader := rg.leader().(*stateAdder) sm := rg.nonLeader().(*stateAdder) for i := 0; i < 1000; i++ { delta := rand.Int63n(222) expectedTotal += delta leader.proposeDelta(delta) if i == 250 { // Let some things catchup. time.Sleep(50 * time.Millisecond) // Snapshot leader and stop and snapshot a member. leader.snapshot(t) sm.snapshot(t) sm.stop() } } // Restart. sm.restart() // Wait for all members to have the correct state. rg.waitOnTotal(t, expectedTotal) } func TestNRGAppendEntryEncode(t *testing.T) { ae := &appendEntry{ term: 1, pindex: 0, } // Test leader should be _EMPTY_ or exactly idLen long ae.leader = "foo_bar_baz" _, err := ae.encode(nil) require_Error(t, err, errLeaderLen) // Empty ok (noLeader) ae.leader = noLeader // _EMPTY_ _, err = ae.encode(nil) require_NoError(t, err) ae.leader = "DEREK123" _, err = ae.encode(nil) require_NoError(t, err) // Buffer reuse var rawSmall [32]byte var rawBigger [64]byte b := rawSmall[:] ae.encode(b) if b[0] != 0 { t.Fatalf("Expected arg buffer to not be used") } b = rawBigger[:] ae.encode(b) if b[0] == 0 { t.Fatalf("Expected arg buffer to be used") } // Test max number of entries. for i := 0; i < math.MaxUint16+1; i++ { ae.entries = append(ae.entries, &Entry{EntryNormal, nil}) } _, err = ae.encode(b) require_Error(t, err, errTooManyEntries) } func TestNRGAppendEntryDecode(t *testing.T) { ae := &appendEntry{ leader: "12345678", term: 1, pindex: 0, } for i := 0; i < math.MaxUint16; i++ { ae.entries = append(ae.entries, &Entry{EntryNormal, nil}) } buf, err := ae.encode(nil) require_NoError(t, err) // Truncate buffer first. var node *raft short := buf[0 : len(buf)-1024] _, err = node.decodeAppendEntry(short, nil, _EMPTY_) require_Error(t, err, errBadAppendEntry) for i := 0; i < 100; i++ { b := copyBytes(buf) // modifying the header (idx < 42) will not result in an error by decodeAppendEntry bi := 42 + rand.Intn(len(b)-42) if b[bi] != 0 && bi != 40 { b[bi] = 0 _, err = node.decodeAppendEntry(b, nil, _EMPTY_) require_Error(t, err, errBadAppendEntry) } } } func TestNRGRecoverFromFollowingNoLeader(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Find out what term we are on. term := rg.leader().node().Term() // Start by pausing all of the nodes. This will stop them from // processing new entries. for _, n := range rg { n.node().PauseApply() } // Now drain all of the ApplyQ entries from them, which will stop // them from automatically trying to follow a previous leader if // they happened to have received an apply entry from one. Then // we're going to force them into a state where they are all // followers but they don't have a leader. for _, n := range rg { rn := n.node().(*raft) rn.ApplyQ().drain() rn.switchToFollower(noLeader) } // Resume the nodes. for _, n := range rg { n.node().ResumeApply() } // Wait a while. The nodes should notice that they haven't heard // from a leader lately and will switch to voting. After an // election we should find a new leader and be on a new term. rg.waitOnLeader() require_True(t, rg.leader() != nil) require_NotEqual(t, rg.leader().node().Term(), term) } func TestNRGInlineStepdown(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // When StepDown() completes, we should not be the leader. Before, // this would not be guaranteed as the stepdown could be processed // some time later. n := rg.leader().node().(*raft) require_NoError(t, n.StepDown()) require_NotEqual(t, n.State(), Leader) } func TestNRGObserverMode(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Put all of the followers into observer mode. In this state // they will not participate in an election but they will continue // to apply incoming commits. for _, n := range rg { if n.node().Leader() { continue } n.node().SetObserver(true) } // Propose a change from the leader. adder := rg.leader().(*stateAdder) adder.proposeDelta(1) adder.proposeDelta(2) adder.proposeDelta(3) // Wait for the followers to apply it. rg.waitOnTotal(t, 6) // Confirm the followers are still just observers and weren't // reset out of that state for some reason. for _, n := range rg { if n.node().Leader() { continue } require_True(t, n.node().IsObserver()) } } func TestNRGAEFromOldLeader(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Listen out for catchup requests. ch := make(chan *nats.Msg, 16) _, err := nc.ChanSubscribe(fmt.Sprintf(raftCatchupReply, ">"), ch) require_NoError(t, err) // Start next term so that we can reuse term 1 in the next step. leader := rg.leader().node().(*raft) leader.StepDown() time.Sleep(time.Millisecond * 100) rg.waitOnLeader() require_Equal(t, leader.Term(), 2) leader = rg.leader().node().(*raft) // Send an append entry with an outdated term. Beforehand, doing // so would have caused a WAL reset and then would have triggered // a Raft-level catchup. ae := &appendEntry{ term: 1, pindex: 0, leader: leader.id, reply: nc.NewRespInbox(), } payload, err := ae.encode(nil) require_NoError(t, err) resp, err := nc.Request(leader.asubj, payload, time.Second) require_NoError(t, err) // Wait for the response, the server should have rejected it. ar := leader.decodeAppendEntryResponse(resp.Data) require_NotNil(t, ar) require_Equal(t, ar.success, false) // No catchup should happen at this point because no reset should // have happened. require_NoChanRead(t, ch, time.Second*2) } // TestNRGSimpleElection tests that a simple election succeeds. It is // simple because the group hasn't processed any entries and hasn't // suffered any interruptions of any kind, therefore there should be // no way that the conditions for granting the votes can fail. func TestNRGSimpleElection(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 9) defer c.shutdown() c.waitOnLeader() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 9, newStateAdder) rg.waitOnLeader() voteReqs := make(chan *nats.Msg, 1) voteResps := make(chan *nats.Msg, len(rg)-1) // Keep a record of the term when we started. leader := rg.leader().node().(*raft) startTerm := leader.term // Subscribe to the vote request subject, this should be the // same across all nodes in the group. _, err := nc.ChanSubscribe(leader.vsubj, voteReqs) require_NoError(t, err) // Subscribe to all of the vote response inboxes for all nodes // in the Raft group, as they can differ. for _, n := range rg { rn := n.node().(*raft) _, err = nc.ChanSubscribe(rn.vreply, voteResps) require_NoError(t, err) } // Step down, this will start a new voting session. require_NoError(t, rg.leader().node().StepDown()) // Wait for a vote request to come in. msg := require_ChanRead(t, voteReqs, time.Second) vr := decodeVoteRequest(msg.Data, msg.Reply) require_True(t, vr != nil) require_NotEqual(t, vr.candidate, _EMPTY_) // The leader should have bumped their term in order to start // an election. require_Equal(t, vr.term, startTerm+1) require_Equal(t, vr.lastTerm, startTerm) // Wait for all of the vote responses to come in. There should // be as many vote responses as there are followers. for i := 0; i < len(rg)-1; i++ { msg := require_ChanRead(t, voteResps, time.Second) re := decodeVoteResponse(msg.Data) require_True(t, re != nil) // Ignore old vote responses that could be in-flight. if re.term < vr.term { i-- continue } // The vote should have been granted. require_Equal(t, re.granted, true) // The node granted the vote, therefore the term in the vote // response should have advanced as well. require_Equal(t, re.term, vr.term) require_Equal(t, re.term, startTerm+1) } // Everyone in the group should have voted for our candidate // and arrived at the term from the vote request. for _, n := range rg { rn := n.node().(*raft) require_Equal(t, rn.term, vr.term) require_Equal(t, rn.term, startTerm+1) require_Equal(t, rn.vote, vr.candidate) } } func TestNRGSwitchStateClearsQueues(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.servers[0] // RunBasicJetStreamServer not available n := &raft{ prop: newIPQueue[*proposedEntry](s, "prop"), resp: newIPQueue[*appendEntryResponse](s, "resp"), leadc: make(chan bool, 1), // for switchState } n.state.Store(int32(Leader)) require_Equal(t, n.prop.len(), 0) require_Equal(t, n.resp.len(), 0) n.prop.push(&proposedEntry{}) n.resp.push(&appendEntryResponse{}) require_Equal(t, n.prop.len(), 1) require_Equal(t, n.resp.len(), 1) n.switchState(Follower) require_Equal(t, n.prop.len(), 0) require_Equal(t, n.resp.len(), 0) } func TestNRGStepDownOnSameTermDoesntClearVote(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() lsm := rg.leader().(*stateAdder) leader := lsm.node().(*raft) follower := rg.nonLeader().node().(*raft) // Make sure we handle the leader change notification from above. require_ChanRead(t, lsm.lch, time.Second) // Subscribe to the append entry subject. sub, err := nc.SubscribeSync(leader.asubj) require_NoError(t, err) // Get the first append entry that we receive. msg, err := sub.NextMsg(time.Second) require_NoError(t, err) require_NoError(t, sub.Unsubscribe()) // We're going to modify the append entry that we received so that // we can send it again with modifications. ae, err := leader.decodeAppendEntry(msg.Data, nil, msg.Reply) require_NoError(t, err) // First of all we're going to try sending an append entry that // has an old term and a fake leader. msg.Reply = follower.areply ae.leader = follower.id ae.term = leader.term - 1 msg.Data, err = ae.encode(msg.Data[:0]) require_NoError(t, err) require_NoError(t, nc.PublishMsg(msg)) // Because the term was old, the fake leader shouldn't matter as // the current leader should ignore it. require_NoChanRead(t, lsm.lch, time.Second) // Now we're going to send it on the same term that the current leader // is on. What we expect to happen is that the leader will step down // but it *shouldn't* clear the vote. ae.term = leader.term msg.Data, err = ae.encode(msg.Data[:0]) require_NoError(t, err) require_NoError(t, nc.PublishMsg(msg)) // Wait for the leader transition and ensure that the vote wasn't // cleared. require_ChanRead(t, lsm.lch, time.Second) require_NotEqual(t, leader.vote, noVote) } func TestNRGUnsuccessfulVoteRequestDoesntResetElectionTimer(t *testing.T) { // This test relies on nodes not hitting their election timer too often, // otherwise the step later where we capture the election time before and // after the failed vote request will flake. origMinTimeout, origMaxTimeout, origHBInterval := minElectionTimeout, maxElectionTimeout, hbInterval minElectionTimeout, maxElectionTimeout, hbInterval = time.Second*5, time.Second*10, time.Second*10 defer func() { minElectionTimeout, maxElectionTimeout, hbInterval = origMinTimeout, origMaxTimeout, origHBInterval }() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) // Because the election timer is quite high, we want to kick a node into // campaigning before it naturally needs to, otherwise the test takes a // long time just to pick a leader. for _, n := range rg { n.node().Campaign() break } rg.waitOnLeader() leader := rg.leader().node().(*raft) follower := rg.nonLeader().node().(*raft) // Set up a new inbox for the vote responses to go to. vsubj, vreply := leader.vsubj, nc.NewInbox() ch := make(chan *nats.Msg, 3) _, err := nc.ChanSubscribe(vreply, ch) require_NoError(t, err) // Keep a track of the last time the election timer was reset before this. // Also build up a vote request that's obviously behind so that the other // nodes should not do anything with it. All locks are taken at the same // time so that it guarantees that both the leader and the follower aren't // operating at the time we take the etlr snapshots. rg.lockAll() leader.resetElect(maxElectionTimeout) follower.resetElect(maxElectionTimeout) leaderOriginal := leader.etlr followerOriginal := follower.etlr vr := &voteRequest{ term: follower.term - 1, lastTerm: follower.term - 1, lastIndex: 0, candidate: follower.id, } rg.unlockAll() // Now send a vote request that's obviously behind. require_NoError(t, nc.PublishMsg(&nats.Msg{ Subject: vsubj, Reply: vreply, Data: vr.encode(), })) // Wait for everyone to respond. require_ChanRead(t, ch, time.Second) require_ChanRead(t, ch, time.Second) require_ChanRead(t, ch, time.Second) // Neither the leader nor our chosen follower should have updated their // election timer as a result of this. rg.lockAll() leaderEqual := leaderOriginal.Equal(leader.etlr) followerEqual := followerOriginal.Equal(follower.etlr) rg.unlockAll() require_True(t, leaderEqual) require_True(t, followerEqual) } func TestNRGUnsuccessfulVoteRequestCampaignEarly(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() nats0 := "S1Nunr6R" // "nats-0" n.etlr = time.Time{} // Simple case: we are follower and vote for a candidate. require_NoError(t, n.processVoteRequest(&voteRequest{term: 1, lastTerm: 0, lastIndex: 0, candidate: nats0})) require_Equal(t, n.term, 1) require_Equal(t, n.vote, nats0) require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it voted. n.etlr = time.Time{} // We are follower and deny vote for outdated candidate. n.pterm, n.pindex = 1, 100 require_NoError(t, n.processVoteRequest(&voteRequest{term: 2, lastTerm: 1, lastIndex: 2, candidate: nats0})) require_Equal(t, n.term, 2) require_Equal(t, n.vote, noVote) require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it starts campaigning. n.etlr = time.Time{} // Switch to candidate. n.pterm, n.pindex = 2, 200 n.switchToCandidate() require_Equal(t, n.term, 3) require_Equal(t, n.State(), Candidate) require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state. n.etlr = time.Time{} // We are candidate and deny vote for outdated candidate. But they were on a more recent term, restart campaign. require_NoError(t, n.processVoteRequest(&voteRequest{term: 4, lastTerm: 1, lastIndex: 2, candidate: nats0})) require_Equal(t, n.term, 4) require_Equal(t, n.vote, noVote) require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it restarts campaigning. n.etlr = time.Time{} // Switch to candidate. n.pterm, n.pindex = 4, 400 n.switchToCandidate() require_Equal(t, n.term, 5) require_Equal(t, n.State(), Candidate) require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state. n.etlr = time.Time{} // We are candidate and deny vote for outdated candidate. Don't start campaigning early. require_NoError(t, n.processVoteRequest(&voteRequest{term: 5, lastTerm: 1, lastIndex: 2, candidate: nats0})) require_Equal(t, n.term, 5) require_Equal(t, n.vote, noVote) // Election timer must NOT be updated as that would mean another candidate that we don't vote // for can short-circuit us by making us restart elections, denying us the ability to become leader. require_Equal(t, n.etlr, time.Time{}) } func TestNRGInvalidTAVDoesntPanic(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Mangle the TAV file to a short length (less than uint64). leader := rg.leader() tav := filepath.Join(leader.node().(*raft).sd, termVoteFile) require_NoError(t, os.WriteFile(tav, []byte{1, 2, 3, 4}, 0644)) // Restart the node. leader.stop() leader.restart() // Before the fix, a crash would have happened before this point. c.waitOnAllCurrent() } func TestNRGAssumeHighTermAfterCandidateIsolation(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Bump the term up on one of the follower nodes by a considerable // amount and force it into the candidate state. This is what happens // after a period of time in isolation. follower := rg.nonLeader().node().(*raft) follower.Lock() follower.term += 100 follower.switchState(Candidate) follower.Unlock() follower.requestVote() time.Sleep(time.Millisecond * 100) // The candidate will shortly send a vote request. When that happens, // the rest of the nodes in the cluster should move up to that term, // even though they will not grant the vote. nterm := follower.term for _, n := range rg { require_Equal(t, n.node().Term(), nterm) } // Have the leader send out a proposal, which will force the candidate // back into follower state. rg.waitOnLeader() rg.leader().(*stateAdder).proposeDelta(1) rg.waitOnTotal(t, 1) // The candidate should have switched to a follower on a term equal to // or newer than the candidate had. for _, n := range rg { require_NotEqual(t, n.node().State(), Candidate) require_True(t, n.node().Term() >= nterm) } } // Test to make sure this does not cause us to truncate our wal or enter catchup state. func TestNRGHeartbeatOnLeaderChange(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() for i := 0; i < 10; i++ { // Restart the leader. leader := rg.leader().(*stateAdder) leader.proposeDelta(22) leader.proposeDelta(-11) leader.proposeDelta(-10) // Must observe forward progress, so each iteration will check +1 total. rg.waitOnTotal(t, int64(i+1)) leader.stop() leader.restart() rg.waitOnLeader() } } func TestNRGElectionTimerAfterObserver(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() for _, n := range rg { n.node().SetObserver(true) } time.Sleep(maxElectionTimeout) before := time.Now() for _, n := range rg { n.node().SetObserver(false) } time.Sleep(maxCampaignTimeout) for _, n := range rg { rn := n.node().(*raft) rn.RLock() etlr := rn.etlr rn.RUnlock() require_True(t, etlr.After(before)) } } func TestNRGSystemClientCleanupFromAccount(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() s := c.randomServer() sacc := s.SystemAccount() numClients := func() int { sacc.mu.RLock() defer sacc.mu.RUnlock() return len(sacc.clients) } start := numClients() var all []smGroup for i := 0; i < 5; i++ { rgName := fmt.Sprintf("TEST-%d", i) rg := c.createRaftGroup(rgName, 3, newStateAdder) all = append(all, rg) rg.waitOnLeader() } for _, rg := range all { for _, sm := range rg { sm.node().Stop() } for _, sm := range rg { sm.node().WaitForStop() } } finish := numClients() require_Equal(t, start, finish) } func TestNRGLeavesObserverAfterPause(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() n := rg.nonLeader().node().(*raft) checkState := func(observer, pobserver bool) { t.Helper() n.RLock() defer n.RUnlock() require_Equal(t, n.observer, observer) require_Equal(t, n.pobserver, pobserver) } // Assume this has happened because of jetstream_cluster_migrate // or similar. n.SetObserver(true) checkState(true, false) // Now something like a catchup has started, but since we were // already in observer mode, pobserver is set to true. n.PauseApply() checkState(true, true) // Now jetstream_cluster_migrate is happy that the leafnodes are // back up so it tries to leave observer mode, but the catchup // hasn't finished yet. This will instead set pobserver to false. n.SetObserver(false) checkState(true, false) // The catchup finishes, so we should correctly leave the observer // state by setting observer to the pobserver value. n.ResumeApply() checkState(false, false) } func TestNRGCandidateDoesntRevertTermAfterOldAE(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Bump the term up a few times. for i := 0; i < 3; i++ { rg.leader().node().StepDown() time.Sleep(time.Millisecond * 50) // Needed because stepdowns not synchronous rg.waitOnLeader() } leader := rg.leader().node().(*raft) follower := rg.nonLeader().node().(*raft) // Sanity check that we are where we expect to be. require_Equal(t, leader.term, 4) require_Equal(t, follower.term, 4) // At this point the active term is 4 and pterm is 4, force the // term up to 9. This won't bump the pterm. rg.lockAll() for _, n := range rg { n.node().(*raft).term += 5 } rg.unlockAll() // Build an AE that has a term newer than the pterm but older than // the term. Give it to the follower in candidate state. ae := newAppendEntry(leader.id, 6, leader.commit, leader.pterm, leader.pindex, nil) follower.switchToCandidate() follower.processAppendEntry(ae, nil) // The candidate must not have reverted back to term 6. require_NotEqual(t, follower.term, 6) } func TestNRGTermDoesntRollBackToPtermOnCatchup(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() // Propose some entries so that we have entries in the log that have pterm 1. lsm := rg.leader().(*stateAdder) for i := 0; i < 5; i++ { lsm.proposeDelta(1) rg.waitOnTotal(t, int64(i)+1) } // Check that everyone is where they are supposed to be. rg.lockAll() for _, n := range rg { rn := n.node().(*raft) require_Equal(t, rn.term, 1) require_Equal(t, rn.pterm, 1) require_Equal(t, rn.pindex, 6) } rg.unlockAll() // Force a stepdown so that we move up to term 2. rg.leader().node().(*raft).switchToFollower(noLeader) rg.waitOnLeader() leader := rg.leader().node().(*raft) // Now make sure everyone has moved up to term 2. Additionally we're // going to prune back the follower logs to term 1 as this is what will // create the right conditions for the catchup. rg.lockAll() for _, n := range rg { rn := n.node().(*raft) require_Equal(t, rn.term, 2) if !rn.Leader() { rn.truncateWAL(1, 6) require_Equal(t, rn.term, 2) // rn.term must stay the same require_Equal(t, rn.pterm, 1) require_Equal(t, rn.pindex, 6) } } // This will make followers run a catchup. ae := newAppendEntry(leader.id, leader.term, leader.commit, leader.pterm, leader.pindex, nil) rg.unlockAll() arInbox := nc.NewRespInbox() arCh := make(chan *nats.Msg, 2) _, err := nc.ChanSubscribe(arInbox, arCh) require_NoError(t, err) // In order to trip this condition, we need to send an append entry that // will trick the followers into running a catchup. In the process they // were setting the term back to pterm which is incorrect. b, err := ae.encode(nil) require_NoError(t, err) require_NoError(t, nc.PublishMsg(&nats.Msg{ Subject: fmt.Sprintf(raftAppendSubj, "TEST"), Reply: arInbox, Data: b, })) // Wait for both followers to respond to the append entry and then verify // that none of the nodes should have reverted back to term 1. require_ChanRead(t, arCh, time.Second*5) // First follower require_ChanRead(t, arCh, time.Second*5) // Second follower for _, n := range rg { require_NotEqual(t, n.node().Term(), 1) } } func TestNRGNoResetOnAppendEntryResponse(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() c.waitOnAllCurrent() leader := rg.leader().node().(*raft) follower := rg.nonLeader().node().(*raft) lsm := rg.leader().(*stateAdder) // Subscribe for append entries that aren't heartbeats and respond to // each of them as though it's a non-success and with a higher term. // The higher term in this case is what would cause the leader previously // to reset the entire log which it shouldn't do. _, err := nc.Subscribe(fmt.Sprintf(raftAppendSubj, "TEST"), func(msg *nats.Msg) { if ae, err := follower.decodeAppendEntry(msg.Data, nil, msg.Reply); err == nil && len(ae.entries) > 0 { ar := newAppendEntryResponse(ae.term+1, ae.commit, follower.id, false) require_NoError(t, msg.Respond(ar.encode(nil))) } }) require_NoError(t, err) // Generate an append entry that the subscriber above can respond to. c.waitOnAllCurrent() lsm.proposeDelta(5) rg.waitOnTotal(t, 5) // The was-leader should now have stepped down, make sure that it // didn't blow away its log in the process. rg.lockAll() defer rg.unlockAll() require_Equal(t, leader.State(), Follower) require_NotEqual(t, leader.pterm, 0) require_NotEqual(t, leader.pindex, 0) } func TestNRGCandidateDontStepdownDueToLeaderOfPreviousTerm(t *testing.T) { // This test relies on nodes not hitting their election timer too often. origMinTimeout, origMaxTimeout, origHBInterval := minElectionTimeout, maxElectionTimeout, hbInterval minElectionTimeout, maxElectionTimeout, hbInterval = time.Second*5, time.Second*10, time.Second defer func() { minElectionTimeout, maxElectionTimeout, hbInterval = origMinTimeout, origMaxTimeout, origHBInterval }() c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() c.waitOnLeader() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() var ( candidatePterm uint64 = 50 candidatePindex uint64 = 70 candidateTerm uint64 = 100 ) // Create a candidate that has received entries while they were a follower in a previous term candidate := rg.nonLeader().node().(*raft) candidate.Lock() candidate.switchState(Candidate) candidate.pterm = candidatePterm candidate.pindex = candidatePindex candidate.term = candidateTerm candidate.Unlock() // Leader term is behind candidate leader := rg.leader().node().(*raft) leader.Lock() leader.term = candidatePterm leader.pterm = candidatePterm leader.pindex = candidatePindex leader.Unlock() // Subscribe to the append entry subject. sub, err := nc.SubscribeSync(leader.asubj) require_NoError(t, err) // Get the first append entry that we receive, should be heartbeat from leader of prev term msg, err := sub.NextMsg(5 * time.Second) require_NoError(t, err) // Stop nodes from progressing so we can check state rg.lockAll() defer rg.unlockAll() // Decode the append entry ae, err := leader.decodeAppendEntry(msg.Data, nil, msg.Reply) require_NoError(t, err) // Check that the append entry is from the leader require_Equal(t, ae.leader, leader.id) // Check that it came from the leader before it updated its term with the response from the candidate require_Equal(t, ae.term, candidatePterm) // Check that the candidate hasn't stepped down require_Equal(t, candidate.State(), Candidate) // Check that the candidate's term is still ahead of the leader's term require_True(t, candidate.term > ae.term) } func TestNRGRemoveLeaderPeerDeadlockBug(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() n := rg.leader().node().(*raft) leader := n.ID() // Propose to remove the leader as a peer. Will lead to a deadlock with bug. require_NoError(t, n.ProposeRemovePeer(leader)) rg.waitOnLeader() checkFor(t, 10*time.Second, 200*time.Millisecond, func() error { nl := n.GroupLeader() if nl != leader { return nil } return errors.New("Leader has not moved") }) } func TestNRGWALEntryWithoutQuorumMustTruncate(t *testing.T) { tests := []struct { title string modify func(rg smGroup) }{ { // state equals, only need to remove the entry title: "equal", modify: func(rg smGroup) {}, }, { // state diverged, need to replace the entry title: "diverged", modify: func(rg smGroup) { rg.leader().(*stateAdder).proposeDelta(11) }, }, } for _, test := range tests { t.Run(test.title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() var err error var scratch [1024]byte // Simulate leader storing an AppendEntry in WAL but being hard killed before it can propose to its peers. n := rg.leader().node().(*raft) esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} n.Lock() ae := n.buildAppendEntry(entries) ae.buf, err = ae.encode(scratch[:]) require_NoError(t, err) err = n.storeToWAL(ae) n.Unlock() require_NoError(t, err) // Stop the leader so it moves to another one. n.shutdown() // Wait for another leader to be picked rg.waitOnLeader() // Make a modification, specific to this test. test.modify(rg) // Restart the previous leader that contains the stored AppendEntry without quorum. for _, a := range rg { if a.node().ID() == n.ID() { sa := a.(*stateAdder) sa.restart() break } } // The previous leader's WAL should truncate to remove the AppendEntry only it has. // Eventually all WALs for all peers must match. checkFor(t, 5*time.Second, 200*time.Millisecond, func() error { var expected [][]byte for _, a := range rg { an := a.node().(*raft) var state StreamState an.wal.FastState(&state) if len(expected) > 0 && int(state.LastSeq-state.FirstSeq+1) != len(expected) { return fmt.Errorf("WAL is different: too many entries") } // Loop over all entries in the WAL, checking if the contents for all RAFT nodes are equal. for index := state.FirstSeq; index <= state.LastSeq; index++ { ae, err := an.loadEntry(index) if err != nil { return err } seq := int(index) if len(expected) < seq { expected = append(expected, ae.buf) } else if !bytes.Equal(expected[seq-1], ae.buf) { return fmt.Errorf("WAL is different: stored bytes differ") } } } return nil }) }) } } func TestNRGTermNoDecreaseAfterWALReset(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() l := rg.leader().node().(*raft) l.Lock() l.term = 20 l.Unlock() esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} l.Lock() ae := l.buildAppendEntry(entries) l.Unlock() for _, f := range rg { if f.node().ID() != l.ID() { fn := f.node().(*raft) fn.processAppendEntry(ae, fn.aesub) require_Equal(t, fn.term, 20) // Follower's term gets upped as expected. } } // Lower the term, simulating the followers receiving a message from an old term/leader. ae.term = 3 for _, f := range rg { if f.node().ID() != l.ID() { fn := f.node().(*raft) fn.processAppendEntry(ae, fn.aesub) require_Equal(t, fn.term, 20) // Follower should reject and the term stays the same. fn.Lock() fn.resetWAL() fn.Unlock() fn.processAppendEntry(ae, fn.aesub) require_Equal(t, fn.term, 20) // Follower should reject again, even after reset, term stays the same. } } } func TestNRGPendingAppendEntryCacheInvalidation(t *testing.T) { for _, test := range []struct { title string entries int }{ {title: "empty", entries: 1}, {title: "at limit", entries: paeDropThreshold}, {title: "full", entries: paeDropThreshold + 1}, } { t.Run(test.title, func(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() rg := c.createMemRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() l := rg.leader() l.(*stateAdder).proposeDelta(1) rg.waitOnTotal(t, 1) // Fill up the cache with N entries. // The contents don't matter as they should never be applied. rg.lockAll() for _, s := range rg { n := s.node().(*raft) for i := 0; i < test.entries; i++ { n.pae[n.pindex+uint64(1+i)] = newAppendEntry("", 0, 0, 0, 0, nil) } } rg.unlockAll() l.(*stateAdder).proposeDelta(1) rg.waitOnTotal(t, 2) }) } } func TestNRGCatchupDoesNotTruncateUncommittedEntriesWithQuorum(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" nats1 := "yrzKKRBu" // "nats-1" // Timeline, for first leader aeInitial := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) aeUncommitted := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeNoQuorum := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries}) // Timeline, after leader change aeMissed := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 2, entries: entries}) aeCatchupTrigger := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 3, entries: entries}) aeHeartbeat2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 4, entries: nil}) aeHeartbeat3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 4, pterm: 2, pindex: 4, entries: nil}) // Initial case is simple, just store the entry. n.processAppendEntry(aeInitial, n.aesub) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Heartbeat, makes sure commit moves up. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 1) // We get one entry that has quorum (but we don't know that yet), so it stays uncommitted for a bit. n.processAppendEntry(aeUncommitted, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We get one entry that has NO quorum (but we don't know that yet). n.processAppendEntry(aeNoQuorum, n.aesub) require_Equal(t, n.wal.State().Msgs, 3) entry, err = n.loadEntry(3) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We've just had a leader election, and we missed one message from the previous leader. // We should truncate the last message. n.processAppendEntry(aeCatchupTrigger, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) require_True(t, n.catchup == nil) // We get a heartbeat that prompts us to catchup. n.processAppendEntry(aeHeartbeat2, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) require_Equal(t, n.commit, 1) // Commit should not change, as we missed an item. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 1) // n.pterm require_Equal(t, n.catchup.pindex, 2) // n.pindex // We now notice the leader indicated a different entry at the (no quorum) index, should save that. n.processAppendEntry(aeMissed, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 3) require_True(t, n.catchup != nil) // We now get the entry that initially triggered us to catchup, it should be added. n.processAppendEntry(aeCatchupTrigger, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 4) require_True(t, n.catchup != nil) entry, err = n.loadEntry(4) require_NoError(t, err) require_Equal(t, entry.leader, nats1) // Heartbeat, makes sure we commit (and reset catchup, as we're now up-to-date). n.processAppendEntry(aeHeartbeat3, n.aesub) require_Equal(t, n.commit, 4) require_True(t, n.catchup == nil) } func TestNRGCatchupCanTruncateMultipleEntriesWithoutQuorum(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" nats1 := "yrzKKRBu" // "nats-1" // Timeline, for first leader aeInitial := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) aeNoQuorum1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeNoQuorum2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries}) // Timeline, after leader change aeMissed1 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeMissed2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 2, entries: entries}) aeCatchupTrigger := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 3, entries: entries}) aeHeartbeat2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 4, entries: nil}) aeHeartbeat3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 4, pterm: 2, pindex: 4, entries: nil}) // Initial case is simple, just store the entry. n.processAppendEntry(aeInitial, n.aesub) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Heartbeat, makes sure commit moves up. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 1) // We get one entry that has NO quorum (but we don't know that yet). n.processAppendEntry(aeNoQuorum1, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We get another entry that has NO quorum (but we don't know that yet). n.processAppendEntry(aeNoQuorum2, n.aesub) require_Equal(t, n.wal.State().Msgs, 3) entry, err = n.loadEntry(3) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We've just had a leader election, and we missed messages from the previous leader. // We should truncate the last message. n.processAppendEntry(aeCatchupTrigger, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) require_True(t, n.catchup == nil) // We get a heartbeat that prompts us to catchup. n.processAppendEntry(aeHeartbeat2, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) require_Equal(t, n.commit, 1) // Commit should not change, as we missed an item. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 1) // n.pterm require_Equal(t, n.catchup.pindex, 2) // n.pindex // We now notice the leader indicated a different entry at the (no quorum) index. We should truncate again. n.processAppendEntry(aeMissed2, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 1) require_True(t, n.catchup == nil) // We get a heartbeat that prompts us to catchup. n.processAppendEntry(aeHeartbeat2, n.aesub) require_Equal(t, n.wal.State().Msgs, 1) require_Equal(t, n.commit, 1) // Commit should not change, as we missed an item. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 1) // n.pterm require_Equal(t, n.catchup.pindex, 1) // n.pindex // We now get caught up with the missed messages. n.processAppendEntry(aeMissed1, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 2) require_True(t, n.catchup != nil) n.processAppendEntry(aeMissed2, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 3) require_True(t, n.catchup != nil) // We now get the entry that initially triggered us to catchup, it should be added. n.processAppendEntry(aeCatchupTrigger, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 4) require_True(t, n.catchup != nil) entry, err = n.loadEntry(4) require_NoError(t, err) require_Equal(t, entry.leader, nats1) // Heartbeat, makes sure we commit (and reset catchup, as we're now up-to-date). n.processAppendEntry(aeHeartbeat3, n.aesub) require_Equal(t, n.commit, 4) require_True(t, n.catchup == nil) } func TestNRGCatchupDoesNotTruncateCommittedEntriesDuringRedelivery(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil}) aeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries}) aeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil}) // Initial case is simple, just store the entry. n.processAppendEntry(aeMsg1, n.aesub) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Deliver a message. n.processAppendEntry(aeMsg2, n.aesub) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Heartbeat, makes sure commit moves up. n.processAppendEntry(aeHeartbeat1, n.aesub) require_Equal(t, n.commit, 2) // Deliver another message. n.processAppendEntry(aeMsg3, n.aesub) require_Equal(t, n.wal.State().Msgs, 3) entry, err = n.loadEntry(3) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Simulate receiving an old entry as a redelivery. We should not truncate as that lowers our commit. n.processAppendEntry(aeMsg1, n.aesub) require_Equal(t, n.commit, 2) // Heartbeat, makes sure we commit. n.processAppendEntry(aeHeartbeat2, n.aesub) require_Equal(t, n.commit, 3) } func TestNRGCatchupFromNewLeaderWithIncorrectPterm(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 0, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) // Heartbeat, triggers catchup. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 0) // Commit should not change, as we missed an item. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 0) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex // First catchup message has the incorrect pterm, stop catchup and re-trigger later with the correct pterm. n.processAppendEntry(aeMsg, n.catchup.sub) require_True(t, n.catchup == nil) require_Equal(t, n.pterm, 1) require_Equal(t, n.pindex, 0) // Heartbeat, triggers catchup. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 0) // Commit should not change, as we missed an item. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 1) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex // Now we get the message again and can continue to store it. n.processAppendEntry(aeMsg, n.catchup.sub) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Now heartbeat is able to commit the entry. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 1) } func TestNRGDontRemoveSnapshotIfTruncateToApplied(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) // Initial case is simple, just store the entry. n.processAppendEntry(aeMsg, n.aesub) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Heartbeat, makes sure commit moves up. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 1) require_Equal(t, n.pterm, 1) // Simulate upper layer calling down to apply. n.Applied(1) // Install snapshot and check it exists. err = n.InstallSnapshot(nil) require_NoError(t, err) snapshots := path.Join(n.sd, snapshotsDir) files, err := os.ReadDir(snapshots) require_NoError(t, err) require_Equal(t, len(files), 1) // Truncate and check snapshot is kept. n.truncateWAL(n.pterm, n.applied) files, err = os.ReadDir(snapshots) require_NoError(t, err) require_Equal(t, len(files), 1) } func TestNRGDontSwitchToCandidateWithInflightSnapshot(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample snapshot entry, the content doesn't matter. snapshotEntries := []*Entry{ newEntry(EntrySnapshot, nil), newEntry(EntryPeerState, encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})), } nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeTriggerCatchup := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) aeCatchupSnapshot := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: snapshotEntries}) // Switch follower into catchup. n.processAppendEntry(aeTriggerCatchup, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 0) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex // Follower receives a snapshot, marking a snapshot as inflight as the apply queue is async. n.processAppendEntry(aeCatchupSnapshot, n.catchup.sub) require_Equal(t, n.pindex, 1) require_Equal(t, n.commit, 1) // Try to switch to candidate, it should be blocked since the snapshot is not processed yet. n.switchToCandidate() require_Equal(t, n.State(), Follower) // Simulate snapshot being processed by the upper layer. n.Applied(1) // Retry becoming candidate, snapshot is processed so can now do so. n.switchToCandidate() require_Equal(t, n.State(), Candidate) } func TestNRGDontSwitchToCandidateWithMultipleInflightSnapshots(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample snapshot entry, the content doesn't matter. snapshotEntries := []*Entry{ newEntry(EntrySnapshot, nil), newEntry(EntryPeerState, encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})), } nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeSnapshot1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: snapshotEntries}) aeSnapshot2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: snapshotEntries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil}) // Simulate snapshots being sent to us. n.processAppendEntry(aeSnapshot1, n.aesub) require_Equal(t, n.pindex, 1) require_Equal(t, n.commit, 0) require_Equal(t, n.applied, 0) n.processAppendEntry(aeSnapshot2, n.aesub) require_Equal(t, n.pindex, 2) require_Equal(t, n.commit, 1) require_Equal(t, n.applied, 0) n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.pindex, 2) require_Equal(t, n.commit, 2) require_Equal(t, n.applied, 0) for i := uint64(1); i <= 2; i++ { // Try to switch to candidate, it should be blocked since the snapshot is not processed yet. n.switchToCandidate() require_Equal(t, n.State(), Follower) // Simulate snapshot being processed by the upper layer. n.Applied(i) } // Retry becoming candidate, all snapshots processed so can now do so. n.switchToCandidate() require_Equal(t, n.State(), Candidate) } func TestNRGRecoverPindexPtermOnlyIfLogNotEmpty(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() gn := rg[0].(*stateAdder) rn := rg[0].node().(*raft) // Delete the msgs and snapshots, leaving the only remaining trace // of the term in the TAV file. store := filepath.Join(gn.cfg.Store) require_NoError(t, rn.wal.Truncate(0)) require_NoError(t, os.RemoveAll(filepath.Join(store, "msgs"))) require_NoError(t, os.RemoveAll(filepath.Join(store, "snapshots"))) for _, gn := range rg { gn.stop() } rg[0].restart() rn = rg[0].node().(*raft) // Both should be zero as, without any snapshots or log entries, // the log is considered empty and therefore we account as such. require_Equal(t, rn.pterm, 0) require_Equal(t, rn.pindex, 0) } func TestNRGCancelCatchupWhenDetectingHigherTermDuringVoteRequest(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline. aeCatchupTrigger := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) // Truncate to simulate we missed one message and need to catchup. n.processAppendEntry(aeCatchupTrigger, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 0) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex // Process first message as part of the catchup. catchupSub := n.catchup.sub n.processAppendEntry(aeMsg1, catchupSub) require_True(t, n.catchup != nil) // Receiving a vote request should cancel our catchup. // Otherwise, we could receive catchup messages after this that provides the previous leader with quorum. // If the new leader doesn't have these entries, the previous leader would desync since it would commit them. err := n.processVoteRequest(&voteRequest{2, 1, 1, nats0, "reply"}) require_NoError(t, err) require_True(t, n.catchup == nil) } func TestNRGMultipleStopsDontPanic(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() defer func() { p := recover() require_True(t, p == nil) }() for i := 0; i < 10; i++ { n.Stop() } } func TestNRGTruncateDownToCommitted(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" nats1 := "yrzKKRBu" // "nats-1" // Timeline, we are leader aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) // Timeline, after leader change aeMsg3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 0, pterm: 1, pindex: 1, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 2, entries: nil}) // Simply receive first message. n.processAppendEntry(aeMsg1, n.aesub) require_Equal(t, n.commit, 0) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Receive second message, which commits the first message. n.processAppendEntry(aeMsg2, n.aesub) require_Equal(t, n.commit, 1) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We receive an entry from another leader, should truncate down to commit / remove the second message. // After doing so, we should also be able to immediately store the message after. n.processAppendEntry(aeMsg3, n.aesub) require_Equal(t, n.commit, 1) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats1) // Heartbeat moves commit up. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 2) } type mockWALTruncateAlwaysFails struct { WAL } func (m mockWALTruncateAlwaysFails) Truncate(seq uint64) error { return errors.New("test: truncate always fails") } func TestNRGTruncateDownToCommittedWhenTruncateFails(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() n.Lock() n.wal = mockWALTruncateAlwaysFails{n.wal} n.Unlock() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" nats1 := "yrzKKRBu" // "nats-1" // Timeline, we are leader aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) // Timeline, after leader change aeMsg3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 0, pterm: 1, pindex: 1, entries: entries}) // Simply receive first message. n.processAppendEntry(aeMsg1, n.aesub) require_Equal(t, n.commit, 0) require_Equal(t, n.wal.State().Msgs, 1) entry, err := n.loadEntry(1) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // Receive second message, which commits the first message. n.processAppendEntry(aeMsg2, n.aesub) require_Equal(t, n.commit, 1) require_Equal(t, n.wal.State().Msgs, 2) entry, err = n.loadEntry(2) require_NoError(t, err) require_Equal(t, entry.leader, nats0) // We receive an entry from another leader, should truncate down to commit / remove the second message. // But, truncation fails so should register that and not change pindex/pterm. bindex, bterm := n.pindex, n.pterm n.processAppendEntry(aeMsg3, n.aesub) require_Error(t, n.werr, errors.New("test: truncate always fails")) require_Equal(t, bindex, n.pindex) require_Equal(t, bterm, n.pterm) } func TestNRGForwardProposalResponse(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() nc, _ := jsClientConnect(t, c.leader(), nats.UserInfo("admin", "s3cr3t!")) defer nc.Close() rg := c.createRaftGroup("TEST", 3, newStateAdder) rg.waitOnLeader() n := rg.nonLeader().node().(*raft) psubj := n.psubj data := make([]byte, binary.MaxVarintLen64) dn := binary.PutVarint(data, int64(123)) _, err := nc.Request(psubj, data[:dn], time.Second*5) require_NoError(t, err) rg.waitOnTotal(t, 123) } func TestNRGMemoryWALEmptiesSnapshotsDir(t *testing.T) { n, c := initSingleMemRaftNodeWithCluster(t) defer c.shutdown() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline aeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil}) // Simply receive first message. n.processAppendEntry(aeMsg, n.aesub) require_Equal(t, n.pindex, 1) require_Equal(t, n.commit, 0) // Heartbeat moves commit up. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.commit, 1) // Manually call back down to applied, and then snapshot. n.Applied(1) err := n.InstallSnapshot(nil) require_NoError(t, err) // Stop current node and restart it. n.Stop() n.WaitForStop() s := c.servers[0] ms, err := newMemStore(&StreamConfig{Name: "TEST", Storage: MemoryStorage}) require_NoError(t, err) cfg := &RaftConfig{Name: "TEST", Store: n.sd, Log: ms} n, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{}) require_NoError(t, err) // Since the WAL is in-memory, the snapshots dir should've been emptied upon restart. files, err := os.ReadDir(filepath.Join(n.sd, snapshotsDir)) require_NoError(t, err) require_Len(t, len(files), 0) } func TestNRGHealthCheckWaitForCatchup(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil}) // Switch follower into catchup. n.processAppendEntry(aeHeartbeat, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 0) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex require_Equal(t, n.catchup.cterm, aeHeartbeat.term) require_Equal(t, n.catchup.cindex, aeHeartbeat.pindex) // Catchup first message. n.processAppendEntry(aeMsg1, n.catchup.sub) require_Equal(t, n.pindex, 1) require_False(t, n.Healthy()) // Catchup second message. n.processAppendEntry(aeMsg2, n.catchup.sub) require_Equal(t, n.pindex, 2) require_Equal(t, n.commit, 1) require_False(t, n.Healthy()) // If we apply the entry sooner than we receive the next catchup message, // should not mark as healthy since we're still in catchup. n.Applied(1) require_False(t, n.Healthy()) // Catchup third message. n.processAppendEntry(aeMsg3, n.catchup.sub) require_Equal(t, n.pindex, 3) require_Equal(t, n.commit, 2) n.Applied(2) require_False(t, n.Healthy()) // Heartbeat stops catchup. n.processAppendEntry(aeHeartbeat, n.aesub) require_True(t, n.catchup == nil) require_Equal(t, n.pindex, 3) require_Equal(t, n.commit, 3) require_False(t, n.Healthy()) // Still need to wait for the last entry to be applied. n.Applied(3) require_True(t, n.Healthy()) } func TestNRGHealthCheckWaitForDoubleCatchup(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil}) aeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries}) aeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil}) // Switch follower into catchup. n.processAppendEntry(aeHeartbeat1, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 0) // n.pterm require_Equal(t, n.catchup.pindex, 0) // n.pindex require_Equal(t, n.catchup.cterm, aeHeartbeat1.term) require_Equal(t, n.catchup.cindex, aeHeartbeat1.pindex) // Catchup first message. n.processAppendEntry(aeMsg1, n.catchup.sub) require_Equal(t, n.pindex, 1) require_False(t, n.Healthy()) // We miss this message, since we're catching up. n.processAppendEntry(aeMsg3, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.pindex, 1) require_False(t, n.Healthy()) // We also miss the heartbeat, since we're catching up. n.processAppendEntry(aeHeartbeat2, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.pindex, 1) require_False(t, n.Healthy()) // Catchup second message, this will stop catchup. n.processAppendEntry(aeMsg2, n.catchup.sub) require_Equal(t, n.pindex, 2) require_Equal(t, n.commit, 1) n.Applied(1) require_False(t, n.Healthy()) // We expect to still be in catchup, waiting for a heartbeat or new append entry to reset. require_True(t, n.catchup != nil) require_Equal(t, n.catchup.cterm, aeHeartbeat1.term) require_Equal(t, n.catchup.cindex, aeHeartbeat1.pindex) // We now get a 'future' heartbeat, should restart catchup. n.processAppendEntry(aeHeartbeat2, n.aesub) require_True(t, n.catchup != nil) require_Equal(t, n.catchup.pterm, 1) // n.pterm require_Equal(t, n.catchup.pindex, 2) // n.pindex require_Equal(t, n.catchup.cterm, aeHeartbeat2.term) require_Equal(t, n.catchup.cindex, aeHeartbeat2.pindex) require_False(t, n.Healthy()) // Catchup third message. n.processAppendEntry(aeMsg3, n.catchup.sub) require_Equal(t, n.pindex, 3) require_Equal(t, n.commit, 2) n.Applied(2) require_False(t, n.Healthy()) // Heartbeat stops catchup. n.processAppendEntry(aeHeartbeat2, n.aesub) require_True(t, n.catchup == nil) require_Equal(t, n.pindex, 3) require_Equal(t, n.commit, 3) require_False(t, n.Healthy()) // Still need to wait for the last entry to be applied. n.Applied(3) require_True(t, n.Healthy()) } func TestNRGHealthCheckWaitForPendingCommitsWhenPaused(t *testing.T) { n, cleanup := initSingleMemRaftNode(t) defer cleanup() // Create a sample entry, the content doesn't matter, just that it's stored. esm := encodeStreamMsgAllowCompress("foo", "_INBOX.foo", nil, nil, 0, 0, true) entries := []*Entry{newEntry(EntryNormal, esm)} nats0 := "S1Nunr6R" // "nats-0" // Timeline aeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries}) aeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries}) aeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil}) // Process first message. n.processAppendEntry(aeMsg1, n.aesub) require_Equal(t, n.pindex, 1) require_False(t, n.Healthy()) // Process second message, moves commit up. n.processAppendEntry(aeMsg2, n.aesub) require_Equal(t, n.pindex, 2) require_False(t, n.Healthy()) // We're healthy once we've applied the first message. n.Applied(1) require_True(t, n.Healthy()) // If we're paused we still are healthy if there are no pending commits. err := n.PauseApply() require_NoError(t, err) require_True(t, n.Healthy()) // Heartbeat marks second message to be committed. n.processAppendEntry(aeHeartbeat, n.aesub) require_Equal(t, n.pindex, 2) require_False(t, n.Healthy()) // Resuming apply commits the message. n.ResumeApply() require_NoError(t, err) require_False(t, n.Healthy()) // But still waiting for it to be applied before marking healthy. n.Applied(2) require_True(t, n.Healthy()) } nats-server-2.10.27/server/rate_counter.go000066400000000000000000000024041477524627100205020ustar00rootroot00000000000000// Copyright 2021-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "sync" "time" ) type rateCounter struct { limit int64 count int64 blocked uint64 end time.Time interval time.Duration mu sync.Mutex } func newRateCounter(limit int64) *rateCounter { return &rateCounter{ limit: limit, interval: time.Second, } } func (r *rateCounter) allow() bool { now := time.Now() r.mu.Lock() if now.After(r.end) { r.count = 0 r.end = now.Add(r.interval) } else { r.count++ } allow := r.count < r.limit if !allow { r.blocked++ } r.mu.Unlock() return allow } func (r *rateCounter) countBlocked() uint64 { r.mu.Lock() blocked := r.blocked r.blocked = 0 r.mu.Unlock() return blocked } nats-server-2.10.27/server/rate_counter_test.go000066400000000000000000000024651477524627100215500ustar00rootroot00000000000000// Copyright 2021-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "testing" "time" ) func TestRateCounter(t *testing.T) { counter := newRateCounter(10) counter.interval = 100 * time.Millisecond var i int for i = 0; i < 10; i++ { if !counter.allow() { t.Errorf("counter should allow (iteration %d)", i) } } for i = 0; i < 5; i++ { if counter.allow() { t.Errorf("counter should not allow (iteration %d)", i) } } blocked := counter.countBlocked() if blocked != 5 { t.Errorf("Expected blocked = 5, got %d", blocked) } blocked = counter.countBlocked() if blocked != 0 { t.Errorf("Expected blocked = 0, got %d", blocked) } time.Sleep(150 * time.Millisecond) if !counter.allow() { t.Errorf("Expected true after current time window expired") } } nats-server-2.10.27/server/reload.go000066400000000000000000002262271477524627100172710ustar00rootroot00000000000000// Copyright 2017-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "cmp" "crypto/tls" "errors" "fmt" "net/url" "reflect" "slices" "strings" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nuid" ) // FlagSnapshot captures the server options as specified by CLI flags at // startup. This should not be modified once the server has started. var FlagSnapshot *Options type reloadContext struct { oldClusterPerms *RoutePermissions } // option is a hot-swappable configuration setting. type option interface { // Apply the server option. Apply(server *Server) // IsLoggingChange indicates if this option requires reloading the logger. IsLoggingChange() bool // IsTraceLevelChange indicates if this option requires reloading cached trace level. // Clients store trace level separately. IsTraceLevelChange() bool // IsAuthChange indicates if this option requires reloading authorization. IsAuthChange() bool // IsTLSChange indicates if this option requires reloading TLS. IsTLSChange() bool // IsClusterPermsChange indicates if this option requires reloading // cluster permissions. IsClusterPermsChange() bool // IsClusterPoolSizeOrAccountsChange indicates if this option requires // special handling for changes in cluster's pool size or accounts list. IsClusterPoolSizeOrAccountsChange() bool // IsJetStreamChange inidicates a change in the servers config for JetStream. // Account changes will be handled separately in reloadAuthorization. IsJetStreamChange() bool // Indicates a change in the server that requires publishing the server's statz IsStatszChange() bool } // noopOption is a base struct that provides default no-op behaviors. type noopOption struct{} func (n noopOption) IsLoggingChange() bool { return false } func (n noopOption) IsTraceLevelChange() bool { return false } func (n noopOption) IsAuthChange() bool { return false } func (n noopOption) IsTLSChange() bool { return false } func (n noopOption) IsClusterPermsChange() bool { return false } func (n noopOption) IsClusterPoolSizeOrAccountsChange() bool { return false } func (n noopOption) IsJetStreamChange() bool { return false } func (n noopOption) IsStatszChange() bool { return false } // loggingOption is a base struct that provides default option behaviors for // logging-related options. type loggingOption struct { noopOption } func (l loggingOption) IsLoggingChange() bool { return true } // traceLevelOption is a base struct that provides default option behaviors for // tracelevel-related options. type traceLevelOption struct { loggingOption } func (l traceLevelOption) IsTraceLevelChange() bool { return true } // traceOption implements the option interface for the `trace` setting. type traceOption struct { traceLevelOption newValue bool } // Apply is a no-op because logging will be reloaded after options are applied. func (t *traceOption) Apply(server *Server) { server.Noticef("Reloaded: trace = %v", t.newValue) } // traceOption implements the option interface for the `trace` setting. type traceVerboseOption struct { traceLevelOption newValue bool } // Apply is a no-op because logging will be reloaded after options are applied. func (t *traceVerboseOption) Apply(server *Server) { server.Noticef("Reloaded: trace_verbose = %v", t.newValue) } // debugOption implements the option interface for the `debug` setting. type debugOption struct { loggingOption newValue bool } // Apply is mostly a no-op because logging will be reloaded after options are applied. // However we will kick the raft nodes if they exist to reload. func (d *debugOption) Apply(server *Server) { server.Noticef("Reloaded: debug = %v", d.newValue) server.reloadDebugRaftNodes(d.newValue) } // logtimeOption implements the option interface for the `logtime` setting. type logtimeOption struct { loggingOption newValue bool } // Apply is a no-op because logging will be reloaded after options are applied. func (l *logtimeOption) Apply(server *Server) { server.Noticef("Reloaded: logtime = %v", l.newValue) } // logtimeUTCOption implements the option interface for the `logtime_utc` setting. type logtimeUTCOption struct { loggingOption newValue bool } // Apply is a no-op because logging will be reloaded after options are applied. func (l *logtimeUTCOption) Apply(server *Server) { server.Noticef("Reloaded: logtime_utc = %v", l.newValue) } // logfileOption implements the option interface for the `log_file` setting. type logfileOption struct { loggingOption newValue string } // Apply is a no-op because logging will be reloaded after options are applied. func (l *logfileOption) Apply(server *Server) { server.Noticef("Reloaded: log_file = %v", l.newValue) } // syslogOption implements the option interface for the `syslog` setting. type syslogOption struct { loggingOption newValue bool } // Apply is a no-op because logging will be reloaded after options are applied. func (s *syslogOption) Apply(server *Server) { server.Noticef("Reloaded: syslog = %v", s.newValue) } // remoteSyslogOption implements the option interface for the `remote_syslog` // setting. type remoteSyslogOption struct { loggingOption newValue string } // Apply is a no-op because logging will be reloaded after options are applied. func (r *remoteSyslogOption) Apply(server *Server) { server.Noticef("Reloaded: remote_syslog = %v", r.newValue) } // tlsOption implements the option interface for the `tls` setting. type tlsOption struct { noopOption newValue *tls.Config } // Apply the tls change. func (t *tlsOption) Apply(server *Server) { server.mu.Lock() tlsRequired := t.newValue != nil server.info.TLSRequired = tlsRequired && !server.getOpts().AllowNonTLS message := "disabled" if tlsRequired { server.info.TLSVerify = (t.newValue.ClientAuth == tls.RequireAndVerifyClientCert) message = "enabled" } server.mu.Unlock() server.Noticef("Reloaded: tls = %s", message) } func (t *tlsOption) IsTLSChange() bool { return true } // tlsTimeoutOption implements the option interface for the tls `timeout` // setting. type tlsTimeoutOption struct { noopOption newValue float64 } // Apply is a no-op because the timeout will be reloaded after options are // applied. func (t *tlsTimeoutOption) Apply(server *Server) { server.Noticef("Reloaded: tls timeout = %v", t.newValue) } // tlsPinnedCertOption implements the option interface for the tls `pinned_certs` setting. type tlsPinnedCertOption struct { noopOption newValue PinnedCertSet } // Apply is a no-op because the pinned certs will be reloaded after options are applied. func (t *tlsPinnedCertOption) Apply(server *Server) { server.Noticef("Reloaded: %d pinned_certs", len(t.newValue)) } // tlsHandshakeFirst implements the option interface for the tls `handshake first` setting. type tlsHandshakeFirst struct { noopOption newValue bool } // Apply is a no-op because the timeout will be reloaded after options are applied. func (t *tlsHandshakeFirst) Apply(server *Server) { server.Noticef("Reloaded: Client TLS handshake first: %v", t.newValue) } // tlsHandshakeFirstFallback implements the option interface for the tls `handshake first fallback delay` setting. type tlsHandshakeFirstFallback struct { noopOption newValue time.Duration } // Apply is a no-op because the timeout will be reloaded after options are applied. func (t *tlsHandshakeFirstFallback) Apply(server *Server) { server.Noticef("Reloaded: Client TLS handshake first fallback delay: %v", t.newValue) } // authOption is a base struct that provides default option behaviors. type authOption struct { noopOption } func (o authOption) IsAuthChange() bool { return true } // usernameOption implements the option interface for the `username` setting. type usernameOption struct { authOption } // Apply is a no-op because authorization will be reloaded after options are // applied. func (u *usernameOption) Apply(server *Server) { server.Noticef("Reloaded: authorization username") } // passwordOption implements the option interface for the `password` setting. type passwordOption struct { authOption } // Apply is a no-op because authorization will be reloaded after options are // applied. func (p *passwordOption) Apply(server *Server) { server.Noticef("Reloaded: authorization password") } // authorizationOption implements the option interface for the `token` // authorization setting. type authorizationOption struct { authOption } // Apply is a no-op because authorization will be reloaded after options are // applied. func (a *authorizationOption) Apply(server *Server) { server.Noticef("Reloaded: authorization token") } // authTimeoutOption implements the option interface for the authorization // `timeout` setting. type authTimeoutOption struct { noopOption // Not authOption because this is a no-op; will be reloaded with options. newValue float64 } // Apply is a no-op because the timeout will be reloaded after options are // applied. func (a *authTimeoutOption) Apply(server *Server) { server.Noticef("Reloaded: authorization timeout = %v", a.newValue) } // tagsOption implements the option interface for the `tags` setting. type tagsOption struct { noopOption // Not authOption because this is a no-op; will be reloaded with options. } func (u *tagsOption) Apply(server *Server) { server.Noticef("Reloaded: tags") } func (u *tagsOption) IsStatszChange() bool { return true } // usersOption implements the option interface for the authorization `users` // setting. type usersOption struct { authOption } func (u *usersOption) Apply(server *Server) { server.Noticef("Reloaded: authorization users") } // nkeysOption implements the option interface for the authorization `users` // setting. type nkeysOption struct { authOption } func (u *nkeysOption) Apply(server *Server) { server.Noticef("Reloaded: authorization nkey users") } // clusterOption implements the option interface for the `cluster` setting. type clusterOption struct { authOption newValue ClusterOpts permsChanged bool accsAdded []string accsRemoved []string poolSizeChanged bool compressChanged bool } // Apply the cluster change. func (c *clusterOption) Apply(s *Server) { // TODO: support enabling/disabling clustering. s.mu.Lock() tlsRequired := c.newValue.TLSConfig != nil s.routeInfo.TLSRequired = tlsRequired s.routeInfo.TLSVerify = tlsRequired s.routeInfo.AuthRequired = c.newValue.Username != "" if c.newValue.NoAdvertise { s.routeInfo.ClientConnectURLs = nil s.routeInfo.WSConnectURLs = nil } else { s.routeInfo.ClientConnectURLs = s.clientConnectURLs s.routeInfo.WSConnectURLs = s.websocket.connectURLs } s.setRouteInfoHostPortAndIP() var routes []*client if c.compressChanged { co := &s.getOpts().Cluster.Compression newMode := co.Mode s.forEachRoute(func(r *client) { r.mu.Lock() // Skip routes that are "not supported" (because they will never do // compression) or the routes that have already the new compression // mode. if r.route.compression == CompressionNotSupported || r.route.compression == newMode { r.mu.Unlock() return } // We need to close the route if it had compression "off" or the new // mode is compression "off", or if the new mode is "accept", because // these require negotiation. if r.route.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept { routes = append(routes, r) } else if newMode == CompressionS2Auto { // If the mode is "s2_auto", we need to check if there is really // need to change, and at any rate, we want to save the actual // compression level here, not s2_auto. r.updateS2AutoCompressionLevel(co, &r.route.compression) } else { // Simply change the compression writer r.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...) r.route.compression = newMode } r.mu.Unlock() }) } s.mu.Unlock() if c.newValue.Name != "" && c.newValue.Name != s.ClusterName() { s.setClusterName(c.newValue.Name) } for _, r := range routes { r.closeConnection(ClientClosed) } s.Noticef("Reloaded: cluster") if tlsRequired && c.newValue.TLSConfig.InsecureSkipVerify { s.Warnf(clusterTLSInsecureWarning) } } func (c *clusterOption) IsClusterPermsChange() bool { return c.permsChanged } func (c *clusterOption) IsClusterPoolSizeOrAccountsChange() bool { return c.poolSizeChanged || len(c.accsAdded) > 0 || len(c.accsRemoved) > 0 } func (c *clusterOption) diffPoolAndAccounts(old *ClusterOpts) { c.poolSizeChanged = c.newValue.PoolSize != old.PoolSize addLoop: for _, na := range c.newValue.PinnedAccounts { for _, oa := range old.PinnedAccounts { if na == oa { continue addLoop } } c.accsAdded = append(c.accsAdded, na) } removeLoop: for _, oa := range old.PinnedAccounts { for _, na := range c.newValue.PinnedAccounts { if oa == na { continue removeLoop } } c.accsRemoved = append(c.accsRemoved, oa) } } // routesOption implements the option interface for the cluster `routes` // setting. type routesOption struct { noopOption add []*url.URL remove []*url.URL } // Apply the route changes by adding and removing the necessary routes. func (r *routesOption) Apply(server *Server) { server.mu.Lock() routes := make([]*client, server.numRoutes()) i := 0 server.forEachRoute(func(r *client) { routes[i] = r i++ }) // If there was a change, notify monitoring code that it should // update the route URLs if /varz endpoint is inspected. if len(r.add)+len(r.remove) > 0 { server.varzUpdateRouteURLs = true } server.mu.Unlock() // Remove routes. for _, remove := range r.remove { for _, client := range routes { var url *url.URL client.mu.Lock() if client.route != nil { url = client.route.url } client.mu.Unlock() if url != nil && urlsAreEqual(url, remove) { // Do not attempt to reconnect when route is removed. client.setNoReconnect() client.closeConnection(RouteRemoved) server.Noticef("Removed route %v", remove) } } } // Add routes. server.mu.Lock() server.solicitRoutes(r.add, server.getOpts().Cluster.PinnedAccounts) server.mu.Unlock() server.Noticef("Reloaded: cluster routes") } // maxConnOption implements the option interface for the `max_connections` // setting. type maxConnOption struct { noopOption newValue int } // Apply the max connections change by closing random connections til we are // below the limit if necessary. func (m *maxConnOption) Apply(server *Server) { server.mu.Lock() var ( clients = make([]*client, len(server.clients)) i = 0 ) // Map iteration is random, which allows us to close random connections. for _, client := range server.clients { clients[i] = client i++ } server.mu.Unlock() if m.newValue > 0 && len(clients) > m.newValue { // Close connections til we are within the limit. var ( numClose = len(clients) - m.newValue closed = 0 ) for _, client := range clients { client.maxConnExceeded() closed++ if closed >= numClose { break } } server.Noticef("Closed %d connections to fall within max_connections", closed) } server.Noticef("Reloaded: max_connections = %v", m.newValue) } // pidFileOption implements the option interface for the `pid_file` setting. type pidFileOption struct { noopOption newValue string } // Apply the setting by logging the pid to the new file. func (p *pidFileOption) Apply(server *Server) { if p.newValue == "" { return } if err := server.logPid(); err != nil { server.Errorf("Failed to write pidfile: %v", err) } server.Noticef("Reloaded: pid_file = %v", p.newValue) } // portsFileDirOption implements the option interface for the `portFileDir` setting. type portsFileDirOption struct { noopOption oldValue string newValue string } func (p *portsFileDirOption) Apply(server *Server) { server.deletePortsFile(p.oldValue) server.logPorts() server.Noticef("Reloaded: ports_file_dir = %v", p.newValue) } // maxControlLineOption implements the option interface for the // `max_control_line` setting. type maxControlLineOption struct { noopOption newValue int32 } // Apply the setting by updating each client. func (m *maxControlLineOption) Apply(server *Server) { mcl := int32(m.newValue) server.mu.Lock() for _, client := range server.clients { atomic.StoreInt32(&client.mcl, mcl) } server.mu.Unlock() server.Noticef("Reloaded: max_control_line = %d", mcl) } // maxPayloadOption implements the option interface for the `max_payload` // setting. type maxPayloadOption struct { noopOption newValue int32 } // Apply the setting by updating the server info and each client. func (m *maxPayloadOption) Apply(server *Server) { server.mu.Lock() server.info.MaxPayload = m.newValue for _, client := range server.clients { atomic.StoreInt32(&client.mpay, int32(m.newValue)) } server.mu.Unlock() server.Noticef("Reloaded: max_payload = %d", m.newValue) } // pingIntervalOption implements the option interface for the `ping_interval` // setting. type pingIntervalOption struct { noopOption newValue time.Duration } // Apply is a no-op because the ping interval will be reloaded after options // are applied. func (p *pingIntervalOption) Apply(server *Server) { server.Noticef("Reloaded: ping_interval = %s", p.newValue) } // maxPingsOutOption implements the option interface for the `ping_max` // setting. type maxPingsOutOption struct { noopOption newValue int } // Apply is a no-op because the ping interval will be reloaded after options // are applied. func (m *maxPingsOutOption) Apply(server *Server) { server.Noticef("Reloaded: ping_max = %d", m.newValue) } // writeDeadlineOption implements the option interface for the `write_deadline` // setting. type writeDeadlineOption struct { noopOption newValue time.Duration } // Apply is a no-op because the write deadline will be reloaded after options // are applied. func (w *writeDeadlineOption) Apply(server *Server) { server.Noticef("Reloaded: write_deadline = %s", w.newValue) } // clientAdvertiseOption implements the option interface for the `client_advertise` setting. type clientAdvertiseOption struct { noopOption newValue string } // Apply the setting by updating the server info and regenerate the infoJSON byte array. func (c *clientAdvertiseOption) Apply(server *Server) { server.mu.Lock() server.setInfoHostPort() server.mu.Unlock() server.Noticef("Reload: client_advertise = %s", c.newValue) } // accountsOption implements the option interface. // Ensure that authorization code is executed if any change in accounts type accountsOption struct { authOption } // Apply is a no-op. Changes will be applied in reloadAuthorization func (a *accountsOption) Apply(s *Server) { s.Noticef("Reloaded: accounts") } // For changes to a server's config. type jetStreamOption struct { noopOption newValue bool } func (a *jetStreamOption) Apply(s *Server) { s.Noticef("Reloaded: JetStream") } func (jso jetStreamOption) IsJetStreamChange() bool { return true } func (jso jetStreamOption) IsStatszChange() bool { return true } type ocspOption struct { tlsOption newValue *OCSPConfig } func (a *ocspOption) Apply(s *Server) { s.Noticef("Reloaded: OCSP") } type ocspResponseCacheOption struct { tlsOption newValue *OCSPResponseCacheConfig } func (a *ocspResponseCacheOption) Apply(s *Server) { s.Noticef("Reloaded OCSP peer cache") } // connectErrorReports implements the option interface for the `connect_error_reports` // setting. type connectErrorReports struct { noopOption newValue int } // Apply is a no-op because the value will be reloaded after options are applied. func (c *connectErrorReports) Apply(s *Server) { s.Noticef("Reloaded: connect_error_reports = %v", c.newValue) } // connectErrorReports implements the option interface for the `connect_error_reports` // setting. type reconnectErrorReports struct { noopOption newValue int } // Apply is a no-op because the value will be reloaded after options are applied. func (r *reconnectErrorReports) Apply(s *Server) { s.Noticef("Reloaded: reconnect_error_reports = %v", r.newValue) } // maxTracedMsgLenOption implements the option interface for the `max_traced_msg_len` setting. type maxTracedMsgLenOption struct { noopOption newValue int } // Apply the setting by updating the maximum traced message length. func (m *maxTracedMsgLenOption) Apply(server *Server) { server.mu.Lock() defer server.mu.Unlock() server.opts.MaxTracedMsgLen = m.newValue server.Noticef("Reloaded: max_traced_msg_len = %d", m.newValue) } type mqttAckWaitReload struct { noopOption newValue time.Duration } func (o *mqttAckWaitReload) Apply(s *Server) { s.Noticef("Reloaded: MQTT ack_wait = %v", o.newValue) } type mqttMaxAckPendingReload struct { noopOption newValue uint16 } func (o *mqttMaxAckPendingReload) Apply(s *Server) { s.mqttUpdateMaxAckPending(o.newValue) s.Noticef("Reloaded: MQTT max_ack_pending = %v", o.newValue) } type mqttStreamReplicasReload struct { noopOption newValue int } func (o *mqttStreamReplicasReload) Apply(s *Server) { s.Noticef("Reloaded: MQTT stream_replicas = %v", o.newValue) } type mqttConsumerReplicasReload struct { noopOption newValue int } func (o *mqttConsumerReplicasReload) Apply(s *Server) { s.Noticef("Reloaded: MQTT consumer_replicas = %v", o.newValue) } type mqttConsumerMemoryStorageReload struct { noopOption newValue bool } func (o *mqttConsumerMemoryStorageReload) Apply(s *Server) { s.Noticef("Reloaded: MQTT consumer_memory_storage = %v", o.newValue) } type mqttInactiveThresholdReload struct { noopOption newValue time.Duration } func (o *mqttInactiveThresholdReload) Apply(s *Server) { s.Noticef("Reloaded: MQTT consumer_inactive_threshold = %v", o.newValue) } type profBlockRateReload struct { noopOption newValue int } func (o *profBlockRateReload) Apply(s *Server) { s.setBlockProfileRate(o.newValue) s.Noticef("Reloaded: prof_block_rate = %v", o.newValue) } type leafNodeOption struct { noopOption tlsFirstChanged bool compressionChanged bool } func (l *leafNodeOption) Apply(s *Server) { opts := s.getOpts() if l.tlsFirstChanged { s.Noticef("Reloaded: LeafNode TLS HandshakeFirst value is: %v", opts.LeafNode.TLSHandshakeFirst) for _, r := range opts.LeafNode.Remotes { s.Noticef("Reloaded: LeafNode Remote to %v TLS HandshakeFirst value is: %v", r.URLs, r.TLSHandshakeFirst) } } if l.compressionChanged { var leafs []*client acceptSideCompOpts := &opts.LeafNode.Compression s.mu.RLock() // First, update our internal leaf remote configurations with the new // compress options. // Since changing the remotes (as in adding/removing) is currently not // supported, we know that we should have the same number in Options // than in leafRemoteCfgs, but to be sure, use the max size. max := len(opts.LeafNode.Remotes) if l := len(s.leafRemoteCfgs); l < max { max = l } for i := 0; i < max; i++ { lr := s.leafRemoteCfgs[i] lr.Lock() lr.Compression = opts.LeafNode.Remotes[i].Compression lr.Unlock() } for _, l := range s.leafs { var co *CompressionOpts l.mu.Lock() if r := l.leaf.remote; r != nil { co = &r.Compression } else { co = acceptSideCompOpts } newMode := co.Mode // Skip leaf connections that are "not supported" (because they // will never do compression) or the ones that have already the // new compression mode. if l.leaf.compression == CompressionNotSupported || l.leaf.compression == newMode { l.mu.Unlock() continue } // We need to close the connections if it had compression "off" or the new // mode is compression "off", or if the new mode is "accept", because // these require negotiation. if l.leaf.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept { leafs = append(leafs, l) } else if newMode == CompressionS2Auto { // If the mode is "s2_auto", we need to check if there is really // need to change, and at any rate, we want to save the actual // compression level here, not s2_auto. l.updateS2AutoCompressionLevel(co, &l.leaf.compression) } else { // Simply change the compression writer l.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...) l.leaf.compression = newMode } l.mu.Unlock() } s.mu.RUnlock() // Close the connections for which negotiation is required. for _, l := range leafs { l.closeConnection(ClientClosed) } s.Noticef("Reloaded: LeafNode compression settings") } } type noFastProdStallReload struct { noopOption noStall bool } func (l *noFastProdStallReload) Apply(s *Server) { var not string if l.noStall { not = "not " } s.Noticef("Reloaded: fast producers will %sbe stalled", not) } // Compares options and disconnects clients that are no longer listed in pinned certs. Lock must not be held. func (s *Server) recheckPinnedCerts(curOpts *Options, newOpts *Options) { s.mu.Lock() disconnectClients := []*client{} protoToPinned := map[int]PinnedCertSet{} if !reflect.DeepEqual(newOpts.TLSPinnedCerts, curOpts.TLSPinnedCerts) { protoToPinned[NATS] = curOpts.TLSPinnedCerts } if !reflect.DeepEqual(newOpts.MQTT.TLSPinnedCerts, curOpts.MQTT.TLSPinnedCerts) { protoToPinned[MQTT] = curOpts.MQTT.TLSPinnedCerts } if !reflect.DeepEqual(newOpts.Websocket.TLSPinnedCerts, curOpts.Websocket.TLSPinnedCerts) { protoToPinned[WS] = curOpts.Websocket.TLSPinnedCerts } for _, c := range s.clients { if c.kind != CLIENT { continue } if pinned, ok := protoToPinned[c.clientType()]; ok { if !c.matchesPinnedCert(pinned) { disconnectClients = append(disconnectClients, c) } } } checkClients := func(kind int, clients map[uint64]*client, set PinnedCertSet) { for _, c := range clients { if c.kind == kind && !c.matchesPinnedCert(set) { disconnectClients = append(disconnectClients, c) } } } if !reflect.DeepEqual(newOpts.LeafNode.TLSPinnedCerts, curOpts.LeafNode.TLSPinnedCerts) { checkClients(LEAF, s.leafs, newOpts.LeafNode.TLSPinnedCerts) } if !reflect.DeepEqual(newOpts.Cluster.TLSPinnedCerts, curOpts.Cluster.TLSPinnedCerts) { s.forEachRoute(func(c *client) { if !c.matchesPinnedCert(newOpts.Cluster.TLSPinnedCerts) { disconnectClients = append(disconnectClients, c) } }) } if s.gateway.enabled && reflect.DeepEqual(newOpts.Gateway.TLSPinnedCerts, curOpts.Gateway.TLSPinnedCerts) { gw := s.gateway gw.RLock() for _, c := range gw.out { if !c.matchesPinnedCert(newOpts.Gateway.TLSPinnedCerts) { disconnectClients = append(disconnectClients, c) } } checkClients(GATEWAY, gw.in, newOpts.Gateway.TLSPinnedCerts) gw.RUnlock() } s.mu.Unlock() if len(disconnectClients) > 0 { s.Noticef("Disconnect %d clients due to pinned certs reload", len(disconnectClients)) for _, c := range disconnectClients { c.closeConnection(TLSHandshakeError) } } } // Reload reads the current configuration file and calls out to ReloadOptions // to apply the changes. This returns an error if the server was not started // with a config file or an option which doesn't support hot-swapping was changed. func (s *Server) Reload() error { s.mu.Lock() configFile := s.configFile s.mu.Unlock() if configFile == "" { return errors.New("can only reload config when a file is provided using -c or --config") } newOpts, err := ProcessConfigFile(configFile) if err != nil { // TODO: Dump previous good config to a .bak file? return err } return s.ReloadOptions(newOpts) } // ReloadOptions applies any supported options from the provided Options // type. This returns an error if an option which doesn't support // hot-swapping was changed. // The provided Options type should not be re-used afterwards. // Either use Options.Clone() to pass a copy, or make a new one. func (s *Server) ReloadOptions(newOpts *Options) error { s.reloadMu.Lock() defer s.reloadMu.Unlock() s.mu.Lock() curOpts := s.getOpts() // Wipe trusted keys if needed when we have an operator. if len(curOpts.TrustedOperators) > 0 && len(curOpts.TrustedKeys) > 0 { curOpts.TrustedKeys = nil } clientOrgPort := curOpts.Port clusterOrgPort := curOpts.Cluster.Port gatewayOrgPort := curOpts.Gateway.Port leafnodesOrgPort := curOpts.LeafNode.Port websocketOrgPort := curOpts.Websocket.Port mqttOrgPort := curOpts.MQTT.Port s.mu.Unlock() // In case "-cluster ..." was provided through the command line, this will // properly set the Cluster.Host/Port etc... if l := curOpts.Cluster.ListenStr; l != _EMPTY_ { newOpts.Cluster.ListenStr = l overrideCluster(newOpts) } // Apply flags over config file settings. newOpts = MergeOptions(newOpts, FlagSnapshot) // Need more processing for boolean flags... if FlagSnapshot != nil { applyBoolFlags(newOpts, FlagSnapshot) } setBaselineOptions(newOpts) // setBaselineOptions sets Port to 0 if set to -1 (RANDOM port) // If that's the case, set it to the saved value when the accept loop was // created. if newOpts.Port == 0 { newOpts.Port = clientOrgPort } // We don't do that for cluster, so check against -1. if newOpts.Cluster.Port == -1 { newOpts.Cluster.Port = clusterOrgPort } if newOpts.Gateway.Port == -1 { newOpts.Gateway.Port = gatewayOrgPort } if newOpts.LeafNode.Port == -1 { newOpts.LeafNode.Port = leafnodesOrgPort } if newOpts.Websocket.Port == -1 { newOpts.Websocket.Port = websocketOrgPort } if newOpts.MQTT.Port == -1 { newOpts.MQTT.Port = mqttOrgPort } if err := s.reloadOptions(curOpts, newOpts); err != nil { return err } s.recheckPinnedCerts(curOpts, newOpts) s.mu.Lock() s.configTime = time.Now().UTC() s.updateVarzConfigReloadableFields(s.varz) s.mu.Unlock() return nil } func applyBoolFlags(newOpts, flagOpts *Options) { // Reset fields that may have been set to `true` in // MergeOptions() when some of the flags default to `true` // but have not been explicitly set and therefore value // from config file should take precedence. for name, val := range newOpts.inConfig { f := reflect.ValueOf(newOpts).Elem() names := strings.Split(name, ".") for _, name := range names { f = f.FieldByName(name) } f.SetBool(val) } // Now apply value (true or false) from flags that have // been explicitly set in command line for name, val := range flagOpts.inCmdLine { f := reflect.ValueOf(newOpts).Elem() names := strings.Split(name, ".") for _, name := range names { f = f.FieldByName(name) } f.SetBool(val) } } // reloadOptions reloads the server config with the provided options. If an // option that doesn't support hot-swapping is changed, this returns an error. func (s *Server) reloadOptions(curOpts, newOpts *Options) error { // Apply to the new options some of the options that may have been set // that can't be configured in the config file (this can happen in // applications starting NATS Server programmatically). newOpts.CustomClientAuthentication = curOpts.CustomClientAuthentication newOpts.CustomRouterAuthentication = curOpts.CustomRouterAuthentication changed, err := s.diffOptions(newOpts) if err != nil { return err } if len(changed) != 0 { if err := validateOptions(newOpts); err != nil { return err } } // Create a context that is used to pass special info that we may need // while applying the new options. ctx := reloadContext{oldClusterPerms: curOpts.Cluster.Permissions} s.setOpts(newOpts) s.applyOptions(&ctx, changed) return nil } // For the purpose of comparing, impose a order on slice data types where order does not matter func imposeOrder(value any) error { switch value := value.(type) { case []*Account: slices.SortFunc(value, func(i, j *Account) int { return cmp.Compare(i.Name, j.Name) }) for _, a := range value { slices.SortFunc(a.imports.streams, func(i, j *streamImport) int { return cmp.Compare(i.acc.Name, j.acc.Name) }) } case []*User: slices.SortFunc(value, func(i, j *User) int { return cmp.Compare(i.Username, j.Username) }) case []*NkeyUser: slices.SortFunc(value, func(i, j *NkeyUser) int { return cmp.Compare(i.Nkey, j.Nkey) }) case []*url.URL: slices.SortFunc(value, func(i, j *url.URL) int { return cmp.Compare(i.String(), j.String()) }) case []string: slices.Sort(value) case []*jwt.OperatorClaims: slices.SortFunc(value, func(i, j *jwt.OperatorClaims) int { return cmp.Compare(i.Issuer, j.Issuer) }) case GatewayOpts: slices.SortFunc(value.Gateways, func(i, j *RemoteGatewayOpts) int { return cmp.Compare(i.Name, j.Name) }) case WebsocketOpts: slices.Sort(value.AllowedOrigins) case string, bool, uint8, uint16, int, int32, int64, time.Duration, float64, nil, LeafNodeOpts, ClusterOpts, *tls.Config, PinnedCertSet, *URLAccResolver, *MemAccResolver, *DirAccResolver, *CacheDirAccResolver, Authentication, MQTTOpts, jwt.TagList, *OCSPConfig, map[string]string, JSLimitOpts, StoreCipher, *OCSPResponseCacheConfig: // explicitly skipped types case *AuthCallout: default: // this will fail during unit tests return fmt.Errorf("OnReload, sort or explicitly skip type: %s", reflect.TypeOf(value)) } return nil } // diffOptions returns a slice containing options which have been changed. If // an option that doesn't support hot-swapping is changed, this returns an // error. func (s *Server) diffOptions(newOpts *Options) ([]option, error) { var ( oldConfig = reflect.ValueOf(s.getOpts()).Elem() newConfig = reflect.ValueOf(newOpts).Elem() diffOpts = []option{} // Need to keep track of whether JS is being disabled // to prevent changing limits at runtime. jsEnabled = s.JetStreamEnabled() disableJS bool jsMemLimitsChanged bool jsFileLimitsChanged bool jsStoreDirChanged bool ) for i := 0; i < oldConfig.NumField(); i++ { field := oldConfig.Type().Field(i) // field.PkgPath is empty for exported fields, and is not for unexported ones. // We skip the unexported fields. if field.PkgPath != _EMPTY_ { continue } var ( oldValue = oldConfig.Field(i).Interface() newValue = newConfig.Field(i).Interface() ) if err := imposeOrder(oldValue); err != nil { return nil, err } if err := imposeOrder(newValue); err != nil { return nil, err } optName := strings.ToLower(field.Name) // accounts and users (referencing accounts) will always differ as accounts // contain internal state, say locks etc..., so we don't bother here. // This also avoids races with atomic stats counters if optName != "accounts" && optName != "users" { if changed := !reflect.DeepEqual(oldValue, newValue); !changed { // Check to make sure we are running JetStream if we think we should be. if optName == "jetstream" && newValue.(bool) { if !jsEnabled { diffOpts = append(diffOpts, &jetStreamOption{newValue: true}) } } continue } } switch optName { case "traceverbose": diffOpts = append(diffOpts, &traceVerboseOption{newValue: newValue.(bool)}) case "trace": diffOpts = append(diffOpts, &traceOption{newValue: newValue.(bool)}) case "debug": diffOpts = append(diffOpts, &debugOption{newValue: newValue.(bool)}) case "logtime": diffOpts = append(diffOpts, &logtimeOption{newValue: newValue.(bool)}) case "logtimeutc": diffOpts = append(diffOpts, &logtimeUTCOption{newValue: newValue.(bool)}) case "logfile": diffOpts = append(diffOpts, &logfileOption{newValue: newValue.(string)}) case "syslog": diffOpts = append(diffOpts, &syslogOption{newValue: newValue.(bool)}) case "remotesyslog": diffOpts = append(diffOpts, &remoteSyslogOption{newValue: newValue.(string)}) case "tlsconfig": diffOpts = append(diffOpts, &tlsOption{newValue: newValue.(*tls.Config)}) case "tlstimeout": diffOpts = append(diffOpts, &tlsTimeoutOption{newValue: newValue.(float64)}) case "tlspinnedcerts": diffOpts = append(diffOpts, &tlsPinnedCertOption{newValue: newValue.(PinnedCertSet)}) case "tlshandshakefirst": diffOpts = append(diffOpts, &tlsHandshakeFirst{newValue: newValue.(bool)}) case "tlshandshakefirstfallback": diffOpts = append(diffOpts, &tlsHandshakeFirstFallback{newValue: newValue.(time.Duration)}) case "username": diffOpts = append(diffOpts, &usernameOption{}) case "password": diffOpts = append(diffOpts, &passwordOption{}) case "tags": diffOpts = append(diffOpts, &tagsOption{}) case "authorization": diffOpts = append(diffOpts, &authorizationOption{}) case "authtimeout": diffOpts = append(diffOpts, &authTimeoutOption{newValue: newValue.(float64)}) case "users": diffOpts = append(diffOpts, &usersOption{}) case "nkeys": diffOpts = append(diffOpts, &nkeysOption{}) case "cluster": newClusterOpts := newValue.(ClusterOpts) oldClusterOpts := oldValue.(ClusterOpts) if err := validateClusterOpts(oldClusterOpts, newClusterOpts); err != nil { return nil, err } co := &clusterOption{ newValue: newClusterOpts, permsChanged: !reflect.DeepEqual(newClusterOpts.Permissions, oldClusterOpts.Permissions), compressChanged: !reflect.DeepEqual(oldClusterOpts.Compression, newClusterOpts.Compression), } co.diffPoolAndAccounts(&oldClusterOpts) // If there are added accounts, first make sure that we can look them up. // If we can't let's fail the reload. for _, acc := range co.accsAdded { if _, err := s.LookupAccount(acc); err != nil { return nil, fmt.Errorf("unable to add account %q to the list of dedicated routes: %v", acc, err) } } // If pool_size has been set to negative (but was not before), then let's // add the system account to the list of removed accounts (we don't have // to check if already there, duplicates are ok in that case). if newClusterOpts.PoolSize < 0 && oldClusterOpts.PoolSize >= 0 { if sys := s.SystemAccount(); sys != nil { co.accsRemoved = append(co.accsRemoved, sys.GetName()) } } diffOpts = append(diffOpts, co) case "routes": add, remove := diffRoutes(oldValue.([]*url.URL), newValue.([]*url.URL)) diffOpts = append(diffOpts, &routesOption{add: add, remove: remove}) case "maxconn": diffOpts = append(diffOpts, &maxConnOption{newValue: newValue.(int)}) case "pidfile": diffOpts = append(diffOpts, &pidFileOption{newValue: newValue.(string)}) case "portsfiledir": diffOpts = append(diffOpts, &portsFileDirOption{newValue: newValue.(string), oldValue: oldValue.(string)}) case "maxcontrolline": diffOpts = append(diffOpts, &maxControlLineOption{newValue: newValue.(int32)}) case "maxpayload": diffOpts = append(diffOpts, &maxPayloadOption{newValue: newValue.(int32)}) case "pinginterval": diffOpts = append(diffOpts, &pingIntervalOption{newValue: newValue.(time.Duration)}) case "maxpingsout": diffOpts = append(diffOpts, &maxPingsOutOption{newValue: newValue.(int)}) case "writedeadline": diffOpts = append(diffOpts, &writeDeadlineOption{newValue: newValue.(time.Duration)}) case "clientadvertise": cliAdv := newValue.(string) if cliAdv != "" { // Validate ClientAdvertise syntax if _, _, err := parseHostPort(cliAdv, 0); err != nil { return nil, fmt.Errorf("invalid ClientAdvertise value of %s, err=%v", cliAdv, err) } } diffOpts = append(diffOpts, &clientAdvertiseOption{newValue: cliAdv}) case "accounts": diffOpts = append(diffOpts, &accountsOption{}) case "resolver", "accountresolver", "accountsresolver": // We can't move from no resolver to one. So check for that. if (oldValue == nil && newValue != nil) || (oldValue != nil && newValue == nil) { return nil, fmt.Errorf("config reload does not support moving to or from an account resolver") } diffOpts = append(diffOpts, &accountsOption{}) case "accountresolvertlsconfig": diffOpts = append(diffOpts, &accountsOption{}) case "gateway": // Not supported for now, but report warning if configuration of gateway // is actually changed so that user knows that it won't take effect. // Any deep-equal is likely to fail for when there is a TLSConfig. so // remove for the test. tmpOld := oldValue.(GatewayOpts) tmpNew := newValue.(GatewayOpts) tmpOld.TLSConfig = nil tmpNew.TLSConfig = nil tmpOld.tlsConfigOpts = nil tmpNew.tlsConfigOpts = nil // Need to do the same for remote gateways' TLS configs. // But we can't just set remotes' TLSConfig to nil otherwise this // would lose the real TLS configuration. tmpOld.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpOld.Gateways) tmpNew.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpNew.Gateways) // If there is really a change prevents reload. if !reflect.DeepEqual(tmpOld, tmpNew) { // See TODO(ik) note below about printing old/new values. return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } case "leafnode": // Similar to gateways tmpOld := oldValue.(LeafNodeOpts) tmpNew := newValue.(LeafNodeOpts) tmpOld.TLSConfig = nil tmpNew.TLSConfig = nil tmpOld.tlsConfigOpts = nil tmpNew.tlsConfigOpts = nil // We will allow TLSHandshakeFirst to me config reloaded. First, // we just want to detect if there was a change in the leafnodes{} // block, and if not, we will check the remotes. handshakeFirstChanged := tmpOld.TLSHandshakeFirst != tmpNew.TLSHandshakeFirst // If changed, set them (in the temporary variables) to false so that the // rest of the comparison does not fail. if handshakeFirstChanged { tmpOld.TLSHandshakeFirst, tmpNew.TLSHandshakeFirst = false, false } else if len(tmpOld.Remotes) == len(tmpNew.Remotes) { // Since we don't support changes in the remotes, we will do a // simple pass to see if there was a change of this field. for i := 0; i < len(tmpOld.Remotes); i++ { if tmpOld.Remotes[i].TLSHandshakeFirst != tmpNew.Remotes[i].TLSHandshakeFirst { handshakeFirstChanged = true break } } } // We also support config reload for compression. Check if it changed before // blanking them out for the deep-equal check at the end. compressionChanged := !reflect.DeepEqual(tmpOld.Compression, tmpNew.Compression) if compressionChanged { tmpOld.Compression, tmpNew.Compression = CompressionOpts{}, CompressionOpts{} } else if len(tmpOld.Remotes) == len(tmpNew.Remotes) { // Same that for tls first check, do the remotes now. for i := 0; i < len(tmpOld.Remotes); i++ { if !reflect.DeepEqual(tmpOld.Remotes[i].Compression, tmpNew.Remotes[i].Compression) { compressionChanged = true break } } } // Need to do the same for remote leafnodes' TLS configs. // But we can't just set remotes' TLSConfig to nil otherwise this // would lose the real TLS configuration. tmpOld.Remotes = copyRemoteLNConfigForReloadCompare(tmpOld.Remotes) tmpNew.Remotes = copyRemoteLNConfigForReloadCompare(tmpNew.Remotes) // Special check for leafnode remotes changes which are not supported right now. leafRemotesChanged := func(a, b LeafNodeOpts) bool { if len(a.Remotes) != len(b.Remotes) { return true } // Check whether all remotes URLs are still the same. for _, oldRemote := range a.Remotes { var found bool if oldRemote.LocalAccount == _EMPTY_ { oldRemote.LocalAccount = globalAccountName } for _, newRemote := range b.Remotes { // Bind to global account in case not defined. if newRemote.LocalAccount == _EMPTY_ { newRemote.LocalAccount = globalAccountName } if reflect.DeepEqual(oldRemote, newRemote) { found = true break } } if !found { return true } } return false } // First check whether remotes changed at all. If they did not, // skip them in the complete equal check. if !leafRemotesChanged(tmpOld, tmpNew) { tmpOld.Remotes = nil tmpNew.Remotes = nil } // Special check for auth users to detect changes. // If anything is off will fall through and fail below. // If we detect they are semantically the same we nil them out // to pass the check below. if tmpOld.Users != nil || tmpNew.Users != nil { if len(tmpOld.Users) == len(tmpNew.Users) { oua := make(map[string]*User, len(tmpOld.Users)) nua := make(map[string]*User, len(tmpOld.Users)) for _, u := range tmpOld.Users { oua[u.Username] = u } for _, u := range tmpNew.Users { nua[u.Username] = u } same := true for uname, u := range oua { // If we can not find new one with same name, drop through to fail. nu, ok := nua[uname] if !ok { same = false break } // If username or password or account different break. if u.Username != nu.Username || u.Password != nu.Password || u.Account.GetName() != nu.Account.GetName() { same = false break } } // We can nil out here. if same { tmpOld.Users, tmpNew.Users = nil, nil } } } // If there is really a change prevents reload. if !reflect.DeepEqual(tmpOld, tmpNew) { // See TODO(ik) note below about printing old/new values. return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } diffOpts = append(diffOpts, &leafNodeOption{ tlsFirstChanged: handshakeFirstChanged, compressionChanged: compressionChanged, }) case "jetstream": new := newValue.(bool) old := oldValue.(bool) if new != old { diffOpts = append(diffOpts, &jetStreamOption{newValue: new}) } // Mark whether JS will be disabled. disableJS = !new case "storedir": new := newValue.(string) old := oldValue.(string) modified := new != old // Check whether JS is being disabled and/or storage dir attempted to change. if jsEnabled && modified { if new == _EMPTY_ { // This means that either JS is being disabled or it is using an temp dir. // Allow the change but error in case JS was not disabled. jsStoreDirChanged = true } else { return nil, fmt.Errorf("config reload not supported for jetstream storage directory") } } case "jetstreammaxmemory", "jetstreammaxstore": old := oldValue.(int64) new := newValue.(int64) // Check whether JS is being disabled and/or limits are being changed. var ( modified = new != old fromUnset = old == -1 fromSet = !fromUnset toUnset = new == -1 toSet = !toUnset ) if jsEnabled && modified { // Cannot change limits from dynamic storage at runtime. switch { case fromSet && toUnset: // Limits changed but it may mean that JS is being disabled, // keep track of the change and error in case it is not. switch optName { case "jetstreammaxmemory": jsMemLimitsChanged = true case "jetstreammaxstore": jsFileLimitsChanged = true default: return nil, fmt.Errorf("config reload not supported for jetstream max memory and store") } case fromUnset && toSet: // Prevent changing from dynamic max memory / file at runtime. return nil, fmt.Errorf("config reload not supported for jetstream dynamic max memory and store") default: return nil, fmt.Errorf("config reload not supported for jetstream max memory and store") } } case "websocket": // Similar to gateways tmpOld := oldValue.(WebsocketOpts) tmpNew := newValue.(WebsocketOpts) tmpOld.TLSConfig, tmpOld.tlsConfigOpts = nil, nil tmpNew.TLSConfig, tmpNew.tlsConfigOpts = nil, nil // If there is really a change prevents reload. if !reflect.DeepEqual(tmpOld, tmpNew) { // See TODO(ik) note below about printing old/new values. return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } case "mqtt": diffOpts = append(diffOpts, &mqttAckWaitReload{newValue: newValue.(MQTTOpts).AckWait}) diffOpts = append(diffOpts, &mqttMaxAckPendingReload{newValue: newValue.(MQTTOpts).MaxAckPending}) diffOpts = append(diffOpts, &mqttStreamReplicasReload{newValue: newValue.(MQTTOpts).StreamReplicas}) diffOpts = append(diffOpts, &mqttConsumerReplicasReload{newValue: newValue.(MQTTOpts).ConsumerReplicas}) diffOpts = append(diffOpts, &mqttConsumerMemoryStorageReload{newValue: newValue.(MQTTOpts).ConsumerMemoryStorage}) diffOpts = append(diffOpts, &mqttInactiveThresholdReload{newValue: newValue.(MQTTOpts).ConsumerInactiveThreshold}) // Nil out/set to 0 the options that we allow to be reloaded so that // we only fail reload if some that we don't support are changed. tmpOld := oldValue.(MQTTOpts) tmpNew := newValue.(MQTTOpts) tmpOld.TLSConfig, tmpOld.tlsConfigOpts, tmpOld.AckWait, tmpOld.MaxAckPending, tmpOld.StreamReplicas, tmpOld.ConsumerReplicas, tmpOld.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false tmpOld.ConsumerInactiveThreshold = 0 tmpNew.TLSConfig, tmpNew.tlsConfigOpts, tmpNew.AckWait, tmpNew.MaxAckPending, tmpNew.StreamReplicas, tmpNew.ConsumerReplicas, tmpNew.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false tmpNew.ConsumerInactiveThreshold = 0 if !reflect.DeepEqual(tmpOld, tmpNew) { // See TODO(ik) note below about printing old/new values. return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } tmpNew.AckWait = newValue.(MQTTOpts).AckWait tmpNew.MaxAckPending = newValue.(MQTTOpts).MaxAckPending tmpNew.StreamReplicas = newValue.(MQTTOpts).StreamReplicas tmpNew.ConsumerReplicas = newValue.(MQTTOpts).ConsumerReplicas tmpNew.ConsumerMemoryStorage = newValue.(MQTTOpts).ConsumerMemoryStorage tmpNew.ConsumerInactiveThreshold = newValue.(MQTTOpts).ConsumerInactiveThreshold case "connecterrorreports": diffOpts = append(diffOpts, &connectErrorReports{newValue: newValue.(int)}) case "reconnecterrorreports": diffOpts = append(diffOpts, &reconnectErrorReports{newValue: newValue.(int)}) case "nolog", "nosigs": // Ignore NoLog and NoSigs options since they are not parsed and only used in // testing. continue case "disableshortfirstping": newOpts.DisableShortFirstPing = oldValue.(bool) continue case "maxtracedmsglen": diffOpts = append(diffOpts, &maxTracedMsgLenOption{newValue: newValue.(int)}) case "port": // check to see if newValue == 0 and continue if so. if newValue == 0 { // ignore RANDOM_PORT continue } fallthrough case "noauthuser": if oldValue != _EMPTY_ && newValue == _EMPTY_ { for _, user := range newOpts.Users { if user.Username == oldValue { return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } } } else { return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } case "systemaccount": if oldValue != DEFAULT_SYSTEM_ACCOUNT || newValue != _EMPTY_ { return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } case "ocspconfig": diffOpts = append(diffOpts, &ocspOption{newValue: newValue.(*OCSPConfig)}) case "ocspcacheconfig": diffOpts = append(diffOpts, &ocspResponseCacheOption{newValue: newValue.(*OCSPResponseCacheConfig)}) case "profblockrate": new := newValue.(int) old := oldValue.(int) if new != old { diffOpts = append(diffOpts, &profBlockRateReload{newValue: new}) } case "nofastproducerstall": diffOpts = append(diffOpts, &noFastProdStallReload{noStall: newValue.(bool)}) default: // TODO(ik): Implement String() on those options to have a nice print. // %v is difficult to figure what's what, %+v print private fields and // would print passwords. Tried json.Marshal but it is too verbose for // the URL array. // Bail out if attempting to reload any unsupported options. return nil, fmt.Errorf("config reload not supported for %s: old=%v, new=%v", field.Name, oldValue, newValue) } } // If not disabling JS but limits have changed then it is an error. if !disableJS { if jsMemLimitsChanged || jsFileLimitsChanged { return nil, fmt.Errorf("config reload not supported for jetstream max memory and max store") } if jsStoreDirChanged { return nil, fmt.Errorf("config reload not supported for jetstream storage dir") } } return diffOpts, nil } func copyRemoteGWConfigsWithoutTLSConfig(current []*RemoteGatewayOpts) []*RemoteGatewayOpts { l := len(current) if l == 0 { return nil } rgws := make([]*RemoteGatewayOpts, 0, l) for _, rcfg := range current { cp := *rcfg cp.TLSConfig = nil cp.tlsConfigOpts = nil rgws = append(rgws, &cp) } return rgws } func copyRemoteLNConfigForReloadCompare(current []*RemoteLeafOpts) []*RemoteLeafOpts { l := len(current) if l == 0 { return nil } rlns := make([]*RemoteLeafOpts, 0, l) for _, rcfg := range current { cp := *rcfg cp.TLSConfig = nil cp.tlsConfigOpts = nil cp.TLSHandshakeFirst = false // This is set only when processing a CONNECT, so reset here so that we // don't fail the DeepEqual comparison. cp.TLS = false // For now, remove DenyImports/Exports since those get modified at runtime // to add JS APIs. cp.DenyImports, cp.DenyExports = nil, nil // Remove compression mode cp.Compression = CompressionOpts{} rlns = append(rlns, &cp) } return rlns } func (s *Server) applyOptions(ctx *reloadContext, opts []option) { var ( reloadLogging = false reloadAuth = false reloadClusterPerms = false reloadClientTrcLvl = false reloadJetstream = false jsEnabled = false isStatszChange = false co *clusterOption ) for _, opt := range opts { opt.Apply(s) if opt.IsLoggingChange() { reloadLogging = true } if opt.IsTraceLevelChange() { reloadClientTrcLvl = true } if opt.IsAuthChange() { reloadAuth = true } if opt.IsClusterPoolSizeOrAccountsChange() { co = opt.(*clusterOption) } if opt.IsClusterPermsChange() { reloadClusterPerms = true } if opt.IsJetStreamChange() { reloadJetstream = true jsEnabled = opt.(*jetStreamOption).newValue } if opt.IsStatszChange() { isStatszChange = true } } if reloadLogging { s.ConfigureLogger() } if reloadClientTrcLvl { s.reloadClientTraceLevel() } if reloadAuth { s.reloadAuthorization() } if reloadClusterPerms { s.reloadClusterPermissions(ctx.oldClusterPerms) } newOpts := s.getOpts() // If we need to reload cluster pool/per-account, then co will be not nil if co != nil { s.reloadClusterPoolAndAccounts(co, newOpts) } if reloadJetstream { if !jsEnabled { s.DisableJetStream() } else if !s.JetStreamEnabled() { if err := s.restartJetStream(); err != nil { s.Warnf("Can't start JetStream: %v", err) } } // Make sure to reset the internal loop's version of JS. s.resetInternalLoopInfo() } if isStatszChange { s.sendStatszUpdate() } // For remote gateways and leafnodes, make sure that their TLS configuration // is updated (since the config is "captured" early and changes would otherwise // not be visible). if s.gateway.enabled { s.gateway.updateRemotesTLSConfig(newOpts) } if len(newOpts.LeafNode.Remotes) > 0 { s.updateRemoteLeafNodesTLSConfig(newOpts) } // Always restart OCSP monitoring on reload. if err := s.reloadOCSP(); err != nil { s.Warnf("Can't restart OCSP features: %v", err) } s.Noticef("Reloaded server configuration") } // This will send a reset to the internal send loop. func (s *Server) resetInternalLoopInfo() { var resetCh chan struct{} s.mu.Lock() if s.sys != nil { // can't hold the lock as go routine reading it may be waiting for lock as well resetCh = s.sys.resetCh } s.mu.Unlock() if resetCh != nil { resetCh <- struct{}{} } } // Update all cached debug and trace settings for every client func (s *Server) reloadClientTraceLevel() { opts := s.getOpts() if opts.NoLog { return } // Create a list of all clients. // Update their trace level when not holding server or gateway lock s.mu.Lock() clientCnt := 1 + len(s.clients) + len(s.grTmpClients) + s.numRoutes() + len(s.leafs) s.mu.Unlock() s.gateway.RLock() clientCnt += len(s.gateway.in) + len(s.gateway.outo) s.gateway.RUnlock() clients := make([]*client, 0, clientCnt) s.mu.Lock() if s.eventsEnabled() { clients = append(clients, s.sys.client) } cMaps := []map[uint64]*client{s.clients, s.grTmpClients, s.leafs} for _, m := range cMaps { for _, c := range m { clients = append(clients, c) } } s.forEachRoute(func(c *client) { clients = append(clients, c) }) s.mu.Unlock() s.gateway.RLock() for _, c := range s.gateway.in { clients = append(clients, c) } clients = append(clients, s.gateway.outo...) s.gateway.RUnlock() for _, c := range clients { // client.trace is commonly read while holding the lock c.mu.Lock() c.setTraceLevel() c.mu.Unlock() } } // reloadAuthorization reconfigures the server authorization settings, // disconnects any clients who are no longer authorized, and removes any // unauthorized subscriptions. func (s *Server) reloadAuthorization() { // This map will contain the names of accounts that have their streams // import configuration changed. var awcsti map[string]struct{} checkJetStream := false opts := s.getOpts() s.mu.Lock() deletedAccounts := make(map[string]*Account) // This can not be changed for now so ok to check server's trustedKeys unlocked. // If plain configured accounts, process here. if s.trustedKeys == nil { // Make a map of the configured account names so we figure out the accounts // that should be removed later on. configAccs := make(map[string]struct{}, len(opts.Accounts)) for _, acc := range opts.Accounts { configAccs[acc.GetName()] = struct{}{} } // Now range over existing accounts and keep track of the ones deleted // so some cleanup can be made after releasing the server lock. s.accounts.Range(func(k, v any) bool { an, acc := k.(string), v.(*Account) // Exclude default and system account from this test since those // may not actually be in opts.Accounts. if an == DEFAULT_GLOBAL_ACCOUNT || an == DEFAULT_SYSTEM_ACCOUNT { return true } // Check check if existing account is still in opts.Accounts. if _, ok := configAccs[an]; !ok { deletedAccounts[an] = acc s.accounts.Delete(k) } return true }) // This will update existing and add new ones. awcsti, _ = s.configureAccounts(true) s.configureAuthorization() // Double check any JetStream configs. checkJetStream = s.getJetStream() != nil } else if opts.AccountResolver != nil { s.configureResolver() if _, ok := s.accResolver.(*MemAccResolver); ok { // Check preloads so we can issue warnings etc if needed. s.checkResolvePreloads() // With a memory resolver we want to do something similar to configured accounts. // We will walk the accounts and delete them if they are no longer present via fetch. // If they are present we will force a claim update to process changes. s.accounts.Range(func(k, v any) bool { acc := v.(*Account) // Skip global account. if acc == s.gacc { return true } accName := acc.GetName() // Release server lock for following actions s.mu.Unlock() accClaims, claimJWT, _ := s.fetchAccountClaims(accName) if accClaims != nil { if err := s.updateAccountWithClaimJWT(acc, claimJWT); err != nil { s.Noticef("Reloaded: deleting account [bad claims]: %q", accName) s.accounts.Delete(k) } } else { s.Noticef("Reloaded: deleting account [removed]: %q", accName) s.accounts.Delete(k) } // Regrab server lock. s.mu.Lock() return true }) } } var ( cclientsa [64]*client cclients = cclientsa[:0] clientsa [64]*client clients = clientsa[:0] routesa [64]*client routes = routesa[:0] ) // Gather clients that changed accounts. We will close them and they // will reconnect, doing the right thing. for _, client := range s.clients { if s.clientHasMovedToDifferentAccount(client) { cclients = append(cclients, client) } else { clients = append(clients, client) } } s.forEachRoute(func(route *client) { routes = append(routes, route) }) // Check here for any system/internal clients which will not be in the servers map of normal clients. if s.sys != nil && s.sys.account != nil && !opts.NoSystemAccount { s.accounts.Store(s.sys.account.Name, s.sys.account) } s.accounts.Range(func(k, v any) bool { acc := v.(*Account) acc.mu.RLock() // Check for sysclients accounting, ignore the system account. if acc.sysclients > 0 && (s.sys == nil || s.sys.account != acc) { for c := range acc.clients { if c.kind != CLIENT && c.kind != LEAF { clients = append(clients, c) } } } acc.mu.RUnlock() return true }) var resetCh chan struct{} if s.sys != nil { // can't hold the lock as go routine reading it may be waiting for lock as well resetCh = s.sys.resetCh } s.mu.Unlock() // Clear some timers and remove service import subs for deleted accounts. for _, acc := range deletedAccounts { acc.mu.Lock() clearTimer(&acc.etmr) clearTimer(&acc.ctmr) for _, se := range acc.exports.services { se.clearResponseThresholdTimer() } acc.mu.Unlock() acc.removeAllServiceImportSubs() } if resetCh != nil { resetCh <- struct{}{} } // Check that publish retained messages sources are still allowed to publish. s.mqttCheckPubRetainedPerms() // Close clients that have moved accounts for _, client := range cclients { client.closeConnection(ClientClosed) } for _, c := range clients { // Disconnect any unauthorized clients. // Ignore internal clients. if (c.kind == CLIENT || c.kind == LEAF) && !s.isClientAuthorized(c) { c.authViolation() continue } // Check to make sure account is correct. c.swapAccountAfterReload() // Remove any unauthorized subscriptions and check for account imports. c.processSubsOnConfigReload(awcsti) } for _, route := range routes { // Disconnect any unauthorized routes. // Do this only for routes that were accepted, not initiated // because in the later case, we don't have the user name/password // of the remote server. if !route.isSolicitedRoute() && !s.isRouterAuthorized(route) { route.setNoReconnect() route.authViolation() } } if res := s.AccountResolver(); res != nil { res.Reload() } // We will double check all JetStream configs on a reload. if checkJetStream { if err := s.enableJetStreamAccounts(); err != nil { s.Errorf(err.Error()) } } } // Returns true if given client current account has changed (or user // no longer exist) in the new config, false if the user did not // change accounts. // Server lock is held on entry. func (s *Server) clientHasMovedToDifferentAccount(c *client) bool { var ( nu *NkeyUser u *User ) c.mu.Lock() defer c.mu.Unlock() if c.opts.Nkey != _EMPTY_ { if s.nkeys != nil { nu = s.nkeys[c.opts.Nkey] } } else if c.opts.Username != _EMPTY_ { if s.users != nil { u = s.users[c.opts.Username] } } else { return false } // Get the current account name var curAccName string if c.acc != nil { curAccName = c.acc.Name } if nu != nil && nu.Account != nil { return curAccName != nu.Account.Name } else if u != nil && u.Account != nil { return curAccName != u.Account.Name } // user/nkey no longer exists. return true } // reloadClusterPermissions reconfigures the cluster's permssions // and set the permissions to all existing routes, sending an // update INFO protocol so that remote can resend their local // subs if needed, and sending local subs matching cluster's // import subjects. func (s *Server) reloadClusterPermissions(oldPerms *RoutePermissions) { s.mu.Lock() newPerms := s.getOpts().Cluster.Permissions routes := make(map[uint64]*client, s.numRoutes()) // Get all connected routes s.forEachRoute(func(route *client) { route.mu.Lock() routes[route.cid] = route route.mu.Unlock() }) // If new permissions is nil, then clear routeInfo import/export if newPerms == nil { s.routeInfo.Import = nil s.routeInfo.Export = nil } else { s.routeInfo.Import = newPerms.Import s.routeInfo.Export = newPerms.Export } infoJSON := generateInfoJSON(&s.routeInfo) s.mu.Unlock() // Close connections for routes that don't understand async INFO. for _, route := range routes { route.mu.Lock() close := route.opts.Protocol < RouteProtoInfo cid := route.cid route.mu.Unlock() if close { route.closeConnection(RouteRemoved) delete(routes, cid) } } // If there are no route left, we are done if len(routes) == 0 { return } // Fake clients to test cluster permissions oldPermsTester := &client{} oldPermsTester.setRoutePermissions(oldPerms) newPermsTester := &client{} newPermsTester.setRoutePermissions(newPerms) var ( _localSubs [4096]*subscription subsNeedSUB = map[*client][]*subscription{} subsNeedUNSUB = map[*client][]*subscription{} deleteRoutedSubs []*subscription ) getRouteForAccount := func(accName string, poolIdx int) *client { for _, r := range routes { r.mu.Lock() ok := (poolIdx >= 0 && poolIdx == r.route.poolIdx) || (string(r.route.accName) == accName) || r.route.noPool r.mu.Unlock() if ok { return r } } return nil } // First set the new permissions on all routes. for _, route := range routes { route.mu.Lock() route.setRoutePermissions(newPerms) route.mu.Unlock() } // Then, go over all accounts and gather local subscriptions that need to be // sent over as SUB or removed as UNSUB, and routed subscriptions that need // to be dropped due to export permissions. s.accounts.Range(func(_, v any) bool { acc := v.(*Account) acc.mu.RLock() accName, sl, poolIdx := acc.Name, acc.sl, acc.routePoolIdx acc.mu.RUnlock() // Get the route handling this account. If no route or sublist, bail out. route := getRouteForAccount(accName, poolIdx) if route == nil || sl == nil { return true } localSubs := _localSubs[:0] sl.localSubs(&localSubs, false) // Go through all local subscriptions for _, sub := range localSubs { // Get all subs that can now be imported subj := string(sub.subject) couldImportThen := oldPermsTester.canImport(subj) canImportNow := newPermsTester.canImport(subj) if canImportNow { // If we could not before, then will need to send a SUB protocol. if !couldImportThen { subsNeedSUB[route] = append(subsNeedSUB[route], sub) } } else if couldImportThen { // We were previously able to import this sub, but now // we can't so we need to send an UNSUB protocol subsNeedUNSUB[route] = append(subsNeedUNSUB[route], sub) } } deleteRoutedSubs = deleteRoutedSubs[:0] route.mu.Lock() pa, _, hasSubType := route.getRoutedSubKeyInfo() for key, sub := range route.subs { // If this is not a pinned-account route, we need to get the // account name from the key to see if we collect this sub. if !pa { if an := getAccNameFromRoutedSubKey(sub, key, hasSubType); an != accName { continue } } // If we can't export, we need to drop the subscriptions that // we have on behalf of this route. // Need to make a string cast here since canExport call sl.Match() subj := string(sub.subject) if !route.canExport(subj) { // We can use bytesToString() here. delete(route.subs, bytesToString(sub.sid)) deleteRoutedSubs = append(deleteRoutedSubs, sub) } } route.mu.Unlock() // Remove as a batch all the subs that we have removed from each route. sl.RemoveBatch(deleteRoutedSubs) return true }) // Send an update INFO, which will allow remote server to show // our current route config in monitoring and resend subscriptions // that we now possibly allow with a change of Export permissions. for _, route := range routes { route.mu.Lock() route.enqueueProto(infoJSON) // Now send SUB and UNSUB protocols as needed. if subs, ok := subsNeedSUB[route]; ok && len(subs) > 0 { route.sendRouteSubProtos(subs, false, nil) } if unsubs, ok := subsNeedUNSUB[route]; ok && len(unsubs) > 0 { route.sendRouteUnSubProtos(unsubs, false, nil) } route.mu.Unlock() } } func (s *Server) reloadClusterPoolAndAccounts(co *clusterOption, opts *Options) { s.mu.Lock() // Prevent adding new routes until we are ready to do so. s.routesReject = true var ch chan struct{} // For accounts that have been added to the list of dedicated routes, // send a protocol to their current assigned routes to allow the // other side to prepare for the changes. if len(co.accsAdded) > 0 { protosSent := 0 s.accAddedReqID = nuid.Next() for _, an := range co.accsAdded { if s.accRoutes == nil { s.accRoutes = make(map[string]map[string]*client) } // In case a config reload was first done on another server, // we may have already switched this account to a dedicated route. // But we still want to send the protocol over the routes that // would have otherwise handled it. if _, ok := s.accRoutes[an]; !ok { s.accRoutes[an] = make(map[string]*client) } if a, ok := s.accounts.Load(an); ok { acc := a.(*Account) acc.mu.Lock() sl := acc.sl // Get the current route pool index before calling setRouteInfo. rpi := acc.routePoolIdx // Switch to per-account route if not already done. if rpi >= 0 { s.setRouteInfo(acc) } else { // If it was transitioning, make sure we set it to the state // that indicates that it has a dedicated route if rpi == accTransitioningToDedicatedRoute { acc.routePoolIdx = accDedicatedRoute } // Otherwise get the route pool index it would have been before // the move so we can send the protocol to those routes. rpi = s.computeRoutePoolIdx(acc) } acc.mu.Unlock() // Generate the INFO protocol to send indicating that this account // is being moved to a dedicated route. ri := Info{ RoutePoolSize: s.routesPoolSize, RouteAccount: an, RouteAccReqID: s.accAddedReqID, } proto := generateInfoJSON(&ri) // Go over each remote's route at pool index `rpi` and remove // remote subs for this account and send the protocol. s.forEachRouteIdx(rpi, func(r *client) bool { r.mu.Lock() // Exclude routes to servers that don't support pooling. if !r.route.noPool { if subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 { sl.RemoveBatch(subs) } r.enqueueProto(proto) protosSent++ } r.mu.Unlock() return true }) } } if protosSent > 0 { s.accAddedCh = make(chan struct{}, protosSent) ch = s.accAddedCh } } // Collect routes that need to be closed. routes := make(map[*client]struct{}) // Collect the per-account routes that need to be closed. if len(co.accsRemoved) > 0 { for _, an := range co.accsRemoved { if remotes, ok := s.accRoutes[an]; ok && remotes != nil { for _, r := range remotes { if r != nil { r.setNoReconnect() routes[r] = struct{}{} } } } } } // If the pool size has changed, we need to close all pooled routes. if co.poolSizeChanged { s.forEachNonPerAccountRoute(func(r *client) { routes[r] = struct{}{} }) } // If there are routes to close, we need to release the server lock. // Same if we need to wait on responses from the remotes when // processing new per-account routes. if len(routes) > 0 || len(ch) > 0 { s.mu.Unlock() for done := false; !done && len(ch) > 0; { select { case <-ch: case <-time.After(2 * time.Second): s.Warnf("Timed out waiting for confirmation from all routes regarding per-account routes changes") done = true } } for r := range routes { r.closeConnection(RouteRemoved) } s.mu.Lock() } // Clear the accAddedCh/ReqID fields in case they were set. s.accAddedReqID, s.accAddedCh = _EMPTY_, nil // Now that per-account routes that needed to be closed are closed, // remove them from s.accRoutes. Doing so before would prevent // removeRoute() to do proper cleanup because the route would not // be found in s.accRoutes. for _, an := range co.accsRemoved { delete(s.accRoutes, an) // Do not lookup and call setRouteInfo() on the accounts here. // We need first to set the new s.routesPoolSize value and // anyway, there is no need to do here if the pool size has // changed (since it will be called for all accounts). } // We have already added the accounts to s.accRoutes that needed to // be added. // We should always have at least the system account with a dedicated route, // but in case we have a configuration that disables pooling and without // a system account, possibly set the accRoutes to nil. if len(opts.Cluster.PinnedAccounts) == 0 { s.accRoutes = nil } // Now deal with pool size updates. if ps := opts.Cluster.PoolSize; ps > 0 { s.routesPoolSize = ps s.routeInfo.RoutePoolSize = ps } else { s.routesPoolSize = 1 s.routeInfo.RoutePoolSize = 0 } // If the pool size has changed, we need to recompute all accounts' route // pool index. Note that the added/removed accounts will be reset there // too, but that's ok (we could use a map to exclude them, but not worth it). if co.poolSizeChanged { s.accounts.Range(func(_, v any) bool { acc := v.(*Account) acc.mu.Lock() s.setRouteInfo(acc) acc.mu.Unlock() return true }) } else if len(co.accsRemoved) > 0 { // For accounts that no longer have a dedicated route, we need to send // the subsriptions on the existing pooled routes for those accounts. for _, an := range co.accsRemoved { if a, ok := s.accounts.Load(an); ok { acc := a.(*Account) acc.mu.Lock() // First call this which will assign a new route pool index. s.setRouteInfo(acc) // Get the value so we can send the subscriptions interest // on all routes with this pool index. rpi := acc.routePoolIdx acc.mu.Unlock() s.forEachRouteIdx(rpi, func(r *client) bool { // We have the guarantee that if the route exists, it // is not a new one that would have been created when // we released the server lock if some routes needed // to be closed, because we have set s.routesReject // to `true` at the top of this function. s.sendSubsToRoute(r, rpi, an) return true }) } } } // Allow routes to be accepted now. s.routesReject = false // If there is a pool size change or added accounts, solicit routes now. if co.poolSizeChanged || len(co.accsAdded) > 0 { s.solicitRoutes(opts.Routes, co.accsAdded) } s.mu.Unlock() } // validateClusterOpts ensures the new ClusterOpts does not change some of the // fields that do not support reload. func validateClusterOpts(old, new ClusterOpts) error { if old.Host != new.Host { return fmt.Errorf("config reload not supported for cluster host: old=%s, new=%s", old.Host, new.Host) } if old.Port != new.Port { return fmt.Errorf("config reload not supported for cluster port: old=%d, new=%d", old.Port, new.Port) } // Validate Cluster.Advertise syntax if new.Advertise != "" { if _, _, err := parseHostPort(new.Advertise, 0); err != nil { return fmt.Errorf("invalid Cluster.Advertise value of %s, err=%v", new.Advertise, err) } } return nil } // diffRoutes diffs the old routes and the new routes and returns the ones that // should be added and removed from the server. func diffRoutes(old, new []*url.URL) (add, remove []*url.URL) { // Find routes to remove. removeLoop: for _, oldRoute := range old { for _, newRoute := range new { if urlsAreEqual(oldRoute, newRoute) { continue removeLoop } } remove = append(remove, oldRoute) } // Find routes to add. addLoop: for _, newRoute := range new { for _, oldRoute := range old { if urlsAreEqual(oldRoute, newRoute) { continue addLoop } } add = append(add, newRoute) } return add, remove } nats-server-2.10.27/server/reload_test.go000066400000000000000000005355701477524627100203340ustar00rootroot00000000000000// Copyright 2017-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "encoding/base64" "encoding/json" "flag" "fmt" "io" "log" "net" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "reflect" "runtime" "strings" "sync" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func newServerWithConfig(t *testing.T, configFile string) (*Server, *Options, string) { t.Helper() content, err := os.ReadFile(configFile) if err != nil { t.Fatalf("Error loading file: %v", err) } return newServerWithContent(t, content) } func newServerWithContent(t *testing.T, content []byte) (*Server, *Options, string) { t.Helper() opts, tmpFile := newOptionsFromContent(t, content) return New(opts), opts, tmpFile } func newOptionsFromContent(t *testing.T, content []byte) (*Options, string) { t.Helper() tmpFile := createConfFile(t, content) opts, err := ProcessConfigFile(tmpFile) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoSigs = true return opts, tmpFile } func createConfFile(t testing.TB, content []byte) string { t.Helper() conf := createTempFile(t, _EMPTY_) fName := conf.Name() conf.Close() if err := os.WriteFile(fName, content, 0666); err != nil { t.Fatalf("Error writing conf file: %v", err) } return fName } func runReloadServerWithConfig(t *testing.T, configFile string) (*Server, *Options, string) { t.Helper() content, err := os.ReadFile(configFile) if err != nil { t.Fatalf("Error loading file: %v", err) } return runReloadServerWithContent(t, content) } func runReloadServerWithContent(t *testing.T, content []byte) (*Server, *Options, string) { t.Helper() opts, tmpFile := newOptionsFromContent(t, content) opts.NoLog = true opts.NoSigs = true s := RunServer(opts) return s, opts, tmpFile } func changeCurrentConfigContent(t *testing.T, curConfig, newConfig string) { t.Helper() content, err := os.ReadFile(newConfig) if err != nil { t.Fatalf("Error loading file: %v", err) } changeCurrentConfigContentWithNewContent(t, curConfig, content) } func changeCurrentConfigContentWithNewContent(t *testing.T, curConfig string, content []byte) { t.Helper() if err := os.WriteFile(curConfig, content, 0666); err != nil { t.Fatalf("Error writing config: %v", err) } } // Ensure Reload returns an error when attempting to reload a server that did // not start with a config file. func TestConfigReloadNoConfigFile(t *testing.T) { server := New(&Options{NoSigs: true}) loaded := server.ConfigTime() if server.Reload() == nil { t.Fatal("Expected Reload to return an error") } if reloaded := server.ConfigTime(); reloaded != loaded { t.Fatalf("ConfigTime is incorrect.\nexpected: %s\ngot: %s", loaded, reloaded) } } // Ensure Reload returns an error when attempting to change an option which // does not support reloading. func TestConfigReloadUnsupported(t *testing.T) { server, _, config := newServerWithConfig(t, "./configs/reload/test.conf") defer server.Shutdown() loaded := server.ConfigTime() golden := &Options{ ConfigFile: config, Host: "0.0.0.0", Port: 2233, AuthTimeout: float64(AUTH_TIMEOUT / time.Second), Debug: false, Trace: false, Logtime: false, MaxControlLine: 4096, MaxPayload: 1048576, MaxConn: 65536, PingInterval: 2 * time.Minute, MaxPingsOut: 2, WriteDeadline: 10 * time.Second, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: -1, }, NoSigs: true, } setBaselineOptions(golden) checkOptionsEqual(t, golden, server.getOpts()) // Change config file to bad config. changeCurrentConfigContent(t, config, "./configs/reload/reload_unsupported.conf") // This should fail because `cluster` host cannot be changed. if err := server.Reload(); err == nil { t.Fatal("Expected Reload to return an error") } // Ensure config didn't change. checkOptionsEqual(t, golden, server.getOpts()) if reloaded := server.ConfigTime(); reloaded != loaded { t.Fatalf("ConfigTime is incorrect.\nexpected: %s\ngot: %s", loaded, reloaded) } } // This checks that if we change an option that does not support hot-swapping // we get an error. Using `listen` for now (test may need to be updated if // server is changed to support change of listen spec). func TestConfigReloadUnsupportedHotSwapping(t *testing.T) { server, _, config := newServerWithContent(t, []byte("listen: 127.0.0.1:-1")) defer server.Shutdown() loaded := server.ConfigTime() time.Sleep(time.Millisecond) // Change config file with unsupported option hot-swap changeCurrentConfigContentWithNewContent(t, config, []byte("listen: 127.0.0.1:9999")) // This should fail because `listen` host cannot be changed. if err := server.Reload(); err == nil || !strings.Contains(err.Error(), "not supported") { t.Fatalf("Expected Reload to return a not supported error, got %v", err) } if reloaded := server.ConfigTime(); reloaded != loaded { t.Fatalf("ConfigTime is incorrect.\nexpected: %s\ngot: %s", loaded, reloaded) } } // Ensure Reload returns an error when reloading from a bad config file. func TestConfigReloadInvalidConfig(t *testing.T) { server, _, config := newServerWithConfig(t, "./configs/reload/test.conf") defer server.Shutdown() loaded := server.ConfigTime() golden := &Options{ ConfigFile: config, Host: "0.0.0.0", Port: 2233, AuthTimeout: float64(AUTH_TIMEOUT / time.Second), Debug: false, Trace: false, Logtime: false, MaxControlLine: 4096, MaxPayload: 1048576, MaxConn: 65536, PingInterval: 2 * time.Minute, MaxPingsOut: 2, WriteDeadline: 10 * time.Second, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: -1, }, NoSigs: true, } setBaselineOptions(golden) checkOptionsEqual(t, golden, server.getOpts()) // Change config file to bad config. changeCurrentConfigContent(t, config, "./configs/reload/invalid.conf") // This should fail because the new config should not parse. if err := server.Reload(); err == nil { t.Fatal("Expected Reload to return an error") } // Ensure config didn't change. checkOptionsEqual(t, golden, server.getOpts()) if reloaded := server.ConfigTime(); reloaded != loaded { t.Fatalf("ConfigTime is incorrect.\nexpected: %s\ngot: %s", loaded, reloaded) } } // Ensure Reload returns nil and the config is changed on success. func TestConfigReload(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/test.conf") defer removeFile(t, "nats-server.pid") defer removeFile(t, "nats-server.log") defer server.Shutdown() var content []byte if runtime.GOOS != "windows" { content = []byte(` remote_syslog: "udp://127.0.0.1:514" # change on reload syslog: true # enable on reload `) } platformConf := filepath.Join(filepath.Dir(config), "platform.conf") if err := os.WriteFile(platformConf, content, 0666); err != nil { t.Fatalf("Unable to write config file: %v", err) } loaded := server.ConfigTime() golden := &Options{ ConfigFile: config, Host: "0.0.0.0", Port: 2233, AuthTimeout: float64(AUTH_TIMEOUT / time.Second), Debug: false, Trace: false, NoLog: true, Logtime: false, MaxControlLine: 4096, MaxPayload: 1048576, MaxConn: 65536, PingInterval: 2 * time.Minute, MaxPingsOut: 2, WriteDeadline: 10 * time.Second, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: server.ClusterAddr().Port, }, NoSigs: true, } setBaselineOptions(golden) checkOptionsEqual(t, golden, opts) // Change config file to new config. changeCurrentConfigContent(t, config, "./configs/reload/reload.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure config changed. updated := server.getOpts() if !updated.Trace { t.Fatal("Expected Trace to be true") } if !updated.Debug { t.Fatal("Expected Debug to be true") } if !updated.Logtime { t.Fatal("Expected Logtime to be true") } if !updated.LogtimeUTC { t.Fatal("Expected LogtimeUTC to be true") } if runtime.GOOS != "windows" { if !updated.Syslog { t.Fatal("Expected Syslog to be true") } if updated.RemoteSyslog != "udp://127.0.0.1:514" { t.Fatalf("RemoteSyslog is incorrect.\nexpected: udp://127.0.0.1:514\ngot: %s", updated.RemoteSyslog) } } if updated.LogFile != "nats-server.log" { t.Fatalf("LogFile is incorrect.\nexpected: nats-server.log\ngot: %s", updated.LogFile) } if updated.TLSConfig == nil { t.Fatal("Expected TLSConfig to be non-nil") } if !server.info.TLSRequired { t.Fatal("Expected TLSRequired to be true") } if !server.info.TLSVerify { t.Fatal("Expected TLSVerify to be true") } if updated.Username != "tyler" { t.Fatalf("Username is incorrect.\nexpected: tyler\ngot: %s", updated.Username) } if updated.Password != "T0pS3cr3t" { t.Fatalf("Password is incorrect.\nexpected: T0pS3cr3t\ngot: %s", updated.Password) } if updated.AuthTimeout != 2 { t.Fatalf("AuthTimeout is incorrect.\nexpected: 2\ngot: %f", updated.AuthTimeout) } if !server.info.AuthRequired { t.Fatal("Expected AuthRequired to be true") } if !updated.Cluster.NoAdvertise { t.Fatal("Expected NoAdvertise to be true") } if updated.Cluster.PingInterval != 20*time.Second { t.Fatalf("Cluster PingInterval is incorrect.\nexpected: 20s\ngot: %v", updated.Cluster.PingInterval) } if updated.Cluster.MaxPingsOut != 8 { t.Fatalf("Cluster MaxPingsOut is incorrect.\nexpected: 6\ngot: %v", updated.Cluster.MaxPingsOut) } if updated.PidFile != "nats-server.pid" { t.Fatalf("PidFile is incorrect.\nexpected: nats-server.pid\ngot: %s", updated.PidFile) } if updated.MaxControlLine != 512 { t.Fatalf("MaxControlLine is incorrect.\nexpected: 512\ngot: %d", updated.MaxControlLine) } if updated.PingInterval != 5*time.Second { t.Fatalf("PingInterval is incorrect.\nexpected 5s\ngot: %s", updated.PingInterval) } if updated.MaxPingsOut != 1 { t.Fatalf("MaxPingsOut is incorrect.\nexpected 1\ngot: %d", updated.MaxPingsOut) } if updated.WriteDeadline != 3*time.Second { t.Fatalf("WriteDeadline is incorrect.\nexpected 3s\ngot: %s", updated.WriteDeadline) } if updated.MaxPayload != 1024 { t.Fatalf("MaxPayload is incorrect.\nexpected 1024\ngot: %d", updated.MaxPayload) } if reloaded := server.ConfigTime(); !reloaded.After(loaded) { t.Fatalf("ConfigTime is incorrect.\nexpected greater than: %s\ngot: %s", loaded, reloaded) } } // Ensure Reload supports TLS config changes. Test this by starting a server // with TLS enabled, connect to it to verify, reload config using a different // key pair and client verification enabled, ensure reconnect fails, then // ensure reconnect succeeds when the client provides a cert. func TestConfigReloadRotateTLS(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/tls_test.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() // Rotate cert and enable client verification. changeCurrentConfigContent(t, config, "./configs/reload/tls_verify_test.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when client presents cert. cert := nats.ClientCert("./configs/certs/cert.new.pem", "./configs/certs/key.new.pem") conn, err := nats.Connect(addr, cert, nats.RootCAs("./configs/certs/cert.new.pem")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the original connection can still publish/receive. if err := nc.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } nc.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } } // Ensure Reload supports enabling TLS. Test this by starting a server without // TLS enabled, connect to it to verify, reload config with TLS enabled, ensure // reconnect fails, then ensure reconnect succeeds when using secure. func TestConfigReloadEnableTLS(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } nc.Close() // Enable TLS. changeCurrentConfigContent(t, config, "./configs/reload/tls_test.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting is OK (we need to skip server cert verification since // the library is not doing that by default now). nc, err = nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})) if err != nil { t.Fatalf("Error creating client: %v", err) } nc.Close() } // Ensure Reload supports disabling TLS. Test this by starting a server with // TLS enabled, connect to it to verify, reload config with TLS disabled, // ensure reconnect fails, then ensure reconnect succeeds when connecting // without secure. func TestConfigReloadDisableTLS(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/tls_test.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})) if err != nil { t.Fatalf("Error creating client: %v", err) } nc.Close() // Disable TLS. changeCurrentConfigContent(t, config, "./configs/reload/basic.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when not using secure. nc, err = nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } nc.Close() } func TestConfigReloadRotateTLSMultiCert(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/tls_multi_cert_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) rawCerts := make(chan []byte, 3) nc, err := nats.Connect(addr, nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { rawCerts <- s.PeerCertificates[0].Raw return nil }, InsecureSkipVerify: true, })) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() // Rotate cert and enable client verification. changeCurrentConfigContent(t, config, "./configs/reload/tls_multi_cert_2.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when client presents cert. cert := nats.ClientCert("../test/configs/certs/client-cert.pem", "../test/configs/certs/client-key.pem") conn, err := nats.Connect(addr, cert, nats.RootCAs("../test/configs/certs/ca.pem"), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { rawCerts <- s.PeerCertificates[0].Raw return nil }, })) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the original connection can still publish/receive. if err := nc.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } nc.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } // Rotate cert and disable client verification. changeCurrentConfigContent(t, config, "./configs/reload/tls_multi_cert_3.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } nc, err = nats.Connect(addr, nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { rawCerts <- s.PeerCertificates[0].Raw return nil }, InsecureSkipVerify: true, })) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() sub, err = nc.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() certA := <-rawCerts certB := <-rawCerts certC := <-rawCerts if !bytes.Equal(certA, certB) { t.Error("Expected the same cert") } if bytes.Equal(certB, certC) { t.Error("Expected a different cert") } } // Ensure Reload supports single user authentication config changes. Test this // by starting a server with authentication enabled, connect to it to verify, // reload config using a different username/password, ensure reconnect fails, // then ensure reconnect succeeds when using the correct credentials. func TestConfigReloadRotateUserAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/single_user_authentication_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("tyler", "T0pS3cr3t")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() disconnected := make(chan struct{}, 1) asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) nc.SetDisconnectHandler(func(*nats.Conn) { disconnected <- struct{}{} }) // Change user credentials. changeCurrentConfigContent(t, config, "./configs/reload/single_user_authentication_2.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.UserInfo("tyler", "T0pS3cr3t")); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.UserInfo("derek", "passw0rd")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. // Note that it is possible that client gets EOF and not able to // process async error, so don't fail if we don't get it. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(time.Second): // Give it up to 1 sec. } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(2 * time.Second): t.Fatal("Expected connection to be disconnected") } } // Ensure Reload supports enabling single user authentication. Test this by // starting a server with authentication disabled, connect to it to verify, // reload config using with a username/password, ensure reconnect fails, then // ensure reconnect succeeds when using the correct credentials. func TestConfigReloadEnableUserAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() disconnected := make(chan struct{}, 1) asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) nc.SetDisconnectHandler(func(*nats.Conn) { disconnected <- struct{}{} }) // Enable authentication. changeCurrentConfigContent(t, config, "./configs/reload/single_user_authentication_1.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.UserInfo("tyler", "T0pS3cr3t")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. // Note that it is possible that client gets EOF and not able to // process async error, so don't fail if we don't get it. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(time.Second): } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(2 * time.Second): t.Fatal("Expected connection to be disconnected") } } // Ensure Reload supports disabling single user authentication. Test this by // starting a server with authentication enabled, connect to it to verify, // reload config using with authentication disabled, then ensure connecting // with no credentials succeeds. func TestConfigReloadDisableUserAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/single_user_authentication_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("tyler", "T0pS3cr3t")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { t.Fatalf("Client received an unexpected error: %v", err) }) // Disable authentication. changeCurrentConfigContent(t, config, "./configs/reload/basic.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting succeeds with no credentials. conn, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() } // Ensure Reload supports token authentication config changes. Test this by // starting a server with token authentication enabled, connect to it to // verify, reload config using a different token, ensure reconnect fails, then // ensure reconnect succeeds when using the correct token. func TestConfigReloadRotateTokenAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/token_authentication_1.conf") defer server.Shutdown() disconnected := make(chan struct{}) asyncErr := make(chan error) eh := func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err } dh := func(*nats.Conn) { disconnected <- struct{}{} } // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.Token("T0pS3cr3t"), nats.ErrorHandler(eh), nats.DisconnectHandler(dh)) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() // Change authentication token. changeCurrentConfigContent(t, config, "./configs/reload/token_authentication_2.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.Token("T0pS3cr3t")); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.Token("passw0rd")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(2 * time.Second): t.Fatal("Expected authorization error") } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(2 * time.Second): t.Fatal("Expected connection to be disconnected") } } // Ensure Reload supports enabling token authentication. Test this by starting // a server with authentication disabled, connect to it to verify, reload // config using with a token, ensure reconnect fails, then ensure reconnect // succeeds when using the correct token. func TestConfigReloadEnableTokenAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() disconnected := make(chan struct{}, 1) asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) nc.SetDisconnectHandler(func(*nats.Conn) { disconnected <- struct{}{} }) // Enable authentication. changeCurrentConfigContent(t, config, "./configs/reload/token_authentication_1.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.Token("T0pS3cr3t")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. // Note that it is possible that client gets EOF and not able to // process async error, so don't fail if we don't get it. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(time.Second): } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(2 * time.Second): t.Fatal("Expected connection to be disconnected") } } // Ensure Reload supports disabling single token authentication. Test this by // starting a server with authentication enabled, connect to it to verify, // reload config using with authentication disabled, then ensure connecting // with no token succeeds. func TestConfigReloadDisableTokenAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/token_authentication_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.Token("T0pS3cr3t")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { t.Fatalf("Client received an unexpected error: %v", err) }) // Disable authentication. changeCurrentConfigContent(t, config, "./configs/reload/basic.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting succeeds with no credentials. conn, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() } // Ensure Reload supports users authentication config changes. Test this by // starting a server with users authentication enabled, connect to it to // verify, reload config using a different user, ensure reconnect fails, then // ensure reconnect succeeds when using the correct credentials. func TestConfigReloadRotateUsersAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/multiple_users_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("alice", "foo")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() disconnected := make(chan struct{}, 1) asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) nc.SetDisconnectHandler(func(*nats.Conn) { disconnected <- struct{}{} }) // These credentials won't change. nc2, err := nats.Connect(addr, nats.UserInfo("bob", "bar")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc2.Close() sub, err := nc2.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() // Change users credentials. changeCurrentConfigContent(t, config, "./configs/reload/multiple_users_2.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr, nats.UserInfo("alice", "foo")); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.UserInfo("alice", "baz")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. // Note that it is possible that client gets EOF and not able to // process async error, so don't fail if we don't get it. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(time.Second): } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(2 * time.Second): t.Fatal("Expected connection to be disconnected") } // Ensure the connection using unchanged credentials can still // publish/receive. if err := nc2.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } nc2.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } } // Ensure Reload supports enabling users authentication. Test this by starting // a server with authentication disabled, connect to it to verify, reload // config using with users, ensure reconnect fails, then ensure reconnect // succeeds when using the correct credentials. func TestConfigReloadEnableUsersAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() disconnected := make(chan struct{}, 1) asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) nc.SetDisconnectHandler(func(*nats.Conn) { disconnected <- struct{}{} }) // Enable authentication. changeCurrentConfigContent(t, config, "./configs/reload/multiple_users_1.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting fails. if _, err := nats.Connect(addr); err == nil { t.Fatal("Expected connect to fail") } // Ensure connecting succeeds when using new credentials. conn, err := nats.Connect(addr, nats.UserInfo("alice", "foo")) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() // Ensure the previous connection received an authorization error. // Note that it is possible that client gets EOF and not able to // process async error, so don't fail if we don't get it. select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(time.Second): } // Ensure the previous connection was disconnected. select { case <-disconnected: case <-time.After(5 * time.Second): t.Fatal("Expected connection to be disconnected") } } // Ensure Reload supports disabling users authentication. Test this by starting // a server with authentication enabled, connect to it to verify, // reload config using with authentication disabled, then ensure connecting // with no credentials succeeds. func TestConfigReloadDisableUsersAuthentication(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/multiple_users_1.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("alice", "foo")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { t.Fatalf("Client received an unexpected error: %v", err) }) // Disable authentication. changeCurrentConfigContent(t, config, "./configs/reload/basic.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure connecting succeeds with no credentials. conn, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } conn.Close() } // Ensure Reload supports changing permissions. Test this by starting a server // with a user configured with certain permissions, test publish and subscribe, // reload config with new permissions, ensure the previous subscription was // closed and publishes fail, then ensure the new permissions succeed. func TestConfigReloadChangePermissions(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/authorization_1.conf") defer server.Shutdown() addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("bob", "bar")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) // Ensure we can publish and receive messages as a sanity check. sub, err := nc.SubscribeSync("_INBOX.>") if err != nil { t.Fatalf("Error subscribing: %v", err) } nc.Flush() conn, err := nats.Connect(addr, nats.UserInfo("alice", "foo")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer conn.Close() sub2, err := conn.SubscribeSync("req.foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } if err := conn.Publish("_INBOX.foo", []byte("hello")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } if err := nc.Publish("req.foo", []byte("world")); err != nil { t.Fatalf("Error publishing message: %v", err) } nc.Flush() msg, err = sub2.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "world" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("world"), msg.Data) } // Susan will subscribe to two subjects, both will succeed but a send to foo.bar should not succeed // however PUBLIC.foo should. sconn, err := nats.Connect(addr, nats.UserInfo("susan", "baz")) if err != nil { t.Fatalf("Error creating client: %v", err) } defer sconn.Close() asyncErr2 := make(chan error, 1) sconn.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr2 <- err }) fooSub, err := sconn.SubscribeSync("foo.*") if err != nil { t.Fatalf("Error subscribing: %v", err) } sconn.Flush() // Publishing from bob on foo.bar should not come through. if err := conn.Publish("foo.bar", []byte("hello")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() _, err = fooSub.NextMsg(100 * time.Millisecond) if err != nats.ErrTimeout { t.Fatalf("Received a message we shouldn't have") } pubSub, err := sconn.SubscribeSync("PUBLIC.*") if err != nil { t.Fatalf("Error subscribing: %v", err) } sconn.Flush() select { case err := <-asyncErr2: t.Fatalf("Received unexpected error for susan: %v", err) default: } // This should work ok with original config. if err := conn.Publish("PUBLIC.foo", []byte("hello monkey")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() msg, err = pubSub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello monkey" { t.Fatalf("Msg is incorrect.\nexpected: %q\ngot: %q", "hello monkey", msg.Data) } /////////////////////////////////////////// // Change permissions. /////////////////////////////////////////// changeCurrentConfigContent(t, config, "./configs/reload/authorization_2.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure we receive an error for the subscription that is no longer authorized. // In this test, since connection is not closed by the server, // the client must receive an -ERR select { case err := <-asyncErr: if !strings.Contains(strings.ToLower(err.Error()), "permissions violation for subscription to \"_inbox.>\"") { t.Fatalf("Expected permissions violation error, got %v", err) } case <-time.After(5 * time.Second): t.Fatal("Expected permissions violation error") } // Ensure we receive an error when publishing to req.foo and we no longer // receive messages on _INBOX.>. if err := nc.Publish("req.foo", []byte("hola")); err != nil { t.Fatalf("Error publishing message: %v", err) } nc.Flush() if err := conn.Publish("_INBOX.foo", []byte("mundo")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() select { case err := <-asyncErr: if !strings.Contains(strings.ToLower(err.Error()), "permissions violation for publish to \"req.foo\"") { t.Fatalf("Expected permissions violation error, got %v", err) } case <-time.After(5 * time.Second): t.Fatal("Expected permissions violation error") } queued, _, err := sub2.Pending() if err != nil { t.Fatalf("Failed to get pending messaged: %v", err) } if queued != 0 { t.Fatalf("Pending is incorrect.\nexpected: 0\ngot: %d", queued) } queued, _, err = sub.Pending() if err != nil { t.Fatalf("Failed to get pending messaged: %v", err) } if queued != 0 { t.Fatalf("Pending is incorrect.\nexpected: 0\ngot: %d", queued) } // Ensure we can publish to _INBOX.foo.bar and subscribe to _INBOX.foo.>. sub, err = nc.SubscribeSync("_INBOX.foo.>") if err != nil { t.Fatalf("Error subscribing: %v", err) } nc.Flush() if err := nc.Publish("_INBOX.foo.bar", []byte("testing")); err != nil { t.Fatalf("Error publishing message: %v", err) } nc.Flush() msg, err = sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "testing" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("testing"), msg.Data) } select { case err := <-asyncErr: t.Fatalf("Received unexpected error: %v", err) default: } // Now check susan again. // // This worked ok with original config but should not deliver a message now. if err := conn.Publish("PUBLIC.foo", []byte("hello monkey")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() _, err = pubSub.NextMsg(100 * time.Millisecond) if err != nats.ErrTimeout { t.Fatalf("Received a message we shouldn't have") } // Now check foo.bar, which did not work before but should work now.. if err := conn.Publish("foo.bar", []byte("hello?")); err != nil { t.Fatalf("Error publishing message: %v", err) } conn.Flush() msg, err = fooSub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving msg: %v", err) } if string(msg.Data) != "hello?" { t.Fatalf("Msg is incorrect.\nexpected: %q\ngot: %q", "hello?", msg.Data) } // Once last check for no errors. sconn.Flush() select { case err := <-asyncErr2: t.Fatalf("Received unexpected error for susan: %v", err) default: } } // Ensure Reload returns an error when attempting to change cluster address // host. func TestConfigReloadClusterHostUnsupported(t *testing.T) { server, _, config := runReloadServerWithConfig(t, "./configs/reload/srv_a_1.conf") defer server.Shutdown() // Attempt to change cluster listen host. changeCurrentConfigContent(t, config, "./configs/reload/srv_c_1.conf") // This should fail because cluster address cannot be changed. if err := server.Reload(); err == nil { t.Fatal("Expected Reload to return an error") } } // Ensure Reload returns an error when attempting to change cluster address // port. func TestConfigReloadClusterPortUnsupported(t *testing.T) { server, _, config := runReloadServerWithConfig(t, "./configs/reload/srv_a_1.conf") defer server.Shutdown() // Attempt to change cluster listen port. changeCurrentConfigContent(t, config, "./configs/reload/srv_b_1.conf") // This should fail because cluster address cannot be changed. if err := server.Reload(); err == nil { t.Fatal("Expected Reload to return an error") } } // Ensure Reload supports enabling route authorization. Test this by starting // two servers in a cluster without authorization, ensuring messages flow // between them, then reloading with authorization and ensuring messages no // longer flow until reloading with the correct credentials. func TestConfigReloadEnableClusterAuthorization(t *testing.T) { srvb, srvbOpts, srvbConfig := runReloadServerWithConfig(t, "./configs/reload/srv_b_1.conf") defer srvb.Shutdown() srva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, "./configs/reload/srv_a_1.conf") defer srva.Shutdown() checkClusterFormed(t, srva, srvb) srvaAddr := fmt.Sprintf("nats://%s:%d", srvaOpts.Host, srvaOpts.Port) srvaConn, err := nats.Connect(srvaAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvaConn.Close() sub, err := srvaConn.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() if err := srvaConn.Flush(); err != nil { t.Fatalf("Error flushing: %v", err) } srvbAddr := fmt.Sprintf("nats://%s:%d", srvbOpts.Host, srvbOpts.Port) srvbConn, err := nats.Connect(srvbAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvbConn.Close() if numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, numRoutes) } // Ensure messages flow through the cluster as a sanity check. if err := srvbConn.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } // Enable route authorization. changeCurrentConfigContent(t, srvbConfig, "./configs/reload/srv_b_2.conf") if err := srvb.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } checkNumRoutes(t, srvb, 0) // Ensure messages no longer flow through the cluster. for i := 0; i < 5; i++ { if err := srvbConn.Publish("foo", []byte("world")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() } if _, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout { t.Fatalf("Expected ErrTimeout, got %v", err) } // Reload Server A with correct route credentials. changeCurrentConfigContent(t, srvaConfig, "./configs/reload/srv_a_2.conf") if err := srva.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } checkClusterFormed(t, srva, srvb) if numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, numRoutes) } // Ensure messages flow through the cluster now. if err := srvbConn.Publish("foo", []byte("hola")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err = sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hola" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hola"), msg.Data) } } // Ensure Reload supports disabling route authorization. Test this by starting // two servers in a cluster with authorization, ensuring messages flow // between them, then reloading without authorization and ensuring messages // still flow. func TestConfigReloadDisableClusterAuthorization(t *testing.T) { srvb, srvbOpts, srvbConfig := runReloadServerWithConfig(t, "./configs/reload/srv_b_2.conf") defer srvb.Shutdown() srva, srvaOpts, _ := runReloadServerWithConfig(t, "./configs/reload/srv_a_2.conf") defer srva.Shutdown() checkClusterFormed(t, srva, srvb) srvaAddr := fmt.Sprintf("nats://%s:%d", srvaOpts.Host, srvaOpts.Port) srvaConn, err := nats.Connect(srvaAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvaConn.Close() sub, err := srvaConn.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() if err := srvaConn.Flush(); err != nil { t.Fatalf("Error flushing: %v", err) } srvbAddr := fmt.Sprintf("nats://%s:%d", srvbOpts.Host, srvbOpts.Port) srvbConn, err := nats.Connect(srvbAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvbConn.Close() if numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, numRoutes) } // Ensure messages flow through the cluster as a sanity check. if err := srvbConn.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } // Disable route authorization. changeCurrentConfigContent(t, srvbConfig, "./configs/reload/srv_b_1.conf") if err := srvb.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } checkClusterFormed(t, srva, srvb) if numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, numRoutes) } // Ensure messages still flow through the cluster. if err := srvbConn.Publish("foo", []byte("hola")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err = sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hola" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hola"), msg.Data) } } // Ensure Reload supports changing cluster routes. Test this by starting // two servers in a cluster, ensuring messages flow between them, then // reloading with a different route and ensuring messages flow through the new // cluster. func TestConfigReloadClusterRoutes(t *testing.T) { srvb, srvbOpts, _ := runReloadServerWithConfig(t, "./configs/reload/srv_b_1.conf") defer srvb.Shutdown() srva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, "./configs/reload/srv_a_1.conf") defer srva.Shutdown() checkClusterFormed(t, srva, srvb) srvcOpts, err := ProcessConfigFile("./configs/reload/srv_c_1.conf") if err != nil { t.Fatalf("Error processing config file: %v", err) } srvcOpts.NoLog = true srvcOpts.NoSigs = true srvc := RunServer(srvcOpts) defer srvc.Shutdown() srvaAddr := fmt.Sprintf("nats://%s:%d", srvaOpts.Host, srvaOpts.Port) srvaConn, err := nats.Connect(srvaAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvaConn.Close() sub, err := srvaConn.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() if err := srvaConn.Flush(); err != nil { t.Fatalf("Error flushing: %v", err) } srvbAddr := fmt.Sprintf("nats://%s:%d", srvbOpts.Host, srvbOpts.Port) srvbConn, err := nats.Connect(srvbAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvbConn.Close() if numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE { t.Fatalf("Expected %d route, got %d", DEFAULT_ROUTE_POOL_SIZE, numRoutes) } // Ensure consumer on srvA is propagated to srvB checkExpectedSubs(t, 1, srvb) // Ensure messages flow through the cluster as a sanity check. if err := srvbConn.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err := sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } // Reload cluster routes. changeCurrentConfigContent(t, srvaConfig, "./configs/reload/srv_a_3.conf") if err := srva.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Kill old route server. srvbConn.Close() srvb.Shutdown() checkClusterFormed(t, srva, srvc) srvcAddr := fmt.Sprintf("nats://%s:%d", srvcOpts.Host, srvcOpts.Port) srvcConn, err := nats.Connect(srvcAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvcConn.Close() // Ensure messages flow through the new cluster. for i := 0; i < 5; i++ { if err := srvcConn.Publish("foo", []byte("hola")); err != nil { t.Fatalf("Error publishing: %v", err) } srvcConn.Flush() } msg, err = sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hola" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hola"), msg.Data) } } // Ensure Reload supports removing a solicited route. In this case from A->B // Test this by starting two servers in a cluster, ensuring messages flow between them. // Then stop server B, and have server A continue to try to connect. Reload A with a config // that removes the route and make sure it does not connect to server B when its restarted. func TestConfigReloadClusterRemoveSolicitedRoutes(t *testing.T) { srvb, srvbOpts := RunServerWithConfig("./configs/reload/srv_b_1.conf") defer srvb.Shutdown() srva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, "./configs/reload/srv_a_1.conf") defer srva.Shutdown() checkClusterFormed(t, srva, srvb) srvaAddr := fmt.Sprintf("nats://%s:%d", srvaOpts.Host, srvaOpts.Port) srvaConn, err := nats.Connect(srvaAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvaConn.Close() sub, err := srvaConn.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } defer sub.Unsubscribe() if err := srvaConn.Flush(); err != nil { t.Fatalf("Error flushing: %v", err) } checkExpectedSubs(t, 1, srvb) srvbAddr := fmt.Sprintf("nats://%s:%d", srvbOpts.Host, srvbOpts.Port) srvbConn, err := nats.Connect(srvbAddr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer srvbConn.Close() if err := srvbConn.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } srvbConn.Flush() msg, err := sub.NextMsg(5 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } if string(msg.Data) != "hello" { t.Fatalf("Msg is incorrect.\nexpected: %+v\ngot: %+v", []byte("hello"), msg.Data) } // Now stop server B. srvb.Shutdown() // Wait til route is dropped. checkNumRoutes(t, srva, 0) // Now change config for server A to not solicit a route to server B. changeCurrentConfigContent(t, srvaConfig, "./configs/reload/srv_a_4.conf") if err := srva.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Restart server B. srvb, _ = RunServerWithConfig("./configs/reload/srv_b_1.conf") defer srvb.Shutdown() // We should not have a cluster formed here. numRoutes := 0 deadline := time.Now().Add(2 * DEFAULT_ROUTE_RECONNECT) for time.Now().Before(deadline) { if numRoutes = srva.NumRoutes(); numRoutes != 0 { break } else { time.Sleep(100 * time.Millisecond) } } if numRoutes != 0 { t.Fatalf("Expected 0 routes for server A, got %d", numRoutes) } } func reloadUpdateConfig(t *testing.T, s *Server, conf, content string) { t.Helper() if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error creating config file: %v", err) } if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } } func TestConfigReloadClusterAdvertise(t *testing.T) { s, _, conf := runReloadServerWithContent(t, []byte(` listen: "0.0.0.0:-1" cluster: { listen: "0.0.0.0:-1" } `)) defer s.Shutdown() orgClusterPort := s.ClusterAddr().Port verify := func(expectedHost string, expectedPort int, expectedIP string) { s.mu.Lock() routeInfo := s.routeInfo rij := generateInfoJSON(&routeInfo) s.mu.Unlock() if routeInfo.Host != expectedHost || routeInfo.Port != expectedPort || routeInfo.IP != expectedIP { t.Fatalf("Expected host/port/IP to be %s:%v, %q, got %s:%d, %q", expectedHost, expectedPort, expectedIP, routeInfo.Host, routeInfo.Port, routeInfo.IP) } routeInfoJSON := Info{} err := json.Unmarshal(rij[5:], &routeInfoJSON) // Skip "INFO " if err != nil { t.Fatalf("Error on Unmarshal: %v", err) } // Check that server routeInfoJSON was updated too if !reflect.DeepEqual(routeInfo, routeInfoJSON) { t.Fatalf("Expected routeInfoJSON to be %+v, got %+v", routeInfo, routeInfoJSON) } } // Update config with cluster_advertise reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" cluster: { listen: "0.0.0.0:-1" cluster_advertise: "me:1" } `) verify("me", 1, "nats-route://me:1/") // Update config with cluster_advertise (no port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" cluster: { listen: "0.0.0.0:-1" cluster_advertise: "me" } `) verify("me", orgClusterPort, fmt.Sprintf("nats-route://me:%d/", orgClusterPort)) // Update config with cluster_advertise (-1 port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" cluster: { listen: "0.0.0.0:-1" cluster_advertise: "me:-1" } `) verify("me", orgClusterPort, fmt.Sprintf("nats-route://me:%d/", orgClusterPort)) // Update to remove cluster_advertise reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" cluster: { listen: "0.0.0.0:-1" } `) verify("0.0.0.0", orgClusterPort, "") } func TestConfigReloadClusterNoAdvertise(t *testing.T) { s, _, conf := runReloadServerWithContent(t, []byte(` listen: "0.0.0.0:-1" client_advertise: "me:1" cluster: { listen: "0.0.0.0:-1" } `)) defer s.Shutdown() s.mu.Lock() ccurls := s.routeInfo.ClientConnectURLs s.mu.Unlock() if len(ccurls) != 1 && ccurls[0] != "me:1" { t.Fatalf("Unexpected routeInfo.ClientConnectURLS: %v", ccurls) } // Update config with no_advertise reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" client_advertise: "me:1" cluster: { listen: "0.0.0.0:-1" no_advertise: true } `) s.mu.Lock() ccurls = s.routeInfo.ClientConnectURLs s.mu.Unlock() if len(ccurls) != 0 { t.Fatalf("Unexpected routeInfo.ClientConnectURLS: %v", ccurls) } // Update config with cluster_advertise (no port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" client_advertise: "me:1" cluster: { listen: "0.0.0.0:-1" } `) s.mu.Lock() ccurls = s.routeInfo.ClientConnectURLs s.mu.Unlock() if len(ccurls) != 1 && ccurls[0] != "me:1" { t.Fatalf("Unexpected routeInfo.ClientConnectURLS: %v", ccurls) } } func TestConfigReloadClusterName(t *testing.T) { s, _, conf := runReloadServerWithContent(t, []byte(` listen: "0.0.0.0:-1" cluster: { name: "abc" listen: "0.0.0.0:-1" } `)) defer s.Shutdown() // Update config with a new cluster name. reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" cluster: { name: "xyz" listen: "0.0.0.0:-1" } `) if s.ClusterName() != "xyz" { t.Fatalf("Expected update clustername of \"xyz\", got %q", s.ClusterName()) } } func TestConfigReloadMaxSubsUnsupported(t *testing.T) { s, _, conf := runReloadServerWithContent(t, []byte(` port: -1 max_subs: 1 `)) defer s.Shutdown() if err := os.WriteFile(conf, []byte(`max_subs: 10`), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } if err := s.Reload(); err == nil { t.Fatal("Expected Reload to return an error") } } func TestConfigReloadClientAdvertise(t *testing.T) { s, _, conf := runReloadServerWithContent(t, []byte(`listen: "0.0.0.0:-1"`)) defer s.Shutdown() orgPort := s.Addr().(*net.TCPAddr).Port verify := func(expectedHost string, expectedPort int) { s.mu.Lock() info := s.info s.mu.Unlock() if info.Host != expectedHost || info.Port != expectedPort { stackFatalf(t, "Expected host/port to be %s:%d, got %s:%d", expectedHost, expectedPort, info.Host, info.Port) } } // Update config with ClientAdvertise (port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" client_advertise: "me:1" `) verify("me", 1) // Update config with ClientAdvertise (no port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" client_advertise: "me" `) verify("me", orgPort) // Update config with ClientAdvertise (-1 port specified) reloadUpdateConfig(t, s, conf, ` listen: "0.0.0.0:-1" client_advertise: "me:-1" `) verify("me", orgPort) // Now remove ClientAdvertise to check that original values // are restored. reloadUpdateConfig(t, s, conf, `listen: "0.0.0.0:-1"`) verify("0.0.0.0", orgPort) } // Ensure Reload supports changing the max connections. Test this by starting a // server with no max connections, connecting two clients, reloading with a // max connections of one, and ensuring one client is disconnected. func TestConfigReloadMaxConnections(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Make two connections. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc1, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc1.Close() closed := make(chan struct{}, 1) nc1.SetDisconnectHandler(func(*nats.Conn) { closed <- struct{}{} }) nc2, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc2.Close() nc2.SetDisconnectHandler(func(*nats.Conn) { closed <- struct{}{} }) if numClients := server.NumClients(); numClients != 2 { t.Fatalf("Expected 2 clients, got %d", numClients) } // Set max connections to one. changeCurrentConfigContent(t, config, "./configs/reload/max_connections.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure one connection was closed. select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Expected to be disconnected") } checkClientsCount(t, server, 1) // Ensure new connections fail. _, err = nats.Connect(addr) if err == nil { t.Fatal("Expected error on connect") } } // Ensure reload supports changing the max payload size. Test this by starting // a server with the default size limit, ensuring publishes work, reloading // with a restrictive limit, and ensuring publishing an oversized message fails // and disconnects the client. func TestConfigReloadMaxPayload(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() closed := make(chan struct{}) nc.SetDisconnectHandler(func(*nats.Conn) { closed <- struct{}{} }) conn, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer conn.Close() sub, err := conn.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } conn.Flush() // Ensure we can publish as a sanity check. if err := nc.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } nc.Flush() _, err = sub.NextMsg(2 * time.Second) if err != nil { t.Fatalf("Error receiving message: %v", err) } // Set max payload to one. changeCurrentConfigContent(t, config, "./configs/reload/max_payload.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Ensure oversized messages don't get delivered and the client is // disconnected. if err := nc.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error publishing: %v", err) } nc.Flush() _, err = sub.NextMsg(20 * time.Millisecond) if err != nats.ErrTimeout { t.Fatalf("Expected ErrTimeout, got: %v", err) } select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Expected to be disconnected") } } // Ensure reload supports rotating out files. Test this by starting // a server with log and pid files, reloading new ones, then check that // we can rename and delete the old log/pid files. func TestConfigReloadRotateFiles(t *testing.T) { server, _, config := runReloadServerWithConfig(t, "./configs/reload/file_rotate.conf") defer func() { removeFile(t, "log1.txt") removeFile(t, "nats-server1.pid") }() defer server.Shutdown() // Configure the logger to enable actual logging opts := server.getOpts() opts.NoLog = false server.ConfigureLogger() // Load a config that renames the files. changeCurrentConfigContent(t, config, "./configs/reload/file_rotate1.conf") if err := server.Reload(); err != nil { t.Fatalf("Error reloading config: %v", err) } // Make sure the new files exist. if _, err := os.Stat("log1.txt"); os.IsNotExist(err) { t.Fatalf("Error reloading config, no new file: %v", err) } if _, err := os.Stat("nats-server1.pid"); os.IsNotExist(err) { t.Fatalf("Error reloading config, no new file: %v", err) } // Check that old file can be renamed. if err := os.Rename("log.txt", "log_old.txt"); err != nil { t.Fatalf("Error reloading config, cannot rename file: %v", err) } if err := os.Rename("nats-server.pid", "nats-server_old.pid"); err != nil { t.Fatalf("Error reloading config, cannot rename file: %v", err) } // Check that the old files can be removed after rename. removeFile(t, "log_old.txt") removeFile(t, "nats-server_old.pid") } func TestConfigReloadClusterWorks(t *testing.T) { confBTemplate := ` listen: -1 cluster: { listen: 127.0.0.1:7244 authorization { user: ruser password: pwd timeout: %d } routes = [ nats-route://ruser:pwd@127.0.0.1:7246 ] }` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, 3))) confATemplate := ` listen: -1 cluster: { listen: 127.0.0.1:7246 authorization { user: ruser password: pwd timeout: %d } routes = [ nats-route://ruser:pwd@127.0.0.1:7244 ] }` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, 3))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() srva, _ := RunServerWithConfig(confA) defer srva.Shutdown() // Wait for the cluster to form and capture the connection IDs of each route checkClusterFormed(t, srva, srvb) getCID := func(s *Server) uint64 { s.mu.Lock() defer s.mu.Unlock() if r := getFirstRoute(s); r != nil { return r.cid } return 0 } acid := getCID(srva) bcid := getCID(srvb) // Update auth timeout to force a check of the connected route auth reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, 5)) reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, 5)) // Wait a little bit to ensure that there is no issue with connection // breaking at this point (this was an issue before). time.Sleep(100 * time.Millisecond) // Cluster should still exist checkClusterFormed(t, srva, srvb) // Check that routes were not re-created newacid := getCID(srva) newbcid := getCID(srvb) if newacid != acid { t.Fatalf("Expected server A route ID to be %v, got %v", acid, newacid) } if newbcid != bcid { t.Fatalf("Expected server B route ID to be %v, got %v", bcid, newbcid) } } func TestConfigReloadClusterPerms(t *testing.T) { confATemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { import { allow: %s } export { allow: %s } } } no_sys_acc: true ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `"foo"`, `"foo"`))) srva, _ := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { import { allow: %s } export { allow: %s } } routes = [ "nats://127.0.0.1:%d" ] } no_sys_acc: true ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `"foo"`, `"foo"`, srva.ClusterAddr().Port))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Create a connection on A nca, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", srva.Addr().(*net.TCPAddr).Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nca.Close() // Create a subscription on "foo" and "bar", only "foo" will be also on server B. subFooOnA, err := nca.SubscribeSync("foo") if err != nil { t.Fatalf("Error on subscribe: %v", err) } subBarOnA, err := nca.SubscribeSync("bar") if err != nil { t.Fatalf("Error on subscribe: %v", err) } // Connect on B and do the same ncb, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", srvb.Addr().(*net.TCPAddr).Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncb.Close() // Create a subscription on "foo" and "bar", only "foo" will be also on server B. subFooOnB, err := ncb.SubscribeSync("foo") if err != nil { t.Fatalf("Error on subscribe: %v", err) } subBarOnB, err := ncb.SubscribeSync("bar") if err != nil { t.Fatalf("Error on subscribe: %v", err) } // Check subscriptions on each server. There should be 3 on each server, // foo and bar locally and foo from remote server. checkExpectedSubs(t, 3, srva, srvb) sendMsg := func(t *testing.T, subj string, nc *nats.Conn) { t.Helper() if err := nc.Publish(subj, []byte("msg")); err != nil { t.Fatalf("Error on publish: %v", err) } } checkSub := func(t *testing.T, sub *nats.Subscription, shouldReceive bool) { t.Helper() _, err := sub.NextMsg(100 * time.Millisecond) if shouldReceive && err != nil { t.Fatalf("Expected message on %q, got %v", sub.Subject, err) } else if !shouldReceive && err == nil { t.Fatalf("Expected no message on %q, got one", sub.Subject) } } // Produce from A and check received on both sides sendMsg(t, "foo", nca) checkSub(t, subFooOnA, true) checkSub(t, subFooOnB, true) // Now from B: sendMsg(t, "foo", ncb) checkSub(t, subFooOnA, true) checkSub(t, subFooOnB, true) // Publish on bar from A and make sure only local sub receives sendMsg(t, "bar", nca) checkSub(t, subBarOnA, true) checkSub(t, subBarOnB, false) // Publish on bar from B and make sure only local sub receives sendMsg(t, "bar", ncb) checkSub(t, subBarOnA, false) checkSub(t, subBarOnB, true) // We will now both import/export foo and bar. Start with reloading A. reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`, `["foo", "bar"]`)) // Since B has not been updated yet, the state should remain the same, // that is 3 subs on each server. checkExpectedSubs(t, 3, srva, srvb) // Now update and reload B. Add "baz" for another test down below reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `["foo", "bar", "baz"]`, `["foo", "bar", "baz"]`, srva.ClusterAddr().Port)) // Now 4 on each server checkExpectedSubs(t, 4, srva, srvb) // Make sure that we can receive all messages sendMsg(t, "foo", nca) checkSub(t, subFooOnA, true) checkSub(t, subFooOnB, true) sendMsg(t, "foo", ncb) checkSub(t, subFooOnA, true) checkSub(t, subFooOnB, true) sendMsg(t, "bar", nca) checkSub(t, subBarOnA, true) checkSub(t, subBarOnB, true) sendMsg(t, "bar", ncb) checkSub(t, subBarOnA, true) checkSub(t, subBarOnB, true) // Create subscription on baz on server B. subBazOnB, err := ncb.SubscribeSync("baz") if err != nil { t.Fatalf("Error on subscribe: %v", err) } // Check subscriptions count checkExpectedSubs(t, 5, srvb) checkExpectedSubs(t, 4, srva) sendMsg(t, "baz", nca) checkSub(t, subBazOnB, false) sendMsg(t, "baz", ncb) checkSub(t, subBazOnB, true) // Test UNSUB by denying something that was previously imported reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"foo"`, `["foo", "bar"]`)) // Since A no longer imports "bar", we should have one less subscription // on B (B will have received an UNSUB for bar) checkExpectedSubs(t, 4, srvb) // A, however, should still have same number of subs. checkExpectedSubs(t, 4, srva) // Remove all permissions from A. reloadUpdateConfig(t, srva, confA, ` port: -1 cluster { listen: 127.0.0.1:-1 } no_sys_acc: true `) // Server A should now have baz sub checkExpectedSubs(t, 5, srvb) checkExpectedSubs(t, 5, srva) sendMsg(t, "baz", nca) checkSub(t, subBazOnB, true) sendMsg(t, "baz", ncb) checkSub(t, subBazOnB, true) // Finally, remove permissions from B reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(` port: -1 cluster { listen: 127.0.0.1:-1 routes = [ "nats://127.0.0.1:%d" ] } no_sys_acc: true `, srva.ClusterAddr().Port)) // Check expected subscriptions count. checkExpectedSubs(t, 5, srvb) checkExpectedSubs(t, 5, srva) } func TestConfigReloadClusterPermsImport(t *testing.T) { confATemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { import: { allow: %s } } } no_sys_acc: true ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `["foo", "bar"]`))) srva, _ := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 routes = [ "nats://127.0.0.1:%d" ] } no_sys_acc: true ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, srva.ClusterAddr().Port))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Create a connection on A nca, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", srva.Addr().(*net.TCPAddr).Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nca.Close() // Create a subscription on "foo" and "bar" if _, err := nca.SubscribeSync("foo"); err != nil { t.Fatalf("Error on subscribe: %v", err) } if _, err := nca.SubscribeSync("bar"); err != nil { t.Fatalf("Error on subscribe: %v", err) } checkExpectedSubs(t, 2, srva, srvb) // Drop foo reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`)) checkExpectedSubs(t, 2, srva) checkExpectedSubs(t, 1, srvb) // Add it back reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`)) checkExpectedSubs(t, 2, srva, srvb) // Empty Import means implicit allow reloadUpdateConfig(t, srva, confA, ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { export: ">" } } no_sys_acc: true `) checkExpectedSubs(t, 2, srva, srvb) confATemplate = ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { import: { allow: ["foo", "bar"] deny: %s } } } no_sys_acc: true ` // Now deny all: reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`)) checkExpectedSubs(t, 2, srva) checkExpectedSubs(t, 0, srvb) // Drop foo from the deny list reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`)) checkExpectedSubs(t, 2, srva) checkExpectedSubs(t, 1, srvb) } func TestConfigReloadClusterPermsExport(t *testing.T) { confATemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { export: { allow: %s } } } no_sys_acc: true ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `["foo", "bar"]`))) srva, _ := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 routes = [ "nats://127.0.0.1:%d" ] } no_sys_acc: true ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, srva.ClusterAddr().Port))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Create a connection on B ncb, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", srvb.Addr().(*net.TCPAddr).Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncb.Close() // Create a subscription on "foo" and "bar" if _, err := ncb.SubscribeSync("foo"); err != nil { t.Fatalf("Error on subscribe: %v", err) } if _, err := ncb.SubscribeSync("bar"); err != nil { t.Fatalf("Error on subscribe: %v", err) } checkExpectedSubs(t, 2, srva, srvb) // Drop foo reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`)) checkExpectedSubs(t, 2, srvb) checkExpectedSubs(t, 1, srva) // Add it back reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`)) checkExpectedSubs(t, 2, srva, srvb) // Empty Export means implicit allow reloadUpdateConfig(t, srva, confA, ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { import: ">" } } no_sys_acc: true `) checkExpectedSubs(t, 2, srva, srvb) confATemplate = ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { export: { allow: ["foo", "bar"] deny: %s } } } no_sys_acc: true ` // Now deny all: reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`)) checkExpectedSubs(t, 0, srva) checkExpectedSubs(t, 2, srvb) // Drop foo from the deny list reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`)) checkExpectedSubs(t, 1, srva) checkExpectedSubs(t, 2, srvb) } func TestConfigReloadClusterPermsOldServer(t *testing.T) { confATemplate := ` port: -1 cluster { listen: 127.0.0.1:-1 permissions { export: { allow: %s } } } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `["foo", "bar"]`))) srva, _ := RunServerWithConfig(confA) defer srva.Shutdown() optsB := DefaultOptions() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srva.ClusterAddr().Port)) // Make server B behave like an old server optsB.routeProto = setRouteProtoForTest(RouteProtoZero) srvb := RunServer(optsB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Get the route's connection ID getRouteRID := func() uint64 { rid := uint64(0) srvb.mu.Lock() if r := getFirstRoute(srvb); r != nil { r.mu.Lock() rid = r.cid r.mu.Unlock() } srvb.mu.Unlock() return rid } orgRID := getRouteRID() // Cause a config reload on A reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`)) // Check that new route gets created check := func(t *testing.T) { t.Helper() checkFor(t, 3*time.Second, 15*time.Millisecond, func() error { if rid := getRouteRID(); rid == orgRID { return fmt.Errorf("Route does not seem to have been recreated") } return nil }) } check(t) // Save the current value orgRID = getRouteRID() // Add another server that supports INFO updates optsC := DefaultOptions() optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srva.ClusterAddr().Port)) srvc := RunServer(optsC) defer srvc.Shutdown() checkClusterFormed(t, srva, srvb, srvc) // Cause a config reload on A reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"foo"`)) // Check that new route gets created check(t) } func TestConfigReloadAccountUsers(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { synadia { users = [ {user: derek, password: derek} {user: foo, password: foo} ] } nats.io { users = [ {user: ivan, password: ivan} {user: bar, password: bar} ] } acc_deleted_after_reload { users = [ {user: gone, password: soon} {user: baz, password: baz} {user: bat, password: bat} ] } } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Connect as exisiting users, should work. nc, err := nats.Connect(fmt.Sprintf("nats://derek:derek@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() ch := make(chan bool, 2) cb := func(_ *nats.Conn) { ch <- true } nc2, err := nats.Connect( fmt.Sprintf("nats://ivan:ivan@%s:%d", opts.Host, opts.Port), nats.NoReconnect(), nats.ClosedHandler(cb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc3, err := nats.Connect( fmt.Sprintf("nats://gone:soon@%s:%d", opts.Host, opts.Port), nats.NoReconnect(), nats.ClosedHandler(cb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc3.Close() // These users will be moved from an account to another (to a specific or to global account) // We will create subscriptions to ensure that they are moved to proper sublists too. rch := make(chan bool, 4) rcb := func(_ *nats.Conn) { rch <- true } nc4, err := nats.Connect(fmt.Sprintf("nats://foo:foo@%s:%d", opts.Host, opts.Port), nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc4.Close() if _, err := nc4.SubscribeSync("foo"); err != nil { t.Fatalf("Error on subscribe: %v", err) } nc5, err := nats.Connect(fmt.Sprintf("nats://bar:bar@%s:%d", opts.Host, opts.Port), nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc5.Close() if _, err := nc5.SubscribeSync("bar"); err != nil { t.Fatalf("Error on subscribe: %v", err) } nc6, err := nats.Connect(fmt.Sprintf("nats://baz:baz@%s:%d", opts.Host, opts.Port), nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc6.Close() if _, err := nc6.SubscribeSync("baz"); err != nil { t.Fatalf("Error on subscribe: %v", err) } nc7, err := nats.Connect(fmt.Sprintf("nats://bat:bat@%s:%d", opts.Host, opts.Port), nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc7.Close() if _, err := nc7.SubscribeSync("bat"); err != nil { t.Fatalf("Error on subscribe: %v", err) } // confirm subscriptions before and after reload. var expectedSubs uint32 = 5 sAcc, err := s.LookupAccount("synadia") require_NoError(t, err) sAcc.mu.RLock() n := sAcc.sl.Count() sAcc.mu.RUnlock() if n != expectedSubs { t.Errorf("Synadia account should have %d sub, got %v", expectedSubs, n) } nAcc, err := s.LookupAccount("nats.io") require_NoError(t, err) nAcc.mu.RLock() n = nAcc.sl.Count() nAcc.mu.RUnlock() if n != expectedSubs { t.Errorf("Nats.io account should have %d sub, got %v", expectedSubs, n) } // Remove user from account and whole account reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" authorization { users = [ {user: foo, password: foo} {user: baz, password: baz} ] } accounts { synadia { users = [ {user: derek, password: derek} {user: bar, password: bar} ] } nats.io { users = [ {user: bat, password: bat} ] } } `) // nc2 and nc3 should be closed if err := wait(ch); err != nil { t.Fatal("Did not get the closed callback") } if err := wait(ch); err != nil { t.Fatal("Did not get the closed callback") } // And first connection should still be connected if !nc.IsConnected() { t.Fatal("First connection should still be connected") } // Old account should be gone if _, err := s.LookupAccount("acc_deleted_after_reload"); err == nil { t.Fatal("old account should be gone") } // Check subscriptions. Since most of the users have been // moving accounts, make sure we account for the reconnect for i := 0; i < 4; i++ { if err := wait(rch); err != nil { t.Fatal("Did not get the reconnect cb") } } // Still need to do the tests in a checkFor() because clients // being reconnected does not mean that resent of subscriptions // has already been processed. checkFor(t, 2*time.Second, 100*time.Millisecond, func() error { gAcc, err := s.LookupAccount(globalAccountName) require_NoError(t, err) gAcc.mu.RLock() n := gAcc.sl.Count() fooMatch := gAcc.sl.Match("foo") bazMatch := gAcc.sl.Match("baz") gAcc.mu.RUnlock() // The number of subscriptions should be 4 ($SYS.REQ.USER.INFO, // $SYS.REQ.ACCOUNT.PING.CONNZ, $SYS.REQ.ACCOUNT.PING.STATZ, // $SYS.REQ.SERVER.PING.CONNZ) + 2 (foo and baz) if n != 6 { return fmt.Errorf("Global account should have 6 subs, got %v", n) } if len(fooMatch.psubs) != 1 { return fmt.Errorf("Global account should have foo sub") } if len(bazMatch.psubs) != 1 { return fmt.Errorf("Global account should have baz sub") } sAcc, err := s.LookupAccount("synadia") require_NoError(t, err) sAcc.mu.RLock() n = sAcc.sl.Count() barMatch := sAcc.sl.Match("bar") sAcc.mu.RUnlock() if n != expectedSubs { return fmt.Errorf("Synadia account should have %d sub, got %v", expectedSubs, n) } if len(barMatch.psubs) != 1 { return fmt.Errorf("Synadia account should have bar sub") } nAcc, err := s.LookupAccount("nats.io") require_NoError(t, err) nAcc.mu.RLock() n = nAcc.sl.Count() batMatch := nAcc.sl.Match("bat") nAcc.mu.RUnlock() if n != expectedSubs { return fmt.Errorf("Nats.io account should have %d sub, got %v", expectedSubs, n) } if len(batMatch.psubs) != 1 { return fmt.Errorf("Synadia account should have bar sub") } return nil }) } func TestConfigReloadAccountWithNoChanges(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" system_account: sys accounts { A { users = [{ user: a }] } B { users = [{ user: b }] } C { users = [{ user: c }] } sys { users = [{ user: sys }] } } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() ncA, err := nats.Connect(fmt.Sprintf("nats://a:@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncA.Close() // Confirm default service imports are ok. checkSubs := func(t *testing.T) { resp, err := ncA.Request("$SYS.REQ.ACCOUNT.PING.CONNZ", nil, time.Second) if err != nil { t.Error(err) } if resp == nil || !strings.Contains(string(resp.Data), `"num_connections":1`) { t.Fatal("unexpected data in connz response") } resp, err = ncA.Request("$SYS.REQ.SERVER.PING.CONNZ", nil, time.Second) if err != nil { t.Error(err) } if resp == nil || !strings.Contains(string(resp.Data), `"num_connections":1`) { t.Fatal("unexpected data in connz response") } resp, err = ncA.Request("$SYS.REQ.ACCOUNT.PING.STATZ", nil, time.Second) if err != nil { t.Error(err) } if resp == nil || !strings.Contains(string(resp.Data), `"conns":1`) { t.Fatal("unexpected data in connz response") } } checkSubs(t) before := s.NumSubscriptions() s.Reload() after := s.NumSubscriptions() if before != after { t.Errorf("Number of subscriptions changed after reload: %d -> %d", before, after) } // Confirm this still works after a reload... checkSubs(t) before = s.NumSubscriptions() s.Reload() after = s.NumSubscriptions() if before != after { t.Errorf("Number of subscriptions changed after reload: %d -> %d", before, after) } // Do another extra reload just in case. checkSubs(t) before = s.NumSubscriptions() s.Reload() after = s.NumSubscriptions() if before != after { t.Errorf("Number of subscriptions changed after reload: %d -> %d", before, after) } } func TestConfigReloadAccountNKeyUsers(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { synadia { users = [ # Derek {nkey : UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR} ] } nats.io { users = [ # Ivan {nkey : UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH} ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() synadia, _ := s.LookupAccount("synadia") nats, _ := s.LookupAccount("nats.io") seed1 := []byte("SUAPM67TC4RHQLKBX55NIQXSMATZDOZK6FNEOSS36CAYA7F7TY66LP4BOM") seed2 := []byte("SUAIS5JPX4X4GJ7EIIJEQ56DH2GWPYJRPWN5XJEDENJOZHCBLI7SEPUQDE") kp, _ := nkeys.FromSeed(seed1) pubKey, _ := kp.PublicKey() c, cr, l := newClientForServer(s) defer c.close() // Check for Nonce var info nonceInfo if err := json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err := kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig := base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs := fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } if c.acc != synadia { t.Fatalf("Expected the nkey client's account to match 'synadia', got %v", c.acc) } // Now nats account nkey user. kp, _ = nkeys.FromSeed(seed2) pubKey, _ = kp.PublicKey() c, cr, l = newClientForServer(s) defer c.close() // Check for Nonce err = json.Unmarshal([]byte(l[5:]), &info) if err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } if info.Nonce == "" { t.Fatalf("Expected a non-empty nonce with nkeys defined") } sigraw, err = kp.Sign([]byte(info.Nonce)) if err != nil { t.Fatalf("Failed signing nonce: %v", err) } sig = base64.RawURLEncoding.EncodeToString(sigraw) // PING needed to flush the +OK to us. cs = fmt.Sprintf("CONNECT {\"nkey\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", pubKey, sig) c.parseAsync(cs) l, _ = cr.ReadString('\n') if !strings.HasPrefix(l, "+OK") { t.Fatalf("Expected an OK, got: %v", l) } if c.acc != nats { t.Fatalf("Expected the nkey client's account to match 'nats', got %v", c.acc) } // Remove user from account and whole account reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" authorization { users = [ # Ivan {nkey : UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH} ] } accounts { nats.io { users = [ # Derek {nkey : UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR} ] } } `) s.mu.Lock() nkeys := s.nkeys globalAcc := s.gacc s.mu.Unlock() if n := len(nkeys); n != 2 { t.Fatalf("NKeys map should have 2 users, got %v", n) } derek := nkeys["UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR"] if derek == nil { t.Fatal("NKey for user Derek not found") } if derek.Account == nil || derek.Account.Name != "nats.io" { t.Fatalf("Invalid account for user Derek: %#v", derek.Account) } ivan := nkeys["UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH"] if ivan == nil { t.Fatal("NKey for user Ivan not found") } if ivan.Account != globalAcc { t.Fatalf("Invalid account for user Ivan: %#v", ivan.Account) } if _, err := s.LookupAccount("synadia"); err == nil { t.Fatal("Account Synadia should have been removed") } } func TestConfigReloadAccountStreamsImportExport(t *testing.T) { template := ` listen: "127.0.0.1:-1" accounts { synadia { users [{user: derek, password: foo}] exports = [ {stream: "private.>", accounts: [nats.io]} {stream: %s} ] } nats.io { users [ {user: ivan, password: bar, permissions: {subscribe: {deny: %s}}} ] imports = [ {stream: {account: "synadia", subject: %s}} {stream: {account: "synadia", subject: "private.natsio.*"}, prefix: %s} ] } } no_sys_acc: true ` // synadia account exports "private.>" to nats.io // synadia account exports "foo.*" // user ivan denies subscription on "xxx" // nats.io account imports "foo.*" from synadia // nats.io account imports "private.natsio.*" from synadia with prefix "ivan" conf := createConfFile(t, []byte(fmt.Sprintf(template, `"foo.*"`, `"xxx"`, `"foo.*"`, `"ivan"`))) s, opts := RunServerWithConfig(conf) defer s.Shutdown() derek, err := nats.Connect(fmt.Sprintf("nats://derek:foo@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer derek.Close() checkClientsCount(t, s, 1) ch := make(chan bool, 1) ivan, err := nats.Connect(fmt.Sprintf("nats://ivan:bar@%s:%d", opts.Host, opts.Port), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { if strings.Contains(strings.ToLower(err.Error()), "permissions violation") { ch <- true } })) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ivan.Close() checkClientsCount(t, s, 2) subscribe := func(t *testing.T, nc *nats.Conn, subj string) *nats.Subscription { t.Helper() s, err := nc.SubscribeSync(subj) if err != nil { t.Fatalf("Error on subscribe: %v", err) } return s } subFooBar := subscribe(t, ivan, "foo.bar") subFooBaz := subscribe(t, ivan, "foo.baz") subFooBat := subscribe(t, ivan, "foo.bat") subPriv := subscribe(t, ivan, "ivan.private.natsio.*") ivan.Flush() publish := func(t *testing.T, nc *nats.Conn, subj string) { t.Helper() if err := nc.Publish(subj, []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } } nextMsg := func(t *testing.T, sub *nats.Subscription, expected bool) { t.Helper() dur := 100 * time.Millisecond if expected { dur = time.Second } _, err := sub.NextMsg(dur) if expected && err != nil { t.Fatalf("Expected a message on %s, got %v", sub.Subject, err) } else if !expected && err != nats.ErrTimeout { t.Fatalf("Expected a timeout on %s, got %v", sub.Subject, err) } } // Checks the derek's user sublist for presence of given subject // interest. Boolean says if interest is expected or not. checkSublist := func(t *testing.T, subject string, shouldBeThere bool) { t.Helper() dcli := s.getClient(1) dcli.mu.Lock() r := dcli.acc.sl.Match(subject) dcli.mu.Unlock() if shouldBeThere && len(r.psubs) != 1 { t.Fatalf("%s should have 1 match in derek's sublist, got %v", subject, len(r.psubs)) } else if !shouldBeThere && len(r.psubs) > 0 { t.Fatalf("%s should not be in derek's sublist", subject) } } // Publish on all subjects and the subs should receive and // subjects should be in sublist publish(t, derek, "foo.bar") nextMsg(t, subFooBar, true) checkSublist(t, "foo.bar", true) publish(t, derek, "foo.baz") nextMsg(t, subFooBaz, true) checkSublist(t, "foo.baz", true) publish(t, derek, "foo.bat") nextMsg(t, subFooBat, true) checkSublist(t, "foo.bat", true) publish(t, derek, "private.natsio.foo") nextMsg(t, subPriv, true) checkSublist(t, "private.natsio.foo", true) // Also make sure that intra-account subscription works OK ivanSub := subscribe(t, ivan, "ivan.sub") publish(t, ivan, "ivan.sub") nextMsg(t, ivanSub, true) derekSub := subscribe(t, derek, "derek.sub") publish(t, derek, "derek.sub") nextMsg(t, derekSub, true) // synadia account exports "private.>" to nats.io // synadia account exports "foo.*" // user ivan denies subscription on "foo.bat" // nats.io account imports "foo.baz" from synadia // nats.io account imports "private.natsio.*" from synadia with prefix "yyyy" reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, `"foo.*"`, `"foo.bat"`, `"foo.baz"`, `"yyyy"`)) // Sub on foo.bar should now fail to receive publish(t, derek, "foo.bar") nextMsg(t, subFooBar, false) checkSublist(t, "foo.bar", false) // But foo.baz should be received publish(t, derek, "foo.baz") nextMsg(t, subFooBaz, true) checkSublist(t, "foo.baz", true) // Due to permissions, foo.bat should not publish(t, derek, "foo.bat") nextMsg(t, subFooBat, false) checkSublist(t, "foo.bat", false) // Prefix changed, so should not be received publish(t, derek, "private.natsio.foo") nextMsg(t, subPriv, false) checkSublist(t, "private.natsio.foo", false) // Wait for client notification of permissions error if err := wait(ch); err != nil { t.Fatal("Did not the permissions error") } publish(t, ivan, "ivan.sub") nextMsg(t, ivanSub, true) publish(t, derek, "derek.sub") nextMsg(t, derekSub, true) // Change export so that foo.* is no longer exported // synadia account exports "private.>" to nats.io // synadia account exports "xxx" // user ivan denies subscription on "foo.bat" // nats.io account imports "xxx" from synadia // nats.io account imports "private.natsio.*" from synadia with prefix "ivan" reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, `"xxx"`, `"foo.bat"`, `"xxx"`, `"ivan"`)) publish(t, derek, "foo.bar") nextMsg(t, subFooBar, false) checkSublist(t, "foo.bar", false) publish(t, derek, "foo.baz") nextMsg(t, subFooBaz, false) checkSublist(t, "foo.baz", false) publish(t, derek, "foo.bat") nextMsg(t, subFooBat, false) checkSublist(t, "foo.bat", false) // Prefix changed back, so should receive publish(t, derek, "private.natsio.foo") nextMsg(t, subPriv, true) checkSublist(t, "private.natsio.foo", true) publish(t, ivan, "ivan.sub") nextMsg(t, ivanSub, true) publish(t, derek, "derek.sub") nextMsg(t, derekSub, true) } func TestConfigReloadAccountServicesImportExport(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { synadia { users [{user: derek, password: foo}] exports = [ {service: "pub.request"} {service: "pub.special.request", accounts: [nats.io]} ] } nats.io { users [{user: ivan, password: bar}] imports = [ {service: {account: "synadia", subject: "pub.special.request"}, to: "foo"} {service: {account: "synadia", subject: "pub.request"}, to: "bar"} ] } } cluster { name: "abc" port: -1 } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() opts2 := DefaultOptions() opts2.Cluster.Name = "abc" opts2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", opts.Cluster.Port)) s2 := RunServer(opts2) defer s2.Shutdown() checkClusterFormed(t, s, s2) derek, err := nats.Connect(fmt.Sprintf("nats://derek:foo@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer derek.Close() checkClientsCount(t, s, 1) ivan, err := nats.Connect(fmt.Sprintf("nats://ivan:bar@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ivan.Close() checkClientsCount(t, s, 2) if _, err := derek.Subscribe("pub.special.request", func(m *nats.Msg) { derek.Publish(m.Reply, []byte("reply1")) }); err != nil { t.Fatalf("Error on subscribe: %v", err) } if _, err := derek.Subscribe("pub.request", func(m *nats.Msg) { derek.Publish(m.Reply, []byte("reply2")) }); err != nil { t.Fatalf("Error on subscribe: %v", err) } if _, err := derek.Subscribe("pub.special.request.new", func(m *nats.Msg) { derek.Publish(m.Reply, []byte("reply3")) }); err != nil { t.Fatalf("Error on subscribe: %v", err) } // Also create one that will be used for intra-account communication if _, err := derek.Subscribe("derek.sub", func(m *nats.Msg) { derek.Publish(m.Reply, []byte("private")) }); err != nil { t.Fatalf("Error on subscribe: %v", err) } derek.Flush() // Create an intra-account sub for ivan too if _, err := ivan.Subscribe("ivan.sub", func(m *nats.Msg) { ivan.Publish(m.Reply, []byte("private")) }); err != nil { t.Fatalf("Error on subscribe: %v", err) } // This subscription is just to make sure that we can update // route map without locking issues during reload. natsSubSync(t, ivan, "bar") req := func(t *testing.T, nc *nats.Conn, subj string, reply string) { t.Helper() var timeout time.Duration if reply != "" { timeout = time.Second } else { timeout = 100 * time.Millisecond } msg, err := nc.Request(subj, []byte("request"), timeout) if reply != "" { if err != nil { t.Fatalf("Expected reply %s on subject %s, got %v", reply, subj, err) } if string(msg.Data) != reply { t.Fatalf("Expected reply %s on subject %s, got %s", reply, subj, msg.Data) } } else if err != nats.ErrTimeout && err != nats.ErrNoResponders { t.Fatalf("Expected timeout on subject %s, got %v", subj, err) } } req(t, ivan, "foo", "reply1") req(t, ivan, "bar", "reply2") // This not exported/imported, so should timeout req(t, ivan, "baz", "") // Check intra-account communication req(t, ivan, "ivan.sub", "private") req(t, derek, "derek.sub", "private") reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" accounts { synadia { users [{user: derek, password: foo}] exports = [ {service: "pub.request"} {service: "pub.special.request", accounts: [nats.io]} {service: "pub.special.request.new", accounts: [nats.io]} ] } nats.io { users [{user: ivan, password: bar}] imports = [ {service: {account: "synadia", subject: "pub.special.request"}, to: "foo"} {service: {account: "synadia", subject: "pub.special.request.new"}, to: "baz"} ] } } cluster { name: "abc" port: -1 } `) // This still should work req(t, ivan, "foo", "reply1") // This should not req(t, ivan, "bar", "") // This now should work req(t, ivan, "baz", "reply3") // Check intra-account communication req(t, ivan, "ivan.sub", "private") req(t, derek, "derek.sub", "private") } // As of now, config reload does not support changes for gateways. // However, ensure that if a gateway is defined, one can still // do reload as long as we don't change the gateway spec. func TestConfigReloadNotPreventedByGateways(t *testing.T) { confTemplate := ` listen: "127.0.0.1:-1" %s gateway { name: "A" listen: "127.0.0.1:-1" tls { cert_file: "configs/certs/server.pem" key_file: "configs/certs/key.pem" timeout: %s } gateways [ { name: "B" url: "nats://localhost:8888" } ] } no_sys_acc: true ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, "", "5"))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Cause reload with adding a param that is supported reloadUpdateConfig(t, s, conf, fmt.Sprintf(confTemplate, "max_payload: 100000", "5")) // Now update gateway, should fail to reload. changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, "max_payload: 100000", "3"))) if err := s.Reload(); err == nil || !strings.Contains(err.Error(), "not supported for Gateway") { t.Fatalf("Expected Reload to return a not supported error, got %v", err) } } func TestConfigReloadBoolFlags(t *testing.T) { defer func() { FlagSnapshot = nil }() logfile := filepath.Join(t.TempDir(), "logtime.log") template := ` listen: "127.0.0.1:-1" logfile: "%s" %s ` var opts *Options var err error for _, test := range []struct { name string content string cmdLine []string expected bool val func() bool }{ // Logtime { "logtime_not_in_config_no_override", "", nil, true, func() bool { return opts.Logtime }, }, { "logtime_not_in_config_override_short_true", "", []string{"-T"}, true, func() bool { return opts.Logtime }, }, { "logtime_not_in_config_override_true", "", []string{"-logtime"}, true, func() bool { return opts.Logtime }, }, { "logtime_false_in_config_no_override", "logtime: false", nil, false, func() bool { return opts.Logtime }, }, { "logtime_false_in_config_override_short_true", "logtime: false", []string{"-T"}, true, func() bool { return opts.Logtime }, }, { "logtime_false_in_config_override_true", "logtime: false", []string{"-logtime"}, true, func() bool { return opts.Logtime }, }, { "logtime_true_in_config_no_override", "logtime: true", nil, true, func() bool { return opts.Logtime }, }, { "logtime_true_in_config_override_short_false", "logtime: true", []string{"-T=false"}, false, func() bool { return opts.Logtime }, }, { "logtime_true_in_config_override_false", "logtime: true", []string{"-logtime=false"}, false, func() bool { return opts.Logtime }, }, // Debug { "debug_not_in_config_no_override", "", nil, false, func() bool { return opts.Debug }, }, { "debug_not_in_config_override_short_true", "", []string{"-D"}, true, func() bool { return opts.Debug }, }, { "debug_not_in_config_override_true", "", []string{"-debug"}, true, func() bool { return opts.Debug }, }, { "debug_false_in_config_no_override", "debug: false", nil, false, func() bool { return opts.Debug }, }, { "debug_false_in_config_override_short_true", "debug: false", []string{"-D"}, true, func() bool { return opts.Debug }, }, { "debug_false_in_config_override_true", "debug: false", []string{"-debug"}, true, func() bool { return opts.Debug }, }, { "debug_true_in_config_no_override", "debug: true", nil, true, func() bool { return opts.Debug }, }, { "debug_true_in_config_override_short_false", "debug: true", []string{"-D=false"}, false, func() bool { return opts.Debug }, }, { "debug_true_in_config_override_false", "debug: true", []string{"-debug=false"}, false, func() bool { return opts.Debug }, }, // Trace { "trace_not_in_config_no_override", "", nil, false, func() bool { return opts.Trace }, }, { "trace_not_in_config_override_short_true", "", []string{"-V"}, true, func() bool { return opts.Trace }, }, { "trace_not_in_config_override_true", "", []string{"-trace"}, true, func() bool { return opts.Trace }, }, { "trace_false_in_config_no_override", "trace: false", nil, false, func() bool { return opts.Trace }, }, { "trace_false_in_config_override_short_true", "trace: false", []string{"-V"}, true, func() bool { return opts.Trace }, }, { "trace_false_in_config_override_true", "trace: false", []string{"-trace"}, true, func() bool { return opts.Trace }, }, { "trace_true_in_config_no_override", "trace: true", nil, true, func() bool { return opts.Trace }, }, { "trace_true_in_config_override_short_false", "trace: true", []string{"-V=false"}, false, func() bool { return opts.Trace }, }, { "trace_true_in_config_override_false", "trace: true", []string{"-trace=false"}, false, func() bool { return opts.Trace }, }, // Syslog { "syslog_not_in_config_no_override", "", nil, false, func() bool { return opts.Syslog }, }, { "syslog_not_in_config_override_short_true", "", []string{"-s"}, true, func() bool { return opts.Syslog }, }, { "syslog_not_in_config_override_true", "", []string{"-syslog"}, true, func() bool { return opts.Syslog }, }, { "syslog_false_in_config_no_override", "syslog: false", nil, false, func() bool { return opts.Syslog }, }, { "syslog_false_in_config_override_short_true", "syslog: false", []string{"-s"}, true, func() bool { return opts.Syslog }, }, { "syslog_false_in_config_override_true", "syslog: false", []string{"-syslog"}, true, func() bool { return opts.Syslog }, }, { "syslog_true_in_config_no_override", "syslog: true", nil, true, func() bool { return opts.Syslog }, }, { "syslog_true_in_config_override_short_false", "syslog: true", []string{"-s=false"}, false, func() bool { return opts.Syslog }, }, { "syslog_true_in_config_override_false", "syslog: true", []string{"-syslog=false"}, false, func() bool { return opts.Syslog }, }, // Cluster.NoAdvertise { "cluster_no_advertise_not_in_config_no_override", `cluster { port: -1 }`, nil, false, func() bool { return opts.Cluster.NoAdvertise }, }, { "cluster_no_advertise_not_in_config_override_true", `cluster { port: -1 }`, []string{"-no_advertise"}, true, func() bool { return opts.Cluster.NoAdvertise }, }, { "cluster_no_advertise_false_in_config_no_override", `cluster { port: -1 no_advertise: false }`, nil, false, func() bool { return opts.Cluster.NoAdvertise }, }, { "cluster_no_advertise_false_in_config_override_true", `cluster { port: -1 no_advertise: false }`, []string{"-no_advertise"}, true, func() bool { return opts.Cluster.NoAdvertise }, }, { "cluster_no_advertise_true_in_config_no_override", `cluster { port: -1 no_advertise: true }`, nil, true, func() bool { return opts.Cluster.NoAdvertise }, }, { "cluster_no_advertise_true_in_config_override_false", `cluster { port: -1 no_advertise: true }`, []string{"-no_advertise=false"}, false, func() bool { return opts.Syslog }, }, // -DV override { "debug_trace_not_in_config_dv_override_true", "", []string{"-DV"}, true, func() bool { return opts.Debug && opts.Trace }, }, { "debug_trace_false_in_config_dv_override_true", `debug: false trace: false `, []string{"-DV"}, true, func() bool { return opts.Debug && opts.Trace }, }, { "debug_trace_true_in_config_dv_override_false", `debug: true trace: true `, []string{"-DV=false"}, false, func() bool { return opts.Debug && opts.Trace }, }, { "trace_verbose_true_in_config_override_true", `trace_verbose: true `, nil, true, func() bool { return opts.Trace && opts.TraceVerbose }, }, { "trace_verbose_true_in_config_override_false", `trace_verbose: true `, []string{"--VV=false"}, true, func() bool { return !opts.TraceVerbose }, }, { "trace_verbose_true_in_config_override_false", `trace_verbose: false `, []string{"--VV=true"}, true, func() bool { return opts.TraceVerbose }, }, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(template, logfile, test.content))) fs := flag.NewFlagSet("test", flag.ContinueOnError) var args []string args = append(args, "-c", conf) if test.cmdLine != nil { args = append(args, test.cmdLine...) } opts, err = ConfigureOptions(fs, args, nil, nil, nil) if err != nil { t.Fatalf("Error processing config: %v", err) } opts.NoSigs = true s := RunServer(opts) defer s.Shutdown() if test.val() != test.expected { t.Fatalf("Expected to be set to %v, got %v", test.expected, test.val()) } if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } if test.val() != test.expected { t.Fatalf("Expected to be set to %v, got %v", test.expected, test.val()) } }) } } func TestConfigReloadMaxControlLineWithClients(t *testing.T) { server, opts, config := runReloadServerWithConfig(t, "./configs/reload/basic.conf") defer server.Shutdown() // Ensure we can connect as a sanity check. addr := fmt.Sprintf("nats://%s:%d", opts.Host, server.Addr().(*net.TCPAddr).Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc.Close() // Now grab server's internal client that matches. cid, _ := nc.GetClientID() c := server.getClient(cid) if c == nil { t.Fatalf("Could not look up internal client") } // Check that we have the correct mcl snapshotted into the connected client. getMcl := func(c *client) int32 { c.mu.Lock() defer c.mu.Unlock() return c.mcl } if mcl := getMcl(c); mcl != opts.MaxControlLine { t.Fatalf("Expected snapshot in client for mcl to be same as opts.MaxControlLine, got %d vs %d", mcl, opts.MaxControlLine) } changeCurrentConfigContentWithNewContent(t, config, []byte("listen: 127.0.0.1:-1; max_control_line: 222")) if err := server.Reload(); err != nil { t.Fatalf("Expected Reload to succeed, got %v", err) } // Refresh properly. opts = server.getOpts() if mcl := getMcl(c); mcl != opts.MaxControlLine { t.Fatalf("Expected snapshot in client for mcl to be same as new opts.MaxControlLine, got %d vs %d", mcl, opts.MaxControlLine) } } type testCustomAuth struct{} func (ca *testCustomAuth) Check(c ClientAuthentication) bool { return true } func TestConfigReloadIgnoreCustomAuth(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 `)) opts := LoadConfig(conf) ca := &testCustomAuth{} opts.CustomClientAuthentication = ca opts.CustomRouterAuthentication = ca s := RunServer(opts) defer s.Shutdown() if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } if s.getOpts().CustomClientAuthentication != ca || s.getOpts().CustomRouterAuthentication != ca { t.Fatalf("Custom auth missing") } } func TestConfigReloadLeafNodeRandomPort(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 leafnodes { port: -1 } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() s.mu.Lock() lnPortBefore := s.leafNodeListener.Addr().(*net.TCPAddr).Port s.mu.Unlock() if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } s.mu.Lock() lnPortAfter := s.leafNodeListener.Addr().(*net.TCPAddr).Port s.mu.Unlock() if lnPortBefore != lnPortAfter { t.Fatalf("Expected leafnodes listen port to be same, was %v is now %v", lnPortBefore, lnPortAfter) } } func TestConfigReloadLeafNodeWithTLS(t *testing.T) { template := ` port: -1 %s leaf { listen: "127.0.0.1:-1" tls: { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/server.pem" key_file: "../test/configs/certs/tlsauth/server-key.pem" timeout: 3 } } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(template, ""))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() u, err := url.Parse(fmt.Sprintf("nats://localhost:%d", o1.LeafNode.Port)) if err != nil { t.Fatalf("Error creating url: %v", err) } conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 leaf { remotes [ { url: "%s" tls { ca_file: "../test/configs/certs/tlsauth/ca.pem" cert_file: "../test/configs/certs/tlsauth/client.pem" key_file: "../test/configs/certs/tlsauth/client-key.pem" timeout: 2 } } ] } `, u.String()))) o2, err := ProcessConfigFile(conf2) if err != nil { t.Fatalf("Error processing config file: %v", err) } o2.NoLog, o2.NoSigs = true, true o2.LeafNode.resolver = &testLoopbackResolver{} s2 := RunServer(o2) defer s2.Shutdown() checkFor(t, 3*time.Second, 15*time.Millisecond, func() error { if n := s1.NumLeafNodes(); n != 1 { return fmt.Errorf("Expected 1 leaf node, got %v", n) } return nil }) changeCurrentConfigContentWithNewContent(t, conf1, []byte(fmt.Sprintf(template, "debug: false"))) if err := s1.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } } func TestConfigReloadLeafNodeWithRemotesNoChanges(t *testing.T) { template := ` port: -1 cluster { port: -1 name: "%s" } leaf { remotes [ { urls: [ "nats://127.0.0.1:1234", "nats://127.0.0.1:1235", "nats://127.0.0.1:1236", "nats://127.0.0.1:1237", "nats://127.0.0.1:1238", "nats://127.0.0.1:1239", ] } ] } ` config := fmt.Sprintf(template, "A") conf := createConfFile(t, []byte(config)) o, err := ProcessConfigFile(conf) if err != nil { t.Fatalf("Error processing config file: %v", err) } o.NoLog, o.NoSigs = true, false s := RunServer(o) defer s.Shutdown() config = fmt.Sprintf(template, "B") changeCurrentConfigContentWithNewContent(t, conf, []byte(config)) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } } func TestConfigReloadAndVarz(t *testing.T) { template := ` port: -1 %s ` conf := createConfFile(t, []byte(fmt.Sprintf(template, ""))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() s.mu.Lock() initConfigTime := s.configTime s.mu.Unlock() v, _ := s.Varz(nil) if !v.ConfigLoadTime.Equal(initConfigTime) { t.Fatalf("ConfigLoadTime should be %v, got %v", initConfigTime, v.ConfigLoadTime) } if v.MaxConn != DEFAULT_MAX_CONNECTIONS { t.Fatalf("MaxConn should be %v, got %v", DEFAULT_MAX_CONNECTIONS, v.MaxConn) } changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, "max_connections: 10"))) // Make sure we wait a bit so config load time has a chance to change. time.Sleep(15 * time.Millisecond) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } v, _ = s.Varz(nil) if v.ConfigLoadTime.Equal(initConfigTime) { t.Fatalf("ConfigLoadTime should be different from %v", initConfigTime) } if v.MaxConn != 10 { t.Fatalf("MaxConn should be 10, got %v", v.MaxConn) } } func TestConfigReloadConnectErrReports(t *testing.T) { template := ` port: -1 %s %s ` conf := createConfFile(t, []byte(fmt.Sprintf(template, "", ""))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() opts := s.getOpts() if cer := opts.ConnectErrorReports; cer != DEFAULT_CONNECT_ERROR_REPORTS { t.Fatalf("Expected ConnectErrorReports to be %v, got %v", DEFAULT_CONNECT_ERROR_REPORTS, cer) } if rer := opts.ReconnectErrorReports; rer != DEFAULT_RECONNECT_ERROR_REPORTS { t.Fatalf("Expected ReconnectErrorReports to be %v, got %v", DEFAULT_RECONNECT_ERROR_REPORTS, rer) } changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, "connect_error_reports: 2", "reconnect_error_reports: 3"))) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } opts = s.getOpts() if cer := opts.ConnectErrorReports; cer != 2 { t.Fatalf("Expected ConnectErrorReports to be %v, got %v", 2, cer) } if rer := opts.ReconnectErrorReports; rer != 3 { t.Fatalf("Expected ReconnectErrorReports to be %v, got %v", 3, rer) } } func TestConfigReloadAuthDoesNotBreakRouteInterest(t *testing.T) { s, opts := RunServerWithConfig("./configs/seed_tls.conf") defer s.Shutdown() // Create client and sub interest on seed server. urlSeed := fmt.Sprintf("nats://%s:%d/", opts.Host, opts.Port) nc, err := nats.Connect(urlSeed) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc.Close() ch := make(chan bool) nc.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc.Flush() // Use this to check for message. checkForMsg := func() { t.Helper() select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } // Create second server and form cluster. We will send from here. urlRoute := fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, opts.Cluster.Port) optsA := nextServerOpts(opts) optsA.Routes = RoutesFromStr(urlRoute) sa := RunServer(optsA) defer sa.Shutdown() checkClusterFormed(t, s, sa) checkSubInterest(t, sa, globalAccountName, "foo", time.Second) // Create second client and send message from this one. Interest should be here. urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) nc2, err := nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() // Check that we can send messages. nc2.Publish("foo", nil) checkForMsg() // Now shutdown nc2 and srvA. nc2.Close() sa.Shutdown() // Now force reload on seed server of auth. s.reloadAuthorization() // Restart both server A and client 2. sa = RunServer(optsA) defer sa.Shutdown() checkClusterFormed(t, s, sa) checkSubInterest(t, sa, globalAccountName, "foo", time.Second) nc2, err = nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() // Check that we can still send messages. nc2.Publish("foo", nil) checkForMsg() } func TestConfigReloadAccountResolverTLSConfig(t *testing.T) { kp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(kp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } pub, _ := kp.PublicKey() tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", } tlsConfig, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(ajwt)) })) ts.TLS = tlsConfig ts.StartTLS() defer ts.Close() // Set a dummy logger to prevent tls bad certificate output to stderr. ts.Config.ErrorLog = log.New(&bytes.Buffer{}, "", 0) confTemplate := ` listen: -1 trusted_keys: %s resolver: URL("%s/ngs/v1/accounts/jwt/") %s ` conf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, ` resolver_tls { cert_file: "../test/configs/certs/client-cert.pem" key_file: "../test/configs/certs/client-key.pem" ca_file: "../test/configs/certs/ca.pem" } `))) s, _ := RunServerWithConfig(conf) defer s.Shutdown() changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, ""))) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } if _, err := s.LookupAccount(apub); err == nil { t.Fatal("Expected error during lookup, did not get one") } changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, ` resolver_tls { insecure: true } `))) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } acc, err := s.LookupAccount(apub) if err != nil { t.Fatalf("Error during lookup: %v", err) } if acc == nil { t.Fatalf("Expected to receive an account") } if acc.Name != apub { t.Fatalf("Account name did not match claim key") } } func TestConfigReloadLogging(t *testing.T) { // This test basically starts a server and causes it's configuration to be reloaded 3 times. // Each time, a new log file is created and trace levels are turned, off - on - off. // At the end of the test, all 3 log files are inspected for certain traces. countMatches := func(log []byte, stmts ...string) int { matchCnt := 0 for _, stmt := range stmts { if strings.Contains(string(log), stmt) { matchCnt++ } } return matchCnt } traces := []string{"[TRC]", "[DBG]", "SYSTEM", "MSG_PAYLOAD", "$SYS.SERVER.ACCOUNT"} didTrace := func(log []byte) bool { return countMatches(log, "[INF] Reloaded server configuration") == 1 } tracingAbsent := func(log []byte) bool { return countMatches(log, traces...) == 0 && didTrace(log) } tracingPresent := func(log []byte) bool { return len(traces) == countMatches(log, traces...) && didTrace(log) } check := func(filename string, valid func([]byte) bool) { t.Helper() log, err := os.ReadFile(filename) if err != nil { t.Fatalf("Error reading log file %s: %v\n", filename, err) } if !valid(log) { t.Fatalf("%s is not valid: %s", filename, log) } //t.Logf("%s contains: %s\n", filename, log) } // common configuration setting up system accounts. trace_verbose needs this to cause traces commonCfg := ` port: -1 system_account: sys accounts { sys { users = [ {user: sys, pass: "" } ] } nats.io: { users = [ { user : bar, pass: "pwd" } ] } } ` conf := createConfFile(t, []byte(commonCfg)) defer removeFile(t, "off-pre.log") defer removeFile(t, "on.log") defer removeFile(t, "off-post.log") s, opts := RunServerWithConfig(conf) defer s.Shutdown() reload := func(change string) { t.Helper() changeCurrentConfigContentWithNewContent(t, conf, []byte(commonCfg+` `+change+` `)) if err := s.Reload(); err != nil { t.Fatalf("Error during reload: %v", err) } } traffic := func(cnt int) { t.Helper() // Create client and sub interest on server and create traffic urlSeed := fmt.Sprintf("nats://bar:pwd@%s:%d/", opts.Host, opts.Port) nc, err := nats.Connect(urlSeed) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc.Close() msgs := make(chan *nats.Msg, 1) defer close(msgs) sub, err := nc.ChanSubscribe("foo", msgs) if err != nil { t.Fatalf("Error creating subscriber: %v\n", err) } nc.Flush() for i := 0; i < cnt; i++ { if err := nc.Publish("foo", []byte("bar")); err == nil { <-msgs } } sub.Unsubscribe() nc.Close() } reload("log_file: off-pre.log") traffic(10) // generate NO trace/debug entries in off-pre.log reload(` log_file: on.log debug: true trace_verbose: true `) traffic(10) // generate trace/debug entries in on.log reload(` log_file: off-post.log debug: false trace_verbose: false `) traffic(10) // generate trace/debug entries in off-post.log // check resulting log files for expected content check("off-pre.log", tracingAbsent) check("on.log", tracingPresent) check("off-post.log", tracingAbsent) } func TestConfigReloadValidate(t *testing.T) { confFileName := createConfFile(t, []byte(` listen: "127.0.0.1:-1" no_auth_user: a authorization { users [ {user: "a", password: "a"}, {user: "b", password: "b"} ] } `)) srv, _ := RunServerWithConfig(confFileName) if srv == nil { t.Fatal("Server did not start") } // Induce error by removing the user no_auth_user points to changeCurrentConfigContentWithNewContent(t, confFileName, []byte(` listen: "127.0.0.1:-1" no_auth_user: a authorization { users [ {user: "b", password: "b"} ] } `)) if err := srv.Reload(); err == nil { t.Fatal("Expected error on reload, got none") } else if strings.HasPrefix(err.Error(), " no_auth_user:") { t.Logf("Expected no_auth_user error, got different one %s", err) } srv.Shutdown() } func TestConfigReloadAccounts(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" system_account: SYS accounts { SYS { users = [ {user: sys, password: pwd} ] } ACC { users = [ {user: usr, password: pwd} ] } acc_deleted_after_reload_will_trigger_reload_of_all_accounts { users = [ {user: notused, password: soon} ] } } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() urlSys := fmt.Sprintf("nats://sys:pwd@%s:%d", o.Host, o.Port) urlUsr := fmt.Sprintf("nats://usr:pwd@%s:%d", o.Host, o.Port) oldAcci, ok := s.accounts.Load("SYS") if !ok { t.Fatal("No SYS account") } oldAcc := oldAcci.(*Account) testSrvState := func(oldAcc *Account) { t.Helper() sysAcc := s.SystemAccount() s.mu.Lock() defer s.mu.Unlock() if s.sys == nil || sysAcc == nil { t.Fatal("Expected sys.account to be non-nil") } if sysAcc.Name != "SYS" { t.Fatal("Found wrong sys.account") } if s.opts.SystemAccount != "SYS" { t.Fatal("Found wrong sys.account") } ai, ok := s.accounts.Load(s.opts.SystemAccount) if !ok { t.Fatalf("System account %q not found in s.accounts map", s.opts.SystemAccount) } acc := ai.(*Account) if acc != oldAcc { t.Fatalf("System account pointer was changed during reload, was %p now %p", oldAcc, acc) } if s.sys.client == nil { t.Fatal("Expected sys.client to be non-nil") } s.sys.client.mu.Lock() defer s.sys.client.mu.Unlock() if s.sys.client.acc.Name != "SYS" { t.Fatal("Found wrong sys.account") } if s.sys.client.echo { t.Fatal("Internal clients should always have echo false") } s.sys.account.mu.Lock() if _, ok := s.sys.account.clients[s.sys.client]; !ok { s.sys.account.mu.Unlock() t.Fatal("internal client not present") } s.sys.account.mu.Unlock() } // Below tests use connection names so that they can be checked for. // The test subscribes to ACC only. This avoids receiving own messages. subscribe := func(name string) (*nats.Conn, *nats.Subscription, *nats.Subscription) { t.Helper() c, err := nats.Connect(urlSys, nats.Name(name)) if err != nil { t.Fatalf("Error on connect: %v", err) } subCon, err := c.SubscribeSync("$SYS.ACCOUNT.ACC.CONNECT") if err != nil { t.Fatalf("Error on subscribe CONNECT: %v", err) } subDis, err := c.SubscribeSync("$SYS.ACCOUNT.ACC.DISCONNECT") if err != nil { t.Fatalf("Error on subscribe DISCONNECT: %v", err) } c.Flush() return c, subCon, subDis } recv := func(name string, sub *nats.Subscription) { t.Helper() if msg, err := sub.NextMsg(1 * time.Second); err != nil { t.Fatalf("%s Error on next: %v", name, err) } else { cMsg := ConnectEventMsg{} json.Unmarshal(msg.Data, &cMsg) if cMsg.Client.Name != name { t.Fatalf("%s wrong message: %s", name, string(msg.Data)) } } } triggerSysEvent := func(name string, subs []*nats.Subscription) { t.Helper() ncs1, err := nats.Connect(urlUsr, nats.Name(name)) if err != nil { t.Fatalf("Error on connect: %v", err) } ncs1.Close() for _, sub := range subs { recv(name, sub) // Make sure they are empty. if pending, _, _ := sub.Pending(); pending != 0 { t.Fatalf("Expected no pending, got %d for %+v", pending, sub) } } } testSrvState(oldAcc) c1, s1C, s1D := subscribe("SYS1") defer c1.Close() defer s1C.Unsubscribe() defer s1D.Unsubscribe() triggerSysEvent("BEFORE1", []*nats.Subscription{s1C, s1D}) triggerSysEvent("BEFORE2", []*nats.Subscription{s1C, s1D}) // Remove account to trigger account reload reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" system_account: SYS accounts { SYS { users = [ {user: sys, password: pwd} ] } ACC { users = [ {user: usr, password: pwd} ] } } `) testSrvState(oldAcc) c2, s2C, s2D := subscribe("SYS2") defer c2.Close() defer s2C.Unsubscribe() defer s2D.Unsubscribe() // test new and existing subscriptions triggerSysEvent("AFTER1", []*nats.Subscription{s1C, s1D, s2C, s2D}) triggerSysEvent("AFTER2", []*nats.Subscription{s1C, s1D, s2C, s2D}) } func TestConfigReloadDefaultSystemAccount(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { ACC { users = [ {user: usr, password: pwd} ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() sysAcc := s.SystemAccount() if sysAcc == nil { t.Fatalf("Expected system account to be present") } numSubs := sysAcc.TotalSubs() sname := sysAcc.GetName() testInAccounts := func() { t.Helper() var found bool s.accounts.Range(func(k, v any) bool { acc := v.(*Account) if acc.GetName() == sname { found = true return false } return true }) if !found { t.Fatalf("System account not found in accounts list") } } testInAccounts() if err := s.Reload(); err != nil { t.Fatalf("Unexpected error reloading: %v", err) } sysAcc = s.SystemAccount() if sysAcc == nil { t.Fatalf("Expected system account to still be present") } if sysAcc.TotalSubs() != numSubs { t.Fatalf("Expected %d subs, got %d", numSubs, sysAcc.TotalSubs()) } testInAccounts() } func TestConfigReloadAccountMappings(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { ACC { users = [{user: usr, password: pwd}] mappings = { foo: bar } } } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" accounts { ACC { users = [{user: usr, password: pwd}] mappings = { foo: baz } } } `) nc := natsConnect(t, fmt.Sprintf("nats://usr:pwd@%s:%d", opts.Host, opts.Port)) defer nc.Close() fsub, _ := nc.SubscribeSync("foo") sub, _ := nc.SubscribeSync("baz") nc.Publish("foo", nil) nc.Flush() checkPending := func(sub *nats.Subscription, expected int) { t.Helper() if n, _, _ := sub.Pending(); n != expected { t.Fatalf("Expected %d msgs for %q, but got %d", expected, sub.Subject, n) } } checkPending(fsub, 0) checkPending(sub, 1) // Drain it off if _, err := sub.NextMsg(2 * time.Second); err != nil { t.Fatalf("Error receiving msg: %v", err) } reloadUpdateConfig(t, s, conf, ` listen: "127.0.0.1:-1" accounts { ACC { users = [{user: usr, password: pwd}] } } `) nc.Publish("foo", nil) nc.Flush() checkPending(fsub, 1) checkPending(sub, 0) } func TestConfigReloadWithSysAccountOnly(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" accounts { $SYS { users = [{user: "system",pass: "password"}, {user: "system2",pass: "password2"}] } } `)) defer os.Remove(conf) s, _ := RunServerWithConfig(conf) defer s.Shutdown() dch := make(chan struct{}, 1) nc := natsConnect(t, s.ClientURL(), nats.DisconnectErrHandler(func(_ *nats.Conn, _ error) { dch <- struct{}{} }), nats.NoCallbacksAfterClientClose()) defer nc.Close() // Just reload... if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } // Make sure we did not get disconnected select { case <-dch: t.Fatal("Got disconnected!") case <-time.After(500 * time.Millisecond): // ok } } func TestConfigReloadRouteImportPermissionsWithAccounts(t *testing.T) { for _, test := range []struct { name string poolSize string accounts string }{ {"regular", "pool_size: -1", _EMPTY_}, {"pooling", "pool_size: 5", _EMPTY_}, {"per-account", _EMPTY_, "accounts: [\"A\"]"}, {"pool and per-account", "pool_size: 3", "accounts: [\"A\"]"}, } { t.Run("import "+test.name, func(t *testing.T) { confATemplate := ` server_name: "A" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 permissions { import { allow: %s } export { allow: ">" } } %s %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `"foo"`, test.poolSize, test.accounts))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` server_name: "B" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 name: "local" permissions { import { allow: %s } export { allow: ">" } } routes = [ "nats://127.0.0.1:%d" ] %s %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `"foo"`, optsA.Cluster.Port, test.poolSize, test.accounts))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) ncA := natsConnect(t, srva.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncA.Close() sub1Foo := natsSubSync(t, ncA, "foo") sub2Foo := natsSubSync(t, ncA, "foo") sub1Bar := natsSubSync(t, ncA, "bar") sub2Bar := natsSubSync(t, ncA, "bar") natsFlush(t, ncA) checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubNoInterest(t, srvb, "A", "bar", 2*time.Second) ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncB.Close() check := func(sub *nats.Subscription, expected bool) { t.Helper() if expected { natsNexMsg(t, sub, time.Second) } else { if msg, err := sub.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Should not have gotten the message, got %s/%s", msg.Subject, msg.Data) } } } // Should receive on "foo" natsPub(t, ncB, "foo", []byte("foo1")) check(sub1Foo, true) check(sub2Foo, true) // But not on "bar" natsPub(t, ncB, "bar", []byte("bar1")) check(sub1Bar, false) check(sub2Bar, false) reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"bar"`, test.poolSize, test.accounts)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `"bar"`, optsA.Cluster.Port, test.poolSize, test.accounts)) checkClusterFormed(t, srva, srvb) checkSubNoInterest(t, srvb, "A", "foo", 2*time.Second) checkSubInterest(t, srvb, "A", "bar", 2*time.Second) // Should not receive on foo natsPub(t, ncB, "foo", []byte("foo2")) check(sub1Foo, false) check(sub2Foo, false) // Should be able to receive on bar natsPub(t, ncB, "bar", []byte("bar2")) check(sub1Bar, true) check(sub2Bar, true) // Restore "foo" reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"foo"`, test.poolSize, test.accounts)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `"foo"`, optsA.Cluster.Port, test.poolSize, test.accounts)) checkClusterFormed(t, srva, srvb) checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubNoInterest(t, srvb, "A", "bar", 2*time.Second) // Should receive on "foo" natsPub(t, ncB, "foo", []byte("foo3")) check(sub1Foo, true) check(sub2Foo, true) // But make sure there are no more than what we expect check(sub1Foo, false) check(sub2Foo, false) // And now "bar" should fail natsPub(t, ncB, "bar", []byte("bar3")) check(sub1Bar, false) check(sub2Bar, false) }) } // Check export now for _, test := range []struct { name string poolSize string accounts string }{ {"regular", "pool_size: -1", _EMPTY_}, {"pooling", "pool_size: 5", _EMPTY_}, {"per-account", _EMPTY_, "accounts: [\"A\"]"}, {"pool and per-account", "pool_size: 3", "accounts: [\"A\"]"}, } { t.Run("export "+test.name, func(t *testing.T) { confATemplate := ` server_name: "A" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 permissions { import { allow: ">" } export { allow: %s } } %s %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `"foo"`, test.poolSize, test.accounts))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` server_name: "B" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 name: "local" permissions { import { allow: ">" } export { allow: %s } } routes = [ "nats://127.0.0.1:%d" ] %s %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `"foo"`, optsA.Cluster.Port, test.poolSize, test.accounts))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) ncA := natsConnect(t, srva.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncA.Close() sub1Foo := natsSubSync(t, ncA, "foo") sub2Foo := natsSubSync(t, ncA, "foo") sub1Bar := natsSubSync(t, ncA, "bar") sub2Bar := natsSubSync(t, ncA, "bar") natsFlush(t, ncA) checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubNoInterest(t, srvb, "A", "bar", 2*time.Second) ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncB.Close() check := func(sub *nats.Subscription, expected bool) { t.Helper() if expected { natsNexMsg(t, sub, time.Second) } else { if msg, err := sub.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Should not have gotten the message, got %s/%s", msg.Subject, msg.Data) } } } // Should receive on "foo" natsPub(t, ncB, "foo", []byte("foo1")) check(sub1Foo, true) check(sub2Foo, true) // But not on "bar" natsPub(t, ncB, "bar", []byte("bar1")) check(sub1Bar, false) check(sub2Bar, false) reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `["foo", "bar"]`, test.poolSize, test.accounts)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `["foo", "bar"]`, optsA.Cluster.Port, test.poolSize, test.accounts)) checkClusterFormed(t, srva, srvb) checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubInterest(t, srvb, "A", "bar", 2*time.Second) // Should receive on foo and bar natsPub(t, ncB, "foo", []byte("foo2")) check(sub1Foo, true) check(sub2Foo, true) natsPub(t, ncB, "bar", []byte("bar2")) check(sub1Bar, true) check(sub2Bar, true) // Remove "bar" reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `"foo"`, test.poolSize, test.accounts)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `"foo"`, optsA.Cluster.Port, test.poolSize, test.accounts)) checkClusterFormed(t, srva, srvb) checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubNoInterest(t, srvb, "A", "bar", 2*time.Second) // Should receive on "foo" natsPub(t, ncB, "foo", []byte("foo3")) check(sub1Foo, true) check(sub2Foo, true) // But make sure there are no more than what we expect check(sub1Foo, false) check(sub2Foo, false) // And now "bar" should fail natsPub(t, ncB, "bar", []byte("bar3")) check(sub1Bar, false) check(sub2Bar, false) }) } } func TestConfigReloadRoutePoolAndPerAccount(t *testing.T) { confATemplate := ` port: -1 server_name: "A" accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 %s %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, "pool_size: 3", "accounts: [\"A\"]"))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBCTemplate := ` port: -1 server_name: "%s" accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 name: "local" routes = [ "nats://127.0.0.1:%d" ] %s %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\"]"))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() confC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\"]"))) srvc, _ := RunServerWithConfig(confC) defer srvc.Shutdown() checkClusterFormed(t, srva, srvb, srvc) // We will also create subscriptions for accounts A, B and C on all sides // just to make sure that interest is properly propagated after a reload. // The conns slices will contain connections for accounts A on srva, srvb, // srvc, then B on srva, srvb, etc.. and the subs slices will contain // subscriptions for account A on foo on srva, bar on srvb, baz on srvc, // then for account B on foo on srva, etc... var conns []*nats.Conn var subs []*nats.Subscription for _, user := range []string{"user1", "user2", "user3"} { nc := natsConnect(t, srva.ClientURL(), nats.UserInfo(user, "pwd")) defer nc.Close() conns = append(conns, nc) sub := natsSubSync(t, nc, "foo") subs = append(subs, sub) nc = natsConnect(t, srvb.ClientURL(), nats.UserInfo(user, "pwd")) defer nc.Close() conns = append(conns, nc) sub = natsSubSync(t, nc, "bar") subs = append(subs, sub) nc = natsConnect(t, srvc.ClientURL(), nats.UserInfo(user, "pwd")) defer nc.Close() conns = append(conns, nc) sub = natsSubSync(t, nc, "baz") subs = append(subs, sub) } checkCluster := func() { t.Helper() checkClusterFormed(t, srva, srvb, srvc) for _, acc := range []string{"A", "B", "C"} { // On server A, there should be interest for bar/baz checkSubInterest(t, srva, acc, "bar", 2*time.Second) checkSubInterest(t, srva, acc, "baz", 2*time.Second) // On serer B, there should be interest on foo/baz checkSubInterest(t, srvb, acc, "foo", 2*time.Second) checkSubInterest(t, srvb, acc, "baz", 2*time.Second) // And on server C, interest on foo/bar checkSubInterest(t, srvc, acc, "foo", 2*time.Second) checkSubInterest(t, srvc, acc, "bar", 2*time.Second) } } checkCluster() getAccRouteID := func(acc string) uint64 { s := srva var id uint64 srvbId := srvb.ID() s.mu.RLock() if remotes, ok := s.accRoutes[acc]; ok { // For this test, we will take a single remote, say srvb if r := remotes[srvbId]; r != nil { r.mu.Lock() if string(r.route.accName) == acc { id = r.cid } r.mu.Unlock() } } s.mu.RUnlock() return id } // Capture the route for account "A" raid := getAccRouteID("A") if raid == 0 { t.Fatal("Did not find route for account A") } getRouteIDForAcc := func(acc string) uint64 { s := srva a, _ := s.LookupAccount(acc) if a == nil { return 0 } a.mu.RLock() pidx := a.routePoolIdx a.mu.RUnlock() var id uint64 s.mu.RLock() // For this test, we will take a single remote, say srvb srvbId := srvb.ID() if conns, ok := s.routes[srvbId]; ok { if r := conns[pidx]; r != nil { r.mu.Lock() id = r.cid r.mu.Unlock() } } s.mu.RUnlock() return id } rbid := getRouteIDForAcc("B") if rbid == 0 { t.Fatal("Did not find route for account B") } rcid := getRouteIDForAcc("C") if rcid == 0 { t.Fatal("Did not find route for account C") } rdid := getRouteIDForAcc("D") if rdid == 0 { t.Fatal("Did not find route for account D") } sendAndRecv := func(msg string) { t.Helper() for accIdx := 0; accIdx < 9; accIdx += 3 { natsPub(t, conns[accIdx], "bar", []byte(msg)) m := natsNexMsg(t, subs[accIdx+1], time.Second) checkMsg := func(m *nats.Msg, subj string) { t.Helper() if string(m.Data) != msg { t.Fatalf("For accIdx=%v, subject %q, expected message %q, got %q", accIdx, subj, msg, m.Data) } } checkMsg(m, "bar") natsPub(t, conns[accIdx+1], "baz", []byte(msg)) m = natsNexMsg(t, subs[accIdx+2], time.Second) checkMsg(m, "baz") natsPub(t, conns[accIdx+2], "foo", []byte(msg)) m = natsNexMsg(t, subs[accIdx], time.Second) checkMsg(m, "foo") } } sendAndRecv("0") // Now add accounts "B" and "D" and do a config reload. reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 3", "accounts: [\"A\",\"B\",\"D\"]")) // Even before reloading srvb and srvc, we should already have per-account // routes for accounts B and D being established. The accounts routePoolIdx // should be marked as transitioning. checkAccPoolIdx := func(s *Server, acc string, expected int) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { s.mu.RLock() defer s.mu.RUnlock() if a, ok := s.accounts.Load(acc); ok { acc := a.(*Account) acc.mu.RLock() rpi := acc.routePoolIdx acc.mu.RUnlock() if rpi != expected { return fmt.Errorf("Server %q - Account %q routePoolIdx should be %v, but is %v", s, acc, expected, rpi) } return nil } return fmt.Errorf("Server %q - Account %q not found", s, acc) }) } checkRoutePerAccAlreadyEstablished := func(s *Server, acc string) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { s.mu.RLock() defer s.mu.RUnlock() if _, ok := s.accRoutes[acc]; !ok { return fmt.Errorf("Route for account %q still not established", acc) } return nil }) checkAccPoolIdx(s, acc, accTransitioningToDedicatedRoute) } // Check srvb and srvc for both accounts. for _, s := range []*Server{srvb, srvc} { for _, acc := range []string{"B", "D"} { checkRoutePerAccAlreadyEstablished(s, acc) } } // On srva, the accounts should already have their routePoolIdx set to // the accDedicatedRoute value. for _, acc := range []string{"B", "D"} { checkAccPoolIdx(srva, acc, accDedicatedRoute) } // Now reload the other servers reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\",\"B\",\"D\"]")) reloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\",\"B\",\"D\"]")) checkCluster() // Now check that the accounts B and D are no longer transitioning for _, s := range []*Server{srva, srvb, srvc} { for _, acc := range []string{"B", "D"} { checkAccPoolIdx(s, acc, accDedicatedRoute) } } checkRouteForADidNotChange := func() { t.Helper() if id := getAccRouteID("A"); id != raid { t.Fatalf("Route id for account 'A' was %d, is now %d", raid, id) } } // Verify that the route for account "A" did not change. checkRouteForADidNotChange() // Verify that account "B" has now its own route if id := getAccRouteID("B"); id == 0 { t.Fatal("Did not find route for account B") } // Same for "D". if id := getAccRouteID("D"); id == 0 { t.Fatal("Did not find route for account D") } checkRouteStillPresent := func(id uint64) { t.Helper() srva.mu.RLock() defer srva.mu.RUnlock() srvbId := srvb.ID() for _, r := range srva.routes[srvbId] { if r != nil { r.mu.Lock() found := r.cid == id r.mu.Unlock() if found { return } } } t.Fatalf("Route id %v has been disconnected", id) } // Verify that routes that were dealing with "B", and "D" were not disconnected. // Of course, since "C" was not involved, that route should still be present too. checkRouteStillPresent(rbid) checkRouteStillPresent(rcid) checkRouteStillPresent(rdid) sendAndRecv("1") // Now remove "B" and "D" and verify that route for "A" did not change. reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 3", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\"]")) checkCluster() // Verify that the route for account "A" did not change. checkRouteForADidNotChange() // Verify that there is no dedicated route for account "B" if id := getAccRouteID("B"); id != 0 { t.Fatal("Should not have found a route for account B") } // It should instead be in one of the pooled route, and same // than it was before. if id := getRouteIDForAcc("B"); id != rbid { t.Fatalf("Account B's route was %d, it is now %d", rbid, id) } // Same for "D" if id := getAccRouteID("D"); id != 0 { t.Fatal("Should not have found a route for account D") } if id := getRouteIDForAcc("D"); id != rdid { t.Fatalf("Account D's route was %d, it is now %d", rdid, id) } sendAndRecv("2") // Finally, change pool size and make sure that routes handling B, C and D // were disconnected/reconnected, and that A did not change. reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 5", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 5", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "pool_size: 5", "accounts: [\"A\"]")) checkCluster() checkRouteForADidNotChange() checkRouteDisconnected := func(acc string, oldID uint64) { t.Helper() if id := getRouteIDForAcc(acc); id == oldID { t.Fatalf("Route that was handling account %q did not change", acc) } } checkRouteDisconnected("B", rbid) checkRouteDisconnected("C", rcid) checkRouteDisconnected("D", rdid) sendAndRecv("3") // Now check that there were no duplicates and that all subs have 0 pending messages. for i, sub := range subs { if n, _, _ := sub.Pending(); n != 0 { t.Fatalf("Expected 0 pending messages, got %v for accIdx=%d sub=%q", n, i, sub.Subject) } } } func TestConfigReloadRoutePoolCannotBeDisabledIfAccountsPresent(t *testing.T) { tmpl := ` port: -1 server_name: "%s" accounts { A { users: [{user: "user1", password: "pwd"}] } B { users: [{user: "user2", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 %s %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", "accounts: [\"A\"]", _EMPTY_, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", "accounts: [\"A\"]", _EMPTY_, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) err := os.WriteFile(conf1, []byte(fmt.Sprintf(tmpl, "A", "accounts: [\"A\"]", "pool_size: -1", _EMPTY_)), 0666) require_NoError(t, err) if err := s1.Reload(); err == nil || !strings.Contains(err.Error(), "accounts") { t.Fatalf("Expected error regarding presence of accounts, got %v", err) } // Now remove the accounts too and reload, this should work reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "pool_size: -1", _EMPTY_)) reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", _EMPTY_, "pool_size: -1", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port))) checkClusterFormed(t, s1, s2) ncs2 := natsConnect(t, s2.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncs2.Close() sub := natsSubSync(t, ncs2, "foo") checkSubInterest(t, s1, "A", "foo", time.Second) ncs1 := natsConnect(t, s1.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncs1.Close() natsPub(t, ncs1, "foo", []byte("hello")) natsNexMsg(t, sub, time.Second) // Wait a bit and make sure there are no duplicates time.Sleep(50 * time.Millisecond) if n, _, _ := sub.Pending(); n != 0 { t.Fatalf("Expected no pending messages, got %v", n) } // Finally, verify that the system account is no longer bound to // a dedicated route. For that matter, s.accRoutes should be nil. for _, s := range []*Server{s1, s2} { sys := s.SystemAccount() if sys == nil { t.Fatal("No system account found") } sys.mu.RLock() rpi := sys.routePoolIdx sys.mu.RUnlock() if rpi != 0 { t.Fatalf("Server %q - expected account's routePoolIdx to be 0, got %v", s, rpi) } s.mu.RLock() arNil := s.accRoutes == nil s.mu.RUnlock() if !arNil { t.Fatalf("Server %q - accRoutes expected to be nil, it was not", s) } } } func TestConfigReloadRoutePoolAndPerAccountWithOlderServer(t *testing.T) { confATemplate := ` port: -1 server_name: "A" accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 %s %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, "pool_size: 3", _EMPTY_))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBCTemplate := ` port: -1 server_name: "%s" accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 name: "local" routes = [ "nats://127.0.0.1:%d" ] %s %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", _EMPTY_))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() confC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "pool_size: -1", _EMPTY_))) srvc, _ := RunServerWithConfig(confC) defer srvc.Shutdown() checkClusterFormed(t, srva, srvb, srvc) // Create a connection and sub on B and C ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncB.Close() subB := natsSubSync(t, ncB, "foo") ncC := natsConnect(t, srvc.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncC.Close() subC := natsSubSync(t, ncC, "bar") // Check that on server B, there is interest on "bar" for account A // (coming from server C), and on server C, there is interest on "foo" // for account A (coming from server B). checkCluster := func() { t.Helper() checkClusterFormed(t, srva, srvb, srvc) checkSubInterest(t, srvb, "A", "bar", 2*time.Second) checkSubInterest(t, srvc, "A", "foo", 2*time.Second) } checkCluster() sendAndRecv := func(msg string) { t.Helper() natsPub(t, ncB, "bar", []byte(msg)) if m := natsNexMsg(t, subC, time.Second); string(m.Data) != msg { t.Fatalf("Expected message %q on %q, got %q", msg, "bar", m.Data) } natsPub(t, ncC, "foo", []byte(msg)) if m := natsNexMsg(t, subB, time.Second); string(m.Data) != msg { t.Fatalf("Expected message %q on %q, got %q", msg, "foo", m.Data) } } sendAndRecv("0") // Now add account "A" and do a config reload. We do this only on // server srva and srb since server C really does not change. reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 3", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", "accounts: [\"A\"]")) checkCluster() sendAndRecv("1") // Remove "A" from the accounts list reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 3", _EMPTY_)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", _EMPTY_)) checkCluster() sendAndRecv("2") // Change the pool size reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 5", _EMPTY_)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 5", _EMPTY_)) checkCluster() sendAndRecv("3") // Add account "A" and change the pool size reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 4", "accounts: [\"A\"]")) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 4", "accounts: [\"A\"]")) checkCluster() sendAndRecv("4") // Remove account "A" and change the pool size reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "pool_size: 3", _EMPTY_)) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "pool_size: 3", _EMPTY_)) checkCluster() sendAndRecv("5") } func TestConfigReloadRoutePoolAndPerAccountNoDuplicateSub(t *testing.T) { confATemplate := ` port: -1 server_name: "A" accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 pool_size: 3 %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, _EMPTY_))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBCTemplate := ` port: -1 server_name: "%s" accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 name: "local" routes = [ "nats://127.0.0.1:%d" ] pool_size: 3 %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, _EMPTY_))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() confC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, _EMPTY_))) srvc, _ := RunServerWithConfig(confC) defer srvc.Shutdown() checkClusterFormed(t, srva, srvb, srvc) ncC := natsConnect(t, srvc.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncC.Close() ch := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(1) var subs []*nats.Subscription go func() { defer wg.Done() // Limit the number of subscriptions. From experimentation, the issue would // arise around subscriptions ~700. for i := 0; i < 1000; i++ { if sub, err := ncC.SubscribeSync(fmt.Sprintf("foo.%d", i)); err == nil { subs = append(subs, sub) } select { case <-ch: return default: if i%100 == 0 { time.Sleep(5 * time.Millisecond) } } } }() // Wait a tiny bit before doing the configuration reload. time.Sleep(100 * time.Millisecond) reloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, "accounts: [\"A\"]")) reloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, "B", optsA.Cluster.Port, "accounts: [\"A\"]")) reloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, "C", optsA.Cluster.Port, "accounts: [\"A\"]")) checkClusterFormed(t, srva, srvb, srvc) close(ch) wg.Wait() for _, sub := range subs { checkSubInterest(t, srvb, "A", sub.Subject, 500*time.Millisecond) } ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncB.Close() for _, sub := range subs { natsPub(t, ncB, sub.Subject, []byte("hello")) } // Now make sure that there is only 1 pending message for each sub. // Wait a bit to give a chance to duplicate messages to arrive if // there was a bug that would lead to a sub on each route (the pooled // and the per-account) time.Sleep(250 * time.Millisecond) for _, sub := range subs { if n, _, _ := sub.Pending(); n != 1 { t.Fatalf("Expected only 1 message for subscription on %q, got %v", sub.Subject, n) } } } func TestConfigReloadGlobalAccountWithMappingAndJetStream(t *testing.T) { tmpl := ` listen: 127.0.0.1:-1 server_name: %s jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} mappings { subj.orig: subj.mapped.before.reload } leaf { listen: 127.0.0.1:-1 } cluster { name: %s listen: 127.0.0.1:%d routes = [%s] } # For access to system account. accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } ` c := createJetStreamClusterWithTemplate(t, tmpl, "R3S", 3) defer c.shutdown() nc, js := jsClientConnect(t, c.randomServer()) defer nc.Close() // Verify that mapping works checkMapping := func(expectedSubj string) { t.Helper() sub := natsSubSync(t, nc, "subj.>") defer sub.Unsubscribe() natsPub(t, nc, "subj.orig", nil) msg := natsNexMsg(t, sub, time.Second) if msg.Subject != expectedSubj { t.Fatalf("Expected subject to have been mapped to %q, got %q", expectedSubj, msg.Subject) } } checkMapping("subj.mapped.before.reload") // Create a stream and check that we can get the INFO _, err := js.AddStream(&nats.StreamConfig{ Name: "TEST", Replicas: 3, Subjects: []string{"foo"}, Retention: nats.InterestPolicy, }) require_NoError(t, err) c.waitOnStreamLeader(globalAccountName, "TEST") _, err = js.StreamInfo("TEST") require_NoError(t, err) // Change mapping on all servers and issue reload for i, s := range c.servers { opts := c.opts[i] content, err := os.ReadFile(opts.ConfigFile) require_NoError(t, err) reloadUpdateConfig(t, s, opts.ConfigFile, strings.Replace(string(content), "subj.mapped.before.reload", "subj.mapped.after.reload", 1)) } // Make sure the cluster is still formed checkClusterFormed(t, c.servers...) // Now repeat the test for the subject mapping and stream info checkMapping("subj.mapped.after.reload") _, err = js.StreamInfo("TEST") require_NoError(t, err) } func TestConfigReloadRouteCompression(t *testing.T) { org := testDefaultClusterCompression testDefaultClusterCompression = _EMPTY_ defer func() { testDefaultClusterCompression = org }() tmpl := ` port: -1 server_name: "%s" cluster { port: -1 name: "local" %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() routes := fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port) conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", routes, _EMPTY_))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() // Run a 3rd server but make it as if it was an old server. We want to // make sure that reload of s1 and s2 will not affect routes from s3 to // s1/s2 because these do not support compression. conf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "C", routes, "compression: \"not supported\""))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) // Collect routes' cid from servers so we can check if routes are // recreated when they should and are not when they should not. collect := func(s *Server) map[uint64]struct{} { m := make(map[uint64]struct{}) s.mu.RLock() defer s.mu.RUnlock() s.forEachRoute(func(r *client) { r.mu.Lock() m[r.cid] = struct{}{} r.mu.Unlock() }) return m } s1RouteIDs := collect(s1) s2RouteIDs := collect(s2) s3ID := s3.ID() servers := []*Server{s1, s2} checkCompMode := func(s1Expected, s2Expected string, shouldBeNew bool) { t.Helper() // We wait a bit to make sure that we have routes closed before // checking that the cluster has (re)formed. time.Sleep(100 * time.Millisecond) // First, make sure that the cluster is formed checkClusterFormed(t, s1, s2, s3) // Then check that all routes are with the expected mode. We need to // possibly wait a bit since there is negotiation going on. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { for _, s := range servers { var err error s.mu.RLock() s.forEachRoute(func(r *client) { if err != nil { return } r.mu.Lock() var exp string var m map[uint64]struct{} if r.route.remoteID == s3ID { exp = CompressionNotSupported } else if s == s1 { exp = s1Expected } else { exp = s2Expected } if s == s1 { m = s1RouteIDs } else { m = s2RouteIDs } _, present := m[r.cid] cm := r.route.compression r.mu.Unlock() if cm != exp { err = fmt.Errorf("Expected route %v for server %s to have compression mode %q, got %q", r, s, exp, cm) } sbn := shouldBeNew if exp == CompressionNotSupported { // Override for routes to s3 sbn = false } if sbn && present { err = fmt.Errorf("Expected route %v for server %s to be a new route, but it was already present", r, s) } else if !sbn && !present { err = fmt.Errorf("Expected route %v for server %s to not be new", r, s) } }) s.mu.RUnlock() if err != nil { return err } } s1RouteIDs = collect(s1) s2RouteIDs = collect(s2) return nil }) } // Since both started without any compression setting, we default to // "accept" which means that a server can accept/switch to compression // but not initiate compression, so they should both be "off" checkCompMode(CompressionOff, CompressionOff, false) // Now reload s1 with "on" (s2_fast), since s2 is *configured* with "accept", // they should both be CompressionS2Fast, even before we reload s2. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: on")) checkCompMode(CompressionS2Fast, CompressionS2Fast, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: on")) checkCompMode(CompressionS2Fast, CompressionS2Fast, false) // Move on with "better" reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_better")) // s1 should be at "better", but s2 still at "fast" checkCompMode(CompressionS2Better, CompressionS2Fast, false) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_better")) checkCompMode(CompressionS2Better, CompressionS2Better, false) // Move to "best" reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_best")) checkCompMode(CompressionS2Best, CompressionS2Better, false) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_best")) checkCompMode(CompressionS2Best, CompressionS2Best, false) // Now turn off reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: off")) checkCompMode(CompressionOff, CompressionOff, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: off")) checkCompMode(CompressionOff, CompressionOff, false) // When "off" (and not "accept"), enabling 1 is not enough, the reload // has to be done on both to take effect. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_better")) checkCompMode(CompressionOff, CompressionOff, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_better")) checkCompMode(CompressionS2Better, CompressionS2Better, true) // Try now to have different ones reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_best")) // S1 should be "best" but S2 should have stayed at "better" checkCompMode(CompressionS2Best, CompressionS2Better, false) // If we remove the compression setting, it defaults to "accept", which // in that case we want to have a negotiation and use the remote's compression // level. So connections should be re-created. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, _EMPTY_)) checkCompMode(CompressionS2Better, CompressionS2Better, true) // To avoid flapping, add a little sleep here to make sure we have things // settled before reloading s2. time.Sleep(100 * time.Millisecond) // And if we do the same with s2, then we will end-up with no compression. reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, _EMPTY_)) checkCompMode(CompressionOff, CompressionOff, true) } func TestConfigReloadRouteCompressionS2Auto(t *testing.T) { // This test checks s2_auto specific behavior. It makes sure that we update // only if the rtt_thresholds and current RTT value warrants a change and // also that we actually save in c.route.compression the actual compression // level (not s2_auto). tmpl1 := ` port: -1 server_name: "A" cluster { port: -1 name: "local" pool_size: -1 compression: {mode: s2_auto, rtt_thresholds: [%s]} } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, "50ms, 100ms, 150ms"))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" cluster { port: -1 name: "local" pool_size: -1 compression: s2_fast routes: ["nats://127.0.0.1:%d"] } `, o1.Cluster.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) getCompInfo := func() (string, io.Writer) { var cm string var cw io.Writer s1.mu.RLock() // There should be only 1 route... s1.forEachRemote(func(r *client) { r.mu.Lock() cm = r.route.compression cw = r.out.cw r.mu.Unlock() }) s1.mu.RUnlock() return cm, cw } // Capture the s2 writer from s1 to s2 cm, cw := getCompInfo() // We do a reload but really the mode is still s2_auto (even if the current // compression level may be "uncompressed", "better", etc.. so we don't // expect the writer to have changed. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "100ms, 200ms, 300ms")) if ncm, ncw := getCompInfo(); ncm != cm || ncw != cw { t.Fatalf("Expected compression info to have stayed the same, was %q - %p, got %q - %p", cm, cw, ncm, ncw) } } func TestConfigReloadLeafNodeCompression(t *testing.T) { org := testDefaultLeafNodeCompression testDefaultLeafNodeCompression = _EMPTY_ defer func() { testDefaultLeafNodeCompression = org }() tmpl1 := ` port: -1 server_name: "A" leafnodes { port: -1 %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, "compression: accept"))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() port := o1.LeafNode.Port tmpl2 := ` port: -1 server_name: "%s" leafnodes { remotes [ { url: "nats://127.0.0.1:%d" %s } ] } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, "B", port, "compression: accept"))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() // Run a 3rd server but make it as if it was an old server. We want to // make sure that reload of s1 and s2 will not affect leafnodes from s3 to // s1/s2 because these do not support compression. conf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, "C", port, "compression: \"not supported\""))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkLeafNodeConnected(t, s2) checkLeafNodeConnected(t, s3) checkLeafNodeConnectedCount(t, s1, 2) // Collect leafnodes' cid from servers so we can check if connections are // recreated when they should and are not when they should not. collect := func(s *Server) map[uint64]struct{} { m := make(map[uint64]struct{}) s.mu.RLock() defer s.mu.RUnlock() for _, l := range s.leafs { l.mu.Lock() m[l.cid] = struct{}{} l.mu.Unlock() } return m } s1LeafNodeIDs := collect(s1) s2LeafNodeIDs := collect(s2) servers := []*Server{s1, s2} checkCompMode := func(s1Expected, s2Expected string, shouldBeNew bool) { t.Helper() // We wait a bit to make sure that we have leaf connections closed // before checking that they are properly reconnected. time.Sleep(100 * time.Millisecond) checkLeafNodeConnected(t, s2) checkLeafNodeConnected(t, s3) checkLeafNodeConnectedCount(t, s1, 2) // Check that all leafnodes are with the expected mode. We need to // possibly wait a bit since there is negotiation going on. checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { for _, s := range servers { var err error s.mu.RLock() for _, l := range s.leafs { l.mu.Lock() var exp string var m map[uint64]struct{} if l.leaf.remoteServer == "C" { exp = CompressionNotSupported } else if s == s1 { exp = s1Expected } else { exp = s2Expected } if s == s1 { m = s1LeafNodeIDs } else { m = s2LeafNodeIDs } _, present := m[l.cid] cm := l.leaf.compression l.mu.Unlock() if cm != exp { err = fmt.Errorf("Expected leaf %v for server %s to have compression mode %q, got %q", l, s, exp, cm) } sbn := shouldBeNew if exp == CompressionNotSupported { // Override for routes to s3 sbn = false } if sbn && present { err = fmt.Errorf("Expected leaf %v for server %s to be a new leaf, but it was already present", l, s) } else if !sbn && !present { err = fmt.Errorf("Expected leaf %v for server %s to not be new", l, s) } if err != nil { break } } s.mu.RUnlock() if err != nil { return err } } s1LeafNodeIDs = collect(s1) s2LeafNodeIDs = collect(s2) return nil }) } // Since both started with compression "accept", they should both be set to "off" checkCompMode(CompressionOff, CompressionOff, false) // Now reload s1 with "on" (s2_auto), since s2 is *configured* with "accept", // s1 should be "uncompressed" (due to low RTT), and s2 is in that case set // to s2_fast. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: on")) checkCompMode(CompressionS2Uncompressed, CompressionS2Fast, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: on")) checkCompMode(CompressionS2Uncompressed, CompressionS2Uncompressed, false) // Move on with "better" reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: s2_better")) // s1 should be at "better", but s2 still at "uncompressed" checkCompMode(CompressionS2Better, CompressionS2Uncompressed, false) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: s2_better")) checkCompMode(CompressionS2Better, CompressionS2Better, false) // Move to "best" reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: s2_best")) checkCompMode(CompressionS2Best, CompressionS2Better, false) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: s2_best")) checkCompMode(CompressionS2Best, CompressionS2Best, false) // Now turn off reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: off")) checkCompMode(CompressionOff, CompressionOff, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: off")) checkCompMode(CompressionOff, CompressionOff, false) // When "off" (and not "accept"), enabling 1 is not enough, the reload // has to be done on both to take effect. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: s2_better")) checkCompMode(CompressionOff, CompressionOff, true) // Now reload s2 reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: s2_better")) checkCompMode(CompressionS2Better, CompressionS2Better, true) // Try now to have different ones reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: s2_best")) // S1 should be "best" but S2 should have stayed at "better" checkCompMode(CompressionS2Best, CompressionS2Better, false) // Change the setting to "accept", which in that case we want to have a // negotiation and use the remote's compression level. So connections // should be re-created. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "compression: accept")) checkCompMode(CompressionS2Better, CompressionS2Better, true) // To avoid flapping, add a little sleep here to make sure we have things // settled before reloading s2. time.Sleep(100 * time.Millisecond) // And if we do the same with s2, then we will end-up with no compression. reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, "compression: accept")) checkCompMode(CompressionOff, CompressionOff, true) // Now remove completely and we should default to s2_auto, which means that // s1 should be at "uncompressed" and s2 to "fast". reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, _EMPTY_)) checkCompMode(CompressionS2Uncompressed, CompressionS2Fast, true) // Now with s2, both will be "uncompressed" reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, "B", port, _EMPTY_)) checkCompMode(CompressionS2Uncompressed, CompressionS2Uncompressed, false) } func TestConfigReloadLeafNodeCompressionS2Auto(t *testing.T) { // This test checks s2_auto specific behavior. It makes sure that we update // only if the rtt_thresholds and current RTT value warrants a change and // also that we actually save in c.leaf.compression the actual compression // level (not s2_auto). tmpl1 := ` port: -1 server_name: "A" leafnodes { port: -1 compression: {mode: s2_auto, rtt_thresholds: [%s]} } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, "50ms, 100ms, 150ms"))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "B" leafnodes { remotes [{ url: "nats://127.0.0.1:%d", compression: s2_fast}] } `, o1.LeafNode.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkLeafNodeConnected(t, s2) getCompInfo := func() (string, io.Writer) { var cm string var cw io.Writer s1.mu.RLock() // There should be only 1 leaf... for _, l := range s1.leafs { l.mu.Lock() cm = l.leaf.compression cw = l.out.cw l.mu.Unlock() } s1.mu.RUnlock() return cm, cw } // Capture the s2 writer from s1 to s2 cm, cw := getCompInfo() // We do a reload but really the mode is still s2_auto (even if the current // compression level may be "uncompressed", "better", etc.. so we don't // expect the writer to have changed. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, "100ms, 200ms, 300ms")) if ncm, ncw := getCompInfo(); ncm != cm || ncw != cw { t.Fatalf("Expected compression info to have stayed the same, was %q - %p, got %q - %p", cm, cw, ncm, ncw) } } func TestConfigReloadNoPanicOnShutdown(t *testing.T) { tmpl := ` port: -1 jetstream: true accounts { A { users: [{user: A, password: pwd}] %s } B { users: [{user: B, password: pwd}] } } ` for i := 0; i < 50; i++ { conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_))) s, _ := RunServerWithConfig(conf) // Don't use a defer s.Shutdown() here since it would prevent the panic // to be reported (but the test would still fail because of a runtime timeout). err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, "jetstream: true")), 0666) require_NoError(t, err) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() time.Sleep(10 * time.Millisecond) s.Shutdown() }() time.Sleep(8 * time.Millisecond) err = s.Reload() require_NoError(t, err) wg.Wait() } } nats-server-2.10.27/server/ring.go000066400000000000000000000042461477524627100167550ustar00rootroot00000000000000// Copyright 2018-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server // We wrap to hold onto optional items for /connz. type closedClient struct { ConnInfo subs []SubDetail user string acc string } // Fixed sized ringbuffer for closed connections. type closedRingBuffer struct { total uint64 conns []*closedClient } // Create a new ring buffer with at most max items. func newClosedRingBuffer(max int) *closedRingBuffer { rb := &closedRingBuffer{} rb.conns = make([]*closedClient, max) return rb } // Adds in a new closed connection. If there is no more room, // remove the oldest. func (rb *closedRingBuffer) append(cc *closedClient) { rb.conns[rb.next()] = cc rb.total++ } func (rb *closedRingBuffer) next() int { return int(rb.total % uint64(cap(rb.conns))) } func (rb *closedRingBuffer) len() int { if rb.total > uint64(cap(rb.conns)) { return cap(rb.conns) } return int(rb.total) } func (rb *closedRingBuffer) totalConns() uint64 { return rb.total } // This will return a sorted copy of the list which recipient can // modify. If the contents of the client itself need to be modified, // meaning swapping in any optional items, a copy should be made. We // could introduce a new lock and hold that but since we return this // list inside monitor which allows programatic access, we do not // know when it would be done. func (rb *closedRingBuffer) closedClients() []*closedClient { dup := make([]*closedClient, rb.len()) head := rb.next() if rb.total <= uint64(cap(rb.conns)) || head == 0 { copy(dup, rb.conns[:rb.len()]) } else { fp := rb.conns[head:] sp := rb.conns[:head] copy(dup, fp) copy(dup[len(fp):], sp) } return dup } nats-server-2.10.27/server/ring_test.go000066400000000000000000000035461477524627100200160ustar00rootroot00000000000000// Copyright 2018 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "reflect" "testing" ) func TestRBAppendAndLenAndTotal(t *testing.T) { rb := newClosedRingBuffer(10) for i := 0; i < 5; i++ { rb.append(&closedClient{}) } if rbl := rb.len(); rbl != 5 { t.Fatalf("Expected len of 5, got %d", rbl) } if rbt := rb.totalConns(); rbt != 5 { t.Fatalf("Expected total of 5, got %d", rbt) } for i := 0; i < 25; i++ { rb.append(&closedClient{}) } if rbl := rb.len(); rbl != 10 { t.Fatalf("Expected len of 10, got %d", rbl) } if rbt := rb.totalConns(); rbt != 30 { t.Fatalf("Expected total of 30, got %d", rbt) } } func (cc *closedClient) String() string { return cc.user } func TestRBclosedClients(t *testing.T) { rb := newClosedRingBuffer(10) var ui int addConn := func() { ui++ rb.append(&closedClient{user: fmt.Sprintf("%d", ui)}) } max := 100 master := make([]*closedClient, 0, max) for i := 1; i <= max; i++ { master = append(master, &closedClient{user: fmt.Sprintf("%d", i)}) } testList := func(i int) { ccs := rb.closedClients() start := int(rb.totalConns()) - len(ccs) ms := master[start : start+len(ccs)] if !reflect.DeepEqual(ccs, ms) { t.Fatalf("test %d: List result did not match master: %+v vs %+v", i, ccs, ms) } } for i := 0; i < max; i++ { addConn() testList(i) } } nats-server-2.10.27/server/route.go000066400000000000000000002670411477524627100171600ustar00rootroot00000000000000// Copyright 2013-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "crypto/tls" "encoding/json" "fmt" "math/rand" "net" "net/url" "reflect" "runtime" "strconv" "strings" "sync/atomic" "time" "github.com/klauspost/compress/s2" ) // RouteType designates the router type type RouteType int // Type of Route const ( // This route we learned from speaking to other routes. Implicit RouteType = iota // This route was explicitly configured. Explicit ) const ( // RouteProtoZero is the original Route protocol from 2009. // http://nats.io/documentation/internals/nats-protocol/ RouteProtoZero = iota // RouteProtoInfo signals a route can receive more then the original INFO block. // This can be used to update remote cluster permissions, etc... RouteProtoInfo // RouteProtoV2 is the new route/cluster protocol that provides account support. RouteProtoV2 ) // Include the space for the proto var ( aSubBytes = []byte{'A', '+', ' '} aUnsubBytes = []byte{'A', '-', ' '} rSubBytes = []byte{'R', 'S', '+', ' '} rUnsubBytes = []byte{'R', 'S', '-', ' '} lSubBytes = []byte{'L', 'S', '+', ' '} lUnsubBytes = []byte{'L', 'S', '-', ' '} ) // Used by tests func setRouteProtoForTest(wantedProto int) int { return (wantedProto + 1) * -1 } type route struct { remoteID string remoteName string didSolicit bool retry bool lnoc bool lnocu bool routeType RouteType url *url.URL authRequired bool tlsRequired bool jetstream bool connectURLs []string wsConnURLs []string gatewayURL string leafnodeURL string hash string idHash string // Location of the route in the slice: s.routes[remoteID][]*client. // Initialized to -1 on creation, as to indicate that it is not // added to the list. poolIdx int // If this is set, it means that the route is dedicated for this // account and the account name will not be included in protocols. accName []byte // This is set to true if this is a route connection to an old // server or a server that has pooling completely disabled. noPool bool // Selected compression mode, which may be different from the // server configured mode. compression string } type connectInfo struct { Echo bool `json:"echo"` Verbose bool `json:"verbose"` Pedantic bool `json:"pedantic"` User string `json:"user,omitempty"` Pass string `json:"pass,omitempty"` TLS bool `json:"tls_required"` Headers bool `json:"headers"` Name string `json:"name"` Cluster string `json:"cluster"` Dynamic bool `json:"cluster_dynamic,omitempty"` LNOC bool `json:"lnoc,omitempty"` LNOCU bool `json:"lnocu,omitempty"` // Support for LS- with origin cluster name Gateway string `json:"gateway,omitempty"` } // Route protocol constants const ( ConProto = "CONNECT %s" + _CRLF_ InfoProto = "INFO %s" + _CRLF_ ) const ( // Warning when user configures cluster TLS insecure clusterTLSInsecureWarning = "TLS certificate chain and hostname of solicited routes will not be verified. DO NOT USE IN PRODUCTION!" // The default ping interval is set to 2 minutes, which is fine for client // connections, etc.. but for route compression, the CompressionS2Auto // mode uses RTT measurements (ping/pong) to decide which compression level // to use, we want the interval to not be that high. defaultRouteMaxPingInterval = 30 * time.Second ) // Can be changed for tests var ( routeConnectDelay = DEFAULT_ROUTE_CONNECT routeMaxPingInterval = defaultRouteMaxPingInterval ) // removeReplySub is called when we trip the max on remoteReply subs. func (c *client) removeReplySub(sub *subscription) { if sub == nil { return } // Lookup the account based on sub.sid. if i := bytes.Index(sub.sid, []byte(" ")); i > 0 { // First part of SID for route is account name. if v, ok := c.srv.accounts.Load(bytesToString(sub.sid[:i])); ok { (v.(*Account)).sl.Remove(sub) } c.mu.Lock() delete(c.subs, bytesToString(sub.sid)) c.mu.Unlock() } } func (c *client) processAccountSub(arg []byte) error { if c.kind == GATEWAY { return c.processGatewayAccountSub(string(arg)) } return nil } func (c *client) processAccountUnsub(arg []byte) { if c.kind == GATEWAY { c.processGatewayAccountUnsub(string(arg)) } } // Process an inbound LMSG specification from the remote route. This means // we have an origin cluster and we force header semantics. func (c *client) processRoutedOriginClusterMsgArgs(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_HMSG_ARGS + 1][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } var an []byte if c.kind == ROUTER { if an = c.route.accName; len(an) > 0 && len(args) > 2 { args = append(args[:2], args[1:]...) args[1] = an } } c.pa.arg = arg switch len(args) { case 0, 1, 2, 3, 4: return fmt.Errorf("processRoutedOriginClusterMsgArgs Parse Error: '%s'", args) case 5: c.pa.reply = nil c.pa.queues = nil c.pa.hdb = args[3] c.pa.hdr = parseSize(args[3]) c.pa.szb = args[4] c.pa.size = parseSize(args[4]) case 6: c.pa.reply = args[3] c.pa.queues = nil c.pa.hdb = args[4] c.pa.hdr = parseSize(args[4]) c.pa.szb = args[5] c.pa.size = parseSize(args[5]) default: // args[2] is our reply indicator. Should be + or | normally. if len(args[3]) != 1 { return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'", args[3]) } switch args[3][0] { case '+': c.pa.reply = args[4] case '|': c.pa.reply = nil default: return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'", args[3]) } // Grab header size. c.pa.hdb = args[len(args)-2] c.pa.hdr = parseSize(c.pa.hdb) // Grab size. c.pa.szb = args[len(args)-1] c.pa.size = parseSize(c.pa.szb) // Grab queue names. if c.pa.reply != nil { c.pa.queues = args[5 : len(args)-2] } else { c.pa.queues = args[4 : len(args)-2] } } if c.pa.hdr < 0 { return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Header Size: '%s'", arg) } if c.pa.size < 0 { return fmt.Errorf("processRoutedOriginClusterMsgArgs Bad or Missing Size: '%s'", args) } if c.pa.hdr > c.pa.size { return fmt.Errorf("processRoutedOriginClusterMsgArgs Header Size larger then TotalSize: '%s'", arg) } // Common ones processed after check for arg length c.pa.origin = args[0] c.pa.account = args[1] c.pa.subject = args[2] if len(an) > 0 { c.pa.pacache = c.pa.subject } else { c.pa.pacache = arg[len(args[0])+1 : len(args[0])+len(args[1])+len(args[2])+2] } return nil } // Process an inbound HMSG specification from the remote route. func (c *client) processRoutedHeaderMsgArgs(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_HMSG_ARGS][]byte{} args := a[:0] var an []byte if c.kind == ROUTER { if an = c.route.accName; len(an) > 0 { args = append(args, an) } } start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 0, 1, 2, 3: return fmt.Errorf("processRoutedHeaderMsgArgs Parse Error: '%s'", args) case 4: c.pa.reply = nil c.pa.queues = nil c.pa.hdb = args[2] c.pa.hdr = parseSize(args[2]) c.pa.szb = args[3] c.pa.size = parseSize(args[3]) case 5: c.pa.reply = args[2] c.pa.queues = nil c.pa.hdb = args[3] c.pa.hdr = parseSize(args[3]) c.pa.szb = args[4] c.pa.size = parseSize(args[4]) default: // args[2] is our reply indicator. Should be + or | normally. if len(args[2]) != 1 { return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) } switch args[2][0] { case '+': c.pa.reply = args[3] case '|': c.pa.reply = nil default: return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) } // Grab header size. c.pa.hdb = args[len(args)-2] c.pa.hdr = parseSize(c.pa.hdb) // Grab size. c.pa.szb = args[len(args)-1] c.pa.size = parseSize(c.pa.szb) // Grab queue names. if c.pa.reply != nil { c.pa.queues = args[4 : len(args)-2] } else { c.pa.queues = args[3 : len(args)-2] } } if c.pa.hdr < 0 { return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Header Size: '%s'", arg) } if c.pa.size < 0 { return fmt.Errorf("processRoutedHeaderMsgArgs Bad or Missing Size: '%s'", args) } if c.pa.hdr > c.pa.size { return fmt.Errorf("processRoutedHeaderMsgArgs Header Size larger then TotalSize: '%s'", arg) } // Common ones processed after check for arg length c.pa.account = args[0] c.pa.subject = args[1] if len(an) > 0 { c.pa.pacache = c.pa.subject } else { c.pa.pacache = arg[:len(args[0])+len(args[1])+1] } return nil } // Process an inbound RMSG or LMSG specification from the remote route. func (c *client) processRoutedMsgArgs(arg []byte) error { // Unroll splitArgs to avoid runtime/heap issues a := [MAX_RMSG_ARGS][]byte{} args := a[:0] var an []byte if c.kind == ROUTER { if an = c.route.accName; len(an) > 0 { args = append(args, an) } } start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } c.pa.arg = arg switch len(args) { case 0, 1, 2: return fmt.Errorf("processRoutedMsgArgs Parse Error: '%s'", args) case 3: c.pa.reply = nil c.pa.queues = nil c.pa.szb = args[2] c.pa.size = parseSize(args[2]) case 4: c.pa.reply = args[2] c.pa.queues = nil c.pa.szb = args[3] c.pa.size = parseSize(args[3]) default: // args[2] is our reply indicator. Should be + or | normally. if len(args[2]) != 1 { return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) } switch args[2][0] { case '+': c.pa.reply = args[3] case '|': c.pa.reply = nil default: return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) } // Grab size. c.pa.szb = args[len(args)-1] c.pa.size = parseSize(c.pa.szb) // Grab queue names. if c.pa.reply != nil { c.pa.queues = args[4 : len(args)-1] } else { c.pa.queues = args[3 : len(args)-1] } } if c.pa.size < 0 { return fmt.Errorf("processRoutedMsgArgs Bad or Missing Size: '%s'", args) } // Common ones processed after check for arg length c.pa.account = args[0] c.pa.subject = args[1] if len(an) > 0 { c.pa.pacache = c.pa.subject } else { c.pa.pacache = arg[:len(args[0])+len(args[1])+1] } return nil } // processInboundRoutedMsg is called to process an inbound msg from a route. func (c *client) processInboundRoutedMsg(msg []byte) { // Update statistics c.in.msgs++ // The msg includes the CR_LF, so pull back out for accounting. c.in.bytes += int32(len(msg) - LEN_CR_LF) if c.opts.Verbose { c.sendOK() } // Mostly under testing scenarios. if c.srv == nil { return } // If the subject (c.pa.subject) has the gateway prefix, this function will handle it. if c.handleGatewayReply(msg) { // We are done here. return } acc, r := c.getAccAndResultFromCache() if acc == nil { c.Debugf("Unknown account %q for routed message on subject: %q", c.pa.account, c.pa.subject) return } // Check for no interest, short circuit if so. // This is the fanout scale. if len(r.psubs)+len(r.qsubs) > 0 { c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag) } } // Lock should be held entering here. func (c *client) sendRouteConnect(clusterName string, tlsRequired bool) error { var user, pass string if userInfo := c.route.url.User; userInfo != nil { user = userInfo.Username() pass, _ = userInfo.Password() } s := c.srv cinfo := connectInfo{ Echo: true, Verbose: false, Pedantic: false, User: user, Pass: pass, TLS: tlsRequired, Name: s.info.ID, Headers: s.supportsHeaders(), Cluster: clusterName, Dynamic: s.isClusterNameDynamic(), LNOC: true, } b, err := json.Marshal(cinfo) if err != nil { c.Errorf("Error marshaling CONNECT to route: %v\n", err) return err } c.enqueueProto([]byte(fmt.Sprintf(ConProto, b))) return nil } // Process the info message if we are a route. func (c *client) processRouteInfo(info *Info) { supportsHeaders := c.srv.supportsHeaders() clusterName := c.srv.ClusterName() srvName := c.srv.Name() c.mu.Lock() // Connection can be closed at any time (by auth timeout, etc). // Does not make sense to continue here if connection is gone. if c.route == nil || c.isClosed() { c.mu.Unlock() return } s := c.srv // Detect route to self. if info.ID == s.info.ID { // Need to set this so that the close does the right thing c.route.remoteID = info.ID c.mu.Unlock() c.closeConnection(DuplicateRoute) return } // Detect if we have a mismatch of cluster names. if info.Cluster != "" && info.Cluster != clusterName { c.mu.Unlock() // If we are dynamic we may update our cluster name. // Use other if remote is non dynamic or their name is "bigger" if s.isClusterNameDynamic() && (!info.Dynamic || (strings.Compare(clusterName, info.Cluster) < 0)) { s.setClusterName(info.Cluster) s.removeAllRoutesExcept(info.ID) c.mu.Lock() } else { c.closeConnection(ClusterNameConflict) return } } opts := s.getOpts() didSolicit := c.route.didSolicit // If this is an async INFO from an existing route... if c.flags.isSet(infoReceived) { remoteID := c.route.remoteID // Check if this is an INFO about adding a per-account route during // a configuration reload. if info.RouteAccReqID != _EMPTY_ { c.mu.Unlock() // If there is an account name, then the remote server is telling // us that this account will now have its dedicated route. if an := info.RouteAccount; an != _EMPTY_ { acc, err := s.LookupAccount(an) if err != nil { s.Errorf("Error looking up account %q: %v", an, err) return } s.mu.Lock() if _, ok := s.accRoutes[an]; !ok { s.accRoutes[an] = make(map[string]*client) } acc.mu.Lock() sl := acc.sl rpi := acc.routePoolIdx // Make sure that the account was not already switched. if rpi >= 0 { s.setRouteInfo(acc) // Change the route pool index to indicate that this // account is actually transitioning. This will be used // to suppress possible remote subscription interest coming // in while the transition is happening. acc.routePoolIdx = accTransitioningToDedicatedRoute } else if info.RoutePoolSize == s.routesPoolSize { // Otherwise, and if the other side's pool size matches // ours, get the route pool index that was handling this // account. rpi = s.computeRoutePoolIdx(acc) } acc.mu.Unlock() // Go over each remote's route at pool index `rpi` and remove // remote subs for this account. s.forEachRouteIdx(rpi, func(r *client) bool { r.mu.Lock() // Exclude routes to servers that don't support pooling. if !r.route.noPool { if subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 { sl.RemoveBatch(subs) } } r.mu.Unlock() return true }) // Respond to the remote by clearing the RouteAccount field. info.RouteAccount = _EMPTY_ proto := generateInfoJSON(info) c.mu.Lock() c.enqueueProto(proto) c.mu.Unlock() s.mu.Unlock() } else { // If no account name is specified, this is a response from the // remote. Simply send to the communication channel, if the // request ID matches the current one. s.mu.Lock() if info.RouteAccReqID == s.accAddedReqID && s.accAddedCh != nil { select { case s.accAddedCh <- struct{}{}: default: } } s.mu.Unlock() } // In both cases, we are done here. return } // Check if this is an INFO for gateways... if info.Gateway != _EMPTY_ { c.mu.Unlock() // If this server has no gateway configured, report error and return. if !s.gateway.enabled { // FIXME: Should this be a Fatalf()? s.Errorf("Received information about gateway %q from %s, but gateway is not configured", info.Gateway, remoteID) return } s.processGatewayInfoFromRoute(info, remoteID) return } // We receive an INFO from a server that informs us about another server, // so the info.ID in the INFO protocol does not match the ID of this route. if remoteID != _EMPTY_ && remoteID != info.ID { // We want to know if the existing route supports pooling/pinned-account // or not when processing the implicit route. noPool := c.route.noPool c.mu.Unlock() // Process this implicit route. We will check that it is not an explicit // route and/or that it has not been connected already. s.processImplicitRoute(info, noPool) return } var connectURLs []string var wsConnectURLs []string var updateRoutePerms bool // If we are notified that the remote is going into LDM mode, capture route's connectURLs. if info.LameDuckMode { connectURLs = c.route.connectURLs wsConnectURLs = c.route.wsConnURLs } else { // Update only if we detect a difference updateRoutePerms = !reflect.DeepEqual(c.opts.Import, info.Import) || !reflect.DeepEqual(c.opts.Export, info.Export) } c.mu.Unlock() if updateRoutePerms { s.updateRemoteRoutePerms(c, info) } // If the remote is going into LDM and there are client connect URLs // associated with this route and we are allowed to advertise, remove // those URLs and update our clients. if (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise { s.mu.Lock() s.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs) s.mu.Unlock() } return } // Check if remote has same server name than this server. if !didSolicit && info.Name == srvName { c.mu.Unlock() // This is now an error and we close the connection. We need unique names for JetStream clustering. c.Errorf("Remote server has a duplicate name: %q", info.Name) c.closeConnection(DuplicateServerName) return } // First INFO, check if this server is configured for compression because // if that is the case, we need to negotiate it with the remote server. if needsCompression(opts.Cluster.Compression.Mode) { accName := bytesToString(c.route.accName) // If we did not yet negotiate... if !c.flags.isSet(compressionNegotiated) { // Prevent from getting back here. c.flags.set(compressionNegotiated) // Release client lock since following function will need server lock. c.mu.Unlock() compress, err := s.negotiateRouteCompression(c, didSolicit, accName, info.Compression, opts) if err != nil { c.sendErrAndErr(err.Error()) c.closeConnection(ProtocolViolation) return } if compress { // Done for now, will get back another INFO protocol... return } // No compression because one side does not want/can't, so proceed. c.mu.Lock() } else if didSolicit { // The other side has switched to compression, so we can now set // the first ping timer and send the delayed INFO for situations // where it was not already sent. c.setFirstPingTimer() if !routeShouldDelayInfo(accName, opts) { cm := compressionModeForInfoProtocol(&opts.Cluster.Compression, c.route.compression) // Need to release and then reacquire... c.mu.Unlock() s.sendDelayedRouteInfo(c, accName, cm) c.mu.Lock() } } // Check that the connection did not close if the lock was released. if c.isClosed() { c.mu.Unlock() return } } else { // Coming from an old server, the Compression field would be the empty // string. For servers that are configured with CompressionNotSupported, // this makes them behave as old servers. if info.Compression == _EMPTY_ || opts.Cluster.Compression.Mode == CompressionNotSupported { c.route.compression = CompressionNotSupported } else { c.route.compression = CompressionOff } } // Mark that the INFO protocol has been received, so we can detect updates. c.flags.set(infoReceived) // Get the route's proto version c.opts.Protocol = info.Proto // Headers c.headers = supportsHeaders && info.Headers // Copy over important information. c.route.remoteID = info.ID c.route.authRequired = info.AuthRequired c.route.tlsRequired = info.TLSRequired c.route.gatewayURL = info.GatewayURL c.route.remoteName = info.Name c.route.lnoc = info.LNOC c.route.lnocu = info.LNOCU c.route.jetstream = info.JetStream // When sent through route INFO, if the field is set, it should be of size 1. if len(info.LeafNodeURLs) == 1 { c.route.leafnodeURL = info.LeafNodeURLs[0] } // Compute the hash of this route based on remote server name c.route.hash = getHash(info.Name) // Same with remote server ID (used for GW mapped replies routing). // Use getGWHash since we don't use the same hash len for that // for backward compatibility. c.route.idHash = string(getGWHash(info.ID)) // Copy over permissions as well. c.opts.Import = info.Import c.opts.Export = info.Export // If we do not know this route's URL, construct one on the fly // from the information provided. if c.route.url == nil { // Add in the URL from host and port hp := net.JoinHostPort(info.Host, strconv.Itoa(info.Port)) url, err := url.Parse(fmt.Sprintf("nats-route://%s/", hp)) if err != nil { c.Errorf("Error parsing URL from INFO: %v\n", err) c.mu.Unlock() c.closeConnection(ParseError) return } c.route.url = url } // The incoming INFO from the route will have IP set // if it has Cluster.Advertise. In that case, use that // otherwise construct it from the remote TCP address. if info.IP == _EMPTY_ { // Need to get the remote IP address. switch conn := c.nc.(type) { case *net.TCPConn, *tls.Conn: addr := conn.RemoteAddr().(*net.TCPAddr) info.IP = fmt.Sprintf("nats-route://%s/", net.JoinHostPort(addr.IP.String(), strconv.Itoa(info.Port))) default: info.IP = c.route.url.String() } } // For accounts that are configured to have their own route: // If this is a solicited route, we already have c.route.accName set in createRoute. // For non solicited route (the accept side), we will set the account name that // is present in the INFO protocol. if didSolicit && len(c.route.accName) > 0 { // Set it in the info.RouteAccount so that addRoute can use that // and we properly gossip that this is a route for an account. info.RouteAccount = string(c.route.accName) } else if !didSolicit && info.RouteAccount != _EMPTY_ { c.route.accName = []byte(info.RouteAccount) } accName := string(c.route.accName) // Check to see if we have this remote already registered. // This can happen when both servers have routes to each other. c.mu.Unlock() if added := s.addRoute(c, didSolicit, info, accName); added { if accName != _EMPTY_ { c.Debugf("Registering remote route %q for account %q", info.ID, accName) } else { c.Debugf("Registering remote route %q", info.ID) } } else { c.Debugf("Detected duplicate remote route %q", info.ID) c.closeConnection(DuplicateRoute) } } func (s *Server) negotiateRouteCompression(c *client, didSolicit bool, accName, infoCompression string, opts *Options) (bool, error) { // Negotiate the appropriate compression mode (or no compression) cm, err := selectCompressionMode(opts.Cluster.Compression.Mode, infoCompression) if err != nil { return false, err } c.mu.Lock() // For "auto" mode, set the initial compression mode based on RTT if cm == CompressionS2Auto { if c.rttStart.IsZero() { c.rtt = computeRTT(c.start) } cm = selectS2AutoModeBasedOnRTT(c.rtt, opts.Cluster.Compression.RTTThresholds) } // Keep track of the negotiated compression mode. c.route.compression = cm c.mu.Unlock() // If we end-up doing compression... if needsCompression(cm) { // Generate an INFO with the chosen compression mode. s.mu.Lock() infoProto := s.generateRouteInitialInfoJSON(accName, cm, 0) s.mu.Unlock() // If we solicited, then send this INFO protocol BEFORE switching // to compression writer. However, if we did not, we send it after. c.mu.Lock() if didSolicit { c.enqueueProto(infoProto) // Make sure it is completely flushed (the pending bytes goes to // 0) before proceeding. for c.out.pb > 0 && !c.isClosed() { c.flushOutbound() } } // This is to notify the readLoop that it should switch to a // (de)compression reader. c.in.flags.set(switchToCompression) // Create the compress writer before queueing the INFO protocol for // a route that did not solicit. It will make sure that that proto // is sent with compression on. c.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...) if !didSolicit { c.enqueueProto(infoProto) } // We can now set the ping timer. c.setFirstPingTimer() c.mu.Unlock() return true, nil } // We are not using compression, set the ping timer. c.mu.Lock() c.setFirstPingTimer() c.mu.Unlock() // If this is a solicited route, we need to send the INFO if it was not // done during createRoute() and will not be done in addRoute(). if didSolicit && !routeShouldDelayInfo(accName, opts) { cm = compressionModeForInfoProtocol(&opts.Cluster.Compression, cm) s.sendDelayedRouteInfo(c, accName, cm) } return false, nil } func (s *Server) sendDelayedRouteInfo(c *client, accName, cm string) { s.mu.Lock() infoProto := s.generateRouteInitialInfoJSON(accName, cm, 0) s.mu.Unlock() c.mu.Lock() c.enqueueProto(infoProto) c.mu.Unlock() } // Possibly sends local subscriptions interest to this route // based on changes in the remote's Export permissions. func (s *Server) updateRemoteRoutePerms(c *client, info *Info) { c.mu.Lock() // Interested only on Export permissions for the remote server. // Create "fake" clients that we will use to check permissions // using the old permissions... oldPerms := &RoutePermissions{Export: c.opts.Export} oldPermsTester := &client{} oldPermsTester.setRoutePermissions(oldPerms) // and the new ones. newPerms := &RoutePermissions{Export: info.Export} newPermsTester := &client{} newPermsTester.setRoutePermissions(newPerms) c.opts.Import = info.Import c.opts.Export = info.Export routeAcc, poolIdx, noPool := bytesToString(c.route.accName), c.route.poolIdx, c.route.noPool c.mu.Unlock() var ( _localSubs [4096]*subscription _allSubs [4096]*subscription allSubs = _allSubs[:0] ) s.accounts.Range(func(_, v any) bool { acc := v.(*Account) acc.mu.RLock() accName, sl, accPoolIdx := acc.Name, acc.sl, acc.routePoolIdx acc.mu.RUnlock() // Do this only for accounts handled by this route if (accPoolIdx >= 0 && accPoolIdx == poolIdx) || (routeAcc == accName) || noPool { localSubs := _localSubs[:0] sl.localSubs(&localSubs, false) if len(localSubs) > 0 { allSubs = append(allSubs, localSubs...) } } return true }) if len(allSubs) == 0 { return } c.mu.Lock() c.sendRouteSubProtos(allSubs, false, func(sub *subscription) bool { subj := string(sub.subject) // If the remote can now export but could not before, and this server can import this // subject, then send SUB protocol. if newPermsTester.canExport(subj) && !oldPermsTester.canExport(subj) && c.canImport(subj) { return true } return false }) c.mu.Unlock() } // sendAsyncInfoToClients sends an INFO protocol to all // connected clients that accept async INFO updates. // The server lock is held on entry. func (s *Server) sendAsyncInfoToClients(regCli, wsCli bool) { // If there are no clients supporting async INFO protocols, we are done. // Also don't send if we are shutting down... if s.cproto == 0 || s.isShuttingDown() { return } info := s.copyInfo() for _, c := range s.clients { c.mu.Lock() // Here, we are going to send only to the clients that are fully // registered (server has received CONNECT and first PING). For // clients that are not at this stage, this will happen in the // processing of the first PING (see client.processPing) if ((regCli && !c.isWebsocket()) || (wsCli && c.isWebsocket())) && c.opts.Protocol >= ClientProtoInfo && c.flags.isSet(firstPongSent) { // sendInfo takes care of checking if the connection is still // valid or not, so don't duplicate tests here. c.enqueueProto(c.generateClientInfoJSON(info)) } c.mu.Unlock() } } // This will process implicit route information received from another server. // We will check to see if we have configured or are already connected, // and if so we will ignore. Otherwise we will attempt to connect. func (s *Server) processImplicitRoute(info *Info, routeNoPool bool) { remoteID := info.ID s.mu.Lock() defer s.mu.Unlock() // Don't connect to ourself if remoteID == s.info.ID { return } // Snapshot server options. opts := s.getOpts() // Check if this route already exists if accName := info.RouteAccount; accName != _EMPTY_ { // If we don't support pooling/pinned account, bail. if opts.Cluster.PoolSize <= 0 { return } if remotes, ok := s.accRoutes[accName]; ok { if r := remotes[remoteID]; r != nil { return } } } else if _, exists := s.routes[remoteID]; exists { return } // Check if we have this route as a configured route if s.hasThisRouteConfigured(info) { return } // Initiate the connection, using info.IP instead of info.URL here... r, err := url.Parse(info.IP) if err != nil { s.Errorf("Error parsing URL from INFO: %v\n", err) return } if info.AuthRequired { r.User = url.UserPassword(opts.Cluster.Username, opts.Cluster.Password) } s.startGoRoutine(func() { s.connectToRoute(r, false, true, info.RouteAccount) }) // If we are processing an implicit route from a route that does not // support pooling/pinned-accounts, we won't receive an INFO for each of // the pinned-accounts that we would normally receive. In that case, just // initiate routes for all our configured pinned accounts. if routeNoPool && info.RouteAccount == _EMPTY_ && len(opts.Cluster.PinnedAccounts) > 0 { // Copy since we are going to pass as closure to a go routine. rURL := r for _, an := range opts.Cluster.PinnedAccounts { accName := an s.startGoRoutine(func() { s.connectToRoute(rURL, false, true, accName) }) } } } // hasThisRouteConfigured returns true if info.Host:info.Port is present // in the server's opts.Routes, false otherwise. // Server lock is assumed to be held by caller. func (s *Server) hasThisRouteConfigured(info *Info) bool { routes := s.getOpts().Routes if len(routes) == 0 { return false } // This could possibly be a 0.0.0.0 host so we will also construct a second // url with the host section of the `info.IP` (if present). sPort := strconv.Itoa(info.Port) urlOne := strings.ToLower(net.JoinHostPort(info.Host, sPort)) var urlTwo string if info.IP != _EMPTY_ { if u, _ := url.Parse(info.IP); u != nil { urlTwo = strings.ToLower(net.JoinHostPort(u.Hostname(), sPort)) // Ignore if same than the first if urlTwo == urlOne { urlTwo = _EMPTY_ } } } for _, ri := range routes { rHost := strings.ToLower(ri.Host) if rHost == urlOne { return true } if urlTwo != _EMPTY_ && rHost == urlTwo { return true } } return false } // forwardNewRouteInfoToKnownServers sends the INFO protocol of the new route // to all routes known by this server. In turn, each server will contact this // new route. // Server lock held on entry. func (s *Server) forwardNewRouteInfoToKnownServers(info *Info) { // Note: nonce is not used in routes. // That being said, the info we get is the initial INFO which // contains a nonce, but we now forward this to existing routes, // so clear it now. info.Nonce = _EMPTY_ b, _ := json.Marshal(info) infoJSON := []byte(fmt.Sprintf(InfoProto, b)) s.forEachRemote(func(r *client) { r.mu.Lock() // If this is a new route for a given account, do not send to a server // that does not support pooling/pinned-accounts. if r.route.remoteID != info.ID && (info.RouteAccount == _EMPTY_ || (info.RouteAccount != _EMPTY_ && !r.route.noPool)) { r.enqueueProto(infoJSON) } r.mu.Unlock() }) } // canImport is whether or not we will send a SUB for interest to the other side. // This is for ROUTER connections only. // Lock is held on entry. func (c *client) canImport(subject string) bool { // Use pubAllowed() since this checks Publish permissions which // is what Import maps to. return c.pubAllowedFullCheck(subject, false, true) } // canExport is whether or not we will accept a SUB from the remote for a given subject. // This is for ROUTER connections only. // Lock is held on entry func (c *client) canExport(subject string) bool { // Use canSubscribe() since this checks Subscribe permissions which // is what Export maps to. return c.canSubscribe(subject) } // Initialize or reset cluster's permissions. // This is for ROUTER connections only. // Client lock is held on entry func (c *client) setRoutePermissions(perms *RoutePermissions) { // Reset if some were set if perms == nil { c.perms = nil c.mperms = nil return } // Convert route permissions to user permissions. // The Import permission is mapped to Publish // and Export permission is mapped to Subscribe. // For meaning of Import/Export, see canImport and canExport. p := &Permissions{ Publish: perms.Import, Subscribe: perms.Export, } c.setPermissions(p) } // Type used to hold a list of subs on a per account basis. type asubs struct { acc *Account subs []*subscription } // Returns the account name from the subscription's key. // This is invoked knowing that the key contains an account name, so for a sub // that is not from a pinned-account route. // The `keyHasSubType` boolean indicates that the key starts with the indicator // for leaf or regular routed subscriptions. func getAccNameFromRoutedSubKey(sub *subscription, key string, keyHasSubType bool) string { var accIdx int if keyHasSubType { // Start after the sub type indicator. accIdx = 1 // But if there is an origin, bump its index. if len(sub.origin) > 0 { accIdx = 2 } } return strings.Fields(key)[accIdx] } // Returns if the route is dedicated to an account, its name, and a boolean // that indicates if this route uses the routed subscription indicator at // the beginning of the subscription key. // Lock held on entry. func (c *client) getRoutedSubKeyInfo() (bool, string, bool) { var accName string if an := c.route.accName; len(an) > 0 { accName = string(an) } return accName != _EMPTY_, accName, c.route.lnocu } // removeRemoteSubs will walk the subs and remove them from the appropriate account. func (c *client) removeRemoteSubs() { // We need to gather these on a per account basis. // FIXME(dlc) - We should be smarter about this.. as := map[string]*asubs{} c.mu.Lock() srv := c.srv subs := c.subs c.subs = nil pa, accountName, hasSubType := c.getRoutedSubKeyInfo() c.mu.Unlock() for key, sub := range subs { c.mu.Lock() sub.max = 0 c.mu.Unlock() // If not a pinned-account route, we need to find the account // name from the sub's key. if !pa { accountName = getAccNameFromRoutedSubKey(sub, key, hasSubType) } ase := as[accountName] if ase == nil { if v, ok := srv.accounts.Load(accountName); ok { ase = &asubs{acc: v.(*Account), subs: []*subscription{sub}} as[accountName] = ase } else { continue } } else { ase.subs = append(ase.subs, sub) } delta := int32(1) if len(sub.queue) > 0 { delta = sub.qw } if srv.gateway.enabled { srv.gatewayUpdateSubInterest(accountName, sub, -delta) } ase.acc.updateLeafNodes(sub, -delta) } // Now remove the subs by batch for each account sublist. for _, ase := range as { c.Debugf("Removing %d subscriptions for account %q", len(ase.subs), ase.acc.Name) ase.acc.mu.Lock() ase.acc.sl.RemoveBatch(ase.subs) ase.acc.mu.Unlock() } } // Removes (and returns) the subscriptions from this route's subscriptions map // that belong to the given account. // Lock is held on entry func (c *client) removeRemoteSubsForAcc(name string) []*subscription { var subs []*subscription _, _, hasSubType := c.getRoutedSubKeyInfo() for key, sub := range c.subs { an := getAccNameFromRoutedSubKey(sub, key, hasSubType) if an == name { sub.max = 0 subs = append(subs, sub) delete(c.subs, key) } } return subs } func (c *client) parseUnsubProto(arg []byte, accInProto, hasOrigin bool) ([]byte, string, []byte, []byte, error) { // Indicate any activity, so pub and sub or unsubs. c.in.subs++ args := splitArg(arg) var ( origin []byte accountName string queue []byte subjIdx int ) // If `hasOrigin` is true, then it means this is a LS- with origin in proto. if hasOrigin { // We would not be here if there was not at least 1 field. origin = args[0] subjIdx = 1 } // If there is an account in the protocol, bump the subject index. if accInProto { subjIdx++ } switch len(args) { case subjIdx + 1: case subjIdx + 2: queue = args[subjIdx+1] default: return nil, _EMPTY_, nil, nil, fmt.Errorf("parse error: '%s'", arg) } if accInProto { // If there is an account in the protocol, it is before the subject. accountName = string(args[subjIdx-1]) } return origin, accountName, args[subjIdx], queue, nil } // Indicates no more interest in the given account/subject for the remote side. func (c *client) processRemoteUnsub(arg []byte, leafUnsub bool) (err error) { srv := c.srv if srv == nil { return nil } var accountName string // Assume the account will be in the protocol. accInProto := true c.mu.Lock() originSupport := c.route.lnocu if c.route != nil && len(c.route.accName) > 0 { accountName, accInProto = string(c.route.accName), false } c.mu.Unlock() hasOrigin := leafUnsub && originSupport _, accNameFromProto, subject, _, err := c.parseUnsubProto(arg, accInProto, hasOrigin) if err != nil { return fmt.Errorf("processRemoteUnsub %s", err.Error()) } if accInProto { accountName = accNameFromProto } // Lookup the account var acc *Account if v, ok := srv.accounts.Load(accountName); ok { acc = v.(*Account) } else { c.Debugf("Unknown account %q for subject %q", accountName, subject) return nil } c.mu.Lock() if c.isClosed() { c.mu.Unlock() return nil } _keya := [128]byte{} _key := _keya[:0] var key string if !originSupport { // If it is an LS- or RS-, we use the protocol as-is as the key. key = bytesToString(arg) } else { // We need to prefix with the sub type. if leafUnsub { _key = append(_key, keyRoutedLeafSubByte) } else { _key = append(_key, keyRoutedSubByte) } _key = append(_key, ' ') _key = append(_key, arg...) key = bytesToString(_key) } delta := int32(1) sub, ok := c.subs[key] if ok { delete(c.subs, key) acc.sl.Remove(sub) if len(sub.queue) > 0 { delta = sub.qw } } c.mu.Unlock() // Update gateways and leaf nodes only if the subscription was found. if ok { if srv.gateway.enabled { srv.gatewayUpdateSubInterest(accountName, sub, -delta) } // Now check on leafnode updates. acc.updateLeafNodes(sub, -delta) } if c.opts.Verbose { c.sendOK() } return nil } func (c *client) processRemoteSub(argo []byte, hasOrigin bool) (err error) { // Indicate activity. c.in.subs++ srv := c.srv if srv == nil { return nil } // We copy `argo` to not reference the read buffer. However, we will // prefix with a code that says if the remote sub is for a leaf // (hasOrigin == true) or not to prevent key collisions. Imagine: // "RS+ foo bar baz 1\r\n" => "foo bar baz" (a routed queue sub) // "LS+ foo bar baz\r\n" => "foo bar baz" (a route leaf sub on "baz", // for account "bar" with origin "foo"). // // The sub.sid/key will be set respectively to "R foo bar baz" and // "L foo bar baz". // // We also no longer add the account if it was not present (due to // pinned-account route) since there is no need really. // // For routes to older server, we will still create the "arg" with // the above layout, but we will create the sub.sid/key as before, // that is, not including the origin for LS+ because older server // only send LS- without origin, so we would not be able to find // the sub in the map. c.mu.Lock() accountName := string(c.route.accName) oldStyle := !c.route.lnocu c.mu.Unlock() // Indicate if the account name should be in the protocol. It would be the // case if accountName is empty. accInProto := accountName == _EMPTY_ // Copy so we do not reference a potentially large buffer. // Add 2 more bytes for the routed sub type. arg := make([]byte, 0, 2+len(argo)) if hasOrigin { arg = append(arg, keyRoutedLeafSubByte) } else { arg = append(arg, keyRoutedSubByte) } arg = append(arg, ' ') arg = append(arg, argo...) // Now split to get all fields. Unroll splitArgs to avoid runtime/heap issues. a := [MAX_RSUB_ARGS][]byte{} args := a[:0] start := -1 for i, b := range arg { switch b { case ' ', '\t', '\r', '\n': if start >= 0 { args = append(args, arg[start:i]) start = -1 } default: if start < 0 { start = i } } } if start >= 0 { args = append(args, arg[start:]) } delta := int32(1) sub := &subscription{client: c} // There will always be at least a subject, but its location will depend // on if there is an origin, an account name, etc.. Since we know that // we have added the sub type indicator as the first field, the subject // position will be at minimum at index 1. subjIdx := 1 if hasOrigin { subjIdx++ } if accInProto { subjIdx++ } switch len(args) { case subjIdx + 1: sub.queue = nil case subjIdx + 3: sub.queue = args[subjIdx+1] sub.qw = int32(parseSize(args[subjIdx+2])) // TODO: (ik) We should have a non empty queue name and a queue // weight >= 1. For 2.11, we may want to return an error if that // is not the case, but for now just overwrite `delta` if queue // weight is greater than 1 (it is possible after a reconnect/ // server restart to receive a queue weight > 1 for a new sub). if sub.qw > 1 { delta = sub.qw } default: return fmt.Errorf("processRemoteSub Parse Error: '%s'", arg) } // We know that the number of fields is correct. So we can access args[] based // on where we expect the fields to be. // If there is an origin, it will be at index 1. if hasOrigin { sub.origin = args[1] } // For subject, use subjIdx. sub.subject = args[subjIdx] // If the account name is in the protocol, it will be before the subject. if accInProto { accountName = bytesToString(args[subjIdx-1]) } // Now set the sub.sid from the arg slice. However, we will have a different // one if we use the origin or not. start = 0 end := len(arg) if sub.queue != nil { // Remove the ' ' from the arg length. end -= 1 + len(args[subjIdx+2]) } if oldStyle { // We will start at the account (if present) or at the subject. // We first skip the "R " or "L " start = 2 // And if there is an origin skip that. if hasOrigin { start += len(sub.origin) + 1 } // Here we are pointing at the account (if present), or at the subject. } sub.sid = arg[start:end] // Lookup account while avoiding fetch. // A slow fetch delays subsequent remote messages. It also avoids the expired check (see below). // With all but memory resolver lookup can be delayed or fail. // It is also possible that the account can't be resolved yet. // This does not apply to the memory resolver. // When used, perform the fetch. staticResolver := true if res := srv.AccountResolver(); res != nil { if _, ok := res.(*MemAccResolver); !ok { staticResolver = false } } var acc *Account if staticResolver { acc, _ = srv.LookupAccount(accountName) } else if v, ok := srv.accounts.Load(accountName); ok { acc = v.(*Account) } if acc == nil { // if the option of retrieving accounts later exists, create an expired one. // When a client comes along, expiration will prevent it from being used, // cause a fetch and update the account to what is should be. if staticResolver { c.Errorf("Unknown account %q for remote subject %q", accountName, sub.subject) return } c.Debugf("Unknown account %q for remote subject %q", accountName, sub.subject) var isNew bool if acc, isNew = srv.LookupOrRegisterAccount(accountName); isNew { acc.mu.Lock() acc.expired.Store(true) acc.incomplete = true acc.mu.Unlock() } } c.mu.Lock() if c.isClosed() { c.mu.Unlock() return nil } // Check permissions if applicable. if c.perms != nil && !c.canExport(string(sub.subject)) { c.mu.Unlock() c.Debugf("Can not export %q, ignoring remote subscription request", sub.subject) return nil } // Check if we have a maximum on the number of subscriptions. if c.subsAtLimit() { c.mu.Unlock() c.maxSubsExceeded() return nil } acc.mu.RLock() // For routes (this can be called by leafnodes), check if the account is // transitioning (from pool to dedicated route) and this route is not a // per-account route (route.poolIdx >= 0). If so, ignore this subscription. // Exclude "no pool" routes from this check. if c.kind == ROUTER && !c.route.noPool && acc.routePoolIdx == accTransitioningToDedicatedRoute && c.route.poolIdx >= 0 { acc.mu.RUnlock() c.mu.Unlock() // Do not return an error, which would cause the connection to be closed. return nil } sl := acc.sl acc.mu.RUnlock() // We use the sub.sid for the key of the c.subs map. key := bytesToString(sub.sid) osub := c.subs[key] if osub == nil { c.subs[key] = sub // Now place into the account sl. if err = sl.Insert(sub); err != nil { delete(c.subs, key) c.mu.Unlock() c.Errorf("Could not insert subscription: %v", err) c.sendErr("Invalid Subscription") return nil } } else if sub.queue != nil { // For a queue we need to update the weight. delta = sub.qw - atomic.LoadInt32(&osub.qw) atomic.StoreInt32(&osub.qw, sub.qw) sl.UpdateRemoteQSub(osub) } c.mu.Unlock() if srv.gateway.enabled { srv.gatewayUpdateSubInterest(acc.Name, sub, delta) } // Now check on leafnode updates. acc.updateLeafNodes(sub, delta) if c.opts.Verbose { c.sendOK() } return nil } // Lock is held on entry func (c *client) addRouteSubOrUnsubProtoToBuf(buf []byte, accName string, sub *subscription, isSubProto bool) []byte { // If we have an origin cluster and the other side supports leafnode origin clusters // send an LS+/LS- version instead. if len(sub.origin) > 0 && c.route.lnoc { if isSubProto { buf = append(buf, lSubBytes...) buf = append(buf, sub.origin...) buf = append(buf, ' ') } else { buf = append(buf, lUnsubBytes...) if c.route.lnocu { buf = append(buf, sub.origin...) buf = append(buf, ' ') } } } else { if isSubProto { buf = append(buf, rSubBytes...) } else { buf = append(buf, rUnsubBytes...) } } if len(c.route.accName) == 0 { buf = append(buf, accName...) buf = append(buf, ' ') } buf = append(buf, sub.subject...) if len(sub.queue) > 0 { buf = append(buf, ' ') buf = append(buf, sub.queue...) // Send our weight if we are a sub proto if isSubProto { buf = append(buf, ' ') var b [12]byte var i = len(b) for l := sub.qw; l > 0; l /= 10 { i-- b[i] = digits[l%10] } buf = append(buf, b[i:]...) } } buf = append(buf, CR_LF...) return buf } // sendSubsToRoute will send over our subject interest to // the remote side. For each account we will send the // complete interest for all subjects, both normal as a binary // and queue group weights. // // Server lock held on entry. func (s *Server) sendSubsToRoute(route *client, idx int, account string) { var noPool bool if idx >= 0 { // We need to check if this route is "no_pool" in which case we // need to select all accounts. route.mu.Lock() noPool = route.route.noPool route.mu.Unlock() } // Estimated size of all protocols. It does not have to be accurate at all. var eSize int estimateProtosSize := func(a *Account, addAccountName bool) { if ns := len(a.rm); ns > 0 { var accSize int if addAccountName { accSize = len(a.Name) + 1 } // Proto looks like: "RS+ [ ][ ]\r\n" eSize += ns * (len(rSubBytes) + 1 + accSize) for key := range a.rm { // Key contains "[ ]" eSize += len(key) // In case this is a queue, just add some bytes for the queue weight. // If we want to be accurate, would have to check if "key" has a space, // if so, then figure out how many bytes we need to represent the weight. eSize += 5 } } } // Send over our account subscriptions. accs := make([]*Account, 0, 1024) if idx < 0 || account != _EMPTY_ { if ai, ok := s.accounts.Load(account); ok { a := ai.(*Account) a.mu.RLock() // Estimate size and add account name in protocol if idx is not -1 estimateProtosSize(a, idx >= 0) accs = append(accs, a) a.mu.RUnlock() } } else { s.accounts.Range(func(k, v any) bool { a := v.(*Account) a.mu.RLock() // We are here for regular or pooled routes (not per-account). // So we collect all accounts whose routePoolIdx matches the // one for this route, or only the account provided, or all // accounts if dealing with a "no pool" route. if a.routePoolIdx == idx || noPool { estimateProtosSize(a, true) accs = append(accs, a) } a.mu.RUnlock() return true }) } buf := make([]byte, 0, eSize) route.mu.Lock() for _, a := range accs { a.mu.RLock() for key, n := range a.rm { var origin, qn []byte s := strings.Fields(key) // Subject will always be the second field (index 1). subj := stringToBytes(s[1]) // Check if the key is for a leaf (will be field 0). forLeaf := s[0] == keyRoutedLeafSub // For queue, if not for a leaf, we need 3 fields "R foo bar", // but if for a leaf, we need 4 fields "L foo bar leaf_origin". if l := len(s); (!forLeaf && l == 3) || (forLeaf && l == 4) { qn = stringToBytes(s[2]) } if forLeaf { // The leaf origin will be the last field. origin = stringToBytes(s[len(s)-1]) } // s[1] is the subject and already as a string, so use that // instead of converting back `subj` to a string. if !route.canImport(s[1]) { continue } sub := subscription{origin: origin, subject: subj, queue: qn, qw: n} buf = route.addRouteSubOrUnsubProtoToBuf(buf, a.Name, &sub, true) } a.mu.RUnlock() } if len(buf) > 0 { route.enqueueProto(buf) route.Debugf("Sent local subscriptions to route") } route.mu.Unlock() } // Sends SUBs protocols for the given subscriptions. If a filter is specified, it is // invoked for each subscription. If the filter returns false, the subscription is skipped. // This function may release the route's lock due to flushing of outbound data. A boolean // is returned to indicate if the connection has been closed during this call. // Lock is held on entry. func (c *client) sendRouteSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) { c.sendRouteSubOrUnSubProtos(subs, true, trace, filter) } // Sends UNSUBs protocols for the given subscriptions. If a filter is specified, it is // invoked for each subscription. If the filter returns false, the subscription is skipped. // This function may release the route's lock due to flushing of outbound data. A boolean // is returned to indicate if the connection has been closed during this call. // Lock is held on entry. func (c *client) sendRouteUnSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) { c.sendRouteSubOrUnSubProtos(subs, false, trace, filter) } // Low-level function that sends RS+ or RS- protocols for the given subscriptions. // This can now also send LS+ and LS- for origin cluster based leafnode subscriptions for cluster no-echo. // Use sendRouteSubProtos or sendRouteUnSubProtos instead for clarity. // Lock is held on entry. func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto, trace bool, filter func(sub *subscription) bool) { var ( _buf [1024]byte buf = _buf[:0] ) for _, sub := range subs { if filter != nil && !filter(sub) { continue } // Determine the account. If sub has an ImportMap entry, use that, otherwise scoped to // client. Default to global if all else fails. var accName string if sub.client != nil && sub.client != c { sub.client.mu.Lock() } if sub.im != nil { accName = sub.im.acc.Name } else if sub.client != nil && sub.client.acc != nil { accName = sub.client.acc.Name } else { c.Debugf("Falling back to default account for sending subs") accName = globalAccountName } if sub.client != nil && sub.client != c { sub.client.mu.Unlock() } as := len(buf) buf = c.addRouteSubOrUnsubProtoToBuf(buf, accName, sub, isSubProto) if trace { c.traceOutOp("", buf[as:len(buf)-LEN_CR_LF]) } } c.enqueueProto(buf) } func (s *Server) createRoute(conn net.Conn, rURL *url.URL, accName string) *client { // Snapshot server options. opts := s.getOpts() didSolicit := rURL != nil r := &route{didSolicit: didSolicit, poolIdx: -1} for _, route := range opts.Routes { if rURL != nil && (strings.EqualFold(rURL.Host, route.Host)) { r.routeType = Explicit } } c := &client{srv: s, nc: conn, opts: ClientOpts{}, kind: ROUTER, msubs: -1, mpay: -1, route: r, start: time.Now()} // Is the server configured for compression? compressionConfigured := needsCompression(opts.Cluster.Compression.Mode) var infoJSON []byte // Grab server variables and generates route INFO Json. Note that we set // and reset some of s.routeInfo fields when that happens, so we need // the server write lock. s.mu.Lock() // If we are creating a pooled connection and this is the server soliciting // the connection, we will delay sending the INFO after we have processed // the incoming INFO from the remote. Also delay if configured for compression. delayInfo := didSolicit && (compressionConfigured || routeShouldDelayInfo(accName, opts)) if !delayInfo { infoJSON = s.generateRouteInitialInfoJSON(accName, opts.Cluster.Compression.Mode, 0) } authRequired := s.routeInfo.AuthRequired tlsRequired := s.routeInfo.TLSRequired clusterName := s.info.Cluster tlsName := s.routeTLSName s.mu.Unlock() // Grab lock c.mu.Lock() // Initialize c.initClient() if didSolicit { // Do this before the TLS code, otherwise, in case of failure // and if route is explicit, it would try to reconnect to 'nil'... r.url = rURL r.accName = []byte(accName) } else { c.flags.set(expectConnect) } // Check for TLS if tlsRequired { tlsConfig := opts.Cluster.TLSConfig if didSolicit { // Copy off the config to add in ServerName if we need to. tlsConfig = tlsConfig.Clone() } // Perform (server or client side) TLS handshake. if resetTLSName, err := c.doTLSHandshake("route", didSolicit, rURL, tlsConfig, tlsName, opts.Cluster.TLSTimeout, opts.Cluster.TLSPinnedCerts); err != nil { c.mu.Unlock() if resetTLSName { s.mu.Lock() s.routeTLSName = _EMPTY_ s.mu.Unlock() } return nil } } // Do final client initialization // Initialize the per-account cache. c.in.pacache = make(map[string]*perAccountCache) if didSolicit { // Set permissions associated with the route user (if applicable). // No lock needed since we are already under client lock. c.setRoutePermissions(opts.Cluster.Permissions) } // We can't safely send the pings until we have negotiated compression // with the remote, but we want to protect against a connection that // does not perform the handshake. We will start a timer that will close // the connection as stale based on the ping interval and max out values, // but without actually sending pings. if compressionConfigured { pingInterval := opts.PingInterval pingMax := opts.MaxPingsOut if opts.Cluster.PingInterval > 0 { pingInterval = opts.Cluster.PingInterval } if opts.Cluster.MaxPingsOut > 0 { pingMax = opts.MaxPingsOut } c.watchForStaleConnection(adjustPingInterval(ROUTER, pingInterval), pingMax) } else { // Set the Ping timer c.setFirstPingTimer() } // For routes, the "client" is added to s.routes only when processing // the INFO protocol, that is much later. // In the meantime, if the server shutsdown, there would be no reference // to the client (connection) to be closed, leaving this readLoop // uinterrupted, causing the Shutdown() to wait indefinitively. // We need to store the client in a special map, under a special lock. if !s.addToTempClients(c.cid, c) { c.mu.Unlock() c.setNoReconnect() c.closeConnection(ServerShutdown) return nil } // Check for Auth required state for incoming connections. // Make sure to do this before spinning up readLoop. if authRequired && !didSolicit { ttl := secondsToDuration(opts.Cluster.AuthTimeout) c.setAuthTimer(ttl) } // Spin up the read loop. s.startGoRoutine(func() { c.readLoop(nil) }) // Spin up the write loop. s.startGoRoutine(func() { c.writeLoop() }) if tlsRequired { c.Debugf("TLS handshake complete") cs := c.nc.(*tls.Conn).ConnectionState() c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tlsCipher(cs.CipherSuite)) } // Queue Connect proto if we solicited the connection. if didSolicit { c.Debugf("Route connect msg sent") if err := c.sendRouteConnect(clusterName, tlsRequired); err != nil { c.mu.Unlock() c.closeConnection(ProtocolViolation) return nil } } if !delayInfo { // Send our info to the other side. // Our new version requires dynamic information for accounts and a nonce. c.enqueueProto(infoJSON) } c.mu.Unlock() c.Noticef("Route connection created") return c } func routeShouldDelayInfo(accName string, opts *Options) bool { return accName == _EMPTY_ && opts.Cluster.PoolSize >= 1 } // Generates a nonce and set some route info's fields before marshal'ing into JSON. // To be used only when a route is created (to send the initial INFO protocol). // // Server lock held on entry. func (s *Server) generateRouteInitialInfoJSON(accName, compression string, poolIdx int) []byte { // New proto wants a nonce (although not used in routes, that is, not signed in CONNECT) var raw [nonceLen]byte nonce := raw[:] s.generateNonce(nonce) ri := &s.routeInfo // Override compression with s2_auto instead of actual compression level. if s.getOpts().Cluster.Compression.Mode == CompressionS2Auto { compression = CompressionS2Auto } ri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression = string(nonce), accName, poolIdx, compression infoJSON := generateInfoJSON(&s.routeInfo) // Clear now that it has been serialized. Will prevent nonce to be included in async INFO that we may send. // Same for some other fields. ri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression = _EMPTY_, _EMPTY_, 0, _EMPTY_ return infoJSON } const ( _CRLF_ = "\r\n" _EMPTY_ = "" ) func (s *Server) addRoute(c *client, didSolicit bool, info *Info, accName string) bool { id := info.ID var acc *Account if accName != _EMPTY_ { var err error acc, err = s.LookupAccount(accName) if err != nil { c.sendErrAndErr(fmt.Sprintf("Unable to lookup account %q: %v", accName, err)) c.closeConnection(MissingAccount) return false } } s.mu.Lock() if !s.isRunning() || s.routesReject { s.mu.Unlock() return false } var invProtoErr string opts := s.getOpts() // Assume we are in pool mode if info.RoutePoolSize is set. We may disable // in some cases. pool := info.RoutePoolSize > 0 // This is used to prevent a server with pooling to constantly trying // to connect to a server with no pooling (for instance old server) after // the first connection is established. var noReconnectForOldServer bool // If the remote is an old server, info.RoutePoolSize will be 0, or if // this server's Cluster.PoolSize is negative, we will behave as an old // server and need to handle things differently. if info.RoutePoolSize <= 0 || opts.Cluster.PoolSize < 0 { if accName != _EMPTY_ { invProtoErr = fmt.Sprintf("Not possible to have a dedicated route for account %q between those servers", accName) // In this case, make sure this route does not attempt to reconnect c.setNoReconnect() } else { // We will accept, but treat this remote has "no pool" pool, noReconnectForOldServer = false, true c.mu.Lock() c.route.poolIdx = 0 c.route.noPool = true c.mu.Unlock() // Keep track of number of routes like that. We will use that when // sending subscriptions over routes. s.routesNoPool++ } } else if s.routesPoolSize != info.RoutePoolSize { // The cluster's PoolSize configuration must be an exact match with the remote server. invProtoErr = fmt.Sprintf("Mismatch route pool size: %v vs %v", s.routesPoolSize, info.RoutePoolSize) } else if didSolicit { // For solicited route, the incoming's RoutePoolIdx should not be set. if info.RoutePoolIdx != 0 { invProtoErr = fmt.Sprintf("Route pool index should not be set but is set to %v", info.RoutePoolIdx) } } else if info.RoutePoolIdx < 0 || info.RoutePoolIdx >= s.routesPoolSize { // For non solicited routes, if the remote sends a RoutePoolIdx, make // sure it is a valid one (in range of the pool size). invProtoErr = fmt.Sprintf("Invalid route pool index: %v - pool size is %v", info.RoutePoolIdx, info.RoutePoolSize) } if invProtoErr != _EMPTY_ { s.mu.Unlock() c.sendErrAndErr(invProtoErr) c.closeConnection(ProtocolViolation) return false } // If accName is set, we are dealing with a per-account connection. if accName != _EMPTY_ { // When an account has its own route, it will be an error if the given // account name is not found in s.accRoutes map. conns, exists := s.accRoutes[accName] if !exists { s.mu.Unlock() c.sendErrAndErr(fmt.Sprintf("No route for account %q", accName)) c.closeConnection(ProtocolViolation) return false } remote, exists := conns[id] if !exists { conns[id] = c c.mu.Lock() idHash := c.route.idHash cid := c.cid if c.last.IsZero() { c.last = time.Now() } if acc != nil { c.acc = acc } c.mu.Unlock() // Store this route with key being the route id hash + account name s.storeRouteByHash(idHash+accName, c) // Now that we have registered the route, we can remove from the temp map. s.removeFromTempClients(cid) // Notify other routes about this new route s.forwardNewRouteInfoToKnownServers(info) // Send subscription interest s.sendSubsToRoute(c, -1, accName) } else { handleDuplicateRoute(remote, c, true) } s.mu.Unlock() return !exists } var remote *client // That will be the position of the connection in the slice, we initialize // to -1 to indicate that no space was found. idx := -1 // This will be the size (or number of connections) in a given slice. sz := 0 // Check if we know about the remote server conns, exists := s.routes[id] if !exists { // No, create a slice for route connections of the size of the pool // or 1 when not in pool mode. conns = make([]*client, s.routesPoolSize) // Track this slice for this remote server. s.routes[id] = conns // Set the index to info.RoutePoolIdx because if this is a solicited // route, this value will be 0, which is what we want, otherwise, we // will use whatever index the remote has chosen. idx = info.RoutePoolIdx } else if pool { // The remote was found. If this is a non solicited route, we will place // the connection in the pool at the index given by info.RoutePoolIdx. // But if there is already one, close this incoming connection as a // duplicate. if !didSolicit { idx = info.RoutePoolIdx if remote = conns[idx]; remote != nil { handleDuplicateRoute(remote, c, false) s.mu.Unlock() return false } // Look if there is a solicited route in the pool. If there is one, // they should all be, so stop at the first. if url, rtype, hasSolicited := hasSolicitedRoute(conns); hasSolicited { upgradeRouteToSolicited(c, url, rtype) } } else { // If we solicit, upgrade to solicited all non-solicited routes that // we may have registered. c.mu.Lock() url := c.route.url rtype := c.route.routeType c.mu.Unlock() for _, r := range conns { upgradeRouteToSolicited(r, url, rtype) } } // For all cases (solicited and not) we need to count how many connections // we already have, and for solicited route, we will find a free spot in // the slice. for i, r := range conns { if idx == -1 && r == nil { idx = i } else if r != nil { sz++ } } } else { remote = conns[0] } // If there is a spot, idx will be greater or equal to 0. if idx >= 0 { c.mu.Lock() c.route.connectURLs = info.ClientConnectURLs c.route.wsConnURLs = info.WSConnectURLs c.route.poolIdx = idx rtype := c.route.routeType cid := c.cid idHash := c.route.idHash rHash := c.route.hash rn := c.route.remoteName url := c.route.url // For solicited routes, we need now to send the INFO protocol. if didSolicit { c.enqueueProto(s.generateRouteInitialInfoJSON(_EMPTY_, c.route.compression, idx)) } if c.last.IsZero() { c.last = time.Now() } c.mu.Unlock() // Add to the slice and bump the count of connections for this remote conns[idx] = c sz++ // This boolean will indicate that we are registering the only // connection in non pooled situation or we stored the very first // connection for a given remote server. doOnce := !pool || sz == 1 if doOnce { // check to be consistent and future proof. but will be same domain if s.sameDomain(info.Domain) { s.nodeToInfo.Store(rHash, nodeInfo{rn, s.info.Version, s.info.Cluster, info.Domain, id, nil, nil, nil, false, info.JetStream, false}) } } // Store this route using the hash as the key if pool { idHash += strconv.Itoa(idx) } s.storeRouteByHash(idHash, c) // Now that we have registered the route, we can remove from the temp map. s.removeFromTempClients(cid) if doOnce { // If the INFO contains a Gateway URL, add it to the list for our cluster. if info.GatewayURL != _EMPTY_ && s.addGatewayURL(info.GatewayURL) { s.sendAsyncGatewayInfo() } // we don't need to send if the only route is the one we just accepted. if len(s.routes) > 1 { // Now let the known servers know about this new route s.forwardNewRouteInfoToKnownServers(info) } // Send info about the known gateways to this route. s.sendGatewayConfigsToRoute(c) // Unless disabled, possibly update the server's INFO protocol // and send to clients that know how to handle async INFOs. if !opts.Cluster.NoAdvertise { s.addConnectURLsAndSendINFOToClients(info.ClientConnectURLs, info.WSConnectURLs) } // Add the remote's leafnodeURL to our list of URLs and send the update // to all LN connections. (Note that when coming from a route, LeafNodeURLs // is an array of size 1 max). if len(info.LeafNodeURLs) == 1 && s.addLeafNodeURL(info.LeafNodeURLs[0]) { s.sendAsyncLeafNodeInfo() } } // Send the subscriptions interest. s.sendSubsToRoute(c, idx, _EMPTY_) // In pool mode, if we did not yet reach the cap, try to connect a new connection if pool && didSolicit && sz != s.routesPoolSize { s.startGoRoutine(func() { select { case <-time.After(time.Duration(rand.Intn(100)) * time.Millisecond): case <-s.quitCh: // Doing this here and not as a defer because connectToRoute is also // calling s.grWG.Done() on exit, so we do this only if we don't // invoke connectToRoute(). s.grWG.Done() return } s.connectToRoute(url, rtype == Explicit, true, _EMPTY_) }) } } s.mu.Unlock() if pool { if idx == -1 { // Was full, so need to close connection c.Debugf("Route pool size reached, closing extra connection to %q", id) handleDuplicateRoute(nil, c, true) return false } return true } // This is for non-pool mode at this point. if exists { handleDuplicateRoute(remote, c, noReconnectForOldServer) } return !exists } func hasSolicitedRoute(conns []*client) (*url.URL, RouteType, bool) { var url *url.URL var rtype RouteType for _, r := range conns { if r == nil { continue } r.mu.Lock() if r.route.didSolicit { url = r.route.url rtype = r.route.routeType } r.mu.Unlock() if url != nil { return url, rtype, true } } return nil, 0, false } func upgradeRouteToSolicited(r *client, url *url.URL, rtype RouteType) { if r == nil { return } r.mu.Lock() if !r.route.didSolicit { r.route.didSolicit = true r.route.url = url } if rtype == Explicit { r.route.routeType = Explicit } r.mu.Unlock() } func handleDuplicateRoute(remote, c *client, setNoReconnect bool) { // We used to clear some fields when closing a duplicate connection // to prevent sending INFO protocols for the remotes to update // their leafnode/gateway URLs. This is no longer needed since // removeRoute() now does the right thing of doing that only when // the closed connection was an added route connection. c.mu.Lock() didSolicit := c.route.didSolicit url := c.route.url rtype := c.route.routeType if setNoReconnect { c.flags.set(noReconnect) } c.mu.Unlock() if remote == nil { return } remote.mu.Lock() if didSolicit && !remote.route.didSolicit { remote.route.didSolicit = true remote.route.url = url } // The extra route might be an configured explicit route // so keep the state that the remote was configured. if rtype == Explicit { remote.route.routeType = rtype } // This is to mitigate the issue where both sides add the route // on the opposite connection, and therefore end-up with both // connections being dropped. remote.route.retry = true remote.mu.Unlock() } // Import filter check. func (c *client) importFilter(sub *subscription) bool { if c.perms == nil { return true } return c.canImport(string(sub.subject)) } // updateRouteSubscriptionMap will make sure to update the route map for the subscription. Will // also forward to all routes if needed. func (s *Server) updateRouteSubscriptionMap(acc *Account, sub *subscription, delta int32) { if acc == nil || sub == nil { return } // We only store state on local subs for transmission across all other routes. if sub.client == nil || sub.client.kind == ROUTER || sub.client.kind == GATEWAY { return } if sub.si { return } // Copy to hold outside acc lock. var n int32 var ok bool isq := len(sub.queue) > 0 accLock := func() { // Not required for code correctness, but helps reduce the number of // updates sent to the routes when processing high number of concurrent // queue subscriptions updates (sub/unsub). // See https://github.com/nats-io/nats-server/pull/1126 for more details. if isq { acc.sqmu.Lock() } acc.mu.Lock() } accUnlock := func() { acc.mu.Unlock() if isq { acc.sqmu.Unlock() } } accLock() // This is non-nil when we know we are in cluster mode. rm, lqws := acc.rm, acc.lqws if rm == nil { accUnlock() return } // Create the subscription key which will prevent collisions between regular // and leaf routed subscriptions. See keyFromSubWithOrigin() for details. key := keyFromSubWithOrigin(sub) // Decide whether we need to send an update out to all the routes. update := isq // This is where we do update to account. For queues we need to take // special care that this order of updates is same as what is sent out // over routes. if n, ok = rm[key]; ok { n += delta if n <= 0 { delete(rm, key) if isq { delete(lqws, key) } update = true // Update for deleting (N->0) } else { rm[key] = n } } else if delta > 0 { n = delta rm[key] = delta update = true // Adding a new entry for normal sub means update (0->1) } accUnlock() if !update { return } // If we are sending a queue sub, make a copy and place in the queue weight. // FIXME(dlc) - We can be smarter here and avoid copying and acquiring the lock. if isq { sub.client.mu.Lock() nsub := *sub sub.client.mu.Unlock() nsub.qw = n sub = &nsub } // We need to send out this update. Gather routes var _routes [32]*client routes := _routes[:0] s.mu.RLock() // The account's routePoolIdx field is set/updated under the server lock // (but also the account's lock). So we don't need to acquire the account's // lock here to get the value. if poolIdx := acc.routePoolIdx; poolIdx < 0 { if conns, ok := s.accRoutes[acc.Name]; ok { for _, r := range conns { routes = append(routes, r) } } if s.routesNoPool > 0 { // We also need to look for "no pool" remotes (that is, routes to older // servers or servers that have explicitly disabled pooling). s.forEachRemote(func(r *client) { r.mu.Lock() if r.route.noPool { routes = append(routes, r) } r.mu.Unlock() }) } } else { // We can't use s.forEachRouteIdx here since we want to check/get the // "no pool" route ONLY if we don't find a route at the given `poolIdx`. for _, conns := range s.routes { if r := conns[poolIdx]; r != nil { routes = append(routes, r) } else if s.routesNoPool > 0 { // Check if we have a "no pool" route at index 0, and if so, it // means that for this remote, we have a single connection because // that server does not have pooling. if r := conns[0]; r != nil { r.mu.Lock() if r.route.noPool { routes = append(routes, r) } r.mu.Unlock() } } } } trace := atomic.LoadInt32(&s.logging.trace) == 1 s.mu.RUnlock() // If we are a queue subscriber we need to make sure our updates are serialized from // potential multiple connections. We want to make sure that the order above is preserved // here but not necessarily all updates need to be sent. We need to block and recheck the // n count with the lock held through sending here. We will suppress duplicate sends of same qw. if isq { // However, we can't hold the acc.mu lock since we allow client.mu.Lock -> acc.mu.Lock // but not the opposite. So use a dedicated lock while holding the route's lock. acc.sqmu.Lock() defer acc.sqmu.Unlock() acc.mu.Lock() n = rm[key] sub.qw = n // Check the last sent weight here. If same, then someone // beat us to it and we can just return here. Otherwise update if ls, ok := lqws[key]; ok && ls == n { acc.mu.Unlock() return } else if n > 0 { lqws[key] = n } acc.mu.Unlock() } // Snapshot into array subs := []*subscription{sub} // Deliver to all routes. for _, route := range routes { route.mu.Lock() // Note that queue unsubs where n > 0 are still // subscribes with a smaller weight. route.sendRouteSubOrUnSubProtos(subs, n > 0, trace, route.importFilter) route.mu.Unlock() } } // This starts the route accept loop in a go routine, unless it // is detected that the server has already been shutdown. // It will also start soliciting explicit routes. func (s *Server) startRouteAcceptLoop() { if s.isShuttingDown() { return } // Snapshot server options. opts := s.getOpts() // Snapshot server options. port := opts.Cluster.Port if port == -1 { port = 0 } // This requires lock, so do this outside of may block. clusterName := s.ClusterName() s.mu.Lock() s.Noticef("Cluster name is %s", clusterName) if s.isClusterNameDynamic() { s.Warnf("Cluster name was dynamically generated, consider setting one") } hp := net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(port)) l, e := natsListen("tcp", hp) s.routeListenerErr = e if e != nil { s.mu.Unlock() s.Fatalf("Error listening on router port: %d - %v", opts.Cluster.Port, e) return } s.Noticef("Listening for route connections on %s", net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) proto := RouteProtoV2 // For tests, we want to be able to make this server behave // as an older server so check this option to see if we should override if opts.routeProto < 0 { // We have a private option that allows test to override the route // protocol. We want this option initial value to be 0, however, // since original proto is RouteProtoZero, tests call setRouteProtoForTest(), // which sets as negative value the (desired proto + 1) * -1. // Here we compute back the real value. proto = (opts.routeProto * -1) - 1 } // Check for TLSConfig tlsReq := opts.Cluster.TLSConfig != nil info := Info{ ID: s.info.ID, Name: s.info.Name, Version: s.info.Version, GoVersion: runtime.Version(), AuthRequired: false, TLSRequired: tlsReq, TLSVerify: tlsReq, MaxPayload: s.info.MaxPayload, JetStream: s.info.JetStream, Proto: proto, GatewayURL: s.getGatewayURL(), Headers: s.supportsHeaders(), Cluster: s.info.Cluster, Domain: s.info.Domain, Dynamic: s.isClusterNameDynamic(), LNOC: true, LNOCU: true, } // For tests that want to simulate old servers, do not set the compression // on the INFO protocol if configured with CompressionNotSupported. if cm := opts.Cluster.Compression.Mode; cm != CompressionNotSupported { info.Compression = cm } if ps := opts.Cluster.PoolSize; ps > 0 { info.RoutePoolSize = ps } // Set this if only if advertise is not disabled if !opts.Cluster.NoAdvertise { info.ClientConnectURLs = s.clientConnectURLs info.WSConnectURLs = s.websocket.connectURLs } // If we have selected a random port... if port == 0 { // Write resolved port back to options. opts.Cluster.Port = l.Addr().(*net.TCPAddr).Port } // Check for Auth items if opts.Cluster.Username != "" { info.AuthRequired = true } // Check for permissions. if opts.Cluster.Permissions != nil { info.Import = opts.Cluster.Permissions.Import info.Export = opts.Cluster.Permissions.Export } // If this server has a LeafNode accept loop, s.leafNodeInfo.IP is, // at this point, set to the host:port for the leafnode accept URL, // taking into account possible advertise setting. Use the LeafNodeURLs // and set this server's leafnode accept URL. This will be sent to // routed servers. if !opts.LeafNode.NoAdvertise && s.leafNodeInfo.IP != _EMPTY_ { info.LeafNodeURLs = []string{s.leafNodeInfo.IP} } s.routeInfo = info // Possibly override Host/Port and set IP based on Cluster.Advertise if err := s.setRouteInfoHostPortAndIP(); err != nil { s.Fatalf("Error setting route INFO with Cluster.Advertise value of %s, err=%v", opts.Cluster.Advertise, err) l.Close() s.mu.Unlock() return } // Setup state that can enable shutdown s.routeListener = l // Warn if using Cluster.Insecure if tlsReq && opts.Cluster.TLSConfig.InsecureSkipVerify { s.Warnf(clusterTLSInsecureWarning) } // Now that we have the port, keep track of all ip:port that resolve to this server. if interfaceAddr, err := net.InterfaceAddrs(); err == nil { var localIPs []string for i := 0; i < len(interfaceAddr); i++ { interfaceIP, _, _ := net.ParseCIDR(interfaceAddr[i].String()) ipStr := interfaceIP.String() if net.ParseIP(ipStr) != nil { localIPs = append(localIPs, ipStr) } } var portStr = strconv.FormatInt(int64(s.routeInfo.Port), 10) for _, ip := range localIPs { ipPort := net.JoinHostPort(ip, portStr) s.routesToSelf[ipPort] = struct{}{} } } // Start the accept loop in a different go routine. go s.acceptConnections(l, "Route", func(conn net.Conn) { s.createRoute(conn, nil, _EMPTY_) }, nil) // Solicit Routes if applicable. This will not block. s.solicitRoutes(opts.Routes, opts.Cluster.PinnedAccounts) s.mu.Unlock() } // Similar to setInfoHostPortAndGenerateJSON, but for routeInfo. func (s *Server) setRouteInfoHostPortAndIP() error { opts := s.getOpts() if opts.Cluster.Advertise != _EMPTY_ { advHost, advPort, err := parseHostPort(opts.Cluster.Advertise, opts.Cluster.Port) if err != nil { return err } s.routeInfo.Host = advHost s.routeInfo.Port = advPort s.routeInfo.IP = fmt.Sprintf("nats-route://%s/", net.JoinHostPort(advHost, strconv.Itoa(advPort))) } else { s.routeInfo.Host = opts.Cluster.Host s.routeInfo.Port = opts.Cluster.Port s.routeInfo.IP = "" } return nil } // StartRouting will start the accept loop on the cluster host:port // and will actively try to connect to listed routes. func (s *Server) StartRouting(clientListenReady chan struct{}) { defer s.grWG.Done() // Wait for the client and and leafnode listen ports to be opened, // and the possible ephemeral ports to be selected. <-clientListenReady // Start the accept loop and solicitation of explicit routes (if applicable) s.startRouteAcceptLoop() } func (s *Server) reConnectToRoute(rURL *url.URL, rtype RouteType, accName string) { tryForEver := rtype == Explicit // If A connects to B, and B to A (regardless if explicit or // implicit - due to auto-discovery), and if each server first // registers the route on the opposite TCP connection, the // two connections will end-up being closed. // Add some random delay to reduce risk of repeated failures. delay := time.Duration(rand.Intn(100)) * time.Millisecond if tryForEver { delay += DEFAULT_ROUTE_RECONNECT } select { case <-time.After(delay): case <-s.quitCh: s.grWG.Done() return } s.connectToRoute(rURL, tryForEver, false, accName) } // Checks to make sure the route is still valid. func (s *Server) routeStillValid(rURL *url.URL) bool { for _, ri := range s.getOpts().Routes { if urlsAreEqual(ri, rURL) { return true } } return false } func (s *Server) connectToRoute(rURL *url.URL, tryForEver, firstConnect bool, accName string) { // Snapshot server options. opts := s.getOpts() defer s.grWG.Done() const connErrFmt = "Error trying to connect to route (attempt %v): %v" s.mu.Lock() resolver := s.routeResolver excludedAddresses := s.routesToSelf s.mu.Unlock() attempts := 0 for s.isRunning() && rURL != nil { if tryForEver { if !s.routeStillValid(rURL) { return } if accName != _EMPTY_ { s.mu.RLock() _, valid := s.accRoutes[accName] s.mu.RUnlock() if !valid { return } } } var conn net.Conn address, err := s.getRandomIP(resolver, rURL.Host, excludedAddresses) if err == errNoIPAvail { // This is ok, we are done. return } if err == nil { s.Debugf("Trying to connect to route on %s (%s)", rURL.Host, address) conn, err = natsDialTimeout("tcp", address, DEFAULT_ROUTE_DIAL) } if err != nil { attempts++ if s.shouldReportConnectErr(firstConnect, attempts) { s.Errorf(connErrFmt, attempts, err) } else { s.Debugf(connErrFmt, attempts, err) } if !tryForEver { if opts.Cluster.ConnectRetries <= 0 { return } if attempts > opts.Cluster.ConnectRetries { return } } select { case <-s.quitCh: return case <-time.After(routeConnectDelay): continue } } if tryForEver && !s.routeStillValid(rURL) { conn.Close() return } // We have a route connection here. // Go ahead and create it and exit this func. s.createRoute(conn, rURL, accName) return } } func (c *client) isSolicitedRoute() bool { c.mu.Lock() defer c.mu.Unlock() return c.kind == ROUTER && c.route != nil && c.route.didSolicit } // Save the first hostname found in route URLs. This will be used in gossip mode // when trying to create a TLS connection by setting the tlsConfig.ServerName. // Lock is held on entry func (s *Server) saveRouteTLSName(routes []*url.URL) { for _, u := range routes { if s.routeTLSName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil { s.routeTLSName = u.Hostname() } } } // Start connection process to provided routes. Each route connection will // be started in a dedicated go routine. // Lock is held on entry func (s *Server) solicitRoutes(routes []*url.URL, accounts []string) { s.saveRouteTLSName(routes) for _, r := range routes { route := r s.startGoRoutine(func() { s.connectToRoute(route, true, true, _EMPTY_) }) } // Now go over possible per-account routes and create them. for _, an := range accounts { for _, r := range routes { route, accName := r, an s.startGoRoutine(func() { s.connectToRoute(route, true, true, accName) }) } } } func (c *client) processRouteConnect(srv *Server, arg []byte, lang string) error { // Way to detect clients that incorrectly connect to the route listen // port. Client provide Lang in the CONNECT protocol while ROUTEs don't. if lang != "" { c.sendErrAndErr(ErrClientConnectedToRoutePort.Error()) c.closeConnection(WrongPort) return ErrClientConnectedToRoutePort } // Unmarshal as a route connect protocol proto := &connectInfo{} if err := json.Unmarshal(arg, proto); err != nil { return err } // Reject if this has Gateway which means that it would be from a gateway // connection that incorrectly connects to the Route port. if proto.Gateway != "" { errTxt := fmt.Sprintf("Rejecting connection from gateway %q on the Route port", proto.Gateway) c.Errorf(errTxt) c.sendErr(errTxt) c.closeConnection(WrongGateway) return ErrWrongGateway } if srv == nil { return ErrServerNotRunning } perms := srv.getOpts().Cluster.Permissions clusterName := srv.ClusterName() // If we have a cluster name set, make sure it matches ours. if proto.Cluster != clusterName { shouldReject := true // If we have a dynamic name we will do additional checks. if srv.isClusterNameDynamic() { if !proto.Dynamic || strings.Compare(clusterName, proto.Cluster) < 0 { // We will take on their name since theirs is configured or higher then ours. srv.setClusterName(proto.Cluster) if !proto.Dynamic { srv.optsMu.Lock() srv.opts.Cluster.Name = proto.Cluster srv.optsMu.Unlock() } c.mu.Lock() remoteID := c.opts.Name c.mu.Unlock() srv.removeAllRoutesExcept(remoteID) shouldReject = false } } if shouldReject { errTxt := fmt.Sprintf("Rejecting connection, cluster name %q does not match %q", proto.Cluster, clusterName) c.Errorf(errTxt) c.sendErr(errTxt) c.closeConnection(ClusterNameConflict) return ErrClusterNameRemoteConflict } } supportsHeaders := c.srv.supportsHeaders() // Grab connection name of remote route. c.mu.Lock() c.route.remoteID = c.opts.Name c.route.lnoc = proto.LNOC c.route.lnocu = proto.LNOCU c.setRoutePermissions(perms) c.headers = supportsHeaders && proto.Headers c.mu.Unlock() return nil } // Called when we update our cluster name during negotiations with remotes. func (s *Server) removeAllRoutesExcept(remoteID string) { s.mu.Lock() routes := make([]*client, 0, s.numRoutes()) for rID, conns := range s.routes { if rID == remoteID { continue } for _, r := range conns { if r != nil { routes = append(routes, r) } } } for _, conns := range s.accRoutes { for rID, r := range conns { if rID == remoteID { continue } routes = append(routes, r) } } s.mu.Unlock() for _, r := range routes { r.closeConnection(ClusterNameConflict) } } func (s *Server) removeRoute(c *client) { s.mu.Lock() defer s.mu.Unlock() var ( rID string lnURL string gwURL string idHash string accName string poolIdx = -1 connectURLs []string wsConnectURLs []string opts = s.getOpts() rURL *url.URL noPool bool didSolicit bool ) c.mu.Lock() cid := c.cid r := c.route if r != nil { rID = r.remoteID lnURL = r.leafnodeURL idHash = r.idHash gwURL = r.gatewayURL poolIdx = r.poolIdx accName = bytesToString(r.accName) if r.noPool { s.routesNoPool-- noPool = true } connectURLs = r.connectURLs wsConnectURLs = r.wsConnURLs rURL = r.url didSolicit = r.didSolicit } c.mu.Unlock() if accName != _EMPTY_ { if conns, ok := s.accRoutes[accName]; ok { if r := conns[rID]; r == c { s.removeRouteByHash(idHash + accName) delete(conns, rID) // Do not remove or set to nil when all remotes have been // removed from the map. The configured accounts must always // be in the accRoutes map and addRoute expects "conns" map // to be created. } } } // If this is still -1, it means that it was not added to the routes // so simply remove from temp clients and we are done. if poolIdx == -1 || accName != _EMPTY_ { s.removeFromTempClients(cid) return } if conns, ok := s.routes[rID]; ok { // If this route was not the one stored, simply remove from the // temporary map and be done. if conns[poolIdx] != c { s.removeFromTempClients(cid) return } conns[poolIdx] = nil // Now check if this was the last connection to be removed. empty := true for _, c := range conns { if c != nil { empty = false break } } // This was the last route for this remote. Remove the remote entry // and possibly send some async INFO protocols regarding gateway // and leafnode URLs. if empty { delete(s.routes, rID) // Since this is the last route for this remote, possibly update // the client connect URLs and send an update to connected // clients. if (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise { s.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs) } // Remove the remote's gateway URL from our list and // send update to inbound Gateway connections. if gwURL != _EMPTY_ && s.removeGatewayURL(gwURL) { s.sendAsyncGatewayInfo() } // Remove the remote's leafNode URL from // our list and send update to LN connections. if lnURL != _EMPTY_ && s.removeLeafNodeURL(lnURL) { s.sendAsyncLeafNodeInfo() } // If this server has pooling/pinned accounts and the route for // this remote was a "no pool" route, attempt to reconnect. if noPool { if s.routesPoolSize > 1 { s.startGoRoutine(func() { s.connectToRoute(rURL, didSolicit, true, _EMPTY_) }) } if len(opts.Cluster.PinnedAccounts) > 0 { for _, an := range opts.Cluster.PinnedAccounts { accName := an s.startGoRoutine(func() { s.connectToRoute(rURL, didSolicit, true, accName) }) } } } } // This is for gateway code. Remove this route from a map that uses // the route hash in combination with the pool index as the key. if s.routesPoolSize > 1 { idHash += strconv.Itoa(poolIdx) } s.removeRouteByHash(idHash) } s.removeFromTempClients(cid) } func (s *Server) isDuplicateServerName(name string) bool { if name == _EMPTY_ { return false } s.mu.RLock() defer s.mu.RUnlock() if s.info.Name == name { return true } for _, conns := range s.routes { for _, r := range conns { if r != nil { r.mu.Lock() duplicate := r.route.remoteName == name r.mu.Unlock() if duplicate { return true } break } } } return false } // Goes over each non-nil route connection for all remote servers // and invokes the function `f`. It does not go over per-account // routes. // Server lock is held on entry. func (s *Server) forEachNonPerAccountRoute(f func(r *client)) { for _, conns := range s.routes { for _, r := range conns { if r != nil { f(r) } } } } // Goes over each non-nil route connection for all remote servers // and invokes the function `f`. This also includes the per-account // routes. // Server lock is held on entry. func (s *Server) forEachRoute(f func(r *client)) { s.forEachNonPerAccountRoute(f) for _, conns := range s.accRoutes { for _, r := range conns { f(r) } } } // Goes over each non-nil route connection at the given pool index // location in the slice and invokes the function `f`. If the // callback returns `true`, this function moves to the next remote. // Otherwise, the iteration over removes stops. // This does not include per-account routes. // Server lock is held on entry. func (s *Server) forEachRouteIdx(idx int, f func(r *client) bool) { for _, conns := range s.routes { if r := conns[idx]; r != nil { if !f(r) { return } } } } // Goes over each remote and for the first non nil route connection, // invokes the function `f`. // Server lock is held on entry. func (s *Server) forEachRemote(f func(r *client)) { for _, conns := range s.routes { for _, r := range conns { if r != nil { f(r) break } } } } nats-server-2.10.27/server/routes_test.go000066400000000000000000003460041477524627100203770ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "context" "crypto/tls" "encoding/json" "fmt" "math/rand" "net" "net/http" "net/http/httptest" "net/url" "os" "reflect" "runtime" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func init() { routeConnectDelay = 15 * time.Millisecond } func checkNumRoutes(t *testing.T, s *Server, expected int) { t.Helper() checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { if nr := s.NumRoutes(); nr != expected { return fmt.Errorf("Expected %v routes, got %v", expected, nr) } return nil }) } func checkSubInterest(t *testing.T, s *Server, accName, subject string, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { acc, err := s.LookupAccount(accName) if err != nil { return fmt.Errorf("error looking up account %q: %v", accName, err) } if acc.SubscriptionInterest(subject) { return nil } return fmt.Errorf("no subscription interest for account %q on %q", accName, subject) }) } func checkSubNoInterest(t *testing.T, s *Server, accName, subject string, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { acc, err := s.LookupAccount(accName) if err != nil { return fmt.Errorf("error looking up account %q: %v", accName, err) } if acc.SubscriptionInterest(subject) { return fmt.Errorf("unexpected subscription interest for account %q on %q", accName, subject) } return nil }) } func TestRouteConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/cluster.conf") if err != nil { t.Fatalf("Received an error reading route config file: %v\n", err) } golden := &Options{ ConfigFile: "./configs/cluster.conf", Host: "127.0.0.1", Port: 4242, Username: "derek", Password: "porkchop", AuthTimeout: 1.0, Cluster: ClusterOpts{ Name: "abc", Host: "127.0.0.1", Port: 4244, Username: "route_user", Password: "top_secret", AuthTimeout: 1.0, NoAdvertise: true, ConnectRetries: 2, }, PidFile: "/tmp/nats-server/nats_cluster_test.pid", authBlockDefined: true, } // Setup URLs r1, _ := url.Parse("nats-route://foo:bar@127.0.0.1:4245") r2, _ := url.Parse("nats-route://foo:bar@127.0.0.1:4246") golden.Routes = []*url.URL{r1, r2} checkOptionsEqual(t, golden, opts) } func TestClusterAdvertise(t *testing.T) { lst, err := natsListen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error starting listener: %v", err) } ch := make(chan error) go func() { c, err := lst.Accept() if err != nil { ch <- err return } c.Close() ch <- nil }() optsA, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true srvA := RunServer(optsA) defer srvA.Shutdown() srvARouteURL := fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, srvA.ClusterAddr().Port) optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(srvARouteURL) srvB := RunServer(optsB) defer srvB.Shutdown() // Wait for these 2 to connect to each other checkClusterFormed(t, srvA, srvB) // Now start server C that connects to A. A should ask B to connect to C, // based on C's URL. But since C configures a Cluster.Advertise, it will connect // to our listener. optsC := nextServerOpts(optsB) optsC.Cluster.Advertise = lst.Addr().String() optsC.ClientAdvertise = "me:1" optsC.Routes = RoutesFromStr(srvARouteURL) srvC := RunServer(optsC) defer srvC.Shutdown() select { case e := <-ch: if e != nil { t.Fatalf("Error: %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Test timed out") } } func TestClusterAdvertiseErrorOnStartup(t *testing.T) { opts := DefaultOptions() // Set invalid address opts.Cluster.Advertise = "addr:::123" testFatalErrorOnStart(t, opts, "Cluster.Advertise") } func TestClientAdvertise(t *testing.T) { optsA, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true srvA := RunServer(optsA) defer srvA.Shutdown() optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) optsB.ClientAdvertise = "me:1" srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() checkFor(t, time.Second, 15*time.Millisecond, func() error { ds := nc.DiscoveredServers() if len(ds) == 1 { if ds[0] == "nats://me:1" { return nil } } return fmt.Errorf("Did not get expected discovered servers: %v", nc.DiscoveredServers()) }) } func TestServerRoutesWithClients(t *testing.T) { optsA, err := ProcessConfigFile("./configs/srv_a.conf") require_NoError(t, err) optsB, err := ProcessConfigFile("./configs/srv_b.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true optsB.NoSigs, optsB.NoLog = true, true srvA := RunServer(optsA) defer srvA.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() ch := make(chan bool) sub, _ := nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc1.QueueSubscribe("foo", "bar", func(m *nats.Msg) {}) nc1.Publish("foo", []byte("Hello")) // Wait for message <-ch sub.Unsubscribe() srvB := RunServer(optsB) defer srvB.Shutdown() // Wait for route to form. checkClusterFormed(t, srvA, srvB) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() nc2.Publish("foo", []byte("Hello")) nc2.Flush() } func TestServerRoutesWithAuthAndBCrypt(t *testing.T) { optsA, err := ProcessConfigFile("./configs/srv_a_bcrypt.conf") require_NoError(t, err) optsB, err := ProcessConfigFile("./configs/srv_b_bcrypt.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true optsB.NoSigs, optsB.NoLog = true, true srvA := RunServer(optsA) defer srvA.Shutdown() srvB := RunServer(optsB) defer srvB.Shutdown() // Wait for route to form. checkClusterFormed(t, srvA, srvB) urlA := fmt.Sprintf("nats://%s:%s@%s:%d/", optsA.Username, optsA.Password, optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%s@%s:%d/", optsB.Username, optsB.Password, optsB.Host, optsB.Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() // Test that we are connected. ch := make(chan bool) sub, err := nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) if err != nil { t.Fatalf("Error creating subscription: %v\n", err) } nc1.Flush() defer sub.Unsubscribe() checkSubInterest(t, srvB, globalAccountName, "foo", time.Second) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() nc2.Publish("foo", []byte("Hello")) nc2.Flush() // Wait for message select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } // Helper function to check that a cluster is formed func checkClusterFormed(t testing.TB, servers ...*Server) { t.Helper() var _enr [8]int enr := _enr[:0] for _, a := range servers { if a.getOpts().Cluster.PoolSize < 0 { enr = append(enr, len(servers)-1) } else { a.mu.RLock() nr := a.routesPoolSize + len(a.accRoutes) a.mu.RUnlock() total := 0 for _, b := range servers { if a == b { continue } if b.getOpts().Cluster.PoolSize < 0 { total++ } else { total += nr } } enr = append(enr, total) } } checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { for i, s := range servers { if numRoutes := s.NumRoutes(); numRoutes != enr[i] { return fmt.Errorf("Expected %d routes for server %q, got %d", enr[i], s, numRoutes) } } return nil }) } // Helper function to generate next opts to make sure no port conflicts etc. func nextServerOpts(opts *Options) *Options { nopts := *opts nopts.Port = -1 nopts.Cluster.Port = -1 nopts.HTTPPort = -1 if nopts.Gateway.Name != "" { nopts.Gateway.Port = -1 } nopts.ServerName = "" return &nopts } func TestSeedSolicitWorks(t *testing.T) { optsSeed, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsSeed.NoSigs, optsSeed.NoLog = true, true optsSeed.NoSystemAccount = true srvSeed := RunServer(optsSeed) defer srvSeed.Shutdown() optsA := nextServerOpts(optsSeed) optsA.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, srvSeed.ClusterAddr().Port)) srvA := RunServer(optsA) defer srvA.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, srvA.ClusterAddr().Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() // Test that we are connected. ch := make(chan bool) nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc1.Flush() optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, srvSeed.ClusterAddr().Port)) srvB := RunServer(optsB) defer srvB.Shutdown() urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, srvB.ClusterAddr().Port) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() checkClusterFormed(t, srvSeed, srvA, srvB) checkExpectedSubs(t, 1, srvB) nc2.Publish("foo", []byte("Hello")) // Wait for message select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } func TestTLSSeedSolicitWorks(t *testing.T) { optsSeed, err := ProcessConfigFile("./configs/seed_tls.conf") require_NoError(t, err) optsSeed.NoSigs, optsSeed.NoLog = true, true optsSeed.NoSystemAccount = true srvSeed := RunServer(optsSeed) defer srvSeed.Shutdown() seedRouteURL := fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, srvSeed.ClusterAddr().Port) optsA := nextServerOpts(optsSeed) optsA.Routes = RoutesFromStr(seedRouteURL) srvA := RunServer(optsA) defer srvA.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, srvA.Addr().(*net.TCPAddr).Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() // Test that we are connected. ch := make(chan bool) nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc1.Flush() optsB := nextServerOpts(optsA) optsB.Routes = RoutesFromStr(seedRouteURL) srvB := RunServer(optsB) defer srvB.Shutdown() urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, srvB.Addr().(*net.TCPAddr).Port) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() checkClusterFormed(t, srvSeed, srvA, srvB) checkExpectedSubs(t, 1, srvB) nc2.Publish("foo", []byte("Hello")) // Wait for message select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } func TestChainedSolicitWorks(t *testing.T) { optsSeed, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsSeed.NoSigs, optsSeed.NoLog = true, true optsSeed.NoSystemAccount = true srvSeed := RunServer(optsSeed) defer srvSeed.Shutdown() seedRouteURL := fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, srvSeed.ClusterAddr().Port) optsA := nextServerOpts(optsSeed) optsA.Routes = RoutesFromStr(seedRouteURL) srvA := RunServer(optsA) defer srvA.Shutdown() urlSeed := fmt.Sprintf("nats://%s:%d/", optsSeed.Host, srvA.Addr().(*net.TCPAddr).Port) nc1, err := nats.Connect(urlSeed) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() // Test that we are connected. ch := make(chan bool) nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc1.Flush() optsB := nextServerOpts(optsA) // Server B connects to A optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, srvA.ClusterAddr().Port)) srvB := RunServer(optsB) defer srvB.Shutdown() urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, srvB.Addr().(*net.TCPAddr).Port) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() checkClusterFormed(t, srvSeed, srvA, srvB) checkExpectedSubs(t, 1, srvB) nc2.Publish("foo", []byte("Hello")) // Wait for message select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } // Helper function to check that a server (or list of servers) have the // expected number of subscriptions. func checkExpectedSubs(t *testing.T, expected int, servers ...*Server) { t.Helper() checkFor(t, 4*time.Second, 10*time.Millisecond, func() error { for _, s := range servers { if numSubs := int(s.NumSubscriptions()); numSubs != expected { return fmt.Errorf("Expected %d subscriptions for server %q, got %d", expected, s.ID(), numSubs) } } return nil }) } func TestTLSChainedSolicitWorks(t *testing.T) { optsSeed, err := ProcessConfigFile("./configs/seed_tls.conf") require_NoError(t, err) optsSeed.NoSigs, optsSeed.NoLog = true, true optsSeed.NoSystemAccount = true srvSeed := RunServer(optsSeed) defer srvSeed.Shutdown() urlSeedRoute := fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, srvSeed.ClusterAddr().Port) optsA := nextServerOpts(optsSeed) optsA.Routes = RoutesFromStr(urlSeedRoute) srvA := RunServer(optsA) defer srvA.Shutdown() urlSeed := fmt.Sprintf("nats://%s:%d/", optsSeed.Host, srvSeed.Addr().(*net.TCPAddr).Port) nc1, err := nats.Connect(urlSeed) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc1.Close() // Test that we are connected. ch := make(chan bool) nc1.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc1.Flush() optsB := nextServerOpts(optsA) // Server B connects to A optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, srvA.ClusterAddr().Port)) srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvSeed, srvA, srvB) checkExpectedSubs(t, 1, srvA, srvB) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, srvB.Addr().(*net.TCPAddr).Port) nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() nc2.Publish("foo", []byte("Hello")) // Wait for message select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } func TestRouteTLSHandshakeError(t *testing.T) { optsSeed, err := ProcessConfigFile("./configs/seed_tls.conf") require_NoError(t, err) optsSeed.NoLog = true optsSeed.NoSigs = true srvSeed := RunServer(optsSeed) defer srvSeed.Shutdown() opts := DefaultOptions() opts.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, optsSeed.Cluster.Port)) srv := RunServer(opts) defer srv.Shutdown() time.Sleep(500 * time.Millisecond) checkNumRoutes(t, srv, 0) } func TestBlockedShutdownOnRouteAcceptLoopFailure(t *testing.T) { opts := DefaultOptions() opts.Cluster.Host = "x.x.x.x" opts.Cluster.Port = 7222 s := New(opts) s.Start() // Wait a second time.Sleep(time.Second) ch := make(chan bool) go func() { s.Shutdown() ch <- true }() timeout := time.NewTimer(5 * time.Second) select { case <-ch: return case <-timeout.C: t.Fatal("Shutdown did not complete") } } func TestRouteUseIPv6(t *testing.T) { opts := DefaultOptions() opts.Cluster.Host = "::" opts.Cluster.Port = 6222 // I believe that there is no IPv6 support on Travis... // Regardless, cannot have this test fail simply because IPv6 is disabled // on the host. hp := net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(opts.Cluster.Port)) _, err := net.ResolveTCPAddr("tcp", hp) if err != nil { t.Skipf("Skipping this test since there is no IPv6 support on this host: %v", err) } s := RunServer(opts) defer s.Shutdown() routeUp := false timeout := time.Now().Add(5 * time.Second) for time.Now().Before(timeout) && !routeUp { // We know that the server is local and listening to // all IPv6 interfaces. Try connect using IPv6 loopback. if conn, err := net.Dial("tcp", "[::1]:6222"); err != nil { // Travis seem to have the server actually listening to 0.0.0.0, // so try with 127.0.0.1 if conn, err := net.Dial("tcp", "127.0.0.1:6222"); err != nil { time.Sleep(time.Second) continue } else { conn.Close() } } else { conn.Close() } routeUp = true } if !routeUp { t.Fatal("Server failed to start route accept loop") } } func TestClientConnectToRoutePort(t *testing.T) { opts := DefaultOptions() // Since client will first connect to the route listen port, set the // cluster's Host to 127.0.0.1 so it works on Windows too, since on // Windows, a client can't use 0.0.0.0 in a connect. opts.Cluster.Host = "127.0.0.1" s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, s.ClusterAddr().Port) clientURL := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // When connecting to the ROUTE port, the client library will receive the // CLIENT port in the INFO protocol. This URL is added to the client's pool // and will be tried after the initial connect failure. So all those // nats.Connect() should succeed. // The only reason for a failure would be if there are too many FDs in time-wait // which would delay the creation of TCP connection. So keep the total of // attempts rather small. total := 10 for i := 0; i < total; i++ { nc, err := nats.Connect(url) if err != nil { t.Fatalf("Unexpected error on connect: %v", err) } defer nc.Close() if nc.ConnectedUrl() != clientURL { t.Fatalf("Expected client to be connected to %v, got %v", clientURL, nc.ConnectedUrl()) } } s.Shutdown() // Try again with NoAdvertise and this time, the client should fail to connect. opts.Cluster.NoAdvertise = true s = RunServer(opts) defer s.Shutdown() for i := 0; i < total; i++ { nc, err := nats.Connect(url) if err == nil { nc.Close() t.Fatal("Expected error on connect, got none") } } } type checkDuplicateRouteLogger struct { sync.Mutex gotDuplicate bool } func (l *checkDuplicateRouteLogger) Noticef(format string, v ...any) {} func (l *checkDuplicateRouteLogger) Errorf(format string, v ...any) {} func (l *checkDuplicateRouteLogger) Warnf(format string, v ...any) {} func (l *checkDuplicateRouteLogger) Fatalf(format string, v ...any) {} func (l *checkDuplicateRouteLogger) Tracef(format string, v ...any) {} func (l *checkDuplicateRouteLogger) Debugf(format string, v ...any) { l.Lock() defer l.Unlock() msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "duplicate remote route") { l.gotDuplicate = true } } func TestRoutesToEachOther(t *testing.T) { optsA := DefaultOptions() optsA.Cluster.Port = 7246 optsA.Routes = RoutesFromStr("nats://127.0.0.1:7247") optsB := DefaultOptions() optsB.Cluster.Port = 7247 optsB.Routes = RoutesFromStr("nats://127.0.0.1:7246") srvALogger := &checkDuplicateRouteLogger{} srvA := New(optsA) srvA.SetLogger(srvALogger, true, false) defer srvA.Shutdown() srvBLogger := &checkDuplicateRouteLogger{} srvB := New(optsB) srvB.SetLogger(srvBLogger, true, false) defer srvB.Shutdown() go srvA.Start() go srvB.Start() start := time.Now() checkClusterFormed(t, srvA, srvB) end := time.Now() srvALogger.Lock() gotIt := srvALogger.gotDuplicate srvALogger.Unlock() if !gotIt { srvBLogger.Lock() gotIt = srvBLogger.gotDuplicate srvBLogger.Unlock() } if gotIt { dur := end.Sub(start) // It should not take too long to have a successful connection // between the 2 servers. if dur > 5*time.Second { t.Logf("Cluster formed, but took a long time: %v", dur) } } else { t.Log("Was not able to get duplicate route this time!") } } func wait(ch chan bool) error { select { case <-ch: return nil case <-time.After(5 * time.Second): } return fmt.Errorf("timeout") } func TestServerPoolUpdatedWhenRouteGoesAway(t *testing.T) { s1Opts := DefaultOptions() s1Opts.ServerName = "A" s1Opts.Host = "127.0.0.1" s1Opts.Port = 4222 s1Opts.Cluster.Host = "127.0.0.1" s1Opts.Cluster.Port = 6222 s1Opts.Routes = RoutesFromStr("nats://127.0.0.1:6223,nats://127.0.0.1:6224") s1 := RunServer(s1Opts) defer s1.Shutdown() s1Url := "nats://127.0.0.1:4222" s2Url := "nats://127.0.0.1:4223" s3Url := "nats://127.0.0.1:4224" ch := make(chan bool, 1) chch := make(chan bool, 1) connHandler := func(_ *nats.Conn) { chch <- true } nc, err := nats.Connect(s1Url, nats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(connHandler), nats.DiscoveredServersHandler(func(_ *nats.Conn) { ch <- true })) if err != nil { t.Fatalf("Error on connect") } defer nc.Close() s2Opts := DefaultOptions() s2Opts.ServerName = "B" s2Opts.Host = "127.0.0.1" s2Opts.Port = s1Opts.Port + 1 s2Opts.Cluster.Host = "127.0.0.1" s2Opts.Cluster.Port = 6223 s2Opts.Routes = RoutesFromStr("nats://127.0.0.1:6222,nats://127.0.0.1:6224") s2 := RunServer(s2Opts) defer s2.Shutdown() // Wait to be notified if err := wait(ch); err != nil { t.Fatal("New server callback was not invoked") } checkPool := func(expected []string) { t.Helper() // Don't use discovered here, but Servers to have the full list. // Also, there may be cases where the mesh is not formed yet, // so try again on failure. checkFor(t, 5*time.Second, 50*time.Millisecond, func() error { ds := nc.Servers() if len(ds) == len(expected) { m := make(map[string]struct{}, len(ds)) for _, url := range ds { m[url] = struct{}{} } ok := true for _, url := range expected { if _, present := m[url]; !present { ok = false break } } if ok { return nil } } return fmt.Errorf("Expected %v, got %v", expected, ds) }) } // Verify that we now know about s2 checkPool([]string{s1Url, s2Url}) s3Opts := DefaultOptions() s3Opts.ServerName = "C" s3Opts.Host = "127.0.0.1" s3Opts.Port = s2Opts.Port + 1 s3Opts.Cluster.Host = "127.0.0.1" s3Opts.Cluster.Port = 6224 s3Opts.Routes = RoutesFromStr("nats://127.0.0.1:6222,nats://127.0.0.1:6223") s3 := RunServer(s3Opts) defer s3.Shutdown() // Wait to be notified if err := wait(ch); err != nil { t.Fatal("New server callback was not invoked") } // Verify that we now know about s3 checkPool([]string{s1Url, s2Url, s3Url}) // Stop s1. Since this was passed to the Connect() call, this one should // still be present. s1.Shutdown() // Wait for reconnect if err := wait(chch); err != nil { t.Fatal("Reconnect handler not invoked") } checkPool([]string{s1Url, s2Url, s3Url}) // Check the server we reconnected to. reConnectedTo := nc.ConnectedUrl() expected := []string{s1Url} if reConnectedTo == s2Url { s2.Shutdown() expected = append(expected, s3Url) } else if reConnectedTo == s3Url { s3.Shutdown() expected = append(expected, s2Url) } else { t.Fatalf("Unexpected server client has reconnected to: %v", reConnectedTo) } // Wait for reconnect if err := wait(chch); err != nil { t.Fatal("Reconnect handler not invoked") } // The implicit server that we just shutdown should have been removed from the pool checkPool(expected) nc.Close() } func TestRouteFailedConnRemovedFromTmpMap(t *testing.T) { for _, test := range []struct { name string poolSize int }{ {"no pooling", -1}, {"pool 1", 1}, {"pool 3", 3}, } { t.Run(test.name, func(t *testing.T) { optsA, err := ProcessConfigFile("./configs/srv_a.conf") require_NoError(t, err) optsA.NoSigs, optsA.NoLog = true, true optsA.Cluster.PoolSize = test.poolSize optsB, err := ProcessConfigFile("./configs/srv_b.conf") require_NoError(t, err) optsB.NoSigs, optsB.NoLog = true, true optsB.Cluster.PoolSize = test.poolSize srvA := New(optsA) defer srvA.Shutdown() srvB := New(optsB) defer srvB.Shutdown() // Start this way to increase chance of having the two connect // to each other at the same time. This will cause one of the // route to be dropped. wg := &sync.WaitGroup{} wg.Add(2) go func() { srvA.Start() wg.Done() }() go func() { srvB.Start() wg.Done() }() checkClusterFormed(t, srvA, srvB) // Ensure that maps are empty checkMap := func(s *Server) { checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { s.grMu.Lock() l := len(s.grTmpClients) s.grMu.Unlock() if l != 0 { return fmt.Errorf("grTmpClients map should be empty, got %v", l) } return nil }) } checkMap(srvA) checkMap(srvB) srvB.Shutdown() srvA.Shutdown() wg.Wait() }) } } func getFirstRoute(s *Server) *client { for _, conns := range s.routes { for _, r := range conns { if r != nil { return r } } } return nil } func TestRoutePermsAppliedOnInboundAndOutboundRoute(t *testing.T) { perms := &RoutePermissions{ Import: &SubjectPermission{ Allow: []string{"imp.foo"}, Deny: []string{"imp.bar"}, }, Export: &SubjectPermission{ Allow: []string{"exp.foo"}, Deny: []string{"exp.bar"}, }, } optsA, err := ProcessConfigFile("./configs/seed.conf") require_NoError(t, err) optsA.NoLog = true optsA.NoSigs = true optsA.Cluster.Permissions = perms srva := RunServer(optsA) defer srva.Shutdown() optsB := DefaultOptions() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) srvb := RunServer(optsB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Ensure permission is properly set check := func(t *testing.T, s *Server) { t.Helper() var route *client s.mu.Lock() route = getFirstRoute(s) s.mu.Unlock() route.mu.Lock() perms := route.perms route.mu.Unlock() if perms == nil { t.Fatal("Expected perms to be set") } if perms.pub.allow == nil || perms.pub.allow.Count() != 1 { t.Fatal("unexpected pub allow perms") } if r := perms.pub.allow.Match("imp.foo"); len(r.psubs) != 1 { t.Fatal("unexpected pub allow match") } if perms.pub.deny == nil || perms.pub.deny.Count() != 1 { t.Fatal("unexpected pub deny perms") } if r := perms.pub.deny.Match("imp.bar"); len(r.psubs) != 1 { t.Fatal("unexpected pub deny match") } if perms.sub.allow == nil || perms.sub.allow.Count() != 1 { t.Fatal("unexpected sub allow perms") } if r := perms.sub.allow.Match("exp.foo"); len(r.psubs) != 1 { t.Fatal("unexpected sub allow match") } if perms.sub.deny == nil || perms.sub.deny.Count() != 1 { t.Fatal("unexpected sub deny perms") } if r := perms.sub.deny.Match("exp.bar"); len(r.psubs) != 1 { t.Fatal("unexpected sub deny match") } } // First check when permissions are set on the server accepting the route connection check(t, srva) srvb.Shutdown() srva.Shutdown() optsA.Cluster.Permissions = nil optsB.Cluster.Permissions = perms srva = RunServer(optsA) defer srva.Shutdown() srvb = RunServer(optsB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) // Now check for permissions set on server initiating the route connection check(t, srvb) } func TestRouteSendLocalSubsWithLowMaxPending(t *testing.T) { optsA := DefaultOptions() optsA.MaxPayload = 1024 optsA.MaxPending = 1024 optsA.NoSystemAccount = true srvA := RunServer(optsA) defer srvA.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() numSubs := 1000 for i := 0; i < numSubs; i++ { subj := fmt.Sprintf("fo.bar.%d", i) nc.Subscribe(subj, func(_ *nats.Msg) {}) } checkExpectedSubs(t, numSubs, srvA) // Now create a route between B and A optsB := DefaultOptions() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsA.Cluster.Host, optsA.Cluster.Port)) optsB.NoSystemAccount = true srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) // Check that all subs have been sent ok checkExpectedSubs(t, numSubs, srvA, srvB) } func TestRouteNoCrashOnAddingSubToRoute(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() numRoutes := routeTargetInit + 2 total := int32(numRoutes) count := int32(0) ch := make(chan bool, 1) cb := func(_ *nats.Msg) { if n := atomic.AddInt32(&count, 1); n == total { ch <- true } } var servers []*Server servers = append(servers, s) seedURL := fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, opts.Cluster.Port) for i := 0; i < numRoutes; i++ { ropts := DefaultOptions() ropts.Routes = RoutesFromStr(seedURL) rs := RunServer(ropts) defer rs.Shutdown() servers = append(servers, rs) // Confirm routes are active before clients connect. for _, srv := range servers { rz, err := srv.Routez(nil) require_NoError(t, err) for i, route := range rz.Routes { if route.LastActivity.IsZero() { t.Errorf("Expected LastActivity to be valid (%d)", i) } } } // Create a sub on each routed server. nc := natsConnect(t, fmt.Sprintf("nats://%s:%d", ropts.Host, ropts.Port)) defer nc.Close() natsSub(t, nc, "foo", cb) } checkClusterFormed(t, servers...) // Make sure all subs are registered in s. gacc := s.globalAccount() gacc.mu.RLock() sl := gacc.sl gacc.mu.RUnlock() checkFor(t, time.Second, 15*time.Millisecond, func() error { var _subs [64]*subscription subs := _subs[:0] sl.All(&subs) var ts int for _, sub := range subs { if string(sub.subject) == "foo" { ts++ } } if ts != int(numRoutes) { return fmt.Errorf("Not all %d routed subs were registered: %d", numRoutes, ts) } return nil }) pubNC := natsConnect(t, fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)) defer pubNC.Close() natsPub(t, pubNC, "foo", []byte("hello world!")) waitCh(t, ch, "Did not get all messages") } func TestRouteRTT(t *testing.T) { ob := DefaultOptions() ob.PingInterval = 15 * time.Millisecond sb := RunServer(ob) defer sb.Shutdown() oa := DefaultOptions() oa.PingInterval = 15 * time.Millisecond oa.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", ob.Cluster.Host, ob.Cluster.Port)) sa := RunServer(oa) defer sa.Shutdown() checkClusterFormed(t, sa, sb) checkRTT := func(t *testing.T, s *Server) time.Duration { t.Helper() var route *client s.mu.Lock() route = getFirstRoute(s) s.mu.Unlock() var rtt time.Duration checkFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error { route.mu.Lock() rtt = route.rtt route.mu.Unlock() if rtt == 0 { return fmt.Errorf("RTT not tracked") } return nil }) return rtt } prevA := checkRTT(t, sa) prevB := checkRTT(t, sb) checkUpdated := func(t *testing.T, s *Server, prev time.Duration) { t.Helper() attempts := 0 timeout := time.Now().Add(2 * firstPingInterval) for time.Now().Before(timeout) { if rtt := checkRTT(t, s); rtt != prev { return } attempts++ if attempts == 5 { // If could be that we are very unlucky // and the RTT is constant. So override // the route's RTT to 0 to see if it gets // updated. s.mu.Lock() if r := getFirstRoute(s); r != nil { r.mu.Lock() r.rtt = 0 r.mu.Unlock() } s.mu.Unlock() } time.Sleep(15 * time.Millisecond) } t.Fatalf("RTT probably not updated") } checkUpdated(t, sa, prevA) checkUpdated(t, sb, prevB) sa.Shutdown() sb.Shutdown() // Now check that initial RTT is computed prior to first PingInterval // Get new options to avoid possible race changing the ping interval. ob = DefaultOptions() ob.PingInterval = time.Minute sb = RunServer(ob) defer sb.Shutdown() oa = DefaultOptions() oa.PingInterval = time.Minute oa.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", ob.Cluster.Host, ob.Cluster.Port)) sa = RunServer(oa) defer sa.Shutdown() checkClusterFormed(t, sa, sb) checkRTT(t, sa) checkRTT(t, sb) } func TestRouteCloseTLSConnection(t *testing.T) { opts := DefaultOptions() opts.DisableShortFirstPing = true opts.Cluster.Name = "A" opts.Cluster.Host = "127.0.0.1" opts.Cluster.Port = -1 opts.Cluster.TLSTimeout = 100 tc := &TLSConfigOpts{ CertFile: "./configs/certs/server.pem", KeyFile: "./configs/certs/key.pem", Insecure: true, } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } opts.Cluster.TLSConfig = tlsConf opts.NoLog = true opts.NoSigs = true s := RunServer(opts) defer s.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Cluster.Host, opts.Cluster.Port) conn, err := net.DialTimeout("tcp", endpoint, 2*time.Second) if err != nil { t.Fatalf("Unexpected error on dial: %v", err) } defer conn.Close() tlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true}) defer tlsConn.Close() if err := tlsConn.Handshake(); err != nil { t.Fatalf("Unexpected error during handshake: %v", err) } connectOp := []byte("CONNECT {\"name\":\"route\",\"verbose\":false,\"pedantic\":false,\"tls_required\":true,\"cluster\":\"A\"}\r\n") if _, err := tlsConn.Write(connectOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } infoOp := []byte("INFO {\"server_id\":\"route\",\"tls_required\":true}\r\n") if _, err := tlsConn.Write(infoOp); err != nil { t.Fatalf("Unexpected error writing CONNECT: %v", err) } if _, err := tlsConn.Write([]byte("PING\r\n")); err != nil { t.Fatalf("Unexpected error writing PING: %v", err) } checkFor(t, time.Second, 15*time.Millisecond, func() error { if s.NumRoutes() != 1 { return fmt.Errorf("No route registered yet") } return nil }) // Get route connection var route *client s.mu.Lock() route = getFirstRoute(s) s.mu.Unlock() // Fill the buffer. We want to timeout on write so that nc.Close() // would block due to a write that cannot complete. buf := make([]byte, 64*1024) done := false for !done { route.nc.SetWriteDeadline(time.Now().Add(time.Second)) if _, err := route.nc.Write(buf); err != nil { done = true } route.nc.SetWriteDeadline(time.Time{}) } ch := make(chan bool) go func() { select { case <-ch: return case <-time.After(3 * time.Second): fmt.Println("!!!! closeConnection is blocked, test will hang !!!") return } }() // Close the route route.closeConnection(SlowConsumerWriteDeadline) ch <- true } func TestRouteClusterNameConflictBetweenStaticAndDynamic(t *testing.T) { o1 := DefaultOptions() o1.Cluster.Name = "AAAAAAAAAAAAAAAAAAAA" // make it alphabetically the "smallest" s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.Cluster.Name = "" // intentional, let it be assigned dynamically o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) } type testRouteResolver struct{} func (r *testRouteResolver) LookupHost(ctx context.Context, host string) ([]string, error) { return []string{"127.0.0.1", "other.host.in.cluster"}, nil } type routeHostLookupLogger struct { DummyLogger errCh chan string ch chan bool count int } func (l *routeHostLookupLogger) Debugf(format string, v ...any) { l.Lock() defer l.Unlock() msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "127.0.0.1:1234") { l.errCh <- msg } else if strings.Contains(msg, "other.host.in.cluster") { if l.count++; l.count == 10 { l.ch <- true } } } func TestRouteIPResolutionAndRouteToSelf(t *testing.T) { o := DefaultOptions() o.Cluster.Port = 1234 r := &testRouteResolver{} o.Cluster.resolver = r o.Routes = RoutesFromStr("nats://routehost:1234") o.Debug = true o.NoLog = false s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } defer s.Shutdown() l := &routeHostLookupLogger{errCh: make(chan string, 1), ch: make(chan bool, 1)} s.SetLogger(l, true, true) s.Start() if err := s.readyForConnections(time.Second); err != nil { t.Fatal(err) } select { case e := <-l.errCh: t.Fatalf("Unexpected trace: %q", e) case <-l.ch: // Ok return } } func TestRouteDuplicateServerName(t *testing.T) { o := DefaultOptions() o.ServerName = "A" s := RunServer(o) defer s.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 1)} s.SetLogger(l, false, false) o2 := DefaultOptions() // Set the same server name on purpose o2.ServerName = "A" o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() // This is an error now so can't wait on cluster formed. select { case w := <-l.errCh: if !strings.Contains(w, "Remote server has a duplicate name") { t.Fatalf("Expected warning about same name, got %q", w) } case <-time.After(5 * time.Second): t.Fatal("Should have gotten a warning regarding duplicate server name") } } func TestRouteLockReleasedOnTLSFailure(t *testing.T) { o1 := DefaultOptions() o1.Cluster.Name = "abc" o1.Cluster.Host = "127.0.0.1" o1.Cluster.Port = -1 o1.Cluster.TLSTimeout = 0.25 tc := &TLSConfigOpts{ CertFile: "./configs/certs/server.pem", KeyFile: "./configs/certs/key.pem", Insecure: true, } tlsConf, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } o1.Cluster.TLSConfig = tlsConf s1 := RunServer(o1) defer s1.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} s1.SetLogger(l, false, false) o2 := DefaultOptions() o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() select { case err := <-l.errCh: if !strings.Contains(err, "TLS") { t.Fatalf("Unexpected error: %v", err) } case <-time.After(time.Second): } s2.Shutdown() // Wait for longer than the TLS timeout and check that tlsTimeout is not stuck time.Sleep(500 * time.Millisecond) buf := make([]byte, 10000) n := runtime.Stack(buf, true) if bytes.Contains(buf[:n], []byte("tlsTimeout")) { t.Fatal("Seem connection lock was not released") } } type localhostResolver struct{} func (r *localhostResolver) LookupHost(ctx context.Context, host string) ([]string, error) { return []string{"127.0.0.1"}, nil } func TestTLSRoutesCertificateImplicitAllowPass(t *testing.T) { testTLSRoutesCertificateImplicitAllow(t, true) } func TestTLSRoutesCertificateImplicitAllowFail(t *testing.T) { testTLSRoutesCertificateImplicitAllow(t, false) } func testTLSRoutesCertificateImplicitAllow(t *testing.T, pass bool) { // Base config for the servers cfg := createTempFile(t, "cfg") cfg.WriteString(fmt.Sprintf(` cluster { tls { cert_file = "../test/configs/certs/tlsauth/server.pem" key_file = "../test/configs/certs/tlsauth/server-key.pem" ca_file = "../test/configs/certs/tlsauth/ca.pem" verify_cert_and_check_known_urls = true insecure = %t timeout = 1 } } `, !pass)) // set insecure to skip verification on the outgoing end if err := cfg.Sync(); err != nil { t.Fatal(err) } cfg.Close() optsA := LoadConfig(cfg.Name()) optsB := LoadConfig(cfg.Name()) routeURLs := "nats://localhost:9935, nats://localhost:9936" if !pass { routeURLs = "nats://127.0.0.1:9935, nats://127.0.0.1:9936" } optsA.Host = "127.0.0.1" optsA.Port = 9335 optsA.Cluster.Name = "xyz" optsA.Cluster.Host = optsA.Host optsA.Cluster.Port = 9935 optsA.Cluster.resolver = &localhostResolver{} optsA.Routes = RoutesFromStr(routeURLs) optsA.NoSystemAccount = true srvA := RunServer(optsA) defer srvA.Shutdown() optsB.Host = "127.0.0.1" optsB.Port = 9336 optsB.Cluster.Name = "xyz" optsB.Cluster.Host = optsB.Host optsB.Cluster.Port = 9936 optsB.Cluster.resolver = &localhostResolver{} optsB.Routes = RoutesFromStr(routeURLs) optsB.NoSystemAccount = true srvB := RunServer(optsB) defer srvB.Shutdown() if pass { checkNumRoutes(t, srvA, DEFAULT_ROUTE_POOL_SIZE) checkNumRoutes(t, srvB, DEFAULT_ROUTE_POOL_SIZE) } else { time.Sleep(1 * time.Second) // the fail case uses the IP, so a short wait is sufficient checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if srvA.NumRoutes() != 0 || srvB.NumRoutes() != 0 { return fmt.Errorf("No route connection expected") } return nil }) } } func TestSubjectRenameViaJetStreamAck(t *testing.T) { s := RunRandClientPortServer(t) defer s.Shutdown() errChan := make(chan error) defer close(errChan) ncPub := natsConnect(t, s.ClientURL(), nats.UserInfo("client", "pwd"), nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) { errChan <- err })) defer ncPub.Close() require_NoError(t, ncPub.PublishRequest("SVC.ALLOWED", "$JS.ACK.whatever@ADMIN", nil)) select { case err := <-errChan: require_Contains(t, err.Error(), "Permissions Violation for Publish with Reply of") case <-time.After(time.Second): t.Fatalf("Expected error") } } func TestClusterQueueGroupWeightTrackingLeak(t *testing.T) { o := DefaultOptions() o.ServerName = "A" s := RunServer(o) defer s.Shutdown() o2 := DefaultOptions() o2.ServerName = "B" o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() nc := natsConnect(t, s.ClientURL()) defer nc.Close() // Create a queue subscription sub := natsQueueSubSync(t, nc, "foo", "bar") // Check on s0 that we have the proper queue weight info acc := s.GlobalAccount() check := func(present bool, expected int32) { t.Helper() sub := subscription{subject: []byte("foo"), queue: []byte("bar")} key := keyFromSubWithOrigin(&sub) checkFor(t, time.Second, 15*time.Millisecond, func() error { acc.mu.RLock() v, ok := acc.lqws[key] acc.mu.RUnlock() if present { if !ok { return fmt.Errorf("the key is not present") } if v != expected { return fmt.Errorf("lqws doest not contain expected value of %v: %v", expected, v) } } else if ok { return fmt.Errorf("the key is present with value %v and should not be", v) } return nil }) } check(true, 1) // Now unsub, and it should be removed, not just be 0 sub.Unsubscribe() check(false, 0) // Still make sure that the subject interest is gone from both servers. checkSubGone := func(s *Server) { t.Helper() checkFor(t, time.Second, 15*time.Millisecond, func() error { acc := s.GlobalAccount() acc.mu.RLock() res := acc.sl.Match("foo") acc.mu.RUnlock() if res != nil && len(res.qsubs) > 0 { return fmt.Errorf("Found queue sub on foo for server %v", s) } return nil }) } checkSubGone(s) checkSubGone(s2) } type testRouteReconnectLogger struct { DummyLogger ch chan string } func (l *testRouteReconnectLogger) Debugf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "Trying to connect to route") { select { case l.ch <- msg: default: } } } func TestRouteSolicitedReconnectsEvenIfImplicit(t *testing.T) { o1 := DefaultOptions() o1.ServerName = "A" s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.ServerName = "B" o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) // Not strictly required to reconnect, but if the reconnect were to fail for any reason // then the server would retry only once and then stops. So set it to some higher value // and then we will check that the server does not try more than that. o2.Cluster.ConnectRetries = 3 s2 := RunServer(o2) defer s2.Shutdown() o3 := DefaultOptions() o3.ServerName = "C" o3.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o3.Cluster.ConnectRetries = 3 s3 := RunServer(o3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) s2.mu.Lock() s2.forEachRoute(func(r *client) { r.mu.Lock() // Close the route between S2 and S3 (that do not have explicit route to each other) if r.route.remoteID == s3.ID() { r.nc.Close() } r.mu.Unlock() }) s2.mu.Unlock() // Wait a bit to make sure that we don't check for cluster formed too soon (need to make // sure that connection is really removed and reconnect mechanism starts). time.Sleep(500 * time.Millisecond) checkClusterFormed(t, s1, s2, s3) // Now shutdown server 3 and make sure that s2 stops trying to reconnect to s3 at one point l := &testRouteReconnectLogger{ch: make(chan string, 10)} s2.SetLogger(l, true, false) s3.Shutdown() // S2 should retry ConnectRetries+1 times and then stop // Take into account default route pool size and system account dedicated route for i := 0; i < (DEFAULT_ROUTE_POOL_SIZE+1)*(o2.Cluster.ConnectRetries+1); i++ { select { case <-l.ch: case <-time.After(2 * time.Second): t.Fatal("Did not attempt to reconnect") } } // Now it should have stopped (in tests, reconnect delay is down to 15ms, so we don't need // to wait for too long). select { case msg := <-l.ch: t.Fatalf("Unexpected attempt to reconnect: %s", msg) case <-time.After(50 * time.Millisecond): // OK } } func TestRouteSaveTLSName(t *testing.T) { c1Conf := createConfFile(t, []byte(` port: -1 cluster { name: "abc" port: -1 pool_size: -1 tls { cert_file: '../test/configs/certs/server-noip.pem' key_file: '../test/configs/certs/server-key-noip.pem' ca_file: '../test/configs/certs/ca.pem' } } `)) s1, o1 := RunServerWithConfig(c1Conf) defer s1.Shutdown() tmpl := ` port: -1 cluster { name: "abc" port: -1 pool_size: -1 routes: ["nats://%s:%d"] tls { cert_file: '../test/configs/certs/server-noip.pem' key_file: '../test/configs/certs/server-key-noip.pem' ca_file: '../test/configs/certs/ca.pem' } } ` c2And3Conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "localhost", o1.Cluster.Port))) s2, _ := RunServerWithConfig(c2And3Conf) defer s2.Shutdown() checkClusterFormed(t, s1, s2) s3, _ := RunServerWithConfig(c2And3Conf) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) reloadUpdateConfig(t, s2, c2And3Conf, fmt.Sprintf(tmpl, "127.0.0.1", o1.Cluster.Port)) s2.mu.RLock() s2.forEachRoute(func(r *client) { r.mu.Lock() if r.route.routeType == Implicit { r.nc.Close() } r.mu.Unlock() }) s2.mu.RUnlock() checkClusterFormed(t, s1, s2, s3) // Set a logger to capture errors trying to connect after clearing // the routeTLSName and causing a disconnect l := &captureErrorLogger{errCh: make(chan string, 1)} s2.SetLogger(l, false, false) var gotIt bool for i := 0; !gotIt && i < 5; i++ { s2.mu.Lock() s2.routeTLSName = _EMPTY_ s2.forEachRoute(func(r *client) { r.mu.Lock() if r.route.routeType == Implicit { r.nc.Close() } r.mu.Unlock() }) s2.mu.Unlock() select { case <-l.errCh: gotIt = true case <-time.After(time.Second): // Try again } } if !gotIt { t.Fatal("Did not get the handshake error") } // Now get back to localhost in config and reload config and // it should start to work again. reloadUpdateConfig(t, s2, c2And3Conf, fmt.Sprintf(tmpl, "localhost", o1.Cluster.Port)) checkClusterFormed(t, s1, s2, s3) } func TestRoutePoolAndPerAccountErrors(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 cluster { port: -1 accounts: ["abc", "def", "abc"] } `)) o := LoadConfig(conf) if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), "duplicate") { t.Fatalf("Expected error about duplicate, got %v", err) } conf1 := createConfFile(t, []byte(` port: -1 accounts { abc { users: [{user:abc, password: pwd}] } def { users: [{user:def, password: pwd}] } } cluster { port: -1 name: "local" accounts: ["abc"] } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() l := &captureErrorLogger{errCh: make(chan string, 10)} s1.SetLogger(l, false, false) conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 accounts { abc { users: [{user:abc, password: pwd}] } def { users: [{user:def, password: pwd}] } } cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] accounts: ["def"] } `, o1.Cluster.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() for i := 0; i < 2; i++ { select { case e := <-l.errCh: if !strings.Contains(e, "No route for account \"def\"") { t.Fatalf("Expected error about no route for account, got %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Did not get expected error regarding no route for account") } time.Sleep(DEFAULT_ROUTE_RECONNECT + 100*time.Millisecond) } s2.Shutdown() s1.Shutdown() conf1 = createConfFile(t, []byte(` port: -1 cluster { port: -1 name: "local" pool_size: 5 } `)) s1, o1 = RunServerWithConfig(conf1) defer s1.Shutdown() l = &captureErrorLogger{errCh: make(chan string, 10)} s1.SetLogger(l, false, false) conf2 = createConfFile(t, []byte(fmt.Sprintf(` port: -1 cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] pool_size: 3 } `, o1.Cluster.Port))) s2, _ = RunServerWithConfig(conf2) defer s2.Shutdown() for i := 0; i < 2; i++ { select { case e := <-l.errCh: if !strings.Contains(e, "Mismatch route pool size") { t.Fatalf("Expected error about pool size mismatch, got %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Did not get expected error regarding mismatch pool size") } time.Sleep(DEFAULT_ROUTE_RECONNECT + 100*time.Millisecond) } } func TestRoutePool(t *testing.T) { tmpl := ` port: -1 accounts { A { users: [{user: "a", password: "a"}] } B { users: [{user: "b", password: "b"}] } } cluster { port: -1 name: "local" %s pool_size: 2 } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) checkRoutePoolIdx := func(s *Server, accName string, expected int) { t.Helper() a, err := s.LookupAccount(accName) require_NoError(t, err) require_True(t, a != nil) a.mu.RLock() rpi := a.routePoolIdx a.mu.RUnlock() require_True(t, rpi == expected) } checkRoutePoolIdx(s1, "A", 0) checkRoutePoolIdx(s2, "A", 0) checkRoutePoolIdx(s2, "B", 1) checkRoutePoolIdx(s2, "B", 1) sendAndRecv := func(acc, user, pwd string) { t.Helper() s2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, pwd)) defer s2nc.Close() sub := natsSubSync(t, s2nc, "foo") natsFlush(t, s2nc) s1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, pwd)) defer s1nc.Close() checkSubInterest(t, s1, acc, "foo", time.Second) for i := 0; i < 1000; i++ { natsPub(t, s1nc, "foo", []byte("hello")) } for i := 0; i < 1000; i++ { natsNexMsg(t, sub, time.Second) } // Make sure we don't receive more if msg, err := sub.NextMsg(150 * time.Millisecond); err == nil { t.Fatalf("Unexpected message: %+v", msg) } } sendAndRecv("A", "a", "a") sendAndRecv("B", "b", "b") checkStats := func(s *Server, isOut bool) { t.Helper() s.mu.RLock() defer s.mu.RUnlock() for _, conns := range s.routes { for i, r := range conns { r.mu.Lock() if isOut { if v := r.stats.outMsgs; v < 1000 { r.mu.Unlock() t.Fatalf("Expected at least 1000 in out msgs for route %v, got %v", i+1, v) } } else { if v := r.stats.inMsgs; v < 1000 { r.mu.Unlock() t.Fatalf("Expected at least 1000 in in msgs for route %v, got %v", i+1, v) } } r.mu.Unlock() } } } checkStats(s1, true) checkStats(s2, false) disconnectRoute := func(s *Server, idx int) { t.Helper() attempts := 0 TRY_AGAIN: s.mu.RLock() for _, conns := range s.routes { for i, r := range conns { if i != idx { continue } if r != nil { r.mu.Lock() nc := r.nc r.mu.Unlock() if nc == nil { s.mu.RUnlock() if attempts++; attempts < 10 { time.Sleep(250 * time.Millisecond) goto TRY_AGAIN } t.Fatalf("Route %v net.Conn is nil", i) } nc.Close() } else { s.mu.RUnlock() if attempts++; attempts < 10 { time.Sleep(250 * time.Millisecond) goto TRY_AGAIN } t.Fatalf("Route %v connection is nil", i) } } } s.mu.RUnlock() time.Sleep(250 * time.Millisecond) checkClusterFormed(t, s1, s2) } disconnectRoute(s1, 0) disconnectRoute(s2, 1) } func TestRoutePoolConnectRace(t *testing.T) { for _, test := range []struct { name string poolSize int }{ {"no pool", -1}, {"pool size 1", 1}, {"pool size 5", 5}, } { t.Run(test.name, func(t *testing.T) { // This test will have each server point to each other and that is causing // each one to attempt to connect routes to each other which should lead // to connections needing to be dropped. We make sure that there is still // resolution and there is the expected number of routes. createSrv := func(name string, port int) *Server { o := DefaultOptions() o.Port = -1 o.ServerName = name o.Cluster.PoolSize = test.poolSize o.Cluster.Name = "local" o.Cluster.Port = port o.Routes = RoutesFromStr("nats://127.0.0.1:1234,nats://127.0.0.1:1235,nats://127.0.0.1:1236") s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } return s } s1 := createSrv("A", 1234) s2 := createSrv("B", 1235) s3 := createSrv("C", 1236) l := &captureDebugLogger{dbgCh: make(chan string, 100)} s1.SetLogger(l, true, false) servers := []*Server{s1, s2, s3} for _, s := range servers { go s.Start() defer s.Shutdown() } checkClusterFormed(t, s1, s2, s3) for done, duplicate := false, 0; !done; { select { case e := <-l.dbgCh: if strings.Contains(e, "duplicate") { if duplicate++; duplicate > 20 { t.Fatalf("Routes are constantly reconnecting: %v", e) } } case <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond): // More than reconnect and some, and no reconnect, so we are good. done = true } } // Also, check that they all report as solicited and configured in monitoring. for _, s := range servers { routes, err := s.Routez(nil) require_NoError(t, err) for _, r := range routes.Routes { if !r.DidSolicit { t.Fatalf("All routes should have been marked as solicited, this one was not: %+v", r) } if !r.IsConfigured { t.Fatalf("All routes should have been marked as configured, this one was not: %+v", r) } } } for _, s := range servers { s.Shutdown() s.WaitForShutdown() } }) } } func TestRoutePoolRouteStoredSameIndexBothSides(t *testing.T) { tmpl := ` port: -1 accounts { A { users: [{user: "a", password: "a"}] } B { users: [{user: "b", password: "b"}] } C { users: [{user: "c", password: "c"}] } D { users: [{user: "d", password: "d"}] } } cluster { port: -1 name: "local" %s pool_size: 4 } no_sys_acc: true ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() for i := 0; i < 20; i++ { checkClusterFormed(t, s1, s2) collect := func(s *Server, checkRemoteAddr bool) []string { addrs := make([]string, 0, 4) s.mu.RLock() s.forEachRoute(func(r *client) { var addr string r.mu.Lock() if r.nc != nil { if checkRemoteAddr { addr = r.nc.RemoteAddr().String() } else { addr = r.nc.LocalAddr().String() } addrs = append(addrs, addr) } r.mu.Unlock() }) s.mu.RUnlock() return addrs } addrsS1 := collect(s1, true) addrsS2 := collect(s2, false) if len(addrsS1) != 4 || len(addrsS2) != 4 { // It could be that connections were not ready (r.nc is nil in collect()) // if that is the case, try again. i-- continue } if !reflect.DeepEqual(addrsS1, addrsS2) { t.Fatalf("Connections not stored at same index:\ns1=%v\ns2=%v", addrsS1, addrsS2) } s1.mu.RLock() s1.forEachRoute(func(r *client) { r.mu.Lock() if r.nc != nil { r.nc.Close() } r.mu.Unlock() }) s1.mu.RUnlock() } } type captureRMsgTrace struct { DummyLogger sync.Mutex traces *bytes.Buffer out []string } func (l *captureRMsgTrace) Tracef(format string, args ...any) { l.Lock() defer l.Unlock() msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "[RMSG ") { l.traces.WriteString(msg) l.out = append(l.out, msg) } } func TestRoutePerAccount(t *testing.T) { akp1, _ := nkeys.CreateAccount() acc1, _ := akp1.PublicKey() akp2, _ := nkeys.CreateAccount() acc2, _ := akp2.PublicKey() tmpl := ` port: -1 accounts { %s { users: [{user: "a", password: "a"}] } %s { users: [{user: "b", password: "b"}] } } cluster { port: -1 name: "local" %s accounts: ["%s"] } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, acc1, acc2, _EMPTY_, acc2))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, acc1, acc2, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), acc2))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() l := &captureRMsgTrace{traces: &bytes.Buffer{}} s2.SetLogger(l, false, true) checkClusterFormed(t, s1, s2) disconnectRoute := func(s *Server) { t.Helper() attempts := 0 TRY_AGAIN: s.mu.RLock() if conns, ok := s.accRoutes[acc2]; ok { for _, r := range conns { if r != nil { r.mu.Lock() nc := r.nc r.mu.Unlock() if nc == nil { s.mu.RUnlock() if attempts++; attempts < 10 { time.Sleep(250 * time.Millisecond) goto TRY_AGAIN } t.Fatal("Route net.Conn is nil") } nc.Close() } else { s.mu.RUnlock() if attempts++; attempts < 10 { time.Sleep(250 * time.Millisecond) goto TRY_AGAIN } t.Fatal("Route connection is nil") } } } s.mu.RUnlock() time.Sleep(250 * time.Millisecond) checkClusterFormed(t, s1, s2) } disconnectRoute(s1) disconnectRoute(s2) sendAndRecv := func(acc, user, pwd string) { t.Helper() s2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, pwd)) defer s2nc.Close() sub := natsSubSync(t, s2nc, "foo") natsFlush(t, s2nc) s1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, pwd)) defer s1nc.Close() checkSubInterest(t, s1, acc, "foo", time.Second) for i := 0; i < 10; i++ { natsPub(t, s1nc, "foo", []byte("hello")) } for i := 0; i < 10; i++ { natsNexMsg(t, sub, time.Second) } // Make sure we don't receive more if msg, err := sub.NextMsg(150 * time.Millisecond); err == nil { t.Fatalf("Unexpected message: %+v", msg) } } sendAndRecv(acc1, "a", "a") sendAndRecv(acc2, "b", "b") l.Lock() traces := l.traces.String() out := append([]string(nil), l.out...) l.Unlock() // We should not have any "[RMSG " if strings.Contains(traces, fmt.Sprintf("[RMSG %s", acc2)) { var outStr string for _, l := range out { outStr += l + "\r\n" } t.Fatalf("Should not have included account %q in protocol, got:\n%s", acc2, outStr) } } func TestRoutePerAccountImplicit(t *testing.T) { tmpl := ` port: -1 accounts { A { users: [{user: "a", password: "a"}] } B { users: [{user: "b", password: "b"}] } } cluster { port: -1 name: "local" accounts: ["A"] %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2And3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2And3) defer s2.Shutdown() checkClusterFormed(t, s1, s2) s3, _ := RunServerWithConfig(conf2And3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) // On s3, close the per-account routes from s2 s3.mu.RLock() for _, conns := range s3.accRoutes { for rem, r := range conns { if rem != s2.ID() { continue } r.mu.Lock() if r.nc != nil { r.nc.Close() } r.mu.Unlock() } } s3.mu.RUnlock() // Wait a bit to make sure there is a disconnect, then check the cluster is ok time.Sleep(250 * time.Millisecond) checkClusterFormed(t, s1, s2, s3) } func TestRoutePerAccountDefaultForSysAccount(t *testing.T) { tmpl := ` port: -1 accounts { A { users: [{user: "a", password: "a"}] } B { users: [{user: "b", password: "b"}] } } cluster { port: -1 name: "local" %s %s %s } %s ` for _, test := range []struct { name string accounts string sysAcc string noSysAcc bool }{ {"default sys no accounts", _EMPTY_, _EMPTY_, false}, {"default sys in accounts", "accounts: [\"$SYS\"]", _EMPTY_, false}, {"default sys with other accounts", "accounts: [\"A\",\"$SYS\"]", _EMPTY_, false}, {"explicit sys no accounts", _EMPTY_, "system_account: B", false}, {"explicit sys in accounts", "accounts: [\"B\"]", "system_account: B", false}, {"explicit sys with other accounts", "accounts: [\"B\",\"A\"]", "system_account: B", false}, {"no system account no accounts", _EMPTY_, "no_sys_acc: true", true}, {"no system account with accounts", "accounts: [\"A\"]", "no_sys_acc: true", true}, } { t.Run(test.name, func(t *testing.T) { conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.accounts, _EMPTY_, test.sysAcc))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.accounts, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), test.sysAcc))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) checkSysAccRoute := func(s *Server) { t.Helper() var name string acc := s.SystemAccount() if test.noSysAcc { if acc != nil { t.Fatalf("Should not be any system account, got %q", acc.GetName()) } // We will check that there is no accRoutes for the default // system account name name = DEFAULT_SYSTEM_ACCOUNT } else { acc.mu.RLock() pi := acc.routePoolIdx name = acc.Name acc.mu.RUnlock() if pi != -1 { t.Fatalf("System account %q should have route pool index==-1, got %v", name, pi) } } s.mu.RLock() _, ok := s.accRoutes[name] s.mu.RUnlock() if test.noSysAcc { if ok { t.Fatalf("System account %q should not have its own route, since NoSystemAccount was specified", name) } } else if !ok { t.Fatalf("System account %q should be present in accRoutes, it was not", name) } } checkSysAccRoute(s1) checkSysAccRoute(s2) // Check that this is still the case after a config reload reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "pool_size: 4", test.accounts, _EMPTY_, test.sysAcc)) reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "pool_size: 4", test.accounts, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), test.sysAcc)) checkSysAccRoute(s1) checkSysAccRoute(s2) }) } } func TestRoutePerAccountConnectRace(t *testing.T) { // This test will have each server point to each other and that is causing // each one to attempt to connect routes to each other which should lead // to connections needing to be dropped. We make sure that there is still // resolution and there is the expected number of routes. createSrv := func(name string, port int) *Server { o := DefaultOptions() o.Port = -1 o.ServerName = name o.Accounts = []*Account{NewAccount("A")} o.NoSystemAccount = true o.Cluster.PoolSize = 1 o.Cluster.PinnedAccounts = []string{"A"} o.Cluster.Name = "local" o.Cluster.Port = port o.Routes = RoutesFromStr("nats://127.0.0.1:1234,nats://127.0.0.1:1235,nats://127.0.0.1:1236") s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } return s } s1 := createSrv("A", 1234) s2 := createSrv("B", 1235) s3 := createSrv("C", 1236) l := &captureDebugLogger{dbgCh: make(chan string, 100)} s1.SetLogger(l, true, false) servers := []*Server{s1, s2, s3} for _, s := range servers { go s.Start() defer s.Shutdown() } checkClusterFormed(t, s1, s2, s3) for done, duplicate := false, 0; !done; { select { case e := <-l.dbgCh: if strings.Contains(e, "duplicate") { if duplicate++; duplicate > 10 { t.Fatalf("Routes are constantly reconnecting: %v", e) } } case <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond): // More than reconnect and some, and no reconnect, so we are good. done = true } } // Also, check that they all report as solicited and configured in monitoring. for _, s := range servers { routes, err := s.Routez(nil) require_NoError(t, err) for _, r := range routes.Routes { if !r.DidSolicit { t.Fatalf("All routes should have been marked as solicited, this one was not: %+v", r) } if !r.IsConfigured { t.Fatalf("All routes should have been marked as configured, this one was not: %+v", r) } } } for _, s := range servers { s.Shutdown() s.WaitForShutdown() } } func TestRoutePerAccountGossipWorks(t *testing.T) { tmplA := ` port: -1 server_name: "A" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: %d name: "local" accounts: ["A"] %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() tmplBC := ` port: -1 server_name: "%s" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" %s accounts: ["A"] } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Now connect s3 to s1 and make sure that s2 connects properly to s3. conf3 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, "C", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) s3.Shutdown() s2.Shutdown() s1.Shutdown() // Slightly different version where s2 is connecting to s1, while s1 // connects to s3 (and s3 does not solicit connections). conf1 = createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, _EMPTY_))) s1, o1 = RunServerWithConfig(conf1) defer s1.Shutdown() conf2 = createConfFile(t, []byte(fmt.Sprintf(tmplBC, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ = RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Start s3 first that will simply accept connections conf3 = createConfFile(t, []byte(fmt.Sprintf(tmplBC, "C", _EMPTY_))) s3, o3 := RunServerWithConfig(conf3) defer s3.Shutdown() // Now config reload s1 so that it points to s3. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmplA, o1.Cluster.Port, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o3.Cluster.Port))) checkClusterFormed(t, s1, s2, s3) } func TestRoutePerAccountGossipWorksWithOldServerNotSeed(t *testing.T) { tmplA := ` port: -1 server_name: "A" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: %d name: "local" accounts: ["A"] } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() // Here, server "B" will have no pooling/accounts. tmplB := ` port: -1 server_name: "B" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] pool_size: -1 } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmplB, o1.Cluster.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) l := &captureErrorLogger{errCh: make(chan string, 100)} s2.SetLogger(l, false, false) // Now connect s3 to s1. Server s1 should not gossip to s2 information // about pinned-account routes or extra routes from the pool. tmplC := ` port: -1 server_name: "C" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" routes: ["nats://127.0.0.1:%d"] accounts: ["A"] } ` conf3 := createConfFile(t, []byte(fmt.Sprintf(tmplC, o1.Cluster.Port))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) // We should not have had s2 try to create dedicated routes for "A" or "$SYS" tm := time.NewTimer(time.Second) defer tm.Stop() for { select { case err := <-l.errCh: if strings.Contains(err, "dedicated route") { t.Fatalf("Server s2 should not have tried to create a dedicated route: %s", err) } case <-tm.C: return } } } func TestRoutePerAccountGossipWorksWithOldServerSeed(t *testing.T) { tmplA := ` port: -1 server_name: "A" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: %d name: "local" pool_size: %d %s } ` // Start with s1 being an "old" server, which does not support pooling/pinned-accounts. conf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, -1, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() tmplBC := ` port: -1 server_name: "%s" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" pool_size: 3 routes: ["nats://127.0.0.1:%d"] accounts: ["A"] } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, "B", o1.Cluster.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Now connect s3 to s1 and make sure that s2 connects properly to s3. conf3 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, "C", o1.Cluster.Port))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) checkRoutes := func(s *Server, expected int) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { if nr := s.NumRoutes(); nr != expected { return fmt.Errorf("Server %q should have %v routes, got %v", s.Name(), expected, nr) } return nil }) } // Since s1 has no pooling/pinned-accounts, there should be only 2 routes, // one to s2 and one to s3. checkRoutes(s1, 2) // s2 and s3 should have 1 route to s1 and 3(pool)+"A"+"$SYS" == 6 checkRoutes(s2, 6) checkRoutes(s3, 6) s1.Shutdown() // The server s1 will now support pooling and accounts pinning. // Restart the server s1 with the same cluster port otherwise // s2/s3 would not be able to reconnect. conf1 = createConfFile(t, []byte(fmt.Sprintf(tmplA, o1.Cluster.Port, 3, "accounts: [\"A\"]"))) s1, _ = RunServerWithConfig(conf1) defer s1.Shutdown() // Make sure reconnect occurs. We should now have 5 routes. checkClusterFormed(t, s1, s2, s3) // Now all servers should have 3+2 to each other, so 10 total. checkRoutes(s1, 10) checkRoutes(s2, 10) checkRoutes(s3, 10) } func TestRoutePoolPerAccountSubUnsubProtoParsing(t *testing.T) { for _, test := range []struct { name string extra string }{ {"regular", _EMPTY_}, {"pooling", "pool_size: 5"}, {"per-account", "accounts: [\"A\"]"}, } { t.Run(test.name, func(t *testing.T) { confATemplate := ` port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, test.extra))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() confBTemplate := ` port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] } } cluster { listen: 127.0.0.1:-1 routes = [ "nats://127.0.0.1:%d" ] %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, optsA.Cluster.Port, test.extra))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) ncA := natsConnect(t, srva.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncA.Close() for i := 0; i < 2; i++ { var sub *nats.Subscription if i == 0 { sub = natsSubSync(t, ncA, "foo") } else { sub = natsQueueSubSync(t, ncA, "foo", "bar") } checkSubInterest(t, srvb, "A", "foo", 2*time.Second) checkSubs := func(s *Server, queue, expected bool) { t.Helper() acc, err := s.LookupAccount("A") if err != nil { t.Fatalf("Error looking account: %v", err) } checkFor(t, time.Second, 15*time.Millisecond, func() error { acc.mu.RLock() res := acc.sl.Match("foo") acc.mu.RUnlock() if expected { if queue && (len(res.qsubs) == 0 || len(res.psubs) != 0) { return fmt.Errorf("Expected queue sub, did not find it") } else if !queue && (len(res.psubs) == 0 || len(res.qsubs) != 0) { return fmt.Errorf("Expected psub, did not find it") } } else if len(res.psubs)+len(res.qsubs) != 0 { return fmt.Errorf("Unexpected subscription: %+v", res) } return nil }) } checkSubs(srva, i == 1, true) checkSubs(srvb, i == 1, true) sub.Unsubscribe() natsFlush(t, ncA) checkSubs(srva, i == 1, false) checkSubs(srvb, i == 1, false) } }) } } func TestRoutePoolPerAccountStreamImport(t *testing.T) { for _, test := range []struct { name string route string }{ {"regular", _EMPTY_}, {"pooled", "pool_size: 5"}, {"one per account", "accounts: [\"A\"]"}, {"both per account", "accounts: [\"A\", \"B\"]"}, } { t.Run(test.name, func(t *testing.T) { tmplA := ` server_name: "A" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] exports: [{stream: "foo"}] } B { users: [{user: "user2", password: "pwd"}] imports: [{stream: {subject: "foo", account: "A"}}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 %s } ` confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, test.route))) srva, optsA := RunServerWithConfig(confA) defer srva.Shutdown() tmplB := ` server_name: "B" port: -1 accounts { A { users: [{user: "user1", password: "pwd"}] exports: [{stream: "foo"}] } B { users: [{user: "user2", password: "pwd"}] imports: [{stream: {subject: "foo", account: "A"}}] } C { users: [{user: "user3", password: "pwd"}] } D { users: [{user: "user4", password: "pwd"}] } } cluster { name: "local" listen: 127.0.0.1:-1 %s %s } ` confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", optsA.Cluster.Port), test.route))) srvb, _ := RunServerWithConfig(confB) defer srvb.Shutdown() checkClusterFormed(t, srva, srvb) ncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo("user2", "pwd")) defer ncB.Close() sub := natsSubSync(t, ncB, "foo") checkSubInterest(t, srva, "B", "foo", time.Second) checkSubInterest(t, srva, "A", "foo", time.Second) ncA := natsConnect(t, srva.ClientURL(), nats.UserInfo("user1", "pwd")) defer ncA.Close() natsPub(t, ncA, "foo", []byte("hello")) natsNexMsg(t, sub, time.Second) natsUnsub(t, sub) natsFlush(t, ncB) checkFor(t, time.Second, 15*time.Millisecond, func() error { for _, acc := range []string{"A", "B"} { a, err := srva.LookupAccount(acc) if err != nil { return err } a.mu.RLock() r := a.sl.Match("foo") a.mu.RUnlock() if len(r.psubs) != 0 { return fmt.Errorf("Subscription not unsubscribed") } } return nil }) }) } } func TestRoutePoolAndPerAccountWithServiceLatencyNoDataRace(t *testing.T) { // For this test, we want the system (SYS) and SERVICE accounts to be bound // to different routes. So the names and pool size have been chosen accordingly. for _, test := range []struct { name string poolStr string }{ {"pool", "pool_size: 5"}, {"per account", "accounts: [\"SYS\", \"SERVICE\", \"REQUESTOR\"]"}, } { t.Run(test.name, func(t *testing.T) { tmpl := ` port: -1 accounts { SYS { users [{user: "sys", password: "pwd"}] } SERVICE { users [{user: "svc", password: "pwd"}] exports = [ {service: "req.*", latency: {subject: "results"}} ] } REQUESTOR { users [{user: "req", password: "pwd"}] imports = [ {service: {account: "SERVICE", subject: "req.echo"}, to: "request"} ] } } system_account: "SYS" cluster { name: "local" port: -1 %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.poolStr, _EMPTY_))) s1, opts1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.poolStr, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", opts1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Create service provider. nc := natsConnect(t, s1.ClientURL(), nats.UserInfo("svc", "pwd")) defer nc.Close() // The service listener. natsSub(t, nc, "req.echo", func(msg *nats.Msg) { msg.Respond(msg.Data) }) // Listen for metrics rsub := natsSubSync(t, nc, "results") natsFlush(t, nc) checkSubInterest(t, s2, "SERVICE", "results", time.Second) // Create second client and send request from this one. nc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo("req", "pwd")) defer nc2.Close() for i := 0; i < 5; i++ { // Send the request. _, err := nc2.Request("request", []byte("hello"), time.Second) require_NoError(t, err) // Get the latency result natsNexMsg(t, rsub, time.Second) } }) } } func TestRouteParseOriginClusterMsgArgs(t *testing.T) { for _, test := range []struct { racc bool args string pacache string reply string queues [][]byte }{ {true, "ORIGIN foo 12 345\r\n", "MY_ACCOUNT foo", _EMPTY_, nil}, {true, "ORIGIN foo bar 12 345\r\n", "MY_ACCOUNT foo", "bar", nil}, {true, "ORIGIN foo + bar queue1 queue2 12 345\r\n", "MY_ACCOUNT foo", "bar", [][]byte{[]byte("queue1"), []byte("queue2")}}, {true, "ORIGIN foo | queue1 queue2 12 345\r\n", "MY_ACCOUNT foo", _EMPTY_, [][]byte{[]byte("queue1"), []byte("queue2")}}, {false, "ORIGIN MY_ACCOUNT foo 12 345\r\n", "MY_ACCOUNT foo", _EMPTY_, nil}, {false, "ORIGIN MY_ACCOUNT foo bar 12 345\r\n", "MY_ACCOUNT foo", "bar", nil}, {false, "ORIGIN MY_ACCOUNT foo + bar queue1 queue2 12 345\r\n", "MY_ACCOUNT foo", "bar", [][]byte{[]byte("queue1"), []byte("queue2")}}, {false, "ORIGIN MY_ACCOUNT foo | queue1 queue2 12 345\r\n", "MY_ACCOUNT foo", _EMPTY_, [][]byte{[]byte("queue1"), []byte("queue2")}}, } { t.Run(test.args, func(t *testing.T) { c := &client{kind: ROUTER, route: &route{}} if test.racc { c.route.accName = []byte("MY_ACCOUNT") } if err := c.processRoutedOriginClusterMsgArgs([]byte(test.args)); err != nil { t.Fatalf("Error processing: %v", err) } if string(c.pa.origin) != "ORIGIN" { t.Fatalf("Invalid origin: %q", c.pa.origin) } if string(c.pa.account) != "MY_ACCOUNT" { t.Fatalf("Invalid account: %q", c.pa.account) } if string(c.pa.subject) != "foo" { t.Fatalf("Invalid subject: %q", c.pa.subject) } if string(c.pa.reply) != test.reply { t.Fatalf("Invalid reply: %q", c.pa.reply) } if !reflect.DeepEqual(c.pa.queues, test.queues) { t.Fatalf("Invalid queues: %v", c.pa.queues) } if c.pa.hdr != 12 { t.Fatalf("Invalid header size: %v", c.pa.hdr) } if c.pa.size != 345 { t.Fatalf("Invalid size: %v", c.pa.size) } }) } } func TestRoutePoolAndPerAccountOperatorMode(t *testing.T) { _, spub := createKey(t) sysClaim := jwt.NewAccountClaims(spub) sysClaim.Name = "SYS" sysJwt := encodeClaim(t, sysClaim, spub) akp, apub := createKey(t) claima := jwt.NewAccountClaims(apub) ajwt := encodeClaim(t, claima, apub) bkp, bpub := createKey(t) claimb := jwt.NewAccountClaims(bpub) bjwt := encodeClaim(t, claimb, bpub) ckp, cpub := createKey(t) claimc := jwt.NewAccountClaims(cpub) cjwt := encodeClaim(t, claimc, cpub) _, dpub := createKey(t) basePath := "/ngs/v1/accounts/jwt/" ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == basePath { w.Write([]byte("ok")) } else if strings.HasSuffix(r.URL.Path, spub) { w.Write([]byte(sysJwt)) } else if strings.HasSuffix(r.URL.Path, apub) { w.Write([]byte(ajwt)) } else if strings.HasSuffix(r.URL.Path, bpub) { w.Write([]byte(bjwt)) } else if strings.HasSuffix(r.URL.Path, cpub) { w.Write([]byte(cjwt)) } })) defer ts.Close() operator := fmt.Sprintf(` operator: %s system_account: %s resolver: URL("%s%s") `, ojwt, spub, ts.URL, basePath) tmpl := ` listen: 127.0.0.1:-1 server_name: %s cluster { port: -1 name: "local" %s accounts: ["` + apub + `"%s] } ` + operator conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), _EMPTY_))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) checkRoute := func(s *Server, acc string, perAccount bool) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { s.mu.RLock() _, ok := s.accRoutes[acc] s.mu.RUnlock() if perAccount && !ok { return fmt.Errorf("No dedicated route for account %q on server %q", acc, s) } else if !perAccount && ok { return fmt.Errorf("Dedicated route for account %q on server %q", acc, s) } return nil }) } // Route for accounts "apub" and "spub" should be a dedicated route checkRoute(s1, apub, true) checkRoute(s2, apub, true) checkRoute(s1, spub, true) checkRoute(s2, spub, true) // Route for account "bpub" should not checkRoute(s1, bpub, false) checkRoute(s2, bpub, false) checkComm := func(acc string, kp nkeys.KeyPair, subj string) { t.Helper() usr := createUserCreds(t, nil, kp) ncAs2 := natsConnect(t, s2.ClientURL(), usr) defer ncAs2.Close() sub := natsSubSync(t, ncAs2, subj) checkSubInterest(t, s1, acc, subj, time.Second) ncAs1 := natsConnect(t, s1.ClientURL(), usr) defer ncAs1.Close() natsPub(t, ncAs1, subj, nil) natsNexMsg(t, sub, time.Second) } checkComm(apub, akp, "foo") checkComm(bpub, bkp, "bar") // Add account "bpub" in accounts doing a configuration reload reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, fmt.Sprintf(",\"%s\"", bpub))) // Already the route should be moved to a dedicated route, even // before doing the config reload on s2. checkRoute(s1, bpub, true) checkRoute(s2, bpub, true) // Account "apub" should still have its dedicated route checkRoute(s1, apub, true) checkRoute(s2, apub, true) // So the system account checkRoute(s1, spub, true) checkRoute(s2, spub, true) // Let's complete the config reload on srvb reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), fmt.Sprintf(",\"%s\"", bpub))) checkClusterFormed(t, s1, s2) // Check communication on account bpub again. checkComm(bpub, bkp, "baz") // Now add with config reload an account that has not been used yet (cpub). // We will also remove account bpub from the account list. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, fmt.Sprintf(",\"%s\"", cpub))) // Again, check before reloading s2. checkRoute(s1, cpub, true) checkRoute(s2, cpub, true) // Now reload s2 and do other checks. reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port), fmt.Sprintf(",\"%s\"", cpub))) checkClusterFormed(t, s1, s2) checkRoute(s1, bpub, false) checkRoute(s2, bpub, false) checkComm(cpub, ckp, "bat") // Finally, let's try to add an account that the account server rejects. err := os.WriteFile(conf1, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, fmt.Sprintf(",\"%s\",\"%s\"", cpub, dpub))), 0666) require_NoError(t, err) if err := s1.Reload(); err == nil || !strings.Contains(err.Error(), dpub) { t.Fatalf("Expected error about not being able to lookup this account, got %q", err) } } func TestRoutePoolAndPerAccountWithOlderServer(t *testing.T) { tmpl := ` port: -1 server_name: "%s" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" pool_size: 5 accounts: ["A"] %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() // Have s3 explicitly disable pooling (to behave as an old server) conf3 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 server_name: "C" accounts { A { users: [{user: "A", password: "pwd"}] } B { users: [{user: "B", password: "pwd"}] } } cluster { port: -1 name: "local" pool_size: -1 routes: ["nats://127.0.0.1:%d"] } `, o1.Cluster.Port))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) check := func(acc, subj string, subSrv, pubSrv1, pubSrv2 *Server) { t.Helper() ncSub := natsConnect(t, subSrv.ClientURL(), nats.UserInfo(acc, "pwd")) defer ncSub.Close() sub := natsSubSync(t, ncSub, subj) checkSubInterest(t, pubSrv1, acc, subj, time.Second) checkSubInterest(t, pubSrv2, acc, subj, time.Second) pub := func(s *Server) { t.Helper() nc := natsConnect(t, s.ClientURL(), nats.UserInfo(acc, "pwd")) defer nc.Close() natsPub(t, nc, subj, []byte("hello")) natsNexMsg(t, sub, time.Second) } pub(pubSrv1) pub(pubSrv2) } check("A", "subj1", s1, s2, s3) check("A", "subj2", s2, s1, s3) check("A", "subj3", s3, s1, s2) check("B", "subj4", s1, s2, s3) check("B", "subj5", s2, s1, s3) check("B", "subj6", s3, s1, s2) } type testDuplicateRouteLogger struct { DummyLogger ch chan struct{} } func (l *testDuplicateRouteLogger) Noticef(format string, args ...any) { msg := fmt.Sprintf(format, args...) if !strings.Contains(msg, DuplicateRoute.String()) { return } select { case l.ch <- struct{}{}: default: } } // This test will make sure that a server with pooling does not // keep trying to connect to a non pooled (for instance old) server. // Will also make sure that if the old server is simply accepting // connections, and restarted, the server with pooling will connect. func TestRoutePoolWithOlderServerConnectAndReconnect(t *testing.T) { tmplA := ` port: -1 server_name: "A" cluster { port: -1 name: "local" pool_size: 3 %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() l := &testDuplicateRouteLogger{ch: make(chan struct{}, 50)} s1.SetLogger(l, false, false) tmplB := ` port: -1 server_name: "B" cluster { port: %d name: "local" pool_size: -1 %s } ` conf2 := createConfFile(t, []byte(fmt.Sprintf(tmplB, -1, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, o2 := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Now reload configuration of s1 to point to s2. reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmplA, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o2.Cluster.Port))) checkClusterFormed(t, s1, s2) // We could get some, but it should settle. checkRepeatConnect := func() { t.Helper() tm := time.NewTimer(4 * routeConnectDelay) var last time.Time for done := false; !done; { select { case <-l.ch: last = time.Now() case <-tm.C: done = true } } if dur := time.Since(last); dur <= routeConnectDelay { t.Fatalf("Still attempted to connect %v ago", dur) } } checkRepeatConnect() // Now shutdown s2 and restart it without active route to s1. // Check that cluster can still be formed: that is, s1 is // still trying to connect to s2. s2.Shutdown() // Wait for more than a regular reconnect delay attempt. // Note that in test it is set to 15ms. time.Sleep(50 * time.Millisecond) // Restart the server s2 with the same cluster port otherwise // s1 would not be able to reconnect. conf2 = createConfFile(t, []byte(fmt.Sprintf(tmplB, o2.Cluster.Port, _EMPTY_))) s2, _ = RunServerWithConfig(conf2) defer s2.Shutdown() // Make sure reconnect occurs checkClusterFormed(t, s1, s2) // And again, make sure there is no repeat-connect checkRepeatConnect() } func TestRouteCompressionOptions(t *testing.T) { org := testDefaultClusterCompression testDefaultClusterCompression = _EMPTY_ defer func() { testDefaultClusterCompression = org }() tmpl := ` port: -1 cluster { port: -1 compression: %s } ` for _, test := range []struct { name string mode string rttVals []int expected string rtts []time.Duration }{ {"boolean enabled", "true", nil, CompressionS2Fast, nil}, {"string enabled", "enabled", nil, CompressionS2Fast, nil}, {"string EnaBled", "EnaBled", nil, CompressionS2Fast, nil}, {"string on", "on", nil, CompressionS2Fast, nil}, {"string ON", "ON", nil, CompressionS2Fast, nil}, {"string fast", "fast", nil, CompressionS2Fast, nil}, {"string Fast", "Fast", nil, CompressionS2Fast, nil}, {"string s2_fast", "s2_fast", nil, CompressionS2Fast, nil}, {"string s2_Fast", "s2_Fast", nil, CompressionS2Fast, nil}, {"boolean disabled", "false", nil, CompressionOff, nil}, {"string disabled", "disabled", nil, CompressionOff, nil}, {"string DisableD", "DisableD", nil, CompressionOff, nil}, {"string off", "off", nil, CompressionOff, nil}, {"string OFF", "OFF", nil, CompressionOff, nil}, {"better", "better", nil, CompressionS2Better, nil}, {"Better", "Better", nil, CompressionS2Better, nil}, {"s2_better", "s2_better", nil, CompressionS2Better, nil}, {"S2_BETTER", "S2_BETTER", nil, CompressionS2Better, nil}, {"best", "best", nil, CompressionS2Best, nil}, {"BEST", "BEST", nil, CompressionS2Best, nil}, {"s2_best", "s2_best", nil, CompressionS2Best, nil}, {"S2_BEST", "S2_BEST", nil, CompressionS2Best, nil}, {"auto no rtts", "auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"s2_auto no rtts", "s2_auto", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds}, {"auto", "{mode: auto, rtt_thresholds: [%s]}", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}}, {"Auto", "{Mode: Auto, thresholds: [%s]}", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}}, {"s2_auto", "{mode: s2_auto, thresholds: [%s]}", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}}, {"s2_AUTO", "{mode: s2_AUTO, thresholds: [%s]}", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}}, {"s2_auto:-10,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,15", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}}, {"s2_auto:0,5,10", "{mode: s2_auto, thresholds: [%s]}", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}}, {"s2_auto:5,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}}, {"s2_auto:0,0,0,20", "{mode: s2_auto, thresholds: [%s]}", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}}, {"s2_auto:0,10,0,0", "{mode: s2_auto, rtt_thresholds: [%s]}", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}}, } { t.Run(test.name, func(t *testing.T) { var val string if len(test.rttVals) > 0 { var rtts string for i, v := range test.rttVals { if i > 0 { rtts += ", " } rtts += fmt.Sprintf("%dms", v) } val = fmt.Sprintf(test.mode, rtts) } else { val = test.mode } conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val))) s, o := RunServerWithConfig(conf) defer s.Shutdown() if cm := o.Cluster.Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, o.Cluster.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, o.Cluster.Compression.RTTThresholds) } s.Shutdown() o.Cluster.Port = -1 o.Cluster.Compression.Mode = test.mode if len(test.rttVals) > 0 { o.Cluster.Compression.Mode = CompressionS2Auto o.Cluster.Compression.RTTThresholds = o.Cluster.Compression.RTTThresholds[:0] for _, v := range test.rttVals { o.Cluster.Compression.RTTThresholds = append(o.Cluster.Compression.RTTThresholds, time.Duration(v)*time.Millisecond) } } s = RunServer(o) defer s.Shutdown() if cm := o.Cluster.Compression.Mode; cm != test.expected { t.Fatalf("Expected compression value to be %q, got %q", test.expected, cm) } if !reflect.DeepEqual(test.rtts, o.Cluster.Compression.RTTThresholds) { t.Fatalf("Expected RTT tresholds to be %+v, got %+v", test.rtts, o.Cluster.Compression.RTTThresholds) } }) } // Test that with no compression specified, we default to "accept" conf := createConfFile(t, []byte(` port: -1 cluster { port: -1 } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() if cm := o.Cluster.Compression.Mode; cm != CompressionAccept { t.Fatalf("Expected compression value to be %q, got %q", CompressionAccept, cm) } for _, test := range []struct { name string mode string rtts []time.Duration err string }{ {"unsupported mode", "gzip", nil, "unsupported"}, {"not ascending order", "s2_auto", []time.Duration{ 5 * time.Millisecond, 10 * time.Millisecond, 2 * time.Millisecond, }, "ascending"}, {"too many thresholds", "s2_auto", []time.Duration{ 5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 60 * time.Millisecond, }, "more than 4"}, {"all 0", "s2_auto", []time.Duration{0, 0, 0, 0}, "at least one"}, {"single 0", "s2_auto", []time.Duration{0}, "at least one"}, } { t.Run(test.name, func(t *testing.T) { o := DefaultOptions() o.Cluster.Port = -1 o.Cluster.Compression = CompressionOpts{test.mode, test.rtts} if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Unexpected error: %v", err) } }) } } type testConnSentBytes struct { net.Conn sync.RWMutex sent int } func (c *testConnSentBytes) Write(p []byte) (int, error) { n, err := c.Conn.Write(p) c.Lock() c.sent += n c.Unlock() return n, err } func TestRouteCompression(t *testing.T) { tmpl := ` port: -1 server_name: "%s" accounts { A { users: [{user: "a", pass: "pwd"}] } } cluster { name: "local" port: -1 compression: true pool_size: %d %s %s } ` for _, test := range []struct { name string poolSize int accounts string }{ {"no pooling", -1, _EMPTY_}, {"pooling", 3, _EMPTY_}, {"per account", 1, "accounts: [\"A\"]"}, } { t.Run(test.name, func(t *testing.T) { conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "S1", test.poolSize, test.accounts, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "S2", test.poolSize, test.accounts, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) s1.mu.RLock() s1.forEachRoute(func(r *client) { r.mu.Lock() r.nc = &testConnSentBytes{Conn: r.nc} r.mu.Unlock() }) s1.mu.RUnlock() nc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo("a", "pwd")) defer nc2.Close() sub := natsSubSync(t, nc2, "foo") natsFlush(t, nc2) checkSubInterest(t, s1, "A", "foo", time.Second) nc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo("a", "pwd")) defer nc1.Close() var payloads [][]byte count := 26 for i := 0; i < count; i++ { n := rand.Intn(2048) + 1 p := make([]byte, n) for j := 0; j < n; j++ { p[j] = byte(i) + 'A' } payloads = append(payloads, p) natsPub(t, nc1, "foo", p) } totalPayloadSize := 0 for i := 0; i < count; i++ { m := natsNexMsg(t, sub, time.Second) if !bytes.Equal(m.Data, payloads[i]) { t.Fatalf("Expected payload %q - got %q", payloads[i], m.Data) } totalPayloadSize += len(m.Data) } // Also check that the route stats shows that compression likely occurred var out int s1.mu.RLock() if len(test.accounts) > 0 { rems := s1.accRoutes["A"] if rems == nil { t.Fatal("Did not find route for account") } for _, r := range rems { r.mu.Lock() if r.nc != nil { nc := r.nc.(*testConnSentBytes) nc.RLock() out = nc.sent nc.RUnlock() } r.mu.Unlock() break } } else { ai, _ := s1.accounts.Load("A") acc := ai.(*Account) acc.mu.RLock() pi := acc.routePoolIdx acc.mu.RUnlock() s1.forEachRouteIdx(pi, func(r *client) bool { r.mu.Lock() if r.nc != nil { nc := r.nc.(*testConnSentBytes) nc.RLock() out = nc.sent nc.RUnlock() } r.mu.Unlock() return false }) } s1.mu.RUnlock() // Should at least be smaller than totalPayloadSize, use 20%. limit := totalPayloadSize * 80 / 100 if int(out) > limit { t.Fatalf("Expected s1's outBytes to be less than %v, got %v", limit, out) } }) } } func TestRouteCompressionMatrixModes(t *testing.T) { tmpl := ` port: -1 server_name: "%s" cluster { name: "local" port: -1 compression: %s pool_size: -1 %s } ` for _, test := range []struct { name string s1 string s2 string s1Expected string s2Expected string }{ {"off off", "off", "off", CompressionOff, CompressionOff}, {"off accept", "off", "accept", CompressionOff, CompressionOff}, {"off on", "off", "on", CompressionOff, CompressionOff}, {"off better", "off", "better", CompressionOff, CompressionOff}, {"off best", "off", "best", CompressionOff, CompressionOff}, {"accept off", "accept", "off", CompressionOff, CompressionOff}, {"accept accept", "accept", "accept", CompressionOff, CompressionOff}, {"accept on", "accept", "on", CompressionS2Fast, CompressionS2Fast}, {"accept better", "accept", "better", CompressionS2Better, CompressionS2Better}, {"accept best", "accept", "best", CompressionS2Best, CompressionS2Best}, {"on off", "on", "off", CompressionOff, CompressionOff}, {"on accept", "on", "accept", CompressionS2Fast, CompressionS2Fast}, {"on on", "on", "on", CompressionS2Fast, CompressionS2Fast}, {"on better", "on", "better", CompressionS2Fast, CompressionS2Better}, {"on best", "on", "best", CompressionS2Fast, CompressionS2Best}, {"better off", "better", "off", CompressionOff, CompressionOff}, {"better accept", "better", "accept", CompressionS2Better, CompressionS2Better}, {"better on", "better", "on", CompressionS2Better, CompressionS2Fast}, {"better better", "better", "better", CompressionS2Better, CompressionS2Better}, {"better best", "better", "best", CompressionS2Better, CompressionS2Best}, {"best off", "best", "off", CompressionOff, CompressionOff}, {"best accept", "best", "accept", CompressionS2Best, CompressionS2Best}, {"best on", "best", "on", CompressionS2Best, CompressionS2Fast}, {"best better", "best", "better", CompressionS2Best, CompressionS2Better}, {"best best", "best", "best", CompressionS2Best, CompressionS2Best}, } { t.Run(test.name, func(t *testing.T) { conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", test.s1, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", test.s2, fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) nc1 := natsConnect(t, s1.ClientURL()) defer nc1.Close() nc2 := natsConnect(t, s2.ClientURL()) defer nc2.Close() payload := make([]byte, 128) check := func(ncp, ncs *nats.Conn, subj string, s *Server) { t.Helper() sub := natsSubSync(t, ncs, subj) checkSubInterest(t, s, globalAccountName, subj, time.Second) natsPub(t, ncp, subj, payload) natsNexMsg(t, sub, time.Second) for _, srv := range []*Server{s1, s2} { rz, err := srv.Routez(nil) require_NoError(t, err) var expected string if srv == s1 { expected = test.s1Expected } else { expected = test.s2Expected } if cm := rz.Routes[0].Compression; cm != expected { t.Fatalf("Server %s - expected compression %q, got %q", srv, expected, cm) } } } check(nc1, nc2, "foo", s1) check(nc2, nc1, "bar", s2) }) } } func TestRouteCompressionWithOlderServer(t *testing.T) { tmpl := ` port: -1 server_name: "%s" cluster { port: -1 name: "local" %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: \"on\""))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() routes := fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port) conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", routes, "compression: \"not supported\""))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // Make sure that s1 route's compression is "off" s1.mu.RLock() s1.forEachRoute(func(r *client) { r.mu.Lock() cm := r.route.compression r.mu.Unlock() if cm != CompressionNotSupported { s1.mu.RUnlock() t.Fatalf("Compression should be %q, got %q", CompressionNotSupported, cm) } }) s1.mu.RUnlock() } func TestRouteCompressionImplicitRoute(t *testing.T) { tmpl := ` port: -1 server_name: "%s" cluster { port: -1 name: "local" %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() routes := fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port) conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", routes, "compression: \"fast\""))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() conf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "C", routes, "compression: \"best\""))) s3, _ := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) checkComp := func(s *Server, remoteID, expected string) { t.Helper() s.mu.RLock() defer s.mu.RUnlock() var err error s.forEachRoute(func(r *client) { if err != nil { return } var cm string ok := true r.mu.Lock() if r.route.remoteID == remoteID { cm = r.route.compression ok = cm == expected } r.mu.Unlock() if !ok { err = fmt.Errorf("Server %q - expected route to %q to use compression %q, got %q", s, remoteID, expected, cm) } }) } checkComp(s1, s2.ID(), CompressionS2Fast) checkComp(s1, s3.ID(), CompressionS2Best) checkComp(s2, s1.ID(), CompressionS2Fast) checkComp(s2, s3.ID(), CompressionS2Best) checkComp(s3, s1.ID(), CompressionS2Best) checkComp(s3, s2.ID(), CompressionS2Best) } func TestRouteCompressionAuto(t *testing.T) { tmpl := ` port: -1 server_name: "%s" ping_interval: "%s" cluster { port: -1 name: "local" compression: %s %s } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", "10s", "s2_fast", _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() // Start with 0ms RTT np := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port), true) routes := fmt.Sprintf("routes: [\"%s\"]", np.routeURL()) rtts := "{mode: s2_auto, rtt_thresholds: [100ms, 200ms, 300ms]}" conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", "500ms", rtts, routes))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() defer np.stop() checkClusterFormed(t, s1, s2) checkComp := func(expected string) { t.Helper() checkFor(t, 4*time.Second, 50*time.Millisecond, func() error { s2.mu.RLock() defer s2.mu.RUnlock() if n := s2.numRoutes(); n != 4 { return fmt.Errorf("Cluster not formed properly, got %v routes", n) } var err error s2.forEachRoute(func(r *client) { if err != nil { return } r.mu.Lock() cm := r.route.compression r.mu.Unlock() if cm != expected { err = fmt.Errorf("Route %v compression mode expected to be %q, got %q", r, expected, cm) } }) return err }) } checkComp(CompressionS2Uncompressed) // Change the proxy RTT and we should get compression "fast" np.updateRTT(150 * time.Millisecond) checkComp(CompressionS2Fast) // Now 250ms, and get "better" np.updateRTT(250 * time.Millisecond) checkComp(CompressionS2Better) // Above 350 and we should get "best" np.updateRTT(350 * time.Millisecond) checkComp(CompressionS2Best) // Down to 1ms and again should get "uncompressed" np.updateRTT(1 * time.Millisecond) checkComp(CompressionS2Uncompressed) // Do a config reload with disabling uncompressed reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", "500ms", "{mode: s2_auto, rtt_thresholds: [0ms, 100ms, 0ms, 300ms]}", routes)) // Change the RTT back down to 1ms, but we should not go uncompressed, // we should have "fast" compression. np.updateRTT(1 * time.Millisecond) checkComp(CompressionS2Fast) // Now bump to 150ms and we should be using "best", not the "better" mode np.updateRTT(150 * time.Millisecond) checkComp(CompressionS2Best) // Try 400ms and we should still be using "best" np.updateRTT(400 * time.Millisecond) checkComp(CompressionS2Best) // Try other variations reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", "500ms", "{mode: s2_auto, rtt_thresholds: [50ms, 150ms, 0ms, 0ms]}", routes)) np.updateRTT(0 * time.Millisecond) checkComp(CompressionS2Uncompressed) np.updateRTT(100 * time.Millisecond) checkComp(CompressionS2Fast) // Since we expect the same compression level, just wait before doing // the update and the next check. time.Sleep(100 * time.Millisecond) np.updateRTT(250 * time.Millisecond) checkComp(CompressionS2Fast) // Now disable compression on s1 reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", "10s", "off", _EMPTY_)) // Wait a bit to make sure we don't check for cluster too soon since // we expect a disconnect. time.Sleep(100 * time.Millisecond) checkClusterFormed(t, s1, s2) // Now change the RTT values in the proxy. np.updateRTT(0 * time.Millisecond) // Now check that s2 also shows as "off". Wait for some ping intervals. time.Sleep(200 * time.Millisecond) checkComp(CompressionOff) } func TestRoutePings(t *testing.T) { routeMaxPingInterval = 50 * time.Millisecond defer func() { routeMaxPingInterval = defaultRouteMaxPingInterval }() o1 := DefaultOptions() s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) ch := make(chan struct{}, 1) s1.mu.RLock() s1.forEachRemote(func(r *client) { r.mu.Lock() r.nc = &capturePingConn{r.nc, ch} r.mu.Unlock() }) s1.mu.RUnlock() for i := 0; i < 5; i++ { select { case <-ch: case <-time.After(250 * time.Millisecond): t.Fatalf("Did not send PING") } } } func TestRouteCustomPing(t *testing.T) { pingInterval := 50 * time.Millisecond o1 := DefaultOptions() o1.Cluster.PingInterval = pingInterval o1.Cluster.MaxPingsOut = 2 s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.Cluster.PingInterval = pingInterval o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) ch := make(chan struct{}, 1) s1.mu.RLock() s1.forEachRemote(func(r *client) { r.mu.Lock() r.nc = &capturePingConn{r.nc, ch} r.mu.Unlock() }) s1.mu.RUnlock() for i := 0; i < 5; i++ { select { case <-ch: case <-time.After(250 * time.Millisecond): t.Fatalf("Did not send PING") } } } func TestRouteNoLeakOnSlowConsumer(t *testing.T) { o1 := DefaultOptions() o1.Cluster.PoolSize = -1 s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.Cluster.PoolSize = -1 o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) // For any route connections on the first server, drop the write // deadline down and then get the client to try sending something. // This should result in an effectively immediate write timeout, // which will surface as a slow consumer. s1.mu.Lock() for _, cl := range s1.routes { for _, c := range cl { c.mu.Lock() c.out.wdl = time.Nanosecond c.mu.Unlock() c.sendRTTPing() } } s1.mu.Unlock() // By now the routes should have gone down, so check that there // aren't any routes listed still. checkFor(t, time.Millisecond*500, time.Millisecond*25, func() error { if nc := s1.NumRoutes(); nc != 0 { return fmt.Errorf("Server 1 should have no route connections, got %v", nc) } if nc := s2.NumRoutes(); nc != 0 { return fmt.Errorf("Server 2 should have no route connections, got %v", nc) } return nil }) var got, expected int64 got = s1.NumSlowConsumers() expected = 1 if got != expected { t.Errorf("got: %d, expected: %d", got, expected) } got = int64(s1.NumSlowConsumersRoutes()) if got != expected { t.Errorf("got: %d, expected: %d", got, expected) } got = int64(s1.NumSlowConsumersClients()) expected = 0 if got != expected { t.Errorf("got: %d, expected: %d", got, expected) } varz, err := s1.Varz(nil) if err != nil { t.Fatal(err) } if varz.SlowConsumersStats.Clients != 0 { t.Error("Expected no slow consumer clients") } if varz.SlowConsumersStats.Routes != 1 { t.Error("Expected a slow consumer route") } } func TestRouteNoAppSubLeakOnSlowConsumer(t *testing.T) { o1 := DefaultOptions() o1.MaxPending = 1024 o1.MaxPayload = int32(o1.MaxPending) s1 := RunServer(o1) defer s1.Shutdown() o2 := DefaultOptions() o2.MaxPending = 1024 o2.MaxPayload = int32(o2.MaxPending) o2.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) checkSub := func(expected bool) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { subsz, err := s1.Subsz(&SubszOptions{Subscriptions: true, Account: globalAccountName}) require_NoError(t, err) for _, sub := range subsz.Subs { if sub.Subject == "foo" { if expected { return nil } return fmt.Errorf("Subscription should not have been found: %+v", sub) } } if expected { return fmt.Errorf("Subscription on `foo` not found") } return nil }) } checkRoutedSub := func(expected bool) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { routez, err := s2.Routez(&RoutezOptions{Subscriptions: true}) require_NoError(t, err) for _, route := range routez.Routes { if route.Account != _EMPTY_ { continue } if len(route.Subs) == 1 && route.Subs[0] == "foo" { if expected { return nil } return fmt.Errorf("Subscription should not have been found: %+v", route.Subs) } } if expected { return fmt.Errorf("Did not find `foo` subscription") } return nil }) } checkClosed := func(cid uint64) { t.Helper() checkFor(t, 2*time.Second, 50*time.Millisecond, func() error { connz, err := s1.Connz(&ConnzOptions{State: ConnClosed, CID: cid, Subscriptions: true}) if err != nil { return err } require_Len(t, len(connz.Conns), 1) conn := connz.Conns[0] require_Equal(t, conn.Reason, SlowConsumerPendingBytes.String()) subs := conn.Subs require_Len(t, len(subs), 1) require_Equal[string](t, subs[0], "foo") return nil }) } for i := 0; i < 5; i++ { nc := natsConnect(t, s1.ClientURL()) defer nc.Close() natsSubSync(t, nc, "foo") natsFlush(t, nc) checkSub(true) checkRoutedSub(true) cid, err := nc.GetClientID() require_NoError(t, err) c := s1.getClient(cid) payload := make([]byte, 2048) c.mu.Lock() c.queueOutbound([]byte(fmt.Sprintf("MSG foo 1 2048\r\n%s\r\n", payload))) closed := c.isClosed() c.mu.Unlock() require_True(t, closed) checkSub(false) checkRoutedSub(false) checkClosed(cid) nc.Close() } } func TestRouteNoLeakOnAuthTimeout(t *testing.T) { opts := DefaultOptions() opts.Cluster.Username = "foo" opts.Cluster.Password = "bar" opts.AuthTimeout = 0.01 // Deliberately short timeout s := RunServer(opts) defer s.Shutdown() c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Cluster.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer c.Close() cr := bufio.NewReader(c) // Wait for INFO... line, _, _ := cr.ReadLine() var info serverInfo if err = json.Unmarshal(line[5:], &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } // Wait out the clock so we hit the auth timeout time.Sleep(secondsToDuration(opts.AuthTimeout) * 2) line, _, _ = cr.ReadLine() if string(line) != "-ERR 'Authentication Timeout'" { t.Fatalf("Expected '-ERR 'Authentication Timeout'' but got %q", line) } // There shouldn't be a route entry as we didn't set up. if nc := s.NumRoutes(); nc != 0 { t.Fatalf("Server should have no route connections, got %v", nc) } } func TestRouteNoRaceOnClusterNameNegotiation(t *testing.T) { // Running the test 5 times was consistently producing the race. for i := 0; i < 5; i++ { o1 := DefaultOptions() // Set this cluster name as dynamic o1.Cluster.Name = _EMPTY_ // Increase number of routes and pinned accounts to increase // the number of processRouteConnect() happening in parallel // to produce the race. o1.Cluster.PoolSize = 5 o1.Cluster.PinnedAccounts = []string{"A", "B", "C", "D"} o1.Accounts = []*Account{NewAccount("A"), NewAccount("B"), NewAccount("C"), NewAccount("D")} s1 := RunServer(o1) defer s1.Shutdown() route := RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o2 := DefaultOptions() // Set this one as explicit. Use name that is likely to be "higher" // than the dynamic one. o2.Cluster.Name = "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" o2.Cluster.PoolSize = 5 o2.Cluster.PinnedAccounts = []string{"A", "B", "C", "D"} o2.Accounts = []*Account{NewAccount("A"), NewAccount("B"), NewAccount("C"), NewAccount("D")} o2.Routes = route s2 := RunServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) s2.Shutdown() s1.Shutdown() } } nats-server-2.10.27/server/sendq.go000066400000000000000000000050501477524627100171220ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "strconv" "sync" ) type outMsg struct { subj string rply string hdr []byte msg []byte } type sendq struct { mu sync.Mutex q *ipQueue[*outMsg] s *Server } func (s *Server) newSendQ() *sendq { sq := &sendq{s: s, q: newIPQueue[*outMsg](s, "SendQ")} s.startGoRoutine(sq.internalLoop) return sq } func (sq *sendq) internalLoop() { sq.mu.Lock() s, q := sq.s, sq.q sq.mu.Unlock() defer s.grWG.Done() c := s.createInternalSystemClient() c.registerWithAccount(s.SystemAccount()) c.noIcb = true defer c.closeConnection(ClientClosed) // To optimize for not converting a string to a []byte slice. var ( subj [256]byte rply [256]byte szb [10]byte hdb [10]byte _msg [4096]byte msg = _msg[:0] ) for s.isRunning() { select { case <-s.quitCh: return case <-q.ch: pms := q.pop() for _, pm := range pms { c.pa.subject = append(subj[:0], pm.subj...) c.pa.size = len(pm.msg) + len(pm.hdr) c.pa.szb = append(szb[:0], strconv.Itoa(c.pa.size)...) if len(pm.rply) > 0 { c.pa.reply = append(rply[:0], pm.rply...) } else { c.pa.reply = nil } msg = msg[:0] if len(pm.hdr) > 0 { c.pa.hdr = len(pm.hdr) c.pa.hdb = append(hdb[:0], strconv.Itoa(c.pa.hdr)...) msg = append(msg, pm.hdr...) msg = append(msg, pm.msg...) msg = append(msg, _CRLF_...) } else { c.pa.hdr = -1 c.pa.hdb = nil msg = append(msg, pm.msg...) msg = append(msg, _CRLF_...) } c.processInboundClientMsg(msg) c.pa.szb = nil outMsgPool.Put(pm) } // TODO: should this be in the for-loop instead? c.flushClients(0) q.recycle(&pms) } } } var outMsgPool = sync.Pool{ New: func() any { return &outMsg{} }, } func (sq *sendq) send(subj, rply string, hdr, msg []byte) { if sq == nil { return } out := outMsgPool.Get().(*outMsg) out.subj, out.rply = subj, rply out.hdr = append(out.hdr[:0], hdr...) out.msg = append(out.msg[:0], msg...) sq.q.push(out) } nats-server-2.10.27/server/server.go000066400000000000000000003766631477524627100173430ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "context" "crypto/tls" "encoding/json" "errors" "flag" "fmt" "hash/fnv" "io" "log" "math/rand" "net" "net/http" "net/url" "regexp" "runtime/pprof" "unicode" // Allow dynamic profiling. _ "net/http/pprof" "os" "path" "path/filepath" "runtime" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" "github.com/nats-io/nats-server/v2/logger" ) const ( // Interval for the first PING for non client connections. firstPingInterval = time.Second // This is for the first ping for client connections. firstClientPingInterval = 2 * time.Second ) // Info is the information sent to clients, routes, gateways, and leaf nodes, // to help them understand information about this server. type Info struct { ID string `json:"server_id"` Name string `json:"server_name"` Version string `json:"version"` Proto int `json:"proto"` GitCommit string `json:"git_commit,omitempty"` GoVersion string `json:"go"` Host string `json:"host"` Port int `json:"port"` Headers bool `json:"headers"` AuthRequired bool `json:"auth_required,omitempty"` TLSRequired bool `json:"tls_required,omitempty"` TLSVerify bool `json:"tls_verify,omitempty"` TLSAvailable bool `json:"tls_available,omitempty"` MaxPayload int32 `json:"max_payload"` JetStream bool `json:"jetstream,omitempty"` IP string `json:"ip,omitempty"` CID uint64 `json:"client_id,omitempty"` ClientIP string `json:"client_ip,omitempty"` Nonce string `json:"nonce,omitempty"` Cluster string `json:"cluster,omitempty"` Dynamic bool `json:"cluster_dynamic,omitempty"` Domain string `json:"domain,omitempty"` ClientConnectURLs []string `json:"connect_urls,omitempty"` // Contains URLs a client can connect to. WSConnectURLs []string `json:"ws_connect_urls,omitempty"` // Contains URLs a ws client can connect to. LameDuckMode bool `json:"ldm,omitempty"` Compression string `json:"compression,omitempty"` // Route Specific Import *SubjectPermission `json:"import,omitempty"` Export *SubjectPermission `json:"export,omitempty"` LNOC bool `json:"lnoc,omitempty"` LNOCU bool `json:"lnocu,omitempty"` InfoOnConnect bool `json:"info_on_connect,omitempty"` // When true the server will respond to CONNECT with an INFO ConnectInfo bool `json:"connect_info,omitempty"` // When true this is the server INFO response to CONNECT RoutePoolSize int `json:"route_pool_size,omitempty"` RoutePoolIdx int `json:"route_pool_idx,omitempty"` RouteAccount string `json:"route_account,omitempty"` RouteAccReqID string `json:"route_acc_add_reqid,omitempty"` // Gateways Specific Gateway string `json:"gateway,omitempty"` // Name of the origin Gateway (sent by gateway's INFO) GatewayURLs []string `json:"gateway_urls,omitempty"` // Gateway URLs in the originating cluster (sent by gateway's INFO) GatewayURL string `json:"gateway_url,omitempty"` // Gateway URL on that server (sent by route's INFO) GatewayCmd byte `json:"gateway_cmd,omitempty"` // Command code for the receiving server to know what to do GatewayCmdPayload []byte `json:"gateway_cmd_payload,omitempty"` // Command payload when needed GatewayNRP bool `json:"gateway_nrp,omitempty"` // Uses new $GNR. prefix for mapped replies GatewayIOM bool `json:"gateway_iom,omitempty"` // Indicate that all accounts will be switched to InterestOnly mode "right away" // LeafNode Specific LeafNodeURLs []string `json:"leafnode_urls,omitempty"` // LeafNode URLs that the server can reconnect to. RemoteAccount string `json:"remote_account,omitempty"` // Lets the other side know the remote account that they bind to. XKey string `json:"xkey,omitempty"` // Public server's x25519 key. } // Server is our main struct. type Server struct { // Fields accessed with atomic operations need to be 64-bit aligned gcid uint64 // How often user logon fails due to the issuer account not being pinned. pinnedAccFail uint64 stats scStats mu sync.RWMutex reloadMu sync.RWMutex // Write-locked when a config reload is taking place ONLY kp nkeys.KeyPair xkp nkeys.KeyPair xpub string info Info configFile string optsMu sync.RWMutex opts *Options running atomic.Bool shutdown atomic.Bool listener net.Listener listenerErr error gacc *Account sys *internal sysAcc atomic.Pointer[Account] js atomic.Pointer[jetStream] isMetaLeader atomic.Bool jsClustered atomic.Bool accounts sync.Map tmpAccounts sync.Map // Temporarily stores accounts that are being built activeAccounts int32 accResolver AccountResolver clients map[uint64]*client routes map[string][]*client routesPoolSize int // Configured pool size routesReject bool // During reload, we may want to reject adding routes until some conditions are met routesNoPool int // Number of routes that don't use pooling (connecting to older server for instance) accRoutes map[string]map[string]*client // Key is account name, value is key=remoteID/value=route connection accRouteByHash sync.Map // Key is account name, value is nil or a pool index accAddedCh chan struct{} accAddedReqID string leafs map[uint64]*client users map[string]*User nkeys map[string]*NkeyUser totalClients uint64 closed *closedRingBuffer done chan bool start time.Time http net.Listener httpHandler http.Handler httpBasePath string profiler net.Listener httpReqStats map[string]uint64 routeListener net.Listener routeListenerErr error routeInfo Info routeResolver netResolver routesToSelf map[string]struct{} routeTLSName string leafNodeListener net.Listener leafNodeListenerErr error leafNodeInfo Info leafNodeInfoJSON []byte leafURLsMap refCountedUrlSet leafNodeOpts struct { resolver netResolver dialTimeout time.Duration } leafRemoteCfgs []*leafNodeCfg leafRemoteAccounts sync.Map leafNodeEnabled bool leafDisableConnect bool // Used in test only quitCh chan struct{} startupComplete chan struct{} shutdownComplete chan struct{} // Tracking Go routines grMu sync.Mutex grTmpClients map[uint64]*client grRunning bool grWG sync.WaitGroup // to wait on various go routines cproto int64 // number of clients supporting async INFO configTime time.Time // last time config was loaded logging struct { sync.RWMutex logger Logger trace int32 debug int32 traceSysAcc int32 } clientConnectURLs []string // Used internally for quick look-ups. clientConnectURLsMap refCountedUrlSet lastCURLsUpdate int64 // For Gateways gatewayListener net.Listener // Accept listener gatewayListenerErr error gateway *srvGateway // Used by tests to check that http.Servers do // not set any timeout. monitoringServer *http.Server profilingServer *http.Server // LameDuck mode ldm bool ldmCh chan bool // Trusted public operator keys. trustedKeys []string // map of trusted keys to operator setting StrictSigningKeyUsage strictSigningKeyUsage map[string]struct{} // We use this to minimize mem copies for requests to monitoring // endpoint /varz (when it comes from http). varzMu sync.Mutex varz *Varz // This is set during a config reload if we detect that we have // added/removed routes. The monitoring code then check that // to know if it should update the cluster's URLs array. varzUpdateRouteURLs bool // Keeps a sublist of of subscriptions attached to leafnode connections // for the $GNR.*.*.*.> subject so that a server can send back a mapped // gateway reply. gwLeafSubs *Sublist // Used for expiration of mapped GW replies gwrm struct { w int32 ch chan time.Duration m sync.Map } // For eventIDs eventIds *nuid.NUID // Websocket structure websocket srvWebsocket // MQTT structure mqtt srvMQTT // OCSP monitoring ocsps []*OCSPMonitor // OCSP peer verification (at least one TLS block) ocspPeerVerify bool // OCSP response cache ocsprc OCSPResponseCache // exporting account name the importer experienced issues with incompleteAccExporterMap sync.Map // Holds cluster name under different lock for mapping cnMu sync.RWMutex cn string // For registering raft nodes with the server. rnMu sync.RWMutex raftNodes map[string]RaftNode // For mapping from a raft node name back to a server name and cluster. Node has to be in the same domain. nodeToInfo sync.Map // For out of resources to not log errors too fast. rerrMu sync.Mutex rerrLast time.Time connRateCounter *rateCounter // If there is a system account configured, to still support the $G account, // the server will create a fake user and add it to the list of users. // Keep track of what that user name is for config reload purposes. sysAccOnlyNoAuthUser string // IPQueues map ipQueues sync.Map // To limit logging frequency rateLimitLogging sync.Map rateLimitLoggingCh chan time.Duration // Total outstanding catchup bytes in flight. gcbMu sync.RWMutex gcbOut int64 gcbOutMax int64 // Taken from JetStreamMaxCatchup or defaultMaxTotalCatchupOutBytes // A global chanel to kick out stalled catchup sequences. gcbKick chan struct{} // Total outbound syncRequests syncOutSem chan struct{} // Queue to process JS API requests that come from routes (or gateways) jsAPIRoutedReqs *ipQueue[*jsAPIRoutedReq] } // For tracking JS nodes. type nodeInfo struct { name string version string cluster string domain string id string tags jwt.TagList cfg *JetStreamConfig stats *JetStreamStats offline bool js bool binarySnapshots bool } // Make sure all are 64bits for atomic use type stats struct { inMsgs int64 outMsgs int64 inBytes int64 outBytes int64 slowConsumers int64 } // scStats includes the total and per connection counters of Slow Consumers. type scStats struct { clients atomic.Uint64 routes atomic.Uint64 leafs atomic.Uint64 gateways atomic.Uint64 } // This is used by tests so we can run all server tests with a default route // or leafnode compression mode. For instance: // go test -race -v ./server -cluster_compression=fast var ( testDefaultClusterCompression string testDefaultLeafNodeCompression string ) // Compression modes. const ( CompressionNotSupported = "not supported" CompressionOff = "off" CompressionAccept = "accept" CompressionS2Auto = "s2_auto" CompressionS2Uncompressed = "s2_uncompressed" CompressionS2Fast = "s2_fast" CompressionS2Better = "s2_better" CompressionS2Best = "s2_best" ) // defaultCompressionS2AutoRTTThresholds is the default of RTT thresholds for // the CompressionS2Auto mode. var defaultCompressionS2AutoRTTThresholds = []time.Duration{ // [0..10ms] -> CompressionS2Uncompressed 10 * time.Millisecond, // ]10ms..50ms] -> CompressionS2Fast 50 * time.Millisecond, // ]50ms..100ms] -> CompressionS2Better 100 * time.Millisecond, // ]100ms..] -> CompressionS2Best } // For a given user provided string, matches to one of the compression mode // constant and updates the provided string to that constant. Returns an // error if the provided compression mode is not known. // The parameter `chosenModeForOn` indicates which compression mode to use // when the user selects "on" (or enabled, true, etc..). This is because // we may have different defaults depending on where the compression is used. func validateAndNormalizeCompressionOption(c *CompressionOpts, chosenModeForOn string) error { if c == nil { return nil } cmtl := strings.ToLower(c.Mode) // First, check for the "on" case so that we set to the default compression // mode for that. The other switch/case will finish setup if needed (for // instance if the default mode is s2Auto). switch cmtl { case "on", "enabled", "true": cmtl = chosenModeForOn default: } // Check (again) with the proper mode. switch cmtl { case "not supported", "not_supported": c.Mode = CompressionNotSupported case "disabled", "off", "false": c.Mode = CompressionOff case "accept": c.Mode = CompressionAccept case "auto", "s2_auto": var rtts []time.Duration if len(c.RTTThresholds) == 0 { rtts = defaultCompressionS2AutoRTTThresholds } else { for _, n := range c.RTTThresholds { // Do not error on negative, but simply set to 0 if n < 0 { n = 0 } // Make sure they are properly ordered. However, it is possible // to have a "0" anywhere in the list to indicate that this // compression level should not be used. if l := len(rtts); l > 0 && n != 0 { for _, v := range rtts { if n < v { return fmt.Errorf("RTT threshold values %v should be in ascending order", c.RTTThresholds) } } } rtts = append(rtts, n) } if len(rtts) > 0 { // Trim 0 that are at the end. stop := -1 for i := len(rtts) - 1; i >= 0; i-- { if rtts[i] != 0 { stop = i break } } rtts = rtts[:stop+1] } if len(rtts) > 4 { // There should be at most values for "uncompressed", "fast", // "better" and "best" (when some 0 are present). return fmt.Errorf("compression mode %q should have no more than 4 RTT thresholds: %v", c.Mode, c.RTTThresholds) } else if len(rtts) == 0 { // But there should be at least 1 if the user provided the slice. // We would be here only if it was provided by say with values // being a single or all zeros. return fmt.Errorf("compression mode %q requires at least one RTT threshold", c.Mode) } } c.Mode = CompressionS2Auto c.RTTThresholds = rtts case "fast", "s2_fast": c.Mode = CompressionS2Fast case "better", "s2_better": c.Mode = CompressionS2Better case "best", "s2_best": c.Mode = CompressionS2Best default: return fmt.Errorf("unsupported compression mode %q", c.Mode) } return nil } // Returns `true` if the compression mode `m` indicates that the server // will negotiate compression with the remote server, `false` otherwise. // Note that the provided compression mode is assumed to have been // normalized and validated. func needsCompression(m string) bool { return m != _EMPTY_ && m != CompressionOff && m != CompressionNotSupported } // Compression is asymmetric, meaning that one side can have a different // compression level than the other. However, we need to check for cases // when this server `scm` or the remote `rcm` do not support compression // (say older server, or test to make it behave as it is not), or have // the compression off. // Note that `scm` is assumed to not be "off" or "not supported". func selectCompressionMode(scm, rcm string) (mode string, err error) { if rcm == CompressionNotSupported || rcm == _EMPTY_ { return CompressionNotSupported, nil } switch rcm { case CompressionOff: // If the remote explicitly disables compression, then we won't // use compression. return CompressionOff, nil case CompressionAccept: // If the remote is ok with compression (but is not initiating it), // and if we too are in this mode, then it means no compression. if scm == CompressionAccept { return CompressionOff, nil } // Otherwise use our compression mode. return scm, nil case CompressionS2Auto, CompressionS2Uncompressed, CompressionS2Fast, CompressionS2Better, CompressionS2Best: // This case is here to make sure that if we don't recognize a // compression setting, we error out. if scm == CompressionAccept { // If our compression mode is "accept", then we will use the remote // compression mode, except if it is "auto", in which case we will // default to "fast". This is not a configuration (auto in one // side and accept in the other) that would be recommended. if rcm == CompressionS2Auto { return CompressionS2Fast, nil } // Use their compression mode. return rcm, nil } // Otherwise use our compression mode. return scm, nil default: return _EMPTY_, fmt.Errorf("unsupported route compression mode %q", rcm) } } // If the configured compression mode is "auto" then will return that, // otherwise will return the given `cm` compression mode. func compressionModeForInfoProtocol(co *CompressionOpts, cm string) string { if co.Mode == CompressionS2Auto { return CompressionS2Auto } return cm } // Given a connection RTT and a list of thresholds durations, this // function will return an S2 compression level such as "uncompressed", // "fast", "better" or "best". For instance, with the following slice: // [5ms, 10ms, 15ms, 20ms], a RTT of up to 5ms will result // in the compression level "uncompressed", ]5ms..10ms] will result in // "fast" compression, etc.. // However, the 0 value allows for disabling of some compression levels. // For instance, the following slice: [0, 0, 20, 30] means that a RTT of // [0..20ms] would result in the "better" compression - effectively disabling // the use of "uncompressed" and "fast", then anything above 20ms would // result in the use of "best" level (the 30 in the list has no effect // and the list could have been simplified to [0, 0, 20]). func selectS2AutoModeBasedOnRTT(rtt time.Duration, rttThresholds []time.Duration) string { var idx int var found bool for i, d := range rttThresholds { if rtt <= d { idx = i found = true break } } if !found { // If we did not find but we have all levels, then use "best", // otherwise use the last one in array. if l := len(rttThresholds); l >= 3 { idx = 3 } else { idx = l - 1 } } switch idx { case 0: return CompressionS2Uncompressed case 1: return CompressionS2Fast case 2: return CompressionS2Better } return CompressionS2Best } // Returns an array of s2 WriterOption based on the route compression mode. // So far we return a single option, but this way we can call s2.NewWriter() // with a nil []s2.WriterOption, but not with a nil s2.WriterOption, so // this is more versatile. func s2WriterOptions(cm string) []s2.WriterOption { _opts := [2]s2.WriterOption{} opts := append( _opts[:0], s2.WriterConcurrency(1), // Stop asynchronous flushing in separate goroutines ) switch cm { case CompressionS2Uncompressed: return append(opts, s2.WriterUncompressed()) case CompressionS2Best: return append(opts, s2.WriterBestCompression()) case CompressionS2Better: return append(opts, s2.WriterBetterCompression()) default: return nil } } // New will setup a new server struct after parsing the options. // DEPRECATED: Use NewServer(opts) func New(opts *Options) *Server { s, _ := NewServer(opts) return s } // NewServer will setup a new server struct after parsing the options. // Could return an error if options can not be validated. // The provided Options type should not be re-used afterwards. // Either use Options.Clone() to pass a copy, or make a new one. func NewServer(opts *Options) (*Server, error) { setBaselineOptions(opts) // Process TLS options, including whether we require client certificates. tlsReq := opts.TLSConfig != nil verify := (tlsReq && opts.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert) // Create our server's nkey identity. kp, _ := nkeys.CreateServer() pub, _ := kp.PublicKey() // Create an xkey for encrypting messages from this server. xkp, _ := nkeys.CreateCurveKeys() xpub, _ := xkp.PublicKey() serverName := pub if opts.ServerName != _EMPTY_ { serverName = opts.ServerName } httpBasePath := normalizeBasePath(opts.HTTPBasePath) // Validate some options. This is here because we cannot assume that // server will always be started with configuration parsing (that could // report issues). Its options can be (incorrectly) set by hand when // server is embedded. If there is an error, return nil. if err := validateOptions(opts); err != nil { return nil, err } info := Info{ ID: pub, XKey: xpub, Version: VERSION, Proto: PROTO, GitCommit: gitCommit, GoVersion: runtime.Version(), Name: serverName, Host: opts.Host, Port: opts.Port, AuthRequired: false, TLSRequired: tlsReq && !opts.AllowNonTLS, TLSVerify: verify, MaxPayload: opts.MaxPayload, JetStream: opts.JetStream, Headers: !opts.NoHeaderSupport, Cluster: opts.Cluster.Name, Domain: opts.JetStreamDomain, } if tlsReq && !info.TLSRequired { info.TLSAvailable = true } now := time.Now() s := &Server{ kp: kp, xkp: xkp, xpub: xpub, configFile: opts.ConfigFile, info: info, opts: opts, done: make(chan bool, 1), start: now, configTime: now, gwLeafSubs: NewSublistWithCache(), httpBasePath: httpBasePath, eventIds: nuid.New(), routesToSelf: make(map[string]struct{}), httpReqStats: make(map[string]uint64), // Used to track HTTP requests rateLimitLoggingCh: make(chan time.Duration, 1), leafNodeEnabled: opts.LeafNode.Port != 0 || len(opts.LeafNode.Remotes) > 0, syncOutSem: make(chan struct{}, maxConcurrentSyncRequests), } // Fill up the maximum in flight syncRequests for this server. // Used in JetStream catchup semantics. for i := 0; i < maxConcurrentSyncRequests; i++ { s.syncOutSem <- struct{}{} } if opts.TLSRateLimit > 0 { s.connRateCounter = newRateCounter(opts.tlsConfigOpts.RateLimit) } // Trusted root operator keys. if !s.processTrustedKeys() { return nil, fmt.Errorf("Error processing trusted operator keys") } // If we have solicited leafnodes but no clustering and no clustername. // However we may need a stable clustername so use the server name. if len(opts.LeafNode.Remotes) > 0 && opts.Cluster.Port == 0 && opts.Cluster.Name == _EMPTY_ { opts.Cluster.Name = opts.ServerName } if opts.Cluster.Name != _EMPTY_ { // Also place into mapping cn with cnMu lock. s.cnMu.Lock() s.cn = opts.Cluster.Name s.cnMu.Unlock() } s.mu.Lock() defer s.mu.Unlock() // Place ourselves in the JetStream nodeInfo if needed. if opts.JetStream { ourNode := getHash(serverName) s.nodeToInfo.Store(ourNode, nodeInfo{ serverName, VERSION, opts.Cluster.Name, opts.JetStreamDomain, info.ID, opts.Tags, &JetStreamConfig{MaxMemory: opts.JetStreamMaxMemory, MaxStore: opts.JetStreamMaxStore, CompressOK: true}, nil, false, true, true, }) } s.routeResolver = opts.Cluster.resolver if s.routeResolver == nil { s.routeResolver = net.DefaultResolver } // Used internally for quick look-ups. s.clientConnectURLsMap = make(refCountedUrlSet) s.websocket.connectURLsMap = make(refCountedUrlSet) s.leafURLsMap = make(refCountedUrlSet) // Ensure that non-exported options (used in tests) are properly set. s.setLeafNodeNonExportedOptions() // Setup OCSP Stapling and OCSP Peer. This will abort server from starting if there // are no valid staples and OCSP Stapling policy is set to Always or MustStaple. if err := s.enableOCSP(); err != nil { return nil, err } // Call this even if there is no gateway defined. It will // initialize the structure so we don't have to check for // it to be nil or not in various places in the code. if err := s.newGateway(opts); err != nil { return nil, err } // If we have a cluster definition but do not have a cluster name, create one. if opts.Cluster.Port != 0 && opts.Cluster.Name == _EMPTY_ { s.info.Cluster = nuid.Next() } else if opts.Cluster.Name != _EMPTY_ { // Likewise here if we have a cluster name set. s.info.Cluster = opts.Cluster.Name } // This is normally done in the AcceptLoop, once the // listener has been created (possibly with random port), // but since some tests may expect the INFO to be properly // set after New(), let's do it now. s.setInfoHostPort() // For tracking clients s.clients = make(map[uint64]*client) // For tracking closed clients. s.closed = newClosedRingBuffer(opts.MaxClosedClients) // For tracking connections that are not yet registered // in s.routes, but for which readLoop has started. s.grTmpClients = make(map[uint64]*client) // For tracking routes and their remote ids s.initRouteStructures(opts) // For tracking leaf nodes. s.leafs = make(map[uint64]*client) // Used to kick out all go routines possibly waiting on server // to shutdown. s.quitCh = make(chan struct{}) // Closed when startup is complete. ReadyForConnections() will block on // this before checking the presence of listening sockets. s.startupComplete = make(chan struct{}) // Closed when Shutdown() is complete. Allows WaitForShutdown() to block // waiting for complete shutdown. s.shutdownComplete = make(chan struct{}) // Check for configured account resolvers. if err := s.configureResolver(); err != nil { return nil, err } // If there is an URL account resolver, do basic test to see if anyone is home. if ar := opts.AccountResolver; ar != nil { if ur, ok := ar.(*URLAccResolver); ok { if _, err := ur.Fetch(_EMPTY_); err != nil { return nil, err } } } // For other resolver: // In operator mode, when the account resolver depends on an external system and // the system account can't be fetched, inject a temporary one. if ar := s.accResolver; len(opts.TrustedOperators) == 1 && ar != nil && opts.SystemAccount != _EMPTY_ && opts.SystemAccount != DEFAULT_SYSTEM_ACCOUNT { if _, ok := ar.(*MemAccResolver); !ok { s.mu.Unlock() var a *Account // perform direct lookup to avoid warning trace if _, err := fetchAccount(ar, opts.SystemAccount); err == nil { a, _ = s.lookupAccount(opts.SystemAccount) } s.mu.Lock() if a == nil { sac := NewAccount(opts.SystemAccount) sac.Issuer = opts.TrustedOperators[0].Issuer sac.signingKeys = map[string]jwt.Scope{} sac.signingKeys[opts.SystemAccount] = nil s.registerAccountNoLock(sac) } } } // For tracking accounts if _, err := s.configureAccounts(false); err != nil { return nil, err } // Used to setup Authorization. s.configureAuthorization() // Start signal handler s.handleSignals() return s, nil } // Initializes route structures based on pooling and/or per-account routes. // // Server lock is held on entry func (s *Server) initRouteStructures(opts *Options) { s.routes = make(map[string][]*client) if ps := opts.Cluster.PoolSize; ps > 0 { s.routesPoolSize = ps } else { s.routesPoolSize = 1 } // If we have per-account routes, we create accRoutes and initialize it // with nil values. The presence of an account as the key will allow us // to know if a given account is supposed to have dedicated routes. if l := len(opts.Cluster.PinnedAccounts); l > 0 { s.accRoutes = make(map[string]map[string]*client, l) for _, acc := range opts.Cluster.PinnedAccounts { s.accRoutes[acc] = make(map[string]*client) } } } func (s *Server) logRejectedTLSConns() { defer s.grWG.Done() t := time.NewTicker(time.Second) defer t.Stop() for { select { case <-s.quitCh: return case <-t.C: blocked := s.connRateCounter.countBlocked() if blocked > 0 { s.Warnf("Rejected %d connections due to TLS rate limiting", blocked) } } } } // clusterName returns our cluster name which could be dynamic. func (s *Server) ClusterName() string { s.mu.RLock() cn := s.info.Cluster s.mu.RUnlock() return cn } // Grabs cluster name with cluster name specific lock. func (s *Server) cachedClusterName() string { s.cnMu.RLock() cn := s.cn s.cnMu.RUnlock() return cn } // setClusterName will update the cluster name for this server. func (s *Server) setClusterName(name string) { s.mu.Lock() var resetCh chan struct{} if s.sys != nil && s.info.Cluster != name { // can't hold the lock as go routine reading it may be waiting for lock as well resetCh = s.sys.resetCh } s.info.Cluster = name s.routeInfo.Cluster = name // Need to close solicited leaf nodes. The close has to be done outside of the server lock. var leafs []*client for _, c := range s.leafs { c.mu.Lock() if c.leaf != nil && c.leaf.remote != nil { leafs = append(leafs, c) } c.mu.Unlock() } s.mu.Unlock() // Also place into mapping cn with cnMu lock. s.cnMu.Lock() s.cn = name s.cnMu.Unlock() for _, l := range leafs { l.closeConnection(ClusterNameConflict) } if resetCh != nil { resetCh <- struct{}{} } s.Noticef("Cluster name updated to %s", name) } // Return whether the cluster name is dynamic. func (s *Server) isClusterNameDynamic() bool { // We need to lock the whole "Cluster.Name" check and not use s.getOpts() // because otherwise this could cause a data race with setting the name in // route.go's processRouteConnect(). s.optsMu.RLock() dynamic := s.opts.Cluster.Name == _EMPTY_ s.optsMu.RUnlock() return dynamic } // Returns our configured serverName. func (s *Server) serverName() string { return s.getOpts().ServerName } // ClientURL returns the URL used to connect clients. Helpful in testing // when we designate a random client port (-1). func (s *Server) ClientURL() string { // FIXME(dlc) - should we add in user and pass if defined single? opts := s.getOpts() var u url.URL u.Scheme = "nats" if opts.TLSConfig != nil { u.Scheme = "tls" } u.Host = net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port)) return u.String() } func validateCluster(o *Options) error { if o.Cluster.Compression.Mode != _EMPTY_ { if err := validateAndNormalizeCompressionOption(&o.Cluster.Compression, CompressionS2Fast); err != nil { return err } } if err := validatePinnedCerts(o.Cluster.TLSPinnedCerts); err != nil { return fmt.Errorf("cluster: %v", err) } // Check that cluster name if defined matches any gateway name. if o.Gateway.Name != "" && o.Gateway.Name != o.Cluster.Name { if o.Cluster.Name != "" { return ErrClusterNameConfigConflict } // Set this here so we do not consider it dynamic. o.Cluster.Name = o.Gateway.Name } if l := len(o.Cluster.PinnedAccounts); l > 0 { if o.Cluster.PoolSize < 0 { return fmt.Errorf("pool_size cannot be negative if pinned accounts are specified") } m := make(map[string]struct{}, l) for _, a := range o.Cluster.PinnedAccounts { if _, exists := m[a]; exists { return fmt.Errorf("found duplicate account name %q in pinned accounts list %q", a, o.Cluster.PinnedAccounts) } m[a] = struct{}{} } } return nil } func validatePinnedCerts(pinned PinnedCertSet) error { re := regexp.MustCompile("^[a-f0-9]{64}$") for certId := range pinned { entry := strings.ToLower(certId) if !re.MatchString(entry) { return fmt.Errorf("error parsing 'pinned_certs' key %s does not look like lower case hex-encoded sha256 of DER encoded SubjectPublicKeyInfo", entry) } } return nil } func validateOptions(o *Options) error { if o.LameDuckDuration > 0 && o.LameDuckGracePeriod >= o.LameDuckDuration { return fmt.Errorf("lame duck grace period (%v) should be strictly lower than lame duck duration (%v)", o.LameDuckGracePeriod, o.LameDuckDuration) } if int64(o.MaxPayload) > o.MaxPending { return fmt.Errorf("max_payload (%v) cannot be higher than max_pending (%v)", o.MaxPayload, o.MaxPending) } // Check that the trust configuration is correct. if err := validateTrustedOperators(o); err != nil { return err } // Check on leaf nodes which will require a system // account when gateways are also configured. if err := validateLeafNode(o); err != nil { return err } // Check that authentication is properly configured. if err := validateAuth(o); err != nil { return err } // Check that gateway is properly configured. Returns no error // if there is no gateway defined. if err := validateGatewayOptions(o); err != nil { return err } // Check that cluster name if defined matches any gateway name. if err := validateCluster(o); err != nil { return err } if err := validateMQTTOptions(o); err != nil { return err } if err := validateJetStreamOptions(o); err != nil { return err } // Finally check websocket options. return validateWebsocketOptions(o) } func (s *Server) getOpts() *Options { s.optsMu.RLock() opts := s.opts s.optsMu.RUnlock() return opts } func (s *Server) setOpts(opts *Options) { s.optsMu.Lock() s.opts = opts s.optsMu.Unlock() } func (s *Server) globalAccount() *Account { s.mu.RLock() gacc := s.gacc s.mu.RUnlock() return gacc } // Used to setup or update Accounts. // Returns a map that indicates which accounts have had their stream imports // changed (in case of an update in configuration reload). // Lock is held upon entry, but will be released/reacquired in this function. func (s *Server) configureAccounts(reloading bool) (map[string]struct{}, error) { awcsti := make(map[string]struct{}) // Create the global account. if s.gacc == nil { s.gacc = NewAccount(globalAccountName) s.registerAccountNoLock(s.gacc) } opts := s.getOpts() // We need to track service imports since we can not swap them out (unsub and re-sub) // until the proper server struct accounts have been swapped in properly. Doing it in // place could lead to data loss or server panic since account under new si has no real // account and hence no sublist, so will panic on inbound message. siMap := make(map[*Account][][]byte) // Check opts and walk through them. We need to copy them here // so that we do not keep a real one sitting in the options. for _, acc := range opts.Accounts { var a *Account create := true // For the global account, we want to skip the reload process // and fall back into the "create" case which will in that // case really be just an update (shallowCopy will make sure // that mappings are copied over). if reloading && acc.Name != globalAccountName { if ai, ok := s.accounts.Load(acc.Name); ok { a = ai.(*Account) // Before updating the account, check if stream imports have changed. if !a.checkStreamImportsEqual(acc) { awcsti[acc.Name] = struct{}{} } a.mu.Lock() // Collect the sids for the service imports since we are going to // replace with new ones. var sids [][]byte for _, si := range a.imports.services { if si.sid != nil { sids = append(sids, si.sid) } } // Setup to process later if needed. if len(sids) > 0 || len(acc.imports.services) > 0 { siMap[a] = sids } // Now reset all export/imports fields since they are going to be // filled in shallowCopy() a.imports.streams, a.imports.services = nil, nil a.exports.streams, a.exports.services = nil, nil // We call shallowCopy from the account `acc` (the one in Options) // and pass `a` (our existing account) to get it updated. acc.shallowCopy(a) a.mu.Unlock() create = false } } // Track old mappings if global account. var oldGMappings []*mapping if create { if acc.Name == globalAccountName { a = s.gacc a.mu.Lock() oldGMappings = append(oldGMappings, a.mappings...) a.mu.Unlock() } else { a = NewAccount(acc.Name) } // Locking matters in the case of an update of the global account a.mu.Lock() acc.shallowCopy(a) a.mu.Unlock() // Will be a no-op in case of the global account since it is already registered. s.registerAccountNoLock(a) } // The `acc` account is stored in options, not in the server, and these can be cleared. acc.sl, acc.clients, acc.mappings = nil, nil, nil // Check here if we have been reloaded and we have a global account with mappings that may have changed. // If we have leafnodes they need to be updated. if reloading && a == s.gacc { a.mu.Lock() mappings := make(map[string]*mapping) if len(a.mappings) > 0 && a.nleafs > 0 { for _, em := range a.mappings { mappings[em.src] = em } } a.mu.Unlock() if len(mappings) > 0 || len(oldGMappings) > 0 { a.lmu.RLock() for _, lc := range a.lleafs { for _, em := range mappings { lc.forceAddToSmap(em.src) } // Remove any old ones if needed. for _, em := range oldGMappings { // Only remove if not in the new ones. if _, ok := mappings[em.src]; !ok { lc.forceRemoveFromSmap(em.src) } } } a.lmu.RUnlock() } } // If we see an account defined using $SYS we will make sure that is set as system account. if acc.Name == DEFAULT_SYSTEM_ACCOUNT && opts.SystemAccount == _EMPTY_ { opts.SystemAccount = DEFAULT_SYSTEM_ACCOUNT } } // Now that we have this we need to remap any referenced accounts in // import or export maps to the new ones. swapApproved := func(ea *exportAuth) { for sub, a := range ea.approved { var acc *Account if v, ok := s.accounts.Load(a.Name); ok { acc = v.(*Account) } ea.approved[sub] = acc } } var numAccounts int s.accounts.Range(func(k, v any) bool { numAccounts++ acc := v.(*Account) acc.mu.Lock() // Exports for _, se := range acc.exports.streams { if se != nil { swapApproved(&se.exportAuth) } } for _, se := range acc.exports.services { if se != nil { // Swap over the bound account for service exports. if se.acc != nil { if v, ok := s.accounts.Load(se.acc.Name); ok { se.acc = v.(*Account) } } swapApproved(&se.exportAuth) } } // Imports for _, si := range acc.imports.streams { if v, ok := s.accounts.Load(si.acc.Name); ok { si.acc = v.(*Account) } } for _, si := range acc.imports.services { if v, ok := s.accounts.Load(si.acc.Name); ok { si.acc = v.(*Account) // It is possible to allow for latency tracking inside your // own account, so lock only when not the same account. if si.acc == acc { si.se = si.acc.getServiceExport(si.to) continue } si.acc.mu.RLock() si.se = si.acc.getServiceExport(si.to) si.acc.mu.RUnlock() } } // Make sure the subs are running, but only if not reloading. if len(acc.imports.services) > 0 && acc.ic == nil && !reloading { acc.ic = s.createInternalAccountClient() acc.ic.acc = acc // Need to release locks to invoke this function. acc.mu.Unlock() s.mu.Unlock() acc.addAllServiceImportSubs() s.mu.Lock() acc.mu.Lock() } acc.updated = time.Now() acc.mu.Unlock() return true }) // Check if we need to process service imports pending from above. // This processing needs to be after we swap in the real accounts above. for acc, sids := range siMap { c := acc.ic for _, sid := range sids { c.processUnsub(sid) } acc.addAllServiceImportSubs() s.mu.Unlock() s.registerSystemImports(acc) s.mu.Lock() } // Set the system account if it was configured. // Otherwise create a default one. if opts.SystemAccount != _EMPTY_ { // Lock may be acquired in lookupAccount, so release to call lookupAccount. s.mu.Unlock() acc, err := s.lookupAccount(opts.SystemAccount) s.mu.Lock() if err == nil && s.sys != nil && acc != s.sys.account { // sys.account.clients (including internal client)/respmap/etc... are transferred separately s.sys.account = acc s.sysAcc.Store(acc) } if err != nil { return awcsti, fmt.Errorf("error resolving system account: %v", err) } // If we have defined a system account here check to see if its just us and the $G account. // We would do this to add user/pass to the system account. If this is the case add in // no-auth-user for $G. // Only do this if non-operator mode and we did not have an authorization block defined. if len(opts.TrustedOperators) == 0 && numAccounts == 2 && opts.NoAuthUser == _EMPTY_ && !opts.authBlockDefined { // If we come here from config reload, let's not recreate the fake user name otherwise // it will cause currently clients to be disconnected. uname := s.sysAccOnlyNoAuthUser if uname == _EMPTY_ { // Create a unique name so we do not collide. var b [8]byte rn := rand.Int63() for i, l := 0, rn; i < len(b); i++ { b[i] = digits[l%base] l /= base } uname = fmt.Sprintf("nats-%s", b[:]) s.sysAccOnlyNoAuthUser = uname } opts.Users = append(opts.Users, &User{Username: uname, Password: uname[6:], Account: s.gacc}) opts.NoAuthUser = uname } } // Add any required exports from system account. if s.sys != nil { s.mu.Unlock() s.addSystemAccountExports(s.sys.account) s.mu.Lock() } return awcsti, nil } // Setup the account resolver. For memory resolver, make sure the JWTs are // properly formed but do not enforce expiration etc. // Lock is held on entry, but may be released/reacquired during this call. func (s *Server) configureResolver() error { opts := s.getOpts() s.accResolver = opts.AccountResolver if opts.AccountResolver != nil { // For URL resolver, set the TLSConfig if specified. if opts.AccountResolverTLSConfig != nil { if ar, ok := opts.AccountResolver.(*URLAccResolver); ok { if t, ok := ar.c.Transport.(*http.Transport); ok { t.CloseIdleConnections() t.TLSClientConfig = opts.AccountResolverTLSConfig.Clone() } } } if len(opts.resolverPreloads) > 0 { // Lock ordering is account resolver -> server, so we need to release // the lock and reacquire it when done with account resolver's calls. ar := s.accResolver s.mu.Unlock() defer s.mu.Lock() if ar.IsReadOnly() { return fmt.Errorf("resolver preloads only available for writeable resolver types MEM/DIR/CACHE_DIR") } for k, v := range opts.resolverPreloads { _, err := jwt.DecodeAccountClaims(v) if err != nil { return fmt.Errorf("preload account error for %q: %v", k, err) } ar.Store(k, v) } } } return nil } // This will check preloads for validation issues. func (s *Server) checkResolvePreloads() { opts := s.getOpts() // We can just check the read-only opts versions here, that way we do not need // to grab server lock or access s.accResolver. for k, v := range opts.resolverPreloads { claims, err := jwt.DecodeAccountClaims(v) if err != nil { s.Errorf("Preloaded account [%s] not valid", k) continue } // Check if it is expired. vr := jwt.CreateValidationResults() claims.Validate(vr) if vr.IsBlocking(true) { s.Warnf("Account [%s] has validation issues:", k) for _, v := range vr.Issues { s.Warnf(" - %s", v.Description) } } } } // Determines if we are in pre NATS 2.0 setup with no accounts. func (s *Server) globalAccountOnly() bool { var hasOthers bool if s.trustedKeys != nil { return false } s.mu.RLock() s.accounts.Range(func(k, v any) bool { acc := v.(*Account) // Ignore global and system if acc == s.gacc || (s.sys != nil && acc == s.sys.account) { return true } hasOthers = true return false }) s.mu.RUnlock() return !hasOthers } // Determines if this server is in standalone mode, meaning no routes or gateways. func (s *Server) standAloneMode() bool { opts := s.getOpts() return opts.Cluster.Port == 0 && opts.Gateway.Port == 0 } func (s *Server) configuredRoutes() int { return len(s.getOpts().Routes) } // activePeers is used in bootstrapping raft groups like the JetStream meta controller. func (s *Server) ActivePeers() (peers []string) { s.nodeToInfo.Range(func(k, v any) bool { si := v.(nodeInfo) if !si.offline { peers = append(peers, k.(string)) } return true }) return peers } // isTrustedIssuer will check that the issuer is a trusted public key. // This is used to make sure an account was signed by a trusted operator. func (s *Server) isTrustedIssuer(issuer string) bool { s.mu.RLock() defer s.mu.RUnlock() // If we are not running in trusted mode and there is no issuer, that is ok. if s.trustedKeys == nil && issuer == _EMPTY_ { return true } for _, tk := range s.trustedKeys { if tk == issuer { return true } } return false } // processTrustedKeys will process binary stamped and // options-based trusted nkeys. Returns success. func (s *Server) processTrustedKeys() bool { s.strictSigningKeyUsage = map[string]struct{}{} opts := s.getOpts() if trustedKeys != _EMPTY_ && !s.initStampedTrustedKeys() { return false } else if opts.TrustedKeys != nil { for _, key := range opts.TrustedKeys { if !nkeys.IsValidPublicOperatorKey(key) { return false } } s.trustedKeys = append([]string(nil), opts.TrustedKeys...) for _, claim := range opts.TrustedOperators { if !claim.StrictSigningKeyUsage { continue } for _, key := range claim.SigningKeys { s.strictSigningKeyUsage[key] = struct{}{} } } } return true } // checkTrustedKeyString will check that the string is a valid array // of public operator nkeys. func checkTrustedKeyString(keys string) []string { tks := strings.Fields(keys) if len(tks) == 0 { return nil } // Walk all the keys and make sure they are valid. for _, key := range tks { if !nkeys.IsValidPublicOperatorKey(key) { return nil } } return tks } // initStampedTrustedKeys will check the stamped trusted keys // and will set the server field 'trustedKeys'. Returns whether // it succeeded or not. func (s *Server) initStampedTrustedKeys() bool { // Check to see if we have an override in options, which will cause us to fail. if len(s.getOpts().TrustedKeys) > 0 { return false } tks := checkTrustedKeyString(trustedKeys) if len(tks) == 0 { return false } s.trustedKeys = tks return true } // PrintAndDie is exported for access in other packages. func PrintAndDie(msg string) { fmt.Fprintln(os.Stderr, msg) os.Exit(1) } // PrintServerAndExit will print our version and exit. func PrintServerAndExit() { fmt.Printf("nats-server: v%s\n", VERSION) os.Exit(0) } // ProcessCommandLineArgs takes the command line arguments // validating and setting flags for handling in case any // sub command was present. func ProcessCommandLineArgs(cmd *flag.FlagSet) (showVersion bool, showHelp bool, err error) { if len(cmd.Args()) > 0 { arg := cmd.Args()[0] switch strings.ToLower(arg) { case "version": return true, false, nil case "help": return false, true, nil default: return false, false, fmt.Errorf("unrecognized command: %q", arg) } } return false, false, nil } // Public version. func (s *Server) Running() bool { return s.isRunning() } // Protected check on running state func (s *Server) isRunning() bool { return s.running.Load() } func (s *Server) logPid() error { pidStr := strconv.Itoa(os.Getpid()) return os.WriteFile(s.getOpts().PidFile, []byte(pidStr), defaultFilePerms) } // numReservedAccounts will return the number of reserved accounts configured in the server. // Currently this is 1, one for the global default account. func (s *Server) numReservedAccounts() int { return 1 } // NumActiveAccounts reports number of active accounts on this server. func (s *Server) NumActiveAccounts() int32 { return atomic.LoadInt32(&s.activeAccounts) } // incActiveAccounts() just adds one under lock. func (s *Server) incActiveAccounts() { atomic.AddInt32(&s.activeAccounts, 1) } // decActiveAccounts() just subtracts one under lock. func (s *Server) decActiveAccounts() { atomic.AddInt32(&s.activeAccounts, -1) } // This should be used for testing only. Will be slow since we have to // range over all accounts in the sync.Map to count. func (s *Server) numAccounts() int { count := 0 s.mu.RLock() s.accounts.Range(func(k, v any) bool { count++ return true }) s.mu.RUnlock() return count } // NumLoadedAccounts returns the number of loaded accounts. func (s *Server) NumLoadedAccounts() int { return s.numAccounts() } // LookupOrRegisterAccount will return the given account if known or create a new entry. func (s *Server) LookupOrRegisterAccount(name string) (account *Account, isNew bool) { s.mu.Lock() defer s.mu.Unlock() if v, ok := s.accounts.Load(name); ok { return v.(*Account), false } acc := NewAccount(name) s.registerAccountNoLock(acc) return acc, true } // RegisterAccount will register an account. The account must be new // or this call will fail. func (s *Server) RegisterAccount(name string) (*Account, error) { s.mu.Lock() defer s.mu.Unlock() if _, ok := s.accounts.Load(name); ok { return nil, ErrAccountExists } acc := NewAccount(name) s.registerAccountNoLock(acc) return acc, nil } // SetSystemAccount will set the internal system account. // If root operators are present it will also check validity. func (s *Server) SetSystemAccount(accName string) error { // Lookup from sync.Map first. if v, ok := s.accounts.Load(accName); ok { return s.setSystemAccount(v.(*Account)) } // If we are here we do not have local knowledge of this account. // Do this one by hand to return more useful error. ac, jwt, err := s.fetchAccountClaims(accName) if err != nil { return err } acc := s.buildInternalAccount(ac) acc.claimJWT = jwt // Due to race, we need to make sure that we are not // registering twice. if racc := s.registerAccount(acc); racc != nil { return nil } return s.setSystemAccount(acc) } // SystemAccount returns the system account if set. func (s *Server) SystemAccount() *Account { return s.sysAcc.Load() } // GlobalAccount returns the global account. // Default clients will use the global account. func (s *Server) GlobalAccount() *Account { s.mu.RLock() defer s.mu.RUnlock() return s.gacc } // SetDefaultSystemAccount will create a default system account if one is not present. func (s *Server) SetDefaultSystemAccount() error { if _, isNew := s.LookupOrRegisterAccount(DEFAULT_SYSTEM_ACCOUNT); !isNew { return nil } s.Debugf("Created system account: %q", DEFAULT_SYSTEM_ACCOUNT) return s.SetSystemAccount(DEFAULT_SYSTEM_ACCOUNT) } // Assign a system account. Should only be called once. // This sets up a server to send and receive messages from // inside the server itself. func (s *Server) setSystemAccount(acc *Account) error { if acc == nil { return ErrMissingAccount } // Don't try to fix this here. if acc.IsExpired() { return ErrAccountExpired } // If we are running with trusted keys for an operator // make sure we check the account is legit. if !s.isTrustedIssuer(acc.Issuer) { return ErrAccountValidation } s.mu.Lock() if s.sys != nil { s.mu.Unlock() return ErrAccountExists } // This is here in an attempt to quiet the race detector and not have to place // locks on fast path for inbound messages and checking service imports. acc.mu.Lock() if acc.imports.services == nil { acc.imports.services = make(map[string]*serviceImport) } acc.mu.Unlock() s.sys = &internal{ account: acc, client: s.createInternalSystemClient(), seq: 1, sid: 1, servers: make(map[string]*serverUpdate), replies: make(map[string]msgHandler), sendq: newIPQueue[*pubMsg](s, "System sendQ"), recvq: newIPQueue[*inSysMsg](s, "System recvQ"), recvqp: newIPQueue[*inSysMsg](s, "System recvQ Pings"), resetCh: make(chan struct{}), sq: s.newSendQ(), statsz: statsHBInterval, orphMax: 5 * eventsHBInterval, chkOrph: 3 * eventsHBInterval, } recvq, recvqp := s.sys.recvq, s.sys.recvqp s.sys.wg.Add(1) s.mu.Unlock() // Store in atomic for fast lookup. s.sysAcc.Store(acc) // Register with the account. s.sys.client.registerWithAccount(acc) s.addSystemAccountExports(acc) // Start our internal loop to serialize outbound messages. // We do our own wg here since we will stop first during shutdown. go s.internalSendLoop(&s.sys.wg) // Start the internal loop for inbound messages. go s.internalReceiveLoop(recvq) // Start the internal loop for inbound STATSZ/Ping messages. go s.internalReceiveLoop(recvqp) // Start up our general subscriptions s.initEventTracking() // Track for dead remote servers. s.wrapChk(s.startRemoteServerSweepTimer)() // Send out statsz updates periodically. s.wrapChk(s.startStatszTimer)() // If we have existing accounts make sure we enable account tracking. s.mu.Lock() s.accounts.Range(func(k, v any) bool { acc := v.(*Account) s.enableAccountTracking(acc) return true }) s.mu.Unlock() return nil } // Creates an internal system client. func (s *Server) createInternalSystemClient() *client { return s.createInternalClient(SYSTEM) } // Creates an internal jetstream client. func (s *Server) createInternalJetStreamClient() *client { return s.createInternalClient(JETSTREAM) } // Creates an internal client for Account. func (s *Server) createInternalAccountClient() *client { return s.createInternalClient(ACCOUNT) } // Internal clients. kind should be SYSTEM or JETSTREAM func (s *Server) createInternalClient(kind int) *client { if kind != SYSTEM && kind != JETSTREAM && kind != ACCOUNT { return nil } now := time.Now() c := &client{srv: s, kind: kind, opts: internalOpts, msubs: -1, mpay: -1, start: now, last: now} c.initClient() c.echo = false c.headers = true c.flags.set(noReconnect) return c } // Determine if accounts should track subscriptions for // efficient propagation. // Lock should be held on entry. func (s *Server) shouldTrackSubscriptions() bool { opts := s.getOpts() return (opts.Cluster.Port != 0 || opts.Gateway.Port != 0) } // Invokes registerAccountNoLock under the protection of the server lock. // That is, server lock is acquired/released in this function. // See registerAccountNoLock for comment on returned value. func (s *Server) registerAccount(acc *Account) *Account { s.mu.Lock() racc := s.registerAccountNoLock(acc) s.mu.Unlock() return racc } // Helper to set the sublist based on preferences. func (s *Server) setAccountSublist(acc *Account) { if acc != nil && acc.sl == nil { opts := s.getOpts() if opts != nil && opts.NoSublistCache { acc.sl = NewSublistNoCache() } else { acc.sl = NewSublistWithCache() } } } // Registers an account in the server. // Due to some locking considerations, we may end-up trying // to register the same account twice. This function will // then return the already registered account. // Lock should be held on entry. func (s *Server) registerAccountNoLock(acc *Account) *Account { // We are under the server lock. Lookup from map, if present // return existing account. if a, _ := s.accounts.Load(acc.Name); a != nil { s.tmpAccounts.Delete(acc.Name) return a.(*Account) } // Finish account setup and store. s.setAccountSublist(acc) acc.mu.Lock() s.setRouteInfo(acc) if acc.clients == nil { acc.clients = make(map[*client]struct{}) } // If we are capable of routing we will track subscription // information for efficient interest propagation. // During config reload, it is possible that account was // already created (global account), so use locking and // make sure we create only if needed. // TODO(dlc)- Double check that we need this for GWs. if acc.rm == nil && s.opts != nil && s.shouldTrackSubscriptions() { acc.rm = make(map[string]int32) acc.lqws = make(map[string]int32) } acc.srv = s acc.updated = time.Now() accName := acc.Name jsEnabled := len(acc.jsLimits) > 0 acc.mu.Unlock() if opts := s.getOpts(); opts != nil && len(opts.JsAccDefaultDomain) > 0 { if defDomain, ok := opts.JsAccDefaultDomain[accName]; ok { if jsEnabled { s.Warnf("Skipping Default Domain %q, set for JetStream enabled account %q", defDomain, accName) } else if defDomain != _EMPTY_ { for src, dest := range generateJSMappingTable(defDomain) { // flip src and dest around so the domain is inserted s.Noticef("Adding default domain mapping %q -> %q to account %q %p", dest, src, accName, acc) if err := acc.AddMapping(dest, src); err != nil { s.Errorf("Error adding JetStream default domain mapping: %v", err) } } } } } s.accounts.Store(acc.Name, acc) s.tmpAccounts.Delete(acc.Name) s.enableAccountTracking(acc) // Can not have server lock here. s.mu.Unlock() s.registerSystemImports(acc) // Starting 2.9.0, we are phasing out the optimistic mode, so change // the account to interest-only mode (except if instructed not to do // it in some tests). if s.gateway.enabled && !gwDoNotForceInterestOnlyMode { s.switchAccountToInterestMode(acc.GetName()) } s.mu.Lock() return nil } // Sets the account's routePoolIdx depending on presence or not of // pooling or per-account routes. Also updates a map used by // gateway code to retrieve a route based on some route hash. // // Both Server and Account lock held on entry. func (s *Server) setRouteInfo(acc *Account) { // If there is a dedicated route configured for this account if _, ok := s.accRoutes[acc.Name]; ok { // We want the account name to be in the map, but we don't // need a value (we could store empty string) s.accRouteByHash.Store(acc.Name, nil) // Set the route pool index to -1 so that it is easy when // ranging over accounts to exclude those accounts when // trying to get accounts for a given pool index. acc.routePoolIdx = accDedicatedRoute } else { // If pool size more than 1, we will compute a hash code and // use modulo to assign to an index of the pool slice. For 1 // and below, all accounts will be bound to the single connection // at index 0. acc.routePoolIdx = s.computeRoutePoolIdx(acc) if s.routesPoolSize > 1 { s.accRouteByHash.Store(acc.Name, acc.routePoolIdx) } } } // Returns a route pool index for this account based on the given pool size. // Account lock is held on entry (account's name is accessed but immutable // so could be called without account's lock). // Server lock held on entry. func (s *Server) computeRoutePoolIdx(acc *Account) int { if s.routesPoolSize <= 1 { return 0 } h := fnv.New32a() h.Write([]byte(acc.Name)) sum32 := h.Sum32() return int((sum32 % uint32(s.routesPoolSize))) } // lookupAccount is a function to return the account structure // associated with an account name. // Lock MUST NOT be held upon entry. func (s *Server) lookupAccount(name string) (*Account, error) { var acc *Account if v, ok := s.accounts.Load(name); ok { acc = v.(*Account) } if acc != nil { // If we are expired and we have a resolver, then // return the latest information from the resolver. if acc.IsExpired() { s.Debugf("Requested account [%s] has expired", name) if s.AccountResolver() != nil { if err := s.updateAccount(acc); err != nil { // This error could mask expired, so just return expired here. return nil, ErrAccountExpired } } else { return nil, ErrAccountExpired } } return acc, nil } // If we have a resolver see if it can fetch the account. if s.AccountResolver() == nil { return nil, ErrMissingAccount } return s.fetchAccount(name) } // LookupAccount is a public function to return the account structure // associated with name. func (s *Server) LookupAccount(name string) (*Account, error) { return s.lookupAccount(name) } // This will fetch new claims and if found update the account with new claims. // Lock MUST NOT be held upon entry. func (s *Server) updateAccount(acc *Account) error { acc.mu.RLock() // TODO(dlc) - Make configurable if !acc.incomplete && time.Since(acc.updated) < time.Second { acc.mu.RUnlock() s.Debugf("Requested account update for [%s] ignored, too soon", acc.Name) return ErrAccountResolverUpdateTooSoon } acc.mu.RUnlock() claimJWT, err := s.fetchRawAccountClaims(acc.Name) if err != nil { return err } return s.updateAccountWithClaimJWT(acc, claimJWT) } // updateAccountWithClaimJWT will check and apply the claim update. // Lock MUST NOT be held upon entry. func (s *Server) updateAccountWithClaimJWT(acc *Account, claimJWT string) error { if acc == nil { return ErrMissingAccount } acc.mu.RLock() sameClaim := acc.claimJWT != _EMPTY_ && acc.claimJWT == claimJWT && !acc.incomplete acc.mu.RUnlock() if sameClaim { s.Debugf("Requested account update for [%s], same claims detected", acc.Name) return nil } accClaims, _, err := s.verifyAccountClaims(claimJWT) if err == nil && accClaims != nil { acc.mu.Lock() // if an account is updated with a different operator signing key, we want to // show a consistent issuer. acc.Issuer = accClaims.Issuer if acc.Name != accClaims.Subject { acc.mu.Unlock() return ErrAccountValidation } acc.mu.Unlock() s.UpdateAccountClaims(acc, accClaims) acc.mu.Lock() // needs to be set after update completed. // This causes concurrent calls to return with sameClaim=true if the change is effective. acc.claimJWT = claimJWT acc.mu.Unlock() return nil } return err } // fetchRawAccountClaims will grab raw account claims iff we have a resolver. // Lock is NOT held upon entry. func (s *Server) fetchRawAccountClaims(name string) (string, error) { accResolver := s.AccountResolver() if accResolver == nil { return _EMPTY_, ErrNoAccountResolver } // Need to do actual Fetch start := time.Now() claimJWT, err := fetchAccount(accResolver, name) fetchTime := time.Since(start) if fetchTime > time.Second { s.Warnf("Account [%s] fetch took %v", name, fetchTime) } else { s.Debugf("Account [%s] fetch took %v", name, fetchTime) } if err != nil { s.Warnf("Account fetch failed: %v", err) return "", err } return claimJWT, nil } // fetchAccountClaims will attempt to fetch new claims if a resolver is present. // Lock is NOT held upon entry. func (s *Server) fetchAccountClaims(name string) (*jwt.AccountClaims, string, error) { claimJWT, err := s.fetchRawAccountClaims(name) if err != nil { return nil, _EMPTY_, err } var claim *jwt.AccountClaims claim, claimJWT, err = s.verifyAccountClaims(claimJWT) if claim != nil && claim.Subject != name { return nil, _EMPTY_, ErrAccountValidation } return claim, claimJWT, err } // verifyAccountClaims will decode and validate any account claims. func (s *Server) verifyAccountClaims(claimJWT string) (*jwt.AccountClaims, string, error) { accClaims, err := jwt.DecodeAccountClaims(claimJWT) if err != nil { return nil, _EMPTY_, err } if !s.isTrustedIssuer(accClaims.Issuer) { return nil, _EMPTY_, ErrAccountValidation } vr := jwt.CreateValidationResults() accClaims.Validate(vr) if vr.IsBlocking(true) { return nil, _EMPTY_, ErrAccountValidation } return accClaims, claimJWT, nil } // This will fetch an account from a resolver if defined. // Lock is NOT held upon entry. func (s *Server) fetchAccount(name string) (*Account, error) { accClaims, claimJWT, err := s.fetchAccountClaims(name) if accClaims == nil { return nil, err } acc := s.buildInternalAccount(accClaims) // Due to possible race, if registerAccount() returns a non // nil account, it means the same account was already // registered and we should use this one. if racc := s.registerAccount(acc); racc != nil { // Update with the new claims in case they are new. if err = s.updateAccountWithClaimJWT(racc, claimJWT); err != nil { return nil, err } return racc, nil } // The sub imports may have been setup but will not have had their // subscriptions properly setup. Do that here. var needImportSubs bool acc.mu.Lock() acc.claimJWT = claimJWT if len(acc.imports.services) > 0 { if acc.ic == nil { acc.ic = s.createInternalAccountClient() acc.ic.acc = acc } needImportSubs = true } acc.mu.Unlock() // Do these outside the lock. if needImportSubs { acc.addAllServiceImportSubs() } return acc, nil } // Start up the server, this will not block. // // WaitForShutdown can be used to block and wait for the server to shutdown properly if needed // after calling s.Shutdown() func (s *Server) Start() { s.Noticef("Starting nats-server") gc := gitCommit if gc == _EMPTY_ { gc = "not set" } // Snapshot server options. opts := s.getOpts() clusterName := s.ClusterName() s.Noticef(" Version: %s", VERSION) s.Noticef(" Git: [%s]", gc) s.Debugf(" Go build: %s", s.info.GoVersion) if clusterName != _EMPTY_ { s.Noticef(" Cluster: %s", clusterName) } s.Noticef(" Name: %s", s.info.Name) if opts.JetStream { s.Noticef(" Node: %s", getHash(s.info.Name)) } s.Noticef(" ID: %s", s.info.ID) defer s.Noticef("Server is ready") // Check for insecure configurations. s.checkAuthforWarnings() // Avoid RACE between Start() and Shutdown() s.running.Store(true) s.mu.Lock() // Update leafNodeEnabled in case options have changed post NewServer() // and before Start() (we should not be able to allow that, but server has // direct reference to user-provided options - at least before a Reload() is // performed. s.leafNodeEnabled = opts.LeafNode.Port != 0 || len(opts.LeafNode.Remotes) > 0 s.mu.Unlock() s.grMu.Lock() s.grRunning = true s.grMu.Unlock() s.startRateLimitLogExpiration() // Pprof http endpoint for the profiler. if opts.ProfPort != 0 { s.StartProfiler() } else { // It's still possible to access this profile via a SYS endpoint, so set // this anyway. (Otherwise StartProfiler would have called it.) s.setBlockProfileRate(opts.ProfBlockRate) } if opts.ConfigFile != _EMPTY_ { s.Noticef("Using configuration file: %s", opts.ConfigFile) } hasOperators := len(opts.TrustedOperators) > 0 if hasOperators { s.Noticef("Trusted Operators") } for _, opc := range opts.TrustedOperators { s.Noticef(" System : %q", opc.Audience) s.Noticef(" Operator: %q", opc.Name) s.Noticef(" Issued : %v", time.Unix(opc.IssuedAt, 0)) switch opc.Expires { case 0: s.Noticef(" Expires : Never") default: s.Noticef(" Expires : %v", time.Unix(opc.Expires, 0)) } } if hasOperators && opts.SystemAccount == _EMPTY_ { s.Warnf("Trusted Operators should utilize a System Account") } if opts.MaxPayload > MAX_PAYLOAD_MAX_SIZE { s.Warnf("Maximum payloads over %v are generally discouraged and could lead to poor performance", friendlyBytes(int64(MAX_PAYLOAD_MAX_SIZE))) } if len(opts.JsAccDefaultDomain) > 0 { s.Warnf("The option `default_js_domain` is a temporary backwards compatibility measure and will be removed") } // If we have a memory resolver, check the accounts here for validation exceptions. // This allows them to be logged right away vs when they are accessed via a client. if hasOperators && len(opts.resolverPreloads) > 0 { s.checkResolvePreloads() } // Log the pid to a file. if opts.PidFile != _EMPTY_ { if err := s.logPid(); err != nil { s.Fatalf("Could not write pidfile: %v", err) return } } // Setup system account which will start the eventing stack. if sa := opts.SystemAccount; sa != _EMPTY_ { if err := s.SetSystemAccount(sa); err != nil { s.Fatalf("Can't set system account: %v", err) return } } else if !opts.NoSystemAccount { // We will create a default system account here. s.SetDefaultSystemAccount() } // Start monitoring before enabling other subsystems of the // server to be able to monitor during startup. if err := s.StartMonitoring(); err != nil { s.Fatalf("Can't start monitoring: %v", err) return } // Start up resolver machinery. if ar := s.AccountResolver(); ar != nil { if err := ar.Start(s); err != nil { s.Fatalf("Could not start resolver: %v", err) return } // In operator mode, when the account resolver depends on an external system and // the system account is the bootstrapping account, start fetching it. if len(opts.TrustedOperators) == 1 && opts.SystemAccount != _EMPTY_ && opts.SystemAccount != DEFAULT_SYSTEM_ACCOUNT { opts := s.getOpts() _, isMemResolver := ar.(*MemAccResolver) if v, ok := s.accounts.Load(opts.SystemAccount); !isMemResolver && ok && v.(*Account).claimJWT == _EMPTY_ { s.Noticef("Using bootstrapping system account") s.startGoRoutine(func() { defer s.grWG.Done() t := time.NewTicker(time.Second) defer t.Stop() for { select { case <-s.quitCh: return case <-t.C: sacc := s.SystemAccount() if claimJWT, err := fetchAccount(ar, opts.SystemAccount); err != nil { continue } else if err = s.updateAccountWithClaimJWT(sacc, claimJWT); err != nil { continue } s.Noticef("System account fetched and updated") return } } }) } } } // Start expiration of mapped GW replies, regardless if // this server is configured with gateway or not. s.startGWReplyMapExpiration() // Check if JetStream has been enabled. This needs to be after // the system account setup above. JetStream will create its // own system account if one is not present. if opts.JetStream { // Make sure someone is not trying to enable on the system account. if sa := s.SystemAccount(); sa != nil && len(sa.jsLimits) > 0 { s.Fatalf("Not allowed to enable JetStream on the system account") } cfg := &JetStreamConfig{ StoreDir: opts.StoreDir, SyncInterval: opts.SyncInterval, SyncAlways: opts.SyncAlways, MaxMemory: opts.JetStreamMaxMemory, MaxStore: opts.JetStreamMaxStore, Domain: opts.JetStreamDomain, CompressOK: true, UniqueTag: opts.JetStreamUniqueTag, } if err := s.EnableJetStream(cfg); err != nil { s.Fatalf("Can't start JetStream: %v", err) return } } else { // Check to see if any configured accounts have JetStream enabled. sa, ga := s.SystemAccount(), s.GlobalAccount() var hasSys, hasGlobal bool var total int s.accounts.Range(func(k, v any) bool { total++ acc := v.(*Account) if acc == sa { hasSys = true } else if acc == ga { hasGlobal = true } acc.mu.RLock() hasJs := len(acc.jsLimits) > 0 acc.mu.RUnlock() if hasJs { s.checkJetStreamExports() acc.enableAllJetStreamServiceImportsAndMappings() } return true }) // If we only have the system account and the global account and we are not standalone, // go ahead and enable JS on $G in case we are in simple mixed mode setup. if total == 2 && hasSys && hasGlobal && !s.standAloneMode() { ga.mu.Lock() ga.jsLimits = map[string]JetStreamAccountLimits{ _EMPTY_: dynamicJSAccountLimits, } ga.mu.Unlock() s.checkJetStreamExports() ga.enableAllJetStreamServiceImportsAndMappings() } } // Start OCSP Stapling monitoring for TLS certificates if enabled. Hook TLS handshake for // OCSP check on peers (LEAF and CLIENT kind) if enabled. s.startOCSPMonitoring() // Configure OCSP Response Cache for peer OCSP checks if enabled. s.initOCSPResponseCache() // Start up gateway if needed. Do this before starting the routes, because // we want to resolve the gateway host:port so that this information can // be sent to other routes. if opts.Gateway.Port != 0 { s.startGateways() } // Start websocket server if needed. Do this before starting the routes, and // leaf node because we want to resolve the gateway host:port so that this // information can be sent to other routes. if opts.Websocket.Port != 0 { s.startWebsocketServer() } // Start up listen if we want to accept leaf node connections. if opts.LeafNode.Port != 0 { // Will resolve or assign the advertise address for the leafnode listener. // We need that in StartRouting(). s.startLeafNodeAcceptLoop() } // Solicit remote servers for leaf node connections. if len(opts.LeafNode.Remotes) > 0 { s.solicitLeafNodeRemotes(opts.LeafNode.Remotes) if opts.Cluster.Name == opts.ServerName && strings.ContainsFunc(opts.Cluster.Name, unicode.IsSpace) { s.Warnf("Server name has spaces and used as the cluster name, leaf remotes may not connect properly") } } // TODO (ik): I wanted to refactor this by starting the client // accept loop first, that is, it would resolve listen spec // in place, but start the accept-for-loop in a different go // routine. This would get rid of the synchronization between // this function and StartRouting, which I also would have wanted // to refactor, but both AcceptLoop() and StartRouting() have // been exported and not sure if that would break users using them. // We could mark them as deprecated and remove in a release or two... // The Routing routine needs to wait for the client listen // port to be opened and potential ephemeral port selected. clientListenReady := make(chan struct{}) // MQTT if opts.MQTT.Port != 0 { s.startMQTT() } // Start up routing as well if needed. if opts.Cluster.Port != 0 { s.startGoRoutine(func() { s.StartRouting(clientListenReady) }) } if opts.PortsFileDir != _EMPTY_ { s.logPorts() } if opts.TLSRateLimit > 0 { s.startGoRoutine(s.logRejectedTLSConns) } // We've finished starting up. close(s.startupComplete) // Wait for clients. if !opts.DontListen { s.AcceptLoop(clientListenReady) } // Bring OSCP Response cache online after accept loop started in anticipation of NATS-enabled cache types s.startOCSPResponseCache() } func (s *Server) isShuttingDown() bool { return s.shutdown.Load() } // Shutdown will shutdown the server instance by kicking out the AcceptLoop // and closing all associated clients. func (s *Server) Shutdown() { if s == nil { return } // This is for JetStream R1 Pull Consumers to allow signaling // that pending pull requests are invalid. s.signalPullConsumers() // Transfer off any raft nodes that we are a leader by stepping them down. s.stepdownRaftNodes() // Shutdown the eventing system as needed. // This is done first to send out any messages for // account status. We will also clean up any // eventing items associated with accounts. s.shutdownEventing() // Prevent issues with multiple calls. if s.isShuttingDown() { return } s.mu.Lock() s.Noticef("Initiating Shutdown...") accRes := s.accResolver opts := s.getOpts() s.shutdown.Store(true) s.running.Store(false) s.grMu.Lock() s.grRunning = false s.grMu.Unlock() s.mu.Unlock() if accRes != nil { accRes.Close() } // Now check and shutdown jetstream. s.shutdownJetStream() // Now shutdown the nodes s.shutdownRaftNodes() s.mu.Lock() conns := make(map[uint64]*client) // Copy off the clients for i, c := range s.clients { conns[i] = c } // Copy off the connections that are not yet registered // in s.routes, but for which the readLoop has started s.grMu.Lock() for i, c := range s.grTmpClients { conns[i] = c } s.grMu.Unlock() // Copy off the routes s.forEachRoute(func(r *client) { r.mu.Lock() conns[r.cid] = r r.mu.Unlock() }) // Copy off the gateways s.getAllGatewayConnections(conns) // Copy off the leaf nodes for i, c := range s.leafs { conns[i] = c } // Number of done channel responses we expect. doneExpected := 0 // Kick client AcceptLoop() if s.listener != nil { doneExpected++ s.listener.Close() s.listener = nil } // Kick websocket server doneExpected += s.closeWebsocketServer() // Kick MQTT accept loop if s.mqtt.listener != nil { doneExpected++ s.mqtt.listener.Close() s.mqtt.listener = nil } // Kick leafnodes AcceptLoop() if s.leafNodeListener != nil { doneExpected++ s.leafNodeListener.Close() s.leafNodeListener = nil } // Kick route AcceptLoop() if s.routeListener != nil { doneExpected++ s.routeListener.Close() s.routeListener = nil } // Kick Gateway AcceptLoop() if s.gatewayListener != nil { doneExpected++ s.gatewayListener.Close() s.gatewayListener = nil } // Kick HTTP monitoring if its running if s.http != nil { doneExpected++ s.http.Close() s.http = nil } // Kick Profiling if its running if s.profiler != nil { doneExpected++ s.profiler.Close() } s.mu.Unlock() // Release go routines that wait on that channel close(s.quitCh) // Close client and route connections for _, c := range conns { c.setNoReconnect() c.closeConnection(ServerShutdown) } // Block until the accept loops exit for doneExpected > 0 { <-s.done doneExpected-- } // Wait for go routines to be done. s.grWG.Wait() if opts.PortsFileDir != _EMPTY_ { s.deletePortsFile(opts.PortsFileDir) } s.Noticef("Server Exiting..") // Stop OCSP Response Cache if s.ocsprc != nil { s.ocsprc.Stop(s) } // Close logger if applicable. It allows tests on Windows // to be able to do proper cleanup (delete log file). s.logging.RLock() log := s.logging.logger s.logging.RUnlock() if log != nil { if l, ok := log.(*logger.Logger); ok { l.Close() } } // Notify that the shutdown is complete close(s.shutdownComplete) } // Close the websocket server if running. If so, returns 1, else 0. // Server lock held on entry. func (s *Server) closeWebsocketServer() int { ws := &s.websocket ws.mu.Lock() hs := ws.server if hs != nil { ws.server = nil ws.listener = nil } ws.mu.Unlock() if hs != nil { hs.Close() return 1 } return 0 } // WaitForShutdown will block until the server has been fully shutdown. func (s *Server) WaitForShutdown() { <-s.shutdownComplete } // AcceptLoop is exported for easier testing. func (s *Server) AcceptLoop(clr chan struct{}) { // If we were to exit before the listener is setup properly, // make sure we close the channel. defer func() { if clr != nil { close(clr) } }() if s.isShuttingDown() { return } // Snapshot server options. opts := s.getOpts() // Setup state that can enable shutdown s.mu.Lock() hp := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) l, e := natsListen("tcp", hp) s.listenerErr = e if e != nil { s.mu.Unlock() s.Fatalf("Error listening on port: %s, %q", hp, e) return } s.Noticef("Listening for client connections on %s", net.JoinHostPort(opts.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port))) // Alert of TLS enabled. if opts.TLSConfig != nil { s.Noticef("TLS required for client connections") if opts.TLSHandshakeFirst && opts.TLSHandshakeFirstFallback == 0 { s.Warnf("Clients that are not using \"TLS Handshake First\" option will fail to connect") } } // If server was started with RANDOM_PORT (-1), opts.Port would be equal // to 0 at the beginning this function. So we need to get the actual port if opts.Port == 0 { // Write resolved port back to options. opts.Port = l.Addr().(*net.TCPAddr).Port } // Now that port has been set (if it was set to RANDOM), set the // server's info Host/Port with either values from Options or // ClientAdvertise. if err := s.setInfoHostPort(); err != nil { s.Fatalf("Error setting server INFO with ClientAdvertise value of %s, err=%v", opts.ClientAdvertise, err) l.Close() s.mu.Unlock() return } // Keep track of client connect URLs. We may need them later. s.clientConnectURLs = s.getClientConnectURLs() s.listener = l go s.acceptConnections(l, "Client", func(conn net.Conn) { s.createClient(conn) }, func(_ error) bool { if s.isLameDuckMode() { // Signal that we are not accepting new clients s.ldmCh <- true // Now wait for the Shutdown... <-s.quitCh return true } return false }) s.mu.Unlock() // Let the caller know that we are ready close(clr) clr = nil } // InProcessConn returns an in-process connection to the server, // avoiding the need to use a TCP listener for local connectivity // within the same process. This can be used regardless of the // state of the DontListen option. func (s *Server) InProcessConn() (net.Conn, error) { pl, pr := net.Pipe() if !s.startGoRoutine(func() { s.createClientInProcess(pl) s.grWG.Done() }) { pl.Close() pr.Close() return nil, fmt.Errorf("failed to create connection") } return pr, nil } func (s *Server) acceptConnections(l net.Listener, acceptName string, createFunc func(conn net.Conn), errFunc func(err error) bool) { tmpDelay := ACCEPT_MIN_SLEEP for { conn, err := l.Accept() if err != nil { if errFunc != nil && errFunc(err) { return } if tmpDelay = s.acceptError(acceptName, err, tmpDelay); tmpDelay < 0 { break } continue } tmpDelay = ACCEPT_MIN_SLEEP if !s.startGoRoutine(func() { s.reloadMu.RLock() createFunc(conn) s.reloadMu.RUnlock() s.grWG.Done() }) { conn.Close() } } s.Debugf(acceptName + " accept loop exiting..") s.done <- true } // This function sets the server's info Host/Port based on server Options. // Note that this function may be called during config reload, this is why // Host/Port may be reset to original Options if the ClientAdvertise option // is not set (since it may have previously been). func (s *Server) setInfoHostPort() error { // When this function is called, opts.Port is set to the actual listen // port (if option was originally set to RANDOM), even during a config // reload. So use of s.opts.Port is safe. opts := s.getOpts() if opts.ClientAdvertise != _EMPTY_ { h, p, err := parseHostPort(opts.ClientAdvertise, opts.Port) if err != nil { return err } s.info.Host = h s.info.Port = p } else { s.info.Host = opts.Host s.info.Port = opts.Port } return nil } // StartProfiler is called to enable dynamic profiling. func (s *Server) StartProfiler() { if s.isShuttingDown() { return } // Snapshot server options. opts := s.getOpts() port := opts.ProfPort // Check for Random Port if port == -1 { port = 0 } s.mu.Lock() hp := net.JoinHostPort(opts.Host, strconv.Itoa(port)) l, err := net.Listen("tcp", hp) if err != nil { s.mu.Unlock() s.Fatalf("error starting profiler: %s", err) return } s.Noticef("profiling port: %d", l.Addr().(*net.TCPAddr).Port) srv := &http.Server{ Addr: hp, Handler: http.DefaultServeMux, MaxHeaderBytes: 1 << 20, ReadTimeout: time.Second * 5, } s.profiler = l s.profilingServer = srv s.setBlockProfileRate(opts.ProfBlockRate) go func() { // if this errors out, it's probably because the server is being shutdown err := srv.Serve(l) if err != nil { if !s.isShuttingDown() { s.Fatalf("error starting profiler: %s", err) } } srv.Close() s.done <- true }() s.mu.Unlock() } func (s *Server) setBlockProfileRate(rate int) { // Passing i ProfBlockRate <= 0 here will disable or > 0 will enable. runtime.SetBlockProfileRate(rate) if rate > 0 { s.Warnf("Block profiling is enabled (rate %d), this may have a performance impact", rate) } } // StartHTTPMonitoring will enable the HTTP monitoring port. // DEPRECATED: Should use StartMonitoring. func (s *Server) StartHTTPMonitoring() { s.startMonitoring(false) } // StartHTTPSMonitoring will enable the HTTPS monitoring port. // DEPRECATED: Should use StartMonitoring. func (s *Server) StartHTTPSMonitoring() { s.startMonitoring(true) } // StartMonitoring starts the HTTP or HTTPs server if needed. func (s *Server) StartMonitoring() error { // Snapshot server options. opts := s.getOpts() // Specifying both HTTP and HTTPS ports is a misconfiguration if opts.HTTPPort != 0 && opts.HTTPSPort != 0 { return fmt.Errorf("can't specify both HTTP (%v) and HTTPs (%v) ports", opts.HTTPPort, opts.HTTPSPort) } var err error if opts.HTTPPort != 0 { err = s.startMonitoring(false) } else if opts.HTTPSPort != 0 { if opts.TLSConfig == nil { return fmt.Errorf("TLS cert and key required for HTTPS") } err = s.startMonitoring(true) } return err } // HTTP endpoints const ( RootPath = "/" VarzPath = "/varz" ConnzPath = "/connz" RoutezPath = "/routez" GatewayzPath = "/gatewayz" LeafzPath = "/leafz" SubszPath = "/subsz" StackszPath = "/stacksz" AccountzPath = "/accountz" AccountStatzPath = "/accstatz" JszPath = "/jsz" HealthzPath = "/healthz" IPQueuesPath = "/ipqueuesz" RaftzPath = "/raftz" ) func (s *Server) basePath(p string) string { return path.Join(s.httpBasePath, p) } type captureHTTPServerLog struct { s *Server prefix string } func (cl *captureHTTPServerLog) Write(p []byte) (int, error) { var buf [128]byte var b = buf[:0] b = append(b, []byte(cl.prefix)...) offset := 0 if bytes.HasPrefix(p, []byte("http:")) { offset = 6 } b = append(b, p[offset:]...) cl.s.Errorf(string(b)) return len(p), nil } // The TLS configuration is passed to the listener when the monitoring // "server" is setup. That prevents TLS configuration updates on reload // from being used. By setting this function in tls.Config.GetConfigForClient // we instruct the TLS handshake to ask for the tls configuration to be // used for a specific client. We don't care which client, we always use // the same TLS configuration. func (s *Server) getMonitoringTLSConfig(_ *tls.ClientHelloInfo) (*tls.Config, error) { opts := s.getOpts() tc := opts.TLSConfig.Clone() tc.ClientAuth = tls.NoClientCert return tc, nil } // Start the monitoring server func (s *Server) startMonitoring(secure bool) error { if s.isShuttingDown() { return nil } // Snapshot server options. opts := s.getOpts() var ( hp string err error httpListener net.Listener port int ) monitorProtocol := "http" if secure { monitorProtocol += "s" port = opts.HTTPSPort if port == -1 { port = 0 } hp = net.JoinHostPort(opts.HTTPHost, strconv.Itoa(port)) config := opts.TLSConfig.Clone() if !s.ocspPeerVerify { config.GetConfigForClient = s.getMonitoringTLSConfig config.ClientAuth = tls.NoClientCert } httpListener, err = tls.Listen("tcp", hp, config) } else { port = opts.HTTPPort if port == -1 { port = 0 } hp = net.JoinHostPort(opts.HTTPHost, strconv.Itoa(port)) httpListener, err = net.Listen("tcp", hp) } if err != nil { return fmt.Errorf("can't listen to the monitor port: %v", err) } rport := httpListener.Addr().(*net.TCPAddr).Port s.Noticef("Starting %s monitor on %s", monitorProtocol, net.JoinHostPort(opts.HTTPHost, strconv.Itoa(rport))) mux := http.NewServeMux() // Root mux.HandleFunc(s.basePath(RootPath), s.HandleRoot) // Varz mux.HandleFunc(s.basePath(VarzPath), s.HandleVarz) // Connz mux.HandleFunc(s.basePath(ConnzPath), s.HandleConnz) // Routez mux.HandleFunc(s.basePath(RoutezPath), s.HandleRoutez) // Gatewayz mux.HandleFunc(s.basePath(GatewayzPath), s.HandleGatewayz) // Leafz mux.HandleFunc(s.basePath(LeafzPath), s.HandleLeafz) // Subz mux.HandleFunc(s.basePath(SubszPath), s.HandleSubsz) // Subz alias for backwards compatibility mux.HandleFunc(s.basePath("/subscriptionsz"), s.HandleSubsz) // Stacksz mux.HandleFunc(s.basePath(StackszPath), s.HandleStacksz) // Accountz mux.HandleFunc(s.basePath(AccountzPath), s.HandleAccountz) // Accstatz mux.HandleFunc(s.basePath(AccountStatzPath), s.HandleAccountStatz) // Jsz mux.HandleFunc(s.basePath(JszPath), s.HandleJsz) // Healthz mux.HandleFunc(s.basePath(HealthzPath), s.HandleHealthz) // IPQueuesz mux.HandleFunc(s.basePath(IPQueuesPath), s.HandleIPQueuesz) // Raftz mux.HandleFunc(s.basePath(RaftzPath), s.HandleRaftz) // Do not set a WriteTimeout because it could cause cURL/browser // to return empty response or unable to display page if the // server needs more time to build the response. srv := &http.Server{ Addr: hp, Handler: mux, MaxHeaderBytes: 1 << 20, ErrorLog: log.New(&captureHTTPServerLog{s, "monitoring: "}, _EMPTY_, 0), ReadHeaderTimeout: time.Second * 5, } s.mu.Lock() s.http = httpListener s.httpHandler = mux s.monitoringServer = srv s.mu.Unlock() go func() { if err := srv.Serve(httpListener); err != nil { if !s.isShuttingDown() { s.Fatalf("Error starting monitor on %q: %v", hp, err) } } srv.Close() s.mu.Lock() s.httpHandler = nil s.mu.Unlock() s.done <- true }() return nil } // HTTPHandler returns the http.Handler object used to handle monitoring // endpoints. It will return nil if the server is not configured for // monitoring, or if the server has not been started yet (Server.Start()). func (s *Server) HTTPHandler() http.Handler { s.mu.Lock() defer s.mu.Unlock() return s.httpHandler } // Perform a conditional deep copy due to reference nature of [Client|WS]ConnectURLs. // If updates are made to Info, this function should be consulted and updated. // Assume lock is held. func (s *Server) copyInfo() Info { info := s.info if len(info.ClientConnectURLs) > 0 { info.ClientConnectURLs = append([]string(nil), s.info.ClientConnectURLs...) } if len(info.WSConnectURLs) > 0 { info.WSConnectURLs = append([]string(nil), s.info.WSConnectURLs...) } return info } // tlsMixConn is used when we can receive both TLS and non-TLS connections on same port. type tlsMixConn struct { net.Conn pre *bytes.Buffer } // Read for our mixed multi-reader. func (c *tlsMixConn) Read(b []byte) (int, error) { if c.pre != nil { n, err := c.pre.Read(b) if c.pre.Len() == 0 { c.pre = nil } return n, err } return c.Conn.Read(b) } func (s *Server) createClient(conn net.Conn) *client { return s.createClientEx(conn, false) } func (s *Server) createClientInProcess(conn net.Conn) *client { return s.createClientEx(conn, true) } func (s *Server) createClientEx(conn net.Conn, inProcess bool) *client { // Snapshot server options. opts := s.getOpts() maxPay := int32(opts.MaxPayload) maxSubs := int32(opts.MaxSubs) // For system, maxSubs of 0 means unlimited, so re-adjust here. if maxSubs == 0 { maxSubs = -1 } now := time.Now() c := &client{srv: s, nc: conn, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now} c.registerWithAccount(s.globalAccount()) var info Info var authRequired bool s.mu.Lock() // Grab JSON info string info = s.copyInfo() if s.nonceRequired() { // Nonce handling var raw [nonceLen]byte nonce := raw[:] s.generateNonce(nonce) info.Nonce = string(nonce) } c.nonce = []byte(info.Nonce) authRequired = info.AuthRequired // Check to see if we have auth_required set but we also have a no_auth_user. // If so set back to false. if info.AuthRequired && opts.NoAuthUser != _EMPTY_ && opts.NoAuthUser != s.sysAccOnlyNoAuthUser { info.AuthRequired = false } // Check to see if this is an in-process connection with tls_required. // If so, set as not required, but available. if inProcess && info.TLSRequired { info.TLSRequired = false info.TLSAvailable = true } s.totalClients++ s.mu.Unlock() // Grab lock c.mu.Lock() if authRequired { c.flags.set(expectConnect) } // Initialize c.initClient() c.Debugf("Client connection created") // Save info.TLSRequired value since we may neeed to change it back and forth. orgInfoTLSReq := info.TLSRequired var tlsFirstFallback time.Duration // Check if we should do TLS first. tlsFirst := opts.TLSConfig != nil && opts.TLSHandshakeFirst if tlsFirst { // Make sure info.TLSRequired is set to true (it could be false // if AllowNonTLS is enabled). info.TLSRequired = true // Get the fallback delay value if applicable. if f := opts.TLSHandshakeFirstFallback; f > 0 { tlsFirstFallback = f } else if inProcess { // For in-process connection, we will always have a fallback // delay. It allows support for non-TLS, TLS and "TLS First" // in-process clients to successfully connect. tlsFirstFallback = DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY } } // Decide if we are going to require TLS or not and generate INFO json. tlsRequired := info.TLSRequired infoBytes := c.generateClientInfoJSON(info) // Send our information, except if TLS and TLSHandshakeFirst is requested. if !tlsFirst { // Need to be sent in place since writeLoop cannot be started until // TLS handshake is done (if applicable). c.sendProtoNow(infoBytes) } // Unlock to register c.mu.Unlock() // Register with the server. s.mu.Lock() // If server is not running, Shutdown() may have already gathered the // list of connections to close. It won't contain this one, so we need // to bail out now otherwise the readLoop started down there would not // be interrupted. Skip also if in lame duck mode. if !s.isRunning() || s.ldm { // There are some tests that create a server but don't start it, // and use "async" clients and perform the parsing manually. Such // clients would branch here (since server is not running). However, // when a server was really running and has been shutdown, we must // close this connection. if s.isShuttingDown() { conn.Close() } s.mu.Unlock() return c } // If there is a max connections specified, check that adding // this new client would not push us over the max if opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn { s.mu.Unlock() c.maxConnExceeded() return nil } s.clients[c.cid] = c s.mu.Unlock() // Re-Grab lock c.mu.Lock() isClosed := c.isClosed() var pre []byte // We need first to check for "TLS First" fallback delay. if !isClosed && tlsFirstFallback > 0 { // We wait and see if we are getting any data. Since we did not send // the INFO protocol yet, only clients that use TLS first should be // sending data (the TLS handshake). We don't really check the content: // if it is a rogue agent and not an actual client performing the // TLS handshake, the error will be detected when performing the // handshake on our side. pre = make([]byte, 4) c.nc.SetReadDeadline(time.Now().Add(tlsFirstFallback)) n, _ := io.ReadFull(c.nc, pre[:]) c.nc.SetReadDeadline(time.Time{}) // If we get any data (regardless of possible timeout), we will proceed // with the TLS handshake. if n > 0 { pre = pre[:n] } else { // We did not get anything so we will send the INFO protocol. pre = nil // Restore the original info.TLSRequired value if it is // different that the current value and regenerate infoBytes. if orgInfoTLSReq != info.TLSRequired { info.TLSRequired = orgInfoTLSReq infoBytes = c.generateClientInfoJSON(info) } c.sendProtoNow(infoBytes) // Set the boolean to false for the rest of the function. tlsFirst = false // Check closed status again isClosed = c.isClosed() } } // If we have both TLS and non-TLS allowed we need to see which // one the client wants. We'll always allow this for in-process // connections. if !isClosed && !tlsFirst && opts.TLSConfig != nil && (inProcess || opts.AllowNonTLS) { pre = make([]byte, 4) c.nc.SetReadDeadline(time.Now().Add(secondsToDuration(opts.TLSTimeout))) n, _ := io.ReadFull(c.nc, pre[:]) c.nc.SetReadDeadline(time.Time{}) pre = pre[:n] if n > 0 && pre[0] == 0x16 { tlsRequired = true } else { tlsRequired = false } } // Check for TLS if !isClosed && tlsRequired { if s.connRateCounter != nil && !s.connRateCounter.allow() { c.mu.Unlock() c.sendErr("Connection throttling is active. Please try again later.") c.closeConnection(MaxConnectionsExceeded) return nil } // If we have a prebuffer create a multi-reader. if len(pre) > 0 { c.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)} // Clear pre so it is not parsed. pre = nil } // Performs server-side TLS handshake. if err := c.doTLSServerHandshake(_EMPTY_, opts.TLSConfig, opts.TLSTimeout, opts.TLSPinnedCerts); err != nil { c.mu.Unlock() return nil } } // Now, send the INFO if it was delayed if !isClosed && tlsFirst { c.flags.set(didTLSFirst) c.sendProtoNow(infoBytes) // Check closed status isClosed = c.isClosed() } // Connection could have been closed while sending the INFO proto. if isClosed { c.mu.Unlock() // We need to call closeConnection() to make sure that proper cleanup is done. c.closeConnection(WriteError) return nil } // Check for Auth. We schedule this timer after the TLS handshake to avoid // the race where the timer fires during the handshake and causes the // server to write bad data to the socket. See issue #432. if authRequired { c.setAuthTimer(secondsToDuration(opts.AuthTimeout)) } // Do final client initialization // Set the Ping timer. Will be reset once connect was received. c.setPingTimer() // Spin up the read loop. s.startGoRoutine(func() { c.readLoop(pre) }) // Spin up the write loop. s.startGoRoutine(func() { c.writeLoop() }) if tlsRequired { c.Debugf("TLS handshake complete") cs := c.nc.(*tls.Conn).ConnectionState() c.Debugf("TLS version %s, cipher suite %s", tlsVersion(cs.Version), tlsCipher(cs.CipherSuite)) } c.mu.Unlock() return c } // This will save off a closed client in a ring buffer such that // /connz can inspect. Useful for debugging, etc. func (s *Server) saveClosedClient(c *client, nc net.Conn, subs map[string]*subscription, reason ClosedState) { now := time.Now() s.accountDisconnectEvent(c, now, reason.String()) c.mu.Lock() cc := &closedClient{} cc.fill(c, nc, now, false) // Note that cc.fill is using len(c.subs), which may have been set to nil by now, // so replace cc.NumSubs with len(subs). cc.NumSubs = uint32(len(subs)) cc.Stop = &now cc.Reason = reason.String() // Do subs, do not place by default in main ConnInfo if len(subs) > 0 { cc.subs = make([]SubDetail, 0, len(subs)) for _, sub := range subs { cc.subs = append(cc.subs, newSubDetail(sub)) } } // Hold user as well. cc.user = c.getRawAuthUser() // Hold account name if not the global account. if c.acc != nil && c.acc.Name != globalAccountName { cc.acc = c.acc.Name } cc.JWT = c.opts.JWT cc.IssuerKey = issuerForClient(c) cc.Tags = c.tags cc.NameTag = c.nameTag c.mu.Unlock() // Place in the ring buffer s.mu.Lock() if s.closed != nil { s.closed.append(cc) } s.mu.Unlock() } // Adds to the list of client and websocket clients connect URLs. // If there was a change, an INFO protocol is sent to registered clients // that support async INFO protocols. // Server lock held on entry. func (s *Server) addConnectURLsAndSendINFOToClients(curls, wsurls []string) { s.updateServerINFOAndSendINFOToClients(curls, wsurls, true) } // Removes from the list of client and websocket clients connect URLs. // If there was a change, an INFO protocol is sent to registered clients // that support async INFO protocols. // Server lock held on entry. func (s *Server) removeConnectURLsAndSendINFOToClients(curls, wsurls []string) { s.updateServerINFOAndSendINFOToClients(curls, wsurls, false) } // Updates the list of client and websocket clients connect URLs and if any change // sends an async INFO update to clients that support it. // Server lock held on entry. func (s *Server) updateServerINFOAndSendINFOToClients(curls, wsurls []string, add bool) { remove := !add // Will return true if we need alter the server's Info object. updateMap := func(urls []string, m refCountedUrlSet) bool { wasUpdated := false for _, url := range urls { if add && m.addUrl(url) { wasUpdated = true } else if remove && m.removeUrl(url) { wasUpdated = true } } return wasUpdated } cliUpdated := updateMap(curls, s.clientConnectURLsMap) wsUpdated := updateMap(wsurls, s.websocket.connectURLsMap) updateInfo := func(infoURLs *[]string, urls []string, m refCountedUrlSet) { // Recreate the info's slice from the map *infoURLs = (*infoURLs)[:0] // Add this server client connect ULRs first... *infoURLs = append(*infoURLs, urls...) // Then the ones from the map for url := range m { *infoURLs = append(*infoURLs, url) } } if cliUpdated { updateInfo(&s.info.ClientConnectURLs, s.clientConnectURLs, s.clientConnectURLsMap) } if wsUpdated { updateInfo(&s.info.WSConnectURLs, s.websocket.connectURLs, s.websocket.connectURLsMap) } if cliUpdated || wsUpdated { // Update the time of this update s.lastCURLsUpdate = time.Now().UnixNano() // Send to all registered clients that support async INFO protocols. s.sendAsyncInfoToClients(cliUpdated, wsUpdated) } } // Handle closing down a connection when the handshake has timedout. func tlsTimeout(c *client, conn *tls.Conn) { c.mu.Lock() closed := c.isClosed() c.mu.Unlock() // Check if already closed if closed { return } cs := conn.ConnectionState() if !cs.HandshakeComplete { c.Errorf("TLS handshake timeout") c.sendErr("Secure Connection - TLS Required") c.closeConnection(TLSHandshakeError) } } // Seems silly we have to write these func tlsVersion(ver uint16) string { switch ver { case tls.VersionTLS10: return "1.0" case tls.VersionTLS11: return "1.1" case tls.VersionTLS12: return "1.2" case tls.VersionTLS13: return "1.3" } return fmt.Sprintf("Unknown [0x%x]", ver) } func tlsVersionFromString(ver string) (uint16, error) { switch ver { case "1.0": return tls.VersionTLS10, nil case "1.1": return tls.VersionTLS11, nil case "1.2": return tls.VersionTLS12, nil case "1.3": return tls.VersionTLS13, nil } return 0, fmt.Errorf("unknown version: %v", ver) } // We use hex here so we don't need multiple versions func tlsCipher(cs uint16) string { name, present := cipherMapByID[cs] if present { return name } return fmt.Sprintf("Unknown [0x%x]", cs) } // Remove a client or route from our internal accounting. func (s *Server) removeClient(c *client) { // kind is immutable, so can check without lock switch c.kind { case CLIENT: c.mu.Lock() cid := c.cid updateProtoInfoCount := false if c.kind == CLIENT && c.opts.Protocol >= ClientProtoInfo { updateProtoInfoCount = true } c.mu.Unlock() s.mu.Lock() delete(s.clients, cid) if updateProtoInfoCount { s.cproto-- } s.mu.Unlock() case ROUTER: s.removeRoute(c) case GATEWAY: s.removeRemoteGatewayConnection(c) case LEAF: s.removeLeafNodeConnection(c) } } func (s *Server) removeFromTempClients(cid uint64) { s.grMu.Lock() delete(s.grTmpClients, cid) s.grMu.Unlock() } func (s *Server) addToTempClients(cid uint64, c *client) bool { added := false s.grMu.Lock() if s.grRunning { s.grTmpClients[cid] = c added = true } s.grMu.Unlock() return added } ///////////////////////////////////////////////////////////////// // These are some helpers for accounting in functional tests. ///////////////////////////////////////////////////////////////// // NumRoutes will report the number of registered routes. func (s *Server) NumRoutes() int { s.mu.RLock() defer s.mu.RUnlock() return s.numRoutes() } // numRoutes will report the number of registered routes. // Server lock held on entry func (s *Server) numRoutes() int { var nr int s.forEachRoute(func(c *client) { nr++ }) return nr } // NumRemotes will report number of registered remotes. func (s *Server) NumRemotes() int { s.mu.RLock() defer s.mu.RUnlock() return s.numRemotes() } // numRemotes will report number of registered remotes. // Server lock held on entry func (s *Server) numRemotes() int { return len(s.routes) } // NumLeafNodes will report number of leaf node connections. func (s *Server) NumLeafNodes() int { s.mu.RLock() defer s.mu.RUnlock() return len(s.leafs) } // NumClients will report the number of registered clients. func (s *Server) NumClients() int { s.mu.RLock() defer s.mu.RUnlock() return len(s.clients) } // GetClient will return the client associated with cid. func (s *Server) GetClient(cid uint64) *client { return s.getClient(cid) } // getClient will return the client associated with cid. func (s *Server) getClient(cid uint64) *client { s.mu.RLock() defer s.mu.RUnlock() return s.clients[cid] } // GetLeafNode returns the leafnode associated with the cid. func (s *Server) GetLeafNode(cid uint64) *client { s.mu.RLock() defer s.mu.RUnlock() return s.leafs[cid] } // NumSubscriptions will report how many subscriptions are active. func (s *Server) NumSubscriptions() uint32 { s.mu.RLock() defer s.mu.RUnlock() return s.numSubscriptions() } // numSubscriptions will report how many subscriptions are active. // Lock should be held. func (s *Server) numSubscriptions() uint32 { var subs int s.accounts.Range(func(k, v any) bool { acc := v.(*Account) subs += acc.TotalSubs() return true }) return uint32(subs) } // NumSlowConsumers will report the number of slow consumers. func (s *Server) NumSlowConsumers() int64 { return atomic.LoadInt64(&s.slowConsumers) } // NumSlowConsumersClients will report the number of slow consumers clients. func (s *Server) NumSlowConsumersClients() uint64 { return s.scStats.clients.Load() } // NumSlowConsumersRoutes will report the number of slow consumers routes. func (s *Server) NumSlowConsumersRoutes() uint64 { return s.scStats.routes.Load() } // NumSlowConsumersGateways will report the number of slow consumers leafs. func (s *Server) NumSlowConsumersGateways() uint64 { return s.scStats.gateways.Load() } // NumSlowConsumersLeafs will report the number of slow consumers leafs. func (s *Server) NumSlowConsumersLeafs() uint64 { return s.scStats.leafs.Load() } // ConfigTime will report the last time the server configuration was loaded. func (s *Server) ConfigTime() time.Time { s.mu.RLock() defer s.mu.RUnlock() return s.configTime } // Addr will return the net.Addr object for the current listener. func (s *Server) Addr() net.Addr { s.mu.RLock() defer s.mu.RUnlock() if s.listener == nil { return nil } return s.listener.Addr() } // MonitorAddr will return the net.Addr object for the monitoring listener. func (s *Server) MonitorAddr() *net.TCPAddr { s.mu.RLock() defer s.mu.RUnlock() if s.http == nil { return nil } return s.http.Addr().(*net.TCPAddr) } // ClusterAddr returns the net.Addr object for the route listener. func (s *Server) ClusterAddr() *net.TCPAddr { s.mu.RLock() defer s.mu.RUnlock() if s.routeListener == nil { return nil } return s.routeListener.Addr().(*net.TCPAddr) } // ProfilerAddr returns the net.Addr object for the profiler listener. func (s *Server) ProfilerAddr() *net.TCPAddr { s.mu.RLock() defer s.mu.RUnlock() if s.profiler == nil { return nil } return s.profiler.Addr().(*net.TCPAddr) } func (s *Server) readyForConnections(d time.Duration) error { // Snapshot server options. opts := s.getOpts() type info struct { ok bool err error } chk := make(map[string]info) end := time.Now().Add(d) for time.Now().Before(end) { s.mu.RLock() chk["server"] = info{ok: s.listener != nil || opts.DontListen, err: s.listenerErr} chk["route"] = info{ok: (opts.Cluster.Port == 0 || s.routeListener != nil), err: s.routeListenerErr} chk["gateway"] = info{ok: (opts.Gateway.Name == _EMPTY_ || s.gatewayListener != nil), err: s.gatewayListenerErr} chk["leafnode"] = info{ok: (opts.LeafNode.Port == 0 || s.leafNodeListener != nil), err: s.leafNodeListenerErr} chk["websocket"] = info{ok: (opts.Websocket.Port == 0 || s.websocket.listener != nil), err: s.websocket.listenerErr} chk["mqtt"] = info{ok: (opts.MQTT.Port == 0 || s.mqtt.listener != nil), err: s.mqtt.listenerErr} s.mu.RUnlock() var numOK int for _, inf := range chk { if inf.ok { numOK++ } } if numOK == len(chk) { // In the case of DontListen option (no accept loop), we still want // to make sure that Start() has done all the work, so we wait on // that. if opts.DontListen { select { case <-s.startupComplete: case <-time.After(d): return fmt.Errorf("failed to be ready for connections after %s: startup did not complete", d) } } return nil } if d > 25*time.Millisecond { time.Sleep(25 * time.Millisecond) } } failed := make([]string, 0, len(chk)) for name, inf := range chk { if inf.ok && inf.err != nil { failed = append(failed, fmt.Sprintf("%s(ok, but %s)", name, inf.err)) } if !inf.ok && inf.err == nil { failed = append(failed, name) } if !inf.ok && inf.err != nil { failed = append(failed, fmt.Sprintf("%s(%s)", name, inf.err)) } } return fmt.Errorf( "failed to be ready for connections after %s: %s", d, strings.Join(failed, ", "), ) } // ReadyForConnections returns `true` if the server is ready to accept clients // and, if routing is enabled, route connections. If after the duration // `dur` the server is still not ready, returns `false`. func (s *Server) ReadyForConnections(dur time.Duration) bool { return s.readyForConnections(dur) == nil } // Quick utility to function to tell if the server supports headers. func (s *Server) supportsHeaders() bool { if s == nil { return false } return !(s.getOpts().NoHeaderSupport) } // ID returns the server's ID func (s *Server) ID() string { return s.info.ID } // NodeName returns the node name for this server. func (s *Server) NodeName() string { return getHash(s.info.Name) } // Name returns the server's name. This will be the same as the ID if it was not set. func (s *Server) Name() string { return s.info.Name } func (s *Server) String() string { return s.info.Name } type pprofLabels map[string]string func setGoRoutineLabels(tags ...pprofLabels) { var labels []string for _, m := range tags { for k, v := range m { labels = append(labels, k, v) } } if len(labels) > 0 { pprof.SetGoroutineLabels( pprof.WithLabels(context.Background(), pprof.Labels(labels...)), ) } } func (s *Server) startGoRoutine(f func(), tags ...pprofLabels) bool { var started bool s.grMu.Lock() defer s.grMu.Unlock() if s.grRunning { s.grWG.Add(1) go func() { setGoRoutineLabels(tags...) f() }() started = true } return started } func (s *Server) numClosedConns() int { s.mu.RLock() defer s.mu.RUnlock() return s.closed.len() } func (s *Server) totalClosedConns() uint64 { s.mu.RLock() defer s.mu.RUnlock() return s.closed.totalConns() } func (s *Server) closedClients() []*closedClient { s.mu.RLock() defer s.mu.RUnlock() return s.closed.closedClients() } // getClientConnectURLs returns suitable URLs for clients to connect to the listen // port based on the server options' Host and Port. If the Host corresponds to // "any" interfaces, this call returns the list of resolved IP addresses. // If ClientAdvertise is set, returns the client advertise host and port. // The server lock is assumed held on entry. func (s *Server) getClientConnectURLs() []string { // Snapshot server options. opts := s.getOpts() // Ignore error here since we know that if there is client advertise, the // parseHostPort is correct because we did it right before calling this // function in Server.New(). urls, _ := s.getConnectURLs(opts.ClientAdvertise, opts.Host, opts.Port) return urls } // Generic version that will return an array of URLs based on the given // advertise, host and port values. func (s *Server) getConnectURLs(advertise, host string, port int) ([]string, error) { urls := make([]string, 0, 1) // short circuit if advertise is set if advertise != "" { h, p, err := parseHostPort(advertise, port) if err != nil { return nil, err } urls = append(urls, net.JoinHostPort(h, strconv.Itoa(p))) } else { sPort := strconv.Itoa(port) _, ips, err := s.getNonLocalIPsIfHostIsIPAny(host, true) for _, ip := range ips { urls = append(urls, net.JoinHostPort(ip, sPort)) } if err != nil || len(urls) == 0 { // We are here if s.opts.Host is not "0.0.0.0" nor "::", or if for some // reason we could not add any URL in the loop above. // We had a case where a Windows VM was hosed and would have err == nil // and not add any address in the array in the loop above, and we // ended-up returning 0.0.0.0, which is problematic for Windows clients. // Check for 0.0.0.0 or :: specifically, and ignore if that's the case. if host == "0.0.0.0" || host == "::" { s.Errorf("Address %q can not be resolved properly", host) } else { urls = append(urls, net.JoinHostPort(host, sPort)) } } } return urls, nil } // Returns an array of non local IPs if the provided host is // 0.0.0.0 or ::. It returns the first resolved if `all` is // false. // The boolean indicate if the provided host was 0.0.0.0 (or ::) // so that if the returned array is empty caller can decide // what to do next. func (s *Server) getNonLocalIPsIfHostIsIPAny(host string, all bool) (bool, []string, error) { ip := net.ParseIP(host) // If this is not an IP, we are done if ip == nil { return false, nil, nil } // If this is not 0.0.0.0 or :: we have nothing to do. if !ip.IsUnspecified() { return false, nil, nil } s.Debugf("Get non local IPs for %q", host) var ips []string ifaces, _ := net.Interfaces() for _, i := range ifaces { addrs, _ := i.Addrs() for _, addr := range addrs { switch v := addr.(type) { case *net.IPNet: ip = v.IP case *net.IPAddr: ip = v.IP } ipStr := ip.String() // Skip non global unicast addresses if !ip.IsGlobalUnicast() || ip.IsUnspecified() { ip = nil continue } s.Debugf(" ip=%s", ipStr) ips = append(ips, ipStr) if !all { break } } } return true, ips, nil } // if the ip is not specified, attempt to resolve it func resolveHostPorts(addr net.Listener) []string { hostPorts := make([]string, 0) hp := addr.Addr().(*net.TCPAddr) port := strconv.Itoa(hp.Port) if hp.IP.IsUnspecified() { var ip net.IP ifaces, _ := net.Interfaces() for _, i := range ifaces { addrs, _ := i.Addrs() for _, addr := range addrs { switch v := addr.(type) { case *net.IPNet: ip = v.IP hostPorts = append(hostPorts, net.JoinHostPort(ip.String(), port)) case *net.IPAddr: ip = v.IP hostPorts = append(hostPorts, net.JoinHostPort(ip.String(), port)) default: continue } } } } else { hostPorts = append(hostPorts, net.JoinHostPort(hp.IP.String(), port)) } return hostPorts } // format the address of a net.Listener with a protocol func formatURL(protocol string, addr net.Listener) []string { hostports := resolveHostPorts(addr) for i, hp := range hostports { hostports[i] = fmt.Sprintf("%s://%s", protocol, hp) } return hostports } // Ports describes URLs that the server can be contacted in type Ports struct { Nats []string `json:"nats,omitempty"` Monitoring []string `json:"monitoring,omitempty"` Cluster []string `json:"cluster,omitempty"` Profile []string `json:"profile,omitempty"` WebSocket []string `json:"websocket,omitempty"` } // PortsInfo attempts to resolve all the ports. If after maxWait the ports are not // resolved, it returns nil. Otherwise it returns a Ports struct // describing ports where the server can be contacted func (s *Server) PortsInfo(maxWait time.Duration) *Ports { if s.readyForListeners(maxWait) { opts := s.getOpts() s.mu.RLock() tls := s.info.TLSRequired listener := s.listener httpListener := s.http clusterListener := s.routeListener profileListener := s.profiler wsListener := s.websocket.listener wss := s.websocket.tls s.mu.RUnlock() ports := Ports{} if listener != nil { natsProto := "nats" if tls { natsProto = "tls" } ports.Nats = formatURL(natsProto, listener) } if httpListener != nil { monProto := "http" if opts.HTTPSPort != 0 { monProto = "https" } ports.Monitoring = formatURL(monProto, httpListener) } if clusterListener != nil { clusterProto := "nats" if opts.Cluster.TLSConfig != nil { clusterProto = "tls" } ports.Cluster = formatURL(clusterProto, clusterListener) } if profileListener != nil { ports.Profile = formatURL("http", profileListener) } if wsListener != nil { protocol := wsSchemePrefix if wss { protocol = wsSchemePrefixTLS } ports.WebSocket = formatURL(protocol, wsListener) } return &ports } return nil } // Returns the portsFile. If a non-empty dirHint is provided, the dirHint // path is used instead of the server option value func (s *Server) portFile(dirHint string) string { dirname := s.getOpts().PortsFileDir if dirHint != "" { dirname = dirHint } if dirname == _EMPTY_ { return _EMPTY_ } return filepath.Join(dirname, fmt.Sprintf("%s_%d.ports", filepath.Base(os.Args[0]), os.Getpid())) } // Delete the ports file. If a non-empty dirHint is provided, the dirHint // path is used instead of the server option value func (s *Server) deletePortsFile(hintDir string) { portsFile := s.portFile(hintDir) if portsFile != "" { if err := os.Remove(portsFile); err != nil { s.Errorf("Error cleaning up ports file %s: %v", portsFile, err) } } } // Writes a file with a serialized Ports to the specified ports_file_dir. // The name of the file is `exename_pid.ports`, typically nats-server_pid.ports. // if ports file is not set, this function has no effect func (s *Server) logPorts() { opts := s.getOpts() portsFile := s.portFile(opts.PortsFileDir) if portsFile != _EMPTY_ { go func() { info := s.PortsInfo(5 * time.Second) if info == nil { s.Errorf("Unable to resolve the ports in the specified time") return } data, err := json.Marshal(info) if err != nil { s.Errorf("Error marshaling ports file: %v", err) return } if err := os.WriteFile(portsFile, data, 0666); err != nil { s.Errorf("Error writing ports file (%s): %v", portsFile, err) return } }() } } // waits until a calculated list of listeners is resolved or a timeout func (s *Server) readyForListeners(dur time.Duration) bool { end := time.Now().Add(dur) for time.Now().Before(end) { s.mu.RLock() listeners := s.serviceListeners() s.mu.RUnlock() if len(listeners) == 0 { return false } ok := true for _, l := range listeners { if l == nil { ok = false break } } if ok { return true } select { case <-s.quitCh: return false case <-time.After(25 * time.Millisecond): // continue - unable to select from quit - we are still running } } return false } // returns a list of listeners that are intended for the process // if the entry is nil, the interface is yet to be resolved func (s *Server) serviceListeners() []net.Listener { listeners := make([]net.Listener, 0) opts := s.getOpts() listeners = append(listeners, s.listener) if opts.Cluster.Port != 0 { listeners = append(listeners, s.routeListener) } if opts.HTTPPort != 0 || opts.HTTPSPort != 0 { listeners = append(listeners, s.http) } if opts.ProfPort != 0 { listeners = append(listeners, s.profiler) } if opts.Websocket.Port != 0 { listeners = append(listeners, s.websocket.listener) } return listeners } // Returns true if in lame duck mode. func (s *Server) isLameDuckMode() bool { s.mu.RLock() defer s.mu.RUnlock() return s.ldm } // LameDuckShutdown will perform a lame duck shutdown of NATS, whereby // the client listener is closed, existing client connections are // kicked, Raft leaderships are transferred, JetStream is shutdown // and then finally shutdown the the NATS Server itself. // This function blocks and will not return until the NATS Server // has completed the entire shutdown operation. func (s *Server) LameDuckShutdown() { s.lameDuckMode() } // This function will close the client listener then close the clients // at some interval to avoid a reconnect storm. // We will also transfer any raft leaders and shutdown JetStream. func (s *Server) lameDuckMode() { s.mu.Lock() // Check if there is actually anything to do if s.isShuttingDown() || s.ldm || s.listener == nil { s.mu.Unlock() return } s.Noticef("Entering lame duck mode, stop accepting new clients") s.ldm = true s.sendLDMShutdownEventLocked() expected := 1 s.listener.Close() s.listener = nil expected += s.closeWebsocketServer() s.ldmCh = make(chan bool, expected) opts := s.getOpts() gp := opts.LameDuckGracePeriod // For tests, we want the grace period to be in some cases bigger // than the ldm duration, so to by-pass the validateOptions() check, // we use negative number and flip it here. if gp < 0 { gp *= -1 } s.mu.Unlock() // If we are running any raftNodes transfer leaders. if hadTransfers := s.transferRaftLeaders(); hadTransfers { // They will transfer leadership quickly, but wait here for a second. select { case <-time.After(time.Second): case <-s.quitCh: return } } // Now check and shutdown jetstream. s.shutdownJetStream() // Now shutdown the nodes s.shutdownRaftNodes() // Wait for accept loops to be done to make sure that no new // client can connect for i := 0; i < expected; i++ { <-s.ldmCh } s.mu.Lock() // Need to recheck few things if s.isShuttingDown() || len(s.clients) == 0 { s.mu.Unlock() // If there is no client, we need to call Shutdown() to complete // the LDMode. If server has been shutdown while lock was released, // calling Shutdown() should be no-op. s.Shutdown() return } dur := int64(opts.LameDuckDuration) dur -= int64(gp) if dur <= 0 { dur = int64(time.Second) } numClients := int64(len(s.clients)) batch := 1 // Sleep interval between each client connection close. var si int64 if numClients != 0 { si = dur / numClients } if si < 1 { // Should not happen (except in test with very small LD duration), but // if there are too many clients, batch the number of close and // use a tiny sleep interval that will result in yield likely. si = 1 batch = int(numClients / dur) } else if si > int64(time.Second) { // Conversely, there is no need to sleep too long between clients // and spread say 10 clients for the 2min duration. Sleeping no // more than 1sec. si = int64(time.Second) } // Now capture all clients clients := make([]*client, 0, len(s.clients)) for _, client := range s.clients { clients = append(clients, client) } // Now that we know that no new client can be accepted, // send INFO to routes and clients to notify this state. s.sendLDMToRoutes() s.sendLDMToClients() s.mu.Unlock() t := time.NewTimer(gp) // Delay start of closing of client connections in case // we have several servers that we want to signal to enter LD mode // and not have their client reconnect to each other. select { case <-t.C: s.Noticef("Closing existing clients") case <-s.quitCh: t.Stop() return } for i, client := range clients { client.closeConnection(ServerShutdown) if i == len(clients)-1 { break } if batch == 1 || i%batch == 0 { // We pick a random interval which will be at least si/2 v := rand.Int63n(si) if v < si/2 { v = si / 2 } t.Reset(time.Duration(v)) // Sleep for given interval or bail out if kicked by Shutdown(). select { case <-t.C: case <-s.quitCh: t.Stop() return } } } s.Shutdown() s.WaitForShutdown() } // Send an INFO update to routes with the indication that this server is in LDM mode. // Server lock is held on entry. func (s *Server) sendLDMToRoutes() { s.routeInfo.LameDuckMode = true infoJSON := generateInfoJSON(&s.routeInfo) s.forEachRemote(func(r *client) { r.mu.Lock() r.enqueueProto(infoJSON) r.mu.Unlock() }) // Clear now so that we notify only once, should we have to send other INFOs. s.routeInfo.LameDuckMode = false } // Send an INFO update to clients with the indication that this server is in // LDM mode and with only URLs of other nodes. // Server lock is held on entry. func (s *Server) sendLDMToClients() { s.info.LameDuckMode = true // Clear this so that if there are further updates, we don't send our URLs. s.clientConnectURLs = s.clientConnectURLs[:0] if s.websocket.connectURLs != nil { s.websocket.connectURLs = s.websocket.connectURLs[:0] } // Reset content first. s.info.ClientConnectURLs = s.info.ClientConnectURLs[:0] s.info.WSConnectURLs = s.info.WSConnectURLs[:0] // Only add the other nodes if we are allowed to. if !s.getOpts().Cluster.NoAdvertise { for url := range s.clientConnectURLsMap { s.info.ClientConnectURLs = append(s.info.ClientConnectURLs, url) } for url := range s.websocket.connectURLsMap { s.info.WSConnectURLs = append(s.info.WSConnectURLs, url) } } // Send to all registered clients that support async INFO protocols. s.sendAsyncInfoToClients(true, true) // We now clear the info.LameDuckMode flag so that if there are // cluster updates and we send the INFO, we don't have the boolean // set which would cause multiple LDM notifications to clients. s.info.LameDuckMode = false } // If given error is a net.Error and is temporary, sleeps for the given // delay and double it, but cap it to ACCEPT_MAX_SLEEP. The sleep is // interrupted if the server is shutdown. // An error message is displayed depending on the type of error. // Returns the new (or unchanged) delay, or a negative value if the // server has been or is being shutdown. func (s *Server) acceptError(acceptName string, err error, tmpDelay time.Duration) time.Duration { if !s.isRunning() { return -1 } //lint:ignore SA1019 We want to retry on a bunch of errors here. if ne, ok := err.(net.Error); ok && ne.Temporary() { // nolint:staticcheck s.Errorf("Temporary %s Accept Error(%v), sleeping %dms", acceptName, ne, tmpDelay/time.Millisecond) select { case <-time.After(tmpDelay): case <-s.quitCh: return -1 } tmpDelay *= 2 if tmpDelay > ACCEPT_MAX_SLEEP { tmpDelay = ACCEPT_MAX_SLEEP } } else { s.Errorf("%s Accept error: %v", acceptName, err) } return tmpDelay } var errNoIPAvail = errors.New("no IP available") func (s *Server) getRandomIP(resolver netResolver, url string, excludedAddresses map[string]struct{}) (string, error) { host, port, err := net.SplitHostPort(url) if err != nil { return "", err } // If already an IP, skip. if net.ParseIP(host) != nil { return url, nil } ips, err := resolver.LookupHost(context.Background(), host) if err != nil { return "", fmt.Errorf("lookup for host %q: %v", host, err) } if len(excludedAddresses) > 0 { for i := 0; i < len(ips); i++ { ip := ips[i] addr := net.JoinHostPort(ip, port) if _, excluded := excludedAddresses[addr]; excluded { if len(ips) == 1 { ips = nil break } ips[i] = ips[len(ips)-1] ips = ips[:len(ips)-1] i-- } } if len(ips) == 0 { return "", errNoIPAvail } } var address string if len(ips) == 0 { s.Warnf("Unable to get IP for %s, will try with %s: %v", host, url, err) address = url } else { var ip string if len(ips) == 1 { ip = ips[0] } else { ip = ips[rand.Int31n(int32(len(ips)))] } // add the port address = net.JoinHostPort(ip, port) } return address, nil } // Returns true for the first attempt and depending on the nature // of the attempt (first connect or a reconnect), when the number // of attempts is equal to the configured report attempts. func (s *Server) shouldReportConnectErr(firstConnect bool, attempts int) bool { opts := s.getOpts() if firstConnect { if attempts == 1 || attempts%opts.ConnectErrorReports == 0 { return true } return false } if attempts == 1 || attempts%opts.ReconnectErrorReports == 0 { return true } return false } func (s *Server) updateRemoteSubscription(acc *Account, sub *subscription, delta int32) { s.updateRouteSubscriptionMap(acc, sub, delta) if s.gateway.enabled { s.gatewayUpdateSubInterest(acc.Name, sub, delta) } acc.updateLeafNodes(sub, delta) } func (s *Server) startRateLimitLogExpiration() { interval := time.Second s.startGoRoutine(func() { defer s.grWG.Done() ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-s.quitCh: return case interval = <-s.rateLimitLoggingCh: ticker.Reset(interval) case <-ticker.C: s.rateLimitLogging.Range(func(k, v any) bool { start := v.(time.Time) if time.Since(start) >= interval { s.rateLimitLogging.Delete(k) } return true }) } } }) } func (s *Server) changeRateLimitLogInterval(d time.Duration) { if d <= 0 { return } select { case s.rateLimitLoggingCh <- d: default: } } // DisconnectClientByID disconnects a client by connection ID func (s *Server) DisconnectClientByID(id uint64) error { if s == nil { return ErrServerNotRunning } if client := s.getClient(id); client != nil { client.closeConnection(Kicked) return nil } else if client = s.GetLeafNode(id); client != nil { client.closeConnection(Kicked) return nil } return errors.New("no such client or leafnode id") } // LDMClientByID sends a Lame Duck Mode info message to a client by connection ID func (s *Server) LDMClientByID(id uint64) error { if s == nil { return ErrServerNotRunning } s.mu.RLock() c := s.clients[id] if c == nil { s.mu.RUnlock() return errors.New("no such client id") } info := s.copyInfo() info.LameDuckMode = true s.mu.RUnlock() c.mu.Lock() defer c.mu.Unlock() if c.opts.Protocol >= ClientProtoInfo && c.flags.isSet(firstPongSent) { // sendInfo takes care of checking if the connection is still // valid or not, so don't duplicate tests here. c.Debugf("Sending Lame Duck Mode info to client") c.enqueueProto(c.generateClientInfoJSON(info)) return nil } else { return errors.New("client does not support Lame Duck Mode or is not ready to receive the notification") } } nats-server-2.10.27/server/server_test.go000066400000000000000000001725341477524627100203710ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "context" "crypto/tls" "encoding/json" "errors" "flag" "fmt" "io" "net" "net/url" "os" "reflect" "runtime" "slices" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats.go" ) func checkForErr(totalWait, sleepDur time.Duration, f func() error) error { timeout := time.Now().Add(totalWait) var err error for time.Now().Before(timeout) { err = f() if err == nil { return nil } time.Sleep(sleepDur) } return err } func checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) { t.Helper() err := checkForErr(totalWait, sleepDur, f) if err != nil { t.Fatal(err.Error()) } } func DefaultOptions() *Options { return &Options{ Host: "127.0.0.1", Port: -1, HTTPPort: -1, Cluster: ClusterOpts{Port: -1, Name: "abc"}, NoLog: true, NoSigs: true, Debug: true, Trace: true, } } // New Go Routine based server func RunServer(opts *Options) *Server { if opts == nil { opts = DefaultOptions() } s, err := NewServer(opts) if err != nil || s == nil { panic(fmt.Sprintf("No NATS Server object returned: %v", err)) } if !opts.NoLog { s.ConfigureLogger() } // Run server in Go routine. s.Start() // Wait for accept loop(s) to be started if err := s.readyForConnections(10 * time.Second); err != nil { panic(err) } return s } // LoadConfig loads a configuration from a filename func LoadConfig(configFile string) (opts *Options) { opts, err := ProcessConfigFile(configFile) if err != nil { panic(fmt.Sprintf("Error processing configuration file: %v", err)) } opts.NoSigs, opts.NoLog = true, opts.LogFile == _EMPTY_ return } // RunServerWithConfig starts a new Go routine based server with a configuration file. func RunServerWithConfig(configFile string) (srv *Server, opts *Options) { opts = LoadConfig(configFile) srv = RunServer(opts) return } func TestVersionMatchesTag(t *testing.T) { tag := os.Getenv("TRAVIS_TAG") // Travis started to return '' when no tag is set. Support both now. if tag == "" || tag == "''" { t.SkipNow() } // We expect a tag of the form vX.Y.Z. If that's not the case, // we need someone to have a look. So fail if first letter is not // a `v` if tag[0] != 'v' { t.Fatalf("Expect tag to start with `v`, tag is: %s", tag) } // Strip the `v` from the tag for the version comparison. if VERSION != tag[1:] { t.Fatalf("Version (%s) does not match tag (%s)", VERSION, tag[1:]) } // Check that the version dynamically set via ldflags matches the version // from the server previous to releasing. if serverVersion == _EMPTY_ { t.Fatal("Version missing in ldflags") } // Unlike VERSION constant, serverVersion is prefixed with a 'v' // since it should be the same as the git tag. expected := "v" + VERSION if serverVersion != _EMPTY_ && expected != serverVersion { t.Fatalf("Version (%s) does not match ldflags version (%s)", expected, serverVersion) } } func TestStartProfiler(t *testing.T) { s := New(DefaultOptions()) s.StartProfiler() s.mu.Lock() s.profiler.Close() s.mu.Unlock() } func TestStartupAndShutdown(t *testing.T) { opts := DefaultOptions() opts.NoSystemAccount = true s := RunServer(opts) defer s.Shutdown() if !s.isRunning() { t.Fatal("Could not run server") } // Debug stuff. numRoutes := s.NumRoutes() if numRoutes != 0 { t.Fatalf("Expected numRoutes to be 0 vs %d\n", numRoutes) } numRemotes := s.NumRemotes() if numRemotes != 0 { t.Fatalf("Expected numRemotes to be 0 vs %d\n", numRemotes) } numClients := s.NumClients() if numClients != 0 && numClients != 1 { t.Fatalf("Expected numClients to be 1 or 0 vs %d\n", numClients) } numSubscriptions := s.NumSubscriptions() if numSubscriptions != 0 { t.Fatalf("Expected numSubscriptions to be 0 vs %d\n", numSubscriptions) } } func TestTLSVersions(t *testing.T) { for _, test := range []struct { name string value uint16 expected string }{ {"1.0", tls.VersionTLS10, "1.0"}, {"1.1", tls.VersionTLS11, "1.1"}, {"1.2", tls.VersionTLS12, "1.2"}, {"1.3", tls.VersionTLS13, "1.3"}, {"unknown", 0x999, "Unknown [0x999]"}, } { t.Run(test.name, func(t *testing.T) { if v := tlsVersion(test.value); v != test.expected { t.Fatalf("Expected value 0x%x to be %q, got %q", test.value, test.expected, v) } }) } } func TestTLSMinVersionConfig(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" tls { cert_file: "../test/configs/certs/server-cert.pem" key_file: "../test/configs/certs/server-key.pem" timeout: 1 min_version: %s } ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, `"1.3"`))) s, o := RunServerWithConfig(conf) defer s.Shutdown() connect := func(t *testing.T, tlsConf *tls.Config, expectedErr error) { t.Helper() opts := []nats.Option{} if tlsConf != nil { opts = append(opts, nats.Secure(tlsConf)) } opts = append(opts, nats.RootCAs("../test/configs/certs/ca.pem")) nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", o.Port), opts...) if err == nil { defer nc.Close() } if expectedErr == nil { if err != nil { t.Fatalf("Unexpected error: %v", err) } } else if err == nil || err.Error() != expectedErr.Error() { nc.Close() t.Fatalf("Expected error %v, got: %v", expectedErr, err) } } // Cannot connect with client requiring a lower minimum TLS Version. connect(t, &tls.Config{ MaxVersion: tls.VersionTLS12, }, errors.New(`remote error: tls: protocol version not supported`)) // Should connect since matching minimum TLS version. connect(t, &tls.Config{ MinVersion: tls.VersionTLS13, }, nil) // Reloading with invalid values should fail. if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.0"`)), 0666); err != nil { t.Fatalf("Error creating config file: %v", err) } if err := s.Reload(); err == nil { t.Fatalf("Expected reload to fail: %v", err) } // Reloading with original values and no changes should be ok. if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.3"`)), 0666); err != nil { t.Fatalf("Error creating config file: %v", err) } if err := s.Reload(); err != nil { t.Fatalf("Unexpected error reloading TLS version: %v", err) } // Reloading with a new minimum lower version. if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.2"`)), 0666); err != nil { t.Fatalf("Error creating config file: %v", err) } if err := s.Reload(); err != nil { t.Fatalf("Unexpected error reloading: %v", err) } // Should connect since now matching minimum TLS version. connect(t, &tls.Config{ MaxVersion: tls.VersionTLS12, }, nil) connect(t, &tls.Config{ MinVersion: tls.VersionTLS13, }, nil) // Setting unsupported TLS versions if err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `"1.4"`)), 0666); err != nil { t.Fatalf("Error creating config file: %v", err) } if err := s.Reload(); err == nil || !strings.Contains(err.Error(), `unknown version: 1.4`) { t.Fatalf("Unexpected error reloading: %v", err) } tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", Timeout: 4.0, MinVersion: tls.VersionTLS11, } _, err := GenTLSConfig(tc) if err == nil || err.Error() != `unsupported minimum TLS version: TLS 1.1` { t.Fatalf("Expected error generating TLS config: %v", err) } } func TestTLSCipher(t *testing.T) { if strings.Compare(tlsCipher(0x0005), "TLS_RSA_WITH_RC4_128_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x000a), "TLS_RSA_WITH_3DES_EDE_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x002f), "TLS_RSA_WITH_AES_128_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x0035), "TLS_RSA_WITH_AES_256_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc007), "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc009), "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc00a), "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc011), "TLS_ECDHE_RSA_WITH_RC4_128_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc012), "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc013), "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc014), "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA") != 0 { t.Fatalf("IUnknownnvalid tls cipher") } if strings.Compare(tlsCipher(0xc02f), "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc02b), "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc030), "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0xc02c), "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x1301), "TLS_AES_128_GCM_SHA256") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x1302), "TLS_AES_256_GCM_SHA384") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x1303), "TLS_CHACHA20_POLY1305_SHA256") != 0 { t.Fatalf("Invalid tls cipher") } if strings.Compare(tlsCipher(0x9999), "Unknown [0x9999]") != 0 { t.Fatalf("Expected an unknown cipher") } } func TestGetConnectURLs(t *testing.T) { opts := DefaultOptions() opts.Port = 4222 var globalIP net.IP checkGlobalConnectURLs := func() { s := New(opts) defer s.Shutdown() s.mu.Lock() urls := s.getClientConnectURLs() s.mu.Unlock() if len(urls) == 0 { t.Fatalf("Expected to get a list of urls, got none for listen addr: %v", opts.Host) } for _, u := range urls { tcpaddr, err := net.ResolveTCPAddr("tcp", u) if err != nil { t.Fatalf("Error resolving: %v", err) } ip := tcpaddr.IP if !ip.IsGlobalUnicast() { t.Fatalf("IP %v is not global", ip.String()) } if ip.IsUnspecified() { t.Fatalf("IP %v is unspecified", ip.String()) } addr := strings.TrimSuffix(u, ":4222") if addr == opts.Host { t.Fatalf("Returned url is not right: %v", u) } if globalIP == nil { globalIP = ip } } } listenAddrs := []string{"0.0.0.0", "::"} for _, listenAddr := range listenAddrs { opts.Host = listenAddr checkGlobalConnectURLs() } checkConnectURLsHasOnlyOne := func() { s := New(opts) defer s.Shutdown() s.mu.Lock() urls := s.getClientConnectURLs() s.mu.Unlock() if len(urls) != 1 { t.Fatalf("Expected one URL, got %v", urls) } tcpaddr, err := net.ResolveTCPAddr("tcp", urls[0]) if err != nil { t.Fatalf("Error resolving: %v", err) } ip := tcpaddr.IP if ip.String() != opts.Host { t.Fatalf("Expected connect URL to be %v, got %v", opts.Host, ip.String()) } } singleConnectReturned := []string{"127.0.0.1", "::1"} if globalIP != nil { singleConnectReturned = append(singleConnectReturned, globalIP.String()) } for _, listenAddr := range singleConnectReturned { opts.Host = listenAddr checkConnectURLsHasOnlyOne() } } func TestInfoServerNameDefaultsToPK(t *testing.T) { opts := DefaultOptions() opts.Port = 4222 opts.ClientAdvertise = "nats.example.com" s := New(opts) defer s.Shutdown() if s.info.Name != s.info.ID { t.Fatalf("server info hostname is incorrect, got: '%v' expected: '%v'", s.info.Name, s.info.ID) } } func TestInfoServerNameIsSettable(t *testing.T) { opts := DefaultOptions() opts.Port = 4222 opts.ClientAdvertise = "nats.example.com" opts.ServerName = "test_server_name" s := New(opts) defer s.Shutdown() if s.info.Name != "test_server_name" { t.Fatalf("server info hostname is incorrect, got: '%v' expected: 'test_server_name'", s.info.Name) } } func TestClientAdvertiseConnectURL(t *testing.T) { opts := DefaultOptions() opts.Port = 4222 opts.ClientAdvertise = "nats.example.com" s := New(opts) defer s.Shutdown() s.mu.Lock() urls := s.getClientConnectURLs() s.mu.Unlock() if len(urls) != 1 { t.Fatalf("Expected to get one url, got none: %v with ClientAdvertise %v", opts.Host, opts.ClientAdvertise) } if urls[0] != "nats.example.com:4222" { t.Fatalf("Expected to get '%s', got: '%v'", "nats.example.com:4222", urls[0]) } s.Shutdown() opts.ClientAdvertise = "nats.example.com:7777" s = New(opts) s.mu.Lock() urls = s.getClientConnectURLs() s.mu.Unlock() if len(urls) != 1 { t.Fatalf("Expected to get one url, got none: %v with ClientAdvertise %v", opts.Host, opts.ClientAdvertise) } if urls[0] != "nats.example.com:7777" { t.Fatalf("Expected 'nats.example.com:7777', got: '%v'", urls[0]) } if s.info.Host != "nats.example.com" { t.Fatalf("Expected host to be set to nats.example.com") } if s.info.Port != 7777 { t.Fatalf("Expected port to be set to 7777") } s.Shutdown() opts = DefaultOptions() opts.Port = 0 opts.ClientAdvertise = "nats.example.com:7777" s = New(opts) if s.info.Host != "nats.example.com" && s.info.Port != 7777 { t.Fatalf("Expected Client Advertise Host:Port to be nats.example.com:7777, got: %s:%d", s.info.Host, s.info.Port) } s.Shutdown() } func TestClientAdvertiseInCluster(t *testing.T) { optsA := DefaultOptions() optsA.ClientAdvertise = "srvA:4222" srvA := RunServer(optsA) defer srvA.Shutdown() nc := natsConnect(t, srvA.ClientURL()) defer nc.Close() optsB := DefaultOptions() optsB.ClientAdvertise = "srvBC:4222" optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Cluster.Port)) srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) checkURLs := func(expected string) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { srvs := nc.DiscoveredServers() for _, u := range srvs { if u == expected { return nil } } return fmt.Errorf("Url %q not found in %q", expected, srvs) }) } checkURLs("nats://srvBC:4222") optsC := DefaultOptions() optsC.ClientAdvertise = "srvBC:4222" optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Cluster.Port)) srvC := RunServer(optsC) defer srvC.Shutdown() checkClusterFormed(t, srvA, srvB, srvC) checkURLs("nats://srvBC:4222") srvB.Shutdown() checkNumRoutes(t, srvA, DEFAULT_ROUTE_POOL_SIZE+1) checkURLs("nats://srvBC:4222") } func TestClientAdvertiseErrorOnStartup(t *testing.T) { opts := DefaultOptions() // Set invalid address opts.ClientAdvertise = "addr:::123" testFatalErrorOnStart(t, opts, "ClientAdvertise") } func TestNoDeadlockOnStartFailure(t *testing.T) { opts := DefaultOptions() opts.Host = "x.x.x.x" // bad host opts.Port = 4222 opts.HTTPHost = opts.Host opts.Cluster.Host = "127.0.0.1" opts.Cluster.Port = -1 opts.ProfPort = -1 s := New(opts) // This should return since it should fail to start a listener // on x.x.x.x:4222 ch := make(chan struct{}) go func() { s.Start() close(ch) }() select { case <-ch: case <-time.After(time.Second): t.Fatalf("Start() should have returned due to failure to start listener") } // We should be able to shutdown s.Shutdown() } func TestMaxConnections(t *testing.T) { opts := DefaultOptions() opts.MaxConn = 1 s := RunServer(opts) defer s.Shutdown() addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc.Close() nc2, err := nats.Connect(addr) if err == nil { nc2.Close() t.Fatal("Expected connection to fail") } } func TestMaxSubscriptions(t *testing.T) { opts := DefaultOptions() opts.MaxSubs = 10 s := RunServer(opts) defer s.Shutdown() addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc.Close() for i := 0; i < 10; i++ { _, err := nc.Subscribe(fmt.Sprintf("foo.%d", i), func(*nats.Msg) {}) if err != nil { t.Fatalf("Error subscribing: %v\n", err) } } // This should cause the error. nc.Subscribe("foo.22", func(*nats.Msg) {}) nc.Flush() if err := nc.LastError(); err == nil { t.Fatal("Expected an error but got none\n") } } func TestProcessCommandLineArgs(t *testing.T) { var host string var port int cmd := flag.NewFlagSet("nats-server", flag.ExitOnError) cmd.StringVar(&host, "a", "0.0.0.0", "Host.") cmd.IntVar(&port, "p", 4222, "Port.") cmd.Parse([]string{"-a", "127.0.0.1", "-p", "9090"}) showVersion, showHelp, err := ProcessCommandLineArgs(cmd) if err != nil { t.Errorf("Expected no errors, got: %s", err) } if showVersion || showHelp { t.Errorf("Expected not having to handle subcommands") } cmd.Parse([]string{"version"}) showVersion, showHelp, err = ProcessCommandLineArgs(cmd) if err != nil { t.Errorf("Expected no errors, got: %s", err) } if !showVersion { t.Errorf("Expected having to handle version command") } if showHelp { t.Errorf("Expected not having to handle help command") } cmd.Parse([]string{"help"}) showVersion, showHelp, err = ProcessCommandLineArgs(cmd) if err != nil { t.Errorf("Expected no errors, got: %s", err) } if showVersion { t.Errorf("Expected not having to handle version command") } if !showHelp { t.Errorf("Expected having to handle help command") } cmd.Parse([]string{"foo", "-p", "9090"}) _, _, err = ProcessCommandLineArgs(cmd) if err == nil { t.Errorf("Expected an error handling the command arguments") } } func TestRandomPorts(t *testing.T) { opts := DefaultOptions() opts.HTTPPort = -1 opts.Port = -1 s := RunServer(opts) defer s.Shutdown() if s.Addr() == nil || s.Addr().(*net.TCPAddr).Port <= 0 { t.Fatal("Should have dynamically assigned server port.") } if s.Addr() == nil || s.Addr().(*net.TCPAddr).Port == 4222 { t.Fatal("Should not have dynamically assigned default port: 4222.") } if s.MonitorAddr() == nil || s.MonitorAddr().Port <= 0 { t.Fatal("Should have dynamically assigned monitoring port.") } } func TestNilMonitoringPort(t *testing.T) { opts := DefaultOptions() opts.HTTPPort = 0 opts.HTTPSPort = 0 s := RunServer(opts) defer s.Shutdown() if s.MonitorAddr() != nil { t.Fatal("HttpAddr should be nil.") } } type DummyAuth struct { t *testing.T needNonce bool deadline time.Time register bool } func (d *DummyAuth) Check(c ClientAuthentication) bool { if d.needNonce && len(c.GetNonce()) == 0 { d.t.Fatalf("Expected a nonce but received none") } else if !d.needNonce && len(c.GetNonce()) > 0 { d.t.Fatalf("Received a nonce when none was expected") } if c.GetOpts().Username != "valid" { return false } if !d.register { return true } u := &User{ Username: c.GetOpts().Username, ConnectionDeadline: d.deadline, } c.RegisterUser(u) return true } func TestCustomClientAuthentication(t *testing.T) { testAuth := func(t *testing.T, nonce bool) { clientAuth := &DummyAuth{t: t, needNonce: nonce} opts := DefaultOptions() opts.CustomClientAuthentication = clientAuth opts.AlwaysEnableNonce = nonce s := RunServer(opts) defer s.Shutdown() addr := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(addr, nats.UserInfo("valid", "")) if err != nil { t.Fatalf("Expected client to connect, got: %s", err) } nc.Close() if _, err := nats.Connect(addr, nats.UserInfo("invalid", "")); err == nil { t.Fatal("Expected client to fail to connect") } } t.Run("with nonce", func(t *testing.T) { testAuth(t, true) }) t.Run("without nonce", func(t *testing.T) { testAuth(t, false) }) } func TestCustomRouterAuthentication(t *testing.T) { opts := DefaultOptions() opts.CustomRouterAuthentication = &DummyAuth{} opts.Cluster.Host = "127.0.0.1" s := RunServer(opts) defer s.Shutdown() clusterPort := s.ClusterAddr().Port opts2 := DefaultOptions() opts2.Cluster.Host = "127.0.0.1" opts2.Routes = RoutesFromStr(fmt.Sprintf("nats://invalid@127.0.0.1:%d", clusterPort)) s2 := RunServer(opts2) defer s2.Shutdown() // s2 will attempt to connect to s, which should reject. // Keep in mind that s2 will try again... time.Sleep(50 * time.Millisecond) checkNumRoutes(t, s2, 0) opts3 := DefaultOptions() opts3.Cluster.Host = "127.0.0.1" opts3.Routes = RoutesFromStr(fmt.Sprintf("nats://valid@127.0.0.1:%d", clusterPort)) s3 := RunServer(opts3) defer s3.Shutdown() checkClusterFormed(t, s, s3) // Default pool size + 1 for system account checkNumRoutes(t, s3, DEFAULT_ROUTE_POOL_SIZE+1) } func TestMonitoringNoTimeout(t *testing.T) { s := runMonitorServer() defer s.Shutdown() s.mu.Lock() srv := s.monitoringServer s.mu.Unlock() if srv == nil { t.Fatalf("Monitoring server not set") } if srv.ReadTimeout != 0 { t.Fatalf("ReadTimeout should not be set, was set to %v", srv.ReadTimeout) } if srv.WriteTimeout != 0 { t.Fatalf("WriteTimeout should not be set, was set to %v", srv.WriteTimeout) } } func TestProfilingNoTimeout(t *testing.T) { opts := DefaultOptions() opts.ProfPort = -1 s := RunServer(opts) defer s.Shutdown() paddr := s.ProfilerAddr() if paddr == nil { t.Fatalf("Profiler not started") } pport := paddr.Port if pport <= 0 { t.Fatalf("Expected profiler port to be set, got %v", pport) } s.mu.Lock() srv := s.profilingServer s.mu.Unlock() if srv == nil { t.Fatalf("Profiling server not set") } if srv.ReadTimeout != time.Second*5 { t.Fatalf("ReadTimeout should not be set, was set to %v", srv.ReadTimeout) } if srv.WriteTimeout != 0 { t.Fatalf("WriteTimeout should not be set, was set to %v", srv.WriteTimeout) } } func TestLameDuckOptionsValidation(t *testing.T) { o := DefaultOptions() o.LameDuckDuration = 5 * time.Second o.LameDuckGracePeriod = 10 * time.Second s, err := NewServer(o) if s != nil { s.Shutdown() } if err == nil || !strings.Contains(err.Error(), "should be strictly lower") { t.Fatalf("Expected error saying that ldm grace period should be lower than ldm duration, got %v", err) } } func testSetLDMGracePeriod(o *Options, val time.Duration) { // For tests, we set the grace period as a negative value // so we can have a grace period bigger than the total duration. // When validating options, we would not be able to run the // server without this trick. o.LameDuckGracePeriod = val * -1 } func TestLameDuckMode(t *testing.T) { optsA := DefaultOptions() testSetLDMGracePeriod(optsA, time.Nanosecond) optsA.Cluster.Host = "127.0.0.1" srvA := RunServer(optsA) defer srvA.Shutdown() // Check that if there is no client, server is shutdown srvA.lameDuckMode() if !srvA.isShuttingDown() { t.Fatalf("Server should have shutdown") } optsA.LameDuckDuration = 10 * time.Nanosecond srvA = RunServer(optsA) defer srvA.Shutdown() optsB := DefaultOptions() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) total := 50 connectClients := func() []*nats.Conn { ncs := make([]*nats.Conn, 0, total) for i := 0; i < total; i++ { nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port), nats.ReconnectWait(50*time.Millisecond)) if err != nil { t.Fatalf("Error on connect: %v", err) } ncs = append(ncs, nc) } return ncs } stopClientsAndSrvB := func(ncs []*nats.Conn) { for _, nc := range ncs { nc.Close() } srvB.Shutdown() } ncs := connectClients() checkClientsCount(t, srvA, total) checkClientsCount(t, srvB, 0) start := time.Now() srvA.lameDuckMode() // Make sure that nothing bad happens if called twice srvA.lameDuckMode() // Wait that shutdown completes elapsed := time.Since(start) // It should have taken more than the allotted time of 10ms since we had 50 clients. if elapsed <= optsA.LameDuckDuration { t.Fatalf("Expected to take more than %v, got %v", optsA.LameDuckDuration, elapsed) } checkClientsCount(t, srvA, 0) checkClientsCount(t, srvB, total) // Check closed status on server A // Connections are saved in go routines, so although we have evaluated the number // of connections in the server A to be 0, the polling of connection closed may // need a bit more time. checkFor(t, time.Second, 15*time.Millisecond, func() error { cz := pollConz(t, srvA, 1, "", &ConnzOptions{State: ConnClosed}) if n := len(cz.Conns); n != total { return fmt.Errorf("expected %v closed connections, got %v", total, n) } return nil }) cz := pollConz(t, srvA, 1, "", &ConnzOptions{State: ConnClosed}) if n := len(cz.Conns); n != total { t.Fatalf("Expected %v closed connections, got %v", total, n) } for _, c := range cz.Conns { checkReason(t, c.Reason, ServerShutdown) } stopClientsAndSrvB(ncs) optsA.LameDuckDuration = time.Second srvA = RunServer(optsA) defer srvA.Shutdown() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvB = RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) ncs = connectClients() checkClientsCount(t, srvA, total) checkClientsCount(t, srvB, 0) start = time.Now() go srvA.lameDuckMode() // Check that while in lameDuckMode, it is not possible to connect // to the server. Wait to be in LD mode first checkFor(t, 500*time.Millisecond, 15*time.Millisecond, func() error { srvA.mu.Lock() ldm := srvA.ldm srvA.mu.Unlock() if !ldm { return fmt.Errorf("Did not reach lame duck mode") } return nil }) if _, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)); err != nats.ErrNoServers { t.Fatalf("Expected %v, got %v", nats.ErrNoServers, err) } srvA.grWG.Wait() elapsed = time.Since(start) checkClientsCount(t, srvA, 0) checkClientsCount(t, srvB, total) if elapsed > time.Duration(float64(optsA.LameDuckDuration)*1.1) { t.Fatalf("Expected to not take more than %v, got %v", optsA.LameDuckDuration, elapsed) } stopClientsAndSrvB(ncs) // Now check that we can shutdown server while in LD mode. optsA.LameDuckDuration = 60 * time.Second srvA = RunServer(optsA) defer srvA.Shutdown() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvB = RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) ncs = connectClients() checkClientsCount(t, srvA, total) checkClientsCount(t, srvB, 0) start = time.Now() go srvA.lameDuckMode() time.Sleep(100 * time.Millisecond) srvA.Shutdown() elapsed = time.Since(start) // Make sure that it did not take that long if elapsed > time.Second { t.Fatalf("Took too long: %v", elapsed) } checkClientsCount(t, srvA, 0) checkClientsCount(t, srvB, total) stopClientsAndSrvB(ncs) // Now test that we introduce delay before starting closing client connections. // This allow to "signal" multiple servers and avoid their clients to reconnect // to a server that is going to be going in LD mode. testSetLDMGracePeriod(optsA, 100*time.Millisecond) optsA.LameDuckDuration = 10 * time.Millisecond srvA = RunServer(optsA) defer srvA.Shutdown() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) testSetLDMGracePeriod(optsB, 100*time.Millisecond) optsB.LameDuckDuration = 10 * time.Millisecond srvB = RunServer(optsB) defer srvB.Shutdown() optsC := DefaultOptions() optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) testSetLDMGracePeriod(optsC, 100*time.Millisecond) optsC.LameDuckGracePeriod = -100 * time.Millisecond optsC.LameDuckDuration = 10 * time.Millisecond srvC := RunServer(optsC) defer srvC.Shutdown() checkClusterFormed(t, srvA, srvB, srvC) rt := int32(0) nc, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), nats.ReconnectWait(15*time.Millisecond), nats.ReconnectHandler(func(*nats.Conn) { atomic.AddInt32(&rt, 1) })) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() go srvA.lameDuckMode() // Wait a bit, but less than lameDuckModeInitialDelay that we set in this // test to 100ms. time.Sleep(30 * time.Millisecond) go srvB.lameDuckMode() srvA.grWG.Wait() srvB.grWG.Wait() checkClientsCount(t, srvC, 1) checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if n := atomic.LoadInt32(&rt); n != 1 { return fmt.Errorf("Expected client to reconnect only once, got %v", n) } return nil }) } func TestLameDuckModeInfo(t *testing.T) { optsA := testWSOptions() optsA.Cluster.Name = "abc" optsA.Cluster.Host = "127.0.0.1" optsA.Cluster.Port = -1 // Ensure that initial delay is set very high so that we can // check that some events occur as expected before the client // is disconnected. testSetLDMGracePeriod(optsA, 5*time.Second) optsA.LameDuckDuration = 50 * time.Millisecond optsA.DisableShortFirstPing = true srvA := RunServer(optsA) defer srvA.Shutdown() curla := fmt.Sprintf("127.0.0.1:%d", optsA.Port) wscurla := fmt.Sprintf("127.0.0.1:%d", optsA.Websocket.Port) c, err := net.Dial("tcp", curla) if err != nil { t.Fatalf("Error connecting: %v", err) } defer c.Close() client := bufio.NewReaderSize(c, maxBufSize) wsconn, wsclient := testWSCreateClient(t, false, false, optsA.Websocket.Host, optsA.Websocket.Port) defer wsconn.Close() getInfo := func(ws bool) *serverInfo { t.Helper() var l string var err error if ws { l = string(testWSReadFrame(t, wsclient)) } else { l, err = client.ReadString('\n') if err != nil { t.Fatalf("Error receiving info from server: %v\n", err) } } var info serverInfo if err = json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatalf("Could not parse INFO json: %v\n", err) } return &info } getInfo(false) c.Write([]byte("CONNECT {\"protocol\":1,\"verbose\":false}\r\nPING\r\n")) client.ReadString('\n') optsB := testWSOptions() optsB.Cluster.Name = "abc" optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) checkConnectURLs := func(expected [][]string) *serverInfo { t.Helper() var si *serverInfo for i, ws := range []bool{false, true} { slices.Sort(expected[i]) si = getInfo(ws) slices.Sort(si.ConnectURLs) if !reflect.DeepEqual(expected[i], si.ConnectURLs) { t.Fatalf("Expected %q, got %q", expected, si.ConnectURLs) } } return si } curlb := fmt.Sprintf("127.0.0.1:%d", optsB.Port) wscurlb := fmt.Sprintf("127.0.0.1:%d", optsB.Websocket.Port) expected := [][]string{{curla, curlb}, {wscurla, wscurlb}} checkConnectURLs(expected) optsC := testWSOptions() testSetLDMGracePeriod(optsA, 5*time.Second) optsC.Cluster.Name = "abc" optsC.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvC := RunServer(optsC) defer srvC.Shutdown() checkClusterFormed(t, srvA, srvB, srvC) curlc := fmt.Sprintf("127.0.0.1:%d", optsC.Port) wscurlc := fmt.Sprintf("127.0.0.1:%d", optsC.Websocket.Port) expected = [][]string{{curla, curlb, curlc}, {wscurla, wscurlb, wscurlc}} checkConnectURLs(expected) optsD := testWSOptions() optsD.Cluster.Name = "abc" optsD.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srvA.ClusterAddr().Port)) srvD := RunServer(optsD) defer srvD.Shutdown() checkClusterFormed(t, srvA, srvB, srvC, srvD) curld := fmt.Sprintf("127.0.0.1:%d", optsD.Port) wscurld := fmt.Sprintf("127.0.0.1:%d", optsD.Websocket.Port) expected = [][]string{{curla, curlb, curlc, curld}, {wscurla, wscurlb, wscurlc, wscurld}} checkConnectURLs(expected) // Now lame duck server A and C. We should have client connected to A // receive info that A is in LDM without A's URL, but also receive // an update with C's URL gone. // But first we need to create a client to C because otherwise the // LDM signal will just shut it down because it would have no client. nc, err := nats.Connect(srvC.ClientURL()) if err != nil { t.Fatalf("Failed to connect: %v", err) } defer nc.Close() nc.Flush() start := time.Now() wg := sync.WaitGroup{} wg.Add(2) go func() { defer wg.Done() srvA.lameDuckMode() }() expected = [][]string{{curlb, curlc, curld}, {wscurlb, wscurlc, wscurld}} si := checkConnectURLs(expected) if !si.LameDuckMode { t.Fatal("Expected LameDuckMode to be true, it was not") } // Start LDM for server C. This should send an update to A // which in turn should remove C from the list of URLs and // update its client. go func() { defer wg.Done() srvC.lameDuckMode() }() expected = [][]string{{curlb, curld}, {wscurlb, wscurld}} si = checkConnectURLs(expected) // This update should not say that it is LDM. if si.LameDuckMode { t.Fatal("Expected LameDuckMode to be false, it was true") } // Now shutdown D, and we also should get an update. srvD.Shutdown() expected = [][]string{{curlb}, {wscurlb}} si = checkConnectURLs(expected) // This update should not say that it is LDM. if si.LameDuckMode { t.Fatal("Expected LameDuckMode to be false, it was true") } if time.Since(start) > 2*time.Second { t.Fatalf("Did not get the expected events prior of server A and C shutting down") } // Now explicitly shutdown srvA. When a server shutdown, it closes all its // connections. For routes, it means that it is going to remove the remote's // URL from its map. We want to make sure that in that case, server does not // actually send an updated INFO to its clients. srvA.Shutdown() // Expect nothing to be received on the client connection. if l, err := client.ReadString('\n'); err == nil { t.Fatalf("Expected connection to fail, instead got %q", l) } c.Close() nc.Close() // Don't need to wait for actual disconnect of clients. srvC.Shutdown() wg.Wait() } func TestServerValidateGatewaysOptions(t *testing.T) { baseOpt := testDefaultOptionsForGateway("A") u, _ := url.Parse("host:5222") g := &RemoteGatewayOpts{ URLs: []*url.URL{u}, } baseOpt.Gateway.Gateways = append(baseOpt.Gateway.Gateways, g) for _, test := range []struct { name string opts func() *Options expectedErr string }{ { name: "gateway_has_no_name", opts: func() *Options { o := baseOpt.Clone() o.Gateway.Name = "" return o }, expectedErr: "has no name", }, { name: "gateway_has_no_port", opts: func() *Options { o := baseOpt.Clone() o.Gateway.Port = 0 return o }, expectedErr: "no port specified", }, { name: "gateway_dst_has_no_name", opts: func() *Options { o := baseOpt.Clone() return o }, expectedErr: "has no name", }, { name: "gateway_dst_urls_is_nil", opts: func() *Options { o := baseOpt.Clone() o.Gateway.Gateways[0].Name = "B" o.Gateway.Gateways[0].URLs = nil return o }, expectedErr: "has no URL", }, { name: "gateway_dst_urls_is_empty", opts: func() *Options { o := baseOpt.Clone() o.Gateway.Gateways[0].Name = "B" o.Gateway.Gateways[0].URLs = []*url.URL{} return o }, expectedErr: "has no URL", }, } { t.Run(test.name, func(t *testing.T) { if err := validateOptions(test.opts()); err == nil || !strings.Contains(err.Error(), test.expectedErr) { t.Fatalf("Expected error about %q, got %v", test.expectedErr, err) } }) } } func TestAcceptError(t *testing.T) { o := DefaultOptions() s := New(o) s.running.Store(true) defer s.Shutdown() orgDelay := time.Hour delay := s.acceptError("Test", fmt.Errorf("any error"), orgDelay) if delay != orgDelay { t.Fatalf("With this type of error, delay should have stayed same, got %v", delay) } // Create any net.Error and make it a temporary ne := &net.DNSError{IsTemporary: true} orgDelay = 10 * time.Millisecond delay = s.acceptError("Test", ne, orgDelay) if delay != 2*orgDelay { t.Fatalf("Expected delay to double, got %v", delay) } // Now check the max orgDelay = 60 * ACCEPT_MAX_SLEEP / 100 delay = s.acceptError("Test", ne, orgDelay) if delay != ACCEPT_MAX_SLEEP { t.Fatalf("Expected delay to double, got %v", delay) } wg := sync.WaitGroup{} wg.Add(1) start := time.Now() go func() { s.acceptError("Test", ne, orgDelay) wg.Done() }() time.Sleep(100 * time.Millisecond) // This should kick out the sleep in acceptError s.Shutdown() if dur := time.Since(start); dur >= ACCEPT_MAX_SLEEP { t.Fatalf("Shutdown took too long: %v", dur) } wg.Wait() if d := s.acceptError("Test", ne, orgDelay); d >= 0 { t.Fatalf("Expected delay to be negative, got %v", d) } } func TestServerShutdownDuringStart(t *testing.T) { o := DefaultOptions() o.ServerName = "server" o.DisableShortFirstPing = true o.Accounts = []*Account{NewAccount("$SYS")} o.SystemAccount = "$SYS" o.Cluster.Name = "abc" o.Cluster.Host = "127.0.0.1" o.Cluster.Port = -1 o.Gateway.Name = "abc" o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 o.LeafNode.Host = "127.0.0.1" o.LeafNode.Port = -1 o.Websocket.Host = "127.0.0.1" o.Websocket.Port = -1 o.Websocket.HandshakeTimeout = 1 o.Websocket.NoTLS = true o.MQTT.Host = "127.0.0.1" o.MQTT.Port = -1 // We are going to test that if the server is shutdown // while Start() runs (in this case, before), we don't // start the listeners and therefore leave accept loops // hanging. s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } s.Shutdown() // Start() should not block, but just in case, start in // different go routine. ch := make(chan struct{}, 1) go func() { s.Start() close(ch) }() select { case <-ch: case <-time.After(time.Second): t.Fatal("Start appear to have blocked after server was shutdown") } // Now make sure that none of the listeners have been created listeners := []string{} s.mu.Lock() if s.listener != nil { listeners = append(listeners, "client") } if s.routeListener != nil { listeners = append(listeners, "route") } if s.gatewayListener != nil { listeners = append(listeners, "gateway") } if s.leafNodeListener != nil { listeners = append(listeners, "leafnode") } if s.websocket.listener != nil { listeners = append(listeners, "websocket") } if s.mqtt.listener != nil { listeners = append(listeners, "mqtt") } s.mu.Unlock() if len(listeners) > 0 { lst := "" for i, l := range listeners { if i > 0 { lst += ", " } lst += l } t.Fatalf("Following listeners have been created: %s", lst) } } type myDummyDNSResolver struct { ips []string err error } func (r *myDummyDNSResolver) LookupHost(ctx context.Context, host string) ([]string, error) { if r.err != nil { return nil, r.err } return r.ips, nil } func TestGetRandomIP(t *testing.T) { s := &Server{} resolver := &myDummyDNSResolver{} // no port... if _, err := s.getRandomIP(resolver, "noport", nil); err == nil || !strings.Contains(err.Error(), "port") { t.Fatalf("Expected error about port missing, got %v", err) } resolver.err = fmt.Errorf("on purpose") if _, err := s.getRandomIP(resolver, "localhost:4222", nil); err == nil || !strings.Contains(err.Error(), "on purpose") { t.Fatalf("Expected error about no port, got %v", err) } resolver.err = nil a, err := s.getRandomIP(resolver, "localhost:4222", nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } if a != "localhost:4222" { t.Fatalf("Expected address to be %q, got %q", "localhost:4222", a) } resolver.ips = []string{"1.2.3.4"} a, err = s.getRandomIP(resolver, "localhost:4222", nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } if a != "1.2.3.4:4222" { t.Fatalf("Expected address to be %q, got %q", "1.2.3.4:4222", a) } // Check for randomness resolver.ips = []string{"1.2.3.4", "2.2.3.4", "3.2.3.4"} dist := [3]int{} for i := 0; i < 100; i++ { ip, err := s.getRandomIP(resolver, "localhost:4222", nil) if err != nil { t.Fatalf("Unexpected error: %v", err) } v := int(ip[0]-'0') - 1 dist[v]++ } low := 20 high := 47 for i, d := range dist { if d == 0 || d == 100 { t.Fatalf("Unexpected distribution for ip %v, got %v", i, d) } else if d < low || d > high { t.Logf("Warning: out of expected range [%v,%v] for ip %v, got %v", low, high, i, d) } } // Check IP exclusions excludedIPs := map[string]struct{}{"1.2.3.4:4222": {}} for i := 0; i < 100; i++ { ip, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ip[0] == '1' { t.Fatalf("Should not have returned this ip: %q", ip) } } excludedIPs["2.2.3.4:4222"] = struct{}{} for i := 0; i < 100; i++ { ip, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs) if err != nil { t.Fatalf("Unexpected error: %v", err) } if ip[0] != '3' { t.Fatalf("Should only have returned '3.2.3.4', got returned %q", ip) } } excludedIPs["3.2.3.4:4222"] = struct{}{} for i := 0; i < 100; i++ { if _, err := s.getRandomIP(resolver, "localhost:4222", excludedIPs); err != errNoIPAvail { t.Fatalf("Unexpected error: %v", err) } } // Now check that exclusion takes into account the port number. resolver.ips = []string{"127.0.0.1"} excludedIPs = map[string]struct{}{"127.0.0.1:4222": {}} for i := 0; i < 100; i++ { if _, err := s.getRandomIP(resolver, "localhost:4223", excludedIPs); err == errNoIPAvail { t.Fatal("Should not have failed") } } } type shortWriteConn struct { net.Conn } func (swc *shortWriteConn) Write(b []byte) (int, error) { // Limit the write to 10 bytes at a time. short := false max := len(b) if max > 10 { max = 10 short = true } n, err := swc.Conn.Write(b[:max]) if err == nil && short { return n, io.ErrShortWrite } return n, err } func TestClientWriteLoopStall(t *testing.T) { opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() errCh := make(chan error, 1) url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) { select { case errCh <- e: default: } })) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatalf("Error on subscribe: %v", err) } nc.Flush() cid, _ := nc.GetClientID() sender, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer sender.Close() c := s.getClient(cid) c.mu.Lock() c.nc = &shortWriteConn{Conn: c.nc} c.mu.Unlock() sender.Publish("foo", make([]byte, 100)) if _, err := sub.NextMsg(3 * time.Second); err != nil { t.Fatalf("WriteLoop has stalled!") } // Make sure that we did not get any async error select { case e := <-errCh: t.Fatalf("Got error: %v", e) case <-time.After(250 * time.Millisecond): } } func TestInsecureSkipVerifyWarning(t *testing.T) { checkWarnReported := func(t *testing.T, o *Options, expectedWarn string) { t.Helper() s, err := NewServer(o) if err != nil { t.Fatalf("Error on new server: %v", err) } l := &captureWarnLogger{warn: make(chan string, 1)} s.SetLogger(l, false, false) wg := sync.WaitGroup{} wg.Add(1) go func() { s.Start() wg.Done() }() if err := s.readyForConnections(time.Second); err != nil { t.Fatal(err) } select { case w := <-l.warn: if !strings.Contains(w, expectedWarn) { t.Fatalf("Expected warning %q, got %q", expectedWarn, w) } case <-time.After(2 * time.Second): t.Fatalf("Did not get warning %q", expectedWarn) } s.Shutdown() wg.Wait() } tc := &TLSConfigOpts{} tc.CertFile = "../test/configs/certs/server-cert.pem" tc.KeyFile = "../test/configs/certs/server-key.pem" tc.CaFile = "../test/configs/certs/ca.pem" tc.Insecure = true config, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } o := DefaultOptions() o.Cluster.Name = "A" o.Cluster.Port = -1 o.Cluster.TLSConfig = config.Clone() checkWarnReported(t, o, clusterTLSInsecureWarning) // Remove the route setting o.Cluster.Port = 0 o.Cluster.TLSConfig = nil // Configure LeafNode with no TLS in the main block first, but only with remotes. o.LeafNode.Port = -1 rurl, _ := url.Parse("nats://127.0.0.1:1234") o.LeafNode.Remotes = []*RemoteLeafOpts{ { URLs: []*url.URL{rurl}, TLSConfig: config.Clone(), }, } checkWarnReported(t, o, leafnodeTLSInsecureWarning) // Now add to main block. o.LeafNode.TLSConfig = config.Clone() checkWarnReported(t, o, leafnodeTLSInsecureWarning) // Now remove remote and check warning still reported o.LeafNode.Remotes = nil checkWarnReported(t, o, leafnodeTLSInsecureWarning) // Remove the LN setting o.LeafNode.Port = 0 o.LeafNode.TLSConfig = nil // Configure GW with no TLS in main block first, but only with remotes o.Gateway.Name = "A" o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 o.Gateway.Gateways = []*RemoteGatewayOpts{ { Name: "B", URLs: []*url.URL{rurl}, TLSConfig: config.Clone(), }, } checkWarnReported(t, o, gatewayTLSInsecureWarning) // Now add to main block. o.Gateway.TLSConfig = config.Clone() checkWarnReported(t, o, gatewayTLSInsecureWarning) // Now remove remote and check warning still reported o.Gateway.Gateways = nil checkWarnReported(t, o, gatewayTLSInsecureWarning) } func TestConnectErrorReports(t *testing.T) { // On Windows, an attempt to connect to a port that has no listener will // take whatever timeout specified in DialTimeout() before failing. // So skip for now. if runtime.GOOS == "windows" { t.Skip() } // Check that default report attempts is as expected opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() if ra := s.getOpts().ConnectErrorReports; ra != DEFAULT_CONNECT_ERROR_REPORTS { t.Fatalf("Expected default value to be %v, got %v", DEFAULT_CONNECT_ERROR_REPORTS, ra) } tmpFile := createTempFile(t, "") log := tmpFile.Name() tmpFile.Close() remoteURLs := RoutesFromStr("nats://127.0.0.1:1234") opts = DefaultOptions() opts.ConnectErrorReports = 3 opts.Cluster.Port = -1 opts.Routes = remoteURLs opts.NoLog = false opts.LogFile = log opts.Logtime = true opts.Debug = true s = RunServer(opts) defer s.Shutdown() checkContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { content, err := os.ReadFile(log) if err != nil { return fmt.Errorf("Error reading log file: %v", err) } present := bytes.Contains(content, []byte(fmt.Sprintf("%s (attempt %d)", txt, attempt))) if shouldBeThere && !present { return fmt.Errorf("Did not find expected log statement (%s) for attempt %d: %s", txt, attempt, content) } else if !shouldBeThere && present { return fmt.Errorf("Log statement (%s) for attempt %d should not be present: %s", txt, attempt, content) } return nil }) } type testConnect struct { name string attempt int errExpected bool } for _, test := range []testConnect{ {"route_attempt_1", 1, true}, {"route_attempt_2", 2, false}, {"route_attempt_3", 3, true}, {"route_attempt_4", 4, false}, {"route_attempt_6", 6, true}, {"route_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected checkContent(t, "[DBG] Error trying to connect to route", test.attempt, debugExpected) checkContent(t, "[ERR] Error trying to connect to route", test.attempt, test.errExpected) }) } s.Shutdown() removeFile(t, log) // Now try with leaf nodes opts.Cluster.Port = 0 opts.Routes = nil opts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{remoteURLs[0]}}} opts.LeafNode.ReconnectInterval = 15 * time.Millisecond s = RunServer(opts) defer s.Shutdown() checkLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { content, err := os.ReadFile(log) if err != nil { return fmt.Errorf("Error reading log file: %v", err) } present := bytes.Contains(content, []byte(fmt.Sprintf("%s %q (attempt %d)", txt, host, attempt))) if shouldBeThere && !present { return fmt.Errorf("Did not find expected log statement (%s %q) for attempt %d: %s", txt, host, attempt, content) } else if !shouldBeThere && present { return fmt.Errorf("Log statement (%s %q) for attempt %d should not be present: %s", txt, host, attempt, content) } return nil }) } for _, test := range []testConnect{ {"leafnode_attempt_1", 1, true}, {"leafnode_attempt_2", 2, false}, {"leafnode_attempt_3", 3, true}, {"leafnode_attempt_4", 4, false}, {"leafnode_attempt_6", 6, true}, {"leafnode_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected checkLeafContent(t, "[DBG] Error trying to connect as leafnode to remote server", remoteURLs[0].Host, test.attempt, debugExpected) checkLeafContent(t, "[ERR] Error trying to connect as leafnode to remote server", remoteURLs[0].Host, test.attempt, test.errExpected) }) } s.Shutdown() removeFile(t, log) // Now try with gateways opts.LeafNode.Remotes = nil opts.Cluster.Name = "A" opts.Gateway.Name = "A" opts.Gateway.Port = -1 opts.Gateway.Gateways = []*RemoteGatewayOpts{ { Name: "B", URLs: remoteURLs, }, } opts.gatewaysSolicitDelay = 15 * time.Millisecond s = RunServer(opts) defer s.Shutdown() for _, test := range []testConnect{ {"gateway_attempt_1", 1, true}, {"gateway_attempt_2", 2, false}, {"gateway_attempt_3", 3, true}, {"gateway_attempt_4", 4, false}, {"gateway_attempt_6", 6, true}, {"gateway_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected infoExpected := test.errExpected // For gateways, we also check our notice that we attempt to connect checkContent(t, "[DBG] Connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, debugExpected) checkContent(t, "[INF] Connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, infoExpected) checkContent(t, "[DBG] Error connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, debugExpected) checkContent(t, "[ERR] Error connecting to explicit gateway \"B\" (127.0.0.1:1234) at 127.0.0.1:1234", test.attempt, test.errExpected) }) } } func TestReconnectErrorReports(t *testing.T) { // On Windows, an attempt to connect to a port that has no listener will // take whatever timeout specified in DialTimeout() before failing. // So skip for now. if runtime.GOOS == "windows" { t.Skip() } // Check that default report attempts is as expected opts := DefaultOptions() s := RunServer(opts) defer s.Shutdown() if ra := s.getOpts().ReconnectErrorReports; ra != DEFAULT_RECONNECT_ERROR_REPORTS { t.Fatalf("Expected default value to be %v, got %v", DEFAULT_RECONNECT_ERROR_REPORTS, ra) } tmpFile := createTempFile(t, "") log := tmpFile.Name() tmpFile.Close() csOpts := DefaultOptions() csOpts.Cluster.Port = -1 cs := RunServer(csOpts) defer cs.Shutdown() opts = DefaultOptions() opts.ReconnectErrorReports = 3 opts.Cluster.Port = -1 opts.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", cs.ClusterAddr().Port)) opts.NoLog = false opts.LogFile = log opts.Logtime = true opts.Debug = true s = RunServer(opts) defer s.Shutdown() // Wait for cluster to be formed checkClusterFormed(t, s, cs) // Now shutdown the server s connected to. cs.Shutdown() // Specifically for route test, wait at least reconnect interval before checking logs time.Sleep(DEFAULT_ROUTE_RECONNECT) checkContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { content, err := os.ReadFile(log) if err != nil { return fmt.Errorf("Error reading log file: %v", err) } present := bytes.Contains(content, []byte(fmt.Sprintf("%s (attempt %d)", txt, attempt))) if shouldBeThere && !present { return fmt.Errorf("Did not find expected log statement (%s) for attempt %d: %s", txt, attempt, content) } else if !shouldBeThere && present { return fmt.Errorf("Log statement (%s) for attempt %d should not be present: %s", txt, attempt, content) } return nil }) } type testConnect struct { name string attempt int errExpected bool } for _, test := range []testConnect{ {"route_attempt_1", 1, true}, {"route_attempt_2", 2, false}, {"route_attempt_3", 3, true}, {"route_attempt_4", 4, false}, {"route_attempt_6", 6, true}, {"route_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected checkContent(t, "[DBG] Error trying to connect to route", test.attempt, debugExpected) checkContent(t, "[ERR] Error trying to connect to route", test.attempt, test.errExpected) }) } s.Shutdown() removeFile(t, log) // Now try with leaf nodes csOpts.Cluster.Port = 0 csOpts.Cluster.Name = _EMPTY_ csOpts.LeafNode.Host = "127.0.0.1" csOpts.LeafNode.Port = -1 cs = RunServer(csOpts) defer cs.Shutdown() opts.Cluster.Port = 0 opts.Cluster.Name = _EMPTY_ opts.Routes = nil u, _ := url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", csOpts.LeafNode.Port)) opts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}} opts.LeafNode.ReconnectInterval = 15 * time.Millisecond s = RunServer(opts) defer s.Shutdown() checkLeafNodeConnected(t, s) // Now shutdown the server s is connected to cs.Shutdown() checkLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { content, err := os.ReadFile(log) if err != nil { return fmt.Errorf("Error reading log file: %v", err) } present := bytes.Contains(content, []byte(fmt.Sprintf("%s %q (attempt %d)", txt, host, attempt))) if shouldBeThere && !present { return fmt.Errorf("Did not find expected log statement (%s %q) for attempt %d: %s", txt, host, attempt, content) } else if !shouldBeThere && present { return fmt.Errorf("Log statement (%s %q) for attempt %d should not be present: %s", txt, host, attempt, content) } return nil }) } for _, test := range []testConnect{ {"leafnode_attempt_1", 1, true}, {"leafnode_attempt_2", 2, false}, {"leafnode_attempt_3", 3, true}, {"leafnode_attempt_4", 4, false}, {"leafnode_attempt_6", 6, true}, {"leafnode_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected checkLeafContent(t, "[DBG] Error trying to connect as leafnode to remote server", u.Host, test.attempt, debugExpected) checkLeafContent(t, "[ERR] Error trying to connect as leafnode to remote server", u.Host, test.attempt, test.errExpected) }) } s.Shutdown() removeFile(t, log) // Now try with gateways csOpts.LeafNode.Port = 0 csOpts.Cluster.Name = "B" csOpts.Gateway.Name = "B" csOpts.Gateway.Port = -1 cs = RunServer(csOpts) opts.LeafNode.Remotes = nil opts.Cluster.Name = "A" opts.Gateway.Name = "A" opts.Gateway.Port = -1 remoteGWPort := cs.GatewayAddr().Port u, _ = url.Parse(fmt.Sprintf("nats://127.0.0.1:%d", remoteGWPort)) opts.Gateway.Gateways = []*RemoteGatewayOpts{ { Name: "B", URLs: []*url.URL{u}, }, } opts.gatewaysSolicitDelay = 15 * time.Millisecond s = RunServer(opts) defer s.Shutdown() waitForOutboundGateways(t, s, 1, 2*time.Second) waitForInboundGateways(t, s, 1, 2*time.Second) // Now stop server s is connecting to cs.Shutdown() connTxt := fmt.Sprintf("Connecting to explicit gateway \"B\" (127.0.0.1:%d) at 127.0.0.1:%d", remoteGWPort, remoteGWPort) dbgConnTxt := fmt.Sprintf("[DBG] %s", connTxt) infConnTxt := fmt.Sprintf("[INF] %s", connTxt) errTxt := fmt.Sprintf("Error connecting to explicit gateway \"B\" (127.0.0.1:%d) at 127.0.0.1:%d", remoteGWPort, remoteGWPort) dbgErrTxt := fmt.Sprintf("[DBG] %s", errTxt) errErrTxt := fmt.Sprintf("[ERR] %s", errTxt) for _, test := range []testConnect{ {"gateway_attempt_1", 1, true}, {"gateway_attempt_2", 2, false}, {"gateway_attempt_3", 3, true}, {"gateway_attempt_4", 4, false}, {"gateway_attempt_6", 6, true}, {"gateway_attempt_7", 7, false}, } { t.Run(test.name, func(t *testing.T) { debugExpected := !test.errExpected infoExpected := test.errExpected // For gateways, we also check our notice that we attempt to connect checkContent(t, dbgConnTxt, test.attempt, debugExpected) checkContent(t, infConnTxt, test.attempt, infoExpected) checkContent(t, dbgErrTxt, test.attempt, debugExpected) checkContent(t, errErrTxt, test.attempt, test.errExpected) }) } } func TestServerLogsConfigurationFile(t *testing.T) { file := createTempFile(t, "nats_server_log_") file.Close() conf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 logfile: '%s' `, file.Name()))) o := LoadConfig(conf) o.ConfigFile = file.Name() o.NoLog = false s := RunServer(o) s.Shutdown() log, err := os.ReadFile(file.Name()) if err != nil { t.Fatalf("Error reading log file: %v", err) } if !bytes.Contains(log, []byte(fmt.Sprintf("Using configuration file: %s", file.Name()))) { t.Fatalf("Config file location was not reported in log: %s", log) } } func TestServerRateLimitLogging(t *testing.T) { s := RunServer(DefaultOptions()) defer s.Shutdown() s.changeRateLimitLogInterval(100 * time.Millisecond) l := &captureWarnLogger{warn: make(chan string, 100)} s.SetLogger(l, false, false) s.RateLimitWarnf("Warning number 1") s.RateLimitWarnf("Warning number 2") s.rateLimitFormatWarnf("warning value %d", 1) s.RateLimitWarnf("Warning number 1") s.RateLimitWarnf("Warning number 2") s.rateLimitFormatWarnf("warning value %d", 2) checkLog := func(c1, c2 *client) { t.Helper() nb1 := "Warning number 1" nb2 := "Warning number 2" nbv := "warning value" gotOne := 0 gotTwo := 0 gotFormat := 0 for done := false; !done; { select { case w := <-l.warn: if strings.Contains(w, nb1) { gotOne++ } else if strings.Contains(w, nb2) { gotTwo++ } else if strings.Contains(w, nbv) { gotFormat++ } case <-time.After(150 * time.Millisecond): done = true } } if gotOne != 1 { t.Fatalf("Should have had only 1 warning for nb1, got %v", gotOne) } if gotTwo != 1 { t.Fatalf("Should have had only 1 warning for nb2, got %v", gotTwo) } if gotFormat != 1 { t.Fatalf("Should have had only 1 warning for format, got %v", gotFormat) } // Wait for more than the expiration interval time.Sleep(200 * time.Millisecond) if c1 == nil { s.RateLimitWarnf("%s", nb1) s.rateLimitFormatWarnf("warning value %d", 1) } else { c1.RateLimitWarnf("%s", nb1) c2.RateLimitWarnf("%s", nb1) c1.rateLimitFormatWarnf("warning value %d", 1) } gotOne = 0 gotFormat = 0 for { select { case w := <-l.warn: if strings.Contains(w, nb1) { gotOne++ } else if strings.Contains(w, nbv) { gotFormat++ } case <-time.After(200 * time.Millisecond): if gotOne == 0 { t.Fatalf("Warning was still suppressed") } else if gotOne > 1 { t.Fatalf("Should have had only 1 warning for nb1, got %v", gotOne) } else if gotFormat == 0 { t.Fatalf("Warning was still suppressed") } else if gotFormat > 1 { t.Fatalf("Should have had only 1 warning for format, got %v", gotFormat) } else { // OK! we are done return } } } } checkLog(nil, nil) nc1 := natsConnect(t, s.ClientURL(), nats.Name("c1")) defer nc1.Close() nc2 := natsConnect(t, s.ClientURL(), nats.Name("c2")) defer nc2.Close() var c1 *client var c2 *client s.mu.Lock() for _, cli := range s.clients { cli.mu.Lock() switch cli.opts.Name { case "c1": c1 = cli case "c2": c2 = cli } cli.mu.Unlock() if c1 != nil && c2 != nil { break } } s.mu.Unlock() if c1 == nil || c2 == nil { t.Fatal("Did not find the clients") } // Wait for more than the expiration interval time.Sleep(200 * time.Millisecond) c1.RateLimitWarnf("Warning number 1") c1.RateLimitWarnf("Warning number 2") c1.rateLimitFormatWarnf("warning value %d", 1) c2.RateLimitWarnf("Warning number 1") c2.RateLimitWarnf("Warning number 2") c2.rateLimitFormatWarnf("warning value %d", 2) checkLog(c1, c2) } // https://github.com/nats-io/nats-server/discussions/4535 func TestServerAuthBlockAndSysAccounts(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: s-test authorization { users = [ { user: "u", password: "pass"} ] } accounts { $SYS: { users: [ { user: admin, password: pwd } ] } } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // This should work of course. nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("u", "pass")) require_NoError(t, err) defer nc.Close() // This should not. _, err = nats.Connect(s.ClientURL()) require_Error(t, err, nats.ErrAuthorization, errors.New("nats: Authorization Violation")) } // https://github.com/nats-io/nats-server/issues/5396 func TestServerConfigLastLineComments(t *testing.T) { conf := createConfFile(t, []byte(` { "listen": "0.0.0.0:4222" } # wibble `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // This should work of course. nc, err := nats.Connect(s.ClientURL()) require_NoError(t, err) defer nc.Close() } func TestServerClientURL(t *testing.T) { for host, expected := range map[string]string{ "host.com": "nats://host.com:12345", "1.2.3.4": "nats://1.2.3.4:12345", "2000::1": "nats://[2000::1]:12345", } { o := DefaultOptions() o.Host = host o.Port = 12345 s, err := NewServer(o) require_NoError(t, err) require_Equal(t, s.ClientURL(), expected) } } nats-server-2.10.27/server/service.go000066400000000000000000000016621477524627100174550ustar00rootroot00000000000000// Copyright 2012-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows // +build !windows package server // Run starts the NATS server. This wrapper function allows Windows to add a // hook for running NATS as a service. func Run(server *Server) error { server.Start() return nil } // isWindowsService indicates if NATS is running as a Windows service. func isWindowsService() bool { return false } nats-server-2.10.27/server/service_test.go000066400000000000000000000023201477524627100205040ustar00rootroot00000000000000// Copyright 2012-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows // +build !windows package server import ( "testing" "time" ) func TestRun(t *testing.T) { var ( s = New(DefaultOptions()) started = make(chan error, 1) errC = make(chan error, 1) ) go func() { errC <- Run(s) }() go func() { if err := s.readyForConnections(time.Second); err != nil { started <- err return } s.Shutdown() close(started) }() select { case err := <-errC: if err != nil { t.Fatalf("Unexpected error: %v", err) } case <-time.After(2 * time.Second): t.Fatal("Timed out") } if err := <-started; err != nil { t.Fatalf("Unexpected error: %v", err) } } nats-server-2.10.27/server/service_windows.go000066400000000000000000000073701477524627100212310ustar00rootroot00000000000000// Copyright 2012-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "os" "time" "golang.org/x/sys/windows/svc" ) const ( reopenLogCode = 128 reopenLogCmd = svc.Cmd(reopenLogCode) ldmCode = 129 ldmCmd = svc.Cmd(ldmCode) acceptReopenLog = svc.Accepted(reopenLogCode) ) var serviceName = "nats-server" // SetServiceName allows setting a different service name func SetServiceName(name string) { serviceName = name } // winServiceWrapper implements the svc.Handler interface for implementing // nats-server as a Windows service. type winServiceWrapper struct { server *Server } var dockerized = false var startupDelay = 10 * time.Second func init() { if v, exists := os.LookupEnv("NATS_DOCKERIZED"); exists && v == "1" { dockerized = true } } // Execute will be called by the package code at the start of // the service, and the service will exit once Execute completes. // Inside Execute you must read service change requests from r and // act accordingly. You must keep service control manager up to date // about state of your service by writing into s as required. // args contains service name followed by argument strings passed // to the service. // You can provide service exit code in exitCode return parameter, // with 0 being "no error". You can also indicate if exit code, // if any, is service specific or not by using svcSpecificEC // parameter. func (w *winServiceWrapper) Execute(args []string, changes <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { status <- svc.Status{State: svc.StartPending} go w.server.Start() if v, exists := os.LookupEnv("NATS_STARTUP_DELAY"); exists { if delay, err := time.ParseDuration(v); err == nil { startupDelay = delay } else { w.server.Errorf("Failed to parse \"%v\" as a duration for startup: %s", v, err) } } // Wait for accept loop(s) to be started if !w.server.ReadyForConnections(startupDelay) { // Failed to start. return false, 1 } status <- svc.Status{ State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.AcceptParamChange | acceptReopenLog, } loop: for change := range changes { switch change.Cmd { case svc.Interrogate: status <- change.CurrentStatus case svc.Stop, svc.Shutdown: w.server.Shutdown() break loop case reopenLogCmd: // File log re-open for rotating file logs. w.server.ReOpenLogFile() case ldmCmd: go w.server.lameDuckMode() case svc.ParamChange: if err := w.server.Reload(); err != nil { w.server.Errorf("Failed to reload server configuration: %s", err) } default: w.server.Debugf("Unexpected control request: %v", change.Cmd) } } status <- svc.Status{State: svc.StopPending} return false, 0 } // Run starts the NATS server as a Windows service. func Run(server *Server) error { if dockerized { server.Start() return nil } isWindowsService, err := svc.IsWindowsService() if err != nil { return err } if !isWindowsService { server.Start() return nil } return svc.Run(serviceName, &winServiceWrapper{server}) } // isWindowsService indicates if NATS is running as a Windows service. func isWindowsService() bool { if dockerized { return false } isWindowsService, _ := svc.IsWindowsService() return isWindowsService } nats-server-2.10.27/server/service_windows_test.go000066400000000000000000000065701477524627100222710ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package server import ( "errors" "fmt" "os" "testing" "time" "golang.org/x/sys/windows/svc" ) // TestWinServiceWrapper reproduces the tests for service, // with extra complication for windows API func TestWinServiceWrapper(t *testing.T) { /* Since the windows API can't be tested through just the Run func, a basic mock checks the intractions of the server, as happens in svc.Run. This test checks : - that the service fails to start within an unreasonable timeframe (serverC) - that the service signals its state correctly to windows with svc.StartPending (mockC) - that no other signal is sent to the windows service API(mockC) */ var ( wsw = &winServiceWrapper{New(DefaultOptions())} args = make([]string, 0) serverC = make(chan error, 1) mockC = make(chan error, 1) changes = make(chan svc.ChangeRequest) status = make(chan svc.Status) ) time.Sleep(time.Millisecond) os.Setenv("NATS_STARTUP_DELAY", "1ms") // purposefly small // prepare mock expectations wsm := &winSvcMock{status: status} wsm.Expect(svc.StartPending) go func() { mockC <- wsm.Listen(50*time.Millisecond, t) close(mockC) }() go func() { // ensure failure with these conditions _, exitCode := wsw.Execute(args, changes, status) if exitCode == 0 { // expect error serverC <- errors.New("Should have exitCode != 0") } wsw.server.Shutdown() close(serverC) }() for expectedErrs := 2; expectedErrs >= 0; { select { case err := <-mockC: if err != nil { t.Fatalf("windows.svc mock: %v", err) } else { expectedErrs-- } case err := <-serverC: if err != nil { t.Fatalf("Server behavior: %v", err) } else { expectedErrs-- } case <-time.After(2 * time.Second): t.Fatal("Test timed out") } } } // winSvcMock mocks part of the golang.org/x/sys/windows/svc // execution stack, listening to svc.Status on its chan. type winSvcMock struct { status chan svc.Status expectedSt []svc.State } // Expect allows to prepare a winSvcMock to receive a specific type of StatusMessage func (w *winSvcMock) Expect(st svc.State) { w.expectedSt = append(w.expectedSt, st) } // Listen is the mock's mainloop, expects messages to comply with previous Expect(). func (w *winSvcMock) Listen(dur time.Duration, t *testing.T) error { t.Helper() timeout := time.NewTimer(dur) defer timeout.Stop() for idx, state := range w.expectedSt { select { case status := <-w.status: if status.State == state { t.Logf("message %d on status, OK\n", idx) continue } else { return fmt.Errorf("message to winsock: expected %v, got %v", state, status.State) } case <-timeout.C: return errors.New("Mock timed out") } } select { case <-timeout.C: return nil case st := <-w.status: return fmt.Errorf("extra message to winsock: got %v", st) } } nats-server-2.10.27/server/signal.go000066400000000000000000000121451477524627100172700ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !wasm // +build !windows,!wasm package server import ( "errors" "fmt" "os" "os/exec" "os/signal" "strconv" "strings" "syscall" ) var processName = "nats-server" // SetProcessName allows to change the expected name of the process. func SetProcessName(name string) { processName = name } // Signal Handling func (s *Server) handleSignals() { if s.getOpts().NoSigs { return } c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP) go func() { for { select { case sig := <-c: s.Debugf("Trapped %q signal", sig) switch sig { case syscall.SIGINT: s.Shutdown() s.WaitForShutdown() os.Exit(0) case syscall.SIGTERM: // Shutdown unless graceful shutdown already in progress. s.mu.Lock() ldm := s.ldm s.mu.Unlock() if !ldm { s.Shutdown() s.WaitForShutdown() os.Exit(1) } case syscall.SIGUSR1: // File log re-open for rotating file logs. s.ReOpenLogFile() case syscall.SIGUSR2: go s.lameDuckMode() case syscall.SIGHUP: // Config reload. if err := s.Reload(); err != nil { s.Errorf("Failed to reload server configuration: %s", err) } } case <-s.quitCh: return } } }() } // ProcessSignal sends the given signal command to the given process. If pidStr // is empty, this will send the signal to the single running instance of // nats-server. If multiple instances are running, pidStr can be a globular // expression ending with '*'. This returns an error if the given process is // not running or the command is invalid. func ProcessSignal(command Command, pidExpr string) error { var ( err error errStr string pids = make([]int, 1) pidStr = strings.TrimSuffix(pidExpr, "*") isGlob = strings.HasSuffix(pidExpr, "*") ) // Validate input if given if pidStr != "" { if pids[0], err = strconv.Atoi(pidStr); err != nil { return fmt.Errorf("invalid pid: %s", pidStr) } } // Gather all PIDs unless the input is specific if pidStr == "" || isGlob { if pids, err = resolvePids(); err != nil { return err } } // Multiple instances are running and the input is not an expression if len(pids) > 1 && !isGlob { errStr = fmt.Sprintf("multiple %s processes running:", processName) for _, p := range pids { errStr += fmt.Sprintf("\n%d", p) } return errors.New(errStr) } // No instances are running if len(pids) == 0 { return fmt.Errorf("no %s processes running", processName) } var signum syscall.Signal if signum, err = CommandToSignal(command); err != nil { return err } for _, pid := range pids { if _pidStr := strconv.Itoa(pid); _pidStr != pidStr && pidStr != "" { if !isGlob || !strings.HasPrefix(_pidStr, pidStr) { continue } } if err = kill(pid, signum); err != nil { errStr += fmt.Sprintf("\nsignal %q %d: %s", command, pid, err) } } if errStr != "" { return errors.New(errStr) } return nil } // Translates a command to a signal number func CommandToSignal(command Command) (syscall.Signal, error) { switch command { case CommandStop: return syscall.SIGKILL, nil case CommandQuit: return syscall.SIGINT, nil case CommandReopen: return syscall.SIGUSR1, nil case CommandReload: return syscall.SIGHUP, nil case commandLDMode: return syscall.SIGUSR2, nil case commandTerm: return syscall.SIGTERM, nil default: return 0, fmt.Errorf("unknown signal %q", command) } } // resolvePids returns the pids for all running nats-server processes. func resolvePids() ([]int, error) { // If pgrep isn't available, this will just bail out and the user will be // required to specify a pid. output, err := pgrep() if err != nil { switch err.(type) { case *exec.ExitError: // ExitError indicates non-zero exit code, meaning no processes // found. break default: return nil, errors.New("unable to resolve pid, try providing one") } } var ( myPid = os.Getpid() pidStrs = strings.Split(string(output), "\n") pids = make([]int, 0, len(pidStrs)) ) for _, pidStr := range pidStrs { if pidStr == "" { continue } pid, err := strconv.Atoi(pidStr) if err != nil { return nil, errors.New("unable to resolve pid, try providing one") } // Ignore the current process. if pid == myPid { continue } pids = append(pids, pid) } return pids, nil } var kill = func(pid int, signal syscall.Signal) error { return syscall.Kill(pid, signal) } var pgrep = func() ([]byte, error) { return exec.Command("pgrep", processName).Output() } nats-server-2.10.27/server/signal_test.go000066400000000000000000000266031477524627100203330ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows // +build !windows package server import ( "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "syscall" "testing" "time" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats.go" ) func TestSignalToReOpenLogFile(t *testing.T) { logFile := filepath.Join(t.TempDir(), "test.log") opts := &Options{ Host: "127.0.0.1", Port: -1, NoSigs: false, LogFile: logFile, } s := RunServer(opts) defer s.SetLogger(nil, false, false) defer s.Shutdown() // Set the file log fileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC)) s.SetLogger(fileLog, false, false) // Add a trace expectedStr := "This is a Notice" s.Noticef(expectedStr) buf, err := os.ReadFile(logFile) if err != nil { t.Fatalf("Error reading file: %v", err) } if !strings.Contains(string(buf), expectedStr) { t.Fatalf("Expected log to contain %q, got %q", expectedStr, string(buf)) } // Rename the file if err := os.Rename(logFile, logFile+".bak"); err != nil { t.Fatalf("Unable to rename file: %v", err) } // This should cause file to be reopened. syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) // Wait a bit for action to be performed time.Sleep(500 * time.Millisecond) buf, err = os.ReadFile(logFile) if err != nil { t.Fatalf("Error reading file: %v", err) } expectedStr = "File log re-opened" if !strings.Contains(string(buf), expectedStr) { t.Fatalf("Expected log to contain %q, got %q", expectedStr, string(buf)) } } func TestSignalToReloadConfig(t *testing.T) { opts, err := ProcessConfigFile("./configs/reload/basic.conf") if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoLog = true s := RunServer(opts) defer s.Shutdown() // Repeat test to make sure that server services signals more than once... for i := 0; i < 2; i++ { loaded := s.ConfigTime() // Wait a bit to ensure ConfigTime changes. time.Sleep(5 * time.Millisecond) // This should cause config to be reloaded. syscall.Kill(syscall.Getpid(), syscall.SIGHUP) // Wait a bit for action to be performed time.Sleep(500 * time.Millisecond) if reloaded := s.ConfigTime(); !reloaded.After(loaded) { t.Fatalf("ConfigTime is incorrect.\nexpected greater than: %s\ngot: %s", loaded, reloaded) } } } func TestProcessSignalNoProcesses(t *testing.T) { pgrepBefore := pgrep pgrep = func() ([]byte, error) { return nil, &exec.ExitError{} } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "") if err == nil { t.Fatal("Expected error") } expectedStr := "no nats-server processes running" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalMultipleProcesses(t *testing.T) { pid := os.Getpid() pgrepBefore := pgrep pgrep = func() ([]byte, error) { return []byte(fmt.Sprintf("123\n456\n%d\n", pid)), nil } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "") if err == nil { t.Fatal("Expected error") } expectedStr := "multiple nats-server processes running:\n123\n456" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalMultipleProcessesGlob(t *testing.T) { pid := os.Getpid() pgrepBefore := pgrep pgrep = func() ([]byte, error) { return []byte(fmt.Sprintf("123\n456\n%d\n", pid)), nil } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "*") if err == nil { t.Fatal("Expected error") } expectedStr := "\nsignal \"stop\" 123: no such process" expectedStr += "\nsignal \"stop\" 456: no such process" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalMultipleProcessesGlobPartial(t *testing.T) { pid := os.Getpid() pgrepBefore := pgrep pgrep = func() ([]byte, error) { return []byte(fmt.Sprintf("123\n124\n456\n%d\n", pid)), nil } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "12*") if err == nil { t.Fatal("Expected error") } expectedStr := "\nsignal \"stop\" 123: no such process" expectedStr += "\nsignal \"stop\" 124: no such process" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalPgrepError(t *testing.T) { pgrepBefore := pgrep pgrep = func() ([]byte, error) { return nil, errors.New("error") } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "") if err == nil { t.Fatal("Expected error") } expectedStr := "unable to resolve pid, try providing one" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalPgrepMangled(t *testing.T) { pgrepBefore := pgrep pgrep = func() ([]byte, error) { return []byte("12x"), nil } defer func() { pgrep = pgrepBefore }() err := ProcessSignal(CommandStop, "") if err == nil { t.Fatal("Expected error") } expectedStr := "unable to resolve pid, try providing one" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalResolveSingleProcess(t *testing.T) { pid := os.Getpid() pgrepBefore := pgrep pgrep = func() ([]byte, error) { return []byte(fmt.Sprintf("123\n%d\n", pid)), nil } defer func() { pgrep = pgrepBefore }() killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGKILL { t.Fatalf("signal is incorrect.\nexpected: killed\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(CommandStop, ""); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalInvalidCommand(t *testing.T) { err := ProcessSignal(Command("invalid"), "123") if err == nil { t.Fatal("Expected error") } expectedStr := "unknown signal \"invalid\"" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalInvalidPid(t *testing.T) { err := ProcessSignal(CommandStop, "abc") if err == nil { t.Fatal("Expected error") } expectedStr := "invalid pid: abc" if err.Error() != expectedStr { t.Fatalf("Error is incorrect.\nexpected: %s\ngot: %s", expectedStr, err.Error()) } } func TestProcessSignalQuitProcess(t *testing.T) { killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGINT { t.Fatalf("signal is incorrect.\nexpected: interrupt\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(CommandQuit, "123"); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalTermProcess(t *testing.T) { killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGTERM { t.Fatalf("signal is incorrect.\nexpected: interrupt\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(commandTerm, "123"); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalReopenProcess(t *testing.T) { killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGUSR1 { t.Fatalf("signal is incorrect.\nexpected: user defined signal 1\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(CommandReopen, "123"); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalReloadProcess(t *testing.T) { killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGHUP { t.Fatalf("signal is incorrect.\nexpected: hangup\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(CommandReload, "123"); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalLameDuckMode(t *testing.T) { killBefore := kill called := false kill = func(pid int, signal syscall.Signal) error { called = true if pid != 123 { t.Fatalf("pid is incorrect.\nexpected: 123\ngot: %d", pid) } if signal != syscall.SIGUSR2 { t.Fatalf("signal is incorrect.\nexpected: sigusr2\ngot: %v", signal) } return nil } defer func() { kill = killBefore }() if err := ProcessSignal(commandLDMode, "123"); err != nil { t.Fatalf("ProcessSignal failed: %v", err) } if !called { t.Fatal("Expected kill to be called") } } func TestProcessSignalTermDuringLameDuckMode(t *testing.T) { opts := &Options{ Host: "127.0.0.1", Port: -1, NoSigs: false, NoLog: true, LameDuckDuration: 2 * time.Second, LameDuckGracePeriod: 1 * time.Second, } s := RunServer(opts) defer s.Shutdown() // Create single NATS Connection which will cause the server // to delay the shutdown. doneCh := make(chan struct{}) nc, err := nats.Connect(fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port), nats.DisconnectHandler(func(*nats.Conn) { close(doneCh) }), ) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Trigger lame duck based shutdown. go s.lameDuckMode() // Wait for client to be disconnected. select { case <-doneCh: break case <-time.After(3 * time.Second): t.Fatalf("Timed out waiting for client to disconnect") } // Termination signal should not cause server to shutdown // while in lame duck mode already. syscall.Kill(syscall.Getpid(), syscall.SIGTERM) // Wait for server shutdown due to lame duck shutdown. timeoutCh := make(chan error) timer := time.AfterFunc(3*time.Second, func() { timeoutCh <- errors.New("Timed out waiting for server shutdown") }) for range time.NewTicker(1 * time.Millisecond).C { select { case err := <-timeoutCh: t.Fatal(err) default: } if !s.isRunning() { timer.Stop() break } } } nats-server-2.10.27/server/signal_wasm.go000066400000000000000000000013541477524627100203170ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build wasm // +build wasm package server func (s *Server) handleSignals() { } func ProcessSignal(command Command, service string) error { return nil } nats-server-2.10.27/server/signal_windows.go000066400000000000000000000046471477524627100210520ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "os" "os/signal" "syscall" "time" "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/mgr" ) // Signal Handling func (s *Server) handleSignals() { if s.getOpts().NoSigs { return } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { for { select { case sig := <-c: s.Debugf("Trapped %q signal", sig) s.Shutdown() os.Exit(0) case <-s.quitCh: return } } }() } // ProcessSignal sends the given signal command to the running nats-server service. // If service is empty, this signals the "nats-server" service. This returns an // error is the given service is not running or the command is invalid. func ProcessSignal(command Command, service string) error { if service == "" { service = serviceName } m, err := mgr.Connect() if err != nil { return err } defer m.Disconnect() s, err := m.OpenService(service) if err != nil { return fmt.Errorf("could not access service: %v", err) } defer s.Close() var ( cmd svc.Cmd to svc.State ) switch command { case CommandStop, CommandQuit: cmd = svc.Stop to = svc.Stopped case CommandReopen: cmd = reopenLogCmd to = svc.Running case CommandReload: cmd = svc.ParamChange to = svc.Running case commandLDMode: cmd = ldmCmd to = svc.Running default: return fmt.Errorf("unknown signal %q", command) } status, err := s.Control(cmd) if err != nil { return fmt.Errorf("could not send control=%d: %v", cmd, err) } timeout := time.Now().Add(10 * time.Second) for status.State != to { if timeout.Before(time.Now()) { return fmt.Errorf("timeout waiting for service to go to state=%d", to) } time.Sleep(300 * time.Millisecond) status, err = s.Query() if err != nil { return fmt.Errorf("could not retrieve service status: %v", err) } } return nil } nats-server-2.10.27/server/split_test.go000066400000000000000000000406031477524627100202050ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "net" "testing" ) func TestSplitBufferSubOp(t *testing.T) { cli, trash := net.Pipe() defer cli.Close() defer trash.Close() s := &Server{gacc: NewAccount(globalAccountName)} if err := s.newGateway(DefaultOptions()); err != nil { t.Fatalf("Error creating gateways: %v", err) } s.registerAccount(s.gacc) c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), nc: cli} subop := []byte("SUB foo 1\r\n") subop1 := subop[:6] subop2 := subop[6:] if err := c.parse(subop1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != SUB_ARG { t.Fatalf("Expected SUB_ARG state vs %d\n", c.state) } if err := c.parse(subop2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } r := s.gacc.sl.Match("foo") if r == nil || len(r.psubs) != 1 { t.Fatalf("Did not match subscription properly: %+v\n", r) } sub := r.psubs[0] if !bytes.Equal(sub.subject, []byte("foo")) { t.Fatalf("Subject did not match expected 'foo' : '%s'\n", sub.subject) } if !bytes.Equal(sub.sid, []byte("1")) { t.Fatalf("Sid did not match expected '1' : '%s'\n", sub.sid) } if sub.queue != nil { t.Fatalf("Received a non-nil queue: '%s'\n", sub.queue) } } func TestSplitBufferUnsubOp(t *testing.T) { s := &Server{gacc: NewAccount(globalAccountName), gateway: &srvGateway{}} s.registerAccount(s.gacc) c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} subop := []byte("SUB foo 1024\r\n") if err := c.parse(subop); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } unsubop := []byte("UNSUB 1024\r\n") unsubop1 := unsubop[:8] unsubop2 := unsubop[8:] if err := c.parse(unsubop1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != UNSUB_ARG { t.Fatalf("Expected UNSUB_ARG state vs %d\n", c.state) } if err := c.parse(unsubop2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } r := s.gacc.sl.Match("foo") if r != nil && len(r.psubs) != 0 { t.Fatalf("Should be no subscriptions in results: %+v\n", r) } } func TestSplitBufferPubOp(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r") pub1 := pub[:2] pub2 := pub[2:9] pub3 := pub[9:15] pub4 := pub[15:22] pub5 := pub[22:25] pub6 := pub[25:33] pub7 := pub[33:] if err := c.parse(pub1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_PU { t.Fatalf("Expected OP_PU state vs %d\n", c.state) } if err := c.parse(pub2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != PUB_ARG { t.Fatalf("Expected OP_PU state vs %d\n", c.state) } if err := c.parse(pub3); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != PUB_ARG { t.Fatalf("Expected OP_PU state vs %d\n", c.state) } if err := c.parse(pub4); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != PUB_ARG { t.Fatalf("Expected PUB_ARG state vs %d\n", c.state) } if err := c.parse(pub5); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } // Check c.pa if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("PUB arg subject incorrect: '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("INBOX.22")) { t.Fatalf("PUB arg reply subject incorrect: '%s'\n", c.pa.reply) } if c.pa.size != 11 { t.Fatalf("PUB arg msg size incorrect: %d\n", c.pa.size) } if err := c.parse(pub6); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } if err := c.parse(pub7); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_END_N { t.Fatalf("Expected MSG_END_N state vs %d\n", c.state) } } func TestSplitBufferPubOp2(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} pub := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n") pub1 := pub[:30] pub2 := pub[30:] if err := c.parse(pub1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } if err := c.parse(pub2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } } func TestSplitBufferPubOp3(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} pubAll := []byte("PUB foo bar 11\r\nhello world\r\n") pub := pubAll[:16] if err := c.parse(pub); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") } // Simulate next read of network, make sure pub state is saved // until msg payload has cleared. copy(pubAll, "XXXXXXXXXXXXXXXX") if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") } if !bytes.Equal(c.pa.reply, []byte("bar")) { t.Fatalf("Unexpected reply: '%s' vs '%s'\n", c.pa.reply, "bar") } if !bytes.Equal(c.pa.szb, []byte("11")) { t.Fatalf("Unexpected size bytes: '%s' vs '%s'\n", c.pa.szb, "11") } } func TestSplitBufferPubOp4(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} pubAll := []byte("PUB foo 11\r\nhello world\r\n") pub := pubAll[:12] if err := c.parse(pub); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") } // Simulate next read of network, make sure pub state is saved // until msg payload has cleared. copy(pubAll, "XXXXXXXXXXXX") if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Unexpected subject: '%s' vs '%s'\n", c.pa.subject, "foo") } if !bytes.Equal(c.pa.reply, []byte("")) { t.Fatalf("Unexpected reply: '%s' vs '%s'\n", c.pa.reply, "") } if !bytes.Equal(c.pa.szb, []byte("11")) { t.Fatalf("Unexpected size bytes: '%s' vs '%s'\n", c.pa.szb, "11") } } func TestSplitBufferPubOp5(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} pubAll := []byte("PUB foo 11\r\nhello world\r\n") // Splits need to be on MSG_END_R now too, so make sure we check that. // Split between \r and \n pub := pubAll[:len(pubAll)-1] if err := c.parse(pub); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.msgBuf == nil { t.Fatalf("msgBuf should not be nil!\n") } if !bytes.Equal(c.msgBuf, []byte("hello world\r")) { t.Fatalf("c.msgBuf did not snaphot the msg") } } func TestSplitConnectArg(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} connectAll := []byte("CONNECT {\"verbose\":false,\"tls_required\":false," + "\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n") argJSON := connectAll[8:] c1 := connectAll[:5] c2 := connectAll[5:22] c3 := connectAll[22 : len(connectAll)-2] c4 := connectAll[len(connectAll)-2:] if err := c.parse(c1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.argBuf != nil { t.Fatalf("Unexpected argBug placeholder.\n") } if err := c.parse(c2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.argBuf == nil { t.Fatalf("Expected argBug to not be nil.\n") } if !bytes.Equal(c.argBuf, argJSON[:14]) { t.Fatalf("argBuf not correct, received %q, wanted %q\n", argJSON[:14], c.argBuf) } if err := c.parse(c3); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.argBuf == nil { t.Fatalf("Expected argBug to not be nil.\n") } if !bytes.Equal(c.argBuf, argJSON[:len(argJSON)-2]) { t.Fatalf("argBuf not correct, received %q, wanted %q\n", argJSON[:len(argJSON)-2], c.argBuf) } if err := c.parse(c4); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.argBuf != nil { t.Fatalf("Unexpected argBuf placeholder.\n") } } func TestSplitDanglingArgBuf(t *testing.T) { s := New(&defaultServerOptions) c := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)} // We test to make sure we do not dangle any argBufs after processing // since that could lead to performance issues. // SUB subop := []byte("SUB foo 1\r\n") c.parse(subop[:6]) c.parse(subop[6:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // UNSUB unsubop := []byte("UNSUB 1024\r\n") c.parse(unsubop[:8]) c.parse(unsubop[8:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // PUB pubop := []byte("PUB foo.bar INBOX.22 11\r\nhello world\r\n") c.parse(pubop[:22]) c.parse(pubop[22:25]) if c.argBuf == nil { t.Fatal("Expected a non-nil argBuf!") } c.parse(pubop[25:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // MINUS_ERR errop := []byte("-ERR Too Long\r\n") c.parse(errop[:8]) c.parse(errop[8:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // CONNECT_ARG connop := []byte("CONNECT {\"verbose\":false,\"tls_required\":false," + "\"user\":\"test\",\"pedantic\":true,\"pass\":\"pass\"}\r\n") c.parse(connop[:22]) c.parse(connop[22:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // INFO_ARG infoop := []byte("INFO {\"server_id\":\"id\"}\r\n") c.parse(infoop[:8]) c.parse(infoop[8:]) if c.argBuf != nil { t.Fatalf("Expected c.argBuf to be nil: %q\n", c.argBuf) } // MSG (the client has to be a ROUTE) c = &client{msubs: -1, mpay: -1, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}} msgop := []byte("RMSG $foo foo 5\r\nhello\r\n") c.parse(msgop[:5]) c.parse(msgop[5:10]) if c.argBuf == nil { t.Fatal("Expected a non-nil argBuf") } if string(c.argBuf) != "$foo " { t.Fatalf("Expected argBuf to be \"$foo \", got %q", string(c.argBuf)) } c.parse(msgop[10:]) if c.argBuf != nil { t.Fatalf("Expected argBuf to be nil: %q", c.argBuf) } if c.msgBuf != nil { t.Fatalf("Expected msgBuf to be nil: %q", c.msgBuf) } c.state = OP_START // Parse up-to somewhere in the middle of the payload. // Verify that we have saved the MSG_ARG info c.parse(msgop[:23]) if c.argBuf == nil { t.Fatal("Expected a non-nil argBuf") } if string(c.pa.account) != "$foo" { t.Fatalf("Expected account to be \"$foo\", got %q", c.pa.account) } if string(c.pa.subject) != "foo" { t.Fatalf("Expected subject to be \"foo\", got %q", c.pa.subject) } if string(c.pa.reply) != "" { t.Fatalf("Expected reply to be \"\", got %q", c.pa.reply) } if c.pa.size != 5 { t.Fatalf("Expected sid to 5, got %v", c.pa.size) } // msg buffer should be if c.msgBuf == nil || string(c.msgBuf) != "hello\r" { t.Fatalf("Expected msgBuf to be \"hello\r\", got %q", c.msgBuf) } c.parse(msgop[23:]) // At the end, we should have cleaned-up both arg and msg buffers. if c.argBuf != nil { t.Fatalf("Expected argBuf to be nil: %q", c.argBuf) } if c.msgBuf != nil { t.Fatalf("Expected msgBuf to be nil: %q", c.msgBuf) } } func TestSplitRoutedMsgArg(t *testing.T) { _, c, _ := setupClient() defer c.close() // Allow parser to process RMSG c.kind = ROUTER c.route = &route{} b := make([]byte, 1024) copy(b, []byte("RMSG $G hello.world 6040\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")) c.parse(b) copy(b, []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\r\n")) c.parse(b) wantAccount := "$G" wantSubject := "hello.world" wantSzb := "6040" if string(c.pa.account) != wantAccount { t.Fatalf("Incorrect account: want %q, got %q", wantAccount, c.pa.account) } if string(c.pa.subject) != wantSubject { t.Fatalf("Incorrect subject: want %q, got %q", wantSubject, c.pa.subject) } if string(c.pa.szb) != wantSzb { t.Fatalf("Incorrect szb: want %q, got %q", wantSzb, c.pa.szb) } } func TestSplitBufferMsgOp(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}} msg := []byte("RMSG $G foo.bar _INBOX.22 11\r\nhello world\r") msg1 := msg[:2] msg2 := msg[2:9] msg3 := msg[9:15] msg4 := msg[15:22] msg5 := msg[22:25] msg6 := msg[25:37] msg7 := msg[37:40] msg8 := msg[40:] if err := c.parse(msg1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_M { t.Fatalf("Expected OP_M state vs %d\n", c.state) } if err := c.parse(msg2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_ARG { t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) } if err := c.parse(msg3); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_ARG { t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) } if err := c.parse(msg4); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_ARG { t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) } if err := c.parse(msg5); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_ARG { t.Fatalf("Expected MSG_ARG state vs %d\n", c.state) } if err := c.parse(msg6); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } // Check c.pa if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("MSG arg subject incorrect: '%s'\n", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("_INBOX.22")) { t.Fatalf("MSG arg reply subject incorrect: '%s'\n", c.pa.reply) } if c.pa.size != 11 { t.Fatalf("MSG arg msg size incorrect: %d\n", c.pa.size) } if err := c.parse(msg7); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } if err := c.parse(msg8); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_END_N { t.Fatalf("Expected MSG_END_N state vs %d\n", c.state) } } func TestSplitBufferLeafMsgArg(t *testing.T) { c := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: LEAF} msg := []byte("LMSG foo + bar baz 11\r\n") if err := c.parse(msg); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_PAYLOAD { t.Fatalf("Expected MSG_PAYLOAD state vs %d\n", c.state) } checkPA := func(t *testing.T) { t.Helper() if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Expected subject to be %q, got %q", "foo", c.pa.subject) } if !bytes.Equal(c.pa.reply, []byte("bar")) { t.Fatalf("Expected reply to be %q, got %q", "bar", c.pa.reply) } if n := len(c.pa.queues); n != 1 { t.Fatalf("Expected 1 queue, got %v", n) } if !bytes.Equal(c.pa.queues[0], []byte("baz")) { t.Fatalf("Expected queues to be %q, got %q", "baz", c.pa.queues) } } checkPA(t) // overwrite msg with payload n := copy(msg, []byte("fffffffffff")) if err := c.parse(msg[:n]); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != MSG_END_R { t.Fatalf("Expected MSG_END_R state vs %d\n", c.state) } checkPA(t) // Finish processing copy(msg, []byte("\r\n")) if err := c.parse(msg[:2]); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } if c.pa.subject != nil || c.pa.reply != nil || c.pa.queues != nil || c.pa.size != 0 || c.pa.szb != nil || c.pa.arg != nil { t.Fatalf("parser state not cleaned-up properly: %+v", c.pa) } } nats-server-2.10.27/server/store.go000066400000000000000000000550571477524627100171600ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "encoding/binary" "errors" "fmt" "io" "os" "strings" "time" "unsafe" "github.com/nats-io/nats-server/v2/server/avl" ) // StorageType determines how messages are stored for retention. type StorageType int const ( // File specifies on disk, designated by the JetStream config StoreDir. FileStorage = StorageType(22) // MemoryStorage specifies in memory only. MemoryStorage = StorageType(33) // Any is for internals. AnyStorage = StorageType(44) ) var ( // ErrStoreClosed is returned when the store has been closed ErrStoreClosed = errors.New("store is closed") // ErrStoreMsgNotFound when message was not found but was expected to be. ErrStoreMsgNotFound = errors.New("no message found") // ErrStoreEOF is returned when message seq is greater than the last sequence. ErrStoreEOF = errors.New("stream store EOF") // ErrMaxMsgs is returned when we have discard new as a policy and we reached the message limit. ErrMaxMsgs = errors.New("maximum messages exceeded") // ErrMaxBytes is returned when we have discard new as a policy and we reached the bytes limit. ErrMaxBytes = errors.New("maximum bytes exceeded") // ErrMaxMsgsPerSubject is returned when we have discard new as a policy and we reached the message limit per subject. ErrMaxMsgsPerSubject = errors.New("maximum messages per subject exceeded") // ErrStoreSnapshotInProgress is returned when RemoveMsg or EraseMsg is called // while a snapshot is in progress. ErrStoreSnapshotInProgress = errors.New("snapshot in progress") // ErrMsgTooLarge is returned when a message is considered too large. ErrMsgTooLarge = errors.New("message to large") // ErrStoreWrongType is for when you access the wrong storage type. ErrStoreWrongType = errors.New("wrong storage type") // ErrNoAckPolicy is returned when trying to update a consumer's acks with no ack policy. ErrNoAckPolicy = errors.New("ack policy is none") // ErrInvalidSequence is returned when the sequence is not present in the stream store. ErrInvalidSequence = errors.New("invalid sequence") // ErrSequenceMismatch is returned when storing a raw message and the expected sequence is wrong. ErrSequenceMismatch = errors.New("expected sequence does not match store") // ErrCorruptStreamState ErrCorruptStreamState = errors.New("stream state snapshot is corrupt") ) // StoreMsg is the stored message format for messages that are retained by the Store layer. type StoreMsg struct { subj string hdr []byte msg []byte buf []byte seq uint64 ts int64 } // Used to call back into the upper layers to report on changes in storage resources. // For the cases where its a single message we will also supply sequence number and subject. type StorageUpdateHandler func(msgs, bytes int64, seq uint64, subj string) type StreamStore interface { StoreMsg(subject string, hdr, msg []byte) (uint64, int64, error) StoreRawMsg(subject string, hdr, msg []byte, seq uint64, ts int64) error SkipMsg() uint64 SkipMsgs(seq uint64, num uint64) error LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) LoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) LoadNextMsgMulti(sl *Sublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) LoadLastMsg(subject string, sm *StoreMsg) (*StoreMsg, error) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error) RemoveMsg(seq uint64) (bool, error) EraseMsg(seq uint64) (bool, error) Purge() (uint64, error) PurgeEx(subject string, seq, keep uint64) (uint64, error) Compact(seq uint64) (uint64, error) Truncate(seq uint64) error GetSeqFromTime(t time.Time) uint64 FilteredState(seq uint64, subject string) SimpleState SubjectsState(filterSubject string) map[string]SimpleState SubjectsTotals(filterSubject string) map[string]uint64 NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64) NumPendingMulti(sseq uint64, sl *Sublist, lastPerSubject bool) (total, validThrough uint64) State() StreamState FastState(*StreamState) EncodedStreamState(failed uint64) (enc []byte, err error) SyncDeleted(dbs DeleteBlocks) Type() StorageType RegisterStorageUpdates(StorageUpdateHandler) UpdateConfig(cfg *StreamConfig) error Delete() error Stop() error ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerStore, error) AddConsumer(o ConsumerStore) error RemoveConsumer(o ConsumerStore) error Snapshot(deadline time.Duration, includeConsumers, checkMsgs bool) (*SnapshotResult, error) Utilization() (total, reported uint64, err error) } // RetentionPolicy determines how messages in a set are retained. type RetentionPolicy int const ( // LimitsPolicy (default) means that messages are retained until any given limit is reached. // This could be one of MaxMsgs, MaxBytes, or MaxAge. LimitsPolicy RetentionPolicy = iota // InterestPolicy specifies that when all known consumers have acknowledged a message it can be removed. InterestPolicy // WorkQueuePolicy specifies that when the first worker or subscriber acknowledges the message it can be removed. WorkQueuePolicy ) // Discard Policy determines how we proceed when limits of messages or bytes are hit. The default, DicscardOld will // remove older messages. DiscardNew will fail to store the new message. type DiscardPolicy int const ( // DiscardOld will remove older messages to return to the limits. DiscardOld = iota // DiscardNew will error on a StoreMsg call DiscardNew ) // StreamState is information about the given stream. type StreamState struct { Msgs uint64 `json:"messages"` Bytes uint64 `json:"bytes"` FirstSeq uint64 `json:"first_seq"` FirstTime time.Time `json:"first_ts"` LastSeq uint64 `json:"last_seq"` LastTime time.Time `json:"last_ts"` NumSubjects int `json:"num_subjects,omitempty"` Subjects map[string]uint64 `json:"subjects,omitempty"` NumDeleted int `json:"num_deleted,omitempty"` Deleted []uint64 `json:"deleted,omitempty"` Lost *LostStreamData `json:"lost,omitempty"` Consumers int `json:"consumer_count"` } // SimpleState for filtered subject specific state. type SimpleState struct { Msgs uint64 `json:"messages"` First uint64 `json:"first_seq"` Last uint64 `json:"last_seq"` // Internal usage for when the first needs to be updated before use. firstNeedsUpdate bool // Internal usage for when the last needs to be updated before use. lastNeedsUpdate bool } // LostStreamData indicates msgs that have been lost. type LostStreamData struct { Msgs []uint64 `json:"msgs"` Bytes uint64 `json:"bytes"` } // SnapshotResult contains information about the snapshot. type SnapshotResult struct { Reader io.ReadCloser State StreamState } const ( // Magic is used to identify stream state encodings. streamStateMagic = uint8(42) // Version streamStateVersion = uint8(1) // Magic / Identifier for run length encodings. runLengthMagic = uint8(33) // Magic / Identifier for AVL seqsets. seqSetMagic = uint8(22) ) // Interface for DeleteBlock. // These will be of three types: // 1. AVL seqsets. // 2. Run length encoding of a deleted range. // 3. Legacy []uint64 type DeleteBlock interface { State() (first, last, num uint64) Range(f func(uint64) bool) } type DeleteBlocks []DeleteBlock // StreamReplicatedState represents what is encoded in a binary stream snapshot used // for stream replication in an NRG. type StreamReplicatedState struct { Msgs uint64 Bytes uint64 FirstSeq uint64 LastSeq uint64 Failed uint64 Deleted DeleteBlocks } // Determine if this is an encoded stream state. func IsEncodedStreamState(buf []byte) bool { return len(buf) >= hdrLen && buf[0] == streamStateMagic && buf[1] == streamStateVersion } var ErrBadStreamStateEncoding = errors.New("bad stream state encoding") func DecodeStreamState(buf []byte) (*StreamReplicatedState, error) { ss := &StreamReplicatedState{} if len(buf) < hdrLen || buf[0] != streamStateMagic || buf[1] != streamStateVersion { return nil, ErrBadStreamStateEncoding } var bi = hdrLen readU64 := func() uint64 { if bi < 0 || bi >= len(buf) { bi = -1 return 0 } num, n := binary.Uvarint(buf[bi:]) if n <= 0 { bi = -1 return 0 } bi += n return num } parserFailed := func() bool { return bi < 0 } ss.Msgs = readU64() ss.Bytes = readU64() ss.FirstSeq = readU64() ss.LastSeq = readU64() ss.Failed = readU64() if parserFailed() { return nil, ErrCorruptStreamState } if numDeleted := readU64(); numDeleted > 0 { // If we have some deleted blocks. for l := len(buf); l > bi; { switch buf[bi] { case seqSetMagic: dmap, n, err := avl.Decode(buf[bi:]) if err != nil { return nil, ErrCorruptStreamState } bi += n ss.Deleted = append(ss.Deleted, dmap) case runLengthMagic: bi++ var rl DeleteRange rl.First = readU64() rl.Num = readU64() if parserFailed() { return nil, ErrCorruptStreamState } ss.Deleted = append(ss.Deleted, &rl) default: return nil, ErrCorruptStreamState } } } return ss, nil } // DeleteRange is a run length encoded delete range. type DeleteRange struct { First uint64 Num uint64 } func (dr *DeleteRange) State() (first, last, num uint64) { deletesAfterFirst := dr.Num if deletesAfterFirst > 0 { deletesAfterFirst-- } return dr.First, dr.First + deletesAfterFirst, dr.Num } // Range will range over all the deleted sequences represented by this block. func (dr *DeleteRange) Range(f func(uint64) bool) { for seq := dr.First; seq < dr.First+dr.Num; seq++ { if !f(seq) { return } } } // Legacy []uint64 type DeleteSlice []uint64 func (ds DeleteSlice) State() (first, last, num uint64) { if len(ds) == 0 { return 0, 0, 0 } return ds[0], ds[len(ds)-1], uint64(len(ds)) } // Range will range over all the deleted sequences represented by this []uint64. func (ds DeleteSlice) Range(f func(uint64) bool) { for _, seq := range ds { if !f(seq) { return } } } func (dbs DeleteBlocks) NumDeleted() (total uint64) { for _, db := range dbs { _, _, num := db.State() total += num } return total } // ConsumerStore stores state on consumers for streams. type ConsumerStore interface { SetStarting(sseq uint64) error HasState() bool UpdateDelivered(dseq, sseq, dc uint64, ts int64) error UpdateAcks(dseq, sseq uint64) error UpdateConfig(cfg *ConsumerConfig) error Update(*ConsumerState) error State() (*ConsumerState, error) BorrowState() (*ConsumerState, error) EncodedState() ([]byte, error) Type() StorageType Stop() error Delete() error StreamDelete() error } // SequencePair has both the consumer and the stream sequence. They point to same message. type SequencePair struct { Consumer uint64 `json:"consumer_seq"` Stream uint64 `json:"stream_seq"` } // ConsumerState represents a stored state for a consumer. type ConsumerState struct { // Delivered keeps track of last delivered sequence numbers for both the stream and the consumer. Delivered SequencePair `json:"delivered"` // AckFloor keeps track of the ack floors for both the stream and the consumer. AckFloor SequencePair `json:"ack_floor"` // These are both in stream sequence context. // Pending is for all messages pending and the timestamp for the delivered time. // This will only be present when the AckPolicy is ExplicitAck. Pending map[uint64]*Pending `json:"pending,omitempty"` // This is for messages that have been redelivered, so count > 1. Redelivered map[uint64]uint64 `json:"redelivered,omitempty"` } // Encode consumer state. func encodeConsumerState(state *ConsumerState) []byte { var hdr [seqsHdrSize]byte var buf []byte maxSize := seqsHdrSize if lp := len(state.Pending); lp > 0 { maxSize += lp*(3*binary.MaxVarintLen64) + binary.MaxVarintLen64 } if lr := len(state.Redelivered); lr > 0 { maxSize += lr*(2*binary.MaxVarintLen64) + binary.MaxVarintLen64 } if maxSize == seqsHdrSize { buf = hdr[:seqsHdrSize] } else { buf = make([]byte, maxSize) } // Write header buf[0] = magic buf[1] = 2 n := hdrLen n += binary.PutUvarint(buf[n:], state.AckFloor.Consumer) n += binary.PutUvarint(buf[n:], state.AckFloor.Stream) n += binary.PutUvarint(buf[n:], state.Delivered.Consumer) n += binary.PutUvarint(buf[n:], state.Delivered.Stream) n += binary.PutUvarint(buf[n:], uint64(len(state.Pending))) asflr := state.AckFloor.Stream adflr := state.AckFloor.Consumer // These are optional, but always write len. This is to avoid a truncate inline. if len(state.Pending) > 0 { // To save space we will use now rounded to seconds to be our base timestamp. mints := time.Now().Round(time.Second).Unix() // Write minimum timestamp we found from above. n += binary.PutVarint(buf[n:], mints) for k, v := range state.Pending { n += binary.PutUvarint(buf[n:], k-asflr) n += binary.PutUvarint(buf[n:], v.Sequence-adflr) // Downsample to seconds to save on space. // Subsecond resolution not needed for recovery etc. ts := v.Timestamp / int64(time.Second) n += binary.PutVarint(buf[n:], mints-ts) } } // We always write the redelivered len. n += binary.PutUvarint(buf[n:], uint64(len(state.Redelivered))) // We expect these to be small. if len(state.Redelivered) > 0 { for k, v := range state.Redelivered { n += binary.PutUvarint(buf[n:], k-asflr) n += binary.PutUvarint(buf[n:], v) } } return buf[:n] } // Represents a pending message for explicit ack or ack all. // Sequence is the original consumer sequence. type Pending struct { Sequence uint64 Timestamp int64 } // TemplateStore stores templates. type TemplateStore interface { Store(*streamTemplate) error Delete(*streamTemplate) error } const ( limitsPolicyJSONString = `"limits"` interestPolicyJSONString = `"interest"` workQueuePolicyJSONString = `"workqueue"` ) var ( limitsPolicyJSONBytes = []byte(limitsPolicyJSONString) interestPolicyJSONBytes = []byte(interestPolicyJSONString) workQueuePolicyJSONBytes = []byte(workQueuePolicyJSONString) ) func (rp RetentionPolicy) String() string { switch rp { case LimitsPolicy: return "Limits" case InterestPolicy: return "Interest" case WorkQueuePolicy: return "WorkQueue" default: return "Unknown Retention Policy" } } func (rp RetentionPolicy) MarshalJSON() ([]byte, error) { switch rp { case LimitsPolicy: return limitsPolicyJSONBytes, nil case InterestPolicy: return interestPolicyJSONBytes, nil case WorkQueuePolicy: return workQueuePolicyJSONBytes, nil default: return nil, fmt.Errorf("can not marshal %v", rp) } } func (rp *RetentionPolicy) UnmarshalJSON(data []byte) error { switch string(data) { case limitsPolicyJSONString: *rp = LimitsPolicy case interestPolicyJSONString: *rp = InterestPolicy case workQueuePolicyJSONString: *rp = WorkQueuePolicy default: return fmt.Errorf("can not unmarshal %q", data) } return nil } func (dp DiscardPolicy) String() string { switch dp { case DiscardOld: return "DiscardOld" case DiscardNew: return "DiscardNew" default: return "Unknown Discard Policy" } } func (dp DiscardPolicy) MarshalJSON() ([]byte, error) { switch dp { case DiscardOld: return []byte(`"old"`), nil case DiscardNew: return []byte(`"new"`), nil default: return nil, fmt.Errorf("can not marshal %v", dp) } } func (dp *DiscardPolicy) UnmarshalJSON(data []byte) error { switch strings.ToLower(string(data)) { case `"old"`: *dp = DiscardOld case `"new"`: *dp = DiscardNew default: return fmt.Errorf("can not unmarshal %q", data) } return nil } const ( memoryStorageJSONString = `"memory"` fileStorageJSONString = `"file"` anyStorageJSONString = `"any"` ) var ( memoryStorageJSONBytes = []byte(memoryStorageJSONString) fileStorageJSONBytes = []byte(fileStorageJSONString) anyStorageJSONBytes = []byte(anyStorageJSONString) ) func (st StorageType) String() string { switch st { case MemoryStorage: return "Memory" case FileStorage: return "File" case AnyStorage: return "Any" default: return "Unknown Storage Type" } } func (st StorageType) MarshalJSON() ([]byte, error) { switch st { case MemoryStorage: return memoryStorageJSONBytes, nil case FileStorage: return fileStorageJSONBytes, nil case AnyStorage: return anyStorageJSONBytes, nil default: return nil, fmt.Errorf("can not marshal %v", st) } } func (st *StorageType) UnmarshalJSON(data []byte) error { switch string(data) { case memoryStorageJSONString: *st = MemoryStorage case fileStorageJSONString: *st = FileStorage case anyStorageJSONString: *st = AnyStorage default: return fmt.Errorf("can not unmarshal %q", data) } return nil } const ( ackNonePolicyJSONString = `"none"` ackAllPolicyJSONString = `"all"` ackExplicitPolicyJSONString = `"explicit"` ) var ( ackNonePolicyJSONBytes = []byte(ackNonePolicyJSONString) ackAllPolicyJSONBytes = []byte(ackAllPolicyJSONString) ackExplicitPolicyJSONBytes = []byte(ackExplicitPolicyJSONString) ) func (ap AckPolicy) MarshalJSON() ([]byte, error) { switch ap { case AckNone: return ackNonePolicyJSONBytes, nil case AckAll: return ackAllPolicyJSONBytes, nil case AckExplicit: return ackExplicitPolicyJSONBytes, nil default: return nil, fmt.Errorf("can not marshal %v", ap) } } func (ap *AckPolicy) UnmarshalJSON(data []byte) error { switch string(data) { case ackNonePolicyJSONString: *ap = AckNone case ackAllPolicyJSONString: *ap = AckAll case ackExplicitPolicyJSONString: *ap = AckExplicit default: return fmt.Errorf("can not unmarshal %q", data) } return nil } const ( replayInstantPolicyJSONString = `"instant"` replayOriginalPolicyJSONString = `"original"` ) var ( replayInstantPolicyJSONBytes = []byte(replayInstantPolicyJSONString) replayOriginalPolicyJSONBytes = []byte(replayOriginalPolicyJSONString) ) func (rp ReplayPolicy) MarshalJSON() ([]byte, error) { switch rp { case ReplayInstant: return replayInstantPolicyJSONBytes, nil case ReplayOriginal: return replayOriginalPolicyJSONBytes, nil default: return nil, fmt.Errorf("can not marshal %v", rp) } } func (rp *ReplayPolicy) UnmarshalJSON(data []byte) error { switch string(data) { case replayInstantPolicyJSONString: *rp = ReplayInstant case replayOriginalPolicyJSONString: *rp = ReplayOriginal default: return fmt.Errorf("can not unmarshal %q", data) } return nil } const ( deliverAllPolicyJSONString = `"all"` deliverLastPolicyJSONString = `"last"` deliverNewPolicyJSONString = `"new"` deliverByStartSequenceJSONString = `"by_start_sequence"` deliverByStartTimeJSONString = `"by_start_time"` deliverLastPerPolicyJSONString = `"last_per_subject"` deliverUndefinedJSONString = `"undefined"` ) var ( deliverAllPolicyJSONBytes = []byte(deliverAllPolicyJSONString) deliverLastPolicyJSONBytes = []byte(deliverLastPolicyJSONString) deliverNewPolicyJSONBytes = []byte(deliverNewPolicyJSONString) deliverByStartSequenceJSONBytes = []byte(deliverByStartSequenceJSONString) deliverByStartTimeJSONBytes = []byte(deliverByStartTimeJSONString) deliverLastPerPolicyJSONBytes = []byte(deliverLastPerPolicyJSONString) deliverUndefinedJSONBytes = []byte(deliverUndefinedJSONString) ) func (p *DeliverPolicy) UnmarshalJSON(data []byte) error { switch string(data) { case deliverAllPolicyJSONString, deliverUndefinedJSONString: *p = DeliverAll case deliverLastPolicyJSONString: *p = DeliverLast case deliverLastPerPolicyJSONString: *p = DeliverLastPerSubject case deliverNewPolicyJSONString: *p = DeliverNew case deliverByStartSequenceJSONString: *p = DeliverByStartSequence case deliverByStartTimeJSONString: *p = DeliverByStartTime default: return fmt.Errorf("can not unmarshal %q", data) } return nil } func (p DeliverPolicy) MarshalJSON() ([]byte, error) { switch p { case DeliverAll: return deliverAllPolicyJSONBytes, nil case DeliverLast: return deliverLastPolicyJSONBytes, nil case DeliverLastPerSubject: return deliverLastPerPolicyJSONBytes, nil case DeliverNew: return deliverNewPolicyJSONBytes, nil case DeliverByStartSequence: return deliverByStartSequenceJSONBytes, nil case DeliverByStartTime: return deliverByStartTimeJSONBytes, nil default: return deliverUndefinedJSONBytes, nil } } func isOutOfSpaceErr(err error) bool { return err != nil && (strings.Contains(err.Error(), "no space left")) } // For when our upper layer catchup detects its missing messages from the beginning of the stream. var errFirstSequenceMismatch = errors.New("first sequence mismatch") func isClusterResetErr(err error) bool { return err == errLastSeqMismatch || err == ErrStoreEOF || err == errFirstSequenceMismatch || errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries } // Copy all fields. func (smo *StoreMsg) copy(sm *StoreMsg) { if sm.buf != nil { sm.buf = sm.buf[:0] } sm.buf = append(sm.buf, smo.buf...) // We set cap on header in case someone wants to expand it. sm.hdr, sm.msg = sm.buf[:len(smo.hdr):len(smo.hdr)], sm.buf[len(smo.hdr):] sm.subj, sm.seq, sm.ts = smo.subj, smo.seq, smo.ts } // Clear all fields except underlying buffer but reset that if present to [:0]. func (sm *StoreMsg) clear() { if sm == nil { return } *sm = StoreMsg{_EMPTY_, nil, nil, sm.buf, 0, 0} if len(sm.buf) > 0 { sm.buf = sm.buf[:0] } } // Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer. // So this should be used sparingly when we know the encompassing byte slice's lifetime is the same. func bytesToString(b []byte) string { if len(b) == 0 { return _EMPTY_ } p := unsafe.SliceData(b) return unsafe.String(p, len(b)) } // Same in reverse. Used less often. func stringToBytes(s string) []byte { if len(s) == 0 { return nil } p := unsafe.StringData(s) b := unsafe.Slice(p, len(s)) return b } // Forces a copy of a string, for use in the case that you might have been passed a value when bytesToString was used, // but now you need a separate copy of it to store for longer-term use. func copyString(s string) string { b := make([]byte, len(s)) copy(b, s) return bytesToString(b) } func isPermissionError(err error) bool { return err != nil && os.IsPermission(err) } nats-server-2.10.27/server/store_test.go000066400000000000000000000347541477524627100202200ustar00rootroot00000000000000// Copyright 2012-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !skip_store_tests // +build !skip_store_tests package server import ( "fmt" "testing" ) func testAllStoreAllPermutations(t *testing.T, compressionAndEncryption bool, cfg StreamConfig, fn func(t *testing.T, fs StreamStore)) { t.Run("Memory", func(t *testing.T) { cfg.Storage = MemoryStorage fs, err := newMemStore(&cfg) require_NoError(t, err) defer fs.Stop() fn(t, fs) }) t.Run("File", func(t *testing.T) { cfg.Storage = FileStorage if compressionAndEncryption { testFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) { fs, err := newFileStore(fcfg, cfg) require_NoError(t, err) defer fs.Stop() fn(t, fs) }) } else { fs, err := newFileStore(FileStoreConfig{ StoreDir: t.TempDir(), }, cfg) require_NoError(t, err) defer fs.Stop() fn(t, fs) } }) } func TestStoreMsgLoadNextMsgMulti(t *testing.T) { testAllStoreAllPermutations( t, false, StreamConfig{Name: "zzz", Subjects: []string{"foo.*"}}, func(t *testing.T, fs StreamStore) { // Put 1k msgs in for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) fs.StoreMsg(subj, nil, []byte("ZZZ")) } var smv StoreMsg // Do multi load next with 1 wc entry. sl := NewSublistWithCache() sl.Insert(&subscription{subject: []byte("foo.>")}) for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) require_NoError(t, err) require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } // Now do multi load next with 1000 literal subjects. sl = NewSublistWithCache() for i := 0; i < 1000; i++ { subj := fmt.Sprintf("foo.%d", i) sl.Insert(&subscription{subject: []byte(subj)}) } for i, seq := 0, uint64(1); i < 1000; i++ { sm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv) require_NoError(t, err) require_Equal(t, sm.subj, fmt.Sprintf("foo.%d", i)) require_Equal(t, nseq, seq) seq++ } // Check that we can pull out 3 individuals. sl = NewSublistWithCache() sl.Insert(&subscription{subject: []byte("foo.2")}) sl.Insert(&subscription{subject: []byte("foo.222")}) sl.Insert(&subscription{subject: []byte("foo.999")}) sm, seq, err := fs.LoadNextMsgMulti(sl, 1, &smv) require_NoError(t, err) require_Equal(t, sm.subj, "foo.2") require_Equal(t, seq, 3) sm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) require_NoError(t, err) require_Equal(t, sm.subj, "foo.222") require_Equal(t, seq, 223) sm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) require_NoError(t, err) require_Equal(t, sm.subj, "foo.999") require_Equal(t, seq, 1000) _, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv) require_Error(t, err) require_Equal(t, seq, 1000) }, ) } func TestStoreDeleteSlice(t *testing.T) { ds := DeleteSlice{2} var deletes []uint64 ds.Range(func(seq uint64) bool { deletes = append(deletes, seq) return true }) require_Len(t, len(deletes), 1) require_Equal(t, deletes[0], 2) first, last, num := ds.State() require_Equal(t, first, 2) require_Equal(t, last, 2) require_Equal(t, num, 1) } func TestStoreDeleteRange(t *testing.T) { dr := DeleteRange{First: 2, Num: 1} var deletes []uint64 dr.Range(func(seq uint64) bool { deletes = append(deletes, seq) return true }) require_Len(t, len(deletes), 1) require_Equal(t, deletes[0], 2) first, last, num := dr.State() require_Equal(t, first, 2) require_Equal(t, last, 2) require_Equal(t, num, 1) } func TestStoreSubjectStateConsistency(t *testing.T) { testAllStoreAllPermutations( t, false, StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, func(t *testing.T, fs StreamStore) { getSubjectState := func() SimpleState { t.Helper() ss := fs.SubjectsState("foo") return ss["foo"] } var smp StoreMsg expectFirstSeq := func(eseq uint64) { t.Helper() sm, _, err := fs.LoadNextMsg("foo", false, 0, &smp) require_NoError(t, err) require_Equal(t, sm.seq, eseq) } expectLastSeq := func(eseq uint64) { t.Helper() sm, err := fs.LoadLastMsg("foo", &smp) require_NoError(t, err) require_Equal(t, sm.seq, eseq) } // Publish an initial batch of messages. for i := 0; i < 4; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } // Expect 4 msgs, with first=1, last=4. ss := getSubjectState() require_Equal(t, ss.Msgs, 4) require_Equal(t, ss.First, 1) expectFirstSeq(1) require_Equal(t, ss.Last, 4) expectLastSeq(4) // Remove first message, ss.First is lazy so will only mark ss.firstNeedsUpdate. removed, err := fs.RemoveMsg(1) require_NoError(t, err) require_True(t, removed) // Will update first, so corrects to seq 2. ss = getSubjectState() require_Equal(t, ss.Msgs, 3) require_Equal(t, ss.First, 2) expectFirstSeq(2) require_Equal(t, ss.Last, 4) expectLastSeq(4) // Remove last message, ss.Last is lazy so will only mark ss.lastNeedsUpdate. removed, err = fs.RemoveMsg(4) require_NoError(t, err) require_True(t, removed) // Will update last, so corrects to 3. ss = getSubjectState() require_Equal(t, ss.Msgs, 2) require_Equal(t, ss.First, 2) expectFirstSeq(2) require_Equal(t, ss.Last, 3) expectLastSeq(3) // Remove first message again. removed, err = fs.RemoveMsg(2) require_NoError(t, err) require_True(t, removed) // Since we only have one message left, must update ss.First and ensure ss.Last equals. ss = getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 3) expectFirstSeq(3) require_Equal(t, ss.Last, 3) expectLastSeq(3) // Publish some more messages so we can test another scenario. for i := 0; i < 3; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } // Just check the state is complete again. ss = getSubjectState() require_Equal(t, ss.Msgs, 4) require_Equal(t, ss.First, 3) expectFirstSeq(3) require_Equal(t, ss.Last, 7) expectLastSeq(7) // Remove last sequence, ss.Last is lazy so doesn't get updated. removed, err = fs.RemoveMsg(7) require_NoError(t, err) require_True(t, removed) // Remove first sequence, ss.First is lazy so doesn't get updated. removed, err = fs.RemoveMsg(3) require_NoError(t, err) require_True(t, removed) // Remove (now) first sequence. Both ss.First and ss.Last are lazy and both need to be recalculated later. removed, err = fs.RemoveMsg(5) require_NoError(t, err) require_True(t, removed) // ss.First and ss.Last should both be recalculated and equal each other. ss = getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 6) expectFirstSeq(6) require_Equal(t, ss.Last, 6) expectLastSeq(6) // We store a new message for ss.Last and remove it after, which marks it to be recalculated. _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) removed, err = fs.RemoveMsg(8) require_NoError(t, err) require_True(t, removed) // This will be the new ss.Last message, so reset ss.lastNeedsUpdate _, _, err = fs.StoreMsg("foo", nil, nil) require_NoError(t, err) // ss.First should remain the same, but ss.Last should equal the last message. ss = getSubjectState() require_Equal(t, ss.Msgs, 2) require_Equal(t, ss.First, 6) expectFirstSeq(6) require_Equal(t, ss.Last, 9) expectLastSeq(9) }, ) } func TestStoreSubjectStateConsistencyOptimization(t *testing.T) { testAllStoreAllPermutations( t, false, StreamConfig{Name: "TEST", Subjects: []string{"foo"}}, func(t *testing.T, fs StreamStore) { fillMsgs := func(c int) { t.Helper() for i := 0; i < c; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } } removeMsgs := func(seqs ...uint64) { t.Helper() for _, seq := range seqs { removed, err := fs.RemoveMsg(seq) require_NoError(t, err) require_True(t, removed) } } getSubjectState := func() (ss *SimpleState) { t.Helper() if f, ok := fs.(*fileStore); ok { ss, ok = f.lmb.fss.Find([]byte("foo")) require_True(t, ok) } else if ms, ok := fs.(*memStore); ok { ss, ok = ms.fss.Find([]byte("foo")) require_True(t, ok) } else { t.Fatal("Store not supported") } return ss } var smp StoreMsg expectSeq := func(seq uint64) { t.Helper() sm, _, err := fs.LoadNextMsg("foo", false, 0, &smp) require_NoError(t, err) require_Equal(t, sm.seq, seq) sm, err = fs.LoadLastMsg("foo", &smp) require_NoError(t, err) require_Equal(t, sm.seq, seq) } // results in ss.Last, ss.First is marked lazy (when we hit ss.Msgs-1==1). fillMsgs(3) removeMsgs(2, 1) ss := getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 3) require_Equal(t, ss.Last, 3) require_False(t, ss.firstNeedsUpdate) require_False(t, ss.lastNeedsUpdate) expectSeq(3) // ss.First is marked lazy first, then ss.Last is marked lazy (when we hit ss.Msgs-1==1). fillMsgs(2) removeMsgs(3, 5) ss = getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 3) require_Equal(t, ss.Last, 5) require_True(t, ss.firstNeedsUpdate) require_True(t, ss.lastNeedsUpdate) expectSeq(4) // ss.Last is marked lazy first, then ss.First is marked lazy (when we hit ss.Msgs-1==1). fillMsgs(2) removeMsgs(7, 4) ss = getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 4) require_Equal(t, ss.Last, 7) require_True(t, ss.firstNeedsUpdate) require_True(t, ss.lastNeedsUpdate) expectSeq(6) // ss.Msgs=1, results in ss.First, ss.Last is marked lazy (when we hit ss.Msgs-1==1). fillMsgs(2) removeMsgs(9, 8) ss = getSubjectState() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.First, 6) require_Equal(t, ss.Last, 6) require_False(t, ss.firstNeedsUpdate) require_False(t, ss.lastNeedsUpdate) expectSeq(6) }, ) } func TestStoreMaxMsgsPerUpdateBug(t *testing.T) { config := func() StreamConfig { return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} } testAllStoreAllPermutations( t, false, config(), func(t *testing.T, fs StreamStore) { for i := 0; i < 5; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } ss := fs.State() require_Equal(t, ss.Msgs, 5) require_Equal(t, ss.FirstSeq, 1) require_Equal(t, ss.LastSeq, 5) // Update max messages per-subject from 0 (infinite) to 1. // Since the per-subject limit was not specified before, messages should be removed upon config update. cfg := config() if _, ok := fs.(*fileStore); ok { cfg.Storage = FileStorage } else { cfg.Storage = MemoryStorage } cfg.MaxMsgsPer = 1 err := fs.UpdateConfig(&cfg) require_NoError(t, err) // Only one message should remain. ss = fs.State() require_Equal(t, ss.Msgs, 1) require_Equal(t, ss.FirstSeq, 5) require_Equal(t, ss.LastSeq, 5) // Update max messages per-subject from 0 (infinite) to an invalid value (< -1). cfg.MaxMsgsPer = -2 err = fs.UpdateConfig(&cfg) require_NoError(t, err) require_Equal(t, cfg.MaxMsgsPer, -1) }, ) } func TestStoreCompactCleansUpDmap(t *testing.T) { config := func() StreamConfig { return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} } for cseq := uint64(2); cseq <= 4; cseq++ { t.Run(fmt.Sprintf("Compact(%d)", cseq), func(t *testing.T) { testAllStoreAllPermutations( t, false, config(), func(t *testing.T, fs StreamStore) { dmapEntries := func() int { if fss, ok := fs.(*fileStore); ok { return fss.dmapEntries() } else if mss, ok := fs.(*memStore); ok { mss.mu.RLock() defer mss.mu.RUnlock() return mss.dmap.Size() } else { return 0 } } // Publish messages, should have no interior deletes. for i := 0; i < 3; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } require_Len(t, dmapEntries(), 0) // Removing one message in the middle should be an interior delete. _, err := fs.RemoveMsg(2) require_NoError(t, err) require_Len(t, dmapEntries(), 1) // Compacting must always clean up the interior delete. _, err = fs.Compact(cseq) require_NoError(t, err) require_Len(t, dmapEntries(), 0) // Validate first/last sequence. state := fs.State() fseq := uint64(3) if fseq < cseq { fseq = cseq } require_Equal(t, state.FirstSeq, fseq) require_Equal(t, state.LastSeq, 3) }) }) } } func TestStoreTruncateCleansUpDmap(t *testing.T) { config := func() StreamConfig { return StreamConfig{Name: "TEST", Subjects: []string{"foo"}, MaxMsgsPer: 0} } for tseq := uint64(0); tseq <= 1; tseq++ { t.Run(fmt.Sprintf("Truncate(%d)", tseq), func(t *testing.T) { testAllStoreAllPermutations( t, false, config(), func(t *testing.T, fs StreamStore) { dmapEntries := func() int { if fss, ok := fs.(*fileStore); ok { return fss.dmapEntries() } else if mss, ok := fs.(*memStore); ok { mss.mu.RLock() defer mss.mu.RUnlock() return mss.dmap.Size() } else { return 0 } } // Publish messages, should have no interior deletes. for i := 0; i < 3; i++ { _, _, err := fs.StoreMsg("foo", nil, nil) require_NoError(t, err) } require_Len(t, dmapEntries(), 0) // Removing one message in the middle should be an interior delete. _, err := fs.RemoveMsg(2) require_NoError(t, err) require_Len(t, dmapEntries(), 1) // Truncating must always clean up the interior delete. err = fs.Truncate(tseq) require_NoError(t, err) require_Len(t, dmapEntries(), 0) // Validate first/last sequence. state := fs.State() fseq := uint64(1) if fseq > tseq { fseq = tseq } require_Equal(t, state.FirstSeq, fseq) require_Equal(t, state.LastSeq, tseq) }) }) } } nats-server-2.10.27/server/stream.go000066400000000000000000005302731477524627100173150ustar00rootroot00000000000000// Copyright 2019-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "archive/tar" "bytes" "encoding/json" "errors" "fmt" "io" "math" "math/rand" "os" "path/filepath" "reflect" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/s2" "github.com/nats-io/nats-server/v2/server/gsl" "github.com/nats-io/nuid" ) // StreamConfig will determine the name, subjects and retention policy // for a given stream. If subjects is empty the name will be used. type StreamConfig struct { Name string `json:"name"` Description string `json:"description,omitempty"` Subjects []string `json:"subjects,omitempty"` Retention RetentionPolicy `json:"retention"` MaxConsumers int `json:"max_consumers"` MaxMsgs int64 `json:"max_msgs"` MaxBytes int64 `json:"max_bytes"` MaxAge time.Duration `json:"max_age"` MaxMsgsPer int64 `json:"max_msgs_per_subject"` MaxMsgSize int32 `json:"max_msg_size,omitempty"` Discard DiscardPolicy `json:"discard"` Storage StorageType `json:"storage"` Replicas int `json:"num_replicas"` NoAck bool `json:"no_ack,omitempty"` Template string `json:"template_owner,omitempty"` Duplicates time.Duration `json:"duplicate_window,omitempty"` Placement *Placement `json:"placement,omitempty"` Mirror *StreamSource `json:"mirror,omitempty"` Sources []*StreamSource `json:"sources,omitempty"` Compression StoreCompression `json:"compression"` FirstSeq uint64 `json:"first_seq,omitempty"` // Allow applying a subject transform to incoming messages before doing anything else SubjectTransform *SubjectTransformConfig `json:"subject_transform,omitempty"` // Allow republish of the message after being sequenced and stored. RePublish *RePublish `json:"republish,omitempty"` // Allow higher performance, direct access to get individual messages. E.g. KeyValue AllowDirect bool `json:"allow_direct"` // Allow higher performance and unified direct access for mirrors as well. MirrorDirect bool `json:"mirror_direct"` // Allow KV like semantics to also discard new on a per subject basis DiscardNewPer bool `json:"discard_new_per_subject,omitempty"` // Optional qualifiers. These can not be modified after set to true. // Sealed will seal a stream so no messages can get out or in. Sealed bool `json:"sealed"` // DenyDelete will restrict the ability to delete messages. DenyDelete bool `json:"deny_delete"` // DenyPurge will restrict the ability to purge messages. DenyPurge bool `json:"deny_purge"` // AllowRollup allows messages to be placed into the system and purge // all older messages using a special msg header. AllowRollup bool `json:"allow_rollup_hdrs"` // The following defaults will apply to consumers when created against // this stream, unless overridden manually. // TODO(nat): Can/should we name these better? ConsumerLimits StreamConsumerLimits `json:"consumer_limits"` // Metadata is additional metadata for the Stream. Metadata map[string]string `json:"metadata,omitempty"` } type StreamConsumerLimits struct { InactiveThreshold time.Duration `json:"inactive_threshold,omitempty"` MaxAckPending int `json:"max_ack_pending,omitempty"` } // SubjectTransformConfig is for applying a subject transform (to matching messages) before doing anything else when a new message is received type SubjectTransformConfig struct { Source string `json:"src"` Destination string `json:"dest"` } // RePublish is for republishing messages once committed to a stream. type RePublish struct { Source string `json:"src,omitempty"` Destination string `json:"dest"` HeadersOnly bool `json:"headers_only,omitempty"` } // JSPubAckResponse is a formal response to a publish operation. type JSPubAckResponse struct { Error *ApiError `json:"error,omitempty"` *PubAck } // ToError checks if the response has a error and if it does converts it to an error // avoiding the pitfalls described by https://yourbasic.org/golang/gotcha-why-nil-error-not-equal-nil/ func (r *JSPubAckResponse) ToError() error { if r.Error == nil { return nil } return r.Error } // PubAck is the detail you get back from a publish to a stream that was successful. // e.g. +OK {"stream": "Orders", "seq": 22} type PubAck struct { Stream string `json:"stream"` Sequence uint64 `json:"seq"` Domain string `json:"domain,omitempty"` Duplicate bool `json:"duplicate,omitempty"` } // StreamInfo shows config and current state for this stream. type StreamInfo struct { Config StreamConfig `json:"config"` Created time.Time `json:"created"` State StreamState `json:"state"` Domain string `json:"domain,omitempty"` Cluster *ClusterInfo `json:"cluster,omitempty"` Mirror *StreamSourceInfo `json:"mirror,omitempty"` Sources []*StreamSourceInfo `json:"sources,omitempty"` Alternates []StreamAlternate `json:"alternates,omitempty"` // TimeStamp indicates when the info was gathered TimeStamp time.Time `json:"ts"` } type StreamAlternate struct { Name string `json:"name"` Domain string `json:"domain,omitempty"` Cluster string `json:"cluster"` } // ClusterInfo shows information about the underlying set of servers // that make up the stream or consumer. type ClusterInfo struct { Name string `json:"name,omitempty"` RaftGroup string `json:"raft_group,omitempty"` Leader string `json:"leader,omitempty"` Replicas []*PeerInfo `json:"replicas,omitempty"` } // PeerInfo shows information about all the peers in the cluster that // are supporting the stream or consumer. type PeerInfo struct { Name string `json:"name"` Current bool `json:"current"` Offline bool `json:"offline,omitempty"` Active time.Duration `json:"active"` Lag uint64 `json:"lag,omitempty"` Peer string `json:"peer"` // For migrations. cluster string } // StreamSourceInfo shows information about an upstream stream source. type StreamSourceInfo struct { Name string `json:"name"` External *ExternalStream `json:"external,omitempty"` Lag uint64 `json:"lag"` Active time.Duration `json:"active"` Error *ApiError `json:"error,omitempty"` FilterSubject string `json:"filter_subject,omitempty"` SubjectTransforms []SubjectTransformConfig `json:"subject_transforms,omitempty"` } // StreamSource dictates how streams can source from other streams. type StreamSource struct { Name string `json:"name"` OptStartSeq uint64 `json:"opt_start_seq,omitempty"` OptStartTime *time.Time `json:"opt_start_time,omitempty"` FilterSubject string `json:"filter_subject,omitempty"` SubjectTransforms []SubjectTransformConfig `json:"subject_transforms,omitempty"` External *ExternalStream `json:"external,omitempty"` // Internal iname string // For indexing when stream names are the same for multiple sources. } // ExternalStream allows you to qualify access to a stream source in another account. type ExternalStream struct { ApiPrefix string `json:"api"` DeliverPrefix string `json:"deliver"` } // Stream is a jetstream stream of messages. When we receive a message internally destined // for a Stream we will direct link from the client to this structure. type stream struct { mu sync.RWMutex // Read/write lock for the stream. js *jetStream // The internal *jetStream for the account. jsa *jsAccount // The JetStream account-level information. acc *Account // The account this stream is defined in. srv *Server // The server we are running in. client *client // The internal JetStream client. sysc *client // The internal JetStream system client. // The current last subscription ID for the subscriptions through `client`. // Those subscriptions are for the subjects filters being listened to and captured by the stream. sid atomic.Uint64 pubAck []byte // The template (prefix) to generate the pubAck responses for this stream quickly. outq *jsOutQ // Queue of *jsPubMsg for sending messages. msgs *ipQueue[*inMsg] // Intra-process queue for the ingress of messages. gets *ipQueue[*directGetReq] // Intra-process queue for the direct get requests. store StreamStore // The storage for this stream. ackq *ipQueue[uint64] // Intra-process queue for acks. lseq uint64 // The sequence number of the last message stored in the stream. lmsgId string // The de-duplication message ID of the last message stored in the stream. consumers map[string]*consumer // The consumers for this stream. numFilter int // The number of filtered consumers. cfg StreamConfig // The stream's config. cfgMu sync.RWMutex // Config mutex used to solve some races with consumer code created time.Time // Time the stream was created. stype StorageType // The storage type. tier string // The tier is the number of replicas for the stream (e.g. "R1" or "R3"). ddmap map[string]*ddentry // The dedupe map. ddarr []*ddentry // The dedupe array. ddindex int // The dedupe index. ddtmr *time.Timer // The dedupe timer. qch chan struct{} // The quit channel. mqch chan struct{} // The monitor's quit channel. active bool // Indicates that there are active internal subscriptions (for the subject filters) // and/or mirror/sources consumers are scheduled to be established or already started. ddloaded bool // set to true when the deduplication structures are been built. closed atomic.Bool // Set to true when stop() is called on the stream. // Mirror mirror *sourceInfo // Sources sources map[string]*sourceInfo sourceSetupSchedules map[string]*time.Timer sourcesConsumerSetup *time.Timer smsgs *ipQueue[*inMsg] // Intra-process queue for all incoming sourced messages. // Indicates we have direct consumers. directs int // For input subject transform. itr *subjectTransform // For republishing. tr *subjectTransform // For processing consumers without main stream lock. clsMu sync.RWMutex cList []*consumer // Consumer list. sch chan struct{} // Channel to signal consumers. sigq *ipQueue[*cMsg] // Intra-process queue for the messages to signal to the consumers. csl *gsl.GenericSublist[*consumer] // Consumer subscription list. // For non limits policy streams when they process an ack before the actual msg. // Can happen in stretch clusters, multi-cloud, or during catchup for a restarted server. preAcks map[uint64]map[*consumer]struct{} // TODO(dlc) - Hide everything below behind two pointers. // Clustered mode. sa *streamAssignment // What the meta controller uses to assign streams to peers. node RaftNode // Our RAFT node for the stream's group. catchup atomic.Bool // Used to signal we are in catchup mode. catchups map[string]uint64 // The number of messages that need to be caught per peer. syncSub *subscription // Internal subscription for sync messages (on "$JSC.SYNC"). infoSub *subscription // Internal subscription for stream info requests. clMu sync.Mutex // The mutex for clseq and clfs. clseq uint64 // The current last seq being proposed to the NRG layer. clfs uint64 // The count (offset) of the number of failed NRG sequences used to compute clseq. inflight map[uint64]uint64 // Inflight message sizes per clseq. lqsent time.Time // The time at which the last lost quorum advisory was sent. Used to rate limit. uch chan struct{} // The channel to signal updates to the monitor routine. compressOK bool // True if we can do message compression in RAFT and catchup logic inMonitor bool // True if the monitor routine has been started. // Direct get subscription. directSub *subscription lastBySub *subscription monitorWg sync.WaitGroup // Wait group for the monitor routine. } type sourceInfo struct { name string // The name of the stream being sourced. iname string // The unique index name of this particular source. cname string // The name of the current consumer for this source. sub *subscription // The subscription to the consumer. // (mirrors only) The subscription to the direct get request subject for // the source stream's name on the `_sys_` queue group. dsub *subscription // (mirrors only) The subscription to the direct get last per subject request subject for // the source stream's name on the `_sys_` queue group. lbsub *subscription msgs *ipQueue[*inMsg] // Intra-process queue for incoming messages. sseq uint64 // Last stream message sequence number seen from the source. dseq uint64 // Last delivery (i.e. consumer's) sequence number. lag uint64 // 0 or number of messages pending (as last reported by the consumer) - 1. err *ApiError // The API error that caused the last consumer setup to fail. fails int // The number of times trying to setup the consumer failed. last atomic.Int64 // Time the consumer was created or of last message it received. lreq time.Time // The last time setupMirrorConsumer/setupSourceConsumer was called. qch chan struct{} // Quit channel. sip bool // Setup in progress. wg sync.WaitGroup // WaitGroup for the consumer's go routine. sf string // The subject filter. sfs []string // The subject filters. trs []*subjectTransform // The subject transforms. } // For mirrors and direct get const ( dgetGroup = sysGroup dgetCaughtUpThresh = 10 ) // Headers for published messages. const ( JSMsgId = "Nats-Msg-Id" JSExpectedStream = "Nats-Expected-Stream" JSExpectedLastSeq = "Nats-Expected-Last-Sequence" JSExpectedLastSubjSeq = "Nats-Expected-Last-Subject-Sequence" JSExpectedLastMsgId = "Nats-Expected-Last-Msg-Id" JSStreamSource = "Nats-Stream-Source" JSLastConsumerSeq = "Nats-Last-Consumer" JSLastStreamSeq = "Nats-Last-Stream" JSConsumerStalled = "Nats-Consumer-Stalled" JSMsgRollup = "Nats-Rollup" JSMsgSize = "Nats-Msg-Size" JSResponseType = "Nats-Response-Type" ) // Headers for republished messages and direct gets. const ( JSStream = "Nats-Stream" JSSequence = "Nats-Sequence" JSTimeStamp = "Nats-Time-Stamp" JSSubject = "Nats-Subject" JSLastSequence = "Nats-Last-Sequence" ) // Rollups, can be subject only or all messages. const ( JSMsgRollupSubject = "sub" JSMsgRollupAll = "all" ) const ( jsCreateResponse = "create" ) // Dedupe entry type ddentry struct { id string // The unique message ID provided by the client. seq uint64 // The sequence number of the message. ts int64 // The timestamp of the message. } // Replicas Range const StreamMaxReplicas = 5 // AddStream adds a stream for the given account. func (a *Account) addStream(config *StreamConfig) (*stream, error) { return a.addStreamWithAssignment(config, nil, nil) } // AddStreamWithStore adds a stream for the given account with custome store config options. func (a *Account) addStreamWithStore(config *StreamConfig, fsConfig *FileStoreConfig) (*stream, error) { return a.addStreamWithAssignment(config, fsConfig, nil) } func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileStoreConfig, sa *streamAssignment) (*stream, error) { s, jsa, err := a.checkForJetStream() if err != nil { return nil, err } // If we do not have the stream currently assigned to us in cluster mode we will proceed but warn. // This can happen on startup with restored state where on meta replay we still do not have // the assignment. Running in single server mode this always returns true. if !jsa.streamAssigned(config.Name) { s.Debugf("Stream '%s > %s' does not seem to be assigned to this server", a.Name, config.Name) } // Sensible defaults. cfg, apiErr := s.checkStreamCfg(config, a) if apiErr != nil { return nil, apiErr } singleServerMode := !s.JetStreamIsClustered() && s.standAloneMode() if singleServerMode && cfg.Replicas > 1 { return nil, ApiErrors[JSStreamReplicasNotSupportedErr] } // Make sure we are ok when these are done in parallel. // We used to call Add(1) in the "else" clause of the "if loaded" // statement. This caused a data race because it was possible // that one go routine stores (with count==0) and another routine // gets "loaded==true" and calls wg.Wait() while the other routine // then calls wg.Add(1). It also could mean that two routines execute // the rest of the code concurrently. swg := &sync.WaitGroup{} swg.Add(1) v, loaded := jsa.inflight.LoadOrStore(cfg.Name, swg) wg := v.(*sync.WaitGroup) if loaded { wg.Wait() // This waitgroup is "thrown away" (since there was an existing one). swg.Done() } else { defer func() { jsa.inflight.Delete(cfg.Name) wg.Done() }() } js, isClustered := jsa.jetStreamAndClustered() jsa.mu.Lock() if mset, ok := jsa.streams[cfg.Name]; ok { jsa.mu.Unlock() // Check to see if configs are same. ocfg := mset.config() // set the index name on cfg since it would not contain a value for iname while the return from mset.config() does to ensure the DeepEqual works for _, s := range cfg.Sources { s.setIndexName() } if reflect.DeepEqual(ocfg, cfg) { if sa != nil { mset.setStreamAssignment(sa) } return mset, nil } else { return nil, ApiErrors[JSStreamNameExistErr] } } jsa.usageMu.RLock() selected, tier, hasTier := jsa.selectLimits(cfg.Replicas) jsa.usageMu.RUnlock() reserved := int64(0) if !isClustered { reserved = jsa.tieredReservation(tier, &cfg) } jsa.mu.Unlock() if !hasTier { return nil, NewJSNoLimitsError() } js.mu.RLock() if isClustered { _, reserved = tieredStreamAndReservationCount(js.cluster.streams[a.Name], tier, &cfg) } if err := js.checkAllLimits(&selected, &cfg, reserved, 0); err != nil { js.mu.RUnlock() return nil, err } js.mu.RUnlock() jsa.mu.Lock() // Check for template ownership if present. if cfg.Template != _EMPTY_ && jsa.account != nil { if !jsa.checkTemplateOwnership(cfg.Template, cfg.Name) { jsa.mu.Unlock() return nil, fmt.Errorf("stream not owned by template") } } // If mirror, check if the transforms (if any) are valid. if cfg.Mirror != nil { if len(cfg.Mirror.SubjectTransforms) == 0 { if cfg.Mirror.FilterSubject != _EMPTY_ && !IsValidSubject(cfg.Mirror.FilterSubject) { jsa.mu.Unlock() return nil, fmt.Errorf("subject filter '%s' for the mirror %w", cfg.Mirror.FilterSubject, ErrBadSubject) } } else { for _, st := range cfg.Mirror.SubjectTransforms { if st.Source != _EMPTY_ && !IsValidSubject(st.Source) { jsa.mu.Unlock() return nil, fmt.Errorf("invalid subject transform source '%s' for the mirror: %w", st.Source, ErrBadSubject) } // check the transform, if any, is valid if st.Destination != _EMPTY_ { if _, err = NewSubjectTransform(st.Source, st.Destination); err != nil { jsa.mu.Unlock() return nil, fmt.Errorf("subject transform from '%s' to '%s' for the mirror: %w", st.Source, st.Destination, err) } } } } } // Setup our internal indexed names here for sources and check if the transforms (if any) are valid. for _, ssi := range cfg.Sources { if len(ssi.SubjectTransforms) == 0 { // check the filter, if any, is valid if ssi.FilterSubject != _EMPTY_ && !IsValidSubject(ssi.FilterSubject) { jsa.mu.Unlock() return nil, fmt.Errorf("subject filter '%s' for the source: %w", ssi.FilterSubject, ErrBadSubject) } } else { for _, st := range ssi.SubjectTransforms { if st.Source != _EMPTY_ && !IsValidSubject(st.Source) { jsa.mu.Unlock() return nil, fmt.Errorf("subject filter '%s' for the source: %w", st.Source, ErrBadSubject) } // check the transform, if any, is valid if st.Destination != _EMPTY_ { if _, err = NewSubjectTransform(st.Source, st.Destination); err != nil { jsa.mu.Unlock() return nil, fmt.Errorf("subject transform from '%s' to '%s' for the source: %w", st.Source, st.Destination, err) } } } } } // Check for overlapping subjects with other streams. // These are not allowed for now. if jsa.subjectsOverlap(cfg.Subjects, nil) { jsa.mu.Unlock() return nil, NewJSStreamSubjectOverlapError() } if !hasTier { jsa.mu.Unlock() return nil, fmt.Errorf("no applicable tier found") } // Setup the internal clients. c := s.createInternalJetStreamClient() ic := s.createInternalJetStreamClient() qpfx := fmt.Sprintf("[ACC:%s] stream '%s' ", a.Name, config.Name) mset := &stream{ acc: a, jsa: jsa, cfg: cfg, js: js, srv: s, client: c, sysc: ic, tier: tier, stype: cfg.Storage, consumers: make(map[string]*consumer), msgs: newIPQueue[*inMsg](s, qpfx+"messages"), gets: newIPQueue[*directGetReq](s, qpfx+"direct gets"), qch: make(chan struct{}), mqch: make(chan struct{}), uch: make(chan struct{}, 4), sch: make(chan struct{}, 1), } // Start our signaling routine to process consumers. mset.sigq = newIPQueue[*cMsg](s, qpfx+"obs") // of *cMsg go mset.signalConsumersLoop() // For no-ack consumers when we are interest retention. if cfg.Retention != LimitsPolicy { mset.ackq = newIPQueue[uint64](s, qpfx+"acks") } // Check for input subject transform if cfg.SubjectTransform != nil { tr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination) if err != nil { jsa.mu.Unlock() return nil, fmt.Errorf("stream subject transform from '%s' to '%s': %w", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err) } mset.itr = tr } // Check for RePublish. if cfg.RePublish != nil { tr, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination) if err != nil { jsa.mu.Unlock() return nil, fmt.Errorf("stream republish transform from '%s' to '%s': %w", cfg.RePublish.Source, cfg.RePublish.Destination, err) } // Assign our transform for republishing. mset.tr = tr } storeDir := filepath.Join(jsa.storeDir, streamsDir, cfg.Name) jsa.mu.Unlock() // Bind to the user account. c.registerWithAccount(a) // Bind to the system account. ic.registerWithAccount(s.SystemAccount()) // Create the appropriate storage fsCfg := fsConfig if fsCfg == nil { fsCfg = &FileStoreConfig{} // If we are file based and not explicitly configured // we may be able to auto-tune based on max msgs or bytes. if cfg.Storage == FileStorage { mset.autoTuneFileStorageBlockSize(fsCfg) } } fsCfg.StoreDir = storeDir fsCfg.AsyncFlush = false // Grab configured sync interval. fsCfg.SyncInterval = s.getOpts().SyncInterval fsCfg.SyncAlways = s.getOpts().SyncAlways fsCfg.Compression = config.Compression if err := mset.setupStore(fsCfg); err != nil { mset.stop(true, false) return nil, NewJSStreamStoreFailedError(err) } // Create our pubAck template here. Better than json marshal each time on success. if domain := s.getOpts().JetStreamDomain; domain != _EMPTY_ { mset.pubAck = []byte(fmt.Sprintf("{%q:%q, %q:%q, %q:", "stream", cfg.Name, "domain", domain, "seq")) } else { mset.pubAck = []byte(fmt.Sprintf("{%q:%q, %q:", "stream", cfg.Name, "seq")) } end := len(mset.pubAck) mset.pubAck = mset.pubAck[:end:end] // Set our known last sequence. var state StreamState mset.store.FastState(&state) // Possible race with consumer.setLeader during recovery. mset.mu.Lock() mset.lseq = state.LastSeq mset.mu.Unlock() // If no msgs (new stream), set dedupe state loaded to true. if state.Msgs == 0 { mset.ddloaded = true } // Set our stream assignment if in clustered mode. reserveResources := true if sa != nil { mset.setStreamAssignment(sa) // If the stream is resetting we must not double-account resources, they were already accounted for. js.mu.Lock() if sa.resetting { reserveResources, sa.resetting = false, false } js.mu.Unlock() } // Setup our internal send go routine. mset.setupSendCapabilities() // Reserve resources if MaxBytes present. if reserveResources { mset.js.reserveStreamResources(&mset.cfg) } // Call directly to set leader if not in clustered mode. // This can be called though before we actually setup clustering, so check both. if singleServerMode { if err := mset.setLeader(true); err != nil { mset.stop(true, false) return nil, err } } // This is always true in single server mode. if mset.IsLeader() { // Send advisory. var suppress bool if !s.standAloneMode() && sa == nil { if cfg.Replicas > 1 { suppress = true } } else if sa != nil { suppress = sa.responded } if !suppress { mset.sendCreateAdvisory() } } // Register with our account last. jsa.mu.Lock() jsa.streams[cfg.Name] = mset jsa.mu.Unlock() return mset, nil } // Composes the index name. Contains the stream name, subject filter, and transform destination // when the stream is external we will use the api prefix as part of the index name // (as the same stream name could be used in multiple JS domains) func (ssi *StreamSource) composeIName() string { var iName = ssi.Name if ssi.External != nil { iName = iName + ":" + getHash(ssi.External.ApiPrefix) } source := ssi.FilterSubject destination := fwcs if len(ssi.SubjectTransforms) == 0 { // normalize filter and destination in case they are empty if source == _EMPTY_ { source = fwcs } if destination == _EMPTY_ { destination = fwcs } } else { var sources, destinations []string for _, tr := range ssi.SubjectTransforms { trsrc, trdest := tr.Source, tr.Destination if trsrc == _EMPTY_ { trsrc = fwcs } if trdest == _EMPTY_ { trdest = fwcs } sources = append(sources, trsrc) destinations = append(destinations, trdest) } source = strings.Join(sources, "\f") destination = strings.Join(destinations, "\f") } return strings.Join([]string{iName, source, destination}, " ") } // Sets the index name. func (ssi *StreamSource) setIndexName() { ssi.iname = ssi.composeIName() } func (mset *stream) streamAssignment() *streamAssignment { mset.mu.RLock() defer mset.mu.RUnlock() return mset.sa } func (mset *stream) setStreamAssignment(sa *streamAssignment) { var node RaftNode var peers []string mset.mu.RLock() js := mset.js mset.mu.RUnlock() if js != nil { js.mu.RLock() if sa.Group != nil { node = sa.Group.node peers = sa.Group.Peers } js.mu.RUnlock() } mset.mu.Lock() defer mset.mu.Unlock() mset.sa = sa if sa == nil { return } // Set our node. mset.node = node if mset.node != nil { mset.node.UpdateKnownPeers(peers) } // Setup our info sub here as well for all stream members. This is now by design. if mset.infoSub == nil { isubj := fmt.Sprintf(clusterStreamInfoT, mset.jsa.acc(), mset.cfg.Name) // Note below the way we subscribe here is so that we can send requests to ourselves. mset.infoSub, _ = mset.srv.systemSubscribe(isubj, _EMPTY_, false, mset.sysc, mset.handleClusterStreamInfoRequest) } // Trigger update chan. select { case mset.uch <- struct{}{}: default: } } func (mset *stream) monitorQuitC() <-chan struct{} { if mset == nil { return nil } mset.mu.RLock() defer mset.mu.RUnlock() return mset.mqch } func (mset *stream) updateC() <-chan struct{} { if mset == nil { return nil } mset.mu.RLock() defer mset.mu.RUnlock() return mset.uch } // IsLeader will return if we are the current leader. func (mset *stream) IsLeader() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.isLeader() } // Lock should be held. func (mset *stream) isLeader() bool { if mset.isClustered() { return mset.node.Leader() } return true } // TODO(dlc) - Check to see if we can accept being the leader or we should should step down. func (mset *stream) setLeader(isLeader bool) error { mset.mu.Lock() // If we are here we have a change in leader status. if isLeader { // Make sure we are listening for sync requests. // TODO(dlc) - Original design was that all in sync members of the group would do DQ. if mset.isClustered() { mset.startClusterSubs() } // Setup subscriptions if we were not already the leader. if err := mset.subscribeToStream(); err != nil { if mset.isClustered() { // Stepdown since we have an error. mset.node.StepDown() } mset.mu.Unlock() return err } } else { // cancel timer to create the source consumers if not fired yet if mset.sourcesConsumerSetup != nil { mset.sourcesConsumerSetup.Stop() mset.sourcesConsumerSetup = nil } else { // Stop any source consumers mset.stopSourceConsumers() } // Stop responding to sync requests. mset.stopClusterSubs() // Unsubscribe from direct stream. mset.unsubscribeToStream(false) // Clear catchup state mset.clearAllCatchupPeers() } mset.mu.Unlock() // If we are interest based make sure to check consumers. // This is to make sure we process any outstanding acks. mset.checkInterestState() return nil } // Lock should be held. func (mset *stream) startClusterSubs() { if mset.syncSub == nil { mset.syncSub, _ = mset.srv.systemSubscribe(mset.sa.Sync, _EMPTY_, false, mset.sysc, mset.handleClusterSyncRequest) } } // Lock should be held. func (mset *stream) stopClusterSubs() { if mset.syncSub != nil { mset.srv.sysUnsubscribe(mset.syncSub) mset.syncSub = nil } } // account gets the account for this stream. func (mset *stream) account() *Account { mset.mu.RLock() jsa := mset.jsa mset.mu.RUnlock() if jsa == nil { return nil } return jsa.acc() } // Helper to determine the max msg size for this stream if file based. func (mset *stream) maxMsgSize() uint64 { maxMsgSize := mset.cfg.MaxMsgSize if maxMsgSize <= 0 { // Pull from the account. if mset.jsa != nil { if acc := mset.jsa.acc(); acc != nil { acc.mu.RLock() maxMsgSize = acc.mpay acc.mu.RUnlock() } } // If all else fails use default. if maxMsgSize <= 0 { maxMsgSize = MAX_PAYLOAD_SIZE } } // Now determine an estimation for the subjects etc. maxSubject := -1 for _, subj := range mset.cfg.Subjects { if subjectIsLiteral(subj) { if len(subj) > maxSubject { maxSubject = len(subj) } } } if maxSubject < 0 { const defaultMaxSubject = 256 maxSubject = defaultMaxSubject } // filestore will add in estimates for record headers, etc. return fileStoreMsgSizeEstimate(maxSubject, int(maxMsgSize)) } // If we are file based and the file storage config was not explicitly set // we can autotune block sizes to better match. Our target will be to store 125% // of the theoretical limit. We will round up to nearest 100 bytes as well. func (mset *stream) autoTuneFileStorageBlockSize(fsCfg *FileStoreConfig) { var totalEstSize uint64 // MaxBytes will take precedence for now. if mset.cfg.MaxBytes > 0 { totalEstSize = uint64(mset.cfg.MaxBytes) } else if mset.cfg.MaxMsgs > 0 { // Determine max message size to estimate. totalEstSize = mset.maxMsgSize() * uint64(mset.cfg.MaxMsgs) } else if mset.cfg.MaxMsgsPer > 0 { fsCfg.BlockSize = uint64(defaultKVBlockSize) return } else { // If nothing set will let underlying filestore determine blkSize. return } blkSize := (totalEstSize / 4) + 1 // (25% overhead) // Round up to nearest 100 if m := blkSize % 100; m != 0 { blkSize += 100 - m } if blkSize <= FileStoreMinBlkSize { blkSize = FileStoreMinBlkSize } else if blkSize >= FileStoreMaxBlkSize { blkSize = FileStoreMaxBlkSize } else { blkSize = defaultMediumBlockSize } fsCfg.BlockSize = uint64(blkSize) } // rebuildDedupe will rebuild any dedupe structures needed after recovery of a stream. // Will be called lazily to avoid penalizing startup times. // TODO(dlc) - Might be good to know if this should be checked at all for streams with no // headers and msgId in them. Would need signaling from the storage layer. // Lock should be held. func (mset *stream) rebuildDedupe() { if mset.ddloaded { return } mset.ddloaded = true // We have some messages. Lookup starting sequence by duplicate time window. sseq := mset.store.GetSeqFromTime(time.Now().Add(-mset.cfg.Duplicates)) if sseq == 0 { return } var smv StoreMsg var state StreamState mset.store.FastState(&state) for seq := sseq; seq <= state.LastSeq; seq++ { sm, err := mset.store.LoadMsg(seq, &smv) if err != nil { continue } var msgId string if len(sm.hdr) > 0 { if msgId = getMsgId(sm.hdr); msgId != _EMPTY_ { mset.storeMsgIdLocked(&ddentry{msgId, sm.seq, sm.ts}) } } if seq == state.LastSeq { mset.lmsgId = msgId } } } func (mset *stream) lastSeqAndCLFS() (uint64, uint64) { return mset.lastSeq(), mset.getCLFS() } func (mset *stream) getCLFS() uint64 { if mset == nil { return 0 } mset.clMu.Lock() defer mset.clMu.Unlock() return mset.clfs } func (mset *stream) setCLFS(clfs uint64) { mset.clMu.Lock() mset.clfs = clfs mset.clMu.Unlock() } func (mset *stream) lastSeq() uint64 { mset.mu.RLock() defer mset.mu.RUnlock() return mset.lseq } // Set last seq. // Write lock should be held. func (mset *stream) setLastSeq(lseq uint64) { mset.lseq = lseq } func (mset *stream) sendCreateAdvisory() { mset.mu.RLock() name := mset.cfg.Name template := mset.cfg.Template outq := mset.outq srv := mset.srv mset.mu.RUnlock() if outq == nil { return } // finally send an event that this stream was created m := JSStreamActionAdvisory{ TypedEvent: TypedEvent{ Type: JSStreamActionAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: name, Action: CreateEvent, Template: template, Domain: srv.getOpts().JetStreamDomain, } j, err := json.Marshal(m) if err != nil { return } subj := JSAdvisoryStreamCreatedPre + "." + name outq.sendMsg(subj, j) } func (mset *stream) sendDeleteAdvisoryLocked() { if mset.outq == nil { return } m := JSStreamActionAdvisory{ TypedEvent: TypedEvent{ Type: JSStreamActionAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: mset.cfg.Name, Action: DeleteEvent, Template: mset.cfg.Template, Domain: mset.srv.getOpts().JetStreamDomain, } j, err := json.Marshal(m) if err == nil { subj := JSAdvisoryStreamDeletedPre + "." + mset.cfg.Name mset.outq.sendMsg(subj, j) } } func (mset *stream) sendUpdateAdvisoryLocked() { if mset.outq == nil { return } m := JSStreamActionAdvisory{ TypedEvent: TypedEvent{ Type: JSStreamActionAdvisoryType, ID: nuid.Next(), Time: time.Now().UTC(), }, Stream: mset.cfg.Name, Action: ModifyEvent, Domain: mset.srv.getOpts().JetStreamDomain, } j, err := json.Marshal(m) if err == nil { subj := JSAdvisoryStreamUpdatedPre + "." + mset.cfg.Name mset.outq.sendMsg(subj, j) } } // Created returns created time. func (mset *stream) createdTime() time.Time { mset.mu.RLock() created := mset.created mset.mu.RUnlock() return created } // Internal to allow creation time to be restored. func (mset *stream) setCreatedTime(created time.Time) { mset.mu.Lock() mset.created = created mset.mu.Unlock() } // subjectsOverlap to see if these subjects overlap with existing subjects. // Use only for non-clustered JetStream // RLock minimum should be held. func (jsa *jsAccount) subjectsOverlap(subjects []string, self *stream) bool { for _, mset := range jsa.streams { if self != nil && mset == self { continue } for _, subj := range mset.cfg.Subjects { for _, tsubj := range subjects { if SubjectsCollide(tsubj, subj) { return true } } } } return false } // StreamDefaultDuplicatesWindow default duplicates window. const StreamDefaultDuplicatesWindow = 2 * time.Minute func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account) (StreamConfig, *ApiError) { lim := &s.getOpts().JetStreamLimits if config == nil { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration invalid")) } if !isValidName(config.Name) { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream name is required and can not contain '.', '*', '>'")) } if len(config.Name) > JSMaxNameLen { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream name is too long, maximum allowed is %d", JSMaxNameLen)) } if len(config.Description) > JSMaxDescriptionLen { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream description is too long, maximum allowed is %d", JSMaxDescriptionLen)) } var metadataLen int for k, v := range config.Metadata { metadataLen += len(k) + len(v) } if metadataLen > JSMaxMetadataLen { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream metadata exceeds maximum size of %d bytes", JSMaxMetadataLen)) } cfg := *config // Make file the default. if cfg.Storage == 0 { cfg.Storage = FileStorage } if cfg.Replicas == 0 { cfg.Replicas = 1 } if cfg.Replicas > StreamMaxReplicas { return cfg, NewJSStreamInvalidConfigError(fmt.Errorf("maximum replicas is %d", StreamMaxReplicas)) } if cfg.Replicas < 0 { return cfg, NewJSReplicasCountCannotBeNegativeError() } if cfg.MaxMsgs == 0 { cfg.MaxMsgs = -1 } if cfg.MaxMsgsPer == 0 { cfg.MaxMsgsPer = -1 } if cfg.MaxBytes == 0 { cfg.MaxBytes = -1 } if cfg.MaxMsgSize == 0 { cfg.MaxMsgSize = -1 } if cfg.MaxConsumers == 0 { cfg.MaxConsumers = -1 } if cfg.Duplicates == 0 && cfg.Mirror == nil { maxWindow := StreamDefaultDuplicatesWindow if lim.Duplicates > 0 && maxWindow > lim.Duplicates { maxWindow = lim.Duplicates } if cfg.MaxAge != 0 && cfg.MaxAge < maxWindow { cfg.Duplicates = cfg.MaxAge } else { cfg.Duplicates = maxWindow } } if cfg.MaxAge > 0 && cfg.MaxAge < 100*time.Millisecond { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("max age needs to be >= 100ms")) } if cfg.Duplicates < 0 { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("duplicates window can not be negative")) } // Check that duplicates is not larger then age if set. if cfg.MaxAge != 0 && cfg.Duplicates > cfg.MaxAge { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("duplicates window can not be larger then max age")) } if lim.Duplicates > 0 && cfg.Duplicates > lim.Duplicates { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("duplicates window can not be larger then server limit of %v", lim.Duplicates.String())) } if cfg.Duplicates > 0 && cfg.Duplicates < 100*time.Millisecond { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("duplicates window needs to be >= 100ms")) } if cfg.DenyPurge && cfg.AllowRollup { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("roll-ups require the purge permission")) } // Check for new discard new per subject, we require the discard policy to also be new. if cfg.DiscardNewPer { if cfg.Discard != DiscardNew { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("discard new per subject requires discard new policy to be set")) } if cfg.MaxMsgsPer <= 0 { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("discard new per subject requires max msgs per subject > 0")) } } getStream := func(streamName string) (bool, StreamConfig) { var exists bool var cfg StreamConfig if s.JetStreamIsClustered() { if js, _ := s.getJetStreamCluster(); js != nil { js.mu.RLock() if sa := js.streamAssignment(acc.Name, streamName); sa != nil { cfg = *sa.Config exists = true } js.mu.RUnlock() } } else if mset, err := acc.lookupStream(streamName); err == nil { cfg = mset.cfg exists = true } return exists, cfg } hasStream := func(streamName string) (bool, int32, []string) { exists, cfg := getStream(streamName) return exists, cfg.MaxMsgSize, cfg.Subjects } var streamSubs []string var deliveryPrefixes []string var apiPrefixes []string // Do some pre-checking for mirror config to avoid cycles in clustered mode. if cfg.Mirror != nil { if cfg.FirstSeq > 0 { return StreamConfig{}, NewJSMirrorWithFirstSeqError() } if len(cfg.Subjects) > 0 { return StreamConfig{}, NewJSMirrorWithSubjectsError() } if len(cfg.Sources) > 0 { return StreamConfig{}, NewJSMirrorWithSourcesError() } if cfg.Mirror.FilterSubject != _EMPTY_ && len(cfg.Mirror.SubjectTransforms) != 0 { return StreamConfig{}, NewJSMirrorMultipleFiltersNotAllowedError() } // Check subject filters overlap. for outer, tr := range cfg.Mirror.SubjectTransforms { if !IsValidSubject(tr.Source) { return StreamConfig{}, NewJSMirrorInvalidSubjectFilterError() } for inner, innertr := range cfg.Mirror.SubjectTransforms { if inner != outer && SubjectsCollide(tr.Source, innertr.Source) { return StreamConfig{}, NewJSMirrorOverlappingSubjectFiltersError() } } } // Do not perform checks if External is provided, as it could lead to // checking against itself (if sourced stream name is the same on different JetStream) if cfg.Mirror.External == nil { if !isValidName(cfg.Mirror.Name) { return StreamConfig{}, NewJSMirrorInvalidStreamNameError() } // We do not require other stream to exist anymore, but if we can see it check payloads. exists, maxMsgSize, subs := hasStream(cfg.Mirror.Name) if len(subs) > 0 { streamSubs = append(streamSubs, subs...) } if exists { if cfg.MaxMsgSize > 0 && maxMsgSize > 0 && cfg.MaxMsgSize < maxMsgSize { return StreamConfig{}, NewJSMirrorMaxMessageSizeTooBigError() } } // Determine if we are inheriting direct gets. if exists, ocfg := getStream(cfg.Mirror.Name); exists { cfg.MirrorDirect = ocfg.AllowDirect } else if js := s.getJetStream(); js != nil && js.isClustered() { // Could not find it here. If we are clustered we can look it up. js.mu.RLock() if cc := js.cluster; cc != nil { if as := cc.streams[acc.Name]; as != nil { if sa := as[cfg.Mirror.Name]; sa != nil { cfg.MirrorDirect = sa.Config.AllowDirect } } } js.mu.RUnlock() } } else { if cfg.Mirror.External.DeliverPrefix != _EMPTY_ { deliveryPrefixes = append(deliveryPrefixes, cfg.Mirror.External.DeliverPrefix) } if cfg.Mirror.External.ApiPrefix != _EMPTY_ { apiPrefixes = append(apiPrefixes, cfg.Mirror.External.ApiPrefix) } } } // check for duplicates var iNames = make(map[string]struct{}) for _, src := range cfg.Sources { if !isValidName(src.Name) { return StreamConfig{}, NewJSSourceInvalidStreamNameError() } if _, ok := iNames[src.composeIName()]; !ok { iNames[src.composeIName()] = struct{}{} } else { return StreamConfig{}, NewJSSourceDuplicateDetectedError() } // Do not perform checks if External is provided, as it could lead to // checking against itself (if sourced stream name is the same on different JetStream) if src.External == nil { exists, maxMsgSize, subs := hasStream(src.Name) if len(subs) > 0 { streamSubs = append(streamSubs, subs...) } if exists { if cfg.MaxMsgSize > 0 && maxMsgSize > 0 && cfg.MaxMsgSize < maxMsgSize { return StreamConfig{}, NewJSSourceMaxMessageSizeTooBigError() } } if src.FilterSubject != _EMPTY_ && len(src.SubjectTransforms) != 0 { return StreamConfig{}, NewJSSourceMultipleFiltersNotAllowedError() } for _, tr := range src.SubjectTransforms { err := ValidateMappingDestination(tr.Destination) if err != nil { return StreamConfig{}, NewJSSourceInvalidTransformDestinationError() } } // Check subject filters overlap. for outer, tr := range src.SubjectTransforms { if !IsValidSubject(tr.Source) { return StreamConfig{}, NewJSSourceInvalidSubjectFilterError() } for inner, innertr := range src.SubjectTransforms { if inner != outer && subjectIsSubsetMatch(tr.Source, innertr.Source) { return StreamConfig{}, NewJSSourceOverlappingSubjectFiltersError() } } } continue } else { if src.External.DeliverPrefix != _EMPTY_ { deliveryPrefixes = append(deliveryPrefixes, src.External.DeliverPrefix) } if src.External.ApiPrefix != _EMPTY_ { apiPrefixes = append(apiPrefixes, src.External.ApiPrefix) } } } // check prefix overlap with subjects for _, pfx := range deliveryPrefixes { if !IsValidPublishSubject(pfx) { return StreamConfig{}, NewJSStreamInvalidExternalDeliverySubjError(pfx) } for _, sub := range streamSubs { if SubjectsCollide(sub, fmt.Sprintf("%s.%s", pfx, sub)) { return StreamConfig{}, NewJSStreamExternalDelPrefixOverlapsError(pfx, sub) } } } // check if api prefixes overlap for _, apiPfx := range apiPrefixes { if !IsValidPublishSubject(apiPfx) { return StreamConfig{}, NewJSStreamInvalidConfigError( fmt.Errorf("stream external api prefix %q must be a valid subject without wildcards", apiPfx)) } if SubjectsCollide(apiPfx, JSApiPrefix) { return StreamConfig{}, NewJSStreamExternalApiOverlapError(apiPfx, JSApiPrefix) } } // cycle check for source cycle toVisit := []*StreamConfig{&cfg} visited := make(map[string]struct{}) overlaps := func(subjects []string, filter string) bool { if filter == _EMPTY_ { return true } for _, subject := range subjects { if SubjectsCollide(subject, filter) { return true } } return false } for len(toVisit) > 0 { cfg := toVisit[0] toVisit = toVisit[1:] visited[cfg.Name] = struct{}{} for _, src := range cfg.Sources { if src.External != nil { continue } // We can detect a cycle between streams, but let's double check that the // subjects actually form a cycle. if _, ok := visited[src.Name]; ok { if overlaps(cfg.Subjects, src.FilterSubject) { return StreamConfig{}, NewJSStreamInvalidConfigError(errors.New("detected cycle")) } } else if exists, cfg := getStream(src.Name); exists { toVisit = append(toVisit, &cfg) } } // Avoid cycles hiding behind mirrors if m := cfg.Mirror; m != nil { if m.External == nil { if _, ok := visited[m.Name]; ok { return StreamConfig{}, NewJSStreamInvalidConfigError(errors.New("detected cycle")) } if exists, cfg := getStream(m.Name); exists { toVisit = append(toVisit, &cfg) } } } } if len(cfg.Subjects) == 0 { if cfg.Mirror == nil && len(cfg.Sources) == 0 { cfg.Subjects = append(cfg.Subjects, cfg.Name) } } else { if cfg.Mirror != nil { return StreamConfig{}, NewJSMirrorWithSubjectsError() } // Check for literal duplication of subject interest in config // and no overlap with any JS or SYS API subject space. dset := make(map[string]struct{}, len(cfg.Subjects)) for _, subj := range cfg.Subjects { // Make sure the subject is valid. Check this first. if !IsValidSubject(subj) { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("invalid subject")) } if _, ok := dset[subj]; ok { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("duplicate subjects detected")) } // Check for trying to capture everything. if subj == fwcs { if !cfg.NoAck { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("capturing all subjects requires no-ack to be true")) } // Capturing everything also will require R1. if cfg.Replicas != 1 { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("capturing all subjects requires replicas of 1")) } } // Also check to make sure we do not overlap with our $JS API subjects. if !cfg.NoAck && (subjectIsSubsetMatch(subj, "$JS.>") || subjectIsSubsetMatch(subj, "$JSC.>")) { // We allow an exception for $JS.EVENT.> since these could have been created in the past. if !subjectIsSubsetMatch(subj, "$JS.EVENT.>") { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("subjects that overlap with jetstream api require no-ack to be true")) } } // And the $SYS subjects. if !cfg.NoAck && subjectIsSubsetMatch(subj, "$SYS.>") { if !subjectIsSubsetMatch(subj, "$SYS.ACCOUNT.>") { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("subjects that overlap with system api require no-ack to be true")) } } // Mark for duplicate check. dset[subj] = struct{}{} } } if len(cfg.Subjects) == 0 && len(cfg.Sources) == 0 && cfg.Mirror == nil { return StreamConfig{}, NewJSStreamInvalidConfigError( fmt.Errorf("stream needs at least one configured subject or be a source/mirror")) } // Check for MaxBytes required and it's limit if required, limit := acc.maxBytesLimits(&cfg); required && cfg.MaxBytes <= 0 { return StreamConfig{}, NewJSStreamMaxBytesRequiredError() } else if limit > 0 && cfg.MaxBytes > limit { return StreamConfig{}, NewJSStreamMaxStreamBytesExceededError() } // Now check if we have multiple subjects they we do not overlap ourselves // which would cause duplicate entries (assuming no MsgID). if len(cfg.Subjects) > 1 { for _, subj := range cfg.Subjects { for _, tsubj := range cfg.Subjects { if tsubj != subj && SubjectsCollide(tsubj, subj) { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("subject %q overlaps with %q", subj, tsubj)) } } } } // If we have a republish directive check if we can create a transform here. if cfg.RePublish != nil { // Check to make sure source is a valid subset of the subjects we have. // Also make sure it does not form a cycle. // Empty same as all. if cfg.RePublish.Source == _EMPTY_ { cfg.RePublish.Source = fwcs } var formsCycle bool for _, subj := range cfg.Subjects { if SubjectsCollide(cfg.RePublish.Destination, subj) { formsCycle = true break } } if formsCycle { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration for republish destination forms a cycle")) } if _, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination); err != nil { return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration for republish with transform from '%s' to '%s' not valid", cfg.RePublish.Source, cfg.RePublish.Destination)) } } return cfg, nil } // Config returns the stream's configuration. func (mset *stream) config() StreamConfig { mset.cfgMu.RLock() defer mset.cfgMu.RUnlock() return mset.cfg } func (mset *stream) fileStoreConfig() (FileStoreConfig, error) { mset.mu.Lock() defer mset.mu.Unlock() fs, ok := mset.store.(*fileStore) if !ok { return FileStoreConfig{}, ErrStoreWrongType } return fs.fileStoreConfig(), nil } // Do not hold jsAccount or jetStream lock func (jsa *jsAccount) configUpdateCheck(old, new *StreamConfig, s *Server) (*StreamConfig, error) { cfg, apiErr := s.checkStreamCfg(new, jsa.acc()) if apiErr != nil { return nil, apiErr } // Name must match. if cfg.Name != old.Name { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration name must match original")) } // Can't change MaxConsumers for now. if cfg.MaxConsumers != old.MaxConsumers { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not change MaxConsumers")) } // Can't change storage types. if cfg.Storage != old.Storage { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not change storage type")) } // Can only change retention from limits to interest or back, not to/from work queue for now. if cfg.Retention != old.Retention { if old.Retention == WorkQueuePolicy || cfg.Retention == WorkQueuePolicy { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not change retention policy to/from workqueue")) } } // Can not have a template owner for now. if old.Template != _EMPTY_ { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update not allowed on template owned stream")) } if cfg.Template != _EMPTY_ { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not be owned by a template")) } // Can not change from true to false. if !cfg.Sealed && old.Sealed { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not unseal a sealed stream")) } // Can not change from true to false. if !cfg.DenyDelete && old.DenyDelete { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not cancel deny message deletes")) } // Can not change from true to false. if !cfg.DenyPurge && old.DenyPurge { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not cancel deny purge")) } // Check for mirror changes which are not allowed. if !reflect.DeepEqual(cfg.Mirror, old.Mirror) { return nil, NewJSStreamMirrorNotUpdatableError() } // Check on new discard new per subject. if cfg.DiscardNewPer { if cfg.Discard != DiscardNew { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("discard new per subject requires discard new policy to be set")) } if cfg.MaxMsgsPer <= 0 { return nil, NewJSStreamInvalidConfigError(fmt.Errorf("discard new per subject requires max msgs per subject > 0")) } } // Do some adjustments for being sealed. if cfg.Sealed { cfg.MaxAge = 0 cfg.Discard = DiscardNew cfg.DenyDelete, cfg.DenyPurge = true, true cfg.AllowRollup = false } // Check limits. We need some extra handling to allow updating MaxBytes. // First, let's calculate the difference between the new and old MaxBytes. maxBytesDiff := cfg.MaxBytes - old.MaxBytes if maxBytesDiff < 0 { // If we're updating to a lower MaxBytes (maxBytesDiff is negative), // then set to zero so checkBytesLimits doesn't set addBytes to 1. maxBytesDiff = 0 } // If maxBytesDiff == 0, then that means MaxBytes didn't change. // If maxBytesDiff > 0, then we want to reserve additional bytes. // Save the user configured MaxBytes. newMaxBytes := cfg.MaxBytes maxBytesOffset := int64(0) // We temporarily set cfg.MaxBytes to maxBytesDiff because checkAllLimits // adds cfg.MaxBytes to the current reserved limit and checks if we've gone // over. However, we don't want an addition cfg.MaxBytes, we only want to // reserve the difference between the new and the old values. cfg.MaxBytes = maxBytesDiff // Check limits. js, isClustered := jsa.jetStreamAndClustered() jsa.mu.RLock() acc := jsa.account jsa.usageMu.RLock() selected, tier, hasTier := jsa.selectLimits(cfg.Replicas) if !hasTier && old.Replicas != cfg.Replicas { selected, tier, hasTier = jsa.selectLimits(old.Replicas) } jsa.usageMu.RUnlock() reserved := int64(0) if !isClustered { reserved = jsa.tieredReservation(tier, &cfg) } jsa.mu.RUnlock() if !hasTier { return nil, NewJSNoLimitsError() } js.mu.RLock() defer js.mu.RUnlock() if isClustered { _, reserved = tieredStreamAndReservationCount(js.cluster.streams[acc.Name], tier, &cfg) } // reservation does not account for this stream, hence add the old value if tier == _EMPTY_ && old.Replicas > 1 { reserved += old.MaxBytes * int64(old.Replicas) } else { reserved += old.MaxBytes } if err := js.checkAllLimits(&selected, &cfg, reserved, maxBytesOffset); err != nil { return nil, err } // Restore the user configured MaxBytes. cfg.MaxBytes = newMaxBytes return &cfg, nil } // Update will allow certain configuration properties of an existing stream to be updated. func (mset *stream) update(config *StreamConfig) error { return mset.updateWithAdvisory(config, true) } // Update will allow certain configuration properties of an existing stream to be updated. func (mset *stream) updateWithAdvisory(config *StreamConfig, sendAdvisory bool) error { _, jsa, err := mset.acc.checkForJetStream() if err != nil { return err } mset.mu.RLock() ocfg := mset.cfg s := mset.srv mset.mu.RUnlock() cfg, err := mset.jsa.configUpdateCheck(&ocfg, config, s) if err != nil { return NewJSStreamInvalidConfigError(err, Unless(err)) } // In the event that some of the stream-level limits have changed, yell appropriately // if any of the consumers exceed that limit. updateLimits := ocfg.ConsumerLimits.InactiveThreshold != cfg.ConsumerLimits.InactiveThreshold || ocfg.ConsumerLimits.MaxAckPending != cfg.ConsumerLimits.MaxAckPending if updateLimits { var errorConsumers []string consumers := map[string]*ConsumerConfig{} if mset.js.isClustered() { for _, c := range mset.sa.consumers { consumers[c.Name] = c.Config } } else { for _, c := range mset.consumers { consumers[c.name] = &c.cfg } } for name, ccfg := range consumers { if ccfg.InactiveThreshold > cfg.ConsumerLimits.InactiveThreshold || ccfg.MaxAckPending > cfg.ConsumerLimits.MaxAckPending { errorConsumers = append(errorConsumers, name) } } if len(errorConsumers) > 0 { // TODO(nat): Return a parsable error so that we can surface something // sensible through the JS API. return fmt.Errorf("change to limits violates consumers: %s", strings.Join(errorConsumers, ", ")) } } jsa.mu.RLock() if jsa.subjectsOverlap(cfg.Subjects, mset) { jsa.mu.RUnlock() return NewJSStreamSubjectOverlapError() } jsa.mu.RUnlock() mset.mu.Lock() if mset.isLeader() { // Now check for subject interest differences. current := make(map[string]struct{}, len(ocfg.Subjects)) for _, s := range ocfg.Subjects { current[s] = struct{}{} } // Update config with new values. The store update will enforce any stricter limits. // Now walk new subjects. All of these need to be added, but we will check // the originals first, since if it is in there we can skip, already added. for _, s := range cfg.Subjects { if _, ok := current[s]; !ok { if _, err := mset.subscribeInternal(s, mset.processInboundJetStreamMsg); err != nil { mset.mu.Unlock() return err } } delete(current, s) } // What is left in current needs to be deleted. for s := range current { if err := mset.unsubscribeInternal(s); err != nil { mset.mu.Unlock() return err } } // Check for the Duplicates if cfg.Duplicates != ocfg.Duplicates && mset.ddtmr != nil { // Let it fire right away, it will adjust properly on purge. mset.ddtmr.Reset(time.Microsecond) } // Check for Sources. if len(cfg.Sources) > 0 || len(ocfg.Sources) > 0 { currentIName := make(map[string]struct{}) needsStartingSeqNum := make(map[string]struct{}) for _, s := range ocfg.Sources { currentIName[s.iname] = struct{}{} } for _, s := range cfg.Sources { s.setIndexName() if _, ok := currentIName[s.iname]; !ok { // new source if mset.sources == nil { mset.sources = make(map[string]*sourceInfo) } mset.cfg.Sources = append(mset.cfg.Sources, s) var si *sourceInfo if len(s.SubjectTransforms) == 0 { si = &sourceInfo{name: s.Name, iname: s.iname, sf: s.FilterSubject} } else { si = &sourceInfo{name: s.Name, iname: s.iname} si.trs = make([]*subjectTransform, len(s.SubjectTransforms)) si.sfs = make([]string, len(s.SubjectTransforms)) for i := range s.SubjectTransforms { // err can be ignored as already validated in config check si.sfs[i] = s.SubjectTransforms[i].Source var err error si.trs[i], err = NewSubjectTransform(s.SubjectTransforms[i].Source, s.SubjectTransforms[i].Destination) if err != nil { mset.mu.Unlock() return fmt.Errorf("unable to get subject transform for source: %v", err) } } } mset.sources[s.iname] = si needsStartingSeqNum[s.iname] = struct{}{} } else { // source already exists delete(currentIName, s.iname) } } // What is left in currentIName needs to be deleted. for iName := range currentIName { mset.cancelSourceConsumer(iName) delete(mset.sources, iName) } neededCopy := make(map[string]struct{}, len(needsStartingSeqNum)) for iName := range needsStartingSeqNum { neededCopy[iName] = struct{}{} } mset.setStartingSequenceForSources(needsStartingSeqNum) for iName := range neededCopy { mset.setupSourceConsumer(iName, mset.sources[iName].sseq+1, time.Time{}) } } } // Check for a change in allow direct status. // These will run on all members, so just update as appropriate here. // We do make sure we are caught up under monitorStream() during initial startup. if cfg.AllowDirect != ocfg.AllowDirect { if cfg.AllowDirect { mset.subscribeToDirect() } else { mset.unsubscribeToDirect() } } // Check for changes to RePublish. if cfg.RePublish != nil { // Empty same as all. if cfg.RePublish.Source == _EMPTY_ { cfg.RePublish.Source = fwcs } if cfg.RePublish.Destination == _EMPTY_ { cfg.RePublish.Destination = fwcs } tr, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination) if err != nil { mset.mu.Unlock() return fmt.Errorf("stream configuration for republish from '%s' to '%s': %w", cfg.RePublish.Source, cfg.RePublish.Destination, err) } // Assign our transform for republishing. mset.tr = tr } else { mset.tr = nil } // Check for changes to subject transform if ocfg.SubjectTransform == nil && cfg.SubjectTransform != nil { tr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination) if err != nil { mset.mu.Unlock() return fmt.Errorf("stream configuration for subject transform from '%s' to '%s': %w", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err) } mset.itr = tr } else if ocfg.SubjectTransform != nil && cfg.SubjectTransform != nil && (ocfg.SubjectTransform.Source != cfg.SubjectTransform.Source || ocfg.SubjectTransform.Destination != cfg.SubjectTransform.Destination) { tr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination) if err != nil { mset.mu.Unlock() return fmt.Errorf("stream configuration for subject transform from '%s' to '%s': %w", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err) } mset.itr = tr } else if ocfg.SubjectTransform != nil && cfg.SubjectTransform == nil { mset.itr = nil } js := mset.js if targetTier := tierName(cfg.Replicas); mset.tier != targetTier { // In cases such as R1->R3, only one update is needed jsa.usageMu.RLock() _, ok := jsa.limits[targetTier] jsa.usageMu.RUnlock() if ok { // error never set _, reported, _ := mset.store.Utilization() jsa.updateUsage(mset.tier, mset.stype, -int64(reported)) jsa.updateUsage(targetTier, mset.stype, int64(reported)) mset.tier = targetTier } // else in case the new tier does not exist (say on move), keep the old tier around // a subsequent update to an existing tier will then move from existing past tier to existing new tier } if mset.isLeader() && mset.sa != nil && ocfg.Retention != cfg.Retention && cfg.Retention == InterestPolicy { // Before we can update the retention policy for the consumer, we need // the replica count of all consumers to match the stream. for _, c := range mset.sa.consumers { if c.Config.Replicas > 0 && c.Config.Replicas != cfg.Replicas { mset.mu.Unlock() return fmt.Errorf("consumer %q replica count must be %d", c.Name, cfg.Replicas) } } } // Now update config and store's version of our config. // Although we are under the stream write lock, we will also assign the new // configuration under mset.cfgMu lock. This is so that in places where // mset.mu cannot be acquired (like many cases in consumer.go where code // is under the consumer's lock), and the stream's configuration needs to // be inspected, one can use mset.cfgMu's read lock to do that safely. mset.cfgMu.Lock() mset.cfg = *cfg mset.cfgMu.Unlock() // If we're changing retention and haven't errored because of consumer // replicas by now, whip through and update the consumer retention. if ocfg.Retention != cfg.Retention && cfg.Retention == InterestPolicy { toUpdate := make([]*consumer, 0, len(mset.consumers)) for _, c := range mset.consumers { toUpdate = append(toUpdate, c) } var ss StreamState mset.store.FastState(&ss) mset.mu.Unlock() for _, c := range toUpdate { c.mu.Lock() c.retention = cfg.Retention c.mu.Unlock() if c.retention == InterestPolicy { // If we're switching to interest, force a check of the // interest of existing stream messages. c.checkStateForInterestStream(&ss) } } mset.mu.Lock() } // If we are the leader never suppress update advisory, simply send. if mset.isLeader() && sendAdvisory { mset.sendUpdateAdvisoryLocked() } mset.mu.Unlock() if js != nil { maxBytesDiff := cfg.MaxBytes - ocfg.MaxBytes if maxBytesDiff > 0 { // Reserve the difference js.reserveStreamResources(&StreamConfig{ MaxBytes: maxBytesDiff, Storage: cfg.Storage, }) } else if maxBytesDiff < 0 { // Release the difference js.releaseStreamResources(&StreamConfig{ MaxBytes: -maxBytesDiff, Storage: ocfg.Storage, }) } } mset.store.UpdateConfig(cfg) return nil } // Small helper to return the Name field from mset.cfg, protected by // the mset.cfgMu mutex. This is simply because we have several places // in consumer.go where we need it. func (mset *stream) getCfgName() string { mset.cfgMu.RLock() defer mset.cfgMu.RUnlock() return mset.cfg.Name } // Purge will remove all messages from the stream and underlying store based on the request. func (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err error) { mset.mu.RLock() if mset.closed.Load() { mset.mu.RUnlock() return 0, errStreamClosed } if mset.cfg.Sealed { mset.mu.RUnlock() return 0, errors.New("sealed stream") } store, mlseq := mset.store, mset.lseq mset.mu.RUnlock() if preq != nil { purged, err = mset.store.PurgeEx(preq.Subject, preq.Sequence, preq.Keep) } else { purged, err = mset.store.Purge() } if err != nil { return purged, err } // Grab our stream state. var state StreamState store.FastState(&state) fseq, lseq := state.FirstSeq, state.LastSeq mset.mu.Lock() // Check if our last has moved past what our original last sequence was, if so reset. if lseq > mlseq { mset.setLastSeq(lseq) } // Clear any pending acks below first seq. mset.clearAllPreAcksBelowFloor(fseq) mset.mu.Unlock() // Purge consumers. // Check for filtered purge. if preq != nil && preq.Subject != _EMPTY_ { ss := store.FilteredState(fseq, preq.Subject) fseq = ss.First } mset.clsMu.RLock() for _, o := range mset.cList { start := fseq o.mu.RLock() // we update consumer sequences if: // no subject was specified, we can purge all consumers sequences doPurge := preq == nil || preq.Subject == _EMPTY_ || // consumer filter subject is equal to purged subject // or consumer filter subject is subset of purged subject, // but not the other way around. o.isEqualOrSubsetMatch(preq.Subject) // Check if a consumer has a wider subject space then what we purged var isWider bool if !doPurge && preq != nil && o.isFilteredMatch(preq.Subject) { doPurge, isWider = true, true start = state.FirstSeq } o.mu.RUnlock() if doPurge { o.purge(start, lseq, isWider) } } mset.clsMu.RUnlock() return purged, nil } // RemoveMsg will remove a message from a stream. // FIXME(dlc) - Should pick one and be consistent. func (mset *stream) removeMsg(seq uint64) (bool, error) { return mset.deleteMsg(seq) } // DeleteMsg will remove a message from a stream. func (mset *stream) deleteMsg(seq uint64) (bool, error) { if mset.closed.Load() { return false, errStreamClosed } removed, err := mset.store.RemoveMsg(seq) if err != nil { return removed, err } mset.mu.Lock() mset.clearAllPreAcks(seq) mset.mu.Unlock() return removed, err } // EraseMsg will securely remove a message and rewrite the data with random data. func (mset *stream) eraseMsg(seq uint64) (bool, error) { if mset.closed.Load() { return false, errStreamClosed } removed, err := mset.store.EraseMsg(seq) if err != nil { return removed, err } mset.mu.Lock() mset.clearAllPreAcks(seq) mset.mu.Unlock() return removed, err } // Are we a mirror? func (mset *stream) isMirror() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.cfg.Mirror != nil } func (mset *stream) sourcesInfo() (sis []*StreamSourceInfo) { mset.mu.RLock() defer mset.mu.RUnlock() for _, si := range mset.sources { sis = append(sis, mset.sourceInfo(si)) } return sis } // Lock should be held func (mset *stream) sourceInfo(si *sourceInfo) *StreamSourceInfo { if si == nil { return nil } var ssi = StreamSourceInfo{Name: si.name, Lag: si.lag, Error: si.err, FilterSubject: si.sf} trConfigs := make([]SubjectTransformConfig, len(si.sfs)) for i := range si.sfs { destination := _EMPTY_ if si.trs[i] != nil { destination = si.trs[i].dest } trConfigs[i] = SubjectTransformConfig{si.sfs[i], destination} } ssi.SubjectTransforms = trConfigs // If we have not heard from the source, set Active to -1. if last := si.last.Load(); last == 0 { ssi.Active = -1 } else { ssi.Active = time.Since(time.Unix(0, last)) } var ext *ExternalStream if mset.cfg.Mirror != nil { ext = mset.cfg.Mirror.External } else if ss := mset.streamSource(si.iname); ss != nil && ss.External != nil { ext = ss.External } if ext != nil { ssi.External = &ExternalStream{ ApiPrefix: ext.ApiPrefix, DeliverPrefix: ext.DeliverPrefix, } } return &ssi } // Return our source info for our mirror. func (mset *stream) mirrorInfo() *StreamSourceInfo { mset.mu.RLock() defer mset.mu.RUnlock() return mset.sourceInfo(mset.mirror) } const ( // Our consumer HB interval. sourceHealthHB = 1 * time.Second // How often we check and our stalled interval. sourceHealthCheckInterval = 10 * time.Second ) // Will run as a Go routine to process mirror consumer messages. func (mset *stream) processMirrorMsgs(mirror *sourceInfo, ready *sync.WaitGroup) { s := mset.srv defer func() { mirror.wg.Done() s.grWG.Done() }() // Grab stream quit channel. mset.mu.Lock() msgs, qch, siqch := mirror.msgs, mset.qch, mirror.qch // Set the last seen as now so that we don't fail at the first check. mirror.last.Store(time.Now().UnixNano()) mset.mu.Unlock() // Signal the caller that we have captured the above fields. ready.Done() // Make sure we have valid ipq for msgs. if msgs == nil { mset.mu.Lock() mset.cancelMirrorConsumer() mset.mu.Unlock() return } t := time.NewTicker(sourceHealthCheckInterval) defer t.Stop() for { select { case <-s.quitCh: return case <-qch: return case <-siqch: return case <-msgs.ch: ims := msgs.pop() for _, im := range ims { if !mset.processInboundMirrorMsg(im) { break } im.returnToPool() } msgs.recycle(&ims) case <-t.C: mset.mu.RLock() var stalled bool if mset.mirror != nil { stalled = time.Since(time.Unix(0, mset.mirror.last.Load())) > sourceHealthCheckInterval } isLeader := mset.isLeader() mset.mu.RUnlock() // No longer leader. if !isLeader { mset.mu.Lock() mset.cancelMirrorConsumer() mset.mu.Unlock() return } // We are stalled. if stalled { mset.retryMirrorConsumer() } } } } // Checks that the message is from our current direct consumer. We can not depend on sub comparison // since cross account imports break. func (si *sourceInfo) isCurrentSub(reply string) bool { return si.cname != _EMPTY_ && strings.HasPrefix(reply, jsAckPre) && si.cname == tokenAt(reply, 4) } // processInboundMirrorMsg handles processing messages bound for a stream. func (mset *stream) processInboundMirrorMsg(m *inMsg) bool { mset.mu.Lock() if mset.mirror == nil { mset.mu.Unlock() return false } if !mset.isLeader() { mset.cancelMirrorConsumer() mset.mu.Unlock() return false } isControl := m.isControlMsg() // Ignore from old subscriptions. // The reason we can not just compare subs is that on cross account imports they will not match. if !mset.mirror.isCurrentSub(m.rply) && !isControl { mset.mu.Unlock() return false } // Check for heartbeats and flow control messages. if isControl { var needsRetry bool // Flow controls have reply subjects. if m.rply != _EMPTY_ { mset.handleFlowControl(m) } else { // For idle heartbeats make sure we did not miss anything and check if we are considered stalled. if ldseq := parseInt64(getHeader(JSLastConsumerSeq, m.hdr)); ldseq > 0 && uint64(ldseq) != mset.mirror.dseq { needsRetry = true } else if fcReply := getHeader(JSConsumerStalled, m.hdr); len(fcReply) > 0 { // Other side thinks we are stalled, so send flow control reply. mset.outq.sendMsg(string(fcReply), nil) } } mset.mu.Unlock() if needsRetry { mset.retryMirrorConsumer() } return !needsRetry } sseq, dseq, dc, ts, pending := replyInfo(m.rply) if dc > 1 { mset.mu.Unlock() return false } // Mirror info tracking. olag, osseq, odseq := mset.mirror.lag, mset.mirror.sseq, mset.mirror.dseq if sseq == mset.mirror.sseq+1 { mset.mirror.dseq = dseq mset.mirror.sseq++ } else if sseq <= mset.mirror.sseq { // Ignore older messages. mset.mu.Unlock() return true } else if mset.mirror.cname == _EMPTY_ { mset.mirror.cname = tokenAt(m.rply, 4) mset.mirror.dseq, mset.mirror.sseq = dseq, sseq } else { // If the deliver sequence matches then the upstream stream has expired or deleted messages. if dseq == mset.mirror.dseq+1 { mset.skipMsgs(mset.mirror.sseq+1, sseq-1) mset.mirror.dseq++ mset.mirror.sseq = sseq } else { mset.mu.Unlock() mset.retryMirrorConsumer() return false } } if pending == 0 { mset.mirror.lag = 0 } else { mset.mirror.lag = pending - 1 } // Check if we allow mirror direct here. If so check they we have mostly caught up. // The reason we do not require 0 is if the source is active we may always be slightly behind. if mset.cfg.MirrorDirect && mset.mirror.dsub == nil && pending < dgetCaughtUpThresh { if err := mset.subscribeToMirrorDirect(); err != nil { // Disable since we had problems above. mset.cfg.MirrorDirect = false } } // Do the subject transform if there's one if len(mset.mirror.trs) > 0 { for _, tr := range mset.mirror.trs { if tr == nil { continue } else { tsubj, err := tr.Match(m.subj) if err == nil { m.subj = tsubj break } } } } s, js, stype := mset.srv, mset.js, mset.cfg.Storage node := mset.node mset.mu.Unlock() var err error if node != nil { if js.limitsExceeded(stype) { s.resourcesExceededError() err = ApiErrors[JSInsufficientResourcesErr] } else { err = node.Propose(encodeStreamMsg(m.subj, _EMPTY_, m.hdr, m.msg, sseq-1, ts)) } } else { err = mset.processJetStreamMsg(m.subj, _EMPTY_, m.hdr, m.msg, sseq-1, ts) } if err != nil { if strings.Contains(err.Error(), "no space left") { s.Errorf("JetStream out of space, will be DISABLED") s.DisableJetStream() return false } if err != errLastSeqMismatch { mset.mu.RLock() accName, sname := mset.acc.Name, mset.cfg.Name mset.mu.RUnlock() s.RateLimitWarnf("Error processing inbound mirror message for '%s' > '%s': %v", accName, sname, err) } else { // We may have missed messages, restart. if sseq <= mset.lastSeq() { mset.mu.Lock() mset.mirror.lag = olag mset.mirror.sseq = osseq mset.mirror.dseq = odseq mset.mu.Unlock() return false } else { mset.mu.Lock() mset.mirror.dseq = odseq mset.mirror.sseq = osseq mset.mu.Unlock() mset.retryMirrorConsumer() } } } return err == nil } func (mset *stream) setMirrorErr(err *ApiError) { mset.mu.Lock() if mset.mirror != nil { mset.mirror.err = err } mset.mu.Unlock() } // Cancels a mirror consumer. // // Lock held on entry func (mset *stream) cancelMirrorConsumer() { if mset.mirror == nil { return } mset.cancelSourceInfo(mset.mirror) } // Similar to setupMirrorConsumer except that it will print a debug statement // indicating that there is a retry. // // Lock is acquired in this function func (mset *stream) retryMirrorConsumer() error { mset.mu.Lock() defer mset.mu.Unlock() mset.srv.Debugf("Retrying mirror consumer for '%s > %s'", mset.acc.Name, mset.cfg.Name) mset.cancelMirrorConsumer() return mset.setupMirrorConsumer() } // Lock should be held. func (mset *stream) skipMsgs(start, end uint64) { node, store := mset.node, mset.store // If we are not clustered we can short circuit now with store.SkipMsgs if node == nil { store.SkipMsgs(start, end-start+1) mset.lseq = end return } // FIXME (dlc) - We should allow proposals of DeleteRange, but would need to make sure all peers support. // With syncRequest was easy to add bool into request. var entries []*Entry for seq := start; seq <= end; seq++ { entries = append(entries, newEntry(EntryNormal, encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, seq-1, 0))) // So a single message does not get too big. if len(entries) > 10_000 { node.ProposeMulti(entries) // We need to re-create `entries` because there is a reference // to it in the node's pae map. entries = entries[:0] } } // Send all at once. if len(entries) > 0 { node.ProposeMulti(entries) } } const ( // Base retry backoff duration. retryBackOff = 5 * time.Second // Maximum amount we will wait. retryMaximum = 2 * time.Minute ) // Calculate our backoff based on number of failures. func calculateRetryBackoff(fails int) time.Duration { backoff := time.Duration(retryBackOff) * time.Duration(fails*2) if backoff > retryMaximum { backoff = retryMaximum } return backoff } // This will schedule a call to setupMirrorConsumer, taking into account the last // time it was retried and determine the soonest setupMirrorConsumer can be called // without tripping the sourceConsumerRetryThreshold. We will also take into account // number of failures and will back off our retries. // The mset.mirror pointer has been verified to be not nil by the caller. // // Lock held on entry func (mset *stream) scheduleSetupMirrorConsumerRetry() { // We are trying to figure out how soon we can retry. setupMirrorConsumer will reject // a retry if last was done less than "sourceConsumerRetryThreshold" ago. next := sourceConsumerRetryThreshold - time.Since(mset.mirror.lreq) if next < 0 { // It means that we have passed the threshold and so we are ready to go. next = 0 } // Take into account failures here. next += calculateRetryBackoff(mset.mirror.fails) // Add some jitter. next += time.Duration(rand.Intn(int(100*time.Millisecond))) + 100*time.Millisecond time.AfterFunc(next, func() { mset.mu.Lock() mset.setupMirrorConsumer() mset.mu.Unlock() }) } // How long we wait for a response from a consumer create request for a source or mirror. var srcConsumerWaitTime = 30 * time.Second // Setup our mirror consumer. // Lock should be held. func (mset *stream) setupMirrorConsumer() error { if mset.closed.Load() { return errStreamClosed } if mset.outq == nil { return errors.New("outq required") } // We use to prevent update of a mirror configuration in cluster // mode but not in standalone. This is now fixed. However, without // rejecting the update, it could be that if the source stream was // removed and then later the mirrored stream config changed to // remove mirror configuration, this function would panic when // accessing mset.cfg.Mirror fields. Adding this protection in case // we allow in the future the mirror config to be changed (removed). if mset.cfg.Mirror == nil { return errors.New("invalid mirror configuration") } // If this is the first time if mset.mirror == nil { mset.mirror = &sourceInfo{name: mset.cfg.Mirror.Name} } else { mset.cancelSourceInfo(mset.mirror) mset.mirror.sseq = mset.lseq // If we are no longer the leader stop trying. if !mset.isLeader() { return nil } } mirror := mset.mirror // We want to throttle here in terms of how fast we request new consumers, // or if the previous is still in progress. if last := time.Since(mirror.lreq); last < sourceConsumerRetryThreshold || mirror.sip { mset.scheduleSetupMirrorConsumerRetry() return nil } mirror.lreq = time.Now() // Determine subjects etc. var deliverSubject string ext := mset.cfg.Mirror.External if ext != nil && ext.DeliverPrefix != _EMPTY_ { deliverSubject = strings.ReplaceAll(ext.DeliverPrefix+syncSubject(".M"), "..", ".") } else { deliverSubject = syncSubject("$JS.M") } // Now send off request to create/update our consumer. This will be all API based even in single server mode. // We calculate durable names apriori so we do not need to save them off. var state StreamState mset.store.FastState(&state) req := &CreateConsumerRequest{ Stream: mset.cfg.Mirror.Name, Config: ConsumerConfig{ DeliverSubject: deliverSubject, DeliverPolicy: DeliverByStartSequence, OptStartSeq: state.LastSeq + 1, AckPolicy: AckNone, AckWait: 22 * time.Hour, MaxDeliver: 1, Heartbeat: sourceHealthHB, FlowControl: true, Direct: true, InactiveThreshold: sourceHealthCheckInterval, }, } // Only use start optionals on first time. if state.Msgs == 0 && state.FirstSeq == 0 { req.Config.OptStartSeq = 0 if mset.cfg.Mirror.OptStartSeq > 0 { req.Config.OptStartSeq = mset.cfg.Mirror.OptStartSeq } else if mset.cfg.Mirror.OptStartTime != nil { req.Config.OptStartTime = mset.cfg.Mirror.OptStartTime req.Config.DeliverPolicy = DeliverByStartTime } } if req.Config.OptStartSeq == 0 && req.Config.OptStartTime == nil { // If starting out and lastSeq is 0. req.Config.DeliverPolicy = DeliverAll } // Filters if mset.cfg.Mirror.FilterSubject != _EMPTY_ { req.Config.FilterSubject = mset.cfg.Mirror.FilterSubject mirror.sf = mset.cfg.Mirror.FilterSubject } if lst := len(mset.cfg.Mirror.SubjectTransforms); lst > 0 { sfs := make([]string, lst) trs := make([]*subjectTransform, lst) for i, tr := range mset.cfg.Mirror.SubjectTransforms { // will not fail as already checked before that the transform will work subjectTransform, err := NewSubjectTransform(tr.Source, tr.Destination) if err != nil { mset.srv.Errorf("Unable to get transform for mirror consumer: %v", err) } sfs[i] = tr.Source trs[i] = subjectTransform } mirror.sfs = sfs mirror.trs = trs req.Config.FilterSubjects = sfs } respCh := make(chan *JSApiConsumerCreateResponse, 1) reply := infoReplySubject() crSub, err := mset.subscribeInternal(reply, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { mset.unsubscribe(sub) _, msg := c.msgParts(rmsg) var ccr JSApiConsumerCreateResponse if err := json.Unmarshal(msg, &ccr); err != nil { c.Warnf("JetStream bad mirror consumer create response: %q", msg) mset.setMirrorErr(ApiErrors[JSInvalidJSONErr]) return } select { case respCh <- &ccr: default: } }) if err != nil { mirror.err = NewJSMirrorConsumerSetupFailedError(err, Unless(err)) mset.scheduleSetupMirrorConsumerRetry() return nil } b, _ := json.Marshal(req) var subject string if req.Config.FilterSubject != _EMPTY_ { req.Config.Name = fmt.Sprintf("mirror-%s", createConsumerName()) subject = fmt.Sprintf(JSApiConsumerCreateExT, mset.cfg.Mirror.Name, req.Config.Name, req.Config.FilterSubject) } else { subject = fmt.Sprintf(JSApiConsumerCreateT, mset.cfg.Mirror.Name) } if ext != nil { subject = strings.Replace(subject, JSApiPrefix, ext.ApiPrefix, 1) subject = strings.ReplaceAll(subject, "..", ".") } // Reset mirror.msgs = nil mirror.err = nil mirror.sip = true // Send the consumer create request mset.outq.send(newJSPubMsg(subject, _EMPTY_, reply, nil, b, nil, 0)) go func() { var retry bool defer func() { mset.mu.Lock() // Check that this is still valid and if so, clear the "setup in progress" flag. if mset.mirror != nil { mset.mirror.sip = false // If we need to retry, schedule now if retry { mset.mirror.fails++ // Cancel here since we can not do anything with this consumer at this point. mset.cancelSourceInfo(mset.mirror) mset.scheduleSetupMirrorConsumerRetry() } else { // Clear on success. mset.mirror.fails = 0 } } mset.mu.Unlock() }() // Wait for previous processMirrorMsgs go routine to be completely done. // If none is running, this will not block. mirror.wg.Wait() select { case ccr := <-respCh: mset.mu.Lock() // Mirror config has been removed. if mset.mirror == nil { mset.mu.Unlock() return } ready := sync.WaitGroup{} mirror := mset.mirror mirror.err = nil if ccr.Error != nil || ccr.ConsumerInfo == nil { mset.srv.Warnf("JetStream error response for create mirror consumer: %+v", ccr.Error) mirror.err = ccr.Error // Let's retry as soon as possible, but we are gated by sourceConsumerRetryThreshold retry = true mset.mu.Unlock() return } else { // Setup actual subscription to process messages from our source. qname := fmt.Sprintf("[ACC:%s] stream mirror '%s' of '%s' msgs", mset.acc.Name, mset.cfg.Name, mset.cfg.Mirror.Name) // Create a new queue each time mirror.msgs = newIPQueue[*inMsg](mset.srv, qname) msgs := mirror.msgs sub, err := mset.subscribeInternal(deliverSubject, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { hdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy. mset.queueInbound(msgs, subject, reply, hdr, msg, nil) mirror.last.Store(time.Now().UnixNano()) }) if err != nil { mirror.err = NewJSMirrorConsumerSetupFailedError(err, Unless(err)) retry = true mset.mu.Unlock() return } // Save our sub. mirror.sub = sub // When an upstream stream expires messages or in general has messages that we want // that are no longer available we need to adjust here. var state StreamState mset.store.FastState(&state) // Check if we need to skip messages. if state.LastSeq != ccr.ConsumerInfo.Delivered.Stream { // Check to see if delivered is past our last and we have no msgs. This will help the // case when mirroring a stream that has a very high starting sequence number. if state.Msgs == 0 && ccr.ConsumerInfo.Delivered.Stream > state.LastSeq { mset.store.PurgeEx(_EMPTY_, ccr.ConsumerInfo.Delivered.Stream+1, 0) mset.lseq = ccr.ConsumerInfo.Delivered.Stream } else { mset.skipMsgs(state.LastSeq+1, ccr.ConsumerInfo.Delivered.Stream) } } // Capture consumer name. mirror.cname = ccr.ConsumerInfo.Name mirror.dseq = 0 mirror.sseq = ccr.ConsumerInfo.Delivered.Stream mirror.qch = make(chan struct{}) mirror.wg.Add(1) ready.Add(1) if !mset.srv.startGoRoutine( func() { mset.processMirrorMsgs(mirror, &ready) }, pprofLabels{ "type": "mirror", "account": mset.acc.Name, "stream": mset.cfg.Name, "consumer": mirror.cname, }, ) { ready.Done() } } mset.mu.Unlock() ready.Wait() case <-time.After(srcConsumerWaitTime): mset.unsubscribe(crSub) // We already waited 30 seconds, let's retry now. retry = true } }() return nil } func (mset *stream) streamSource(iname string) *StreamSource { for _, ssi := range mset.cfg.Sources { if ssi.iname == iname { return ssi } } return nil } // Lock should be held. func (mset *stream) retrySourceConsumerAtSeq(iName string, seq uint64) { s := mset.srv s.Debugf("Retrying source consumer for '%s > %s'", mset.acc.Name, mset.cfg.Name) // setupSourceConsumer will check that the source is still configured. mset.setupSourceConsumer(iName, seq, time.Time{}) } // Lock should be held. func (mset *stream) cancelSourceConsumer(iname string) { if si := mset.sources[iname]; si != nil { mset.cancelSourceInfo(si) si.sseq, si.dseq = 0, 0 } } // The `si` has been verified to be not nil. The sourceInfo's sub will // be unsubscribed and set to nil (if not already done) and the // cname will be reset. The message processing's go routine quit channel // will be closed if still opened. // // Lock should be held func (mset *stream) cancelSourceInfo(si *sourceInfo) { if si.sub != nil { mset.unsubscribe(si.sub) si.sub = nil } // In case we had a mirror direct subscription. if si.dsub != nil { mset.unsubscribe(si.dsub) si.dsub = nil } mset.removeInternalConsumer(si) if si.qch != nil { close(si.qch) si.qch = nil } if si.msgs != nil { si.msgs.drain() si.msgs.unregister() } // If we have a schedule setup go ahead and delete that. if t := mset.sourceSetupSchedules[si.iname]; t != nil { t.Stop() delete(mset.sourceSetupSchedules, si.iname) } } const sourceConsumerRetryThreshold = 2 * time.Second // This is the main function to call when needing to setup a new consumer for the source. // It actually only does the scheduling of the execution of trySetupSourceConsumer in order to implement retry backoff // and throttle the number of requests. // Lock should be held. func (mset *stream) setupSourceConsumer(iname string, seq uint64, startTime time.Time) { if mset.sourceSetupSchedules == nil { mset.sourceSetupSchedules = map[string]*time.Timer{} } if _, ok := mset.sourceSetupSchedules[iname]; ok { // If there is already a timer scheduled, we don't need to do anything. return } si := mset.sources[iname] if si == nil || si.sip { // if sourceInfo was removed or setup is in progress, nothing to do return } // First calculate the delay until the next time we can var scheduleDelay time.Duration if !si.lreq.IsZero() { // it's not the very first time we are called, compute the delay // We want to throttle here in terms of how fast we request new consumers if sinceLast := time.Since(si.lreq); sinceLast < sourceConsumerRetryThreshold { scheduleDelay = sourceConsumerRetryThreshold - sinceLast } // Is it a retry? If so, add a backoff if si.fails > 0 { scheduleDelay += calculateRetryBackoff(si.fails) } } // Always add some jitter scheduleDelay += time.Duration(rand.Intn(int(100*time.Millisecond))) + 100*time.Millisecond // Schedule the call to trySetupSourceConsumer mset.sourceSetupSchedules[iname] = time.AfterFunc(scheduleDelay, func() { mset.mu.Lock() defer mset.mu.Unlock() delete(mset.sourceSetupSchedules, iname) mset.trySetupSourceConsumer(iname, seq, startTime) }) } // This is where we will actually try to create a new consumer for the source // Lock should be held. func (mset *stream) trySetupSourceConsumer(iname string, seq uint64, startTime time.Time) { // Ignore if closed or not leader. if mset.closed.Load() || !mset.isLeader() { return } si := mset.sources[iname] if si == nil { return } // Cancel previous instance if applicable mset.cancelSourceInfo(si) ssi := mset.streamSource(iname) if ssi == nil { return } si.lreq = time.Now() // Determine subjects etc. var deliverSubject string ext := ssi.External if ext != nil && ext.DeliverPrefix != _EMPTY_ { deliverSubject = strings.ReplaceAll(ext.DeliverPrefix+syncSubject(".S"), "..", ".") } else { deliverSubject = syncSubject("$JS.S") } req := &CreateConsumerRequest{ Stream: si.name, Config: ConsumerConfig{ DeliverSubject: deliverSubject, AckPolicy: AckNone, AckWait: 22 * time.Hour, MaxDeliver: 1, Heartbeat: sourceHealthHB, FlowControl: true, Direct: true, InactiveThreshold: sourceHealthCheckInterval, }, } // If starting, check any configs. if !startTime.IsZero() && seq > 1 { req.Config.OptStartTime = &startTime req.Config.DeliverPolicy = DeliverByStartTime } else if seq <= 1 { if ssi.OptStartSeq > 0 { req.Config.OptStartSeq = ssi.OptStartSeq req.Config.DeliverPolicy = DeliverByStartSequence } else { // We have not recovered state so check that configured time is less that our first seq time. var state StreamState mset.store.FastState(&state) if ssi.OptStartTime != nil { if !state.LastTime.IsZero() && ssi.OptStartTime.Before(state.LastTime) { req.Config.OptStartTime = &state.LastTime } else { req.Config.OptStartTime = ssi.OptStartTime } req.Config.DeliverPolicy = DeliverByStartTime } else if state.FirstSeq > 1 && !state.LastTime.IsZero() { req.Config.OptStartTime = &state.LastTime req.Config.DeliverPolicy = DeliverByStartTime } } } else { req.Config.OptStartSeq = seq req.Config.DeliverPolicy = DeliverByStartSequence } // Filters if ssi.FilterSubject != _EMPTY_ { req.Config.FilterSubject = ssi.FilterSubject } var filterSubjects []string for _, tr := range ssi.SubjectTransforms { filterSubjects = append(filterSubjects, tr.Source) } req.Config.FilterSubjects = filterSubjects respCh := make(chan *JSApiConsumerCreateResponse, 1) reply := infoReplySubject() crSub, err := mset.subscribeInternal(reply, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { mset.unsubscribe(sub) _, msg := c.msgParts(rmsg) var ccr JSApiConsumerCreateResponse if err := json.Unmarshal(msg, &ccr); err != nil { c.Warnf("JetStream bad source consumer create response: %q", msg) return } select { case respCh <- &ccr: default: } }) if err != nil { si.err = NewJSSourceConsumerSetupFailedError(err, Unless(err)) mset.setupSourceConsumer(iname, seq, startTime) return } var subject string if req.Config.FilterSubject != _EMPTY_ { req.Config.Name = fmt.Sprintf("src-%s", createConsumerName()) subject = fmt.Sprintf(JSApiConsumerCreateExT, si.name, req.Config.Name, req.Config.FilterSubject) } else if len(req.Config.FilterSubjects) == 1 { req.Config.Name = fmt.Sprintf("src-%s", createConsumerName()) // It is necessary to switch to using FilterSubject here as the extended consumer // create API checks for it, so as to not accidentally allow multiple filtered subjects. req.Config.FilterSubject = req.Config.FilterSubjects[0] req.Config.FilterSubjects = nil subject = fmt.Sprintf(JSApiConsumerCreateExT, si.name, req.Config.Name, req.Config.FilterSubject) } else { subject = fmt.Sprintf(JSApiConsumerCreateT, si.name) } if ext != nil { subject = strings.Replace(subject, JSApiPrefix, ext.ApiPrefix, 1) subject = strings.ReplaceAll(subject, "..", ".") } // Marshal request. b, _ := json.Marshal(req) // Reset si.msgs = nil si.err = nil si.sip = true // Send the consumer create request mset.outq.send(newJSPubMsg(subject, _EMPTY_, reply, nil, b, nil, 0)) go func() { var retry bool defer func() { mset.mu.Lock() // Check that this is still valid and if so, clear the "setup in progress" flag. if si := mset.sources[iname]; si != nil { si.sip = false // If we need to retry, schedule now if retry { si.fails++ // Cancel here since we can not do anything with this consumer at this point. mset.cancelSourceInfo(si) mset.setupSourceConsumer(iname, seq, startTime) } else { // Clear on success. si.fails = 0 } } mset.mu.Unlock() }() select { case ccr := <-respCh: mset.mu.Lock() // Check that it has not been removed or canceled (si.sub would be nil) if si := mset.sources[iname]; si != nil { si.err = nil if ccr.Error != nil || ccr.ConsumerInfo == nil { // Note: this warning can happen a few times when starting up the server when sourcing streams are // defined, this is normal as the streams are re-created in no particular order and it is possible // that a stream sourcing another could come up before all of its sources have been recreated. mset.srv.Warnf("JetStream error response for stream %s create source consumer %s: %+v", mset.cfg.Name, si.name, ccr.Error) si.err = ccr.Error // Let's retry as soon as possible, but we are gated by sourceConsumerRetryThreshold retry = true mset.mu.Unlock() return } else { // Check if our shared msg queue and go routine is running or not. if mset.smsgs == nil { qname := fmt.Sprintf("[ACC:%s] stream sources '%s' msgs", mset.acc.Name, mset.cfg.Name) mset.smsgs = newIPQueue[*inMsg](mset.srv, qname) mset.srv.startGoRoutine(func() { mset.processAllSourceMsgs() }, pprofLabels{ "type": "source", "account": mset.acc.Name, "stream": mset.cfg.Name, }, ) } // Setup actual subscription to process messages from our source. if si.sseq != ccr.ConsumerInfo.Delivered.Stream { si.sseq = ccr.ConsumerInfo.Delivered.Stream + 1 } // Capture consumer name. si.cname = ccr.ConsumerInfo.Name // Do not set si.sseq to seq here. si.sseq will be set in processInboundSourceMsg si.dseq = 0 si.qch = make(chan struct{}) // Set the last seen as now so that we don't fail at the first check. si.last.Store(time.Now().UnixNano()) msgs := mset.smsgs sub, err := mset.subscribeInternal(deliverSubject, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { hdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy. mset.queueInbound(msgs, subject, reply, hdr, msg, si) si.last.Store(time.Now().UnixNano()) }) if err != nil { si.err = NewJSSourceConsumerSetupFailedError(err, Unless(err)) retry = true mset.mu.Unlock() return } // Save our sub. si.sub = sub } } mset.mu.Unlock() case <-time.After(srcConsumerWaitTime): mset.unsubscribe(crSub) // We already waited 30 seconds, let's retry now. retry = true } }() } // This will process all inbound source msgs. // We mux them into one go routine to avoid lock contention and high cpu and thread thrashing. // TODO(dlc) make this more then one and pin sources to one of a group. func (mset *stream) processAllSourceMsgs() { s := mset.srv defer s.grWG.Done() mset.mu.RLock() msgs, qch := mset.smsgs, mset.qch mset.mu.RUnlock() t := time.NewTicker(sourceHealthCheckInterval) defer t.Stop() // When we detect we are no longer leader, we will cleanup. // Should always return right after this is called. cleanUp := func() { mset.mu.Lock() defer mset.mu.Unlock() for _, si := range mset.sources { mset.cancelSourceConsumer(si.iname) } mset.smsgs.drain() mset.smsgs.unregister() mset.smsgs = nil } for { select { case <-s.quitCh: return case <-qch: return case <-msgs.ch: ims := msgs.pop() for _, im := range ims { if !mset.processInboundSourceMsg(im.si, im) { // If we are no longer leader bail. if !mset.IsLeader() { cleanUp() return } break } im.returnToPool() } msgs.recycle(&ims) case <-t.C: // If we are no longer leader bail. if !mset.IsLeader() { cleanUp() return } // Check health of all sources. var stalled []*sourceInfo mset.mu.RLock() for _, si := range mset.sources { if time.Since(time.Unix(0, si.last.Load())) > sourceHealthCheckInterval { stalled = append(stalled, si) } } numSources := len(mset.sources) mset.mu.RUnlock() // This can happen on an update when no longer have sources. if numSources == 0 { cleanUp() return } // We do not want to block here so do in separate Go routine. if len(stalled) > 0 { go func() { mset.mu.Lock() defer mset.mu.Unlock() for _, si := range stalled { mset.setupSourceConsumer(si.iname, si.sseq+1, time.Time{}) si.last.Store(time.Now().UnixNano()) } }() } } } } // isControlMsg determines if this is a control message. func (m *inMsg) isControlMsg() bool { return len(m.msg) == 0 && len(m.hdr) > 0 && bytes.HasPrefix(m.hdr, []byte("NATS/1.0 100 ")) } // Sends a reply to a flow control request. func (mset *stream) sendFlowControlReply(reply string) { mset.mu.RLock() if mset.isLeader() && mset.outq != nil { mset.outq.sendMsg(reply, nil) } mset.mu.RUnlock() } // handleFlowControl will properly handle flow control messages for both R==1 and R>1. // Lock should be held. func (mset *stream) handleFlowControl(m *inMsg) { // If we are clustered we will send the flow control message through the replication stack. if mset.isClustered() { mset.node.Propose(encodeStreamMsg(_EMPTY_, m.rply, m.hdr, nil, 0, 0)) } else { mset.outq.sendMsg(m.rply, nil) } } // processInboundSourceMsg handles processing other stream messages bound for this stream. func (mset *stream) processInboundSourceMsg(si *sourceInfo, m *inMsg) bool { mset.mu.Lock() // If we are no longer the leader cancel this subscriber. if !mset.isLeader() { mset.cancelSourceConsumer(si.iname) mset.mu.Unlock() return false } isControl := m.isControlMsg() // Ignore from old subscriptions. if !si.isCurrentSub(m.rply) && !isControl { mset.mu.Unlock() return false } // Check for heartbeats and flow control messages. if isControl { var needsRetry bool // Flow controls have reply subjects. if m.rply != _EMPTY_ { mset.handleFlowControl(m) } else { // For idle heartbeats make sure we did not miss anything. if ldseq := parseInt64(getHeader(JSLastConsumerSeq, m.hdr)); ldseq > 0 && uint64(ldseq) != si.dseq { needsRetry = true mset.retrySourceConsumerAtSeq(si.iname, si.sseq+1) } else if fcReply := getHeader(JSConsumerStalled, m.hdr); len(fcReply) > 0 { // Other side thinks we are stalled, so send flow control reply. mset.outq.sendMsg(string(fcReply), nil) } } mset.mu.Unlock() return !needsRetry } sseq, dseq, dc, _, pending := replyInfo(m.rply) if dc > 1 { mset.mu.Unlock() return false } // Tracking is done here. if dseq == si.dseq+1 { si.dseq++ si.sseq = sseq } else if dseq > si.dseq { if si.cname == _EMPTY_ { si.cname = tokenAt(m.rply, 4) si.dseq, si.sseq = dseq, sseq } else { mset.retrySourceConsumerAtSeq(si.iname, si.sseq+1) mset.mu.Unlock() return false } } else { mset.mu.Unlock() return false } if pending == 0 { si.lag = 0 } else { si.lag = pending - 1 } node := mset.node mset.mu.Unlock() hdr, msg := m.hdr, m.msg // If we are daisy chained here make sure to remove the original one. if len(hdr) > 0 { hdr = removeHeaderIfPresent(hdr, JSStreamSource) // Remove any Nats-Expected- headers as we don't want to validate them. hdr = removeHeaderIfPrefixPresent(hdr, "Nats-Expected-") } // Hold onto the origin reply which has all the metadata. hdr = genHeader(hdr, JSStreamSource, si.genSourceHeader(m.rply)) // Do the subject transform for the source if there's one if len(si.trs) > 0 { for _, tr := range si.trs { if tr == nil { continue } else { tsubj, err := tr.Match(m.subj) if err == nil { m.subj = tsubj break } } } } var err error // If we are clustered we need to propose this message to the underlying raft group. if node != nil { err = mset.processClusteredInboundMsg(m.subj, _EMPTY_, hdr, msg) } else { err = mset.processJetStreamMsg(m.subj, _EMPTY_, hdr, msg, 0, 0) } if err != nil { s := mset.srv if strings.Contains(err.Error(), "no space left") { s.Errorf("JetStream out of space, will be DISABLED") s.DisableJetStream() } else { mset.mu.RLock() accName, sname, iName := mset.acc.Name, mset.cfg.Name, si.iname mset.mu.RUnlock() // Can happen temporarily all the time during normal operations when the sourcing stream // is working queue/interest with a limit and discard new. // TODO - Improve sourcing to WQ with limit and new to use flow control rather than re-creating the consumer. if errors.Is(err, ErrMaxMsgs) { // Do not need to do a full retry that includes finding the last sequence in the stream // for that source. Just re-create starting with the seq we couldn't store instead. mset.mu.Lock() mset.retrySourceConsumerAtSeq(iName, si.sseq) mset.mu.Unlock() } else { // Log some warning for errors other than errLastSeqMismatch or errMaxMsgs. if !errors.Is(err, errLastSeqMismatch) { s.RateLimitWarnf("Error processing inbound source %q for '%s' > '%s': %v", iName, accName, sname, err) } // Retry in all type of errors if we are still leader. if mset.isLeader() { // This will make sure the source is still in mset.sources map, // find the last sequence and then call setupSourceConsumer. iNameMap := map[string]struct{}{iName: {}} mset.setStartingSequenceForSources(iNameMap) mset.mu.Lock() mset.retrySourceConsumerAtSeq(iName, si.sseq+1) mset.mu.Unlock() } } } return false } return true } // Generate a new (2.10) style source header (stream name, sequence number, source filter, source destination transform). func (si *sourceInfo) genSourceHeader(reply string) string { var b strings.Builder iNameParts := strings.Split(si.iname, " ") b.WriteString(iNameParts[0]) b.WriteByte(' ') // Grab sequence as text here from reply subject. var tsa [expectedNumReplyTokens]string start, tokens := 0, tsa[:0] for i := 0; i < len(reply); i++ { if reply[i] == btsep { tokens, start = append(tokens, reply[start:i]), i+1 } } tokens = append(tokens, reply[start:]) seq := "1" // Default if len(tokens) == expectedNumReplyTokens && tokens[0] == "$JS" && tokens[1] == "ACK" { seq = tokens[5] } b.WriteString(seq) b.WriteByte(' ') b.WriteString(iNameParts[1]) b.WriteByte(' ') b.WriteString(iNameParts[2]) return b.String() } // Original version of header that stored ack reply direct. func streamAndSeqFromAckReply(reply string) (string, string, uint64) { tsa := [expectedNumReplyTokens]string{} start, tokens := 0, tsa[:0] for i := 0; i < len(reply); i++ { if reply[i] == btsep { tokens, start = append(tokens, reply[start:i]), i+1 } } tokens = append(tokens, reply[start:]) if len(tokens) != expectedNumReplyTokens || tokens[0] != "$JS" || tokens[1] != "ACK" { return _EMPTY_, _EMPTY_, 0 } return tokens[2], _EMPTY_, uint64(parseAckReplyNum(tokens[5])) } // Extract the stream name, the source index name and the message sequence number from the source header. // Uses the filter and transform arguments to provide backwards compatibility func streamAndSeq(shdr string) (string, string, uint64) { if strings.HasPrefix(shdr, jsAckPre) { return streamAndSeqFromAckReply(shdr) } // New version which is stream index name sequence fields := strings.Split(shdr, " ") nFields := len(fields) if nFields != 2 && nFields <= 3 { return _EMPTY_, _EMPTY_, 0 } if nFields >= 4 { return fields[0], strings.Join([]string{fields[0], fields[2], fields[3]}, " "), uint64(parseAckReplyNum(fields[1])) } else { return fields[0], _EMPTY_, uint64(parseAckReplyNum(fields[1])) } } // Lock should be held. func (mset *stream) setStartingSequenceForSources(iNames map[string]struct{}) { var state StreamState mset.store.FastState(&state) // Do not reset sseq here so we can remember when purge/expiration happens. if state.Msgs == 0 { for iName := range iNames { si := mset.sources[iName] if si == nil { continue } else { si.dseq = 0 } } return } var smv StoreMsg for seq := state.LastSeq; seq >= state.FirstSeq; { sm, err := mset.store.LoadPrevMsg(seq, &smv) if err == ErrStoreEOF || err != nil { break } seq = sm.seq - 1 if len(sm.hdr) == 0 { continue } ss := getHeader(JSStreamSource, sm.hdr) if len(ss) == 0 { continue } streamName, indexName, sseq := streamAndSeq(bytesToString(ss)) if _, ok := iNames[indexName]; ok { si := mset.sources[indexName] si.sseq = sseq si.dseq = 0 delete(iNames, indexName) } else if indexName == _EMPTY_ && streamName != _EMPTY_ { for iName := range iNames { // TODO streamSource is a linear walk, to optimize later if si := mset.sources[iName]; si != nil && streamName == si.name || (mset.streamSource(iName).External != nil && streamName == si.name+":"+getHash(mset.streamSource(iName).External.ApiPrefix)) { si.sseq = sseq si.dseq = 0 delete(iNames, iName) break } } } if len(iNames) == 0 { break } } } // Resets the SourceInfo for all the sources // lock should be held. func (mset *stream) resetSourceInfo() { // Reset if needed. mset.stopSourceConsumers() mset.sources = make(map[string]*sourceInfo) for _, ssi := range mset.cfg.Sources { if ssi.iname == _EMPTY_ { ssi.setIndexName() } var si *sourceInfo if len(ssi.SubjectTransforms) == 0 { si = &sourceInfo{name: ssi.Name, iname: ssi.iname, sf: ssi.FilterSubject} } else { sfs := make([]string, len(ssi.SubjectTransforms)) trs := make([]*subjectTransform, len(ssi.SubjectTransforms)) for i, str := range ssi.SubjectTransforms { tr, err := NewSubjectTransform(str.Source, str.Destination) if err != nil { mset.srv.Errorf("Unable to get subject transform for source: %v", err) } sfs[i] = str.Source trs[i] = tr } si = &sourceInfo{name: ssi.Name, iname: ssi.iname, sfs: sfs, trs: trs} } mset.sources[ssi.iname] = si } } // This will do a reverse scan on startup or leader election // searching for the starting sequence number. // This can be slow in degenerative cases. // Lock should be held. func (mset *stream) startingSequenceForSources() { if len(mset.cfg.Sources) == 0 { return } // Always reset here. mset.resetSourceInfo() var state StreamState mset.store.FastState(&state) // Bail if no messages, meaning no context. if state.Msgs == 0 { return } // For short circuiting return. expected := len(mset.cfg.Sources) seqs := make(map[string]uint64) // Stamp our si seq records on the way out. defer func() { for sname, seq := range seqs { // Ignore if not set. if seq == 0 { continue } if si := mset.sources[sname]; si != nil { si.sseq = seq si.dseq = 0 } } }() update := func(iName string, seq uint64) { // Only update active in case we have older ones in here that got configured out. if si := mset.sources[iName]; si != nil { if _, ok := seqs[iName]; !ok { seqs[iName] = seq } } } var smv StoreMsg for seq := state.LastSeq; ; { sm, err := mset.store.LoadPrevMsg(seq, &smv) if err == ErrStoreEOF || err != nil { break } seq = sm.seq - 1 if len(sm.hdr) == 0 { continue } ss := getHeader(JSStreamSource, sm.hdr) if len(ss) == 0 { continue } streamName, iName, sseq := streamAndSeq(bytesToString(ss)) if iName == _EMPTY_ { // Pre-2.10 message header means it's a match for any source using that stream name for _, ssi := range mset.cfg.Sources { if streamName == ssi.Name || (ssi.External != nil && streamName == ssi.Name+":"+getHash(ssi.External.ApiPrefix)) { update(ssi.iname, sseq) } } } else { update(iName, sseq) } if len(seqs) == expected { return } } } // Setup our source consumers. // Lock should be held. func (mset *stream) setupSourceConsumers() error { if mset.outq == nil { return errors.New("outq required") } // Reset if needed. for _, si := range mset.sources { if si.sub != nil { mset.cancelSourceConsumer(si.iname) } } // If we are no longer the leader, give up if !mset.isLeader() { return nil } mset.startingSequenceForSources() // Setup our consumers at the proper starting position. for _, ssi := range mset.cfg.Sources { if si := mset.sources[ssi.iname]; si != nil { mset.setupSourceConsumer(ssi.iname, si.sseq+1, time.Time{}) } } return nil } // Will create internal subscriptions for the stream. // Lock should be held. func (mset *stream) subscribeToStream() error { if mset.active { return nil } for _, subject := range mset.cfg.Subjects { if _, err := mset.subscribeInternal(subject, mset.processInboundJetStreamMsg); err != nil { return err } } // Check if we need to setup mirroring. if mset.cfg.Mirror != nil { // setup the initial mirror sourceInfo mset.mirror = &sourceInfo{name: mset.cfg.Mirror.Name} sfs := make([]string, len(mset.cfg.Mirror.SubjectTransforms)) trs := make([]*subjectTransform, len(mset.cfg.Mirror.SubjectTransforms)) for i, tr := range mset.cfg.Mirror.SubjectTransforms { // will not fail as already checked before that the transform will work subjectTransform, err := NewSubjectTransform(tr.Source, tr.Destination) if err != nil { mset.srv.Errorf("Unable to get transform for mirror consumer: %v", err) } sfs[i] = tr.Source trs[i] = subjectTransform } mset.mirror.sfs = sfs mset.mirror.trs = trs // delay the actual mirror consumer creation for after a delay mset.scheduleSetupMirrorConsumerRetry() } else if len(mset.cfg.Sources) > 0 && mset.sourcesConsumerSetup == nil { // Setup the initial source infos for the sources mset.resetSourceInfo() // Delay the actual source consumer(s) creation(s) for after a delay if a replicated stream. // If it's an R1, this is done at startup and we will do inline. if mset.cfg.Replicas == 1 { mset.setupSourceConsumers() } else { mset.sourcesConsumerSetup = time.AfterFunc(time.Duration(rand.Intn(int(500*time.Millisecond)))+100*time.Millisecond, func() { mset.mu.Lock() mset.setupSourceConsumers() mset.mu.Unlock() }) } } // Check for direct get access. // We spin up followers for clustered streams in monitorStream(). if mset.cfg.AllowDirect { if err := mset.subscribeToDirect(); err != nil { return err } } mset.active = true return nil } // Lock should be held. func (mset *stream) subscribeToDirect() error { // We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis. if mset.directSub == nil { dsubj := fmt.Sprintf(JSDirectMsgGetT, mset.cfg.Name) if sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetRequest); err == nil { mset.directSub = sub } else { return err } } // Now the one that will have subject appended past stream name. if mset.lastBySub == nil { dsubj := fmt.Sprintf(JSDirectGetLastBySubjectT, mset.cfg.Name, fwcs) // We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis. if sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetLastBySubjectRequest); err == nil { mset.lastBySub = sub } else { return err } } return nil } // Lock should be held. func (mset *stream) unsubscribeToDirect() { if mset.directSub != nil { mset.unsubscribe(mset.directSub) mset.directSub = nil } if mset.lastBySub != nil { mset.unsubscribe(mset.lastBySub) mset.lastBySub = nil } } // Lock should be held. func (mset *stream) subscribeToMirrorDirect() error { if mset.mirror == nil { return nil } // We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis. if mset.mirror.dsub == nil { dsubj := fmt.Sprintf(JSDirectMsgGetT, mset.mirror.name) // We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis. if sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetRequest); err == nil { mset.mirror.dsub = sub } else { return err } } // Now the one that will have subject appended past stream name. if mset.mirror.lbsub == nil { dsubj := fmt.Sprintf(JSDirectGetLastBySubjectT, mset.mirror.name, fwcs) // We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis. if sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetLastBySubjectRequest); err == nil { mset.mirror.lbsub = sub } else { return err } } return nil } // Stop our source consumers. // Lock should be held. func (mset *stream) stopSourceConsumers() { for _, si := range mset.sources { mset.cancelSourceInfo(si) } } // Lock should be held. func (mset *stream) removeInternalConsumer(si *sourceInfo) { if si == nil || si.cname == _EMPTY_ { return } si.cname = _EMPTY_ } // Will unsubscribe from the stream. // Lock should be held. func (mset *stream) unsubscribeToStream(stopping bool) error { for _, subject := range mset.cfg.Subjects { mset.unsubscribeInternal(subject) } if mset.mirror != nil { mset.cancelSourceInfo(mset.mirror) mset.mirror = nil } if len(mset.sources) > 0 { mset.stopSourceConsumers() } // In case we had a direct get subscriptions. if stopping { mset.unsubscribeToDirect() } mset.active = false return nil } // Lock does NOT need to be held, we set the client on setup and never change it at this point. func (mset *stream) subscribeInternal(subject string, cb msgHandler) (*subscription, error) { if mset.closed.Load() { return nil, errStreamClosed } if cb == nil { return nil, errInvalidMsgHandler } c := mset.client sid := int(mset.sid.Add(1)) // Now create the subscription return c.processSub([]byte(subject), nil, []byte(strconv.Itoa(sid)), cb, false) } // Lock does NOT need to be held, we set the client on setup and never change it at this point. func (mset *stream) queueSubscribeInternal(subject, group string, cb msgHandler) (*subscription, error) { if mset.closed.Load() { return nil, errStreamClosed } if cb == nil { return nil, errInvalidMsgHandler } c := mset.client sid := int(mset.sid.Add(1)) // Now create the subscription return c.processSub([]byte(subject), []byte(group), []byte(strconv.Itoa(sid)), cb, false) } // This will unsubscribe us from the exact subject given. // We do not currently track the subs so do not have the sid. // This should be called only on an update. // Lock does NOT need to be held, we set the client on setup and never change it at this point. func (mset *stream) unsubscribeInternal(subject string) error { if mset.closed.Load() { return errStreamClosed } c := mset.client var sid []byte c.mu.Lock() for _, sub := range c.subs { if subject == string(sub.subject) { sid = sub.sid break } } c.mu.Unlock() if sid != nil { return c.processUnsub(sid) } return nil } // Lock should be held. func (mset *stream) unsubscribe(sub *subscription) { if sub == nil || mset.closed.Load() { return } mset.client.processUnsub(sub.sid) } func (mset *stream) setupStore(fsCfg *FileStoreConfig) error { mset.mu.Lock() mset.created = time.Now().UTC() switch mset.cfg.Storage { case MemoryStorage: ms, err := newMemStore(&mset.cfg) if err != nil { mset.mu.Unlock() return err } mset.store = ms case FileStorage: s := mset.srv prf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name) if prf != nil { // We are encrypted here, fill in correct cipher selection. fsCfg.Cipher = s.getOpts().JetStreamCipher } oldprf := s.jsKeyGen(s.getOpts().JetStreamOldKey, mset.acc.Name) cfg := *fsCfg cfg.srv = s fs, err := newFileStoreWithCreated(cfg, mset.cfg, mset.created, prf, oldprf) if err != nil { mset.mu.Unlock() return err } mset.store = fs } // This will fire the callback but we do not require the lock since md will be 0 here. mset.store.RegisterStorageUpdates(mset.storeUpdates) mset.mu.Unlock() return nil } // Called for any updates to the underlying stream. We pass through the bytes to the // jetstream account. We do local processing for stream pending for consumers, but only // for removals. // Lock should not be held. func (mset *stream) storeUpdates(md, bd int64, seq uint64, subj string) { // If we have a single negative update then we will process our consumers for stream pending. // Purge and Store handled separately inside individual calls. if md == -1 && seq > 0 && subj != _EMPTY_ { // We use our consumer list mutex here instead of the main stream lock since it may be held already. mset.clsMu.RLock() if mset.csl != nil { mset.csl.Match(subj, func(o *consumer) { o.decStreamPending(seq, subj) }) } else { for _, o := range mset.cList { o.decStreamPending(seq, subj) } } mset.clsMu.RUnlock() } else if md < 0 { // Batch decrements we need to force consumers to re-calculate num pending. mset.clsMu.RLock() for _, o := range mset.cList { o.streamNumPendingLocked() } mset.clsMu.RUnlock() } if mset.jsa != nil { mset.jsa.updateUsage(mset.tier, mset.stype, bd) } } // NumMsgIds returns the number of message ids being tracked for duplicate suppression. func (mset *stream) numMsgIds() int { mset.mu.Lock() defer mset.mu.Unlock() if !mset.ddloaded { mset.rebuildDedupe() } return len(mset.ddmap) } // checkMsgId will process and check for duplicates. // Lock should be held. func (mset *stream) checkMsgId(id string) *ddentry { if !mset.ddloaded { mset.rebuildDedupe() } if id == _EMPTY_ || len(mset.ddmap) == 0 { return nil } return mset.ddmap[id] } // Will purge the entries that are past the window. // Should be called from a timer. func (mset *stream) purgeMsgIds() { mset.mu.Lock() defer mset.mu.Unlock() now := time.Now().UnixNano() tmrNext := mset.cfg.Duplicates window := int64(tmrNext) for i, dde := range mset.ddarr[mset.ddindex:] { if now-dde.ts >= window { delete(mset.ddmap, dde.id) } else { mset.ddindex += i // Check if we should garbage collect here if we are 1/3 total size. if cap(mset.ddarr) > 3*(len(mset.ddarr)-mset.ddindex) { mset.ddarr = append([]*ddentry(nil), mset.ddarr[mset.ddindex:]...) mset.ddindex = 0 } tmrNext = time.Duration(window - (now - dde.ts)) break } } if len(mset.ddmap) > 0 { // Make sure to not fire too quick const minFire = 50 * time.Millisecond if tmrNext < minFire { tmrNext = minFire } if mset.ddtmr != nil { mset.ddtmr.Reset(tmrNext) } else { mset.ddtmr = time.AfterFunc(tmrNext, mset.purgeMsgIds) } } else { if mset.ddtmr != nil { mset.ddtmr.Stop() mset.ddtmr = nil } mset.ddmap = nil mset.ddarr = nil mset.ddindex = 0 } } // storeMsgIdLocked will store the message id for duplicate detection. // Lock should be held. func (mset *stream) storeMsgIdLocked(dde *ddentry) { if mset.ddmap == nil { mset.ddmap = make(map[string]*ddentry) } mset.ddmap[dde.id] = dde mset.ddarr = append(mset.ddarr, dde) if mset.ddtmr == nil { mset.ddtmr = time.AfterFunc(mset.cfg.Duplicates, mset.purgeMsgIds) } } // Fast lookup of msgId. func getMsgId(hdr []byte) string { return string(getHeader(JSMsgId, hdr)) } // Fast lookup of expected last msgId. func getExpectedLastMsgId(hdr []byte) string { return string(getHeader(JSExpectedLastMsgId, hdr)) } // Fast lookup of expected stream. func getExpectedStream(hdr []byte) string { return string(getHeader(JSExpectedStream, hdr)) } // Fast lookup of expected stream. func getExpectedLastSeq(hdr []byte) (uint64, bool) { bseq := getHeader(JSExpectedLastSeq, hdr) if len(bseq) == 0 { return 0, false } return uint64(parseInt64(bseq)), true } // Fast lookup of rollups. func getRollup(hdr []byte) string { r := getHeader(JSMsgRollup, hdr) if len(r) == 0 { return _EMPTY_ } return strings.ToLower(string(r)) } // Fast lookup of expected stream sequence per subject. func getExpectedLastSeqPerSubject(hdr []byte) (uint64, bool) { bseq := getHeader(JSExpectedLastSubjSeq, hdr) if len(bseq) == 0 { return 0, false } return uint64(parseInt64(bseq)), true } // Signal if we are clustered. Will acquire rlock. func (mset *stream) IsClustered() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.isClustered() } // Lock should be held. func (mset *stream) isClustered() bool { return mset.node != nil } // Used if we have to queue things internally to avoid the route/gw path. type inMsg struct { subj string rply string hdr []byte msg []byte si *sourceInfo } var inMsgPool = sync.Pool{ New: func() any { return &inMsg{} }, } func (im *inMsg) returnToPool() { im.subj, im.rply, im.hdr, im.msg, im.si = _EMPTY_, _EMPTY_, nil, nil, nil inMsgPool.Put(im) } func (mset *stream) queueInbound(ib *ipQueue[*inMsg], subj, rply string, hdr, msg []byte, si *sourceInfo) { im := inMsgPool.Get().(*inMsg) im.subj, im.rply, im.hdr, im.msg, im.si = subj, rply, hdr, msg, si ib.push(im) } var dgPool = sync.Pool{ New: func() any { return &directGetReq{} }, } // For when we need to not inline the request. type directGetReq struct { // Copy of this is correct for this. req JSApiMsgGetRequest reply string } // processDirectGetRequest handles direct get request for stream messages. func (mset *stream) processDirectGetRequest(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if len(reply) == 0 { return } _, msg := c.msgParts(rmsg) if len(msg) == 0 { hdr := []byte("NATS/1.0 408 Empty Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } var req JSApiMsgGetRequest err := json.Unmarshal(msg, &req) if err != nil { hdr := []byte("NATS/1.0 408 Malformed Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } // Check if nothing set. if req.Seq == 0 && req.LastFor == _EMPTY_ && req.NextFor == _EMPTY_ { hdr := []byte("NATS/1.0 408 Empty Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } // Check that we do not have both options set. if req.Seq > 0 && req.LastFor != _EMPTY_ { hdr := []byte("NATS/1.0 408 Bad Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } if req.LastFor != _EMPTY_ && req.NextFor != _EMPTY_ { hdr := []byte("NATS/1.0 408 Bad Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } inlineOk := c.kind != ROUTER && c.kind != GATEWAY && c.kind != LEAF if !inlineOk { dg := dgPool.Get().(*directGetReq) dg.req, dg.reply = req, reply mset.gets.push(dg) } else { mset.getDirectRequest(&req, reply) } } // This is for direct get by last subject which is part of the subject itself. func (mset *stream) processDirectGetLastBySubjectRequest(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { if len(reply) == 0 { return } _, msg := c.msgParts(rmsg) // This version expects no payload. if len(msg) != 0 { hdr := []byte("NATS/1.0 408 Bad Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } // Extract the key. var key string for i, n := 0, 0; i < len(subject); i++ { if subject[i] == btsep { if n == 4 { if start := i + 1; start < len(subject) { key = subject[i+1:] } break } n++ } } if len(key) == 0 { hdr := []byte("NATS/1.0 408 Bad Request\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } req := JSApiMsgGetRequest{LastFor: key} inlineOk := c.kind != ROUTER && c.kind != GATEWAY && c.kind != LEAF if !inlineOk { dg := dgPool.Get().(*directGetReq) dg.req, dg.reply = req, reply mset.gets.push(dg) } else { mset.getDirectRequest(&req, reply) } } // Do actual work on a direct msg request. // This could be called in a Go routine if we are inline for a non-client connection. func (mset *stream) getDirectRequest(req *JSApiMsgGetRequest, reply string) { var svp StoreMsg var sm *StoreMsg var err error mset.mu.RLock() store, name := mset.store, mset.cfg.Name mset.mu.RUnlock() if req.Seq > 0 && req.NextFor == _EMPTY_ { sm, err = store.LoadMsg(req.Seq, &svp) } else if req.NextFor != _EMPTY_ { sm, _, err = store.LoadNextMsg(req.NextFor, subjectHasWildcard(req.NextFor), req.Seq, &svp) } else { sm, err = store.LoadLastMsg(req.LastFor, &svp) } if err != nil { hdr := []byte("NATS/1.0 404 Message Not Found\r\n\r\n") mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0)) return } hdr := sm.hdr ts := time.Unix(0, sm.ts).UTC() if len(hdr) == 0 { const ht = "NATS/1.0\r\nNats-Stream: %s\r\nNats-Subject: %s\r\nNats-Sequence: %d\r\nNats-Time-Stamp: %s\r\n\r\n" hdr = fmt.Appendf(nil, ht, name, sm.subj, sm.seq, ts.Format(time.RFC3339Nano)) } else { hdr = copyBytes(hdr) hdr = genHeader(hdr, JSStream, name) hdr = genHeader(hdr, JSSubject, sm.subj) hdr = genHeader(hdr, JSSequence, strconv.FormatUint(sm.seq, 10)) hdr = genHeader(hdr, JSTimeStamp, ts.Format(time.RFC3339Nano)) } mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, sm.msg, nil, 0)) } // processInboundJetStreamMsg handles processing messages bound for a stream. func (mset *stream) processInboundJetStreamMsg(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) { hdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy. mset.queueInbound(mset.msgs, subject, reply, hdr, msg, nil) } var ( errLastSeqMismatch = errors.New("last sequence mismatch") errMsgIdDuplicate = errors.New("msgid is duplicate") errStreamClosed = errors.New("stream closed") errInvalidMsgHandler = errors.New("undefined message handler") errStreamMismatch = errors.New("expected stream does not match") ) // processJetStreamMsg is where we try to actually process the stream msg. func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, lseq uint64, ts int64) error { if mset.closed.Load() { return errStreamClosed } mset.mu.Lock() s, store := mset.srv, mset.store bumpCLFS := func() { mset.clMu.Lock() mset.clfs++ mset.clMu.Unlock() } // Apply the input subject transform if any if mset.itr != nil { ts, err := mset.itr.Match(subject) if err == nil { // no filtering: if the subject doesn't map the source of the transform, don't change it subject = ts } } var accName string if mset.acc != nil { accName = mset.acc.Name } js, jsa, doAck := mset.js, mset.jsa, !mset.cfg.NoAck name, stype := mset.cfg.Name, mset.cfg.Storage maxMsgSize := int(mset.cfg.MaxMsgSize) numConsumers := len(mset.consumers) interestRetention := mset.cfg.Retention == InterestPolicy // Snapshot if we are the leader and if we can respond. isLeader, isSealed := mset.isLeader(), mset.cfg.Sealed canRespond := doAck && len(reply) > 0 && isLeader var resp = &JSPubAckResponse{} // Bail here if sealed. if isSealed { outq := mset.outq mset.mu.Unlock() bumpCLFS() if canRespond && outq != nil { resp.PubAck = &PubAck{Stream: name} resp.Error = ApiErrors[JSStreamSealedErr] b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return ApiErrors[JSStreamSealedErr] } var buf [256]byte pubAck := append(buf[:0], mset.pubAck...) // If this is a non-clustered msg and we are not considered active, meaning no active subscription, do not process. if lseq == 0 && ts == 0 && !mset.active { mset.mu.Unlock() return nil } // For clustering the lower layers will pass our expected lseq. If it is present check for that here. if lseq > 0 && lseq != (mset.lseq+mset.clfs) { isMisMatch := true // We may be able to recover here if we have no state whatsoever, or we are a mirror. // See if we have to adjust our starting sequence. if mset.lseq == 0 || mset.cfg.Mirror != nil { var state StreamState mset.store.FastState(&state) if state.FirstSeq == 0 { mset.store.Compact(lseq + 1) mset.lseq = lseq isMisMatch = false } } // Really is a mismatch. if isMisMatch { outq := mset.outq mset.mu.Unlock() if canRespond && outq != nil { resp.PubAck = &PubAck{Stream: name} resp.Error = ApiErrors[JSStreamSequenceNotMatchErr] b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return errLastSeqMismatch } } // If we have received this message across an account we may have request information attached. // For now remove. TODO(dlc) - Should this be opt-in or opt-out? if len(hdr) > 0 { hdr = removeHeaderIfPresent(hdr, ClientInfoHdr) } // Process additional msg headers if still present. var msgId string var rollupSub, rollupAll bool isClustered := mset.isClustered() if len(hdr) > 0 { outq := mset.outq // Certain checks have already been performed if in clustered mode, so only check if not. if !isClustered { // Expected stream. if sname := getExpectedStream(hdr); sname != _EMPTY_ && sname != name { mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamNotMatchError() b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return errStreamMismatch } } // Dedupe detection. This is done at the cluster level for dedupe detectiom above the // lower layers. But we still need to pull out the msgId. if msgId = getMsgId(hdr); msgId != _EMPTY_ { // Do real check only if not clustered or traceOnly flag is set. if !isClustered { if dde := mset.checkMsgId(msgId); dde != nil { mset.mu.Unlock() bumpCLFS() if canRespond { response := append(pubAck, strconv.FormatUint(dde.seq, 10)...) response = append(response, ",\"duplicate\": true}"...) outq.sendMsg(reply, response) } return errMsgIdDuplicate } } } // Expected last sequence per subject. if seq, exists := getExpectedLastSeqPerSubject(hdr); exists { // TODO(dlc) - We could make a new store func that does this all in one. var smv StoreMsg var fseq uint64 sm, err := store.LoadLastMsg(subject, &smv) if sm != nil { fseq = sm.seq } if err == ErrStoreMsgNotFound { if seq == 0 { fseq, err = 0, nil } else if mset.isClustered() { // Do not bump clfs in case message was not found and could have been deleted. var ss StreamState store.FastState(&ss) if seq <= ss.LastSeq { fseq, err = seq, nil } } } if err != nil || fseq != seq { mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamWrongLastSequenceError(fseq) b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return fmt.Errorf("last sequence by subject mismatch: %d vs %d", seq, fseq) } } // Expected last sequence. if seq, exists := getExpectedLastSeq(hdr); exists && seq != mset.lseq { mlseq := mset.lseq mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamWrongLastSequenceError(mlseq) b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return fmt.Errorf("last sequence mismatch: %d vs %d", seq, mlseq) } // Expected last msgId. if lmsgId := getExpectedLastMsgId(hdr); lmsgId != _EMPTY_ { if mset.lmsgId == _EMPTY_ && !mset.ddloaded { mset.rebuildDedupe() } if lmsgId != mset.lmsgId { last := mset.lmsgId mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamWrongLastMsgIDError(last) b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return fmt.Errorf("last msgid mismatch: %q vs %q", lmsgId, last) } } // Check for any rollups. if rollup := getRollup(hdr); rollup != _EMPTY_ { if !mset.cfg.AllowRollup || mset.cfg.DenyPurge { mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamRollupFailedError(errors.New("rollup not permitted")) b, _ := json.Marshal(resp) outq.sendMsg(reply, b) } return errors.New("rollup not permitted") } switch rollup { case JSMsgRollupSubject: rollupSub = true case JSMsgRollupAll: rollupAll = true default: mset.mu.Unlock() bumpCLFS() return fmt.Errorf("rollup value invalid: %q", rollup) } } } // Response Ack. var ( response []byte seq uint64 err error ) // Check to see if we are over the max msg size. if maxMsgSize >= 0 && (len(hdr)+len(msg)) > maxMsgSize { mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamMessageExceedsMaximumError() response, _ = json.Marshal(resp) mset.outq.sendMsg(reply, response) } return ErrMaxPayload } if len(hdr) > math.MaxUint16 { mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamHeaderExceedsMaximumError() response, _ = json.Marshal(resp) mset.outq.sendMsg(reply, response) } return ErrMaxPayload } // Check to see if we have exceeded our limits. if js.limitsExceeded(stype) { s.resourcesExceededError() mset.mu.Unlock() bumpCLFS() if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSInsufficientResourcesError() response, _ = json.Marshal(resp) mset.outq.sendMsg(reply, response) } // Stepdown regardless. if node := mset.raftNode(); node != nil { node.StepDown() } return NewJSInsufficientResourcesError() } var noInterest bool // If we are interest based retention and have no consumers then we can skip. if interestRetention { mset.clsMu.RLock() noInterest = numConsumers == 0 || mset.csl == nil || !mset.csl.HasInterest(subject) mset.clsMu.RUnlock() } // Grab timestamp if not already set. if ts == 0 && lseq > 0 { ts = time.Now().UnixNano() } // Skip msg here. if noInterest { mset.lseq = store.SkipMsg() mset.lmsgId = msgId // If we have a msgId make sure to save. if msgId != _EMPTY_ { mset.storeMsgIdLocked(&ddentry{msgId, mset.lseq, ts}) } if canRespond { response = append(pubAck, strconv.FormatUint(mset.lseq, 10)...) response = append(response, '}') mset.outq.sendMsg(reply, response) } mset.mu.Unlock() return nil } // If here we will attempt to store the message. // Assume this will succeed. olmsgId := mset.lmsgId mset.lmsgId = msgId clfs := mset.clfs mset.lseq++ tierName := mset.tier // Republish state if needed. var tsubj string var tlseq uint64 var thdrsOnly bool if mset.tr != nil { tsubj, _ = mset.tr.Match(subject) if mset.cfg.RePublish != nil { thdrsOnly = mset.cfg.RePublish.HeadersOnly } } republish := tsubj != _EMPTY_ && isLeader // If we are republishing grab last sequence for this exact subject. Aids in gap detection for lightweight clients. if republish { var smv StoreMsg if sm, _ := store.LoadLastMsg(subject, &smv); sm != nil { tlseq = sm.seq } } // If clustered this was already checked and we do not want to check here and possibly introduce skew. if !isClustered { if exceeded, err := jsa.wouldExceedLimits(stype, tierName, mset.cfg.Replicas, subject, hdr, msg); exceeded { if err == nil { err = NewJSAccountResourcesExceededError() } s.RateLimitWarnf("JetStream resource limits exceeded for account: %q", accName) if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = err response, _ = json.Marshal(resp) mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0)) } mset.mu.Unlock() return err } } // Store actual msg. if lseq == 0 && ts == 0 { seq, ts, err = store.StoreMsg(subject, hdr, msg) } else { // Make sure to take into account any message assignments that we had to skip (clfs). seq = lseq + 1 - clfs // Check for preAcks and the need to skip vs store. if mset.hasAllPreAcks(seq, subject) { mset.clearAllPreAcks(seq) store.SkipMsg() } else { err = store.StoreRawMsg(subject, hdr, msg, seq, ts) } } if err != nil { if isPermissionError(err) { mset.mu.Unlock() // messages in block cache could be lost in the worst case. // In the clustered mode it is very highly unlikely as a result of replication. mset.srv.DisableJetStream() mset.srv.Warnf("Filesystem permission denied while writing msg, disabling JetStream: %v", err) return err } // If we did not succeed put those values back and increment clfs in case we are clustered. var state StreamState mset.store.FastState(&state) mset.lseq = state.LastSeq mset.lmsgId = olmsgId mset.mu.Unlock() bumpCLFS() switch err { case ErrMaxMsgs, ErrMaxBytes, ErrMaxMsgsPerSubject, ErrMsgTooLarge: s.RateLimitDebugf("JetStream failed to store a msg on stream '%s > %s': %v", accName, name, err) case ErrStoreClosed: default: s.Errorf("JetStream failed to store a msg on stream '%s > %s': %v", accName, name, err) } if canRespond { resp.PubAck = &PubAck{Stream: name} resp.Error = NewJSStreamStoreFailedError(err, Unless(err)) response, _ = json.Marshal(resp) mset.outq.sendMsg(reply, response) } return err } // If we have a msgId make sure to save. // This will replace our estimate from the cluster layer if we are clustered. if msgId != _EMPTY_ { if isClustered && isLeader && mset.ddmap != nil { if dde := mset.ddmap[msgId]; dde != nil { dde.seq, dde.ts = seq, ts } else { mset.storeMsgIdLocked(&ddentry{msgId, seq, ts}) } } else { // R1 or not leader.. mset.storeMsgIdLocked(&ddentry{msgId, seq, ts}) } } // If here we succeeded in storing the message. mset.mu.Unlock() // No errors, this is the normal path. if rollupSub { mset.purge(&JSApiStreamPurgeRequest{Subject: subject, Keep: 1}) } else if rollupAll { mset.purge(&JSApiStreamPurgeRequest{Keep: 1}) } // Check for republish. if republish { const ht = "NATS/1.0\r\nNats-Stream: %s\r\nNats-Subject: %s\r\nNats-Sequence: %d\r\nNats-Time-Stamp: %s\r\nNats-Last-Sequence: %d\r\n\r\n" const htho = "NATS/1.0\r\nNats-Stream: %s\r\nNats-Subject: %s\r\nNats-Sequence: %d\r\nNats-Time-Stamp: %s\r\nNats-Last-Sequence: %d\r\nNats-Msg-Size: %d\r\n\r\n" // When adding to existing headers, will use the fmt.Append version so this skips the headers from above. const hoff = 10 tsStr := time.Unix(0, ts).UTC().Format(time.RFC3339Nano) var rpMsg []byte if len(hdr) == 0 { if !thdrsOnly { hdr = fmt.Appendf(nil, ht, name, subject, seq, tsStr, tlseq) rpMsg = copyBytes(msg) } else { hdr = fmt.Appendf(nil, htho, name, subject, seq, tsStr, tlseq, len(msg)) } } else { // use hdr[:end:end] to make sure as we add we copy the original hdr. end := len(hdr) - LEN_CR_LF if !thdrsOnly { hdr = fmt.Appendf(hdr[:end:end], ht[hoff:], name, subject, seq, tsStr, tlseq) rpMsg = copyBytes(msg) } else { hdr = fmt.Appendf(hdr[:end:end], htho[hoff:], name, subject, seq, tsStr, tlseq, len(msg)) } } mset.outq.send(newJSPubMsg(tsubj, _EMPTY_, _EMPTY_, hdr, rpMsg, nil, seq)) } // Send response here. if canRespond { response = append(pubAck, strconv.FormatUint(seq, 10)...) response = append(response, '}') mset.outq.sendMsg(reply, response) } // Signal consumers for new messages. if numConsumers > 0 { mset.sigq.push(newCMsg(subject, seq)) select { case mset.sch <- struct{}{}: default: } } return nil } // Used to signal inbound message to registered consumers. type cMsg struct { seq uint64 subj string } // Pool to recycle consumer bound msgs. var cMsgPool sync.Pool // Used to queue up consumer bound msgs for signaling. func newCMsg(subj string, seq uint64) *cMsg { var m *cMsg cm := cMsgPool.Get() if cm != nil { m = cm.(*cMsg) } else { m = new(cMsg) } m.subj, m.seq = subj, seq return m } func (m *cMsg) returnToPool() { if m == nil { return } m.subj, m.seq = _EMPTY_, 0 cMsgPool.Put(m) } // Go routine to signal consumers. // Offloaded from stream msg processing. func (mset *stream) signalConsumersLoop() { mset.mu.RLock() s, qch, sch, msgs := mset.srv, mset.qch, mset.sch, mset.sigq mset.mu.RUnlock() for { select { case <-s.quitCh: return case <-qch: return case <-sch: cms := msgs.pop() for _, m := range cms { seq, subj := m.seq, m.subj m.returnToPool() // Signal all appropriate consumers. mset.signalConsumers(subj, seq) } msgs.recycle(&cms) } } } // This will update and signal all consumers that match. func (mset *stream) signalConsumers(subj string, seq uint64) { mset.clsMu.RLock() defer mset.clsMu.RUnlock() csl := mset.csl if csl == nil { return } csl.Match(subj, func(o *consumer) { o.processStreamSignal(seq) }) } // Internal message for use by jetstream subsystem. type jsPubMsg struct { dsubj string // Subject to send to, e.g. _INBOX.xxx reply string StoreMsg o *consumer } var jsPubMsgPool sync.Pool func newJSPubMsg(dsubj, subj, reply string, hdr, msg []byte, o *consumer, seq uint64) *jsPubMsg { var m *jsPubMsg var buf []byte pm := jsPubMsgPool.Get() if pm != nil { m = pm.(*jsPubMsg) buf = m.buf[:0] if hdr != nil { hdr = append(m.hdr[:0], hdr...) } } else { m = new(jsPubMsg) } // When getting something from a pool it is critical that all fields are // initialized. Doing this way guarantees that if someone adds a field to // the structure, the compiler will fail the build if this line is not updated. (*m) = jsPubMsg{dsubj, reply, StoreMsg{subj, hdr, msg, buf, seq, 0}, o} return m } // Gets a jsPubMsg from the pool. func getJSPubMsgFromPool() *jsPubMsg { pm := jsPubMsgPool.Get() if pm != nil { return pm.(*jsPubMsg) } return new(jsPubMsg) } func (pm *jsPubMsg) returnToPool() { if pm == nil { return } pm.subj, pm.dsubj, pm.reply, pm.hdr, pm.msg, pm.o = _EMPTY_, _EMPTY_, _EMPTY_, nil, nil, nil if len(pm.buf) > 0 { pm.buf = pm.buf[:0] } if len(pm.hdr) > 0 { pm.hdr = pm.hdr[:0] } jsPubMsgPool.Put(pm) } func (pm *jsPubMsg) size() int { if pm == nil { return 0 } return len(pm.dsubj) + len(pm.reply) + len(pm.hdr) + len(pm.msg) } // Queue of *jsPubMsg for sending internal system messages. type jsOutQ struct { *ipQueue[*jsPubMsg] } func (q *jsOutQ) sendMsg(subj string, msg []byte) { if q != nil { q.send(newJSPubMsg(subj, _EMPTY_, _EMPTY_, nil, msg, nil, 0)) } } func (q *jsOutQ) send(msg *jsPubMsg) { if q == nil || msg == nil { return } q.push(msg) } func (q *jsOutQ) unregister() { if q == nil { return } q.ipQueue.unregister() } // StoredMsg is for raw access to messages in a stream. type StoredMsg struct { Subject string `json:"subject"` Sequence uint64 `json:"seq"` Header []byte `json:"hdrs,omitempty"` Data []byte `json:"data,omitempty"` Time time.Time `json:"time"` } // This is similar to system semantics but did not want to overload the single system sendq, // or require system account when doing simple setup with jetstream. func (mset *stream) setupSendCapabilities() { mset.mu.Lock() defer mset.mu.Unlock() if mset.outq != nil { return } qname := fmt.Sprintf("[ACC:%s] stream '%s' sendQ", mset.acc.Name, mset.cfg.Name) mset.outq = &jsOutQ{newIPQueue[*jsPubMsg](mset.srv, qname)} go mset.internalLoop() } // Returns the associated account name. func (mset *stream) accName() string { if mset == nil { return _EMPTY_ } mset.mu.RLock() acc := mset.acc mset.mu.RUnlock() return acc.Name } // Name returns the stream name. func (mset *stream) name() string { if mset == nil { return _EMPTY_ } mset.mu.RLock() defer mset.mu.RUnlock() return mset.cfg.Name } func (mset *stream) internalLoop() { mset.mu.RLock() setGoRoutineLabels(pprofLabels{ "account": mset.acc.Name, "stream": mset.cfg.Name, }) s := mset.srv c := s.createInternalJetStreamClient() c.registerWithAccount(mset.acc) defer c.closeConnection(ClientClosed) outq, qch, msgs, gets := mset.outq, mset.qch, mset.msgs, mset.gets // For the ack msgs queue for interest retention. var ( amch chan struct{} ackq *ipQueue[uint64] ) if mset.ackq != nil { ackq, amch = mset.ackq, mset.ackq.ch } mset.mu.RUnlock() // Raw scratch buffer. // This should be rarely used now so can be smaller. var _r [1024]byte // To optimize for not converting a string to a []byte slice. var ( subj [256]byte dsubj [256]byte rply [256]byte szb [10]byte hdb [10]byte ) for { select { case <-outq.ch: pms := outq.pop() for _, pm := range pms { c.pa.subject = append(dsubj[:0], pm.dsubj...) c.pa.deliver = append(subj[:0], pm.subj...) c.pa.size = len(pm.msg) + len(pm.hdr) c.pa.szb = append(szb[:0], strconv.Itoa(c.pa.size)...) if len(pm.reply) > 0 { c.pa.reply = append(rply[:0], pm.reply...) } else { c.pa.reply = nil } // If we have an underlying buf that is the wire contents for hdr + msg, else construct on the fly. var msg []byte if len(pm.buf) > 0 { msg = pm.buf } else { if len(pm.hdr) > 0 { msg = pm.hdr if len(pm.msg) > 0 { msg = _r[:0] msg = append(msg, pm.hdr...) msg = append(msg, pm.msg...) } } else if len(pm.msg) > 0 { // We own this now from a low level buffer perspective so can use directly here. msg = pm.msg } } if len(pm.hdr) > 0 { c.pa.hdr = len(pm.hdr) c.pa.hdb = []byte(strconv.Itoa(c.pa.hdr)) c.pa.hdb = append(hdb[:0], strconv.Itoa(c.pa.hdr)...) } else { c.pa.hdr = -1 c.pa.hdb = nil } msg = append(msg, _CRLF_...) didDeliver, _ := c.processInboundClientMsg(msg) c.pa.szb, c.pa.subject, c.pa.deliver = nil, nil, nil // Check to see if this is a delivery for a consumer and // we failed to deliver the message. If so alert the consumer. if pm.o != nil && pm.seq > 0 && !didDeliver { pm.o.didNotDeliver(pm.seq, pm.dsubj) } pm.returnToPool() } // TODO: Move in the for-loop? c.flushClients(0) outq.recycle(&pms) case <-msgs.ch: // This can possibly change now so needs to be checked here. isClustered := mset.IsClustered() ims := msgs.pop() for _, im := range ims { // If we are clustered we need to propose this message to the underlying raft group. if isClustered { mset.processClusteredInboundMsg(im.subj, im.rply, im.hdr, im.msg) } else { mset.processJetStreamMsg(im.subj, im.rply, im.hdr, im.msg, 0, 0) } im.returnToPool() } msgs.recycle(&ims) case <-gets.ch: dgs := gets.pop() for _, dg := range dgs { mset.getDirectRequest(&dg.req, dg.reply) dgPool.Put(dg) } gets.recycle(&dgs) case <-amch: seqs := ackq.pop() for _, seq := range seqs { mset.ackMsg(nil, seq) } ackq.recycle(&seqs) case <-qch: return case <-s.quitCh: return } } } // Used to break consumers out of their monitorConsumer go routines. func (mset *stream) resetAndWaitOnConsumers() { mset.mu.RLock() consumers := make([]*consumer, 0, len(mset.consumers)) for _, o := range mset.consumers { consumers = append(consumers, o) } mset.mu.RUnlock() for _, o := range consumers { if node := o.raftNode(); node != nil { if o.IsLeader() { node.StepDown() } node.Delete() } if o.isMonitorRunning() { o.monitorWg.Wait() } } } // Internal function to delete a stream. func (mset *stream) delete() error { if mset == nil { return nil } return mset.stop(true, true) } // Internal function to stop or delete the stream. func (mset *stream) stop(deleteFlag, advisory bool) error { mset.mu.RLock() js, jsa, name := mset.js, mset.jsa, mset.cfg.Name mset.mu.RUnlock() if jsa == nil { return NewJSNotEnabledForAccountError() } // Remove from our account map first. jsa.mu.Lock() delete(jsa.streams, name) accName := jsa.account.Name jsa.mu.Unlock() // Kick monitor and collect consumers first. mset.mu.Lock() // Mark closed. mset.closed.Store(true) // Signal to the monitor loop. // Can't use qch here. if mset.mqch != nil { close(mset.mqch) mset.mqch = nil } // Stop responding to sync requests. mset.stopClusterSubs() // Unsubscribe from direct stream. mset.unsubscribeToStream(true) // Our info sub if we spun it up. if mset.infoSub != nil { mset.srv.sysUnsubscribe(mset.infoSub) mset.infoSub = nil } // Clean up consumers. var obs []*consumer for _, o := range mset.consumers { obs = append(obs, o) } mset.clsMu.Lock() mset.consumers, mset.cList, mset.csl = nil, nil, nil mset.clsMu.Unlock() // Check if we are a mirror. if mset.mirror != nil && mset.mirror.sub != nil { mset.unsubscribe(mset.mirror.sub) mset.mirror.sub = nil mset.removeInternalConsumer(mset.mirror) } // Now check for sources. if len(mset.sources) > 0 { for _, si := range mset.sources { mset.cancelSourceConsumer(si.iname) } } mset.mu.Unlock() isShuttingDown := js.isShuttingDown() for _, o := range obs { if !o.isClosed() { // Third flag says do not broadcast a signal. // TODO(dlc) - If we have an err here we don't want to stop // but should we log? o.stopWithFlags(deleteFlag, deleteFlag, false, advisory) if !isShuttingDown { o.monitorWg.Wait() } } } mset.mu.Lock() // Send stream delete advisory after the consumers. if deleteFlag && advisory { mset.sendDeleteAdvisoryLocked() } // Quit channel, do this after sending the delete advisory if mset.qch != nil { close(mset.qch) mset.qch = nil } // Cluster cleanup var sa *streamAssignment if n := mset.node; n != nil { if deleteFlag { n.Delete() sa = mset.sa } else { n.Stop() } } // Cleanup duplicate timer if running. if mset.ddtmr != nil { mset.ddtmr.Stop() mset.ddtmr = nil mset.ddmap = nil mset.ddarr = nil mset.ddindex = 0 } sysc := mset.sysc mset.sysc = nil if deleteFlag { // Unregistering ipQueues do not prevent them from push/pop // just will remove them from the central monitoring map mset.msgs.unregister() mset.ackq.unregister() mset.outq.unregister() mset.sigq.unregister() mset.smsgs.unregister() } // Snapshot store. store := mset.store c := mset.client // Clustered cleanup. mset.mu.Unlock() // Check if the stream assignment has the group node specified. // We need this cleared for if the stream gets reassigned here. if sa != nil { js.mu.Lock() if sa.Group != nil { sa.Group.node = nil } js.mu.Unlock() } if c != nil { c.closeConnection(ClientClosed) } if sysc != nil { sysc.closeConnection(ClientClosed) } if deleteFlag { if store != nil { // Ignore errors. store.Delete() } // Release any resources. js.releaseStreamResources(&mset.cfg) // cleanup directories after the stream accDir := filepath.Join(js.config.StoreDir, accName) // Do cleanup in separate go routine similar to how fs will use purge here.. go func() { // no op if not empty os.Remove(filepath.Join(accDir, streamsDir)) os.Remove(accDir) }() } else if store != nil { // Ignore errors. store.Stop() } return nil } func (mset *stream) getMsg(seq uint64) (*StoredMsg, error) { var smv StoreMsg sm, err := mset.store.LoadMsg(seq, &smv) if err != nil { return nil, err } // This only used in tests directly so no need to pool etc. return &StoredMsg{ Subject: sm.subj, Sequence: sm.seq, Header: sm.hdr, Data: sm.msg, Time: time.Unix(0, sm.ts).UTC(), }, nil } // getConsumers will return a copy of all the current consumers for this stream. func (mset *stream) getConsumers() []*consumer { mset.clsMu.RLock() defer mset.clsMu.RUnlock() return append([]*consumer(nil), mset.cList...) } // Lock should be held for this one. func (mset *stream) numPublicConsumers() int { return len(mset.consumers) - mset.directs } // This returns all consumers that are not DIRECT. func (mset *stream) getPublicConsumers() []*consumer { mset.clsMu.RLock() defer mset.clsMu.RUnlock() var obs []*consumer for _, o := range mset.cList { if !o.cfg.Direct { obs = append(obs, o) } } return obs } // Will check for interest retention and make sure messages // that have been acked are processed and removed. // This will check the ack floors of all consumers, and adjust our first sequence accordingly. func (mset *stream) checkInterestState() { if mset == nil || !mset.isInterestRetention() { // If we are limits based nothing to do. return } var ss StreamState mset.store.FastState(&ss) for _, o := range mset.getConsumers() { o.checkStateForInterestStream(&ss) } } func (mset *stream) isInterestRetention() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.cfg.Retention != LimitsPolicy } // NumConsumers reports on number of active consumers for this stream. func (mset *stream) numConsumers() int { mset.mu.RLock() defer mset.mu.RUnlock() return len(mset.consumers) } // Lock should be held. func (mset *stream) setConsumer(o *consumer) { mset.consumers[o.name] = o if len(o.subjf) > 0 { mset.numFilter++ } if o.cfg.Direct { mset.directs++ } // Now update consumers list as well mset.clsMu.Lock() mset.cList = append(mset.cList, o) if mset.csl == nil { mset.csl = gsl.NewSublist[*consumer]() } for _, sub := range o.signalSubs() { mset.csl.Insert(sub, o) } mset.clsMu.Unlock() } // Lock should be held. func (mset *stream) removeConsumer(o *consumer) { if o.cfg.FilterSubject != _EMPTY_ && mset.numFilter > 0 { mset.numFilter-- } if o.cfg.Direct && mset.directs > 0 { mset.directs-- } if mset.consumers != nil { delete(mset.consumers, o.name) // Now update consumers list as well mset.clsMu.Lock() for i, ol := range mset.cList { if ol == o { mset.cList = append(mset.cList[:i], mset.cList[i+1:]...) break } } // Always remove from the leader sublist. if mset.csl != nil { for _, sub := range o.signalSubs() { mset.csl.Remove(sub, o) } } mset.clsMu.Unlock() } } // swapSigSubs will update signal Subs for a new subject filter. // consumer lock should not be held. func (mset *stream) swapSigSubs(o *consumer, newFilters []string) { mset.clsMu.Lock() o.mu.Lock() if o.closed || o.mset == nil { o.mu.Unlock() mset.clsMu.Unlock() return } if o.sigSubs != nil { if mset.csl != nil { for _, sub := range o.sigSubs { mset.csl.Remove(sub, o) } } o.sigSubs = nil } if o.isLeader() { if mset.csl == nil { mset.csl = gsl.NewSublist[*consumer]() } // If no filters are preset, add fwcs to sublist for that consumer. if newFilters == nil { mset.csl.Insert(fwcs, o) o.sigSubs = append(o.sigSubs, fwcs) // If there are filters, add their subjects to sublist. } else { for _, filter := range newFilters { mset.csl.Insert(filter, o) o.sigSubs = append(o.sigSubs, filter) } } } o.mu.Unlock() mset.clsMu.Unlock() mset.mu.Lock() defer mset.mu.Unlock() if mset.numFilter > 0 && len(o.subjf) > 0 { mset.numFilter-- } if len(newFilters) > 0 { mset.numFilter++ } } // lookupConsumer will retrieve a consumer by name. func (mset *stream) lookupConsumer(name string) *consumer { mset.mu.RLock() defer mset.mu.RUnlock() return mset.consumers[name] } func (mset *stream) numDirectConsumers() (num int) { mset.clsMu.RLock() defer mset.clsMu.RUnlock() // Consumers that are direct are not recorded at the store level. for _, o := range mset.cList { o.mu.RLock() if o.cfg.Direct { num++ } o.mu.RUnlock() } return num } // State will return the current state for this stream. func (mset *stream) state() StreamState { return mset.stateWithDetail(false) } func (mset *stream) stateWithDetail(details bool) StreamState { // mset.store does not change once set, so ok to reference here directly. // We do this elsewhere as well. store := mset.store if store == nil { return StreamState{} } // Currently rely on store for details. if details { return store.State() } // Here we do the fast version. var state StreamState store.FastState(&state) return state } func (mset *stream) Store() StreamStore { mset.mu.RLock() defer mset.mu.RUnlock() return mset.store } // Determines if the new proposed partition is unique amongst all consumers. // Lock should be held. func (mset *stream) partitionUnique(name string, partitions []string) bool { for _, partition := range partitions { for n, o := range mset.consumers { // Skip the consumer being checked. if n == name { continue } if o.subjf == nil { return false } for _, filter := range o.subjf { if SubjectsCollide(partition, filter.subject) { return false } } } } return true } // Lock should be held. func (mset *stream) potentialFilteredConsumers() bool { numSubjects := len(mset.cfg.Subjects) if len(mset.consumers) == 0 || numSubjects == 0 { return false } if numSubjects > 1 || subjectHasWildcard(mset.cfg.Subjects[0]) { return true } return false } // Check if there is no interest in this sequence number across our consumers. // The consumer passed is optional if we are processing the ack for that consumer. // Write lock should be held. func (mset *stream) noInterest(seq uint64, obs *consumer) bool { return !mset.checkForInterest(seq, obs) } // Check if there is no interest in this sequence number and subject across our consumers. // The consumer passed is optional if we are processing the ack for that consumer. // Write lock should be held. func (mset *stream) noInterestWithSubject(seq uint64, subj string, obs *consumer) bool { return !mset.checkForInterestWithSubject(seq, subj, obs) } // Write lock should be held here for the stream to avoid race conditions on state. func (mset *stream) checkForInterest(seq uint64, obs *consumer) bool { var subj string if mset.potentialFilteredConsumers() { pmsg := getJSPubMsgFromPool() defer pmsg.returnToPool() sm, err := mset.store.LoadMsg(seq, &pmsg.StoreMsg) if err != nil { if err == ErrStoreEOF { // Register this as a preAck. mset.registerPreAck(obs, seq) return true } mset.clearAllPreAcks(seq) return false } subj = sm.subj } return mset.checkForInterestWithSubject(seq, subj, obs) } // Checks for interest given a sequence and subject. func (mset *stream) checkForInterestWithSubject(seq uint64, subj string, obs *consumer) bool { for _, o := range mset.consumers { // If this is us or we have a registered preAck for this consumer continue inspecting. if o == obs || mset.hasPreAck(o, seq) { continue } // Check if we need an ack. if o.needAck(seq, subj) { return true } } mset.clearAllPreAcks(seq) return false } // Check if we have a pre-registered ack for this sequence. // Write lock should be held. func (mset *stream) hasPreAck(o *consumer, seq uint64) bool { if o == nil || len(mset.preAcks) == 0 { return false } consumers := mset.preAcks[seq] if len(consumers) == 0 { return false } _, found := consumers[o] return found } // Check if we have all consumers pre-acked for this sequence and subject. // Write lock should be held. func (mset *stream) hasAllPreAcks(seq uint64, subj string) bool { if len(mset.preAcks) == 0 || len(mset.preAcks[seq]) == 0 { return false } // Since these can be filtered and mutually exclusive, // if we have some preAcks we need to check all interest here. return mset.noInterestWithSubject(seq, subj, nil) } // Check if we have all consumers pre-acked. // Write lock should be held. func (mset *stream) clearAllPreAcks(seq uint64) { delete(mset.preAcks, seq) } // Clear all preAcks below floor. // Write lock should be held. func (mset *stream) clearAllPreAcksBelowFloor(floor uint64) { for seq := range mset.preAcks { if seq < floor { delete(mset.preAcks, seq) } } } // This will register an ack for a consumer if it arrives before the actual message. func (mset *stream) registerPreAckLock(o *consumer, seq uint64) { mset.mu.Lock() defer mset.mu.Unlock() mset.registerPreAck(o, seq) } // This will register an ack for a consumer if it arrives before // the actual message. // Write lock should be held. func (mset *stream) registerPreAck(o *consumer, seq uint64) { if o == nil { return } if mset.preAcks == nil { mset.preAcks = make(map[uint64]map[*consumer]struct{}) } if mset.preAcks[seq] == nil { mset.preAcks[seq] = make(map[*consumer]struct{}) } mset.preAcks[seq][o] = struct{}{} } // This will clear an ack for a consumer. // Write lock should be held. func (mset *stream) clearPreAck(o *consumer, seq uint64) { if o == nil || len(mset.preAcks) == 0 { return } if consumers := mset.preAcks[seq]; len(consumers) > 0 { delete(consumers, o) if len(consumers) == 0 { delete(mset.preAcks, seq) } } } // ackMsg is called into from a consumer when we have a WorkQueue or Interest Retention Policy. // Returns whether the message at seq was removed as a result of the ACK. func (mset *stream) ackMsg(o *consumer, seq uint64) bool { if seq == 0 { return false } // Don't make this RLock(). We need to have only 1 running at a time to gauge interest across all consumers. mset.mu.Lock() if mset.closed.Load() || mset.cfg.Retention == LimitsPolicy { mset.mu.Unlock() return false } store := mset.store var state StreamState store.FastState(&state) // If this has arrived before we have processed the message itself. if seq > state.LastSeq { mset.registerPreAck(o, seq) mset.mu.Unlock() // We have not removed the message, but should still signal so we could retry later // since we potentially need to remove it then. return true } // Always clear pre-ack if here. mset.clearPreAck(o, seq) // Make sure this sequence is not below our first sequence. if seq < state.FirstSeq { mset.mu.Unlock() return false } var shouldRemove bool switch mset.cfg.Retention { case WorkQueuePolicy: // Normally we just remove a message when its ack'd here but if we have direct consumers // from sources and/or mirrors we need to make sure they have delivered the msg. shouldRemove = mset.directs <= 0 || mset.noInterest(seq, o) case InterestPolicy: shouldRemove = mset.noInterest(seq, o) } mset.mu.Unlock() // If nothing else to do. if !shouldRemove { return false } // If we are here we should attempt to remove. if _, err := store.RemoveMsg(seq); err == ErrStoreEOF { // This should not happen, but being pedantic. mset.registerPreAckLock(o, seq) } return true } // Snapshot creates a snapshot for the stream and possibly consumers. func (mset *stream) snapshot(deadline time.Duration, checkMsgs, includeConsumers bool) (*SnapshotResult, error) { if mset.closed.Load() { return nil, errStreamClosed } store := mset.store return store.Snapshot(deadline, checkMsgs, includeConsumers) } const snapsDir = "__snapshots__" // RestoreStream will restore a stream from a snapshot. func (a *Account) RestoreStream(ncfg *StreamConfig, r io.Reader) (*stream, error) { if ncfg == nil { return nil, errors.New("nil config on stream restore") } s, jsa, err := a.checkForJetStream() if err != nil { return nil, err } js := jsa.js if js == nil { return nil, NewJSNotEnabledForAccountError() } cfg, apiErr := s.checkStreamCfg(ncfg, a) if apiErr != nil { return nil, apiErr } sd := filepath.Join(jsa.storeDir, snapsDir) if _, err := os.Stat(sd); os.IsNotExist(err) { if err := os.MkdirAll(sd, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create snapshots directory - %v", err) } } sdir, err := os.MkdirTemp(sd, "snap-") if err != nil { return nil, err } if _, err := os.Stat(sdir); os.IsNotExist(err) { if err := os.MkdirAll(sdir, defaultDirPerms); err != nil { return nil, fmt.Errorf("could not create snapshots directory - %v", err) } } defer os.RemoveAll(sdir) logAndReturnError := func() error { a.mu.RLock() err := fmt.Errorf("unexpected content (account=%s)", a.Name) if a.srv != nil { a.srv.Errorf("Stream restore failed due to %v", err) } a.mu.RUnlock() return err } sdirCheck := filepath.Clean(sdir) + string(os.PathSeparator) _, isClustered := jsa.jetStreamAndClustered() jsa.usageMu.RLock() selected, tier, hasTier := jsa.selectLimits(cfg.Replicas) jsa.usageMu.RUnlock() reserved := int64(0) if hasTier { if isClustered { js.mu.RLock() _, reserved = tieredStreamAndReservationCount(js.cluster.streams[a.Name], tier, &cfg) js.mu.RUnlock() } else { reserved = jsa.tieredReservation(tier, &cfg) } } var bc int64 tr := tar.NewReader(s2.NewReader(r)) for { hdr, err := tr.Next() if err == io.EOF { break // End of snapshot } if err != nil { return nil, err } if hdr.Typeflag != tar.TypeReg { return nil, logAndReturnError() } bc += hdr.Size js.mu.RLock() err = js.checkAllLimits(&selected, &cfg, reserved, bc) js.mu.RUnlock() if err != nil { return nil, err } fpath := filepath.Join(sdir, filepath.Clean(hdr.Name)) if !strings.HasPrefix(fpath, sdirCheck) { return nil, logAndReturnError() } os.MkdirAll(filepath.Dir(fpath), defaultDirPerms) fd, err := os.OpenFile(fpath, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, err } _, err = io.Copy(fd, tr) fd.Close() if err != nil { return nil, err } } // Check metadata. // The cfg passed in will be the new identity for the stream. var fcfg FileStreamInfo b, err := os.ReadFile(filepath.Join(sdir, JetStreamMetaFile)) if err != nil { return nil, err } if err := json.Unmarshal(b, &fcfg); err != nil { return nil, err } // Check to make sure names match. if fcfg.Name != cfg.Name { return nil, errors.New("stream names do not match") } // See if this stream already exists. if _, err := a.lookupStream(cfg.Name); err == nil { return nil, NewJSStreamNameExistRestoreFailedError() } // Move into the correct place here. ndir := filepath.Join(jsa.storeDir, streamsDir, cfg.Name) // Remove old one if for some reason it is still here. if _, err := os.Stat(ndir); err == nil { os.RemoveAll(ndir) } // Make sure our destination streams directory exists. if err := os.MkdirAll(filepath.Join(jsa.storeDir, streamsDir), defaultDirPerms); err != nil { return nil, err } // Move into new location. if err := os.Rename(sdir, ndir); err != nil { return nil, err } if cfg.Template != _EMPTY_ { if err := jsa.addStreamNameToTemplate(cfg.Template, cfg.Name); err != nil { return nil, err } } mset, err := a.addStream(&cfg) if err != nil { // Make sure to clean up after ourselves here. os.RemoveAll(ndir) return nil, err } if !fcfg.Created.IsZero() { mset.setCreatedTime(fcfg.Created) } lseq := mset.lastSeq() // Make sure we do an update if the configs have changed. if !reflect.DeepEqual(fcfg.StreamConfig, cfg) { if err := mset.update(&cfg); err != nil { return nil, err } } // Now do consumers. odir := filepath.Join(ndir, consumerDir) ofis, _ := os.ReadDir(odir) for _, ofi := range ofis { metafile := filepath.Join(odir, ofi.Name(), JetStreamMetaFile) metasum := filepath.Join(odir, ofi.Name(), JetStreamMetaFileSum) if _, err := os.Stat(metafile); os.IsNotExist(err) { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } buf, err := os.ReadFile(metafile) if err != nil { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } if _, err := os.Stat(metasum); os.IsNotExist(err) { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } var cfg FileConsumerInfo if err := json.Unmarshal(buf, &cfg); err != nil { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } isEphemeral := !isDurableConsumer(&cfg.ConsumerConfig) if isEphemeral { // This is an ephermal consumer and this could fail on restart until // the consumer can reconnect. We will create it as a durable and switch it. cfg.ConsumerConfig.Durable = ofi.Name() } obs, err := mset.addConsumer(&cfg.ConsumerConfig) if err != nil { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } if isEphemeral { obs.switchToEphemeral() } if !cfg.Created.IsZero() { obs.setCreatedTime(cfg.Created) } obs.mu.Lock() err = obs.readStoredState(lseq) obs.mu.Unlock() if err != nil { mset.stop(true, false) return nil, fmt.Errorf("error restoring consumer [%q]: %v", ofi.Name(), err) } } return mset, nil } // This is to check for dangling messages on interest retention streams. Only called on account enable. // Issue https://github.com/nats-io/nats-server/issues/3612 func (mset *stream) checkForOrphanMsgs() { mset.mu.RLock() consumers := make([]*consumer, 0, len(mset.consumers)) for _, o := range mset.consumers { consumers = append(consumers, o) } accName, stream := mset.acc.Name, mset.cfg.Name var ss StreamState mset.store.FastState(&ss) mset.mu.RUnlock() for _, o := range consumers { if err := o.checkStateForInterestStream(&ss); err == errAckFloorHigherThanLastSeq { o.mu.RLock() s, consumer := o.srv, o.name state, _ := o.store.State() asflr := state.AckFloor.Stream o.mu.RUnlock() // Warn about stream state vs our ack floor. s.RateLimitWarnf("Detected consumer '%s > %s > %s' ack floor %d is ahead of stream's last sequence %d", accName, stream, consumer, asflr, ss.LastSeq) } } } // Check on startup to make sure that consumers replication matches us. // Interest retention requires replication matches. func (mset *stream) checkConsumerReplication() { mset.mu.RLock() defer mset.mu.RUnlock() if mset.cfg.Retention != InterestPolicy { return } s, acc := mset.srv, mset.acc for _, o := range mset.consumers { o.mu.RLock() // Consumer replicas 0 can be a legit config for the replicas and we will inherit from the stream // when this is the case. if mset.cfg.Replicas != o.cfg.Replicas && o.cfg.Replicas != 0 { s.Errorf("consumer '%s > %s > %s' MUST match replication (%d vs %d) of stream with interest policy", acc, mset.cfg.Name, o.cfg.Name, mset.cfg.Replicas, o.cfg.Replicas) } o.mu.RUnlock() } } // Will check if we are running in the monitor already and if not set the appropriate flag. func (mset *stream) checkInMonitor() bool { mset.mu.Lock() defer mset.mu.Unlock() if mset.inMonitor { return true } mset.inMonitor = true return false } // Clear us being in the monitor routine. func (mset *stream) clearMonitorRunning() { mset.mu.Lock() defer mset.mu.Unlock() mset.inMonitor = false } // Check if our monitor is running. func (mset *stream) isMonitorRunning() bool { mset.mu.RLock() defer mset.mu.RUnlock() return mset.inMonitor } nats-server-2.10.27/server/stree/000077500000000000000000000000001477524627100166035ustar00rootroot00000000000000nats-server-2.10.27/server/stree/dump.go000066400000000000000000000035451477524627100201060ustar00rootroot00000000000000// Copyright 2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import ( "fmt" "io" "strings" ) // For dumping out a text representation of a tree. func (t *SubjectTree[T]) Dump(w io.Writer) { t.dump(w, t.root, 0) fmt.Fprintln(w) } // Will dump out a node. func (t *SubjectTree[T]) dump(w io.Writer, n node, depth int) { if n == nil { fmt.Fprintf(w, "EMPTY\n") return } if n.isLeaf() { leaf := n.(*leaf[T]) fmt.Fprintf(w, "%s LEAF: Suffix: %q Value: %+v\n", dumpPre(depth), leaf.suffix, leaf.value) n = nil } else { // We are a node type here, grab meta portion. bn := n.base() fmt.Fprintf(w, "%s %s Prefix: %q\n", dumpPre(depth), n.kind(), bn.prefix) depth++ n.iter(func(n node) bool { t.dump(w, n, depth) return true }) } } // For individual node/leaf dumps. func (n *leaf[T]) kind() string { return "LEAF" } func (n *node4) kind() string { return "NODE4" } func (n *node10) kind() string { return "NODE10" } func (n *node16) kind() string { return "NODE16" } func (n *node48) kind() string { return "NODE48" } func (n *node256) kind() string { return "NODE256" } // Calculates the indendation, etc. func dumpPre(depth int) string { if depth == 0 { return "-- " } else { var b strings.Builder for i := 0; i < depth; i++ { b.WriteString(" ") } b.WriteString("|__ ") return b.String() } } nats-server-2.10.27/server/stree/helper_test.go000066400000000000000000000017411477524627100214530ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import "testing" func require_True(t *testing.T, b bool) { t.Helper() if !b { t.Fatalf("require true, but got false") } } func require_False(t *testing.T, b bool) { t.Helper() if b { t.Fatalf("require false, but got true") } } func require_Equal[T comparable](t *testing.T, a, b T) { t.Helper() if a != b { t.Fatalf("require %T equal, but got: %v != %v", a, a, b) } } nats-server-2.10.27/server/stree/leaf.go000066400000000000000000000046431477524627100200500ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import ( "bytes" ) // Leaf node // Order of struct fields for best memory alignment (as per govet/fieldalignment) type leaf[T any] struct { value T // This could be the whole subject, but most likely just the suffix portion. // We will only store the suffix here and assume all prior prefix paths have // been checked once we arrive at this leafnode. suffix []byte } func newLeaf[T any](suffix []byte, value T) *leaf[T] { return &leaf[T]{value, copyBytes(suffix)} } func (n *leaf[T]) isLeaf() bool { return true } func (n *leaf[T]) base() *meta { return nil } func (n *leaf[T]) match(subject []byte) bool { return bytes.Equal(subject, n.suffix) } func (n *leaf[T]) setSuffix(suffix []byte) { n.suffix = copyBytes(suffix) } func (n *leaf[T]) isFull() bool { return true } func (n *leaf[T]) matchParts(parts [][]byte) ([][]byte, bool) { return matchParts(parts, n.suffix) } func (n *leaf[T]) iter(f func(node) bool) {} func (n *leaf[T]) children() []node { return nil } func (n *leaf[T]) numChildren() uint16 { return 0 } func (n *leaf[T]) path() []byte { return n.suffix } // Not applicable to leafs and should not be called, so panic if we do. func (n *leaf[T]) setPrefix(pre []byte) { panic("setPrefix called on leaf") } func (n *leaf[T]) addChild(_ byte, _ node) { panic("addChild called on leaf") } func (n *leaf[T]) findChild(_ byte) *node { panic("findChild called on leaf") } func (n *leaf[T]) grow() node { panic("grow called on leaf") } func (n *leaf[T]) deleteChild(_ byte) { panic("deleteChild called on leaf") } func (n *leaf[T]) shrink() node { panic("shrink called on leaf") } nats-server-2.10.27/server/stree/node.go000066400000000000000000000026411477524627100200620ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Internal node interface. type node interface { isLeaf() bool base() *meta setPrefix(pre []byte) addChild(c byte, n node) findChild(c byte) *node deleteChild(c byte) isFull() bool grow() node shrink() node matchParts(parts [][]byte) ([][]byte, bool) kind() string iter(f func(node) bool) children() []node numChildren() uint16 path() []byte } type meta struct { prefix []byte size uint16 } func (n *meta) isLeaf() bool { return false } func (n *meta) base() *meta { return n } func (n *meta) setPrefix(pre []byte) { n.prefix = append([]byte(nil), pre...) } func (n *meta) numChildren() uint16 { return n.size } func (n *meta) path() []byte { return n.prefix } // Will match parts against our prefix. func (n *meta) matchParts(parts [][]byte) ([][]byte, bool) { return matchParts(parts, n.prefix) } nats-server-2.10.27/server/stree/node10.go000066400000000000000000000050631477524627100202240ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Node with 10 children // This node size is for the particular case that a part of the subject is numeric // in nature, i.e. it only needs to satisfy the range 0-9 without wasting bytes // Order of struct fields for best memory alignment (as per govet/fieldalignment) type node10 struct { child [10]node meta key [10]byte } func newNode10(prefix []byte) *node10 { nn := &node10{} nn.setPrefix(prefix) return nn } // Currently we do not keep node10 sorted or use bitfields for traversal so just add to the end. // TODO(dlc) - We should revisit here with more detailed benchmarks. func (n *node10) addChild(c byte, nn node) { if n.size >= 10 { panic("node10 full!") } n.key[n.size] = c n.child[n.size] = nn n.size++ } func (n *node10) findChild(c byte) *node { for i := uint16(0); i < n.size; i++ { if n.key[i] == c { return &n.child[i] } } return nil } func (n *node10) isFull() bool { return n.size >= 10 } func (n *node10) grow() node { nn := newNode16(n.prefix) for i := 0; i < 10; i++ { nn.addChild(n.key[i], n.child[i]) } return nn } // Deletes a child from the node. func (n *node10) deleteChild(c byte) { for i, last := uint16(0), n.size-1; i < n.size; i++ { if n.key[i] == c { // Unsorted so just swap in last one here, else nil if last. if i < last { n.key[i] = n.key[last] n.child[i] = n.child[last] n.key[last] = 0 n.child[last] = nil } else { n.key[i] = 0 n.child[i] = nil } n.size-- return } } } // Shrink if needed and return new node, otherwise return nil. func (n *node10) shrink() node { if n.size > 4 { return nil } nn := newNode4(nil) for i := uint16(0); i < n.size; i++ { nn.addChild(n.key[i], n.child[i]) } return nn } // Iterate over all children calling func f. func (n *node10) iter(f func(node) bool) { for i := uint16(0); i < n.size; i++ { if !f(n.child[i]) { return } } } // Return our children as a slice. func (n *node10) children() []node { return n.child[:n.size] } nats-server-2.10.27/server/stree/node16.go000066400000000000000000000046221477524627100202320ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Node with 16 children // Order of struct fields for best memory alignment (as per govet/fieldalignment) type node16 struct { child [16]node meta key [16]byte } func newNode16(prefix []byte) *node16 { nn := &node16{} nn.setPrefix(prefix) return nn } // Currently we do not keep node16 sorted or use bitfields for traversal so just add to the end. // TODO(dlc) - We should revisit here with more detailed benchmarks. func (n *node16) addChild(c byte, nn node) { if n.size >= 16 { panic("node16 full!") } n.key[n.size] = c n.child[n.size] = nn n.size++ } func (n *node16) findChild(c byte) *node { for i := uint16(0); i < n.size; i++ { if n.key[i] == c { return &n.child[i] } } return nil } func (n *node16) isFull() bool { return n.size >= 16 } func (n *node16) grow() node { nn := newNode48(n.prefix) for i := 0; i < 16; i++ { nn.addChild(n.key[i], n.child[i]) } return nn } // Deletes a child from the node. func (n *node16) deleteChild(c byte) { for i, last := uint16(0), n.size-1; i < n.size; i++ { if n.key[i] == c { // Unsorted so just swap in last one here, else nil if last. if i < last { n.key[i] = n.key[last] n.child[i] = n.child[last] n.key[last] = 0 n.child[last] = nil } else { n.key[i] = 0 n.child[i] = nil } n.size-- return } } } // Shrink if needed and return new node, otherwise return nil. func (n *node16) shrink() node { if n.size > 10 { return nil } nn := newNode10(nil) for i := uint16(0); i < n.size; i++ { nn.addChild(n.key[i], n.child[i]) } return nn } // Iterate over all children calling func f. func (n *node16) iter(f func(node) bool) { for i := uint16(0); i < n.size; i++ { if !f(n.child[i]) { return } } } // Return our children as a slice. func (n *node16) children() []node { return n.child[:n.size] } nats-server-2.10.27/server/stree/node256.go000066400000000000000000000035001477524627100203120ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Node with 256 children // Order of struct fields for best memory alignment (as per govet/fieldalignment) type node256 struct { child [256]node meta } func newNode256(prefix []byte) *node256 { nn := &node256{} nn.setPrefix(prefix) return nn } func (n *node256) addChild(c byte, nn node) { n.child[c] = nn n.size++ } func (n *node256) findChild(c byte) *node { if n.child[c] != nil { return &n.child[c] } return nil } func (n *node256) isFull() bool { return false } func (n *node256) grow() node { panic("grow can not be called on node256") } // Deletes a child from the node. func (n *node256) deleteChild(c byte) { if n.child[c] != nil { n.child[c] = nil n.size-- } } // Shrink if needed and return new node, otherwise return nil. func (n *node256) shrink() node { if n.size > 48 { return nil } nn := newNode48(nil) for c, child := range n.child { if child != nil { nn.addChild(byte(c), n.child[c]) } } return nn } // Iterate over all children calling func f. func (n *node256) iter(f func(node) bool) { for i := 0; i < 256; i++ { if n.child[i] != nil { if !f(n.child[i]) { return } } } } // Return our children as a slice. func (n *node256) children() []node { return n.child[:256] } nats-server-2.10.27/server/stree/node4.go000066400000000000000000000043161477524627100201470ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Node with 4 children // Order of struct fields for best memory alignment (as per govet/fieldalignment) type node4 struct { child [4]node meta key [4]byte } func newNode4(prefix []byte) *node4 { nn := &node4{} nn.setPrefix(prefix) return nn } // Currently we do not need to keep sorted for traversal so just add to the end. func (n *node4) addChild(c byte, nn node) { if n.size >= 4 { panic("node4 full!") } n.key[n.size] = c n.child[n.size] = nn n.size++ } func (n *node4) findChild(c byte) *node { for i := uint16(0); i < n.size; i++ { if n.key[i] == c { return &n.child[i] } } return nil } func (n *node4) isFull() bool { return n.size >= 4 } func (n *node4) grow() node { nn := newNode10(n.prefix) for i := 0; i < 4; i++ { nn.addChild(n.key[i], n.child[i]) } return nn } // Deletes a child from the node. func (n *node4) deleteChild(c byte) { for i, last := uint16(0), n.size-1; i < n.size; i++ { if n.key[i] == c { // Unsorted so just swap in last one here, else nil if last. if i < last { n.key[i] = n.key[last] n.child[i] = n.child[last] n.key[last] = 0 n.child[last] = nil } else { n.key[i] = 0 n.child[i] = nil } n.size-- return } } } // Shrink if needed and return new node, otherwise return nil. func (n *node4) shrink() node { if n.size == 1 { return n.child[0] } return nil } // Iterate over all children calling func f. func (n *node4) iter(f func(node) bool) { for i := uint16(0); i < n.size; i++ { if !f(n.child[i]) { return } } } // Return our children as a slice. func (n *node4) children() []node { return n.child[:n.size] } nats-server-2.10.27/server/stree/node48.go000066400000000000000000000047711477524627100202440ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // Node with 48 children // Memory saving vs node256 comes from the fact that the child array is 16 bytes // per `node` entry, so node256's 256*16=4096 vs node48's 256+(48*16)=1024 // Note that `key` is effectively 1-indexed, as 0 means no entry, so offset by 1 // Order of struct fields for best memory alignment (as per govet/fieldalignment) type node48 struct { child [48]node meta key [256]byte } func newNode48(prefix []byte) *node48 { nn := &node48{} nn.setPrefix(prefix) return nn } func (n *node48) addChild(c byte, nn node) { if n.size >= 48 { panic("node48 full!") } n.child[n.size] = nn n.key[c] = byte(n.size + 1) // 1-indexed n.size++ } func (n *node48) findChild(c byte) *node { i := n.key[c] if i == 0 { return nil } return &n.child[i-1] } func (n *node48) isFull() bool { return n.size >= 48 } func (n *node48) grow() node { nn := newNode256(n.prefix) for c := 0; c < len(n.key); c++ { if i := n.key[byte(c)]; i > 0 { nn.addChild(byte(c), n.child[i-1]) } } return nn } // Deletes a child from the node. func (n *node48) deleteChild(c byte) { i := n.key[c] if i == 0 { return } i-- // Adjust for 1-indexing last := byte(n.size - 1) if i < last { n.child[i] = n.child[last] for ic := 0; ic < len(n.key); ic++ { if n.key[byte(ic)] == last+1 { n.key[byte(ic)] = i + 1 break } } } n.child[last] = nil n.key[c] = 0 n.size-- } // Shrink if needed and return new node, otherwise return nil. func (n *node48) shrink() node { if n.size > 16 { return nil } nn := newNode16(nil) for c := 0; c < len(n.key); c++ { if i := n.key[byte(c)]; i > 0 { nn.addChild(byte(c), n.child[i-1]) } } return nn } // Iterate over all children calling func f. func (n *node48) iter(f func(node) bool) { for _, c := range n.child { if c != nil && !f(c) { return } } } // Return our children as a slice. func (n *node48) children() []node { return n.child[:n.size] } nats-server-2.10.27/server/stree/parts.go000066400000000000000000000077441477524627100202770ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import ( "bytes" ) // genParts will break a filter subject up into parts. // We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'. // We do not care about other tokens per se, just parts that are separated by wildcards with an optional end fwc. func genParts(filter []byte, parts [][]byte) [][]byte { var start int for i, e := 0, len(filter)-1; i < len(filter); i++ { if filter[i] == tsep { // See if next token is pwc. Either internal or end pwc. if i < e && filter[i+1] == pwc && (i+2 <= e && filter[i+2] == tsep || i+1 == e) { if i > start { parts = append(parts, filter[start:i+1]) } parts = append(parts, filter[i+1:i+2]) i++ // Skip pwc if i+2 <= e { i++ // Skip next tsep from next part too. } start = i + 1 } else if i < e && filter[i+1] == fwc && i+1 == e { // We have a fwc if i > start { parts = append(parts, filter[start:i+1]) } parts = append(parts, filter[i+1:i+2]) i++ // Skip fwc start = i + 1 } } else if filter[i] == pwc || filter[i] == fwc { // Wildcard must be at the start or preceded by tsep. if prev := i - 1; prev >= 0 && filter[prev] != tsep { continue } // Wildcard must be at the end or followed by tsep. if next := i + 1; next == e || next < e && filter[next] != tsep { continue } // We start with a pwc or fwc. parts = append(parts, filter[i:i+1]) if i+1 <= e { i++ // Skip next tsep from next part too. } start = i + 1 } } if start < len(filter) { // Check to see if we need to eat a leading tsep. if filter[start] == tsep { start++ } parts = append(parts, filter[start:]) } return parts } // Match our parts against a fragment, which could be prefix for nodes or a suffix for leafs. func matchParts(parts [][]byte, frag []byte) ([][]byte, bool) { lf := len(frag) if lf == 0 { return parts, true } var si int lpi := len(parts) - 1 for i, part := range parts { if si >= lf { return parts[i:], true } lp := len(part) // Check for pwc or fwc place holders. if lp == 1 { if part[0] == pwc { index := bytes.IndexByte(frag[si:], tsep) // We are trying to match pwc and did not find our tsep. // Will need to move to next node from caller. if index < 0 { if i == lpi { return nil, true } return parts[i:], true } si += index + 1 continue } else if part[0] == fwc { // If we are here we should be good. return nil, true } } end := min(si+lp, lf) // If part is bigger then the remaining fragment, adjust to a portion on the part. if si+lp > end { // Frag is smaller then part itself. part = part[:end-si] } if !bytes.Equal(part, frag[si:end]) { return parts, false } // If we still have a portion of the fragment left, update and continue. if end < lf { si = end continue } // If we matched a partial, do not move past current part // but update the part to what was consumed. This allows upper layers to continue. if end < si+lp { if end >= lf { parts = append([][]byte{}, parts...) // Create a copy before modifying. parts[i] = parts[i][lf-si:] } else { i++ } return parts[i:], true } if i == lpi { return nil, true } // If we are here we are not the last part which means we have a wildcard // gap, so we need to match anything up to next tsep. si += len(part) } return parts, false } nats-server-2.10.27/server/stree/stree.go000066400000000000000000000277251477524627100202710ustar00rootroot00000000000000// Copyright 2023-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import ( "bytes" "slices" ) // SubjectTree is an adaptive radix trie (ART) for storing subject information on literal subjects. // Will use dynamic nodes, path compression and lazy expansion. // The reason this exists is to not only save some memory in our filestore but to greatly optimize matching // a wildcard subject to certain members, e.g. consumer NumPending calculations. type SubjectTree[T any] struct { root node size int } // NewSubjectTree creates a new SubjectTree with values T. func NewSubjectTree[T any]() *SubjectTree[T] { return &SubjectTree[T]{} } // Size returns the number of elements stored. func (t *SubjectTree[T]) Size() int { if t == nil { return 0 } return t.size } // Will empty out the tree, or if tree is nil create a new one. func (t *SubjectTree[T]) Empty() *SubjectTree[T] { if t == nil { return NewSubjectTree[T]() } t.root, t.size = nil, 0 return t } // Insert a value into the tree. Will return if the value was updated and if so the old value. func (t *SubjectTree[T]) Insert(subject []byte, value T) (*T, bool) { if t == nil { return nil, false } // Make sure we never insert anything with a noPivot byte. if bytes.IndexByte(subject, noPivot) >= 0 { return nil, false } old, updated := t.insert(&t.root, subject, value, 0) if !updated { t.size++ } return old, updated } // Find will find the value and return it or false if it was not found. func (t *SubjectTree[T]) Find(subject []byte) (*T, bool) { if t == nil { return nil, false } var si int for n := t.root; n != nil; { if n.isLeaf() { if ln := n.(*leaf[T]); ln.match(subject[si:]) { return &ln.value, true } return nil, false } // We are a node type here, grab meta portion. if bn := n.base(); len(bn.prefix) > 0 { end := min(si+len(bn.prefix), len(subject)) if !bytes.Equal(subject[si:end], bn.prefix) { return nil, false } // Increment our subject index. si += len(bn.prefix) } if an := n.findChild(pivot(subject, si)); an != nil { n = *an } else { return nil, false } } return nil, false } // Delete will delete the item and return its value, or not found if it did not exist. func (t *SubjectTree[T]) Delete(subject []byte) (*T, bool) { if t == nil { return nil, false } val, deleted := t.delete(&t.root, subject, 0) if deleted { t.size-- } return val, deleted } // Match will match against a subject that can have wildcards and invoke the callback func for each matched value. func (t *SubjectTree[T]) Match(filter []byte, cb func(subject []byte, val *T)) { if t == nil || t.root == nil || len(filter) == 0 || cb == nil { return } // We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'. var raw [16][]byte parts := genParts(filter, raw[:0]) var _pre [256]byte t.match(t.root, parts, _pre[:0], cb) } // IterOrdered will walk all entries in the SubjectTree lexographically. The callback can return false to terminate the walk. func (t *SubjectTree[T]) IterOrdered(cb func(subject []byte, val *T) bool) { if t == nil || t.root == nil { return } var _pre [256]byte t.iter(t.root, _pre[:0], true, cb) } // IterFast will walk all entries in the SubjectTree with no guarantees of ordering. The callback can return false to terminate the walk. func (t *SubjectTree[T]) IterFast(cb func(subject []byte, val *T) bool) { if t == nil || t.root == nil { return } var _pre [256]byte t.iter(t.root, _pre[:0], false, cb) } // Internal methods // Internal call to insert that can be recursive. func (t *SubjectTree[T]) insert(np *node, subject []byte, value T, si int) (*T, bool) { n := *np if n == nil { *np = newLeaf(subject, value) return nil, false } if n.isLeaf() { ln := n.(*leaf[T]) if ln.match(subject[si:]) { // Replace with new value. old := ln.value ln.value = value return &old, true } // Here we need to split this leaf. cpi := commonPrefixLen(ln.suffix, subject[si:]) nn := newNode4(subject[si : si+cpi]) ln.setSuffix(ln.suffix[cpi:]) si += cpi // Make sure we have different pivot, normally this will be the case unless we have overflowing prefixes. if p := pivot(ln.suffix, 0); cpi > 0 && si < len(subject) && p == subject[si] { // We need to split the original leaf. Recursively call into insert. t.insert(np, subject, value, si) // Now add the update version of *np as a child to the new node4. nn.addChild(p, *np) } else { // Can just add this new leaf as a sibling. nl := newLeaf(subject[si:], value) nn.addChild(pivot(nl.suffix, 0), nl) // Add back original. nn.addChild(pivot(ln.suffix, 0), ln) } *np = nn return nil, false } // Non-leaf nodes. bn := n.base() if len(bn.prefix) > 0 { cpi := commonPrefixLen(bn.prefix, subject[si:]) if pli := len(bn.prefix); cpi >= pli { // Move past this node. We look for an existing child node to recurse into. // If one does not exist we can create a new leaf node. si += pli if nn := n.findChild(pivot(subject, si)); nn != nil { return t.insert(nn, subject, value, si) } if n.isFull() { n = n.grow() *np = n } n.addChild(pivot(subject, si), newLeaf(subject[si:], value)) return nil, false } else { // We did not match the prefix completely here. // Calculate new prefix for this node. prefix := subject[si : si+cpi] si += len(prefix) // We will insert a new node4 and attach our current node below after adjusting prefix. nn := newNode4(prefix) // Shift the prefix for our original node. n.setPrefix(bn.prefix[cpi:]) nn.addChild(pivot(bn.prefix[:], 0), n) // Add in our new leaf. nn.addChild(pivot(subject[si:], 0), newLeaf(subject[si:], value)) // Update our node reference. *np = nn } } else { if nn := n.findChild(pivot(subject, si)); nn != nil { return t.insert(nn, subject, value, si) } // No prefix and no matched child, so add in new leafnode as needed. if n.isFull() { n = n.grow() *np = n } n.addChild(pivot(subject, si), newLeaf(subject[si:], value)) } return nil, false } // internal function to recursively find the leaf to delete. Will do compaction if the item is found and removed. func (t *SubjectTree[T]) delete(np *node, subject []byte, si int) (*T, bool) { if t == nil || np == nil || *np == nil || len(subject) == 0 { return nil, false } n := *np if n.isLeaf() { ln := n.(*leaf[T]) if ln.match(subject[si:]) { *np = nil return &ln.value, true } return nil, false } // Not a leaf node. if bn := n.base(); len(bn.prefix) > 0 { if !bytes.Equal(subject[si:si+len(bn.prefix)], bn.prefix) { return nil, false } // Increment our subject index. si += len(bn.prefix) } p := pivot(subject, si) nna := n.findChild(p) if nna == nil { return nil, false } nn := *nna if nn.isLeaf() { ln := nn.(*leaf[T]) if ln.match(subject[si:]) { n.deleteChild(p) if sn := n.shrink(); sn != nil { bn := n.base() // Make sure to set cap so we force an append to copy below. pre := bn.prefix[:len(bn.prefix):len(bn.prefix)] // Need to fix up prefixes/suffixes. if sn.isLeaf() { ln := sn.(*leaf[T]) // Make sure to set cap so we force an append to copy. ln.suffix = append(pre, ln.suffix...) } else { // We are a node here, we need to add in the old prefix. if len(pre) > 0 { bsn := sn.base() sn.setPrefix(append(pre, bsn.prefix...)) } } *np = sn } return &ln.value, true } return nil, false } return t.delete(nna, subject, si) } // Internal function which can be called recursively to match all leaf nodes to a given filter subject which // once here has been decomposed to parts. These parts only care about wildcards, both pwc and fwc. func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subject []byte, val *T)) { // Capture if we are sitting on a terminal fwc. var hasFWC bool if lp := len(parts); lp > 0 && len(parts[lp-1]) > 0 && parts[lp-1][0] == fwc { hasFWC = true } for n != nil { nparts, matched := n.matchParts(parts) // Check if we did not match. if !matched { return } // We have matched here. If we are a leaf and have exhausted all parts or he have a FWC fire callback. if n.isLeaf() { if len(nparts) == 0 || (hasFWC && len(nparts) == 1) { ln := n.(*leaf[T]) cb(append(pre, ln.suffix...), &ln.value) } return } // We have normal nodes here. // We need to append our prefix bn := n.base() if len(bn.prefix) > 0 { // Note that this append may reallocate, but it doesn't modify "pre" at the "match" callsite. pre = append(pre, bn.prefix...) } // Check our remaining parts. if len(nparts) == 0 && !hasFWC { // We are a node with no parts left and we are not looking at a fwc. // We could have a leafnode with no suffix which would be a match. // We could also have a terminal pwc. Check for those here. var hasTermPWC bool if lp := len(parts); lp > 0 && len(parts[lp-1]) == 1 && parts[lp-1][0] == pwc { // If we are sitting on a terminal pwc, put the pwc back and continue. nparts = parts[len(parts)-1:] hasTermPWC = true } for _, cn := range n.children() { if cn == nil { continue } if cn.isLeaf() { ln := cn.(*leaf[T]) if len(ln.suffix) == 0 { cb(append(pre, ln.suffix...), &ln.value) } else if hasTermPWC && bytes.IndexByte(ln.suffix, tsep) < 0 { cb(append(pre, ln.suffix...), &ln.value) } } else if hasTermPWC { // We have terminal pwc so call into match again with the child node. t.match(cn, nparts, pre, cb) } } // Return regardless. return } // If we are sitting on a terminal fwc, put back and continue. if hasFWC && len(nparts) == 0 { nparts = parts[len(parts)-1:] } // Here we are a node type with a partial match. // Check if the first part is a wildcard. fp := nparts[0] p := pivot(fp, 0) // Check if we have a pwc/fwc part here. This will cause us to iterate. if len(fp) == 1 && (p == pwc || p == fwc) { // We need to iterate over all children here for the current node // to see if we match further down. for _, cn := range n.children() { if cn != nil { t.match(cn, nparts, pre, cb) } } return } // Here we have normal traversal, so find the next child. nn := n.findChild(p) if nn == nil { return } n, parts = *nn, nparts } } // Interal iter function to walk nodes in lexigraphical order. func (t *SubjectTree[T]) iter(n node, pre []byte, ordered bool, cb func(subject []byte, val *T) bool) bool { if n.isLeaf() { ln := n.(*leaf[T]) return cb(append(pre, ln.suffix...), &ln.value) } // We are normal node here. bn := n.base() // Note that this append may reallocate, but it doesn't modify "pre" at the "iter" callsite. pre = append(pre, bn.prefix...) // Not everything requires lexicographical sorting, so support a fast path for iterating in // whatever order the stree has things stored instead. if !ordered { for _, cn := range n.children() { if cn == nil { continue } if !t.iter(cn, pre, false, cb) { return false } } return true } // Collect nodes since unsorted. var _nodes [256]node nodes := _nodes[:0] for _, cn := range n.children() { if cn != nil { nodes = append(nodes, cn) } } // Now sort. slices.SortStableFunc(nodes, func(a, b node) int { return bytes.Compare(a.path(), b.path()) }) // Now walk the nodes in order and call into next iter. for i := range nodes { if !t.iter(nodes[i], pre, true, cb) { return false } } return true } nats-server-2.10.27/server/stree/stree_test.go000066400000000000000000000615411477524627100213220ustar00rootroot00000000000000// Copyright 2023-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree import ( crand "crypto/rand" "encoding/hex" "flag" "fmt" "math/rand" "strings" "testing" "time" ) // Print Results: go test -v --args --results // For some benchmarks. var runResults = flag.Bool("results", false, "Enable Results Tests") func TestSubjectTreeBasics(t *testing.T) { st := NewSubjectTree[int]() require_Equal(t, st.Size(), 0) // Single leaf old, updated := st.Insert(b("foo.bar.baz"), 22) require_True(t, old == nil) require_False(t, updated) require_Equal(t, st.Size(), 1) // Find shouldn't work with a wildcard. _, found := st.Find(b("foo.bar.*")) require_False(t, found) // But it should with a literal. Find with single leaf. v, found := st.Find(b("foo.bar.baz")) require_True(t, found) require_Equal(t, *v, 22) // Update single leaf old, updated = st.Insert(b("foo.bar.baz"), 33) require_True(t, old != nil) require_Equal(t, *old, 22) require_True(t, updated) require_Equal(t, st.Size(), 1) // Split the tree old, updated = st.Insert(b("foo.bar"), 22) require_True(t, old == nil) require_False(t, updated) require_Equal(t, st.Size(), 2) // Now we have node4 -> leaf*2 v, found = st.Find(b("foo.bar")) require_True(t, found) require_Equal(t, *v, 22) // Make sure we can still retrieve the original after the split. v, found = st.Find(b("foo.bar.baz")) require_True(t, found) require_Equal(t, *v, 33) } func TestSubjectTreeNodeGrow(t *testing.T) { st := NewSubjectTree[int]() for i := 0; i < 4; i++ { subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) } // We have filled a node4. _, ok := st.root.(*node4) require_True(t, ok) // This one will trigger us to grow. old, updated := st.Insert(b("foo.bar.E"), 22) require_True(t, old == nil) require_False(t, updated) _, ok = st.root.(*node10) require_True(t, ok) for i := 5; i < 10; i++ { subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) } // This one will trigger us to grow. old, updated = st.Insert(b("foo.bar.K"), 22) require_True(t, old == nil) require_False(t, updated) // We have filled a node10. _, ok = st.root.(*node16) require_True(t, ok) for i := 11; i < 16; i++ { subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) } // This one will trigger us to grow. old, updated = st.Insert(b("foo.bar.Q"), 22) require_True(t, old == nil) require_False(t, updated) _, ok = st.root.(*node48) require_True(t, ok) // Fill the node48. for i := 17; i < 48; i++ { subj := b(fmt.Sprintf("foo.bar.%c", 'A'+i)) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) } // This one will trigger us to grow. subj := b(fmt.Sprintf("foo.bar.%c", 'A'+49)) old, updated = st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) _, ok = st.root.(*node256) require_True(t, ok) } func TestSubjectTreeNodePrefixMismatch(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 11) st.Insert(b("foo.bar.B"), 22) st.Insert(b("foo.bar.C"), 33) // Grab current root. Split below will cause update. or := st.root // This one will force a split of the node st.Insert(b("foo.foo.A"), 44) require_True(t, or != st.root) // Now make sure we can retrieve correctly. v, found := st.Find(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 11) v, found = st.Find(b("foo.bar.B")) require_True(t, found) require_Equal(t, *v, 22) v, found = st.Find(b("foo.bar.C")) require_True(t, found) require_Equal(t, *v, 33) v, found = st.Find(b("foo.foo.A")) require_True(t, found) require_Equal(t, *v, 44) } func TestSubjectTreeNodeDelete(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 22) v, found := st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 22) require_Equal(t, st.root, nil) v, found = st.Delete(b("foo.bar.A")) require_False(t, found) require_Equal(t, v, nil) v, found = st.Find(b("foo.foo.A")) require_False(t, found) require_Equal(t, v, nil) // Kick to a node4. st.Insert(b("foo.bar.A"), 11) st.Insert(b("foo.bar.B"), 22) st.Insert(b("foo.bar.C"), 33) // Make sure we can delete and that we shrink back to leaf. v, found = st.Delete(b("foo.bar.C")) require_True(t, found) require_Equal(t, *v, 33) v, found = st.Delete(b("foo.bar.B")) require_True(t, found) require_Equal(t, *v, 22) // We should have shrunk here. require_True(t, st.root.isLeaf()) v, found = st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 11) require_Equal(t, st.root, nil) // Now pop up to a node10 and make sure we can shrink back down. for i := 0; i < 5; i++ { subj := fmt.Sprintf("foo.bar.%c", 'A'+i) st.Insert(b(subj), 22) } _, ok := st.root.(*node10) require_True(t, ok) v, found = st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 22) _, ok = st.root.(*node4) require_True(t, ok) // Now pop up to node16 for i := 0; i < 11; i++ { subj := fmt.Sprintf("foo.bar.%c", 'A'+i) st.Insert(b(subj), 22) } _, ok = st.root.(*node16) require_True(t, ok) v, found = st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 22) _, ok = st.root.(*node10) require_True(t, ok) v, found = st.Find(b("foo.bar.B")) require_True(t, found) require_Equal(t, *v, 22) // Now pop up to node48 st = NewSubjectTree[int]() for i := 0; i < 17; i++ { subj := fmt.Sprintf("foo.bar.%c", 'A'+i) st.Insert(b(subj), 22) } _, ok = st.root.(*node48) require_True(t, ok) v, found = st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 22) _, ok = st.root.(*node16) require_True(t, ok) v, found = st.Find(b("foo.bar.B")) require_True(t, found) require_Equal(t, *v, 22) // Now pop up to node256 st = NewSubjectTree[int]() for i := 0; i < 49; i++ { subj := fmt.Sprintf("foo.bar.%c", 'A'+i) st.Insert(b(subj), 22) } _, ok = st.root.(*node256) require_True(t, ok) v, found = st.Delete(b("foo.bar.A")) require_True(t, found) require_Equal(t, *v, 22) _, ok = st.root.(*node48) require_True(t, ok) v, found = st.Find(b("foo.bar.B")) require_True(t, found) require_Equal(t, *v, 22) } func TestSubjectTreeNodesAndPaths(t *testing.T) { st := NewSubjectTree[int]() check := func(subj string) { t.Helper() v, found := st.Find(b(subj)) require_True(t, found) require_Equal(t, *v, 22) } st.Insert(b("foo.bar.A"), 22) st.Insert(b("foo.bar.B"), 22) st.Insert(b("foo.bar.C"), 22) st.Insert(b("foo.bar"), 22) check("foo.bar.A") check("foo.bar.B") check("foo.bar.C") check("foo.bar") // This will do several things in terms of shrinking and pruning, // want to make sure it gets prefix correct for new top node4. st.Delete(b("foo.bar")) check("foo.bar.A") check("foo.bar.B") check("foo.bar.C") } // Check that we are constructing a proper tree with complex insert patterns. func TestSubjectTreeConstruction(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 2) st.Insert(b("foo.bar.C"), 3) st.Insert(b("foo.baz.A"), 11) st.Insert(b("foo.baz.B"), 22) st.Insert(b("foo.baz.C"), 33) st.Insert(b("foo.bar"), 42) checkNode := func(an *node, kind string, pors string, numChildren uint16) { t.Helper() require_True(t, an != nil) n := *an require_True(t, n != nil) require_Equal(t, n.kind(), kind) require_Equal(t, pors, string(n.path())) require_Equal(t, numChildren, n.numChildren()) } checkNode(&st.root, "NODE4", "foo.ba", 2) nn := st.root.findChild('r') checkNode(nn, "NODE4", "r", 2) checkNode((*nn).findChild(noPivot), "LEAF", "", 0) rnn := (*nn).findChild('.') checkNode(rnn, "NODE4", ".", 3) checkNode((*rnn).findChild('A'), "LEAF", "A", 0) checkNode((*rnn).findChild('B'), "LEAF", "B", 0) checkNode((*rnn).findChild('C'), "LEAF", "C", 0) znn := st.root.findChild('z') checkNode(znn, "NODE4", "z.", 3) checkNode((*znn).findChild('A'), "LEAF", "A", 0) checkNode((*znn).findChild('B'), "LEAF", "B", 0) checkNode((*znn).findChild('C'), "LEAF", "C", 0) // Use st.Dump() if you want a tree print out. // Now delete "foo.bar" and make sure put ourselves back together properly. v, found := st.Delete(b("foo.bar")) require_True(t, found) require_Equal(t, *v, 42) checkNode(&st.root, "NODE4", "foo.ba", 2) nn = st.root.findChild('r') checkNode(nn, "NODE4", "r.", 3) checkNode((*nn).findChild('A'), "LEAF", "A", 0) checkNode((*nn).findChild('B'), "LEAF", "B", 0) checkNode((*nn).findChild('C'), "LEAF", "C", 0) znn = st.root.findChild('z') checkNode(znn, "NODE4", "z.", 3) checkNode((*znn).findChild('A'), "LEAF", "A", 0) checkNode((*znn).findChild('B'), "LEAF", "B", 0) checkNode((*znn).findChild('C'), "LEAF", "C", 0) } func match(t *testing.T, st *SubjectTree[int], filter string, expected int) { t.Helper() var matches []int st.Match(b(filter), func(_ []byte, v *int) { matches = append(matches, *v) }) require_Equal(t, expected, len(matches)) } func TestSubjectTreeMatchLeafOnly(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.baz.A"), 1) // Check all placements of pwc in token space. match(t, st, "foo.bar.*.A", 1) match(t, st, "foo.*.baz.A", 1) match(t, st, "foo.*.*.A", 1) match(t, st, "foo.*.*.*", 1) match(t, st, "*.*.*.*", 1) // Now check fwc. match(t, st, ">", 1) match(t, st, "foo.>", 1) match(t, st, "foo.*.>", 1) match(t, st, "foo.bar.>", 1) match(t, st, "foo.bar.*.>", 1) // Check partials so they do not trigger on leafs. match(t, st, "foo.bar.baz", 0) } func TestSubjectTreeMatchNodes(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 2) st.Insert(b("foo.bar.C"), 3) st.Insert(b("foo.baz.A"), 11) st.Insert(b("foo.baz.B"), 22) st.Insert(b("foo.baz.C"), 33) // Test literals. match(t, st, "foo.bar.A", 1) match(t, st, "foo.baz.A", 1) match(t, st, "foo.bar", 0) // Test internal pwc match(t, st, "foo.*.A", 2) // Test terminal pwc match(t, st, "foo.bar.*", 3) match(t, st, "foo.baz.*", 3) // Check fwc match(t, st, ">", 6) match(t, st, "foo.>", 6) match(t, st, "foo.bar.>", 3) match(t, st, "foo.baz.>", 3) // Make sure we do not have false positives on prefix matches. match(t, st, "foo.ba", 0) // Now add in "foo.bar" to make a more complex tree construction // and re-test. st.Insert(b("foo.bar"), 42) // Test literals. match(t, st, "foo.bar.A", 1) match(t, st, "foo.baz.A", 1) match(t, st, "foo.bar", 1) // Test internal pwc match(t, st, "foo.*.A", 2) // Test terminal pwc match(t, st, "foo.bar.*", 3) match(t, st, "foo.baz.*", 3) // Check fwc match(t, st, ">", 7) match(t, st, "foo.>", 7) match(t, st, "foo.bar.>", 3) match(t, st, "foo.baz.>", 3) } func TestSubjectTreeNoPrefix(t *testing.T) { st := NewSubjectTree[int]() for i := 0; i < 26; i++ { subj := b(fmt.Sprintf("%c", 'A'+i)) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) } n, ok := st.root.(*node48) require_True(t, ok) require_Equal(t, n.numChildren(), 26) v, found := st.Delete(b("B")) require_True(t, found) require_Equal(t, *v, 22) require_Equal(t, n.numChildren(), 25) v, found = st.Delete(b("Z")) require_True(t, found) require_Equal(t, *v, 22) require_Equal(t, n.numChildren(), 24) } func TestSubjectTreePartialTerminalWildcardBugMatch(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-A"), 5) st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-B"), 1) st.Insert(b("STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-C"), 2) match(t, st, "STATE.GLOBAL.CELL1.7PDSGAALXNN000010.*", 3) } func TestSubjectTreeMatchSubjectParam(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 2) st.Insert(b("foo.bar.C"), 3) st.Insert(b("foo.baz.A"), 11) st.Insert(b("foo.baz.B"), 22) st.Insert(b("foo.baz.C"), 33) st.Insert(b("foo.bar"), 42) checkValMap := map[string]int{ "foo.bar.A": 1, "foo.bar.B": 2, "foo.bar.C": 3, "foo.baz.A": 11, "foo.baz.B": 22, "foo.baz.C": 33, "foo.bar": 42, } // Make sure we get a proper subject parameter and it matches our value properly. st.Match([]byte(">"), func(subject []byte, v *int) { if expected, ok := checkValMap[string(subject)]; !ok { t.Fatalf("Unexpected subject parameter: %q", subject) } else if expected != *v { t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) } }) } func TestSubjectTreeMatchRandomDoublePWC(t *testing.T) { st := NewSubjectTree[int]() for i := 1; i <= 10_000; i++ { subj := fmt.Sprintf("foo.%d.%d", rand.Intn(20)+1, i) st.Insert(b(subj), 42) } match(t, st, "foo.*.*", 10_000) // Check with pwc and short interior token. seen, verified := 0, 0 st.Match(b("*.2.*"), func(_ []byte, _ *int) { seen++ }) // Now check via walk to make sure we are right. st.IterOrdered(func(subject []byte, v *int) bool { tokens := strings.Split(string(subject), ".") require_Equal(t, len(tokens), 3) if tokens[1] == "2" { verified++ } return true }) require_Equal(t, seen, verified) seen, verified = 0, 0 st.Match(b("*.*.222"), func(_ []byte, _ *int) { seen++ }) st.IterOrdered(func(subject []byte, v *int) bool { tokens := strings.Split(string(subject), ".") require_Equal(t, len(tokens), 3) if tokens[2] == "222" { verified++ } return true }) require_Equal(t, seen, verified) } func TestSubjectTreeIterOrdered(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 2) st.Insert(b("foo.bar.C"), 3) st.Insert(b("foo.baz.A"), 11) st.Insert(b("foo.baz.B"), 22) st.Insert(b("foo.baz.C"), 33) st.Insert(b("foo.bar"), 42) checkValMap := map[string]int{ "foo.bar.A": 1, "foo.bar.B": 2, "foo.bar.C": 3, "foo.baz.A": 11, "foo.baz.B": 22, "foo.baz.C": 33, "foo.bar": 42, } checkOrder := []string{ "foo.bar", "foo.bar.A", "foo.bar.B", "foo.bar.C", "foo.baz.A", "foo.baz.B", "foo.baz.C", } var received int walk := func(subject []byte, v *int) bool { if expected := checkOrder[received]; expected != string(subject) { t.Fatalf("Expected %q for %d item returned, got %q", expected, received, subject) } received++ require_True(t, v != nil) if expected := checkValMap[string(subject)]; expected != *v { t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) } return true } // Kick in the iter. st.IterOrdered(walk) require_Equal(t, received, len(checkOrder)) // Make sure we can terminate properly. received = 0 st.IterOrdered(func(subject []byte, v *int) bool { received++ return received != 4 }) require_Equal(t, received, 4) } func TestSubjectTreeIterFast(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 2) st.Insert(b("foo.bar.C"), 3) st.Insert(b("foo.baz.A"), 11) st.Insert(b("foo.baz.B"), 22) st.Insert(b("foo.baz.C"), 33) st.Insert(b("foo.bar"), 42) checkValMap := map[string]int{ "foo.bar.A": 1, "foo.bar.B": 2, "foo.bar.C": 3, "foo.baz.A": 11, "foo.baz.B": 22, "foo.baz.C": 33, "foo.bar": 42, } var received int walk := func(subject []byte, v *int) bool { received++ require_True(t, v != nil) if expected := checkValMap[string(subject)]; expected != *v { t.Fatalf("Expected %q to have value of %d, but got %d", subject, expected, *v) } return true } // Kick in the iter. st.IterFast(walk) require_Equal(t, received, len(checkValMap)) // Make sure we can terminate properly. received = 0 st.IterFast(func(subject []byte, v *int) bool { received++ return received != 4 }) require_Equal(t, received, 4) } func TestSubjectTreeInsertSamePivotBug(t *testing.T) { testSubjects := [][]byte{ []byte("0d00.2abbb82c1d.6e16.fa7f85470e.3e46"), []byte("534b12.3486c17249.4dde0666"), []byte("6f26aabd.920ee3.d4d3.5ffc69f6"), []byte("8850.ade3b74c31.aa533f77.9f59.a4bd8415.b3ed7b4111"), []byte("5a75047dcb.5548e845b6.76024a34.14d5b3.80c426.51db871c3a"), []byte("825fa8acfc.5331.00caf8bbbd.107c4b.c291.126d1d010e"), } st := NewSubjectTree[int]() for _, subj := range testSubjects { old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) if _, found := st.Find(subj); !found { t.Fatalf("Could not find subject %q which should be findable", subj) } } } func TestSubjectTreeMatchTsepSecondThenPartialPartBug(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.xxxxx.foo1234.zz"), 22) st.Insert(b("foo.yyy.foo123.zz"), 22) st.Insert(b("foo.yyybar789.zz"), 22) st.Insert(b("foo.yyy.foo12345.zz"), 22) st.Insert(b("foo.yyy.foo12345.yy"), 22) st.Insert(b("foo.yyy.foo123456789.zz"), 22) match(t, st, "foo.*.foo123456789.*", 1) match(t, st, "foo.*.*.zzz.foo.>", 0) } func TestSubjectTreeMatchMultipleWildcardBasic(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("A.B.C.D.0.G.H.I.0"), 22) st.Insert(b("A.B.C.D.1.G.H.I.0"), 22) match(t, st, "A.B.*.D.1.*.*.I.0", 1) } func TestSubjectTreeMatchInvalidWildcard(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.123"), 22) st.Insert(b("one.two.three.four.five"), 22) st.Insert(b("'*.123"), 22) match(t, st, "invalid.>", 0) match(t, st, ">", 3) match(t, st, `'*.*`, 1) match(t, st, `'*.*.*'`, 0) // None of these should match. match(t, st, "`>`", 0) match(t, st, `">"`, 0) match(t, st, `'>'`, 0) match(t, st, `'*.>'`, 0) match(t, st, `'*.>.`, 0) match(t, st, "`invalid.>`", 0) match(t, st, `'*.*'`, 0) } func TestSubjectTreeRandomTrackEntries(t *testing.T) { st := NewSubjectTree[int]() smap := make(map[string]struct{}, 1000) // Make sure all added items can be found. check := func() { t.Helper() for subj := range smap { if _, found := st.Find(b(subj)); !found { t.Fatalf("Could not find subject %q which should be findable", subj) } } } buf := make([]byte, 10) for i := 0; i < 1000; i++ { var sb strings.Builder // 1-6 tokens. numTokens := rand.Intn(6) + 1 for i := 0; i < numTokens; i++ { tlen := rand.Intn(4) + 2 tok := buf[:tlen] crand.Read(tok) sb.WriteString(hex.EncodeToString(tok)) if i != numTokens-1 { sb.WriteString(".") } } subj := sb.String() // Avoid dupes since will cause check to fail after we delete messages. if _, ok := smap[subj]; ok { continue } smap[subj] = struct{}{} old, updated := st.Insert(b(subj), 22) require_True(t, old == nil) require_False(t, updated) require_Equal(t, st.Size(), len(smap)) check() } } // Needs to be longer then internal node prefix, which currently is 24. func TestSubjectTreeLongTokens(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("a1.aaaaaaaaaaaaaaaaaaaaaa0"), 1) st.Insert(b("a2.0"), 2) st.Insert(b("a1.aaaaaaaaaaaaaaaaaaaaaa1"), 3) st.Insert(b("a2.1"), 4) // Simulate purge of a2.> // This required to show bug. st.Delete(b("a2.0")) st.Delete(b("a2.1")) require_Equal(t, st.Size(), 2) v, found := st.Find(b("a1.aaaaaaaaaaaaaaaaaaaaaa0")) require_True(t, found) require_Equal(t, *v, 1) v, found = st.Find(b("a1.aaaaaaaaaaaaaaaaaaaaaa1")) require_True(t, found) require_Equal(t, *v, 3) } func b(s string) []byte { return []byte(s) } func TestSubjectTreeMatchAllPerf(t *testing.T) { if !*runResults { t.Skip() } st := NewSubjectTree[int]() for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("subj.%d.%d", rand.Intn(100)+1, i) st.Insert(b(subj), 22) } for _, f := range [][]byte{ []byte(">"), []byte("subj.>"), []byte("subj.*.*"), []byte("*.*.*"), []byte("subj.1.*"), []byte("subj.1.>"), []byte("subj.*.1"), []byte("*.*.1"), } { start := time.Now() count := 0 st.Match(f, func(_ []byte, _ *int) { count++ }) t.Logf("Match %q took %s and matched %d entries", f, time.Since(start), count) } } func TestSubjectTreeIterPerf(t *testing.T) { if !*runResults { t.Skip() } st := NewSubjectTree[int]() for i := 0; i < 1_000_000; i++ { subj := fmt.Sprintf("subj.%d.%d", rand.Intn(100)+1, i) st.Insert(b(subj), 22) } start := time.Now() count := 0 st.IterOrdered(func(_ []byte, _ *int) bool { count++ return true }) t.Logf("Iter took %s and matched %d entries", time.Since(start), count) } func TestSubjectTreeNode48(t *testing.T) { var a, b, c leaf[int] var n node48 n.addChild('A', &a) require_Equal(t, n.key['A'], 1) require_True(t, n.child[0] != nil) require_Equal(t, n.child[0].(*leaf[int]), &a) require_Equal(t, len(n.children()), 1) child := n.findChild('A') require_True(t, child != nil) require_Equal(t, (*child).(*leaf[int]), &a) n.addChild('B', &b) require_Equal(t, n.key['B'], 2) require_True(t, n.child[1] != nil) require_Equal(t, n.child[1].(*leaf[int]), &b) require_Equal(t, len(n.children()), 2) child = n.findChild('B') require_True(t, child != nil) require_Equal(t, (*child).(*leaf[int]), &b) n.addChild('C', &c) require_Equal(t, n.key['C'], 3) require_True(t, n.child[2] != nil) require_Equal(t, n.child[2].(*leaf[int]), &c) require_Equal(t, len(n.children()), 3) child = n.findChild('C') require_True(t, child != nil) require_Equal(t, (*child).(*leaf[int]), &c) n.deleteChild('A') require_Equal(t, len(n.children()), 2) require_Equal(t, n.key['A'], 0) // Now deleted require_Equal(t, n.key['B'], 2) // Untouched require_Equal(t, n.key['C'], 1) // Where A was child = n.findChild('A') require_Equal(t, child, nil) require_True(t, n.child[0] != nil) require_Equal(t, n.child[0].(*leaf[int]), &c) child = n.findChild('B') require_True(t, child != nil) require_Equal(t, (*child).(*leaf[int]), &b) require_True(t, n.child[1] != nil) require_Equal(t, n.child[1].(*leaf[int]), &b) child = n.findChild('C') require_True(t, child != nil) require_Equal(t, (*child).(*leaf[int]), &c) require_True(t, n.child[2] == nil) var gotB, gotC bool var iterations int n.iter(func(n node) bool { iterations++ if gb, ok := n.(*leaf[int]); ok && &b == gb { gotB = true } if gc, ok := n.(*leaf[int]); ok && &c == gc { gotC = true } return true }) require_Equal(t, iterations, 2) require_True(t, gotB) require_True(t, gotC) // Check for off-by-one on byte 255 as found by staticcheck, see // https://github.com/nats-io/nats-server/pull/5826. n.addChild(255, &c) require_Equal(t, n.key[255], 3) grown := n.grow().(*node256) require_True(t, grown.findChild(255) != nil) shrunk := n.shrink().(*node16) require_True(t, shrunk.findChild(255) != nil) } func TestSubjectTreeMatchNoCallbackDupe(t *testing.T) { st := NewSubjectTree[int]() st.Insert(b("foo.bar.A"), 1) st.Insert(b("foo.bar.B"), 1) st.Insert(b("foo.bar.C"), 1) st.Insert(b("foo.bar.>"), 1) for _, f := range [][]byte{ []byte(">"), []byte("foo.>"), []byte("foo.bar.>"), } { seen := map[string]struct{}{} st.Match(f, func(bsubj []byte, _ *int) { subj := string(bsubj) if _, ok := seen[subj]; ok { t.Logf("Match callback was called twice for %q", subj) } seen[subj] = struct{}{} }) } } func TestSubjectTreeNilNoPanic(t *testing.T) { var st *SubjectTree[int] st.Match([]byte("foo"), func(_ []byte, _ *int) {}) _, found := st.Find([]byte("foo")) require_False(t, found) _, found = st.Delete([]byte("foo")) require_False(t, found) _, found = st.Insert([]byte("foo"), 22) require_False(t, found) } // This bug requires the trailing suffix contain repeating nulls \x00 // and the second subject be longer with more nulls. func TestSubjectTreeInsertLongerLeafSuffixWithTrailingNulls(t *testing.T) { st := NewSubjectTree[int]() subj := []byte("foo.bar.baz_") // add in 10 nulls. for i := 0; i < 10; i++ { subj = append(subj, 0) } st.Insert(subj, 1) // add in 10 more nulls. subj2 := subj for i := 0; i < 10; i++ { subj2 = append(subj, 0) } st.Insert(subj2, 2) // Make sure we can look them up. v, found := st.Find(subj) require_True(t, found) require_Equal(t, *v, 1) v, found = st.Find(subj2) require_True(t, found) require_Equal(t, *v, 2) } // Make sure the system does not insert any subject with the noPivot (DEL) in it. func TestSubjectTreeInsertWithNoPivot(t *testing.T) { st := NewSubjectTree[int]() subj := []byte("foo.bar.baz.") subj = append(subj, noPivot) old, updated := st.Insert(subj, 22) require_True(t, old == nil) require_False(t, updated) require_Equal(t, st.Size(), 0) } // Make sure we don't panic when checking for fwc. func TestSubjectTreeMatchHasFWCNoPanic(t *testing.T) { defer func() { p := recover() require_True(t, p == nil) }() st := NewSubjectTree[int]() subj := []byte("foo") st.Insert(subj, 1) st.Match([]byte("."), func(subject []byte, val *int) {}) } nats-server-2.10.27/server/stree/util.go000066400000000000000000000026511477524627100201130ustar00rootroot00000000000000// Copyright 2023-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stree // For subject matching. const ( pwc = '*' fwc = '>' tsep = '.' ) // Determine index of common prefix. No match at all is 0, etc. func commonPrefixLen(s1, s2 []byte) int { limit := min(len(s1), len(s2)) var i int for ; i < limit; i++ { if s1[i] != s2[i] { break } } return i } // Helper to copy bytes. func copyBytes(src []byte) []byte { if len(src) == 0 { return nil } dst := make([]byte, len(src)) copy(dst, src) return dst } type position interface{ int | uint16 } // No pivot available. const noPivot = byte(127) // Can return 127 (DEL) if we have all the subject as prefixes. // We used to use 0, but when that was in the subject would cause infinite recursion in some situations. func pivot[N position](subject []byte, pos N) byte { if int(pos) >= len(subject) { return noPivot } return subject[pos] } nats-server-2.10.27/server/subject_transform.go000066400000000000000000000530271477524627100215510ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "hash/fnv" "regexp" "strconv" "strings" ) // Subject mapping and transform setups. var ( commaSeparatorRegEx = regexp.MustCompile(`,\s*`) partitionMappingFunctionRegEx = regexp.MustCompile(`{{\s*[pP]artition\s*\((.*)\)\s*}}`) wildcardMappingFunctionRegEx = regexp.MustCompile(`{{\s*[wW]ildcard\s*\((.*)\)\s*}}`) splitFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit[fF]rom[lL]eft\s*\((.*)\)\s*}}`) splitFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit[fF]rom[rR]ight\s*\((.*)\)\s*}}`) sliceFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]lice[fF]rom[lL]eft\s*\((.*)\)\s*}}`) sliceFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]lice[fF]rom[rR]ight\s*\((.*)\)\s*}}`) splitMappingFunctionRegEx = regexp.MustCompile(`{{\s*[sS]plit\s*\((.*)\)\s*}}`) leftMappingFunctionRegEx = regexp.MustCompile(`{{\s*[lL]eft\s*\((.*)\)\s*}}`) rightMappingFunctionRegEx = regexp.MustCompile(`{{\s*[rR]ight\s*\((.*)\)\s*}}`) ) // Enum for the subject mapping subjectTransform function types const ( NoTransform int16 = iota BadTransform Partition Wildcard SplitFromLeft SplitFromRight SliceFromLeft SliceFromRight Split Left Right ) // Transforms for arbitrarily mapping subjects from one to another for maps, tees and filters. // These can also be used for proper mapping on wildcard exports/imports. // These will be grouped and caching and locking are assumed to be in the upper layers. type subjectTransform struct { src, dest string dtoks []string // destination tokens stoks []string // source tokens dtokmftypes []int16 // destination token mapping function types dtokmftokindexesargs [][]int // destination token mapping function array of source token index arguments dtokmfintargs []int32 // destination token mapping function int32 arguments dtokmfstringargs []string // destination token mapping function string arguments } // SubjectTransformer transforms subjects using mappings // // This API is not part of the public API and not subject to SemVer protections type SubjectTransformer interface { // TODO(dlc) - We could add in client here to allow for things like foo -> foo.$ACCOUNT Match(string) (string, error) TransformSubject(subject string) string TransformTokenizedSubject(tokens []string) string } func NewSubjectTransformWithStrict(src, dest string, strict bool) (*subjectTransform, error) { // strict = true for import subject mappings that need to be reversible // (meaning can only use the Wildcard function and must use all the pwcs that are present in the source) // No source given is equivalent to the source being ">" if dest == _EMPTY_ { return nil, nil } if src == _EMPTY_ { src = fwcs } // Both entries need to be valid subjects. sv, stokens, npwcs, hasFwc := subjectInfo(src) dv, dtokens, dnpwcs, dHasFwc := subjectInfo(dest) // Make sure both are valid, match fwc if present and there are no pwcs in the dest subject. if !sv || !dv || dnpwcs > 0 || hasFwc != dHasFwc { return nil, ErrBadSubject } var dtokMappingFunctionTypes []int16 var dtokMappingFunctionTokenIndexes [][]int var dtokMappingFunctionIntArgs []int32 var dtokMappingFunctionStringArgs []string // If the src has partial wildcards then the dest needs to have the token place markers. if npwcs > 0 || hasFwc { // We need to count to make sure that the dest has token holders for the pwcs. sti := make(map[int]int) for i, token := range stokens { if len(token) == 1 && token[0] == pwc { sti[len(sti)+1] = i } } nphs := 0 for _, token := range dtokens { tranformType, transformArgWildcardIndexes, transfomArgInt, transformArgString, err := indexPlaceHolders(token) if err != nil { return nil, err } if strict { if tranformType != NoTransform && tranformType != Wildcard { return nil, &mappingDestinationErr{token, ErrMappingDestinationNotSupportedForImport} } } if npwcs == 0 { if tranformType != NoTransform { return nil, &mappingDestinationErr{token, ErrMappingDestinationIndexOutOfRange} } } if tranformType == NoTransform { dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, NoTransform) dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{-1}) dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, -1) dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_) } else { nphs += len(transformArgWildcardIndexes) // Now build up our runtime mapping from dest to source tokens. var stis []int for _, wildcardIndex := range transformArgWildcardIndexes { if wildcardIndex > npwcs { return nil, &mappingDestinationErr{fmt.Sprintf("%s: [%d]", token, wildcardIndex), ErrMappingDestinationIndexOutOfRange} } stis = append(stis, sti[wildcardIndex]) } dtokMappingFunctionTypes = append(dtokMappingFunctionTypes, tranformType) dtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, stis) dtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt) dtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, transformArgString) } } if strict && nphs < npwcs { // not all wildcards are being used in the destination return nil, &mappingDestinationErr{dest, ErrMappingDestinationNotUsingAllWildcards} } } else { // no wildcards used in the source: check that no transform functions are used in the destination for _, token := range dtokens { tranformType, _, _, _, err := indexPlaceHolders(token) if err != nil { return nil, err } if tranformType != NoTransform { return nil, &mappingDestinationErr{token, ErrMappingDestinationIndexOutOfRange} } } } return &subjectTransform{ src: src, dest: dest, dtoks: dtokens, stoks: stokens, dtokmftypes: dtokMappingFunctionTypes, dtokmftokindexesargs: dtokMappingFunctionTokenIndexes, dtokmfintargs: dtokMappingFunctionIntArgs, dtokmfstringargs: dtokMappingFunctionStringArgs, }, nil } func NewSubjectTransform(src, dest string) (*subjectTransform, error) { return NewSubjectTransformWithStrict(src, dest, false) } func NewSubjectTransformStrict(src, dest string) (*subjectTransform, error) { return NewSubjectTransformWithStrict(src, dest, true) } func getMappingFunctionArgs(functionRegEx *regexp.Regexp, token string) []string { commandStrings := functionRegEx.FindStringSubmatch(token) if len(commandStrings) > 1 { return commaSeparatorRegEx.Split(commandStrings[1], -1) } return nil } // Helper for mapping functions that take a wildcard index and an integer as arguments func transformIndexIntArgsHelper(token string, args []string, transformType int16) (int16, []int, int32, string, error) { if len(args) < 2 { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} } if len(args) > 2 { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} } i, err := strconv.Atoi(strings.Trim(args[0], " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } mappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[1], " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } return transformType, []int{i}, int32(mappingFunctionIntArg), _EMPTY_, nil } // Helper to ingest and index the subjectTransform destination token (e.g. $x or {{}}) in the token // returns a transformation type, and three function arguments: an array of source subject token indexes, // and a single number (e.g. number of partitions, or a slice size), and a string (e.g.a split delimiter) func indexPlaceHolders(token string) (int16, []int, int32, string, error) { length := len(token) if length > 1 { // old $1, $2, etc... mapping format still supported to maintain backwards compatibility if token[0] == '$' { // simple non-partition mapping tp, err := strconv.Atoi(token[1:]) if err != nil { // other things rely on tokens starting with $ so not an error just leave it as is return NoTransform, []int{-1}, -1, _EMPTY_, nil } return Wildcard, []int{tp}, -1, _EMPTY_, nil } // New 'mustache' style mapping if length > 4 && token[0] == '{' && token[1] == '{' && token[length-2] == '}' && token[length-1] == '}' { // wildcard(wildcard token index) (equivalent to $) args := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token) if args != nil { if len(args) == 1 && args[0] == _EMPTY_ { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} } if len(args) == 1 { tokenIndex, err := strconv.Atoi(strings.Trim(args[0], " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } return Wildcard, []int{tokenIndex}, -1, _EMPTY_, nil } else { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} } } // partition(number of partitions, token1, token2, ...) args = getMappingFunctionArgs(partitionMappingFunctionRegEx, token) if args != nil { if len(args) < 2 { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} } if len(args) >= 2 { mappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } var numPositions = len(args[1:]) tokenIndexes := make([]int, numPositions) for ti, t := range args[1:] { i, err := strconv.Atoi(strings.Trim(t, " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } tokenIndexes[ti] = i } return Partition, tokenIndexes, int32(mappingFunctionIntArg), _EMPTY_, nil } } // SplitFromLeft(token, position) args = getMappingFunctionArgs(splitFromLeftMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, SplitFromLeft) } // SplitFromRight(token, position) args = getMappingFunctionArgs(splitFromRightMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, SplitFromRight) } // SliceFromLeft(token, position) args = getMappingFunctionArgs(sliceFromLeftMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, SliceFromLeft) } // SliceFromRight(token, position) args = getMappingFunctionArgs(sliceFromRightMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, SliceFromRight) } // Right(token, length) args = getMappingFunctionArgs(rightMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, Right) } // Left(token, length) args = getMappingFunctionArgs(leftMappingFunctionRegEx, token) if args != nil { return transformIndexIntArgsHelper(token, args, Left) } // split(token, deliminator) args = getMappingFunctionArgs(splitMappingFunctionRegEx, token) if args != nil { if len(args) < 2 { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs} } if len(args) > 2 { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs} } i, err := strconv.Atoi(strings.Trim(args[0], " ")) if err != nil { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg} } if strings.Contains(args[1], " ") || strings.Contains(args[1], tsep) { return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token: token, err: ErrMappingDestinationInvalidArg} } return Split, []int{i}, -1, args[1], nil } return BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrUnknownMappingDestinationFunction} } } return NoTransform, []int{-1}, -1, _EMPTY_, nil } // Helper function to tokenize subjects with partial wildcards into formal transform destinations. // e.g. "foo.*.*" -> "foo.$1.$2" func transformTokenize(subject string) string { // We need to make the appropriate markers for the wildcards etc. i := 1 var nda []string for _, token := range strings.Split(subject, tsep) { if token == pwcs { nda = append(nda, fmt.Sprintf("$%d", i)) i++ } else { nda = append(nda, token) } } return strings.Join(nda, tsep) } // Helper function to go from transform destination to a subject with partial wildcards and ordered list of placeholders // E.g.: // // "bar" -> "bar", [] // "foo.$2.$1" -> "foo.*.*", ["$2","$1"] // "foo.{{wildcard(2)}}.{{wildcard(1)}}" -> "foo.*.*", ["{{wildcard(2)}}","{{wildcard(1)}}"] func transformUntokenize(subject string) (string, []string) { var phs []string var nda []string for _, token := range strings.Split(subject, tsep) { if args := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token); (len(token) > 1 && token[0] == '$' && token[1] >= '1' && token[1] <= '9') || (len(args) == 1 && args[0] != _EMPTY_) { phs = append(phs, token) nda = append(nda, pwcs) } else { nda = append(nda, token) } } return strings.Join(nda, tsep), phs } func tokenizeSubject(subject string) []string { // Tokenize the subject. tsa := [32]string{} tts := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tts = append(tts, subject[start:i]) start = i + 1 } } tts = append(tts, subject[start:]) return tts } // Match will take a literal published subject that is associated with a client and will match and subjectTransform // the subject if possible. // // This API is not part of the public API and not subject to SemVer protections func (tr *subjectTransform) Match(subject string) (string, error) { // Special case: matches any and no no-op subjectTransform. May not be legal config for some features // but specific validations made at subjectTransform create time if (tr.src == fwcs || tr.src == _EMPTY_) && (tr.dest == fwcs || tr.dest == _EMPTY_) { return subject, nil } tts := tokenizeSubject(subject) // TODO(jnm): optimization -> not sure this is actually needed but was there in initial code if !isValidLiteralSubject(tts) { return _EMPTY_, ErrBadSubject } if (tr.src == _EMPTY_ || tr.src == fwcs) || isSubsetMatch(tts, tr.src) { return tr.TransformTokenizedSubject(tts), nil } return _EMPTY_, ErrNoTransforms } // TransformSubject transforms a subject // // This API is not part of the public API and not subject to SemVer protection func (tr *subjectTransform) TransformSubject(subject string) string { return tr.TransformTokenizedSubject(tokenizeSubject(subject)) } func (tr *subjectTransform) getHashPartition(key []byte, numBuckets int) string { h := fnv.New32a() _, _ = h.Write(key) return strconv.Itoa(int(h.Sum32() % uint32(numBuckets))) } // Do a subjectTransform on the subject to the dest subject. func (tr *subjectTransform) TransformTokenizedSubject(tokens []string) string { if len(tr.dtokmftypes) == 0 { return tr.dest } var b strings.Builder // We need to walk destination tokens and create the mapped subject pulling tokens or mapping functions li := len(tr.dtokmftypes) - 1 for i, mfType := range tr.dtokmftypes { if mfType == NoTransform { // Break if fwc if len(tr.dtoks[i]) == 1 && tr.dtoks[i][0] == fwc { break } b.WriteString(tr.dtoks[i]) } else { switch mfType { case Partition: var ( _buffer [64]byte keyForHashing = _buffer[:0] ) for _, sourceToken := range tr.dtokmftokindexesargs[i] { keyForHashing = append(keyForHashing, []byte(tokens[sourceToken])...) } b.WriteString(tr.getHashPartition(keyForHashing, int(tr.dtokmfintargs[i]))) case Wildcard: // simple substitution b.WriteString(tokens[tr.dtokmftokindexesargs[i][0]]) case SplitFromLeft: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) position := int(tr.dtokmfintargs[i]) if position > 0 && position < sourceTokenLen { b.WriteString(sourceToken[:position]) b.WriteString(tsep) b.WriteString(sourceToken[position:]) } else { // too small to split at the requested position: don't split b.WriteString(sourceToken) } case SplitFromRight: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) position := int(tr.dtokmfintargs[i]) if position > 0 && position < sourceTokenLen { b.WriteString(sourceToken[:sourceTokenLen-position]) b.WriteString(tsep) b.WriteString(sourceToken[sourceTokenLen-position:]) } else { // too small to split at the requested position: don't split b.WriteString(sourceToken) } case SliceFromLeft: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) sliceSize := int(tr.dtokmfintargs[i]) if sliceSize > 0 && sliceSize < sourceTokenLen { for i := 0; i+sliceSize <= sourceTokenLen; i += sliceSize { if i != 0 { b.WriteString(tsep) } b.WriteString(sourceToken[i : i+sliceSize]) if i+sliceSize != sourceTokenLen && i+sliceSize+sliceSize > sourceTokenLen { b.WriteString(tsep) b.WriteString(sourceToken[i+sliceSize:]) break } } } else { // too small to slice at the requested size: don't slice b.WriteString(sourceToken) } case SliceFromRight: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) sliceSize := int(tr.dtokmfintargs[i]) if sliceSize > 0 && sliceSize < sourceTokenLen { remainder := sourceTokenLen % sliceSize if remainder > 0 { b.WriteString(sourceToken[:remainder]) b.WriteString(tsep) } for i := remainder; i+sliceSize <= sourceTokenLen; i += sliceSize { b.WriteString(sourceToken[i : i+sliceSize]) if i+sliceSize < sourceTokenLen { b.WriteString(tsep) } } } else { // too small to slice at the requested size: don't slice b.WriteString(sourceToken) } case Split: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] splits := strings.Split(sourceToken, tr.dtokmfstringargs[i]) for j, split := range splits { if split != _EMPTY_ { b.WriteString(split) } if j < len(splits)-1 && splits[j+1] != _EMPTY_ && !(j == 0 && split == _EMPTY_) { b.WriteString(tsep) } } case Left: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) sliceSize := int(tr.dtokmfintargs[i]) if sliceSize > 0 && sliceSize < sourceTokenLen { b.WriteString(sourceToken[0:sliceSize]) } else { // too small to slice at the requested size: don't slice b.WriteString(sourceToken) } case Right: sourceToken := tokens[tr.dtokmftokindexesargs[i][0]] sourceTokenLen := len(sourceToken) sliceSize := int(tr.dtokmfintargs[i]) if sliceSize > 0 && sliceSize < sourceTokenLen { b.WriteString(sourceToken[sourceTokenLen-sliceSize : sourceTokenLen]) } else { // too small to slice at the requested size: don't slice b.WriteString(sourceToken) } } } if i < li { b.WriteByte(btsep) } } // We may have more source tokens available. This happens with ">". if tr.dtoks[len(tr.dtoks)-1] == fwcs { for sli, i := len(tokens)-1, len(tr.stoks)-1; i < len(tokens); i++ { b.WriteString(tokens[i]) if i < sli { b.WriteByte(btsep) } } } return b.String() } // Reverse a subjectTransform. func (tr *subjectTransform) reverse() *subjectTransform { if len(tr.dtokmftokindexesargs) == 0 { rtr, _ := NewSubjectTransformStrict(tr.dest, tr.src) return rtr } // If we are here we need to dynamically get the correct reverse // of this subjectTransform. nsrc, phs := transformUntokenize(tr.dest) var nda []string for _, token := range tr.stoks { if token == pwcs { if len(phs) == 0 { // TODO(dlc) - Should not happen return nil } nda = append(nda, phs[0]) phs = phs[1:] } else { nda = append(nda, token) } } ndest := strings.Join(nda, tsep) rtr, _ := NewSubjectTransformStrict(nsrc, ndest) return rtr } // Will share relevant info regarding the subject. // Returns valid, tokens, num pwcs, has fwc. func subjectInfo(subject string) (bool, []string, int, bool) { if subject == "" { return false, nil, 0, false } npwcs := 0 sfwc := false tokens := strings.Split(subject, tsep) for _, t := range tokens { if len(t) == 0 || sfwc { return false, nil, 0, false } if len(t) > 1 { continue } switch t[0] { case fwc: sfwc = true case pwc: npwcs++ } } return true, tokens, npwcs, sfwc } nats-server-2.10.27/server/subject_transform_test.go000066400000000000000000000216651477524627100226130ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "errors" "reflect" "testing" ) func TestPlaceHolderIndex(t *testing.T) { testString := "$1" transformType, indexes, nbPartitions, _, err := indexPlaceHolders(testString) var position int32 if err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 1 || nbPartitions != -1 { t.Fatalf("Error parsing %s", testString) } testString = "{{partition(10,1,2,3)}}" transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) if err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 { t.Fatalf("Error parsing %s", testString) } testString = "{{ Partition (10,1,2,3) }}" transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) if err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 { t.Fatalf("Error parsing %s", testString) } testString = "{{wildcard(2)}}" transformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString) if err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 2 || nbPartitions != -1 { t.Fatalf("Error parsing %s", testString) } testString = "{{SplitFromLeft(2,1)}}" transformType, indexes, position, _, err = indexPlaceHolders(testString) if err != nil || transformType != SplitFromLeft || len(indexes) != 1 || indexes[0] != 2 || position != 1 { t.Fatalf("Error parsing %s", testString) } testString = "{{SplitFromRight(3,2)}}" transformType, indexes, position, _, err = indexPlaceHolders(testString) if err != nil || transformType != SplitFromRight || len(indexes) != 1 || indexes[0] != 3 || position != 2 { t.Fatalf("Error parsing %s", testString) } testString = "{{SliceFromLeft(2,2)}}" transformType, indexes, sliceSize, _, err := indexPlaceHolders(testString) if err != nil || transformType != SliceFromLeft || len(indexes) != 1 || indexes[0] != 2 || sliceSize != 2 { t.Fatalf("Error parsing %s", testString) } testString = "{{Left(3,2)}}" transformType, indexes, position, _, err = indexPlaceHolders(testString) if err != nil || transformType != Left || len(indexes) != 1 || indexes[0] != 3 || position != 2 { t.Fatalf("Error parsing %s", testString) } testString = "{{Right(3,2)}}" transformType, indexes, position, _, err = indexPlaceHolders(testString) if err != nil || transformType != Right || len(indexes) != 1 || indexes[0] != 3 || position != 2 { t.Fatalf("Error parsing %s", testString) } } func TestSubjectTransformHelpers(t *testing.T) { equals := func(a, b []string) bool { if len(a) != len(b) { return false } for i, v := range a { if v != b[i] { return false } } return true } filter, placeHolders := transformUntokenize("bar") if filter != "bar" || len(placeHolders) != 0 { t.Fatalf("transformUntokenize for not returning expected result") } filter, placeHolders = transformUntokenize("foo.$2.$1") if filter != "foo.*.*" || !equals(placeHolders, []string{"$2", "$1"}) { t.Fatalf("transformUntokenize for not returning expected result") } filter, placeHolders = transformUntokenize("foo.{{wildcard(2)}}.{{wildcard(1)}}") if filter != "foo.*.*" || !equals(placeHolders, []string{"{{wildcard(2)}}", "{{wildcard(1)}}"}) { t.Fatalf("transformUntokenize for not returning expected result") } newReversibleTransform := func(src, dest string) *subjectTransform { tr, err := NewSubjectTransformStrict(src, dest) if err != nil { t.Fatalf("Error getting reversible transform: %s to %s", src, dest) } return tr } tr := newReversibleTransform("foo.*.*", "bar.$2.{{Wildcard(1)}}") subject := "foo.b.a" transformed := tr.TransformSubject(subject) reverse := tr.reverse() if reverse.TransformSubject(transformed) != subject { t.Fatal("Reversed transform subject not matching") } } func TestSubjectTransforms(t *testing.T) { shouldErr := func(src, dest string, strict bool) { t.Helper() if _, err := NewSubjectTransformWithStrict(src, dest, strict); err != ErrBadSubject && !errors.Is(err, ErrInvalidMappingDestination) { t.Fatalf("Did not get an error for src=%q and dest=%q", src, dest) } } // Must be valid subjects. shouldErr("foo..", "bar", false) // Wildcards are allowed in src, but must be matched by token placements on the other side. // e.g. foo.* -> bar.$1. // Need to have as many pwcs as placements on other side shouldErr("foo.*", "bar.*", false) shouldErr("foo.*", "bar.$2", false) // Bad pwc token identifier shouldErr("foo.*", "bar.$1.>", false) // fwcs have to match. shouldErr("foo.>", "bar.baz", false) // fwcs have to match. shouldErr("foo.*.*", "bar.$2", true) // Must place all pwcs. shouldErr("foo.*", "foo.$foo", true) // invalid $ value shouldErr("foo.*", "bar.{{Partition(2,1)}}", true) // can only use Wildcard function (and old-style $x) in import transform shouldErr("foo.*", "foo.{{wildcard(2)}}", false) // Mapping function being passed an out of range wildcard index shouldErr("foo.*", "foo.{{unimplemented(1)}}", false) // Mapping trying to use an unknown mapping function shouldErr("foo.*", "foo.{{partition(10)}}", false) // Not enough arguments passed to the mapping function shouldErr("foo.*", "foo.{{wildcard(foo)}}", false) // Invalid argument passed to the mapping function shouldErr("foo.*", "foo.{{wildcard()}}", false) // Not enough arguments passed to the mapping function shouldErr("foo.*", "foo.{{wildcard(1,2)}}", false) // Too many arguments passed to the mapping function shouldErr("foo.*", "foo.{{ wildcard5) }}", false) // Bad mapping function shouldErr("foo.*", "foo.{{splitLeft(2,2}}", false) // arg out of range shouldErr("foo", "bla.{{wildcard(1)}}", false) // arg out of range with no wildcard in the source shouldBeOK := func(src, dest string, strict bool) *subjectTransform { t.Helper() tr, err := NewSubjectTransformWithStrict(src, dest, strict) if err != nil { t.Fatalf("Got an error %v for src=%q and dest=%q", err, src, dest) } return tr } shouldBeOK("foo.*", "bar.{{Wildcard(1)}}", true) shouldBeOK("foo.*.*", "bar.$2", false) // don't have to use all pwcs. shouldBeOK("foo.*.*", "bar.{{wildcard(1)}}", false) // don't have to use all pwcs. shouldBeOK("foo", "bar", false) shouldBeOK("foo.*.bar.*.baz", "req.$2.$1", false) shouldBeOK("baz.>", "mybaz.>", false) shouldBeOK("*", "{{splitfromleft(1,1)}}", false) shouldBeOK("", "prefix.>", false) shouldBeOK("*.*", "{{partition(10,1,2)}}", false) shouldBeOK("foo.*.*", "foo.{{wildcard(1)}}.{{wildcard(2)}}.{{partition(5,1,2)}}", false) shouldMatch := func(src, dest, sample, expected string) { t.Helper() tr := shouldBeOK(src, dest, false) if tr != nil { s, err := tr.Match(sample) if err != nil { t.Fatalf("Got an error %v when expecting a match for %q to %q", err, sample, expected) } if s != expected { t.Fatalf("Dest does not match what was expected. Got %q, expected %q", s, expected) } } } shouldMatch("", "prefix.>", "foo", "prefix.foo") shouldMatch("foo", "", "foo", "foo") shouldMatch("foo", "bar", "foo", "bar") shouldMatch("foo.*.bar.*.baz", "req.$2.$1", "foo.A.bar.B.baz", "req.B.A") shouldMatch("foo.*.bar.*.baz", "req.{{wildcard(2)}}.{{wildcard(1)}}", "foo.A.bar.B.baz", "req.B.A") shouldMatch("baz.>", "my.pre.>", "baz.1.2.3", "my.pre.1.2.3") shouldMatch("baz.>", "foo.bar.>", "baz.1.2.3", "foo.bar.1.2.3") shouldMatch("*", "foo.bar.$1", "foo", "foo.bar.foo") shouldMatch("*", "{{splitfromleft(1,3)}}", "12345", "123.45") shouldMatch("*", "{{SplitFromRight(1,3)}}", "12345", "12.345") shouldMatch("*", "{{SliceFromLeft(1,3)}}", "1234567890", "123.456.789.0") shouldMatch("*", "{{SliceFromRight(1,3)}}", "1234567890", "1.234.567.890") shouldMatch("*", "{{split(1,-)}}", "-abc-def--ghi-", "abc.def.ghi") shouldMatch("*", "{{split(1,-)}}", "abc-def--ghi-", "abc.def.ghi") shouldMatch("*.*", "{{split(2,-)}}.{{splitfromleft(1,2)}}", "foo.-abc-def--ghij-", "abc.def.ghij.fo.o") // combo + checks split for multiple instance of deliminator and deliminator being at the start or end shouldMatch("*", "{{right(1,1)}}", "1234", "4") shouldMatch("*", "{{right(1,3)}}", "1234", "234") shouldMatch("*", "{{right(1,6)}}", "1234", "1234") shouldMatch("*", "{{left(1,1)}}", "1234", "1") shouldMatch("*", "{{left(1,3)}}", "1234", "123") shouldMatch("*", "{{left(1,6)}}", "1234", "1234") } nats-server-2.10.27/server/sublist.go000066400000000000000000001264161477524627100175070ustar00rootroot00000000000000// Copyright 2016-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "errors" "strings" "sync" "sync/atomic" "unicode/utf8" "github.com/nats-io/nats-server/v2/server/stree" ) // Sublist is a routing mechanism to handle subject distribution and // provides a facility to match subjects from published messages to // interested subscribers. Subscribers can have wildcard subjects to // match multiple published subjects. // Common byte variables for wildcards and token separator. const ( pwc = '*' pwcs = "*" fwc = '>' fwcs = ">" tsep = "." btsep = '.' ) // Sublist related errors var ( ErrInvalidSubject = errors.New("sublist: invalid subject") ErrNotFound = errors.New("sublist: no matches found") ErrNilChan = errors.New("sublist: nil channel") ErrAlreadyRegistered = errors.New("sublist: notification already registered") ) const ( // cacheMax is used to bound limit the frontend cache slCacheMax = 1024 // If we run a sweeper we will drain to this count. slCacheSweep = 256 // plistMin is our lower bounds to create a fast plist for Match. plistMin = 256 ) // SublistResult is a result structure better optimized for queue subs. type SublistResult struct { psubs []*subscription qsubs [][]*subscription // don't make this a map, too expensive to iterate } // A Sublist stores and efficiently retrieves subscriptions. type Sublist struct { sync.RWMutex genid uint64 matches uint64 cacheHits uint64 inserts uint64 removes uint64 root *level cache map[string]*SublistResult ccSweep int32 notify *notifyMaps count uint32 } // notifyMaps holds maps of arrays of channels for notifications // on a change of interest. type notifyMaps struct { insert map[string][]chan<- bool remove map[string][]chan<- bool } // A node contains subscriptions and a pointer to the next level. type node struct { next *level psubs map[*subscription]struct{} qsubs map[string]map[*subscription]struct{} plist []*subscription } // A level represents a group of nodes and special pointers to // wildcard nodes. type level struct { nodes map[string]*node pwc, fwc *node } // Create a new default node. func newNode() *node { return &node{psubs: make(map[*subscription]struct{})} } // Create a new default level. func newLevel() *level { return &level{nodes: make(map[string]*node)} } // In general caching is recommended however in some extreme cases where // interest changes are high, suppressing the cache can help. // https://github.com/nats-io/nats-server/issues/941 // FIXME(dlc) - should be more dynamic at some point based on cache thrashing. // NewSublist will create a default sublist with caching enabled per the flag. func NewSublist(enableCache bool) *Sublist { if enableCache { return &Sublist{root: newLevel(), cache: make(map[string]*SublistResult)} } return &Sublist{root: newLevel()} } // NewSublistWithCache will create a default sublist with caching enabled. func NewSublistWithCache() *Sublist { return NewSublist(true) } // NewSublistNoCache will create a default sublist with caching disabled. func NewSublistNoCache() *Sublist { return NewSublist(false) } // CacheEnabled returns whether or not caching is enabled for this sublist. func (s *Sublist) CacheEnabled() bool { s.RLock() enabled := s.cache != nil s.RUnlock() return enabled } // RegisterNotification will register for notifications when interest for the given // subject changes. The subject must be a literal publish type subject. // The notification is true for when the first interest for a subject is inserted, // and false when all interest in the subject is removed. Note that this interest // needs to be exact and that wildcards will not trigger the notifications. The sublist // will not block when trying to send the notification. Its up to the caller to make // sure the channel send will not block. func (s *Sublist) RegisterNotification(subject string, notify chan<- bool) error { return s.registerNotification(subject, _EMPTY_, notify) } func (s *Sublist) RegisterQueueNotification(subject, queue string, notify chan<- bool) error { return s.registerNotification(subject, queue, notify) } func (s *Sublist) registerNotification(subject, queue string, notify chan<- bool) error { if subjectHasWildcard(subject) { return ErrInvalidSubject } if notify == nil { return ErrNilChan } var hasInterest bool r := s.Match(subject) if len(r.psubs)+len(r.qsubs) > 0 { if queue == _EMPTY_ { for _, sub := range r.psubs { if string(sub.subject) == subject { hasInterest = true break } } } else { for _, qsub := range r.qsubs { qs := qsub[0] if string(qs.subject) == subject && string(qs.queue) == queue { hasInterest = true break } } } } key := keyFromSubjectAndQueue(subject, queue) var err error s.Lock() if s.notify == nil { s.notify = ¬ifyMaps{ insert: make(map[string][]chan<- bool), remove: make(map[string][]chan<- bool), } } // Check which list to add us to. if hasInterest { err = s.addRemoveNotify(key, notify) } else { err = s.addInsertNotify(key, notify) } s.Unlock() if err == nil { sendNotification(notify, hasInterest) } return err } // Lock should be held. func chkAndRemove(key string, notify chan<- bool, ms map[string][]chan<- bool) bool { chs := ms[key] for i, ch := range chs { if ch == notify { chs[i] = chs[len(chs)-1] chs = chs[:len(chs)-1] if len(chs) == 0 { delete(ms, key) } return true } } return false } func (s *Sublist) ClearNotification(subject string, notify chan<- bool) bool { return s.clearNotification(subject, _EMPTY_, notify) } func (s *Sublist) ClearQueueNotification(subject, queue string, notify chan<- bool) bool { return s.clearNotification(subject, queue, notify) } func (s *Sublist) clearNotification(subject, queue string, notify chan<- bool) bool { s.Lock() if s.notify == nil { s.Unlock() return false } key := keyFromSubjectAndQueue(subject, queue) // Check both, start with remove. didRemove := chkAndRemove(key, notify, s.notify.remove) didRemove = didRemove || chkAndRemove(key, notify, s.notify.insert) // Check if everything is gone if len(s.notify.remove)+len(s.notify.insert) == 0 { s.notify = nil } s.Unlock() return didRemove } func sendNotification(ch chan<- bool, hasInterest bool) { select { case ch <- hasInterest: default: } } // Add a new channel for notification in insert map. // Write lock should be held. func (s *Sublist) addInsertNotify(subject string, notify chan<- bool) error { return s.addNotify(s.notify.insert, subject, notify) } // Add a new channel for notification in removal map. // Write lock should be held. func (s *Sublist) addRemoveNotify(subject string, notify chan<- bool) error { return s.addNotify(s.notify.remove, subject, notify) } // Add a new channel for notification. // Write lock should be held. func (s *Sublist) addNotify(m map[string][]chan<- bool, subject string, notify chan<- bool) error { chs := m[subject] if len(chs) > 0 { // Check to see if this chan is already registered. for _, ch := range chs { if ch == notify { return ErrAlreadyRegistered } } } m[subject] = append(chs, notify) return nil } // To generate a key from subject and queue. We just add spc. func keyFromSubjectAndQueue(subject, queue string) string { if len(queue) == 0 { return subject } var sb strings.Builder sb.WriteString(subject) sb.WriteString(" ") sb.WriteString(queue) return sb.String() } // chkForInsertNotification will check to see if we need to notify on this subject. // Write lock should be held. func (s *Sublist) chkForInsertNotification(subject, queue string) { key := keyFromSubjectAndQueue(subject, queue) // All notify subjects are also literal so just do a hash lookup here. if chs := s.notify.insert[key]; len(chs) > 0 { for _, ch := range chs { sendNotification(ch, true) } // Move from the insert map to the remove map. s.notify.remove[key] = append(s.notify.remove[key], chs...) delete(s.notify.insert, key) } } // chkForRemoveNotification will check to see if we need to notify on this subject. // Write lock should be held. func (s *Sublist) chkForRemoveNotification(subject, queue string) { key := keyFromSubjectAndQueue(subject, queue) if chs := s.notify.remove[key]; len(chs) > 0 { // We need to always check that we have no interest anymore. var hasInterest bool r := s.matchNoLock(subject) if len(r.psubs)+len(r.qsubs) > 0 { if queue == _EMPTY_ { for _, sub := range r.psubs { if string(sub.subject) == subject { hasInterest = true break } } } else { for _, qsub := range r.qsubs { qs := qsub[0] if string(qs.subject) == subject && string(qs.queue) == queue { hasInterest = true break } } } } if !hasInterest { for _, ch := range chs { sendNotification(ch, false) } // Move from the remove map to the insert map. s.notify.insert[key] = append(s.notify.insert[key], chs...) delete(s.notify.remove, key) } } } // Insert adds a subscription into the sublist func (s *Sublist) Insert(sub *subscription) error { // copy the subject since we hold this and this might be part of a large byte slice. subject := string(sub.subject) tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) s.Lock() var sfwc, haswc, isnew bool var n *node l := s.root for _, t := range tokens { lt := len(t) if lt == 0 || sfwc { s.Unlock() return ErrInvalidSubject } if lt > 1 { n = l.nodes[t] } else { switch t[0] { case pwc: n = l.pwc haswc = true case fwc: n = l.fwc haswc, sfwc = true, true default: n = l.nodes[t] } } if n == nil { n = newNode() if lt > 1 { l.nodes[t] = n } else { switch t[0] { case pwc: l.pwc = n case fwc: l.fwc = n default: l.nodes[t] = n } } } if n.next == nil { n.next = newLevel() } l = n.next } if sub.queue == nil { n.psubs[sub] = struct{}{} isnew = len(n.psubs) == 1 if n.plist != nil { n.plist = append(n.plist, sub) } else if len(n.psubs) > plistMin { n.plist = make([]*subscription, 0, len(n.psubs)) // Populate for psub := range n.psubs { n.plist = append(n.plist, psub) } } } else { if n.qsubs == nil { n.qsubs = make(map[string]map[*subscription]struct{}) } qname := string(sub.queue) // This is a queue subscription subs, ok := n.qsubs[qname] if !ok { subs = make(map[*subscription]struct{}) n.qsubs[qname] = subs isnew = true } subs[sub] = struct{}{} } s.count++ s.inserts++ s.addToCache(subject, sub) atomic.AddUint64(&s.genid, 1) if s.notify != nil && isnew && !haswc && len(s.notify.insert) > 0 { s.chkForInsertNotification(subject, string(sub.queue)) } s.Unlock() return nil } // Deep copy func copyResult(r *SublistResult) *SublistResult { nr := &SublistResult{} nr.psubs = append([]*subscription(nil), r.psubs...) for _, qr := range r.qsubs { nqr := append([]*subscription(nil), qr...) nr.qsubs = append(nr.qsubs, nqr) } return nr } // Adds a new sub to an existing result. func (r *SublistResult) addSubToResult(sub *subscription) *SublistResult { // Copy since others may have a reference. nr := copyResult(r) if sub.queue == nil { nr.psubs = append(nr.psubs, sub) } else { if i := findQSlot(sub.queue, nr.qsubs); i >= 0 { nr.qsubs[i] = append(nr.qsubs[i], sub) } else { nr.qsubs = append(nr.qsubs, []*subscription{sub}) } } return nr } // addToCache will add the new entry to the existing cache // entries if needed. Assumes write lock is held. // Assumes write lock is held. func (s *Sublist) addToCache(subject string, sub *subscription) { if s.cache == nil { return } // If literal we can direct match. if subjectIsLiteral(subject) { if r := s.cache[subject]; r != nil { s.cache[subject] = r.addSubToResult(sub) } return } for key, r := range s.cache { if matchLiteral(key, subject) { s.cache[key] = r.addSubToResult(sub) } } } // removeFromCache will remove the sub from any active cache entries. // Assumes write lock is held. func (s *Sublist) removeFromCache(subject string) { if s.cache == nil { return } // If literal we can direct match. if subjectIsLiteral(subject) { delete(s.cache, subject) return } // Wildcard here. for key := range s.cache { if matchLiteral(key, subject) { delete(s.cache, key) } } } // a place holder for an empty result. var emptyResult = &SublistResult{} // Match will match all entries to the literal subject. // It will return a set of results for both normal and queue subscribers. func (s *Sublist) Match(subject string) *SublistResult { return s.match(subject, true, false) } // MatchBytes will match all entries to the literal subject. // It will return a set of results for both normal and queue subscribers. func (s *Sublist) MatchBytes(subject []byte) *SublistResult { return s.match(bytesToString(subject), true, true) } // HasInterest will return whether or not there is any interest in the subject. // In cases where more detail is not required, this may be faster than Match. func (s *Sublist) HasInterest(subject string) bool { return s.hasInterest(subject, true, nil, nil) } // NumInterest will return the number of subs/qsubs interested in the subject. // In cases where more detail is not required, this may be faster than Match. func (s *Sublist) NumInterest(subject string) (np, nq int) { s.hasInterest(subject, true, &np, &nq) return } func (s *Sublist) matchNoLock(subject string) *SublistResult { return s.match(subject, false, false) } func (s *Sublist) match(subject string, doLock bool, doCopyOnCache bool) *SublistResult { atomic.AddUint64(&s.matches, 1) // Check cache first. if doLock { s.RLock() } cacheEnabled := s.cache != nil r, ok := s.cache[subject] if doLock { s.RUnlock() } if ok { atomic.AddUint64(&s.cacheHits, 1) return r } tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { if i-start == 0 { return emptyResult } tokens = append(tokens, subject[start:i]) start = i + 1 } } if start >= len(subject) { return emptyResult } tokens = append(tokens, subject[start:]) // FIXME(dlc) - Make shared pool between sublist and client readLoop? result := &SublistResult{} // Get result from the main structure and place into the shared cache. // Hold the read lock to avoid race between match and store. var n int if doLock { if cacheEnabled { s.Lock() } else { s.RLock() } } matchLevel(s.root, tokens, result) // Check for empty result. if len(result.psubs) == 0 && len(result.qsubs) == 0 { result = emptyResult } if cacheEnabled { if doCopyOnCache { subject = copyString(subject) } s.cache[subject] = result n = len(s.cache) } if doLock { if cacheEnabled { s.Unlock() } else { s.RUnlock() } } // Reduce the cache count if we have exceeded our set maximum. if cacheEnabled && n > slCacheMax && atomic.CompareAndSwapInt32(&s.ccSweep, 0, 1) { go s.reduceCacheCount() } return result } func (s *Sublist) hasInterest(subject string, doLock bool, np, nq *int) bool { // Check cache first. if doLock { s.RLock() } var matched bool if s.cache != nil { if r, ok := s.cache[subject]; ok { if np != nil && nq != nil { *np += len(r.psubs) for _, qsub := range r.qsubs { *nq += len(qsub) } } matched = len(r.psubs)+len(r.qsubs) > 0 } } if doLock { s.RUnlock() } if matched { atomic.AddUint64(&s.cacheHits, 1) return true } tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { if i-start == 0 { return false } tokens = append(tokens, subject[start:i]) start = i + 1 } } if start >= len(subject) { return false } tokens = append(tokens, subject[start:]) if doLock { s.RLock() defer s.RUnlock() } return matchLevelForAny(s.root, tokens, np, nq) } // Remove entries in the cache until we are under the maximum. // TODO(dlc) this could be smarter now that its not inline. func (s *Sublist) reduceCacheCount() { defer atomic.StoreInt32(&s.ccSweep, 0) // If we are over the cache limit randomly drop until under the limit. s.Lock() for key := range s.cache { delete(s.cache, key) if len(s.cache) <= slCacheSweep { break } } s.Unlock() } // Helper function for auto-expanding remote qsubs. func isRemoteQSub(sub *subscription) bool { return sub != nil && sub.queue != nil && sub.client != nil && (sub.client.kind == ROUTER || sub.client.kind == LEAF) } // UpdateRemoteQSub should be called when we update the weight of an existing // remote queue sub. func (s *Sublist) UpdateRemoteQSub(sub *subscription) { // We could search to make sure we find it, but probably not worth // it unless we are thrashing the cache. Just remove from our L2 and update // the genid so L1 will be flushed. s.Lock() s.removeFromCache(string(sub.subject)) atomic.AddUint64(&s.genid, 1) s.Unlock() } // This will add in a node's results to the total results. func addNodeToResults(n *node, results *SublistResult) { // Normal subscriptions if n.plist != nil { results.psubs = append(results.psubs, n.plist...) } else { for psub := range n.psubs { results.psubs = append(results.psubs, psub) } } // Queue subscriptions for qname, qr := range n.qsubs { if len(qr) == 0 { continue } // Need to find matching list in results var i int if i = findQSlot([]byte(qname), results.qsubs); i < 0 { i = len(results.qsubs) nqsub := make([]*subscription, 0, len(qr)) results.qsubs = append(results.qsubs, nqsub) } for sub := range qr { if isRemoteQSub(sub) { ns := atomic.LoadInt32(&sub.qw) // Shadow these subscriptions for n := 0; n < int(ns); n++ { results.qsubs[i] = append(results.qsubs[i], sub) } } else { results.qsubs[i] = append(results.qsubs[i], sub) } } } } // We do not use a map here since we want iteration to be past when // processing publishes in L1 on client. So we need to walk sequentially // for now. Keep an eye on this in case we start getting large number of // different queue subscribers for the same subject. func findQSlot(queue []byte, qsl [][]*subscription) int { if queue == nil { return -1 } for i, qr := range qsl { if len(qr) > 0 && bytes.Equal(queue, qr[0].queue) { return i } } return -1 } // matchLevel is used to recursively descend into the trie. func matchLevel(l *level, toks []string, results *SublistResult) { var pwc, n *node for i, t := range toks { if l == nil { return } if l.fwc != nil { addNodeToResults(l.fwc, results) } if pwc = l.pwc; pwc != nil { matchLevel(pwc.next, toks[i+1:], results) } n = l.nodes[t] if n != nil { l = n.next } else { l = nil } } if n != nil { addNodeToResults(n, results) } if pwc != nil { addNodeToResults(pwc, results) } } func matchLevelForAny(l *level, toks []string, np, nq *int) bool { var pwc, n *node for i, t := range toks { if l == nil { return false } if l.fwc != nil { if np != nil && nq != nil { *np += len(l.fwc.psubs) for _, qsub := range l.fwc.qsubs { *nq += len(qsub) } } return true } if pwc = l.pwc; pwc != nil { if match := matchLevelForAny(pwc.next, toks[i+1:], np, nq); match { return true } } n = l.nodes[t] if n != nil { l = n.next } else { l = nil } } if n != nil { if np != nil && nq != nil { *np += len(n.psubs) for _, qsub := range n.qsubs { *nq += len(qsub) } } return len(n.plist) > 0 || len(n.psubs) > 0 || len(n.qsubs) > 0 } if pwc != nil { if np != nil && nq != nil { *np += len(pwc.psubs) for _, qsub := range pwc.qsubs { *nq += len(qsub) } } return len(pwc.plist) > 0 || len(pwc.psubs) > 0 || len(pwc.qsubs) > 0 } return false } // lnt is used to track descent into levels for a removal for pruning. type lnt struct { l *level n *node t string } // Raw low level remove, can do batches with lock held outside. func (s *Sublist) remove(sub *subscription, shouldLock bool, doCacheUpdates bool) error { subject := string(sub.subject) tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) if shouldLock { s.Lock() defer s.Unlock() } var sfwc, haswc bool var n *node l := s.root // Track levels for pruning var lnts [32]lnt levels := lnts[:0] for _, t := range tokens { lt := len(t) if lt == 0 || sfwc { return ErrInvalidSubject } if l == nil { return ErrNotFound } if lt > 1 { n = l.nodes[t] } else { switch t[0] { case pwc: n = l.pwc haswc = true case fwc: n = l.fwc haswc, sfwc = true, true default: n = l.nodes[t] } } if n != nil { levels = append(levels, lnt{l, n, t}) l = n.next } else { l = nil } } removed, last := s.removeFromNode(n, sub) if !removed { return ErrNotFound } s.count-- s.removes++ for i := len(levels) - 1; i >= 0; i-- { l, n, t := levels[i].l, levels[i].n, levels[i].t if n.isEmpty() { l.pruneNode(n, t) } } if doCacheUpdates { s.removeFromCache(subject) atomic.AddUint64(&s.genid, 1) } if s.notify != nil && last && !haswc && len(s.notify.remove) > 0 { s.chkForRemoveNotification(subject, string(sub.queue)) } return nil } // Remove will remove a subscription. func (s *Sublist) Remove(sub *subscription) error { return s.remove(sub, true, true) } // RemoveBatch will remove a list of subscriptions. func (s *Sublist) RemoveBatch(subs []*subscription) error { if len(subs) == 0 { return nil } s.Lock() defer s.Unlock() // TODO(dlc) - We could try to be smarter here for a client going away but the account // has a large number of subscriptions compared to this client. Quick and dirty testing // though said just disabling all the time best for now. // Turn off our cache if enabled. wasEnabled := s.cache != nil s.cache = nil // We will try to remove all subscriptions but will report the first that caused // an error. In other words, we don't bail out at the first error which would // possibly leave a bunch of subscriptions that could have been removed. var err error for _, sub := range subs { if lerr := s.remove(sub, false, false); lerr != nil && err == nil { err = lerr } } // Turn caching back on here. atomic.AddUint64(&s.genid, 1) if wasEnabled { s.cache = make(map[string]*SublistResult) } return err } // pruneNode is used to prune an empty node from the tree. func (l *level) pruneNode(n *node, t string) { if n == nil { return } if n == l.fwc { l.fwc = nil } else if n == l.pwc { l.pwc = nil } else { delete(l.nodes, t) } } // isEmpty will test if the node has any entries. Used // in pruning. func (n *node) isEmpty() bool { if len(n.psubs) == 0 && len(n.qsubs) == 0 { if n.next == nil || n.next.numNodes() == 0 { return true } } return false } // Return the number of nodes for the given level. func (l *level) numNodes() int { num := len(l.nodes) if l.pwc != nil { num++ } if l.fwc != nil { num++ } return num } // Remove the sub for the given node. func (s *Sublist) removeFromNode(n *node, sub *subscription) (found, last bool) { if n == nil { return false, true } if sub.queue == nil { _, found = n.psubs[sub] delete(n.psubs, sub) if found && n.plist != nil { // This will brute force remove the plist to perform // correct behavior. Will get re-populated on a call // to Match as needed. n.plist = nil } return found, len(n.psubs) == 0 } // We have a queue group subscription here qsub := n.qsubs[string(sub.queue)] _, found = qsub[sub] delete(qsub, sub) if len(qsub) == 0 { // This is the last queue subscription interest when len(qsub) == 0, not // when n.qsubs is empty. last = true delete(n.qsubs, string(sub.queue)) } return found, last } // Count returns the number of subscriptions. func (s *Sublist) Count() uint32 { s.RLock() defer s.RUnlock() return s.count } // CacheCount returns the number of result sets in the cache. func (s *Sublist) CacheCount() int { s.RLock() cc := len(s.cache) s.RUnlock() return cc } // SublistStats are public stats for the sublist type SublistStats struct { NumSubs uint32 `json:"num_subscriptions"` NumCache uint32 `json:"num_cache"` NumInserts uint64 `json:"num_inserts"` NumRemoves uint64 `json:"num_removes"` NumMatches uint64 `json:"num_matches"` CacheHitRate float64 `json:"cache_hit_rate"` MaxFanout uint32 `json:"max_fanout"` AvgFanout float64 `json:"avg_fanout"` totFanout int cacheCnt int cacheHits uint64 } func (s *SublistStats) add(stat *SublistStats) { s.NumSubs += stat.NumSubs s.NumCache += stat.NumCache s.NumInserts += stat.NumInserts s.NumRemoves += stat.NumRemoves s.NumMatches += stat.NumMatches s.cacheHits += stat.cacheHits if s.MaxFanout < stat.MaxFanout { s.MaxFanout = stat.MaxFanout } // ignore slStats.AvgFanout, collect the values // it's based on instead s.totFanout += stat.totFanout s.cacheCnt += stat.cacheCnt if s.totFanout > 0 { s.AvgFanout = float64(s.totFanout) / float64(s.cacheCnt) } if s.NumMatches > 0 { s.CacheHitRate = float64(s.cacheHits) / float64(s.NumMatches) } } // Stats will return a stats structure for the current state. func (s *Sublist) Stats() *SublistStats { st := &SublistStats{} s.RLock() cache := s.cache cc := len(s.cache) st.NumSubs = s.count st.NumInserts = s.inserts st.NumRemoves = s.removes s.RUnlock() st.NumCache = uint32(cc) st.NumMatches = atomic.LoadUint64(&s.matches) st.cacheHits = atomic.LoadUint64(&s.cacheHits) if st.NumMatches > 0 { st.CacheHitRate = float64(st.cacheHits) / float64(st.NumMatches) } // whip through cache for fanout stats, this can be off if cache is full and doing evictions. // If this is called frequently, which it should not be, this could hurt performance. if cache != nil { tot, max, clen := 0, 0, 0 s.RLock() for _, r := range s.cache { clen++ l := len(r.psubs) + len(r.qsubs) tot += l if l > max { max = l } } s.RUnlock() st.totFanout = tot st.cacheCnt = clen st.MaxFanout = uint32(max) if tot > 0 { st.AvgFanout = float64(tot) / float64(clen) } } return st } // numLevels will return the maximum number of levels // contained in the Sublist tree. func (s *Sublist) numLevels() int { return visitLevel(s.root, 0) } // visitLevel is used to descend the Sublist tree structure // recursively. func visitLevel(l *level, depth int) int { if l == nil || l.numNodes() == 0 { return depth } depth++ maxDepth := depth for _, n := range l.nodes { if n == nil { continue } newDepth := visitLevel(n.next, depth) if newDepth > maxDepth { maxDepth = newDepth } } if l.pwc != nil { pwcDepth := visitLevel(l.pwc.next, depth) if pwcDepth > maxDepth { maxDepth = pwcDepth } } if l.fwc != nil { fwcDepth := visitLevel(l.fwc.next, depth) if fwcDepth > maxDepth { maxDepth = fwcDepth } } return maxDepth } // Determine if a subject has any wildcard tokens. func subjectHasWildcard(subject string) bool { // This one exits earlier then !subjectIsLiteral(subject) for i, c := range subject { if c == pwc || c == fwc { if (i == 0 || subject[i-1] == btsep) && (i+1 == len(subject) || subject[i+1] == btsep) { return true } } } return false } // Determine if the subject has any wildcards. Fast version, does not check for // valid subject. Used in caching layer. func subjectIsLiteral(subject string) bool { for i, c := range subject { if c == pwc || c == fwc { if (i == 0 || subject[i-1] == btsep) && (i+1 == len(subject) || subject[i+1] == btsep) { return false } } } return true } // IsValidPublishSubject returns true if a subject is valid and a literal, false otherwise func IsValidPublishSubject(subject string) bool { return IsValidSubject(subject) && subjectIsLiteral(subject) } // IsValidSubject returns true if a subject is valid, false otherwise func IsValidSubject(subject string) bool { return isValidSubject(subject, false) } func isValidSubject(subject string, checkRunes bool) bool { if subject == _EMPTY_ { return false } if checkRunes { // Check if we have embedded nulls. if bytes.IndexByte(stringToBytes(subject), 0) >= 0 { return false } // Since casting to a string will always produce valid UTF-8, we need to look for replacement runes. // This signals something is off or corrupt. for _, r := range subject { if r == utf8.RuneError { return false } } } sfwc := false tokens := strings.Split(subject, tsep) for _, t := range tokens { length := len(t) if length == 0 || sfwc { return false } if length > 1 { if strings.ContainsAny(t, "\t\n\f\r ") { return false } continue } switch t[0] { case fwc: sfwc = true case ' ', '\t', '\n', '\r', '\f': return false } } return true } // IsValidLiteralSubject returns true if a subject is valid and literal (no wildcards), false otherwise func IsValidLiteralSubject(subject string) bool { return isValidLiteralSubject(strings.Split(subject, tsep)) } // isValidLiteralSubject returns true if the tokens are valid and literal (no wildcards), false otherwise func isValidLiteralSubject(tokens []string) bool { for _, t := range tokens { if len(t) == 0 { return false } if len(t) > 1 { continue } switch t[0] { case pwc, fwc: return false } } return true } // ValidateMappingDestination returns nil error if the subject is a valid subject mapping destination subject func ValidateMappingDestination(subject string) error { if subject == _EMPTY_ { return nil } subjectTokens := strings.Split(subject, tsep) sfwc := false for _, t := range subjectTokens { length := len(t) if length == 0 || sfwc { return &mappingDestinationErr{t, ErrInvalidMappingDestinationSubject} } if length > 4 && t[0] == '{' && t[1] == '{' && t[length-2] == '}' && t[length-1] == '}' { if !partitionMappingFunctionRegEx.MatchString(t) && !wildcardMappingFunctionRegEx.MatchString(t) && !splitFromLeftMappingFunctionRegEx.MatchString(t) && !splitFromRightMappingFunctionRegEx.MatchString(t) && !sliceFromLeftMappingFunctionRegEx.MatchString(t) && !sliceFromRightMappingFunctionRegEx.MatchString(t) && !splitMappingFunctionRegEx.MatchString(t) { return &mappingDestinationErr{t, ErrUnknownMappingDestinationFunction} } else { continue } } if length == 1 && t[0] == fwc { sfwc = true } else if strings.ContainsAny(t, "\t\n\f\r ") { return ErrInvalidMappingDestinationSubject } } return nil } // Will check tokens and report back if the have any partial or full wildcards. func analyzeTokens(tokens []string) (hasPWC, hasFWC bool) { for _, t := range tokens { if lt := len(t); lt == 0 || lt > 1 { continue } switch t[0] { case pwc: hasPWC = true case fwc: hasFWC = true } } return } // Check on a token basis if they could match. func tokensCanMatch(t1, t2 string) bool { if len(t1) == 0 || len(t2) == 0 { return false } t1c, t2c := t1[0], t2[0] if t1c == pwc || t2c == pwc || t1c == fwc || t2c == fwc { return true } return t1 == t2 } // SubjectsCollide will determine if two subjects could both match a single literal subject. func SubjectsCollide(subj1, subj2 string) bool { if subj1 == subj2 { return true } toks1 := strings.Split(subj1, tsep) toks2 := strings.Split(subj2, tsep) pwc1, fwc1 := analyzeTokens(toks1) pwc2, fwc2 := analyzeTokens(toks2) // if both literal just string compare. l1, l2 := !(pwc1 || fwc1), !(pwc2 || fwc2) if l1 && l2 { return subj1 == subj2 } // So one or both have wildcards. If one is literal than we can do subset matching. if l1 && !l2 { return isSubsetMatch(toks1, subj2) } else if l2 && !l1 { return isSubsetMatch(toks2, subj1) } // Both have wildcards. // If they only have partials then the lengths must match. if !fwc1 && !fwc2 && len(toks1) != len(toks2) { return false } if lt1, lt2 := len(toks1), len(toks2); lt1 != lt2 { // If the shorter one only has partials then these will not collide. if lt1 < lt2 && !fwc1 || lt2 < lt1 && !fwc2 { return false } } stop := len(toks1) if len(toks2) < stop { stop = len(toks2) } // We look for reasons to say no. for i := 0; i < stop; i++ { t1, t2 := toks1[i], toks2[i] if !tokensCanMatch(t1, t2) { return false } } return true } // Returns number of tokens in the subject. func numTokens(subject string) int { var numTokens int if len(subject) == 0 { return 0 } for i := 0; i < len(subject); i++ { if subject[i] == btsep { numTokens++ } } return numTokens + 1 } // Fast way to return an indexed token. // This is one based, so first token is TokenAt(subject, 1) func tokenAt(subject string, index uint8) string { ti, start := uint8(1), 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { if ti == index { return subject[start:i] } start = i + 1 ti++ } } if ti == index { return subject[start:] } return _EMPTY_ } // use similar to append. meaning, the updated slice will be returned func tokenizeSubjectIntoSlice(tts []string, subject string) []string { start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tts = append(tts, subject[start:i]) start = i + 1 } } tts = append(tts, subject[start:]) return tts } // Calls into the function isSubsetMatch() func subjectIsSubsetMatch(subject, test string) bool { tsa := [32]string{} tts := tokenizeSubjectIntoSlice(tsa[:0], subject) return isSubsetMatch(tts, test) } // This will test a subject as an array of tokens against a test subject // Calls into the function isSubsetMatchTokenized func isSubsetMatch(tokens []string, test string) bool { tsa := [32]string{} tts := tokenizeSubjectIntoSlice(tsa[:0], test) return isSubsetMatchTokenized(tokens, tts) } // This will test a subject as an array of tokens against a test subject (also encoded as array of tokens) // and determine if the tokens are matched. Both test subject and tokens // may contain wildcards. So foo.* is a subset match of [">", "*.*", "foo.*"], // but not of foo.bar, etc. func isSubsetMatchTokenized(tokens, test []string) bool { // Walk the target tokens for i, t2 := range test { if i >= len(tokens) { return false } l := len(t2) if l == 0 { return false } if t2[0] == fwc && l == 1 { return true } t1 := tokens[i] l = len(t1) if l == 0 || t1[0] == fwc && l == 1 { return false } if t1[0] == pwc && len(t1) == 1 { m := t2[0] == pwc && len(t2) == 1 if !m { return false } if i >= len(test) { return true } continue } if t2[0] != pwc && strings.Compare(t1, t2) != 0 { return false } } return len(tokens) == len(test) } // matchLiteral is used to test literal subjects, those that do not have any // wildcards, with a target subject. This is used in the cache layer. func matchLiteral(literal, subject string) bool { li := 0 ll := len(literal) ls := len(subject) for i := 0; i < ls; i++ { if li >= ll { return false } // This function has been optimized for speed. // For instance, do not set b:=subject[i] here since // we may bump `i` in this loop to avoid `continue` or // skipping common test in a particular test. // Run Benchmark_SublistMatchLiteral before making any change. switch subject[i] { case pwc: // NOTE: This is not testing validity of a subject, instead ensures // that wildcards are treated as such if they follow some basic rules, // namely that they are a token on their own. if i == 0 || subject[i-1] == btsep { if i == ls-1 { // There is no more token in the subject after this wildcard. // Skip token in literal and expect to not find a separator. for { // End of literal, this is a match. if li >= ll { return true } // Presence of separator, this can't be a match. if literal[li] == btsep { return false } li++ } } else if subject[i+1] == btsep { // There is another token in the subject after this wildcard. // Skip token in literal and expect to get a separator. for { // We found the end of the literal before finding a separator, // this can't be a match. if li >= ll { return false } if literal[li] == btsep { break } li++ } // Bump `i` since we know there is a `.` following, we are // safe. The common test below is going to check `.` with `.` // which is good. A `continue` here is too costly. i++ } } case fwc: // For `>` to be a wildcard, it means being the only or last character // in the string preceded by a `.` if (i == 0 || subject[i-1] == btsep) && i == ls-1 { return true } } if subject[i] != literal[li] { return false } li++ } // Make sure we have processed all of the literal's chars.. return li >= ll } func addLocalSub(sub *subscription, subs *[]*subscription, includeLeafHubs bool) { if sub != nil && sub.client != nil { kind := sub.client.kind if kind == CLIENT || kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT || (includeLeafHubs && sub.client.isHubLeafNode() /* implied kind==LEAF */) { *subs = append(*subs, sub) } } } func (s *Sublist) addNodeToSubs(n *node, subs *[]*subscription, includeLeafHubs bool) { // Normal subscriptions if n.plist != nil { for _, sub := range n.plist { addLocalSub(sub, subs, includeLeafHubs) } } else { for sub := range n.psubs { addLocalSub(sub, subs, includeLeafHubs) } } // Queue subscriptions for _, qr := range n.qsubs { for sub := range qr { addLocalSub(sub, subs, includeLeafHubs) } } } func (s *Sublist) collectLocalSubs(l *level, subs *[]*subscription, includeLeafHubs bool) { for _, n := range l.nodes { s.addNodeToSubs(n, subs, includeLeafHubs) s.collectLocalSubs(n.next, subs, includeLeafHubs) } if l.pwc != nil { s.addNodeToSubs(l.pwc, subs, includeLeafHubs) s.collectLocalSubs(l.pwc.next, subs, includeLeafHubs) } if l.fwc != nil { s.addNodeToSubs(l.fwc, subs, includeLeafHubs) s.collectLocalSubs(l.fwc.next, subs, includeLeafHubs) } } // Return all local client subscriptions. Use the supplied slice. func (s *Sublist) localSubs(subs *[]*subscription, includeLeafHubs bool) { s.RLock() s.collectLocalSubs(s.root, subs, includeLeafHubs) s.RUnlock() } // All is used to collect all subscriptions. func (s *Sublist) All(subs *[]*subscription) { s.RLock() s.collectAllSubs(s.root, subs) s.RUnlock() } func (s *Sublist) addAllNodeToSubs(n *node, subs *[]*subscription) { // Normal subscriptions if n.plist != nil { *subs = append(*subs, n.plist...) } else { for sub := range n.psubs { *subs = append(*subs, sub) } } // Queue subscriptions for _, qr := range n.qsubs { for sub := range qr { *subs = append(*subs, sub) } } } func (s *Sublist) collectAllSubs(l *level, subs *[]*subscription) { for _, n := range l.nodes { s.addAllNodeToSubs(n, subs) s.collectAllSubs(n.next, subs) } if l.pwc != nil { s.addAllNodeToSubs(l.pwc, subs) s.collectAllSubs(l.pwc.next, subs) } if l.fwc != nil { s.addAllNodeToSubs(l.fwc, subs) s.collectAllSubs(l.fwc.next, subs) } } // For a given subject (which may contain wildcards), this call returns all // subscriptions that would match that subject. For instance, suppose that // the sublist contains: foo.bar, foo.bar.baz and foo.baz, ReverseMatch("foo.*") // would return foo.bar and foo.baz. // This is used in situations where the sublist is likely to contain only // literals and one wants to get all the subjects that would have been a match // to a subscription on `subject`. func (s *Sublist) ReverseMatch(subject string) *SublistResult { tsa := [32]string{} tokens := tsa[:0] start := 0 for i := 0; i < len(subject); i++ { if subject[i] == btsep { tokens = append(tokens, subject[start:i]) start = i + 1 } } tokens = append(tokens, subject[start:]) result := &SublistResult{} s.RLock() reverseMatchLevel(s.root, tokens, nil, result) // Check for empty result. if len(result.psubs) == 0 && len(result.qsubs) == 0 { result = emptyResult } s.RUnlock() return result } func reverseMatchLevel(l *level, toks []string, n *node, results *SublistResult) { if l == nil { return } for i, t := range toks { if len(t) == 1 { if t[0] == fwc { getAllNodes(l, results) return } else if t[0] == pwc { for _, n := range l.nodes { reverseMatchLevel(n.next, toks[i+1:], n, results) } if l.pwc != nil { reverseMatchLevel(l.pwc.next, toks[i+1:], n, results) } if l.fwc != nil { getAllNodes(l, results) } return } } // If the sub tree has a fwc at this position, match as well. if l.fwc != nil { getAllNodes(l, results) return } else if l.pwc != nil { reverseMatchLevel(l.pwc.next, toks[i+1:], n, results) } n = l.nodes[t] if n == nil { break } l = n.next } if n != nil { addNodeToResults(n, results) } } func getAllNodes(l *level, results *SublistResult) { if l == nil { return } if l.pwc != nil { addNodeToResults(l.pwc, results) } if l.fwc != nil { addNodeToResults(l.fwc, results) } for _, n := range l.nodes { addNodeToResults(n, results) getAllNodes(n.next, results) } } // IntersectStree will match all items in the given subject tree that // have interest expressed in the given sublist. The callback will only be called // once for each subject, regardless of overlapping subscriptions in the sublist. func IntersectStree[T any](st *stree.SubjectTree[T], sl *Sublist, cb func(subj []byte, entry *T)) { var _subj [255]byte intersectStree(st, sl.root, _subj[:0], cb) } func intersectStree[T any](st *stree.SubjectTree[T], r *level, subj []byte, cb func(subj []byte, entry *T)) { if r.numNodes() == 0 { // For wildcards we can't avoid Match, but if it's a literal subject at // this point, using Find is considerably cheaper. if subjectHasWildcard(bytesToString(subj)) { st.Match(subj, cb) } else if e, ok := st.Find(subj); ok { cb(subj, e) } return } nsubj := subj if len(nsubj) > 0 { nsubj = append(subj, '.') } switch { case r.fwc != nil: // We've reached a full wildcard, do a FWC match on the stree at this point // and don't keep iterating downward. nsubj := append(nsubj, '>') st.Match(nsubj, cb) case r.pwc != nil: // We've found a partial wildcard. We'll keep iterating downwards, but first // check whether there's interest at this level (without triggering dupes) and // match if so. nsubj := append(nsubj, '*') if len(r.pwc.psubs)+len(r.pwc.qsubs) > 0 && r.pwc.next != nil && r.pwc.next.numNodes() > 0 { st.Match(nsubj, cb) } intersectStree(st, r.pwc.next, nsubj, cb) case r.numNodes() > 0: // Normal node with subject literals, keep iterating. for t, n := range r.nodes { nsubj := append(nsubj, t...) intersectStree(st, n.next, nsubj, cb) } } } nats-server-2.10.27/server/sublist_test.go000066400000000000000000001744201477524627100205440ustar00rootroot00000000000000// Copyright 2016-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "errors" "flag" "fmt" "math/rand" "os" "runtime" "strconv" "strings" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/server/stree" "github.com/nats-io/nuid" ) func stackFatalf(t *testing.T, f string, args ...any) { lines := make([]string, 0, 32) msg := fmt.Sprintf(f, args...) lines = append(lines, msg) // Generate the Stack of callers: Skip us and verify* frames. for i := 2; true; i++ { _, file, line, ok := runtime.Caller(i) if !ok { break } msg := fmt.Sprintf("%d - %s:%d", i, file, line) lines = append(lines, msg) } t.Fatalf("%s", strings.Join(lines, "\n")) } func verifyCount(s *Sublist, count uint32, t *testing.T) { t.Helper() if s.Count() != count { t.Fatalf("Count is %d, should be %d", s.Count(), count) } } func verifyLen(r []*subscription, l int, t *testing.T) { t.Helper() if len(r) != l { t.Fatalf("Results len is %d, should be %d", len(r), l) } } func verifyQLen(r [][]*subscription, l int, t *testing.T) { t.Helper() if len(r) != l { t.Fatalf("Queue Results len is %d, should be %d", len(r), l) } } func verifyNumLevels(s *Sublist, expected int, t *testing.T) { t.Helper() dl := s.numLevels() if dl != expected { t.Fatalf("NumLevels is %d, should be %d", dl, expected) } } func verifyQMember(qsubs [][]*subscription, val *subscription, t *testing.T) { t.Helper() verifyMember(qsubs[findQSlot(val.queue, qsubs)], val, t) } func verifyMember(r []*subscription, val *subscription, t *testing.T) { t.Helper() for _, v := range r { if v == nil { continue } if v == val { return } } t.Fatalf("Subscription (%p) for [%s : %s] not found in results", val, val.subject, val.queue) } // Helpers to generate test subscriptions. func newSub(subject string) *subscription { c := &client{kind: CLIENT} return &subscription{client: c, subject: []byte(subject)} } func newQSub(subject, queue string) *subscription { if queue != "" { return &subscription{subject: []byte(subject), queue: []byte(queue)} } return newSub(subject) } func newRemoteQSub(subject, queue string, num int32) *subscription { if queue != "" { c := &client{kind: ROUTER} return &subscription{client: c, subject: []byte(subject), queue: []byte(queue), qw: num} } return newSub(subject) } func TestSublistInit(t *testing.T) { s := NewSublistWithCache() verifyCount(s, 0, t) } func TestSublistInsertCount(t *testing.T) { testSublistInsertCount(t, NewSublistWithCache()) } func TestSublistInsertCountNoCache(t *testing.T) { testSublistInsertCount(t, NewSublistNoCache()) } func testSublistInsertCount(t *testing.T, s *Sublist) { s.Insert(newSub("foo")) s.Insert(newSub("bar")) s.Insert(newSub("foo.bar")) verifyCount(s, 3, t) } func TestSublistSimple(t *testing.T) { testSublistSimple(t, NewSublistWithCache()) } func TestSublistSimpleNoCache(t *testing.T) { testSublistSimple(t, NewSublistNoCache()) } func testSublistSimple(t *testing.T, s *Sublist) { subject := "foo" sub := newSub(subject) s.Insert(sub) r := s.Match(subject) verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub, t) } func TestSublistSimpleMultiTokens(t *testing.T) { testSublistSimpleMultiTokens(t, NewSublistWithCache()) } func TestSublistSimpleMultiTokensNoCache(t *testing.T) { testSublistSimpleMultiTokens(t, NewSublistNoCache()) } func testSublistSimpleMultiTokens(t *testing.T, s *Sublist) { subject := "foo.bar.baz" sub := newSub(subject) s.Insert(sub) r := s.Match(subject) verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub, t) } func TestSublistPartialWildcard(t *testing.T) { testSublistPartialWildcard(t, NewSublistWithCache()) } func TestSublistPartialWildcardNoCache(t *testing.T) { testSublistPartialWildcard(t, NewSublistNoCache()) } func testSublistPartialWildcard(t *testing.T, s *Sublist) { lsub := newSub("a.b.c") psub := newSub("a.*.c") s.Insert(lsub) s.Insert(psub) r := s.Match("a.b.c") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, lsub, t) verifyMember(r.psubs, psub, t) } func TestSublistPartialWildcardAtEnd(t *testing.T) { testSublistPartialWildcardAtEnd(t, NewSublistWithCache()) } func TestSublistPartialWildcardAtEndNoCache(t *testing.T) { testSublistPartialWildcardAtEnd(t, NewSublistNoCache()) } func testSublistPartialWildcardAtEnd(t *testing.T, s *Sublist) { lsub := newSub("a.b.c") psub := newSub("a.b.*") s.Insert(lsub) s.Insert(psub) r := s.Match("a.b.c") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, lsub, t) verifyMember(r.psubs, psub, t) } func TestSublistFullWildcard(t *testing.T) { testSublistFullWildcard(t, NewSublistWithCache()) } func TestSublistFullWildcardNoCache(t *testing.T) { testSublistFullWildcard(t, NewSublistNoCache()) } func testSublistFullWildcard(t *testing.T, s *Sublist) { lsub := newSub("a.b.c") fsub := newSub("a.>") s.Insert(lsub) s.Insert(fsub) r := s.Match("a.b.c") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, lsub, t) verifyMember(r.psubs, fsub, t) r = s.Match("a.>") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, fsub, t) } func TestSublistRemove(t *testing.T) { testSublistRemove(t, NewSublistWithCache()) } func TestSublistRemoveNoCache(t *testing.T) { testSublistRemove(t, NewSublistNoCache()) } func testSublistRemove(t *testing.T, s *Sublist) { subject := "a.b.c.d" sub := newSub(subject) s.Insert(sub) verifyCount(s, 1, t) r := s.Match(subject) verifyLen(r.psubs, 1, t) s.Remove(newSub("a.b.c")) verifyCount(s, 1, t) s.Remove(sub) verifyCount(s, 0, t) r = s.Match(subject) verifyLen(r.psubs, 0, t) } func TestSublistRemoveWildcard(t *testing.T) { testSublistRemoveWildcard(t, NewSublistWithCache()) } func TestSublistRemoveWildcardNoCache(t *testing.T) { testSublistRemoveWildcard(t, NewSublistNoCache()) } func testSublistRemoveWildcard(t *testing.T, s *Sublist) { subject := "a.b.c.d" sub := newSub(subject) psub := newSub("a.b.*.d") fsub := newSub("a.b.>") s.Insert(sub) s.Insert(psub) s.Insert(fsub) verifyCount(s, 3, t) r := s.Match(subject) verifyLen(r.psubs, 3, t) s.Remove(sub) verifyCount(s, 2, t) s.Remove(fsub) verifyCount(s, 1, t) s.Remove(psub) verifyCount(s, 0, t) r = s.Match(subject) verifyLen(r.psubs, 0, t) } func TestSublistRemoveCleanup(t *testing.T) { testSublistRemoveCleanup(t, NewSublistWithCache()) } func TestSublistRemoveCleanupNoCache(t *testing.T) { testSublistRemoveCleanup(t, NewSublistNoCache()) } func testSublistRemoveCleanup(t *testing.T, s *Sublist) { literal := "a.b.c.d.e.f" depth := len(strings.Split(literal, tsep)) sub := newSub(literal) verifyNumLevels(s, 0, t) s.Insert(sub) verifyNumLevels(s, depth, t) s.Remove(sub) verifyNumLevels(s, 0, t) } func TestSublistRemoveCleanupWildcards(t *testing.T) { testSublistRemoveCleanupWildcards(t, NewSublistWithCache()) } func TestSublistRemoveCleanupWildcardsNoCache(t *testing.T) { testSublistRemoveCleanupWildcards(t, NewSublistNoCache()) } func testSublistRemoveCleanupWildcards(t *testing.T, s *Sublist) { subject := "a.b.*.d.e.>" depth := len(strings.Split(subject, tsep)) sub := newSub(subject) verifyNumLevels(s, 0, t) s.Insert(sub) verifyNumLevels(s, depth, t) s.Remove(sub) verifyNumLevels(s, 0, t) } func TestSublistRemoveWithLargeSubs(t *testing.T) { testSublistRemoveWithLargeSubs(t, NewSublistWithCache()) } func TestSublistRemoveWithLargeSubsNoCache(t *testing.T) { testSublistRemoveWithLargeSubs(t, NewSublistNoCache()) } func testSublistRemoveWithLargeSubs(t *testing.T, s *Sublist) { subject := "foo" for i := 0; i < plistMin*2; i++ { sub := newSub(subject) s.Insert(sub) } r := s.Match(subject) verifyLen(r.psubs, plistMin*2, t) // Remove one that is in the middle s.Remove(r.psubs[plistMin]) // Remove first one s.Remove(r.psubs[0]) // Remove last one s.Remove(r.psubs[len(r.psubs)-1]) // Check len again r = s.Match(subject) verifyLen(r.psubs, plistMin*2-3, t) } func TestSublistInvalidSubjectsInsert(t *testing.T) { testSublistInvalidSubjectsInsert(t, NewSublistWithCache()) } func TestSublistInvalidSubjectsInsertNoCache(t *testing.T) { testSublistInvalidSubjectsInsert(t, NewSublistNoCache()) } func TestSublistNoCacheRemoveBatch(t *testing.T) { s := NewSublistNoCache() s.Insert(newSub("foo")) sub := newSub("bar") s.Insert(sub) s.RemoveBatch([]*subscription{sub}) // Now test that this did not turn on cache for i := 0; i < 10; i++ { s.Match("foo") } if s.CacheEnabled() { t.Fatalf("Cache should not be enabled") } } func TestSublistRemoveBatchWithError(t *testing.T) { s := NewSublistNoCache() sub1 := newSub("foo") sub2 := newSub("bar") sub3 := newSub("baz") s.Insert(sub1) s.Insert(sub2) s.Insert(sub3) subNotPresent := newSub("not.inserted") // Try to remove all subs, but include the sub that has not been inserted. err := s.RemoveBatch([]*subscription{subNotPresent, sub1, sub3}) // We expect an error to be returned, but sub1,2 and 3 to have been removed. require_Error(t, err, ErrNotFound) // Make sure that we have only sub2 present verifyCount(s, 1, t) r := s.Match("bar") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub2, t) r = s.Match("foo") verifyLen(r.psubs, 0, t) r = s.Match("baz") verifyLen(r.psubs, 0, t) } func testSublistInvalidSubjectsInsert(t *testing.T, s *Sublist) { // Insert, or subscriptions, can have wildcards, but not empty tokens, // and can not have a FWC that is not the terminal token. // beginning empty token if err := s.Insert(newSub(".foo")); err != ErrInvalidSubject { t.Fatal("Expected invalid subject error") } // trailing empty token if err := s.Insert(newSub("foo.")); err != ErrInvalidSubject { t.Fatal("Expected invalid subject error") } // empty middle token if err := s.Insert(newSub("foo..bar")); err != ErrInvalidSubject { t.Fatal("Expected invalid subject error") } // empty middle token #2 if err := s.Insert(newSub("foo.bar..baz")); err != ErrInvalidSubject { t.Fatal("Expected invalid subject error") } // fwc not terminal if err := s.Insert(newSub("foo.>.bar")); err != ErrInvalidSubject { t.Fatal("Expected invalid subject error") } } func TestSublistCache(t *testing.T) { s := NewSublistWithCache() // Test add a remove logistics subject := "a.b.c.d" sub := newSub(subject) psub := newSub("a.b.*.d") fsub := newSub("a.b.>") s.Insert(sub) r := s.Match(subject) verifyLen(r.psubs, 1, t) s.Insert(psub) s.Insert(fsub) verifyCount(s, 3, t) r = s.Match(subject) verifyLen(r.psubs, 3, t) s.Remove(sub) verifyCount(s, 2, t) s.Remove(fsub) verifyCount(s, 1, t) s.Remove(psub) verifyCount(s, 0, t) // Check that cache is now empty if cc := s.CacheCount(); cc != 0 { t.Fatalf("Cache should be zero, got %d\n", cc) } r = s.Match(subject) verifyLen(r.psubs, 0, t) for i := 0; i < 2*slCacheMax; i++ { s.Match(fmt.Sprintf("foo-%d\n", i)) } checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { if cc := s.CacheCount(); cc > slCacheMax { return fmt.Errorf("Cache should be constrained by cacheMax, got %d for current count", cc) } return nil }) // Test that adding to a wildcard properly adds to the cache. s = NewSublistWithCache() s.Insert(newSub("foo.*")) s.Insert(newSub("foo.bar")) r = s.Match("foo.baz") verifyLen(r.psubs, 1, t) r = s.Match("foo.bar") verifyLen(r.psubs, 2, t) s.Insert(newSub("foo.>")) r = s.Match("foo.bar") verifyLen(r.psubs, 3, t) } func TestSublistBasicQueueResults(t *testing.T) { testSublistBasicQueueResults(t, NewSublistWithCache()) } func TestSublistBasicQueueResultsNoCache(t *testing.T) { testSublistBasicQueueResults(t, NewSublistNoCache()) } func testSublistBasicQueueResults(t *testing.T, s *Sublist) { // Test some basics subject := "foo" sub := newSub(subject) sub1 := newQSub(subject, "bar") sub2 := newQSub(subject, "baz") s.Insert(sub1) r := s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 1, t) verifyLen(r.qsubs[0], 1, t) verifyQMember(r.qsubs, sub1, t) s.Insert(sub2) r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 2, t) verifyLen(r.qsubs[0], 1, t) verifyLen(r.qsubs[1], 1, t) verifyQMember(r.qsubs, sub1, t) verifyQMember(r.qsubs, sub2, t) s.Insert(sub) r = s.Match(subject) verifyLen(r.psubs, 1, t) verifyQLen(r.qsubs, 2, t) verifyLen(r.qsubs[0], 1, t) verifyLen(r.qsubs[1], 1, t) verifyQMember(r.qsubs, sub1, t) verifyQMember(r.qsubs, sub2, t) verifyMember(r.psubs, sub, t) sub3 := newQSub(subject, "bar") sub4 := newQSub(subject, "baz") s.Insert(sub3) s.Insert(sub4) r = s.Match(subject) verifyLen(r.psubs, 1, t) verifyQLen(r.qsubs, 2, t) verifyLen(r.qsubs[0], 2, t) verifyLen(r.qsubs[1], 2, t) verifyQMember(r.qsubs, sub1, t) verifyQMember(r.qsubs, sub2, t) verifyQMember(r.qsubs, sub3, t) verifyQMember(r.qsubs, sub4, t) verifyMember(r.psubs, sub, t) // Now removal s.Remove(sub) r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 2, t) verifyLen(r.qsubs[0], 2, t) verifyLen(r.qsubs[1], 2, t) verifyQMember(r.qsubs, sub1, t) verifyQMember(r.qsubs, sub2, t) s.Remove(sub1) r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 2, t) verifyLen(r.qsubs[findQSlot(sub1.queue, r.qsubs)], 1, t) verifyLen(r.qsubs[findQSlot(sub2.queue, r.qsubs)], 2, t) verifyQMember(r.qsubs, sub2, t) verifyQMember(r.qsubs, sub3, t) verifyQMember(r.qsubs, sub4, t) s.Remove(sub3) // Last one r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 1, t) verifyLen(r.qsubs[0], 2, t) // this is sub2/baz now verifyQMember(r.qsubs, sub2, t) s.Remove(sub2) s.Remove(sub4) r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 0, t) } func checkBool(b, expected bool, t *testing.T) { t.Helper() if b != expected { t.Fatalf("Expected %v, but got %v\n", expected, b) } } func checkError(err, expected error, t *testing.T) { t.Helper() if err != expected && err != nil && !errors.Is(err, expected) { t.Fatalf("Expected %v, but got %v\n", expected, err) } } func TestSublistValidLiteralSubjects(t *testing.T) { checkBool(IsValidLiteralSubject("foo"), true, t) checkBool(IsValidLiteralSubject(".foo"), false, t) checkBool(IsValidLiteralSubject("foo."), false, t) checkBool(IsValidLiteralSubject("foo..bar"), false, t) checkBool(IsValidLiteralSubject("foo.bar.*"), false, t) checkBool(IsValidLiteralSubject("foo.bar.>"), false, t) checkBool(IsValidLiteralSubject("*"), false, t) checkBool(IsValidLiteralSubject(">"), false, t) // The followings have widlcards characters but are not // considered as such because they are not individual tokens. checkBool(IsValidLiteralSubject("foo*"), true, t) checkBool(IsValidLiteralSubject("foo**"), true, t) checkBool(IsValidLiteralSubject("foo.**"), true, t) checkBool(IsValidLiteralSubject("foo*bar"), true, t) checkBool(IsValidLiteralSubject("foo.*bar"), true, t) checkBool(IsValidLiteralSubject("foo*.bar"), true, t) checkBool(IsValidLiteralSubject("*bar"), true, t) checkBool(IsValidLiteralSubject("foo>"), true, t) checkBool(IsValidLiteralSubject("foo>>"), true, t) checkBool(IsValidLiteralSubject("foo.>>"), true, t) checkBool(IsValidLiteralSubject("foo>bar"), true, t) checkBool(IsValidLiteralSubject("foo.>bar"), true, t) checkBool(IsValidLiteralSubject("foo>.bar"), true, t) checkBool(IsValidLiteralSubject(">bar"), true, t) } func TestSublistValidSubjects(t *testing.T) { checkBool(IsValidSubject("."), false, t) checkBool(IsValidSubject(".foo"), false, t) checkBool(IsValidSubject("foo."), false, t) checkBool(IsValidSubject("foo..bar"), false, t) checkBool(IsValidSubject(">.bar"), false, t) checkBool(IsValidSubject("foo.>.bar"), false, t) checkBool(IsValidSubject("foo"), true, t) checkBool(IsValidSubject("foo.bar.*"), true, t) checkBool(IsValidSubject("foo.bar.>"), true, t) checkBool(IsValidSubject("*"), true, t) checkBool(IsValidSubject(">"), true, t) checkBool(IsValidSubject("foo*"), true, t) checkBool(IsValidSubject("foo**"), true, t) checkBool(IsValidSubject("foo.**"), true, t) checkBool(IsValidSubject("foo*bar"), true, t) checkBool(IsValidSubject("foo.*bar"), true, t) checkBool(IsValidSubject("foo*.bar"), true, t) checkBool(IsValidSubject("*bar"), true, t) checkBool(IsValidSubject("foo>"), true, t) checkBool(IsValidSubject("foo.>>"), true, t) checkBool(IsValidSubject("foo>bar"), true, t) checkBool(IsValidSubject("foo.>bar"), true, t) checkBool(IsValidSubject("foo>.bar"), true, t) checkBool(IsValidSubject(">bar"), true, t) // Check for embedded nulls. subj := []byte("foo.bar.baz.") subj = append(subj, 0) checkBool(isValidSubject(string(subj), true), false, t) } func TestSublistMatchLiterals(t *testing.T) { checkBool(matchLiteral("foo", "foo"), true, t) checkBool(matchLiteral("foo", "bar"), false, t) checkBool(matchLiteral("foo", "*"), true, t) checkBool(matchLiteral("foo", ">"), true, t) checkBool(matchLiteral("foo.bar", ">"), true, t) checkBool(matchLiteral("foo.bar", "foo.>"), true, t) checkBool(matchLiteral("foo.bar", "bar.>"), false, t) checkBool(matchLiteral("stats.test.22", "stats.>"), true, t) checkBool(matchLiteral("stats.test.22", "stats.*.*"), true, t) checkBool(matchLiteral("foo.bar", "foo"), false, t) checkBool(matchLiteral("stats.test.foos", "stats.test.foos"), true, t) checkBool(matchLiteral("stats.test.foos", "stats.test.foo"), false, t) checkBool(matchLiteral("stats.test", "stats.test.*"), false, t) checkBool(matchLiteral("stats.test.foos", "stats.*"), false, t) checkBool(matchLiteral("stats.test.foos", "stats.*.*.foos"), false, t) // These are cases where wildcards characters should not be considered // wildcards since they do not follow the rules of wildcards. checkBool(matchLiteral("*bar", "*bar"), true, t) checkBool(matchLiteral("foo*", "foo*"), true, t) checkBool(matchLiteral("foo*bar", "foo*bar"), true, t) checkBool(matchLiteral("foo.***.bar", "foo.***.bar"), true, t) checkBool(matchLiteral(">bar", ">bar"), true, t) checkBool(matchLiteral("foo>", "foo>"), true, t) checkBool(matchLiteral("foo>bar", "foo>bar"), true, t) checkBool(matchLiteral("foo.>>>.bar", "foo.>>>.bar"), true, t) } func TestSubjectIsLiteral(t *testing.T) { checkBool(subjectIsLiteral("foo"), true, t) checkBool(subjectIsLiteral("foo.bar"), true, t) checkBool(subjectIsLiteral("foo*.bar"), true, t) checkBool(subjectIsLiteral("*"), false, t) checkBool(subjectIsLiteral(">"), false, t) checkBool(subjectIsLiteral("foo.*"), false, t) checkBool(subjectIsLiteral("foo.>"), false, t) checkBool(subjectIsLiteral("foo.*.>"), false, t) checkBool(subjectIsLiteral("foo.*.bar"), false, t) checkBool(subjectIsLiteral("foo.bar.>"), false, t) } func TestValidateDestinationSubject(t *testing.T) { checkError(ValidateMappingDestination("foo"), nil, t) checkError(ValidateMappingDestination("foo.bar"), nil, t) checkError(ValidateMappingDestination("*"), nil, t) checkError(ValidateMappingDestination(">"), nil, t) checkError(ValidateMappingDestination("foo.*"), nil, t) checkError(ValidateMappingDestination("foo.>"), nil, t) checkError(ValidateMappingDestination("foo.*.>"), nil, t) checkError(ValidateMappingDestination("foo.*.bar"), nil, t) checkError(ValidateMappingDestination("foo.bar.>"), nil, t) checkError(ValidateMappingDestination("foo.{{wildcard(1)}}"), nil, t) checkError(ValidateMappingDestination("foo.{{ wildcard(1) }}"), nil, t) checkError(ValidateMappingDestination("foo.{{wildcard( 1 )}}"), nil, t) checkError(ValidateMappingDestination("foo.{{partition(2,1)}}"), nil, t) checkError(ValidateMappingDestination("foo.{{SplitFromLeft(2,1)}}"), nil, t) checkError(ValidateMappingDestination("foo.{{SplitFromRight(2,1)}}"), nil, t) checkError(ValidateMappingDestination("foo.{{unknown(1)}}"), ErrInvalidMappingDestination, t) checkError(ValidateMappingDestination("foo..}"), ErrInvalidMappingDestination, t) checkError(ValidateMappingDestination("foo. bar}"), ErrInvalidMappingDestinationSubject, t) } func TestSubjectToken(t *testing.T) { checkToken := func(token, expected string) { t.Helper() if token != expected { t.Fatalf("Expected token of %q, got %q", expected, token) } } checkToken(tokenAt("foo.bar.baz.*", 0), "") checkToken(tokenAt("foo.bar.baz.*", 1), "foo") checkToken(tokenAt("foo.bar.baz.*", 2), "bar") checkToken(tokenAt("foo.bar.baz.*", 3), "baz") checkToken(tokenAt("foo.bar.baz.*", 4), "*") checkToken(tokenAt("foo.bar.baz.*", 5), "") } func TestSublistBadSubjectOnRemove(t *testing.T) { testSublistBadSubjectOnRemove(t, NewSublistWithCache()) } func TestSublistBadSubjectOnRemoveNoCache(t *testing.T) { testSublistBadSubjectOnRemove(t, NewSublistNoCache()) } func testSublistBadSubjectOnRemove(t *testing.T, s *Sublist) { bad := "a.b..d" sub := newSub(bad) if err := s.Insert(sub); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) } if err := s.Remove(sub); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) } badfwc := "a.>.b" if err := s.Remove(newSub(badfwc)); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject, got %v\n", err) } } // This is from bug report #18 func TestSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) { testSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistWithCache()) } func TestSublistTwoTokenPubMatchSingleTokenSubNoCache(t *testing.T) { testSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistNoCache()) } func testSublistTwoTokenPubMatchSingleTokenSub(t *testing.T, s *Sublist) { sub := newSub("foo") s.Insert(sub) r := s.Match("foo") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub, t) r = s.Match("foo.bar") verifyLen(r.psubs, 0, t) } func TestSublistInsertWithWildcardsAsLiterals(t *testing.T) { testSublistInsertWithWildcardsAsLiterals(t, NewSublistWithCache()) } func TestSublistInsertWithWildcardsAsLiteralsNoCache(t *testing.T) { testSublistInsertWithWildcardsAsLiterals(t, NewSublistNoCache()) } func testSublistInsertWithWildcardsAsLiterals(t *testing.T, s *Sublist) { subjects := []string{"foo.*-", "foo.>-"} for _, subject := range subjects { sub := newSub(subject) s.Insert(sub) // Should find no match r := s.Match("foo.bar") verifyLen(r.psubs, 0, t) // Should find a match r = s.Match(subject) verifyLen(r.psubs, 1, t) } } func TestSublistRemoveWithWildcardsAsLiterals(t *testing.T) { testSublistRemoveWithWildcardsAsLiterals(t, NewSublistWithCache()) } func TestSublistRemoveWithWildcardsAsLiteralsNoCache(t *testing.T) { testSublistRemoveWithWildcardsAsLiterals(t, NewSublistNoCache()) } func testSublistRemoveWithWildcardsAsLiterals(t *testing.T, s *Sublist) { subjects := []string{"foo.*-", "foo.>-"} for _, subject := range subjects { sub := newSub(subject) s.Insert(sub) // Should find no match rsub := newSub("foo.bar") s.Remove(rsub) if c := s.Count(); c != 1 { t.Fatalf("Expected sublist to still contain sub, got %v", c) } s.Remove(sub) if c := s.Count(); c != 0 { t.Fatalf("Expected sublist to be empty, got %v", c) } } } func TestSublistRaceOnRemove(t *testing.T) { testSublistRaceOnRemove(t, NewSublistWithCache()) } func TestSublistRaceOnRemoveNoCache(t *testing.T) { testSublistRaceOnRemove(t, NewSublistNoCache()) } func testSublistRaceOnRemove(t *testing.T, s *Sublist) { var ( total = 100 subs = make(map[int]*subscription, total) // use map for randomness ) for i := 0; i < total; i++ { sub := newQSub("foo", "bar") subs[i] = sub } for i := 0; i < 2; i++ { for _, sub := range subs { s.Insert(sub) } // Call Match() once or twice, to make sure we get from cache if i == 1 { s.Match("foo") } // This will be from cache when i==1 r := s.Match("foo") wg := sync.WaitGroup{} wg.Add(1) go func() { for _, sub := range subs { s.Remove(sub) } wg.Done() }() for _, qsub := range r.qsubs { for i := 0; i < len(qsub); i++ { sub := qsub[i] if string(sub.queue) != "bar" { t.Fatalf("Queue name should be bar, got %s", qsub[i].queue) } } } wg.Wait() } // Repeat tests with regular subs for i := 0; i < total; i++ { sub := newSub("foo") subs[i] = sub } for i := 0; i < 2; i++ { for _, sub := range subs { s.Insert(sub) } // Call Match() once or twice, to make sure we get from cache if i == 1 { s.Match("foo") } // This will be from cache when i==1 r := s.Match("foo") wg := sync.WaitGroup{} wg.Add(1) go func() { for _, sub := range subs { s.Remove(sub) } wg.Done() }() for i := 0; i < len(r.psubs); i++ { sub := r.psubs[i] if string(sub.subject) != "foo" { t.Fatalf("Subject should be foo, got %s", sub.subject) } } wg.Wait() } } func TestSublistRaceOnInsert(t *testing.T) { testSublistRaceOnInsert(t, NewSublistWithCache()) } func TestSublistRaceOnInsertNoCache(t *testing.T) { testSublistRaceOnInsert(t, NewSublistNoCache()) } func testSublistRaceOnInsert(t *testing.T, s *Sublist) { var ( total = 100 subs = make(map[int]*subscription, total) // use map for randomness wg sync.WaitGroup ) for i := 0; i < total; i++ { sub := newQSub("foo", "bar") subs[i] = sub } wg.Add(1) go func() { for _, sub := range subs { s.Insert(sub) } wg.Done() }() for i := 0; i < 1000; i++ { r := s.Match("foo") for _, qsubs := range r.qsubs { for _, qsub := range qsubs { if string(qsub.queue) != "bar" { t.Fatalf("Expected queue name to be bar, got %v", string(qsub.queue)) } } } } wg.Wait() // Repeat the test with plain subs for i := 0; i < total; i++ { sub := newSub("foo") subs[i] = sub } wg.Add(1) go func() { for _, sub := range subs { s.Insert(sub) } wg.Done() }() for i := 0; i < 1000; i++ { r := s.Match("foo") for _, sub := range r.psubs { if string(sub.subject) != "foo" { t.Fatalf("Expected subject to be foo, got %v", string(sub.subject)) } } } wg.Wait() } func TestSublistRaceOnMatch(t *testing.T) { s := NewSublistNoCache() s.Insert(newQSub("foo.*", "workers")) s.Insert(newQSub("foo.bar", "workers")) s.Insert(newSub("foo.*")) s.Insert(newSub("foo.bar")) wg := sync.WaitGroup{} wg.Add(2) errCh := make(chan error, 2) f := func() { defer wg.Done() for i := 0; i < 10; i++ { r := s.Match("foo.bar") for _, sub := range r.psubs { if !strings.HasPrefix(string(sub.subject), "foo.") { errCh <- fmt.Errorf("Wrong subject: %s", sub.subject) return } } for _, qsub := range r.qsubs { for _, sub := range qsub { if string(sub.queue) != "workers" { errCh <- fmt.Errorf("Wrong queue name: %s", sub.queue) return } } } } } go f() go f() wg.Wait() select { case e := <-errCh: t.Fatal(e.Error()) default: } } // Remote subscriptions for queue subscribers will be weighted such that a single subscription // is received, but represents all of the queue subscribers on the remote side. func TestSublistRemoteQueueSubscriptions(t *testing.T) { testSublistRemoteQueueSubscriptions(t, NewSublistWithCache()) } func TestSublistRemoteQueueSubscriptionsNoCache(t *testing.T) { testSublistRemoteQueueSubscriptions(t, NewSublistNoCache()) } func testSublistRemoteQueueSubscriptions(t *testing.T, s *Sublist) { // Normals s1 := newQSub("foo", "bar") s2 := newQSub("foo", "bar") s.Insert(s1) s.Insert(s2) // Now do weighted remotes. rs1 := newRemoteQSub("foo", "bar", 10) s.Insert(rs1) rs2 := newRemoteQSub("foo", "bar", 10) s.Insert(rs2) // These are just shadowed in results, so should appear as 4 subs. verifyCount(s, 4, t) r := s.Match("foo") verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 1, t) verifyLen(r.qsubs[0], 22, t) s.Remove(s1) s.Remove(rs1) verifyCount(s, 2, t) // Now make sure our shadowed results are correct after a removal. r = s.Match("foo") verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 1, t) verifyLen(r.qsubs[0], 11, t) // Now do an update to an existing remote sub to update its weight. rs2.qw = 1 s.UpdateRemoteQSub(rs2) // Results should reflect new weight. r = s.Match("foo") verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 1, t) verifyLen(r.qsubs[0], 2, t) } func TestSublistSharedEmptyResult(t *testing.T) { s := NewSublistWithCache() r1 := s.Match("foo") verifyLen(r1.psubs, 0, t) verifyQLen(r1.qsubs, 0, t) r2 := s.Match("bar") verifyLen(r2.psubs, 0, t) verifyQLen(r2.qsubs, 0, t) if r1 != r2 { t.Fatalf("Expected empty result to be a shared result set") } } func TestSublistNoCacheStats(t *testing.T) { s := NewSublistNoCache() s.Insert(newSub("foo")) s.Insert(newSub("bar")) s.Insert(newSub("baz")) s.Insert(newSub("foo.bar.baz")) s.Match("a.b.c") s.Match("bar") stats := s.Stats() if stats.NumCache != 0 { t.Fatalf("Expected 0 for NumCache stat, got %d", stats.NumCache) } } func TestSublistAll(t *testing.T) { s := NewSublistNoCache() subs := []*subscription{ newSub("foo.bar.baz"), newSub("foo"), newSub("baz"), } // alter client's kind subs[0].client.kind = LEAF for _, sub := range subs { s.Insert(sub) } var buf [32]*subscription output := buf[:0] s.All(&output) if len(output) != len(subs) { t.Fatalf("Expected %d for All, got %d", len(subs), len(output)) } } func TestIsSubsetMatch(t *testing.T) { for _, test := range []struct { subject string test string result bool }{ {"foo.bar", "foo.bar", true}, {"foo.*", ">", true}, {"foo.*", "*.*", true}, {"foo.*", "foo.*", true}, {"foo.*", "foo.bar", false}, {"foo.>", ">", true}, {"foo.>", "*.>", true}, {"foo.>", "foo.>", true}, {"foo.>", "foo.bar", false}, {"foo..bar", "foo.*", false}, // Bad subject, we return false {"foo.*", "foo..bar", false}, // Bad subject, we return false } { t.Run("", func(t *testing.T) { if res := subjectIsSubsetMatch(test.subject, test.test); res != test.result { t.Fatalf("Subject %q subset match of %q, should be %v, got %v", test.test, test.subject, test.result, res) } }) } } func TestSublistRegisterInterestNotification(t *testing.T) { s := NewSublistWithCache() ch := make(chan bool, 1) expectErr := func(subject string) { if err := s.RegisterNotification(subject, ch); err != ErrInvalidSubject { t.Fatalf("Expected err, got %v", err) } } // Test that we require a literal subject. expectErr("foo.*") expectErr(">") // Chan needs to be non-nil if err := s.RegisterNotification("foo", nil); err != ErrNilChan { t.Fatalf("Expected err, got %v", err) } // Clearing one that is not there will return false. if s.ClearNotification("foo", ch) { t.Fatalf("Expected to return false on non-existent notification entry") } // This should work properly. if err := s.RegisterNotification("foo", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } tt := time.NewTimer(time.Second) expectBoolWithCh := func(ch chan bool, b bool) { t.Helper() tt.Reset(time.Second) defer tt.Stop() select { case v := <-ch: if v != b { t.Fatalf("Expected %v, got %v", b, v) } case <-tt.C: t.Fatalf("Timeout waiting for expected value") } } expectBool := func(b bool) { t.Helper() expectBoolWithCh(ch, b) } expectFalse := func() { t.Helper() expectBool(false) } expectTrue := func() { t.Helper() expectBool(true) } expectNone := func() { t.Helper() if lch := len(ch); lch != 0 { t.Fatalf("Expected no notifications, had %d and first was %v", lch, <-ch) } } expectOneWithCh := func(ch chan bool) { t.Helper() if len(ch) != 1 { t.Fatalf("Expected 1 notification") } } expectOne := func() { t.Helper() expectOneWithCh(ch) } expectOne() expectFalse() sub := newSub("foo") s.Insert(sub) expectTrue() sub2 := newSub("foo") s.Insert(sub2) expectNone() if err := s.RegisterNotification("bar", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectFalse() sub3 := newSub("foo") s.Insert(sub3) expectNone() // Now remove literals. s.Remove(sub) expectNone() s.Remove(sub2) expectNone() s.Remove(sub3) expectFalse() if err := s.RegisterNotification("test.node", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectOne() expectFalse() tnSub1 := newSub("test.node.already.exist") s.Insert(tnSub1) expectNone() tnSub2 := newSub("test.node") s.Insert(tnSub2) expectTrue() tnSub3 := newSub("test.node") s.Insert(tnSub3) expectNone() s.Remove(tnSub1) expectNone() s.Remove(tnSub2) expectNone() s.Remove(tnSub3) expectFalse() if !s.ClearNotification("test.node", ch) { t.Fatalf("Expected to return true") } sub4 := newSub("bar") s.Insert(sub4) expectTrue() if !s.ClearNotification("bar", ch) { t.Fatalf("Expected to return true") } s.RLock() lnr := len(s.notify.remove) s.RUnlock() if lnr != 0 { t.Fatalf("Expected zero entries for remove notify, got %d", lnr) } if !s.ClearNotification("foo", ch) { t.Fatalf("Expected to return true") } s.RLock() notifyMap := s.notify s.RUnlock() if notifyMap != nil { t.Fatalf("Expected the notify map to be nil") } // Let's do some wildcard checks. // Wildcards will not trigger interest. subpwc := newSub("*") s.Insert(subpwc) expectNone() if err := s.RegisterNotification("foo", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectFalse() s.Insert(sub) expectTrue() s.Remove(sub) expectFalse() s.Remove(subpwc) expectNone() subfwc := newSub(">") s.Insert(subfwc) expectNone() s.Insert(subpwc) expectNone() s.Remove(subpwc) expectNone() s.Remove(subfwc) expectNone() // Test batch subs := []*subscription{sub, sub2, sub3, sub4, subpwc, subfwc} for _, sub := range subs { s.Insert(sub) } expectTrue() s.RemoveBatch(subs) expectOne() expectFalse() // Test queue subs // We know make sure that if you have qualified a queue group it has to match, etc. // Also if you do not specify one they will not trigger. qsub := newQSub("foo.bar.baz", "1") s.Insert(qsub) expectNone() if err := s.RegisterNotification("foo.bar.baz", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectFalse() wcqsub := newQSub("foo.bar.>", "1") s.Insert(wcqsub) expectNone() s.Remove(qsub) expectNone() s.Remove(wcqsub) expectNone() s.Insert(wcqsub) expectNone() if err := s.RegisterQueueNotification("queue.test.node", "q22", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectOne() expectFalse() qsub1 := newQSub("queue.test.node.already.exist", "queue") s.Insert(qsub1) expectNone() qsub2 := newQSub("queue.test.node", "q22") s.Insert(qsub2) expectTrue() qsub3 := newQSub("queue.test.node", "otherqueue") s.Insert(qsub3) expectNone() qsub4 := newQSub("queue.different.node", "q22") s.Insert(qsub4) expectNone() qsub5 := newQSub("queue.test.node", "q22") s.Insert(qsub5) expectNone() s.Remove(qsub3) expectNone() s.Remove(qsub1) expectNone() s.Remove(qsub2) expectNone() s.Remove(qsub4) expectNone() s.Remove(qsub5) expectFalse() if !s.ClearQueueNotification("queue.test.node", "q22", ch) { t.Fatalf("Expected to return true") } if err := s.RegisterQueueNotification("some.subject", "queue1", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } expectOne() expectFalse() qsub1 = newQSub("some.subject", "queue1") s.Insert(qsub1) expectTrue() // Create a second channel for this other queue ch2 := make(chan bool, 1) if err := s.RegisterQueueNotification("some.subject", "queue2", ch2); err != nil { t.Fatalf("Unexpected error: %v", err) } expectOneWithCh(ch2) expectBoolWithCh(ch2, false) qsub2 = newQSub("some.subject", "queue2") s.Insert(qsub2) expectBoolWithCh(ch2, true) // But we should not get notification on queue1 expectNone() s.Remove(qsub1) expectFalse() s.Remove(qsub2) expectBoolWithCh(ch2, false) if !s.ClearQueueNotification("some.subject", "queue1", ch) { t.Fatalf("Expected to return true") } if !s.ClearQueueNotification("some.subject", "queue2", ch2) { t.Fatalf("Expected to return true") } // Test non-blocking notifications. if err := s.RegisterNotification("bar", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } if err := s.RegisterNotification("baz", ch); err != nil { t.Fatalf("Unexpected error: %v", err) } s.Insert(newSub("baz")) s.Insert(newSub("bar")) s.Insert(subpwc) expectOne() expectFalse() } func TestSublistReverseMatch(t *testing.T) { s := NewSublistWithCache() fooSub := newSub("foo") barSub := newSub("bar") fooBarSub := newSub("foo.bar") fooBazSub := newSub("foo.baz") fooBarBazSub := newSub("foo.bar.baz") s.Insert(fooSub) s.Insert(barSub) s.Insert(fooBarSub) s.Insert(fooBazSub) s.Insert(fooBarBazSub) r := s.ReverseMatch("foo") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, fooSub, t) r = s.ReverseMatch("bar") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, barSub, t) r = s.ReverseMatch("*") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, fooSub, t) verifyMember(r.psubs, barSub, t) r = s.ReverseMatch("baz") verifyLen(r.psubs, 0, t) r = s.ReverseMatch("foo.*") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, fooBarSub, t) verifyMember(r.psubs, fooBazSub, t) r = s.ReverseMatch("*.*") verifyLen(r.psubs, 2, t) verifyMember(r.psubs, fooBarSub, t) verifyMember(r.psubs, fooBazSub, t) r = s.ReverseMatch("*.bar") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, fooBarSub, t) r = s.ReverseMatch("*.baz") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, fooBazSub, t) r = s.ReverseMatch("bar.*") verifyLen(r.psubs, 0, t) r = s.ReverseMatch("*.bat") verifyLen(r.psubs, 0, t) r = s.ReverseMatch("foo.>") verifyLen(r.psubs, 3, t) verifyMember(r.psubs, fooBarSub, t) verifyMember(r.psubs, fooBazSub, t) verifyMember(r.psubs, fooBarBazSub, t) r = s.ReverseMatch(">") verifyLen(r.psubs, 5, t) verifyMember(r.psubs, fooSub, t) verifyMember(r.psubs, barSub, t) verifyMember(r.psubs, fooBarSub, t) verifyMember(r.psubs, fooBazSub, t) verifyMember(r.psubs, fooBarBazSub, t) } func TestSublistReverseMatchWider(t *testing.T) { s := NewSublistWithCache() sub := newSub("uplink.*.*.>") s.Insert(sub) r := s.ReverseMatch("uplink.1.*.*.>") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub, t) r = s.ReverseMatch("uplink.1.2.3.>") verifyLen(r.psubs, 1, t) verifyMember(r.psubs, sub, t) } func TestSublistMatchWithEmptyTokens(t *testing.T) { for _, test := range []struct { name string cache bool }{ {"cache", true}, {"no cache", false}, } { t.Run(test.name, func(t *testing.T) { sl := NewSublist(true) sub1 := newSub(">") sub2 := newQSub(">", "queue") sl.Insert(sub1) sl.Insert(sub2) for _, subj := range []string{".foo", "..foo", "foo..", "foo.", "foo..bar", "foo...bar"} { t.Run(subj, func(t *testing.T) { r := sl.Match(subj) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 0, t) }) } }) } } func TestSublistSubjectCollide(t *testing.T) { require_False(t, SubjectsCollide("foo.*", "foo.*.bar.>")) require_False(t, SubjectsCollide("foo.*.bar.>", "foo.*")) require_True(t, SubjectsCollide("foo.*", "foo.foo")) require_True(t, SubjectsCollide("foo.*", "*.foo")) require_True(t, SubjectsCollide("foo.bar.>", "*.bar.foo")) } func TestSublistAddCacheHitRate(t *testing.T) { sl1 := NewSublistWithCache() fooSub := newSub("foo") sl1.Insert(fooSub) for i := 0; i < 4; i++ { sl1.Match("foo") } stats1 := sl1.Stats() require_True(t, stats1.CacheHitRate == 0.75) sl2 := NewSublistWithCache() barSub := newSub("bar") sl2.Insert(barSub) for i := 0; i < 4; i++ { sl2.Match("bar") } stats2 := sl2.Stats() require_True(t, stats2.CacheHitRate == 0.75) ts := &SublistStats{} ts.add(stats1) ts.add(stats2) require_True(t, ts.CacheHitRate == 0.75) } // -- Benchmarks Setup -- var benchSublistSubs []*subscription var benchSublistSl = NewSublistWithCache() // https://github.com/golang/go/issues/31859 func TestMain(m *testing.M) { flag.StringVar(&testDefaultClusterCompression, "cluster_compression", _EMPTY_, "Test with this compression level as the default") flag.StringVar(&testDefaultLeafNodeCompression, "leafnode_compression", _EMPTY_, "Test with this compression level as the default") flag.Parse() initSublist := false flag.Visit(func(f *flag.Flag) { if f.Name == "test.bench" { initSublist = true } }) if initSublist { benchSublistSubs = make([]*subscription, 0, 256*1024) toks := []string{"synadia", "nats", "jetstream", "nkeys", "jwt", "deny", "auth", "drain"} subsInit("", toks) for i := 0; i < len(benchSublistSubs); i++ { benchSublistSl.Insert(benchSublistSubs[i]) } addWildcards() } os.Exit(m.Run()) } func TestSublistHasInterest(t *testing.T) { sl := NewSublistWithCache() fooSub := newSub("foo") sl.Insert(fooSub) // Expect to find that "foo" matches but "bar" doesn't. // At this point nothing should be in the cache. require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("bar")) require_Equal(t, sl.cacheHits, 0) // Now call Match(), which will populate the cache. sl.Match("foo") require_Equal(t, sl.cacheHits, 0) // Future calls to HasInterest() should hit the cache now. for i := uint64(1); i <= 5; i++ { require_True(t, sl.HasInterest("foo")) require_Equal(t, sl.cacheHits, i) } // Call Match on a subject we know there is no match. sl.Match("bar") require_False(t, sl.HasInterest("bar")) // Remove fooSub and check interest again sl.Remove(fooSub) require_False(t, sl.HasInterest("foo")) // Try with some wildcards sub := newSub("foo.*") sl.Insert(sub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) // Remove sub, there should be no interest sl.Remove(sub) require_False(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) sub = newSub("foo.>") sl.Insert(sub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) sl.Remove(sub) require_False(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) sub = newSub("*.>") sl.Insert(sub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.baz")) sl.Remove(sub) sub = newSub("*.bar") sl.Insert(sub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.baz")) sl.Remove(sub) sub = newSub("*") sl.Insert(sub) require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) sl.Remove(sub) // Try with queues now. qsub := newQSub("foo", "bar") sl.Insert(qsub) require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) qsub2 := newQSub("foo", "baz") sl.Insert(qsub2) require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) // Remove first queue sl.Remove(qsub) require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) // Remove last. sl.Remove(qsub2) require_False(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) // With wildcards now qsub = newQSub("foo.*", "bar") sl.Insert(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) // Add another queue to the group qsub2 = newQSub("foo.*", "baz") sl.Insert(qsub2) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) // Remove first queue sl.Remove(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) // Remove last sl.Remove(qsub2) require_False(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) qsub = newQSub("foo.>", "bar") sl.Insert(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) // Add another queue to the group qsub2 = newQSub("foo.>", "baz") sl.Insert(qsub2) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) // Remove first queue sl.Remove(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) // Remove last sl.Remove(qsub2) require_False(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.bar.baz")) qsub = newQSub("*.>", "bar") sl.Insert(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.baz")) sl.Remove(qsub) qsub = newQSub("*.bar", "bar") sl.Insert(qsub) require_False(t, sl.HasInterest("foo")) require_True(t, sl.HasInterest("foo.bar")) require_False(t, sl.HasInterest("foo.baz")) sl.Remove(qsub) qsub = newQSub("*", "bar") sl.Insert(qsub) require_True(t, sl.HasInterest("foo")) require_False(t, sl.HasInterest("foo.bar")) sl.Remove(qsub) } func TestSublistNumInterest(t *testing.T) { sl := NewSublistWithCache() fooSub := newSub("foo") sl.Insert(fooSub) require_NumInterest := func(t *testing.T, subj string, wnp, wnq int) { t.Helper() np, nq := sl.NumInterest(subj) require_Equal(t, np, wnp) require_Equal(t, nq, wnq) } // Expect to find that "foo" matches but "bar" doesn't. // At this point nothing should be in the cache. require_NumInterest(t, "foo", 1, 0) require_NumInterest(t, "bar", 0, 0) require_Equal(t, sl.cacheHits, 0) // Now call Match(), which will populate the cache. sl.Match("foo") require_Equal(t, sl.cacheHits, 0) // Future calls to HasInterest() should hit the cache now. for i := uint64(1); i <= 5; i++ { require_NumInterest(t, "foo", 1, 0) require_Equal(t, sl.cacheHits, i) } // Call Match on a subject we know there is no match. sl.Match("bar") require_NumInterest(t, "bar", 0, 0) // Remove fooSub and check interest again sl.Remove(fooSub) require_NumInterest(t, "foo", 0, 0) // Try with some wildcards sub := newSub("foo.*") sl.Insert(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 1, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) // Remove sub, there should be no interest sl.Remove(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) sub = newSub("foo.>") sl.Insert(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 1, 0) require_NumInterest(t, "foo.bar.baz", 1, 0) sl.Remove(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) sub = newSub("*.>") sl.Insert(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 1, 0) require_NumInterest(t, "foo.bar.baz", 1, 0) sl.Remove(sub) sub = newSub("*.bar") sl.Insert(sub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 1, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) sl.Remove(sub) sub = newSub("*") sl.Insert(sub) require_NumInterest(t, "foo", 1, 0) require_NumInterest(t, "foo.bar", 0, 0) sl.Remove(sub) // Try with queues now. qsub := newQSub("foo", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 1) require_NumInterest(t, "foo.bar", 0, 0) qsub2 := newQSub("foo", "baz") sl.Insert(qsub2) require_NumInterest(t, "foo", 0, 2) require_NumInterest(t, "foo.bar", 0, 0) // Add a second qsub to the second queue group qsub3 := newQSub("foo", "baz") sl.Insert(qsub3) require_NumInterest(t, "foo", 0, 3) require_NumInterest(t, "foo.bar", 0, 0) // Remove first queue sl.Remove(qsub) require_NumInterest(t, "foo", 0, 2) require_NumInterest(t, "foo.bar", 0, 0) // Remove second sl.Remove(qsub2) require_NumInterest(t, "foo", 0, 1) require_NumInterest(t, "foo.bar", 0, 0) // Remove last. sl.Remove(qsub3) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 0) // With wildcards now qsub = newQSub("foo.*", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 0) // Add another queue to the group qsub2 = newQSub("foo.*", "baz") sl.Insert(qsub2) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 2) require_NumInterest(t, "foo.bar.baz", 0, 0) qsub3 = newQSub("foo.*", "baz") sl.Insert(qsub3) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 3) require_NumInterest(t, "foo.bar.baz", 0, 0) // Remove first queue sl.Remove(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 2) require_NumInterest(t, "foo.bar.baz", 0, 0) // Remove second sl.Remove(qsub2) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 0) // Remove last sl.Remove(qsub3) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) // With > wildcard qsub = newQSub("foo.>", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 1) // Add another queue to the group qsub2 = newQSub("foo.>", "baz") sl.Insert(qsub2) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 2) require_NumInterest(t, "foo.bar.baz", 0, 2) // Add another queue to second group. qsub3 = newQSub("foo.>", "baz") sl.Insert(qsub3) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 3) require_NumInterest(t, "foo.bar.baz", 0, 3) // Remove first queue sl.Remove(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 2) require_NumInterest(t, "foo.bar.baz", 0, 2) // Remove second sl.Remove(qsub2) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 1) // Remove last sl.Remove(qsub3) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 0) require_NumInterest(t, "foo.bar.baz", 0, 0) qsub = newQSub("*.>", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 1) sl.Remove(qsub) qsub = newQSub("*.bar", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 0) require_NumInterest(t, "foo.bar", 0, 1) require_NumInterest(t, "foo.bar.baz", 0, 0) sl.Remove(qsub) qsub = newQSub("*", "bar") sl.Insert(qsub) require_NumInterest(t, "foo", 0, 1) require_NumInterest(t, "foo.bar", 0, 0) sl.Remove(qsub) } func TestSublistInterestBasedIntersection(t *testing.T) { st := stree.NewSubjectTree[struct{}]() st.Insert([]byte("one.two.three.four"), struct{}{}) st.Insert([]byte("one.two.three.five"), struct{}{}) st.Insert([]byte("one.two.six"), struct{}{}) st.Insert([]byte("one.two.seven"), struct{}{}) st.Insert([]byte("eight.nine"), struct{}{}) require_NoDuplicates := func(t *testing.T, got map[string]int) { for _, c := range got { require_Equal(t, c, 1) } } t.Run("Literals", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one.two.six")) sl.Insert(newSub("eight.nine")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWC", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one.two.*.*")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWCOverlapping", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one.two.*.four")) sl.Insert(newSub("one.two.*.*")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 2) require_NoDuplicates(t, got) }) t.Run("PWCAll", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("*.*")) sl.Insert(newSub("*.*.*")) sl.Insert(newSub("*.*.*.*")) require_True(t, sl.HasInterest("foo.bar")) require_True(t, sl.HasInterest("foo.bar.baz")) require_True(t, sl.HasInterest("foo.bar.baz.qux")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 5) require_NoDuplicates(t, got) }) t.Run("FWC", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one.>")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 4) require_NoDuplicates(t, got) }) t.Run("FWCOverlapping", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one.two.three.four")) sl.Insert(newSub("one.>")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 4) require_NoDuplicates(t, got) }) t.Run("FWCAll", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub(">")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 5) require_NoDuplicates(t, got) }) t.Run("NoMatch", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 0) }) t.Run("NoMatches", func(t *testing.T) { got := map[string]int{} sl := NewSublistNoCache() sl.Insert(newSub("one")) sl.Insert(newSub("eight")) sl.Insert(newSub("ten")) IntersectStree(st, sl, func(subj []byte, entry *struct{}) { got[string(subj)]++ }) require_Len(t, len(got), 0) }) } func subsInit(pre string, toks []string) { var sub string for _, t := range toks { if len(pre) > 0 { sub = pre + tsep + t } else { sub = t } benchSublistSubs = append(benchSublistSubs, newSub(sub)) if len(strings.Split(sub, tsep)) < 5 { subsInit(sub, toks) } } } func addWildcards() { benchSublistSl.Insert(newSub("cloud.>")) benchSublistSl.Insert(newSub("cloud.nats.component.>")) benchSublistSl.Insert(newSub("cloud.*.*.nkeys.*")) } // -- Benchmarks Setup End -- func Benchmark______________________SublistInsert(b *testing.B) { s := NewSublistWithCache() for i, l := 0, len(benchSublistSubs); i < b.N; i++ { index := i % l s.Insert(benchSublistSubs[index]) } } func Benchmark_______________SublistInsertNoCache(b *testing.B) { s := NewSublistNoCache() for i, l := 0, len(benchSublistSubs); i < b.N; i++ { index := i % l s.Insert(benchSublistSubs[index]) } } func benchSublistTokens(b *testing.B, tokens string) { for i := 0; i < b.N; i++ { benchSublistSl.Match(tokens) } } func Benchmark____________SublistMatchSingleToken(b *testing.B) { benchSublistTokens(b, "synadia") } func Benchmark______________SublistMatchTwoTokens(b *testing.B) { benchSublistTokens(b, "synadia.nats") } func Benchmark____________SublistMatchThreeTokens(b *testing.B) { benchSublistTokens(b, "synadia.nats.jetstream") } func Benchmark_____________SublistMatchFourTokens(b *testing.B) { benchSublistTokens(b, "synadia.nats.jetstream.nkeys") } func Benchmark_SublistMatchFourTokensSingleResult(b *testing.B) { benchSublistTokens(b, "synadia.nats.jetstream.nkeys") } func Benchmark_SublistMatchFourTokensMultiResults(b *testing.B) { benchSublistTokens(b, "cloud.nats.component.router") } func Benchmark_______SublistMissOnLastTokenOfFive(b *testing.B) { benchSublistTokens(b, "synadia.nats.jetstream.nkeys.ZZZZ") } func multiRead(b *testing.B, num int) { var swg, fwg sync.WaitGroup swg.Add(num) fwg.Add(num) s := "synadia.nats.jetstream.nkeys" for i := 0; i < num; i++ { go func() { swg.Done() swg.Wait() n := b.N / num for i := 0; i < n; i++ { benchSublistSl.Match(s) } fwg.Done() }() } swg.Wait() b.ResetTimer() fwg.Wait() } func Benchmark____________Sublist10XMultipleReads(b *testing.B) { multiRead(b, 10) } func Benchmark___________Sublist100XMultipleReads(b *testing.B) { multiRead(b, 100) } func Benchmark__________Sublist1000XMultipleReads(b *testing.B) { multiRead(b, 1000) } func Benchmark________________SublistMatchLiteral(b *testing.B) { cachedSubj := "foo.foo.foo.foo.foo.foo.foo.foo.foo.foo" subjects := []string{ "foo.foo.foo.foo.foo.foo.foo.foo.foo.foo", "foo.foo.foo.foo.foo.foo.foo.foo.foo.>", "foo.foo.foo.foo.foo.foo.foo.foo.>", "foo.foo.foo.foo.foo.foo.foo.>", "foo.foo.foo.foo.foo.foo.>", "foo.foo.foo.foo.foo.>", "foo.foo.foo.foo.>", "foo.foo.foo.>", "foo.foo.>", "foo.>", ">", "foo.foo.foo.foo.foo.foo.foo.foo.foo.*", "foo.foo.foo.foo.foo.foo.foo.foo.*.*", "foo.foo.foo.foo.foo.foo.foo.*.*.*", "foo.foo.foo.foo.foo.foo.*.*.*.*", "foo.foo.foo.foo.foo.*.*.*.*.*", "foo.foo.foo.foo.*.*.*.*.*.*", "foo.foo.foo.*.*.*.*.*.*.*", "foo.foo.*.*.*.*.*.*.*.*", "foo.*.*.*.*.*.*.*.*.*", "*.*.*.*.*.*.*.*.*.*", } b.ResetTimer() for i := 0; i < b.N; i++ { for _, subject := range subjects { if !matchLiteral(cachedSubj, subject) { b.Fatalf("Subject %q no match with %q", cachedSubj, subject) } } } } func Benchmark_____SublistMatch10kSubsWithNoCache(b *testing.B) { var nsubs = 512 s := NewSublistNoCache() subject := "foo" for i := 0; i < nsubs; i++ { s.Insert(newSub(subject)) } b.ResetTimer() for i := 0; i < b.N; i++ { r := s.Match(subject) if len(r.psubs) != nsubs { b.Fatalf("Results len is %d, should be %d", len(r.psubs), nsubs) } } } func removeTest(b *testing.B, singleSubject, doBatch bool, qgroup string) { s := NewSublistWithCache() subject := "foo" subs := make([]*subscription, 0, b.N) for i := 0; i < b.N; i++ { var sub *subscription if singleSubject { sub = newQSub(subject, qgroup) } else { sub = newQSub(fmt.Sprintf("%s.%d\n", subject, i), qgroup) } s.Insert(sub) subs = append(subs, sub) } // Actual test on Remove b.ResetTimer() if doBatch { s.RemoveBatch(subs) } else { for _, sub := range subs { s.Remove(sub) } } } func Benchmark__________SublistRemove1TokenSingle(b *testing.B) { removeTest(b, true, false, "") } func Benchmark___________SublistRemove1TokenBatch(b *testing.B) { removeTest(b, true, true, "") } func Benchmark_________SublistRemove2TokensSingle(b *testing.B) { removeTest(b, false, false, "") } func Benchmark__________SublistRemove2TokensBatch(b *testing.B) { removeTest(b, false, true, "") } func Benchmark________SublistRemove1TokenQGSingle(b *testing.B) { removeTest(b, true, false, "bar") } func Benchmark_________SublistRemove1TokenQGBatch(b *testing.B) { removeTest(b, true, true, "bar") } func removeMultiTest(b *testing.B, singleSubject, doBatch bool) { s := NewSublistWithCache() subject := "foo" var swg, fwg sync.WaitGroup swg.Add(b.N) fwg.Add(b.N) // We will have b.N go routines each with 1k subscriptions. sc := 1000 for i := 0; i < b.N; i++ { go func() { subs := make([]*subscription, 0, sc) for n := 0; n < sc; n++ { var sub *subscription if singleSubject { sub = newSub(subject) } else { sub = newSub(fmt.Sprintf("%s.%d\n", subject, n)) } s.Insert(sub) subs = append(subs, sub) } // Wait to start test swg.Done() swg.Wait() // Actual test on Remove if doBatch { s.RemoveBatch(subs) } else { for _, sub := range subs { s.Remove(sub) } } fwg.Done() }() } swg.Wait() b.ResetTimer() fwg.Wait() } // Check contention rates for remove from multiple Go routines. // Reason for BatchRemove. func Benchmark_________SublistRemove1kSingleMulti(b *testing.B) { removeMultiTest(b, true, false) } // Batch version func Benchmark__________SublistRemove1kBatchMulti(b *testing.B) { removeMultiTest(b, true, true) } func Benchmark__SublistRemove1kSingle2TokensMulti(b *testing.B) { removeMultiTest(b, false, false) } // Batch version func Benchmark___SublistRemove1kBatch2TokensMulti(b *testing.B) { removeMultiTest(b, false, true) } // Cache contention tests func cacheContentionTest(b *testing.B, numMatchers, numAdders, numRemovers int) { var swg, fwg, mwg sync.WaitGroup total := numMatchers + numAdders + numRemovers swg.Add(total) fwg.Add(total) mwg.Add(numMatchers) mu := sync.RWMutex{} subs := make([]*subscription, 0, 8192) quitCh := make(chan struct{}) // Set up a new sublist. subjects will be foo.bar.baz.N s := NewSublistWithCache() mu.Lock() for i := 0; i < 10000; i++ { sub := newSub(fmt.Sprintf("foo.bar.baz.%d", i)) s.Insert(sub) subs = append(subs, sub) } mu.Unlock() // Now warm up the cache for i := 0; i < slCacheMax; i++ { s.Match(fmt.Sprintf("foo.bar.baz.%d", i)) } // Setup go routines. // Adders for i := 0; i < numAdders; i++ { go func() { swg.Done() swg.Wait() for { select { case <-quitCh: fwg.Done() return default: mu.Lock() next := len(subs) subj := "foo.bar.baz." + strconv.FormatInt(int64(next), 10) sub := newSub(subj) subs = append(subs, sub) mu.Unlock() s.Insert(sub) } } }() } // Removers for i := 0; i < numRemovers; i++ { go func() { prand := rand.New(rand.NewSource(time.Now().UnixNano())) swg.Done() swg.Wait() for { select { case <-quitCh: fwg.Done() return default: mu.RLock() lh := len(subs) - 1 index := prand.Intn(lh) sub := subs[index] mu.RUnlock() s.Remove(sub) } } }() } // Matchers for i := 0; i < numMatchers; i++ { go func() { id := nuid.New() swg.Done() swg.Wait() // We will miss on purpose to blow the cache. n := b.N / numMatchers for i := 0; i < n; i++ { subj := "foo.bar.baz." + id.Next() s.Match(subj) } mwg.Done() fwg.Done() }() } swg.Wait() b.ResetTimer() mwg.Wait() b.StopTimer() close(quitCh) fwg.Wait() } func Benchmark____SublistCacheContention10M10A10R(b *testing.B) { cacheContentionTest(b, 10, 10, 10) } func Benchmark_SublistCacheContention100M100A100R(b *testing.B) { cacheContentionTest(b, 100, 100, 100) } func Benchmark____SublistCacheContention1kM1kA1kR(b *testing.B) { cacheContentionTest(b, 1024, 1024, 1024) } func Benchmark_SublistCacheContention10kM10kA10kR(b *testing.B) { cacheContentionTest(b, 10*1024, 10*1024, 10*1024) } func Benchmark______________IsValidLiteralSubject(b *testing.B) { for i := 0; i < b.N; i++ { IsValidLiteralSubject("foo.bar.baz.22") } } func Benchmark___________________subjectIsLiteral(b *testing.B) { for i := 0; i < b.N; i++ { subjectIsLiteral("foo.bar.baz.22") } } nats-server-2.10.27/server/sysmem/000077500000000000000000000000001477524627100167765ustar00rootroot00000000000000nats-server-2.10.27/server/sysmem/mem_bsd.go000066400000000000000000000013731477524627100207370ustar00rootroot00000000000000// Copyright 2019-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build freebsd || openbsd || dragonfly || netbsd // +build freebsd openbsd dragonfly netbsd package sysmem func Memory() int64 { return sysctlInt64("hw.physmem") } nats-server-2.10.27/server/sysmem/mem_darwin.go000066400000000000000000000012761477524627100214550ustar00rootroot00000000000000// Copyright 2019-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build darwin // +build darwin package sysmem func Memory() int64 { return sysctlInt64("hw.memsize") } nats-server-2.10.27/server/sysmem/mem_linux.go000066400000000000000000000014661477524627100213310ustar00rootroot00000000000000// Copyright 2019-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux // +build linux package sysmem import "syscall" func Memory() int64 { var info syscall.Sysinfo_t err := syscall.Sysinfo(&info) if err != nil { return 0 } return int64(info.Totalram) * int64(info.Unit) } nats-server-2.10.27/server/sysmem/mem_wasm.go000066400000000000000000000013071477524627100211330ustar00rootroot00000000000000// Copyright 2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build wasm // +build wasm package sysmem func Memory() int64 { // TODO: We don't know the system memory return 0 } nats-server-2.10.27/server/sysmem/mem_windows.go000066400000000000000000000026101477524627100216540ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows // +build windows package sysmem import ( "unsafe" "golang.org/x/sys/windows" ) var winKernel32 = windows.NewLazySystemDLL("kernel32.dll") var winGlobalMemoryStatusEx = winKernel32.NewProc("GlobalMemoryStatusEx") func init() { if err := winKernel32.Load(); err != nil { panic(err) } if err := winGlobalMemoryStatusEx.Find(); err != nil { panic(err) } } // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex type _memoryStatusEx struct { dwLength uint32 dwMemoryLoad uint32 ullTotalPhys uint64 unused [6]uint64 // ignore rest of struct } func Memory() int64 { msx := &_memoryStatusEx{dwLength: 64} res, _, _ := winGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) if res == 0 { return 0 } return int64(msx.ullTotalPhys) } nats-server-2.10.27/server/sysmem/mem_zos.go000066400000000000000000000013121477524627100207730ustar00rootroot00000000000000// Copyright 2022-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build zos // +build zos package sysmem func Memory() int64 { // TODO: We don't know the system memory return 0 } nats-server-2.10.27/server/sysmem/sysctl.go000066400000000000000000000021131477524627100206430ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build darwin || freebsd || openbsd || dragonfly || netbsd // +build darwin freebsd openbsd dragonfly netbsd package sysmem import ( "syscall" "unsafe" ) func sysctlInt64(name string) int64 { s, err := syscall.Sysctl(name) if err != nil { return 0 } // Make sure it's 8 bytes when we do the cast below. // We were getting fatal error: checkptr: converted pointer straddles multiple allocations in go 1.22.1 on darwin. var b [8]byte copy(b[:], s) return *(*int64)(unsafe.Pointer(&b[0])) } nats-server-2.10.27/server/test_test.go000066400000000000000000000226531477524627100200360ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "cmp" "fmt" "math/rand" "net/url" "os" "reflect" "strings" "testing" "time" ) // DefaultTestOptions are default options for the unit tests. var DefaultTestOptions = Options{ Host: "127.0.0.1", Port: 4222, NoLog: true, NoSigs: true, MaxControlLine: 4096, DisableShortFirstPing: true, } func testDefaultClusterOptionsForLeafNodes() *Options { o := DefaultTestOptions o.Port = -1 o.Cluster.Host = o.Host o.Cluster.Port = -1 o.Gateway.Host = o.Host o.Gateway.Port = -1 o.LeafNode.Host = o.Host o.LeafNode.Port = -1 return &o } func RunRandClientPortServer(t *testing.T) *Server { opts := DefaultTestOptions opts.Port = -1 opts.StoreDir = t.TempDir() return RunServer(&opts) } func require_True(t testing.TB, b bool) { t.Helper() if !b { t.Fatalf("require true, but got false") } } func require_False(t testing.TB, b bool) { t.Helper() if b { t.Fatalf("require false, but got true") } } func require_NoError(t testing.TB, err error) { t.Helper() if err != nil { t.Fatalf("require no error, but got: %v", err) } } func require_NotNil[T any](t testing.TB, v T) { t.Helper() r := reflect.ValueOf(v) switch k := r.Kind(); k { case reflect.Ptr, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func: if r.IsNil() { t.Fatalf("require not nil, but got nil") } } } func require_Contains(t *testing.T, s string, subStrs ...string) { t.Helper() for _, subStr := range subStrs { if !strings.Contains(s, subStr) { t.Fatalf("require %q to be contained in %q", subStr, s) } } } func require_Error(t testing.TB, err error, expected ...error) { t.Helper() if err == nil { t.Fatalf("require error, but got none") } if len(expected) == 0 { return } // Try to strip nats prefix from Go library if present. const natsErrPre = "nats: " eStr := err.Error() if strings.HasPrefix(eStr, natsErrPre) { eStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1) } for _, e := range expected { if err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) { return } } t.Fatalf("Expected one of %v, got '%v'", expected, err) } func require_Equal[T comparable](t testing.TB, a, b T) { t.Helper() if a != b { t.Fatalf("require %T equal, but got: %v != %v", a, a, b) } } func require_NotEqual[T comparable](t testing.TB, a, b T) { t.Helper() if a == b { t.Fatalf("require %T not equal, but got: %v == %v", a, a, b) } } func require_Len(t testing.TB, a, b int) { t.Helper() if a != b { t.Fatalf("require len, but got: %v != %v", a, b) } } func require_LessThan[T cmp.Ordered](t *testing.T, a, b T) { t.Helper() if a >= b { t.Fatalf("require %v to be less than %v", a, b) } } func require_ChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) T { t.Helper() select { case v := <-ch: return v case <-time.After(timeout): t.Fatalf("require read from channel within %v but didn't get anything", timeout) } panic("this shouldn't be possible") } func require_NoChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) { t.Helper() select { case <-ch: t.Fatalf("require no read from channel within %v but got something", timeout) case <-time.After(timeout): } } func checkNatsError(t *testing.T, e *ApiError, id ErrorIdentifier) { t.Helper() ae, ok := ApiErrors[id] if !ok { t.Fatalf("Unknown error ID identifier: %d", id) } if e.ErrCode != ae.ErrCode { t.Fatalf("Did not get NATS Error %d: %+v", e.ErrCode, e) } } // Creates a full cluster with numServers and given name and makes sure its well formed. // Will have Gateways and Leaf Node connections active. func createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster { t.Helper() return createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...) } // Creates a cluster and optionally additional accounts and users. // Will have Gateways and Leaf Node connections active. func createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster { t.Helper() if clusterName == "" || numServers < 1 { t.Fatalf("Bad params") } // Setup some accounts and users. // $SYS is always the system account. And we have default FOO and BAR accounts, as well // as DLC and NGS which do a service import. createAccountsAndUsers := func() ([]*Account, []*User) { if !doAccounts { return []*Account{NewAccount("$SYS")}, nil } sys := NewAccount("$SYS") ngs := NewAccount("NGS") dlc := NewAccount("DLC") foo := NewAccount("FOO") bar := NewAccount("BAR") accounts := []*Account{sys, foo, bar, ngs, dlc} ngs.AddServiceExport("ngs.usage.*", nil) dlc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.dlc") // Setup users users := []*User{ {Username: "dlc", Password: "pass", Permissions: nil, Account: dlc}, {Username: "ngs", Password: "pass", Permissions: nil, Account: ngs}, {Username: "foo", Password: "pass", Permissions: nil, Account: foo}, {Username: "bar", Password: "pass", Permissions: nil, Account: bar}, {Username: "sys", Password: "pass", Permissions: nil, Account: sys}, } return accounts, users } bindGlobal := func(s *Server) { ngs, err := s.LookupAccount("NGS") if err != nil { return } // Bind global to service import gacc, _ := s.LookupAccount("$G") gacc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.$G") } // If we are going to connect to another cluster set that up now for options. var gws []*RemoteGatewayOpts for _, c := range connectTo { // Gateways autodiscover here too, so just need one address from the set. gwAddr := fmt.Sprintf("nats-gw://%s:%d", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port) gwurl, _ := url.Parse(gwAddr) gws = append(gws, &RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}}) } // Make the GWs form faster for the tests. SetGatewaysSolicitDelay(gwSolicit) defer ResetGatewaysSolicitDelay() // Create seed first. o := testDefaultClusterOptionsForLeafNodes() o.Gateway.Name = clusterName o.Gateway.Gateways = gws // All of these need system accounts. o.Accounts, o.Users = createAccountsAndUsers() o.SystemAccount = "$SYS" o.ServerName = fmt.Sprintf("%s1", clusterName) // Run the server s := RunServer(o) bindGlobal(s) c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} c.servers = append(c.servers, s) c.opts = append(c.opts, o) // For connecting to seed server above. routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) rurl, _ := url.Parse(routeAddr) routes := []*url.URL{rurl} for i := 1; i < numServers; i++ { o := testDefaultClusterOptionsForLeafNodes() o.Gateway.Name = clusterName o.Gateway.Gateways = gws o.Routes = routes // All of these need system accounts. o.Accounts, o.Users = createAccountsAndUsers() o.SystemAccount = "$SYS" o.ServerName = fmt.Sprintf("%s%d", clusterName, i+1) s := RunServer(o) bindGlobal(s) c.servers = append(c.servers, s) c.opts = append(c.opts, o) } checkClusterFormed(t, c.servers...) if waitOnGWs { // Wait on gateway connections if we were asked to connect to other gateways. if numGWs := len(connectTo); numGWs > 0 { for _, s := range c.servers { waitForOutboundGateways(t, s, numGWs, 2*time.Second) } } } c.t = t return c } func (c *cluster) shutdown() { if c == nil { return } // Stop any proxies. for _, np := range c.nproxies { np.stop() } // Shutdown and cleanup servers. for i, s := range c.servers { sd := s.StoreDir() s.Shutdown() s.WaitForShutdown() if cf := c.opts[i].ConfigFile; cf != _EMPTY_ { os.Remove(cf) } if sd != _EMPTY_ { sd = strings.TrimSuffix(sd, JetStreamStoreDir) os.RemoveAll(sd) } } } func shutdownCluster(c *cluster) { c.shutdown() } func (c *cluster) randomServer() *Server { return c.randomServerFromCluster(c.name) } func (c *cluster) randomServerFromCluster(cname string) *Server { // Since these can be randomly shutdown in certain tests make sure they are running first. // Copy our servers list and shuffle then walk looking for first running server. cs := append(c.servers[:0:0], c.servers...) rand.Shuffle(len(cs), func(i, j int) { cs[i], cs[j] = cs[j], cs[i] }) for _, s := range cs { if s.Running() && s.ClusterName() == cname { return s } } return nil } func runSolicitLeafServer(lso *Options) (*Server, *Options) { return runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) } func runSolicitLeafServerToURL(surl string) (*Server, *Options) { o := DefaultTestOptions o.Port = -1 o.NoSystemAccount = true rurl, _ := url.Parse(surl) o.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{rurl}}} o.LeafNode.ReconnectInterval = 100 * time.Millisecond return RunServer(&o), &o } nats-server-2.10.27/server/trust_test.go000066400000000000000000000070101477524627100202260ustar00rootroot00000000000000// Copyright 2018-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "fmt" "strings" "testing" ) const ( t1 = "OBYEOZQ46VZMFMNETBAW2H6VGDSOBLP67VUEZJ5LPR3PIZBWWRIY4UI4" t2 = "OAHC7NGAHG3YVPTD6QOUFZGPM2OMU6EOS67O2VHBUOA6BJLPTWFHGLKU" ) func TestStampedTrustedKeys(t *testing.T) { opts := DefaultOptions() defer func() { trustedKeys = "" }() // Set this to a bad key. We require valid operator public keys. trustedKeys = "bad" if s := New(opts); s != nil { s.Shutdown() t.Fatalf("Expected a bad trustedKeys to return nil server") } trustedKeys = t1 s := New(opts) if s == nil { t.Fatalf("Expected non-nil server") } if len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 { t.Fatalf("Trusted Nkeys not setup properly") } trustedKeys = strings.Join([]string{t1, t2}, " ") if s = New(opts); s == nil { t.Fatalf("Expected non-nil server") } if len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 { t.Fatalf("Trusted Nkeys not setup properly") } opts.TrustedKeys = []string{"OVERRIDE ME"} if s = New(opts); s != nil { t.Fatalf("Expected opts.TrustedKeys to return nil server") } } func TestTrustedKeysOptions(t *testing.T) { trustedKeys = "" opts := DefaultOptions() opts.TrustedKeys = []string{"bad"} if s := New(opts); s != nil { s.Shutdown() t.Fatalf("Expected a bad opts.TrustedKeys to return nil server") } opts.TrustedKeys = []string{t1} s := New(opts) if s == nil { t.Fatalf("Expected non-nil server") } if len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 { t.Fatalf("Trusted Nkeys not setup properly via options") } opts.TrustedKeys = []string{t1, t2} if s = New(opts); s == nil { t.Fatalf("Expected non-nil server") } if len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 { t.Fatalf("Trusted Nkeys not setup properly via options") } } func TestTrustConfigOption(t *testing.T) { confFileName := createConfFile(t, []byte(fmt.Sprintf("trusted = %q", t1))) opts, err := ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Error parsing config: %v", err) } if l := len(opts.TrustedKeys); l != 1 { t.Fatalf("Expected 1 trusted key, got %d", l) } if opts.TrustedKeys[0] != t1 { t.Fatalf("Expected trusted key to be %q, got %q", t1, opts.TrustedKeys[0]) } confFileName = createConfFile(t, []byte(fmt.Sprintf("trusted = [%q, %q]", t1, t2))) opts, err = ProcessConfigFile(confFileName) if err != nil { t.Fatalf("Error parsing config: %v", err) } if l := len(opts.TrustedKeys); l != 2 { t.Fatalf("Expected 2 trusted key, got %d", l) } if opts.TrustedKeys[0] != t1 { t.Fatalf("Expected trusted key to be %q, got %q", t1, opts.TrustedKeys[0]) } if opts.TrustedKeys[1] != t2 { t.Fatalf("Expected trusted key to be %q, got %q", t2, opts.TrustedKeys[1]) } // Now do a bad one. confFileName = createConfFile(t, []byte(fmt.Sprintf("trusted = [%q, %q]", t1, "bad"))) _, err = ProcessConfigFile(confFileName) if err == nil { t.Fatalf("Expected an error parsing trust keys with a bad key") } } nats-server-2.10.27/server/util.go000066400000000000000000000220521477524627100167660ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" "context" "encoding/json" "errors" "fmt" "math" "net" "net/url" "reflect" "regexp" "strconv" "strings" "time" ) // This map is used to store URLs string as the key with a reference count as // the value. This is used to handle gossiped URLs such as connect_urls, etc.. type refCountedUrlSet map[string]int // Ascii numbers 0-9 const ( asciiZero = 48 asciiNine = 57 ) var semVerRe = regexp.MustCompile(`\Av?([0-9]+)\.?([0-9]+)?\.?([0-9]+)?`) func versionComponents(version string) (major, minor, patch int, err error) { m := semVerRe.FindStringSubmatch(version) if m == nil { return 0, 0, 0, errors.New("invalid semver") } major, err = strconv.Atoi(m[1]) if err != nil { return -1, -1, -1, err } minor, err = strconv.Atoi(m[2]) if err != nil { return -1, -1, -1, err } patch, err = strconv.Atoi(m[3]) if err != nil { return -1, -1, -1, err } return major, minor, patch, err } func versionAtLeastCheckError(version string, emajor, eminor, epatch int) (bool, error) { major, minor, patch, err := versionComponents(version) if err != nil { return false, err } if major > emajor || (major == emajor && minor > eminor) || (major == emajor && minor == eminor && patch >= epatch) { return true, nil } return false, err } func versionAtLeast(version string, emajor, eminor, epatch int) bool { res, _ := versionAtLeastCheckError(version, emajor, eminor, epatch) return res } // parseSize expects decimal positive numbers. We // return -1 to signal error. func parseSize(d []byte) (n int) { const maxParseSizeLen = 9 //999M l := len(d) if l == 0 || l > maxParseSizeLen { return -1 } var ( i int dec byte ) // Note: Use `goto` here to avoid for loop in order // to have the function be inlined. // See: https://github.com/golang/go/issues/14768 loop: dec = d[i] if dec < asciiZero || dec > asciiNine { return -1 } n = n*10 + (int(dec) - asciiZero) i++ if i < l { goto loop } return n } // parseInt64 expects decimal positive numbers. We // return -1 to signal error func parseInt64(d []byte) (n int64) { if len(d) == 0 { return -1 } for _, dec := range d { if dec < asciiZero || dec > asciiNine { return -1 } n = n*10 + (int64(dec) - asciiZero) } return n } // Helper to move from float seconds to time.Duration func secondsToDuration(seconds float64) time.Duration { ttl := seconds * float64(time.Second) return time.Duration(ttl) } // Parse a host/port string with a default port to use // if none (or 0 or -1) is specified in `hostPort` string. func parseHostPort(hostPort string, defaultPort int) (host string, port int, err error) { if hostPort != "" { host, sPort, err := net.SplitHostPort(hostPort) if ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, "missing port") { // try appending the current port host, sPort, err = net.SplitHostPort(fmt.Sprintf("%s:%d", hostPort, defaultPort)) } if err != nil { return "", -1, err } port, err = strconv.Atoi(strings.TrimSpace(sPort)) if err != nil { return "", -1, err } if port == 0 || port == -1 { port = defaultPort } return strings.TrimSpace(host), port, nil } return "", -1, errors.New("no hostport specified") } // Returns true if URL u1 represents the same URL than u2, // false otherwise. func urlsAreEqual(u1, u2 *url.URL) bool { return reflect.DeepEqual(u1, u2) } // comma produces a string form of the given number in base 10 with // commas after every three orders of magnitude. // // e.g. comma(834142) -> 834,142 // // This function was copied from the github.com/dustin/go-humanize // package and is Copyright Dustin Sallings func comma(v int64) string { sign := "" // Min int64 can't be negated to a usable value, so it has to be special cased. if v == math.MinInt64 { return "-9,223,372,036,854,775,808" } if v < 0 { sign = "-" v = 0 - v } parts := []string{"", "", "", "", "", "", ""} j := len(parts) - 1 for v > 999 { parts[j] = strconv.FormatInt(v%1000, 10) switch len(parts[j]) { case 2: parts[j] = "0" + parts[j] case 1: parts[j] = "00" + parts[j] } v = v / 1000 j-- } parts[j] = strconv.Itoa(int(v)) return sign + strings.Join(parts[j:], ",") } // Adds urlStr to the given map. If the string was already present, simply // bumps the reference count. // Returns true only if it was added for the first time. func (m refCountedUrlSet) addUrl(urlStr string) bool { m[urlStr]++ return m[urlStr] == 1 } // Removes urlStr from the given map. If the string is not present, nothing // is done and false is returned. // If the string was present, its reference count is decreased. Returns true // if this was the last reference, false otherwise. func (m refCountedUrlSet) removeUrl(urlStr string) bool { removed := false if ref, ok := m[urlStr]; ok { if ref == 1 { removed = true delete(m, urlStr) } else { m[urlStr]-- } } return removed } // Returns the unique URLs in this map as a slice func (m refCountedUrlSet) getAsStringSlice() []string { a := make([]string, 0, len(m)) for u := range m { a = append(a, u) } return a } // natsListenConfig provides a common configuration to match the one used by // net.Listen() but with our own defaults. // Go 1.13 introduced default-on TCP keepalives with aggressive timings and // there's no sane portable way in Go with stdlib to split the initial timer // from the retry timer. Linux/BSD defaults are 2hrs/75s and Go sets both // to 15s; the issue re making them indepedently tunable has been open since // 2014 and this code here is being written in 2020. // The NATS protocol has its own L7 PING/PONG keepalive system and the Go // defaults are inappropriate for IoT deployment scenarios. // Replace any NATS-protocol calls to net.Listen(...) with // natsListenConfig.Listen(ctx,...) or use natsListen(); leave calls for HTTP // monitoring, etc, on the default. var natsListenConfig = &net.ListenConfig{ KeepAlive: -1, } // natsListen() is the same as net.Listen() except that TCP keepalives are // disabled (to match Go's behavior before Go 1.13). func natsListen(network, address string) (net.Listener, error) { return natsListenConfig.Listen(context.Background(), network, address) } // natsDialTimeout is the same as net.DialTimeout() except the TCP keepalives // are disabled (to match Go's behavior before Go 1.13). func natsDialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { d := net.Dialer{ Timeout: timeout, KeepAlive: -1, } return d.Dial(network, address) } // redactURLList() returns a copy of a list of URL pointers where each item // in the list will either be the same pointer if the URL does not contain a // password, or to a new object if there is a password. // The intended use-case is for logging lists of URLs safely. func redactURLList(unredacted []*url.URL) []*url.URL { r := make([]*url.URL, len(unredacted)) // In the common case of no passwords, if we don't let the new object leave // this function then GC should be easier. needCopy := false for i := range unredacted { if unredacted[i] == nil { r[i] = nil continue } if _, has := unredacted[i].User.Password(); !has { r[i] = unredacted[i] continue } needCopy = true ru := *unredacted[i] ru.User = url.UserPassword(ru.User.Username(), "xxxxx") r[i] = &ru } if needCopy { return r } return unredacted } // redactURLString() attempts to redact a URL string. func redactURLString(raw string) string { if !strings.ContainsRune(raw, '@') { return raw } u, err := url.Parse(raw) if err != nil { return raw } return u.Redacted() } // getURLsAsString returns a slice of u.Host from the given slice of url.URL's func getURLsAsString(urls []*url.URL) []string { a := make([]string, 0, len(urls)) for _, u := range urls { a = append(a, u.Host) } return a } // copyBytes make a new slice of the same size than `src` and copy its content. // If `src` is nil or its length is 0, then this returns `nil` func copyBytes(src []byte) []byte { if len(src) == 0 { return nil } dst := make([]byte, len(src)) copy(dst, src) return dst } // copyStrings make a new slice of the same size than `src` and copy its content. // If `src` is nil, then this returns `nil` func copyStrings(src []string) []string { if src == nil { return nil } dst := make([]string, len(src)) copy(dst, src) return dst } // Returns a byte slice for the INFO protocol. func generateInfoJSON(info *Info) []byte { b, _ := json.Marshal(info) pcs := [][]byte{[]byte("INFO"), b, []byte(CR_LF)} return bytes.Join(pcs, []byte(" ")) } nats-server-2.10.27/server/util_test.go000066400000000000000000000205301477524627100200240ustar00rootroot00000000000000// Copyright 2012-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "math" "math/rand" "net/url" "reflect" "strconv" "sync" "testing" "time" ) func TestParseSize(t *testing.T) { if parseSize(nil) != -1 { t.Fatal("Should error on nil byte slice") } n := []byte("12345678") if pn := parseSize(n); pn != 12345678 { t.Fatalf("Did not parse %q correctly, res=%d", n, pn) } n = []byte("12345invalid678") if pn := parseSize(n); pn != -1 { t.Fatalf("Should error on %q, res=%d", n, pn) } } func TestParseSInt64(t *testing.T) { if parseInt64(nil) != -1 { t.Fatal("Should error on nil byte slice") } n := []byte("12345678") if pn := parseInt64(n); pn != 12345678 { t.Fatalf("Did not parse %q correctly, res=%d", n, pn) } n = []byte("12345invalid678") if pn := parseInt64(n); pn != -1 { t.Fatalf("Should error on %q, res=%d", n, pn) } } func TestParseHostPort(t *testing.T) { check := func(hostPort string, defaultPort int, expectedHost string, expectedPort int, expectedErr bool) { h, p, err := parseHostPort(hostPort, defaultPort) if expectedErr { if err == nil { stackFatalf(t, "Expected an error, did not get one") } // expected error, so we are done return } if !expectedErr && err != nil { stackFatalf(t, "Unexpected error: %v", err) } if expectedHost != h { stackFatalf(t, "Expected host %q, got %q", expectedHost, h) } if expectedPort != p { stackFatalf(t, "Expected port %d, got %d", expectedPort, p) } } check("addr:1234", 5678, "addr", 1234, false) check(" addr:1234 ", 5678, "addr", 1234, false) check(" addr : 1234 ", 5678, "addr", 1234, false) check("addr", 5678, "addr", 5678, false) check(" addr ", 5678, "addr", 5678, false) check("addr:-1", 5678, "addr", 5678, false) check(" addr:-1 ", 5678, "addr", 5678, false) check(" addr : -1 ", 5678, "addr", 5678, false) check("addr:0", 5678, "addr", 5678, false) check(" addr:0 ", 5678, "addr", 5678, false) check(" addr : 0 ", 5678, "addr", 5678, false) check("addr:addr", 0, "", 0, true) check("addr:::1234", 0, "", 0, true) check("", 0, "", 0, true) } func TestURLsAreEqual(t *testing.T) { check := func(t *testing.T, u1Str, u2Str string, expectedSame bool) { t.Helper() u1, err := url.Parse(u1Str) if err != nil { t.Fatalf("Error parsing url %q: %v", u1Str, err) } u2, err := url.Parse(u2Str) if err != nil { t.Fatalf("Error parsing url %q: %v", u2Str, err) } same := urlsAreEqual(u1, u2) if expectedSame && !same { t.Fatalf("Expected %v and %v to be the same, they were not", u1, u2) } else if !expectedSame && same { t.Fatalf("Expected %v and %v to be different, they were not", u1, u2) } } check(t, "nats://localhost:4222", "nats://localhost:4222", true) check(t, "nats://ivan:pwd@localhost:4222", "nats://ivan:pwd@localhost:4222", true) check(t, "nats://ivan@localhost:4222", "nats://ivan@localhost:4222", true) check(t, "nats://ivan:@localhost:4222", "nats://ivan:@localhost:4222", true) check(t, "nats://host1:4222", "nats://host2:4222", false) } func TestComma(t *testing.T) { type testList []struct { name, got, exp string } l := testList{ {"0", comma(0), "0"}, {"10", comma(10), "10"}, {"100", comma(100), "100"}, {"1,000", comma(1000), "1,000"}, {"10,000", comma(10000), "10,000"}, {"100,000", comma(100000), "100,000"}, {"10,000,000", comma(10000000), "10,000,000"}, {"10,100,000", comma(10100000), "10,100,000"}, {"10,010,000", comma(10010000), "10,010,000"}, {"10,001,000", comma(10001000), "10,001,000"}, {"123,456,789", comma(123456789), "123,456,789"}, {"maxint", comma(9.223372e+18), "9,223,372,000,000,000,000"}, {"math.maxint", comma(math.MaxInt64), "9,223,372,036,854,775,807"}, {"math.minint", comma(math.MinInt64), "-9,223,372,036,854,775,808"}, {"minint", comma(-9.223372e+18), "-9,223,372,000,000,000,000"}, {"-123,456,789", comma(-123456789), "-123,456,789"}, {"-10,100,000", comma(-10100000), "-10,100,000"}, {"-10,010,000", comma(-10010000), "-10,010,000"}, {"-10,001,000", comma(-10001000), "-10,001,000"}, {"-10,000,000", comma(-10000000), "-10,000,000"}, {"-100,000", comma(-100000), "-100,000"}, {"-10,000", comma(-10000), "-10,000"}, {"-1,000", comma(-1000), "-1,000"}, {"-100", comma(-100), "-100"}, {"-10", comma(-10), "-10"}, } failed := false for _, test := range l { if test.got != test.exp { t.Errorf("On %v, expected '%v', but got '%v'", test.name, test.exp, test.got) failed = true } } if failed { t.FailNow() } } func TestURLRedaction(t *testing.T) { redactionFromTo := []struct { Full string Safe string }{ {"nats://foo:bar@example.org", "nats://foo:xxxxx@example.org"}, {"nats://foo@example.org", "nats://foo@example.org"}, {"nats://example.org", "nats://example.org"}, {"nats://example.org/foo?bar=1", "nats://example.org/foo?bar=1"}, } var err error listFull := make([]*url.URL, len(redactionFromTo)) listSafe := make([]*url.URL, len(redactionFromTo)) for i := range redactionFromTo { r := redactURLString(redactionFromTo[i].Full) if r != redactionFromTo[i].Safe { t.Fatalf("Redacting URL [index %d] %q, expected %q got %q", i, redactionFromTo[i].Full, redactionFromTo[i].Safe, r) } if listFull[i], err = url.Parse(redactionFromTo[i].Full); err != nil { t.Fatalf("Redacting URL index %d parse Full failed: %v", i, err) } if listSafe[i], err = url.Parse(redactionFromTo[i].Safe); err != nil { t.Fatalf("Redacting URL index %d parse Safe failed: %v", i, err) } } results := redactURLList(listFull) if !reflect.DeepEqual(results, listSafe) { t.Fatalf("Redacting URL list did not compare equal, even after each URL did") } } func TestVersionAtLeast(t *testing.T) { for _, test := range []struct { version string major int minor int update int result bool }{ {"2.0.0-beta", 1, 9, 9, true}, {"2.0.0", 1, 99, 9, true}, {"2.2.0", 2, 1, 9, true}, {"2.2.2", 2, 2, 2, true}, {"2.2.2", 2, 2, 3, false}, {"2.2.2", 2, 3, 2, false}, {"2.2.2", 3, 2, 2, false}, {"2.22.2", 3, 0, 0, false}, {"2.2.22", 2, 3, 0, false}, {"bad.version", 1, 2, 3, false}, } { t.Run(_EMPTY_, func(t *testing.T) { if res := versionAtLeast(test.version, test.major, test.minor, test.update); res != test.result { t.Fatalf("For check version %q at least %d.%d.%d result should have been %v, got %v", test.version, test.major, test.minor, test.update, test.result, res) } }) } } func BenchmarkParseInt(b *testing.B) { b.SetBytes(1) n := "12345678" for i := 0; i < b.N; i++ { strconv.ParseInt(n, 10, 0) } } func BenchmarkParseSize(b *testing.B) { b.SetBytes(1) n := []byte("12345678") for i := 0; i < b.N; i++ { parseSize(n) } } func deferUnlock(mu *sync.Mutex) { mu.Lock() defer mu.Unlock() // see noDeferUnlock if false { return } } func BenchmarkDeferMutex(b *testing.B) { var mu sync.Mutex b.SetBytes(1) for i := 0; i < b.N; i++ { deferUnlock(&mu) } } func noDeferUnlock(mu *sync.Mutex) { mu.Lock() // prevent staticcheck warning about empty critical section if false { return } mu.Unlock() } func BenchmarkNoDeferMutex(b *testing.B) { var mu sync.Mutex b.SetBytes(1) for i := 0; i < b.N; i++ { noDeferUnlock(&mu) } } func createTestSub() *subscription { return &subscription{ subject: []byte("foo"), queue: []byte("bar"), sid: []byte("22"), } } func BenchmarkArrayRand(b *testing.B) { b.StopTimer() r := rand.New(rand.NewSource(time.Now().UnixNano())) // Create an array of 10 items subs := []*subscription{} for i := 0; i < 10; i++ { subs = append(subs, createTestSub()) } b.StartTimer() for i := 0; i < b.N; i++ { index := r.Intn(len(subs)) _ = subs[index] } } func BenchmarkMapRange(b *testing.B) { b.StopTimer() // Create an map of 10 items subs := map[int]*subscription{} for i := 0; i < 10; i++ { subs[i] = createTestSub() } b.StartTimer() for i := 0; i < b.N; i++ { for range subs { break } } } nats-server-2.10.27/server/websocket.go000066400000000000000000001244351477524627100200070ustar00rootroot00000000000000// Copyright 2020-2025 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bytes" crand "crypto/rand" "crypto/sha1" "crypto/tls" "encoding/base64" "encoding/binary" "errors" "fmt" "io" "log" mrand "math/rand" "net" "net/http" "net/url" "strconv" "strings" "sync" "time" "unicode/utf8" "github.com/klauspost/compress/flate" ) type wsOpCode int const ( // From https://tools.ietf.org/html/rfc6455#section-5.2 wsTextMessage = wsOpCode(1) wsBinaryMessage = wsOpCode(2) wsCloseMessage = wsOpCode(8) wsPingMessage = wsOpCode(9) wsPongMessage = wsOpCode(10) wsFinalBit = 1 << 7 wsRsv1Bit = 1 << 6 // Used for compression, from https://tools.ietf.org/html/rfc7692#section-6 wsRsv2Bit = 1 << 5 wsRsv3Bit = 1 << 4 wsMaskBit = 1 << 7 wsContinuationFrame = 0 wsMaxFrameHeaderSize = 14 // Since LeafNode may need to behave as a client wsMaxControlPayloadSize = 125 wsFrameSizeForBrowsers = 4096 // From experiment, webrowsers behave better with limited frame size wsCompressThreshold = 64 // Don't compress for small buffer(s) wsCloseSatusSize = 2 // From https://tools.ietf.org/html/rfc6455#section-11.7 wsCloseStatusNormalClosure = 1000 wsCloseStatusGoingAway = 1001 wsCloseStatusProtocolError = 1002 wsCloseStatusUnsupportedData = 1003 wsCloseStatusNoStatusReceived = 1005 wsCloseStatusInvalidPayloadData = 1007 wsCloseStatusPolicyViolation = 1008 wsCloseStatusMessageTooBig = 1009 wsCloseStatusInternalSrvError = 1011 wsCloseStatusTLSHandshake = 1015 wsFirstFrame = true wsContFrame = false wsFinalFrame = true wsUncompressedFrame = false wsSchemePrefix = "ws" wsSchemePrefixTLS = "wss" wsNoMaskingHeader = "Nats-No-Masking" wsNoMaskingValue = "true" wsXForwardedForHeader = "X-Forwarded-For" wsNoMaskingFullResponse = wsNoMaskingHeader + ": " + wsNoMaskingValue + CR_LF wsPMCExtension = "permessage-deflate" // per-message compression wsPMCSrvNoCtx = "server_no_context_takeover" wsPMCCliNoCtx = "client_no_context_takeover" wsPMCReqHeaderValue = wsPMCExtension + "; " + wsPMCSrvNoCtx + "; " + wsPMCCliNoCtx wsPMCFullResponse = "Sec-WebSocket-Extensions: " + wsPMCExtension + "; " + wsPMCSrvNoCtx + "; " + wsPMCCliNoCtx + _CRLF_ wsSecProto = "Sec-Websocket-Protocol" wsMQTTSecProtoVal = "mqtt" wsMQTTSecProto = wsSecProto + ": " + wsMQTTSecProtoVal + CR_LF ) var decompressorPool sync.Pool var compressLastBlock = []byte{0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff} // From https://tools.ietf.org/html/rfc6455#section-1.3 var wsGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") // Test can enable this so that server does not support "no-masking" requests. var wsTestRejectNoMasking = false type websocket struct { frames net.Buffers fs int64 closeMsg []byte compress bool closeSent bool browser bool nocompfrag bool // No fragment for compressed frames maskread bool maskwrite bool compressor *flate.Writer cookieJwt string clientIP string } type srvWebsocket struct { mu sync.RWMutex server *http.Server listener net.Listener listenerErr error allowedOrigins map[string]*allowedOrigin // host will be the key sameOrigin bool connectURLs []string connectURLsMap refCountedUrlSet authOverride bool // indicate if there is auth override in websocket config // These are immutable and can be accessed without lock. // This is the case when generating the client INFO. tls bool // True if TLS is required (TLSConfig is specified). host string // Host/IP the webserver is listening on (shortcut to opts.Websocket.Host). port int // Port the webserver is listening on. This is after an ephemeral port may have been selected (shortcut to opts.Websocket.Port). } type allowedOrigin struct { scheme string port string } type wsUpgradeResult struct { conn net.Conn ws *websocket kind int } type wsReadInfo struct { rem int fs bool ff bool fc bool mask bool // Incoming leafnode connections may not have masking. mkpos byte mkey [4]byte cbufs [][]byte coff int } func (r *wsReadInfo) init() { r.fs, r.ff = true, true } // Returns a slice containing `needed` bytes from the given buffer `buf` // starting at position `pos`, and possibly read from the given reader `r`. // When bytes are present in `buf`, the `pos` is incremented by the number // of bytes found up to `needed` and the new position is returned. If not // enough bytes are found, the bytes found in `buf` are copied to the returned // slice and the remaning bytes are read from `r`. func wsGet(r io.Reader, buf []byte, pos, needed int) ([]byte, int, error) { avail := len(buf) - pos if avail >= needed { return buf[pos : pos+needed], pos + needed, nil } b := make([]byte, needed) start := copy(b, buf[pos:]) for start != needed { n, err := r.Read(b[start:cap(b)]) if err != nil { return nil, 0, err } start += n } return b, pos + avail, nil } // Returns true if this connection is from a Websocket client. // Lock held on entry. func (c *client) isWebsocket() bool { return c.ws != nil } // Returns a slice of byte slices corresponding to payload of websocket frames. // The byte slice `buf` is filled with bytes from the connection's read loop. // This function will decode the frame headers and unmask the payload(s). // It is possible that the returned slices point to the given `buf` slice, so // `buf` should not be overwritten until the returned slices have been parsed. // // Client lock MUST NOT be held on entry. func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, error) { var ( bufs [][]byte tmpBuf []byte err error pos int max = len(buf) ) for pos != max { if r.fs { b0 := buf[pos] frameType := wsOpCode(b0 & 0xF) final := b0&wsFinalBit != 0 compressed := b0&wsRsv1Bit != 0 pos++ tmpBuf, pos, err = wsGet(ior, buf, pos, 1) if err != nil { return bufs, err } b1 := tmpBuf[0] // Clients MUST set the mask bit. If not set, reject. // However, LEAF by default will not have masking, unless they are forced to, by configuration. if r.mask && b1&wsMaskBit == 0 { return bufs, c.wsHandleProtocolError("mask bit missing") } // Store size in case it is < 125 r.rem = int(b1 & 0x7F) switch frameType { case wsPingMessage, wsPongMessage, wsCloseMessage: if r.rem > wsMaxControlPayloadSize { return bufs, c.wsHandleProtocolError( fmt.Sprintf("control frame length bigger than maximum allowed of %v bytes", wsMaxControlPayloadSize)) } if !final { return bufs, c.wsHandleProtocolError("control frame does not have final bit set") } case wsTextMessage, wsBinaryMessage: if !r.ff { return bufs, c.wsHandleProtocolError("new message started before final frame for previous message was received") } r.ff = final r.fc = compressed case wsContinuationFrame: // Compressed bit must be only set in the first frame if r.ff || compressed { return bufs, c.wsHandleProtocolError("invalid continuation frame") } r.ff = final default: return bufs, c.wsHandleProtocolError(fmt.Sprintf("unknown opcode %v", frameType)) } switch r.rem { case 126: tmpBuf, pos, err = wsGet(ior, buf, pos, 2) if err != nil { return bufs, err } r.rem = int(binary.BigEndian.Uint16(tmpBuf)) case 127: tmpBuf, pos, err = wsGet(ior, buf, pos, 8) if err != nil { return bufs, err } r.rem = int(binary.BigEndian.Uint64(tmpBuf)) } if r.mask { // Read masking key tmpBuf, pos, err = wsGet(ior, buf, pos, 4) if err != nil { return bufs, err } copy(r.mkey[:], tmpBuf) r.mkpos = 0 } // Handle control messages in place... if wsIsControlFrame(frameType) { pos, err = c.wsHandleControlFrame(r, frameType, ior, buf, pos) if err != nil { return bufs, err } continue } // Done with the frame header r.fs = false } if pos < max { var b []byte var n int n = r.rem if pos+n > max { n = max - pos } b = buf[pos : pos+n] pos += n r.rem -= n // If needed, unmask the buffer if r.mask { r.unmask(b) } addToBufs := true // Handle compressed message if r.fc { // Assume that we may have continuation frames or not the full payload. addToBufs = false // Make a copy of the buffer before adding it to the list // of compressed fragments. r.cbufs = append(r.cbufs, append([]byte(nil), b...)) // When we have the final frame and we have read the full payload, // we can decompress it. if r.ff && r.rem == 0 { b, err = r.decompress() if err != nil { return bufs, err } r.fc = false // Now we can add to `bufs` addToBufs = true } } // For non compressed frames, or when we have decompressed the // whole message. if addToBufs { bufs = append(bufs, b) } // If payload has been fully read, then indicate that next // is the start of a frame. if r.rem == 0 { r.fs = true } } } return bufs, nil } func (r *wsReadInfo) Read(dst []byte) (int, error) { if len(dst) == 0 { return 0, nil } if len(r.cbufs) == 0 { return 0, io.EOF } copied := 0 rem := len(dst) for buf := r.cbufs[0]; buf != nil && rem > 0; { n := len(buf[r.coff:]) if n > rem { n = rem } copy(dst[copied:], buf[r.coff:r.coff+n]) copied += n rem -= n r.coff += n buf = r.nextCBuf() } return copied, nil } func (r *wsReadInfo) nextCBuf() []byte { // We still have remaining data in the first buffer if r.coff != len(r.cbufs[0]) { return r.cbufs[0] } // We read the full first buffer. Reset offset. r.coff = 0 // We were at the last buffer, so we are done. if len(r.cbufs) == 1 { r.cbufs = nil return nil } // Here we move to the next buffer. r.cbufs = r.cbufs[1:] return r.cbufs[0] } func (r *wsReadInfo) ReadByte() (byte, error) { if len(r.cbufs) == 0 { return 0, io.EOF } b := r.cbufs[0][r.coff] r.coff++ r.nextCBuf() return b, nil } func (r *wsReadInfo) decompress() ([]byte, error) { r.coff = 0 // As per https://tools.ietf.org/html/rfc7692#section-7.2.2 // add 0x00, 0x00, 0xff, 0xff and then a final block so that flate reader // does not report unexpected EOF. r.cbufs = append(r.cbufs, compressLastBlock) // Get a decompressor from the pool and bind it to this object (wsReadInfo) // that provides Read() and ReadByte() APIs that will consume the compressed // buffers (r.cbufs). d, _ := decompressorPool.Get().(io.ReadCloser) if d == nil { d = flate.NewReader(r) } else { d.(flate.Resetter).Reset(r, nil) } // This will do the decompression. b, err := io.ReadAll(d) decompressorPool.Put(d) // Now reset the compressed buffers list. r.cbufs = nil return b, err } // Handles the PING, PONG and CLOSE websocket control frames. // // Client lock MUST NOT be held on entry. func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.Reader, buf []byte, pos int) (int, error) { var payload []byte var err error if r.rem > 0 { payload, pos, err = wsGet(nc, buf, pos, r.rem) if err != nil { return pos, err } if r.mask { r.unmask(payload) } r.rem = 0 } switch frameType { case wsCloseMessage: status := wsCloseStatusNoStatusReceived var body string lp := len(payload) // If there is a payload, the status is represented as a 2-byte // unsigned integer (in network byte order). Then, there may be an // optional body. hasStatus, hasBody := lp >= wsCloseSatusSize, lp > wsCloseSatusSize if hasStatus { // Decode the status status = int(binary.BigEndian.Uint16(payload[:wsCloseSatusSize])) // Now if there is a body, capture it and make sure this is a valid UTF-8. if hasBody { body = string(payload[wsCloseSatusSize:]) if !utf8.ValidString(body) { // https://tools.ietf.org/html/rfc6455#section-5.5.1 // If body is present, it must be a valid utf8 status = wsCloseStatusInvalidPayloadData body = "invalid utf8 body in close frame" } } } // If the status indicates that nothing was received, then we don't // send anything back. // From https://datatracker.ietf.org/doc/html/rfc6455#section-7.4 // it says that code 1005 is a reserved value and MUST NOT be set as a // status code in a Close control frame by an endpoint. It is // designated for use in applications expecting a status code to indicate // that no status code was actually present. var clm []byte if status != wsCloseStatusNoStatusReceived { clm = wsCreateCloseMessage(status, body) } c.wsEnqueueControlMessage(wsCloseMessage, clm) if len(clm) > 0 { nbPoolPut(clm) // wsEnqueueControlMessage has taken a copy. } // Return io.EOF so that readLoop will close the connection as ClientClosed // after processing pending buffers. return pos, io.EOF case wsPingMessage: c.wsEnqueueControlMessage(wsPongMessage, payload) case wsPongMessage: // Nothing to do.. } return pos, nil } // Unmask the given slice. func (r *wsReadInfo) unmask(buf []byte) { p := int(r.mkpos) if len(buf) < 16 { for i := 0; i < len(buf); i++ { buf[i] ^= r.mkey[p&3] p++ } r.mkpos = byte(p & 3) return } var k [8]byte for i := 0; i < 8; i++ { k[i] = r.mkey[(p+i)&3] } km := binary.BigEndian.Uint64(k[:]) n := (len(buf) / 8) * 8 for i := 0; i < n; i += 8 { tmp := binary.BigEndian.Uint64(buf[i : i+8]) tmp ^= km binary.BigEndian.PutUint64(buf[i:], tmp) } buf = buf[n:] for i := 0; i < len(buf); i++ { buf[i] ^= r.mkey[p&3] p++ } r.mkpos = byte(p & 3) } // Returns true if the op code corresponds to a control frame. func wsIsControlFrame(frameType wsOpCode) bool { return frameType >= wsCloseMessage } // Create the frame header. // Encodes the frame type and optional compression flag, and the size of the payload. func wsCreateFrameHeader(useMasking, compressed bool, frameType wsOpCode, l int) ([]byte, []byte) { fh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize] n, key := wsFillFrameHeader(fh, useMasking, wsFirstFrame, wsFinalFrame, compressed, frameType, l) return fh[:n], key } func wsFillFrameHeader(fh []byte, useMasking, first, final, compressed bool, frameType wsOpCode, l int) (int, []byte) { var n int var b byte if first { b = byte(frameType) } if final { b |= wsFinalBit } if compressed { b |= wsRsv1Bit } b1 := byte(0) if useMasking { b1 |= wsMaskBit } switch { case l <= 125: n = 2 fh[0] = b fh[1] = b1 | byte(l) case l < 65536: n = 4 fh[0] = b fh[1] = b1 | 126 binary.BigEndian.PutUint16(fh[2:], uint16(l)) default: n = 10 fh[0] = b fh[1] = b1 | 127 binary.BigEndian.PutUint64(fh[2:], uint64(l)) } var key []byte if useMasking { var keyBuf [4]byte if _, err := io.ReadFull(crand.Reader, keyBuf[:4]); err != nil { kv := mrand.Int31() binary.LittleEndian.PutUint32(keyBuf[:4], uint32(kv)) } copy(fh[n:], keyBuf[:4]) key = fh[n : n+4] n += 4 } return n, key } // Invokes wsEnqueueControlMessageLocked under client lock. // // Client lock MUST NOT be held on entry func (c *client) wsEnqueueControlMessage(controlMsg wsOpCode, payload []byte) { c.mu.Lock() c.wsEnqueueControlMessageLocked(controlMsg, payload) c.mu.Unlock() } // Mask the buffer with the given key func wsMaskBuf(key, buf []byte) { for i := 0; i < len(buf); i++ { buf[i] ^= key[i&3] } } // Mask the buffers, as if they were contiguous, with the given key func wsMaskBufs(key []byte, bufs [][]byte) { pos := 0 for i := 0; i < len(bufs); i++ { buf := bufs[i] for j := 0; j < len(buf); j++ { buf[j] ^= key[pos&3] pos++ } } } // Enqueues a websocket control message. // If the control message is a wsCloseMessage, then marks this client // has having sent the close message (since only one should be sent). // This will prevent the generic closeConnection() to enqueue one. // // Client lock held on entry. func (c *client) wsEnqueueControlMessageLocked(controlMsg wsOpCode, payload []byte) { // Control messages are never compressed and their size will be // less than wsMaxControlPayloadSize, which means the frame header // will be only 2 or 6 bytes. useMasking := c.ws.maskwrite sz := 2 if useMasking { sz += 4 } cm := nbPoolGet(sz + len(payload)) cm = cm[:cap(cm)] n, key := wsFillFrameHeader(cm, useMasking, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, controlMsg, len(payload)) cm = cm[:n] // Note that payload is optional. if len(payload) > 0 { cm = append(cm, payload...) if useMasking { wsMaskBuf(key, cm[n:]) } } c.out.pb += int64(len(cm)) if controlMsg == wsCloseMessage { // We can't add the close message to the frames buffers // now. It will be done on a flushOutbound() when there // are no more pending buffers to send. c.ws.closeSent = true c.ws.closeMsg = cm } else { c.ws.frames = append(c.ws.frames, cm) c.ws.fs += int64(len(cm)) } c.flushSignal() } // Enqueues a websocket close message with a status mapped from the given `reason`. // // Client lock held on entry func (c *client) wsEnqueueCloseMessage(reason ClosedState) { var status int switch reason { case ClientClosed: status = wsCloseStatusNormalClosure case AuthenticationTimeout, AuthenticationViolation, SlowConsumerPendingBytes, SlowConsumerWriteDeadline, MaxAccountConnectionsExceeded, MaxConnectionsExceeded, MaxControlLineExceeded, MaxSubscriptionsExceeded, MissingAccount, AuthenticationExpired, Revocation: status = wsCloseStatusPolicyViolation case TLSHandshakeError: status = wsCloseStatusTLSHandshake case ParseError, ProtocolViolation, BadClientProtocolVersion: status = wsCloseStatusProtocolError case MaxPayloadExceeded: status = wsCloseStatusMessageTooBig case WriteError, ReadError, StaleConnection, ServerShutdown: // We used to have WriteError, ReadError and StaleConnection result in // code 1006, which the spec says that it must not be used to set the // status in the close message. So using this one instead. status = wsCloseStatusGoingAway default: status = wsCloseStatusInternalSrvError } body := wsCreateCloseMessage(status, reason.String()) c.wsEnqueueControlMessageLocked(wsCloseMessage, body) nbPoolPut(body) // wsEnqueueControlMessageLocked has taken a copy. } // Create and then enqueue a close message with a protocol error and the // given message. This is invoked when parsing websocket frames. // // Lock MUST NOT be held on entry. func (c *client) wsHandleProtocolError(message string) error { buf := wsCreateCloseMessage(wsCloseStatusProtocolError, message) c.wsEnqueueControlMessage(wsCloseMessage, buf) nbPoolPut(buf) // wsEnqueueControlMessage has taken a copy. return errors.New(message) } // Create a close message with the given `status` and `body`. // If the `body` is more than the maximum allows control frame payload size, // it is truncated and "..." is added at the end (as a hint that message // is not complete). func wsCreateCloseMessage(status int, body string) []byte { // Since a control message payload is limited in size, we // will limit the text and add trailing "..." if truncated. // The body of a Close Message must be preceded with 2 bytes, // so take that into account for limiting the body length. if len(body) > wsMaxControlPayloadSize-2 { body = body[:wsMaxControlPayloadSize-5] body += "..." } buf := nbPoolGet(2 + len(body))[:2+len(body)] // We need to have a 2 byte unsigned int that represents the error status code // https://tools.ietf.org/html/rfc6455#section-5.5.1 binary.BigEndian.PutUint16(buf[:2], uint16(status)) copy(buf[2:], []byte(body)) return buf } // Process websocket client handshake. On success, returns the raw net.Conn that // will be used to create a *client object. // Invoked from the HTTP server listening on websocket port. func (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeResult, error) { kind := CLIENT if r.URL != nil { ep := r.URL.EscapedPath() if strings.HasSuffix(ep, leafNodeWSPath) { kind = LEAF } else if strings.HasSuffix(ep, mqttWSPath) { kind = MQTT } } opts := s.getOpts() // From https://tools.ietf.org/html/rfc6455#section-4.2.1 // Point 1. if r.Method != "GET" { return nil, wsReturnHTTPError(w, r, http.StatusMethodNotAllowed, "request method must be GET") } // Point 2. if r.Host == _EMPTY_ { return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "'Host' missing in request") } // Point 3. if !wsHeaderContains(r.Header, "Upgrade", "websocket") { return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid value for header 'Upgrade'") } // Point 4. if !wsHeaderContains(r.Header, "Connection", "Upgrade") { return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid value for header 'Connection'") } // Point 5. key := r.Header.Get("Sec-Websocket-Key") if key == _EMPTY_ { return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "key missing") } // Point 6. if !wsHeaderContains(r.Header, "Sec-Websocket-Version", "13") { return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid version") } // Others are optional // Point 7. if err := s.websocket.checkOrigin(r); err != nil { return nil, wsReturnHTTPError(w, r, http.StatusForbidden, fmt.Sprintf("origin not allowed: %v", err)) } // Point 8. // We don't have protocols, so ignore. // Point 9. // Extensions, only support for compression at the moment compress := opts.Websocket.Compression if compress { // Simply check if permessage-deflate extension is present. compress, _ = wsPMCExtensionSupport(r.Header, true) } // We will do masking if asked (unless we reject for tests) noMasking := r.Header.Get(wsNoMaskingHeader) == wsNoMaskingValue && !wsTestRejectNoMasking h := w.(http.Hijacker) conn, brw, err := h.Hijack() if err != nil { if conn != nil { conn.Close() } return nil, wsReturnHTTPError(w, r, http.StatusInternalServerError, err.Error()) } if brw.Reader.Buffered() > 0 { conn.Close() return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "client sent data before handshake is complete") } var buf [1024]byte p := buf[:0] // From https://tools.ietf.org/html/rfc6455#section-4.2.2 p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) p = append(p, wsAcceptKey(key)...) p = append(p, _CRLF_...) if compress { p = append(p, wsPMCFullResponse...) } if noMasking { p = append(p, wsNoMaskingFullResponse...) } if kind == MQTT { p = append(p, wsMQTTSecProto...) } p = append(p, _CRLF_...) if _, err = conn.Write(p); err != nil { conn.Close() return nil, err } // If there was a deadline set for the handshake, clear it now. if opts.Websocket.HandshakeTimeout > 0 { conn.SetDeadline(time.Time{}) } // Server always expect "clients" to send masked payload, unless the option // "no-masking" has been enabled. ws := &websocket{compress: compress, maskread: !noMasking} // Check for X-Forwarded-For header if cips, ok := r.Header[wsXForwardedForHeader]; ok { cip := cips[0] if net.ParseIP(cip) != nil { ws.clientIP = cip } } if kind == CLIENT || kind == MQTT { // Indicate if this is likely coming from a browser. if ua := r.Header.Get("User-Agent"); ua != _EMPTY_ && strings.HasPrefix(ua, "Mozilla/") { ws.browser = true // Disable fragmentation of compressed frames for Safari browsers. // Unfortunately, you could be running Chrome on macOS and this // string will contain "Safari/" (along "Chrome/"). However, what // I have found is that actual Safari browser also have "Version/". // So make the combination of the two. ws.nocompfrag = ws.compress && strings.Contains(ua, "Version/") && strings.Contains(ua, "Safari/") } if opts.Websocket.JWTCookie != _EMPTY_ { if c, err := r.Cookie(opts.Websocket.JWTCookie); err == nil && c != nil { ws.cookieJwt = c.Value } } } return &wsUpgradeResult{conn: conn, ws: ws, kind: kind}, nil } // Returns true if the header named `name` contains a token with value `value`. func wsHeaderContains(header http.Header, name string, value string) bool { for _, s := range header[name] { tokens := strings.Split(s, ",") for _, t := range tokens { t = strings.Trim(t, " \t") if strings.EqualFold(t, value) { return true } } } return false } func wsPMCExtensionSupport(header http.Header, checkPMCOnly bool) (bool, bool) { for _, extensionList := range header["Sec-Websocket-Extensions"] { extensions := strings.Split(extensionList, ",") for _, extension := range extensions { extension = strings.Trim(extension, " \t") params := strings.Split(extension, ";") for i, p := range params { p = strings.Trim(p, " \t") if strings.EqualFold(p, wsPMCExtension) { if checkPMCOnly { return true, false } var snc bool var cnc bool for j := i + 1; j < len(params); j++ { p = params[j] p = strings.Trim(p, " \t") if strings.EqualFold(p, wsPMCSrvNoCtx) { snc = true } else if strings.EqualFold(p, wsPMCCliNoCtx) { cnc = true } if snc && cnc { return true, true } } return true, false } } } } return false, false } // Send an HTTP error with the given `status` to the given http response writer `w`. // Return an error created based on the `reason` string. func wsReturnHTTPError(w http.ResponseWriter, r *http.Request, status int, reason string) error { err := fmt.Errorf("%s - websocket handshake error: %s", r.RemoteAddr, reason) w.Header().Set("Sec-Websocket-Version", "13") http.Error(w, http.StatusText(status), status) return err } // If the server is configured to accept any origin, then this function returns // `nil` without checking if the Origin is present and valid. This is also // the case if the request does not have the Origin header. // Otherwise, this will check that the Origin matches the same origin or // any origin in the allowed list. func (w *srvWebsocket) checkOrigin(r *http.Request) error { w.mu.RLock() checkSame := w.sameOrigin listEmpty := len(w.allowedOrigins) == 0 w.mu.RUnlock() if !checkSame && listEmpty { return nil } origin := r.Header.Get("Origin") if origin == _EMPTY_ { origin = r.Header.Get("Sec-Websocket-Origin") } // If the header is not present, we will accept. // From https://datatracker.ietf.org/doc/html/rfc6455#section-1.6 // "Naturally, when the WebSocket Protocol is used by a dedicated client // directly (i.e., not from a web page through a web browser), the origin // model is not useful, as the client can provide any arbitrary origin string." if origin == _EMPTY_ { return nil } u, err := url.ParseRequestURI(origin) if err != nil { return err } oh, op, err := wsGetHostAndPort(u.Scheme == "https", u.Host) if err != nil { return err } // If checking same origin, compare with the http's request's Host. if checkSame { rh, rp, err := wsGetHostAndPort(r.TLS != nil, r.Host) if err != nil { return err } if oh != rh || op != rp { return errors.New("not same origin") } // I guess it is possible to have cases where one wants to check // same origin, but also that the origin is in the allowed list. // So continue with the next check. } if !listEmpty { w.mu.RLock() ao := w.allowedOrigins[oh] w.mu.RUnlock() if ao == nil || u.Scheme != ao.scheme || op != ao.port { return errors.New("not in the allowed list") } } return nil } func wsGetHostAndPort(tls bool, hostport string) (string, string, error) { host, port, err := net.SplitHostPort(hostport) if err != nil { // If error is missing port, then use defaults based on the scheme if ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, "missing port") { err = nil host = hostport if tls { port = "443" } else { port = "80" } } } return strings.ToLower(host), port, err } // Concatenate the key sent by the client with the GUID, then computes the SHA1 hash // and returns it as a based64 encoded string. func wsAcceptKey(key string) string { h := sha1.New() h.Write([]byte(key)) h.Write(wsGUID) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } func wsMakeChallengeKey() (string, error) { p := make([]byte, 16) if _, err := io.ReadFull(crand.Reader, p); err != nil { return _EMPTY_, err } return base64.StdEncoding.EncodeToString(p), nil } // Validate the websocket related options. func validateWebsocketOptions(o *Options) error { wo := &o.Websocket // If no port is defined, we don't care about other options if wo.Port == 0 { return nil } // Enforce TLS... unless NoTLS is set to true. if wo.TLSConfig == nil && !wo.NoTLS { return errors.New("websocket requires TLS configuration") } // Make sure that allowed origins, if specified, can be parsed. for _, ao := range wo.AllowedOrigins { if _, err := url.Parse(ao); err != nil { return fmt.Errorf("unable to parse allowed origin: %v", err) } } // If there is a NoAuthUser, we need to have Users defined and // the user to be present. if wo.NoAuthUser != _EMPTY_ { if err := validateNoAuthUser(o, wo.NoAuthUser); err != nil { return err } } // Token/Username not possible if there are users/nkeys if len(o.Users) > 0 || len(o.Nkeys) > 0 { if wo.Username != _EMPTY_ { return fmt.Errorf("websocket authentication username not compatible with presence of users/nkeys") } if wo.Token != _EMPTY_ { return fmt.Errorf("websocket authentication token not compatible with presence of users/nkeys") } } // Using JWT requires Trusted Keys if wo.JWTCookie != _EMPTY_ { if len(o.TrustedOperators) == 0 && len(o.TrustedKeys) == 0 { return fmt.Errorf("trusted operators or trusted keys configuration is required for JWT authentication via cookie %q", wo.JWTCookie) } } if err := validatePinnedCerts(wo.TLSPinnedCerts); err != nil { return fmt.Errorf("websocket: %v", err) } return nil } // Creates or updates the existing map func (s *Server) wsSetOriginOptions(o *WebsocketOpts) { ws := &s.websocket ws.mu.Lock() defer ws.mu.Unlock() // Copy over the option's same origin boolean ws.sameOrigin = o.SameOrigin // Reset the map. Will help for config reload if/when we support it. ws.allowedOrigins = nil if o.AllowedOrigins == nil { return } for _, ao := range o.AllowedOrigins { // We have previously checked (during options validation) that the urls // are parseable, but if we get an error, report and skip. u, err := url.ParseRequestURI(ao) if err != nil { s.Errorf("error parsing allowed origin: %v", err) continue } h, p, _ := wsGetHostAndPort(u.Scheme == "https", u.Host) if ws.allowedOrigins == nil { ws.allowedOrigins = make(map[string]*allowedOrigin, len(o.AllowedOrigins)) } ws.allowedOrigins[h] = &allowedOrigin{scheme: u.Scheme, port: p} } } // Given the websocket options, we check if any auth configuration // has been provided. If so, possibly create users/nkey users and // store them in s.websocket.users/nkeys. // Also update a boolean that indicates if auth is required for // websocket clients. // Server lock is held on entry. func (s *Server) wsConfigAuth(opts *WebsocketOpts) { ws := &s.websocket // If any of those is specified, we consider that there is an override. ws.authOverride = opts.Username != _EMPTY_ || opts.Token != _EMPTY_ || opts.NoAuthUser != _EMPTY_ } func (s *Server) startWebsocketServer() { if s.isShuttingDown() { return } sopts := s.getOpts() o := &sopts.Websocket s.wsSetOriginOptions(o) var hl net.Listener var proto string var err error port := o.Port if port == -1 { port = 0 } hp := net.JoinHostPort(o.Host, strconv.Itoa(port)) // We are enforcing (when validating the options) the use of TLS, but the // code was originally supporting both modes. The reason for TLS only is // that we expect users to send JWTs with bearer tokens and we want to // avoid the possibility of it being "intercepted". s.mu.Lock() // Do not check o.NoTLS here. If a TLS configuration is available, use it, // regardless of NoTLS. If we don't have a TLS config, it means that the // user has configured NoTLS because otherwise the server would have failed // to start due to options validation. if o.TLSConfig != nil { proto = wsSchemePrefixTLS config := o.TLSConfig.Clone() config.GetConfigForClient = s.wsGetTLSConfig hl, err = tls.Listen("tcp", hp, config) } else { proto = wsSchemePrefix hl, err = net.Listen("tcp", hp) } s.websocket.listenerErr = err if err != nil { s.mu.Unlock() s.Fatalf("Unable to listen for websocket connections: %v", err) return } if port == 0 { o.Port = hl.Addr().(*net.TCPAddr).Port } s.Noticef("Listening for websocket clients on %s://%s:%d", proto, o.Host, o.Port) if proto == wsSchemePrefix { s.Warnf("Websocket not configured with TLS. DO NOT USE IN PRODUCTION!") } // These 3 are immutable and will be accessed without lock by the client // when generating/sending the INFO protocols. s.websocket.tls = proto == wsSchemePrefixTLS s.websocket.host, s.websocket.port = o.Host, o.Port // This will be updated when/if the cluster changes. s.websocket.connectURLs, err = s.getConnectURLs(o.Advertise, o.Host, o.Port) if err != nil { s.Fatalf("Unable to get websocket connect URLs: %v", err) hl.Close() s.mu.Unlock() return } hasLeaf := sopts.LeafNode.Port != 0 mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { res, err := s.wsUpgrade(w, r) if err != nil { s.Errorf(err.Error()) return } switch res.kind { case CLIENT: s.createWSClient(res.conn, res.ws) case MQTT: s.createMQTTClient(res.conn, res.ws) case LEAF: if !hasLeaf { s.Errorf("Not configured to accept leaf node connections") // Silently close for now. If we want to send an error back, we would // need to create the leafnode client anyway, so that is is handling websocket // frames, then send the error to the remote. res.conn.Close() return } s.createLeafNode(res.conn, nil, nil, res.ws) } }) hs := &http.Server{ Addr: hp, Handler: mux, ReadTimeout: o.HandshakeTimeout, ErrorLog: log.New(&captureHTTPServerLog{s, "websocket: "}, _EMPTY_, 0), } s.websocket.mu.Lock() s.websocket.server = hs s.websocket.listener = hl s.websocket.mu.Unlock() go func() { if err := hs.Serve(hl); err != http.ErrServerClosed { s.Fatalf("websocket listener error: %v", err) } if s.isLameDuckMode() { // Signal that we are not accepting new clients s.ldmCh <- true // Now wait for the Shutdown... <-s.quitCh return } s.done <- true }() s.mu.Unlock() } // The TLS configuration is passed to the listener when the websocket // "server" is setup. That prevents TLS configuration updates on reload // from being used. By setting this function in tls.Config.GetConfigForClient // we instruct the TLS handshake to ask for the tls configuration to be // used for a specific client. We don't care which client, we always use // the same TLS configuration. func (s *Server) wsGetTLSConfig(_ *tls.ClientHelloInfo) (*tls.Config, error) { opts := s.getOpts() return opts.Websocket.TLSConfig, nil } // This is similar to createClient() but has some modifications // specific to handle websocket clients. // The comments have been kept to minimum to reduce code size. // Check createClient() for more details. func (s *Server) createWSClient(conn net.Conn, ws *websocket) *client { opts := s.getOpts() maxPay := int32(opts.MaxPayload) maxSubs := int32(opts.MaxSubs) if maxSubs == 0 { maxSubs = -1 } now := time.Now().UTC() c := &client{srv: s, nc: conn, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now, ws: ws} c.registerWithAccount(s.globalAccount()) var info Info var authRequired bool s.mu.Lock() info = s.copyInfo() // Check auth, override if applicable. if !info.AuthRequired { // Set info.AuthRequired since this is what is sent to the client. info.AuthRequired = s.websocket.authOverride } if s.nonceRequired() { var raw [nonceLen]byte nonce := raw[:] s.generateNonce(nonce) info.Nonce = string(nonce) } c.nonce = []byte(info.Nonce) authRequired = info.AuthRequired s.totalClients++ s.mu.Unlock() c.mu.Lock() if authRequired { c.flags.set(expectConnect) } c.initClient() c.Debugf("Client connection created") c.sendProtoNow(c.generateClientInfoJSON(info)) c.mu.Unlock() s.mu.Lock() if !s.isRunning() || s.ldm { if s.isShuttingDown() { conn.Close() } s.mu.Unlock() return c } if opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn { s.mu.Unlock() c.maxConnExceeded() return nil } s.clients[c.cid] = c s.mu.Unlock() c.mu.Lock() // Websocket clients do TLS in the websocket http server. // So no TLS initiation here... if _, ok := conn.(*tls.Conn); ok { c.flags.set(handshakeComplete) } if c.isClosed() { c.mu.Unlock() c.closeConnection(WriteError) return nil } if authRequired { timeout := opts.AuthTimeout // Possibly override with Websocket specific value. if opts.Websocket.AuthTimeout != 0 { timeout = opts.Websocket.AuthTimeout } c.setAuthTimer(secondsToDuration(timeout)) } c.setPingTimer() s.startGoRoutine(func() { c.readLoop(nil) }) s.startGoRoutine(func() { c.writeLoop() }) c.mu.Unlock() return c } func (c *client) wsCollapsePtoNB() (net.Buffers, int64) { nb := c.out.nb var mfs int var usz int if c.ws.browser { mfs = wsFrameSizeForBrowsers } mask := c.ws.maskwrite // Start with possible already framed buffers (that we could have // got from partials or control messages such as ws pings or pongs). bufs := c.ws.frames compress := c.ws.compress if compress && len(nb) > 0 { // First, make sure we don't compress for very small cumulative buffers. for _, b := range nb { usz += len(b) } if usz <= wsCompressThreshold { compress = false if cp := c.ws.compressor; cp != nil { cp.Reset(nil) } } } if compress && len(nb) > 0 { // Overwrite mfs if this connection does not support fragmented compressed frames. if mfs > 0 && c.ws.nocompfrag { mfs = 0 } buf := bytes.NewBuffer(nbPoolGet(usz)) cp := c.ws.compressor if cp == nil { c.ws.compressor, _ = flate.NewWriter(buf, flate.BestSpeed) cp = c.ws.compressor } else { cp.Reset(buf) } var csz int for _, b := range nb { for len(b) > 0 { n, err := cp.Write(b) if err != nil { // Whatever this error is, it'll be handled by the cp.Flush() // call below, as the same error will be returned there. // Let the outer loop return all the buffers back to the pool // and fall through naturally. break } b = b[n:] } nbPoolPut(b) // No longer needed as contents written to compressor. } if err := cp.Flush(); err != nil { c.Errorf("Error during compression: %v", err) c.markConnAsClosed(WriteError) cp.Reset(nil) return nil, 0 } b := buf.Bytes() p := b[:len(b)-4] if mfs > 0 && len(p) > mfs { for first, final := true, false; len(p) > 0; first = false { lp := len(p) if lp > mfs { lp = mfs } else { final = true } // Only the first frame should be marked as compressed, so pass // `first` for the compressed boolean. fh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize] n, key := wsFillFrameHeader(fh, mask, first, final, first, wsBinaryMessage, lp) if mask { wsMaskBuf(key, p[:lp]) } bufs = append(bufs, fh[:n], p[:lp]) csz += n + lp p = p[lp:] } } else { ol := len(p) h, key := wsCreateFrameHeader(mask, true, wsBinaryMessage, ol) if mask { wsMaskBuf(key, p) } if ol > 0 { bufs = append(bufs, h, p) } csz = len(h) + ol } // Make sure that the compressor no longer holds a reference to // the bytes.Buffer, so that the underlying memory gets cleaned // up after flushOutbound/flushAndClose. For this to be safe, we // always cp.Reset(...) before reusing the compressor again. cp.Reset(nil) // Add to pb the compressed data size (including headers), but // remove the original uncompressed data size that was added // during the queueing. c.out.pb += int64(csz) - int64(usz) c.ws.fs += int64(csz) } else if len(nb) > 0 { var total int if mfs > 0 { // We are limiting the frame size. startFrame := func() int { bufs = append(bufs, nbPoolGet(wsMaxFrameHeaderSize)) return len(bufs) - 1 } endFrame := func(idx, size int) { bufs[idx] = bufs[idx][:wsMaxFrameHeaderSize] n, key := wsFillFrameHeader(bufs[idx], mask, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, wsBinaryMessage, size) bufs[idx] = bufs[idx][:n] c.out.pb += int64(n) c.ws.fs += int64(n + size) if mask { wsMaskBufs(key, bufs[idx+1:]) } } fhIdx := startFrame() for i := 0; i < len(nb); i++ { b := nb[i] if total+len(b) <= mfs { buf := nbPoolGet(len(b)) bufs = append(bufs, append(buf, b...)) total += len(b) nbPoolPut(nb[i]) continue } for len(b) > 0 { endStart := total != 0 if endStart { endFrame(fhIdx, total) } total = len(b) if total >= mfs { total = mfs } if endStart { fhIdx = startFrame() } buf := nbPoolGet(total) bufs = append(bufs, append(buf, b[:total]...)) b = b[total:] } nbPoolPut(nb[i]) // No longer needed as copied into smaller frames. } if total > 0 { endFrame(fhIdx, total) } } else { // If there is no limit on the frame size, create a single frame for // all pending buffers. for _, b := range nb { total += len(b) } wsfh, key := wsCreateFrameHeader(mask, false, wsBinaryMessage, total) c.out.pb += int64(len(wsfh)) bufs = append(bufs, wsfh) idx := len(bufs) bufs = append(bufs, nb...) if mask { wsMaskBufs(key, bufs[idx:]) } c.ws.fs += int64(len(wsfh) + total) } } if len(c.ws.closeMsg) > 0 { bufs = append(bufs, c.ws.closeMsg) c.ws.fs += int64(len(c.ws.closeMsg)) c.ws.closeMsg = nil c.ws.compressor = nil } c.ws.frames = nil return bufs, c.ws.fs } func isWSURL(u *url.URL) bool { return strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefix) } func isWSSURL(u *url.URL) bool { return strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefixTLS) } nats-server-2.10.27/server/websocket_test.go000066400000000000000000004012651477524627100210450ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "bufio" "bytes" "crypto/tls" "encoding/base64" "encoding/binary" "encoding/json" "errors" "fmt" "io" "math/rand" "net" "net/http" "net/url" "reflect" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/klauspost/compress/flate" ) type testReader struct { buf []byte pos int max int err error } func (tr *testReader) Read(p []byte) (int, error) { if tr.err != nil { return 0, tr.err } n := len(tr.buf) - tr.pos if n == 0 { return 0, nil } if n > len(p) { n = len(p) } if tr.max > 0 && n > tr.max { n = tr.max } copy(p, tr.buf[tr.pos:tr.pos+n]) tr.pos += n return n, nil } func TestWSGet(t *testing.T) { rb := []byte("012345") tr := &testReader{buf: []byte("6789")} for _, test := range []struct { name string pos int needed int newpos int trmax int result string reterr bool }{ {"fromrb1", 0, 3, 3, 4, "012", false}, // Partial from read buffer {"fromrb2", 3, 2, 5, 4, "34", false}, // Partial from read buffer {"fromrb3", 5, 1, 6, 4, "5", false}, // Partial from read buffer {"fromtr1", 4, 4, 6, 4, "4567", false}, // Partial from read buffer + some of ioReader {"fromtr2", 4, 6, 6, 4, "456789", false}, // Partial from read buffer + all of ioReader {"fromtr3", 4, 6, 6, 2, "456789", false}, // Partial from read buffer + all of ioReader with several reads {"fromtr4", 4, 6, 6, 2, "", true}, // ioReader returns error } { t.Run(test.name, func(t *testing.T) { tr.pos = 0 tr.max = test.trmax if test.reterr { tr.err = fmt.Errorf("on purpose") } res, np, err := wsGet(tr, rb, test.pos, test.needed) if test.reterr { if err == nil { t.Fatalf("Expected error, got none") } if err.Error() != "on purpose" { t.Fatalf("Unexpected error: %v", err) } if np != 0 || res != nil { t.Fatalf("Unexpected returned values: res=%v n=%v", res, np) } return } if err != nil { t.Fatalf("Error on get: %v", err) } if np != test.newpos { t.Fatalf("Expected pos=%v, got %v", test.newpos, np) } if string(res) != test.result { t.Fatalf("Invalid returned content: %s", res) } }) } } func TestWSIsControlFrame(t *testing.T) { for _, test := range []struct { name string code wsOpCode isControl bool }{ {"binary", wsBinaryMessage, false}, {"text", wsTextMessage, false}, {"ping", wsPingMessage, true}, {"pong", wsPongMessage, true}, {"close", wsCloseMessage, true}, } { t.Run(test.name, func(t *testing.T) { if res := wsIsControlFrame(test.code); res != test.isControl { t.Fatalf("Expected %q isControl to be %v, got %v", test.name, test.isControl, res) } }) } } func testWSSimpleMask(key, buf []byte) { for i := 0; i < len(buf); i++ { buf[i] ^= key[i&3] } } func TestWSUnmask(t *testing.T) { key := []byte{1, 2, 3, 4} orgBuf := []byte("this is a clear text") mask := func() []byte { t.Helper() buf := append([]byte(nil), orgBuf...) testWSSimpleMask(key, buf) // First ensure that the content is masked. if bytes.Equal(buf, orgBuf) { t.Fatalf("Masking did not do anything: %q", buf) } return buf } ri := &wsReadInfo{mask: true} ri.init() copy(ri.mkey[:], key) buf := mask() // Unmask in one call ri.unmask(buf) if !bytes.Equal(buf, orgBuf) { t.Fatalf("Unmask error, expected %q, got %q", orgBuf, buf) } // Unmask in multiple calls buf = mask() ri.mkpos = 0 ri.unmask(buf[:3]) ri.unmask(buf[3:11]) ri.unmask(buf[11:]) if !bytes.Equal(buf, orgBuf) { t.Fatalf("Unmask error, expected %q, got %q", orgBuf, buf) } } func TestWSCreateCloseMessage(t *testing.T) { for _, test := range []struct { name string status int psize int truncated bool }{ {"fits", wsCloseStatusInternalSrvError, 10, false}, {"truncated", wsCloseStatusProtocolError, wsMaxControlPayloadSize + 10, true}, } { t.Run(test.name, func(t *testing.T) { payload := make([]byte, test.psize) for i := 0; i < len(payload); i++ { payload[i] = byte('A' + (i % 26)) } res := wsCreateCloseMessage(test.status, string(payload)) if status := binary.BigEndian.Uint16(res[:2]); int(status) != test.status { t.Fatalf("Expected status to be %v, got %v", test.status, status) } psize := len(res) - 2 if !test.truncated { if int(psize) != test.psize { t.Fatalf("Expected size to be %v, got %v", test.psize, psize) } if !bytes.Equal(res[2:], payload) { t.Fatalf("Unexpected result: %q", res[2:]) } return } // Since the payload of a close message contains a 2 byte status, the // actual max text size will be wsMaxControlPayloadSize-2 if int(psize) != wsMaxControlPayloadSize-2 { t.Fatalf("Expected size to be capped to %v, got %v", wsMaxControlPayloadSize-2, psize) } if string(res[len(res)-3:]) != "..." { t.Fatalf("Expected res to have `...` at the end, got %q", res[4:]) } }) } } func TestWSCreateFrameHeader(t *testing.T) { for _, test := range []struct { name string frameType wsOpCode compressed bool len int }{ {"uncompressed 10", wsBinaryMessage, false, 10}, {"uncompressed 600", wsTextMessage, false, 600}, {"uncompressed 100000", wsTextMessage, false, 100000}, {"compressed 10", wsBinaryMessage, true, 10}, {"compressed 600", wsBinaryMessage, true, 600}, {"compressed 100000", wsTextMessage, true, 100000}, } { t.Run(test.name, func(t *testing.T) { res, _ := wsCreateFrameHeader(false, test.compressed, test.frameType, test.len) // The server is always sending the message has a single frame, // so the "final" bit should be set. expected := byte(test.frameType) | wsFinalBit if test.compressed { expected |= wsRsv1Bit } if b := res[0]; b != expected { t.Fatalf("Expected first byte to be %v, got %v", expected, b) } switch { case test.len <= 125: if len(res) != 2 { t.Fatalf("Frame len should be 2, got %v", len(res)) } if res[1] != byte(test.len) { t.Fatalf("Expected len to be in second byte and be %v, got %v", test.len, res[1]) } case test.len < 65536: // 1+1+2 if len(res) != 4 { t.Fatalf("Frame len should be 4, got %v", len(res)) } if res[1] != 126 { t.Fatalf("Second byte value should be 126, got %v", res[1]) } if rl := binary.BigEndian.Uint16(res[2:]); int(rl) != test.len { t.Fatalf("Expected len to be %v, got %v", test.len, rl) } default: // 1+1+8 if len(res) != 10 { t.Fatalf("Frame len should be 10, got %v", len(res)) } if res[1] != 127 { t.Fatalf("Second byte value should be 127, got %v", res[1]) } if rl := binary.BigEndian.Uint64(res[2:]); int(rl) != test.len { t.Fatalf("Expected len to be %v, got %v", test.len, rl) } } }) } } func testWSCreateClientMsg(frameType wsOpCode, frameNum int, final, compressed bool, payload []byte) []byte { if compressed { buf := &bytes.Buffer{} compressor, _ := flate.NewWriter(buf, 1) compressor.Write(payload) compressor.Flush() payload = buf.Bytes() // The last 4 bytes are dropped payload = payload[:len(payload)-4] } frame := make([]byte, 14+len(payload)) if frameNum == 1 { frame[0] = byte(frameType) } if final { frame[0] |= wsFinalBit } if compressed { frame[0] |= wsRsv1Bit } pos := 1 lenPayload := len(payload) switch { case lenPayload <= 125: frame[pos] = byte(lenPayload) | wsMaskBit pos++ case lenPayload < 65536: frame[pos] = 126 | wsMaskBit binary.BigEndian.PutUint16(frame[2:], uint16(lenPayload)) pos += 3 default: frame[1] = 127 | wsMaskBit binary.BigEndian.PutUint64(frame[2:], uint64(lenPayload)) pos += 9 } key := []byte{1, 2, 3, 4} copy(frame[pos:], key) pos += 4 copy(frame[pos:], payload) testWSSimpleMask(key, frame[pos:]) pos += lenPayload return frame[:pos] } func testWSSetupForRead() (*client, *wsReadInfo, *testReader) { ri := &wsReadInfo{mask: true} ri.init() tr := &testReader{} opts := DefaultOptions() opts.MaxPending = MAX_PENDING_SIZE s := &Server{opts: opts} c := &client{srv: s, ws: &websocket{}} c.initClient() return c, ri, tr } func TestWSReadUncompressedFrames(t *testing.T) { c, ri, tr := testWSSetupForRead() // Create 2 WS messages pl1 := []byte("first message") wsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, pl1) pl2 := []byte("second message") wsmsg2 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, pl2) // Add both in single buffer orgrb := append([]byte(nil), wsmsg1...) orgrb = append(orgrb, wsmsg2...) rb := append([]byte(nil), orgrb...) bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 2 { t.Fatalf("Expected 2 buffers, got %v", n) } if !bytes.Equal(bufs[0], pl1) { t.Fatalf("Unexpected content for buffer 1: %s", bufs[0]) } if !bytes.Equal(bufs[1], pl2) { t.Fatalf("Unexpected content for buffer 2: %s", bufs[1]) } // Now reset and try with the read buffer not containing full ws frame c, ri, tr = testWSSetupForRead() rb = append([]byte(nil), orgrb...) // Frame is 1+1+4+'first message'. So say we pass with rb of 11 bytes, // then we should get "first" bufs, err = c.wsRead(ri, tr, rb[:11]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != "first" { t.Fatalf("Unexpected content: %q", bufs[0]) } // Call again with more data.. bufs, err = c.wsRead(ri, tr, rb[11:32]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 2 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != " message" { t.Fatalf("Unexpected content: %q", bufs[0]) } if string(bufs[1]) != "second " { t.Fatalf("Unexpected content: %q", bufs[1]) } // Call with the rest bufs, err = c.wsRead(ri, tr, rb[32:]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != "message" { t.Fatalf("Unexpected content: %q", bufs[0]) } } func TestWSReadCompressedFrames(t *testing.T) { c, ri, tr := testWSSetupForRead() uncompressed := []byte("this is the uncompress data") wsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, uncompressed) rb := append([]byte(nil), wsmsg1...) // Call with some but not all of the payload bufs, err := c.wsRead(ri, tr, rb[:10]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // Call with the rest, only then should we get the uncompressed data. bufs, err = c.wsRead(ri, tr, rb[10:]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if !bytes.Equal(bufs[0], uncompressed) { t.Fatalf("Unexpected content: %s", bufs[0]) } // Stress the fact that we use a pool and want to make sure // that if we get a decompressor from the pool, it is properly reset // with the buffer to decompress. // Since we unmask the read buffer, reset it now and fill it // with 10 compressed frames. rb = nil for i := 0; i < 10; i++ { rb = append(rb, wsmsg1...) } bufs, err = c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 10 { t.Fatalf("Unexpected buffer returned: %v", n) } // Compress a message and send it in several frames. buf := &bytes.Buffer{} compressor, _ := flate.NewWriter(buf, 1) compressor.Write(uncompressed) compressor.Flush() compressed := buf.Bytes() // The last 4 bytes are dropped compressed = compressed[:len(compressed)-4] ncomp := 10 frag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, compressed[:ncomp]) frag1[0] |= wsRsv1Bit frag2 := testWSCreateClientMsg(wsBinaryMessage, 2, true, false, compressed[ncomp:]) rb = append([]byte(nil), frag1...) rb = append(rb, frag2...) bufs, err = c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if !bytes.Equal(bufs[0], uncompressed) { t.Fatalf("Unexpected content: %s", bufs[0]) } } func TestWSReadCompressedFrameCorrupted(t *testing.T) { c, ri, tr := testWSSetupForRead() uncompressed := []byte("this is the uncompress data") wsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, uncompressed) copy(wsmsg1[10:], []byte{1, 2, 3, 4}) rb := append([]byte(nil), wsmsg1...) bufs, err := c.wsRead(ri, tr, rb) if err == nil || !strings.Contains(err.Error(), "corrupt") { t.Fatalf("Expected error about corrupted data, got %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Expected no buffer, got %v", n) } } func TestWSReadVariousFrameSizes(t *testing.T) { for _, test := range []struct { name string size int }{ {"tiny", 100}, {"medium", 1000}, {"large", 70000}, } { t.Run(test.name, func(t *testing.T) { c, ri, tr := testWSSetupForRead() uncompressed := make([]byte, test.size) for i := 0; i < len(uncompressed); i++ { uncompressed[i] = 'A' + byte(i%26) } wsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, uncompressed) rb := append([]byte(nil), wsmsg1...) bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if !bytes.Equal(bufs[0], uncompressed) { t.Fatalf("Unexpected content: %s", bufs[0]) } }) } } func TestWSReadFragmentedFrames(t *testing.T) { c, ri, tr := testWSSetupForRead() payloads := []string{"first", "second", "third"} var rb []byte for i := 0; i < len(payloads); i++ { final := i == len(payloads)-1 frag := testWSCreateClientMsg(wsBinaryMessage, i+1, final, false, []byte(payloads[i])) rb = append(rb, frag...) } bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 3 { t.Fatalf("Unexpected buffer returned: %v", n) } for i, expected := range payloads { if string(bufs[i]) != expected { t.Fatalf("Unexpected content for buf=%v: %s", i, bufs[i]) } } } func TestWSReadPartialFrameHeaderAtEndOfReadBuffer(t *testing.T) { c, ri, tr := testWSSetupForRead() msg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("msg1")) msg2 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("msg2")) rb := append([]byte(nil), msg1...) rb = append(rb, msg2...) // We will pass the first frame + the first byte of the next frame. rbl := rb[:len(msg1)+1] // Make the io reader return the rest of the frame tr.buf = rb[len(msg1)+1:] bufs, err := c.wsRead(ri, tr, rbl) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } // We should not have asked to the io reader more than what is needed for reading // the frame header. Since we had already the first byte in the read buffer, // tr.pos should be 1(size)+4(key)=5 if tr.pos != 5 { t.Fatalf("Expected reader pos to be 5, got %v", tr.pos) } } func TestWSReadPingFrame(t *testing.T) { for _, test := range []struct { name string payload []byte }{ {"without payload", nil}, {"with payload", []byte("optional payload")}, } { t.Run(test.name, func(t *testing.T) { c, ri, tr := testWSSetupForRead() ping := testWSCreateClientMsg(wsPingMessage, 1, true, false, test.payload) rb := append([]byte(nil), ping...) bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // A PONG should have been queued with the payload of the ping c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n == 0 { t.Fatalf("Expected buffers, got %v", n) } if expected := 2 + len(test.payload); expected != len(nb[0]) { t.Fatalf("Expected buffer to be %v bytes long, got %v", expected, len(nb[0])) } b := nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsPongMessage) == 0 { t.Fatalf("Should have been a PONG, it wasn't: %v", b) } if len(test.payload) > 0 { if !bytes.Equal(nb[0][2:], test.payload) { t.Fatalf("Unexpected content: %s", nb[0][2:]) } } }) } } func TestWSReadPongFrame(t *testing.T) { for _, test := range []struct { name string payload []byte }{ {"without payload", nil}, {"with payload", []byte("optional payload")}, } { t.Run(test.name, func(t *testing.T) { c, ri, tr := testWSSetupForRead() pong := testWSCreateClientMsg(wsPongMessage, 1, true, false, test.payload) rb := append([]byte(nil), pong...) bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // Nothing should be sent... c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n != 0 { t.Fatalf("Expected no buffer, got %v", n) } }) } } func TestWSReadCloseFrame(t *testing.T) { for _, test := range []struct { name string noStatus bool payload []byte }{ {"status without payload", false, nil}, {"status with payload", false, []byte("optional payload")}, {"no status no payload", true, nil}, } { t.Run(test.name, func(t *testing.T) { c, ri, tr := testWSSetupForRead() var payload []byte if !test.noStatus { // a close message has a status in 2 bytes + optional payload payload = make([]byte, 2+len(test.payload)) binary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure) if len(test.payload) > 0 { copy(payload[2:], test.payload) } } close := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload) // Have a normal frame prior to close to make sure that wsRead returns // the normal frame along with io.EOF to indicate that wsCloseMessage was received. msg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("msg")) rb := append([]byte(nil), msg...) rb = append(rb, close...) bufs, err := c.wsRead(ri, tr, rb) // It is expected that wsRead returns io.EOF on processing a close. if err != io.EOF { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != "msg" { t.Fatalf("Unexpected content: %s", bufs[0]) } // A CLOSE should have been queued with the payload of the original close message. c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n == 0 { t.Fatalf("Expected buffers, got %v", n) } if test.noStatus { if expected := 2; expected != len(nb[0]) { t.Fatalf("Expected buffer to be %v bytes long, got %v", expected, len(nb[0])) } } else if expected := 2 + 2 + len(test.payload); expected != len(nb[0]) { t.Fatalf("Expected buffer to be %v bytes long, got %v", expected, len(nb[0])) } b := nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsCloseMessage) == 0 { t.Fatalf("Should have been a CLOSE, it wasn't: %v", b) } if !test.noStatus { if status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusNormalClosure { t.Fatalf("Expected status to be %v, got %v", wsCloseStatusNormalClosure, status) } if len(test.payload) > 0 { if !bytes.Equal(nb[0][4:], test.payload) { t.Fatalf("Unexpected content: %s", nb[0][4:]) } } } }) } } func TestWSReadControlFrameBetweebFragmentedFrames(t *testing.T) { c, ri, tr := testWSSetupForRead() frag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, []byte("first")) frag2 := testWSCreateClientMsg(wsBinaryMessage, 2, true, false, []byte("second")) ctrl := testWSCreateClientMsg(wsPongMessage, 1, true, false, nil) rb := append([]byte(nil), frag1...) rb = append(rb, ctrl...) rb = append(rb, frag2...) bufs, err := c.wsRead(ri, tr, rb) if err != nil { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 2 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != "first" { t.Fatalf("Unexpected content: %s", bufs[0]) } if string(bufs[1]) != "second" { t.Fatalf("Unexpected content: %s", bufs[1]) } } func TestWSCloseFrameWithPartialOrInvalid(t *testing.T) { c, ri, tr := testWSSetupForRead() // a close message has a status in 2 bytes + optional payload payloadTxt := []byte("hello") payload := make([]byte, 2+len(payloadTxt)) binary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure) copy(payload[2:], payloadTxt) closeMsg := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload) // We will pass to wsRead a buffer of small capacity that contains // only 1 byte. closeFirtByte := []byte{closeMsg[0]} // Make the io reader return the rest of the frame tr.buf = closeMsg[1:] bufs, err := c.wsRead(ri, tr, closeFirtByte[:]) // It is expected that wsRead returns io.EOF on processing a close. if err != io.EOF { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // A CLOSE should have been queued with the payload of the original close message. c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n == 0 { t.Fatalf("Expected buffers, got %v", n) } if expected := 2 + 2 + len(payloadTxt); expected != len(nb[0]) { t.Fatalf("Expected buffer to be %v bytes long, got %v", expected, len(nb[0])) } b := nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsCloseMessage) == 0 { t.Fatalf("Should have been a CLOSE, it wasn't: %v", b) } if status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusNormalClosure { t.Fatalf("Expected status to be %v, got %v", wsCloseStatusNormalClosure, status) } if !bytes.Equal(nb[0][4:], payloadTxt) { t.Fatalf("Unexpected content: %s", nb[0][4:]) } // Now test close with invalid status size (1 instead of 2 bytes) c, ri, tr = testWSSetupForRead() binary.BigEndian.PutUint16(payload, wsCloseStatusNormalClosure) closeMsg = testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload[:1]) // We will pass to wsRead a buffer of small capacity that contains // only 1 byte. closeFirtByte = []byte{closeMsg[0]} // Make the io reader return the rest of the frame tr.buf = closeMsg[1:] bufs, err = c.wsRead(ri, tr, closeFirtByte[:]) // It is expected that wsRead returns io.EOF on processing a close. if err != io.EOF { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // Since no status was received, the server will send a close frame without // status code nor payload. c.mu.Lock() nb, _ = c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n == 0 { t.Fatalf("Expected buffers, got %v", n) } if expected := 2; expected != len(nb[0]) { t.Fatalf("Expected buffer to be %v bytes long, got %v", expected, len(nb[0])) } b = nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsCloseMessage) == 0 { t.Fatalf("Should have been a CLOSE, it wasn't: %v", b) } } func TestWSReadGetErrors(t *testing.T) { tr := &testReader{err: fmt.Errorf("on purpose")} for _, test := range []struct { lenPayload int rbextra int }{ {10, 1}, {10, 3}, {200, 1}, {200, 2}, {200, 5}, {70000, 1}, {70000, 5}, {70000, 13}, } { t.Run("", func(t *testing.T) { c, ri, _ := testWSSetupForRead() msg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("msg")) frame := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, make([]byte, test.lenPayload)) rb := append([]byte(nil), msg...) rb = append(rb, frame...) bufs, err := c.wsRead(ri, tr, rb[:len(msg)+test.rbextra]) if err == nil || err.Error() != "on purpose" { t.Fatalf("Expected 'on purpose' error, got %v", err) } if n := len(bufs); n != 1 { t.Fatalf("Unexpected buffer returned: %v", n) } if string(bufs[0]) != "msg" { t.Fatalf("Unexpected content: %s", bufs[0]) } }) } } func TestWSHandleControlFrameErrors(t *testing.T) { c, ri, tr := testWSSetupForRead() tr.err = fmt.Errorf("on purpose") // a close message has a status in 2 bytes + optional payload text := []byte("this is a close message") payload := make([]byte, 2+len(text)) binary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure) copy(payload[2:], text) ctrl := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload) bufs, err := c.wsRead(ri, tr, ctrl[:len(ctrl)-4]) if err == nil || err.Error() != "on purpose" { t.Fatalf("Expected 'on purpose' error, got %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } // Alter the content of close message. It is supposed to be valid utf-8. c, ri, tr = testWSSetupForRead() cp := append([]byte(nil), payload...) cp[10] = 0xF1 ctrl = testWSCreateClientMsg(wsCloseMessage, 1, true, false, cp) bufs, err = c.wsRead(ri, tr, ctrl) // We should still receive an EOF but the message enqueued to the client // should contain wsCloseStatusInvalidPayloadData and the error about invalid utf8 if err != io.EOF { t.Fatalf("Unexpected error: %v", err) } if n := len(bufs); n != 0 { t.Fatalf("Unexpected buffer returned: %v", n) } c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n == 0 { t.Fatalf("Expected buffers, got %v", n) } b := nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsCloseMessage) == 0 { t.Fatalf("Should have been a CLOSE, it wasn't: %v", b) } if status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusInvalidPayloadData { t.Fatalf("Expected status to be %v, got %v", wsCloseStatusInvalidPayloadData, status) } if !bytes.Contains(nb[0][4:], []byte("utf8")) { t.Fatalf("Unexpected content: %s", nb[0][4:]) } } func TestWSReadErrors(t *testing.T) { for _, test := range []struct { cframe func() []byte err string nbufs int }{ { func() []byte { msg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("hello")) msg[1] &= ^byte(wsMaskBit) return msg }, "mask bit missing", 1, }, { func() []byte { return testWSCreateClientMsg(wsPingMessage, 1, true, false, make([]byte, 200)) }, "control frame length bigger than maximum allowed", 1, }, { func() []byte { return testWSCreateClientMsg(wsPingMessage, 1, false, false, []byte("hello")) }, "control frame does not have final bit set", 1, }, { func() []byte { frag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, []byte("frag1")) newMsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("new message")) all := append([]byte(nil), frag1...) all = append(all, newMsg...) return all }, "new message started before final frame for previous message was received", 2, }, { func() []byte { frame := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("frame")) frag := testWSCreateClientMsg(wsBinaryMessage, 2, false, false, []byte("continuation")) all := append([]byte(nil), frame...) all = append(all, frag...) return all }, "invalid continuation frame", 2, }, { func() []byte { return testWSCreateClientMsg(wsBinaryMessage, 2, false, true, []byte("frame")) }, "invalid continuation frame", 1, }, { func() []byte { return testWSCreateClientMsg(99, 1, false, false, []byte("hello")) }, "unknown opcode", 1, }, } { t.Run(test.err, func(t *testing.T) { c, ri, tr := testWSSetupForRead() // Add a valid message first msg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("hello")) // Then add the bad frame bad := test.cframe() // Add them both to a read buffer rb := append([]byte(nil), msg...) rb = append(rb, bad...) bufs, err := c.wsRead(ri, tr, rb) if err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Expected error to contain %q, got %q", test.err, err.Error()) } if n := len(bufs); n != test.nbufs { t.Fatalf("Unexpected number of buffers: %v", n) } if string(bufs[0]) != "hello" { t.Fatalf("Unexpected content: %s", bufs[0]) } }) } } func TestWSEnqueueCloseMsg(t *testing.T) { for _, test := range []struct { reason ClosedState status int }{ {ClientClosed, wsCloseStatusNormalClosure}, {AuthenticationTimeout, wsCloseStatusPolicyViolation}, {AuthenticationViolation, wsCloseStatusPolicyViolation}, {SlowConsumerPendingBytes, wsCloseStatusPolicyViolation}, {SlowConsumerWriteDeadline, wsCloseStatusPolicyViolation}, {MaxAccountConnectionsExceeded, wsCloseStatusPolicyViolation}, {MaxConnectionsExceeded, wsCloseStatusPolicyViolation}, {MaxControlLineExceeded, wsCloseStatusPolicyViolation}, {MaxSubscriptionsExceeded, wsCloseStatusPolicyViolation}, {MissingAccount, wsCloseStatusPolicyViolation}, {AuthenticationExpired, wsCloseStatusPolicyViolation}, {Revocation, wsCloseStatusPolicyViolation}, {TLSHandshakeError, wsCloseStatusTLSHandshake}, {ParseError, wsCloseStatusProtocolError}, {ProtocolViolation, wsCloseStatusProtocolError}, {BadClientProtocolVersion, wsCloseStatusProtocolError}, {MaxPayloadExceeded, wsCloseStatusMessageTooBig}, {ServerShutdown, wsCloseStatusGoingAway}, {WriteError, wsCloseStatusGoingAway}, {ReadError, wsCloseStatusGoingAway}, {StaleConnection, wsCloseStatusGoingAway}, {ClosedState(254), wsCloseStatusInternalSrvError}, } { t.Run(test.reason.String(), func(t *testing.T) { c, _, _ := testWSSetupForRead() c.wsEnqueueCloseMessage(test.reason) c.mu.Lock() nb, _ := c.collapsePtoNB() c.mu.Unlock() if n := len(nb); n != 1 { t.Fatalf("Expected 1 buffer, got %v", n) } b := nb[0][0] if b&wsFinalBit == 0 { t.Fatalf("Control frame should have been the final flag, it was not set: %v", b) } if b&byte(wsCloseMessage) == 0 { t.Fatalf("Should have been a CLOSE, it wasn't: %v", b) } if status := binary.BigEndian.Uint16(nb[0][2:4]); int(status) != test.status { t.Fatalf("Expected status to be %v, got %v", test.status, status) } if string(nb[0][4:]) != test.reason.String() { t.Fatalf("Unexpected content: %s", nb[0][4:]) } }) } } type testResponseWriter struct { http.ResponseWriter buf bytes.Buffer headers http.Header err error brw *bufio.ReadWriter conn *testWSFakeNetConn } func (trw *testResponseWriter) Write(p []byte) (int, error) { return trw.buf.Write(p) } func (trw *testResponseWriter) WriteHeader(status int) { trw.buf.WriteString(fmt.Sprintf("%v", status)) } func (trw *testResponseWriter) Header() http.Header { if trw.headers == nil { trw.headers = make(http.Header) } return trw.headers } type testWSFakeNetConn struct { net.Conn wbuf bytes.Buffer err error wsOpened bool isClosed bool deadlineCleared bool } func (c *testWSFakeNetConn) Write(p []byte) (int, error) { if c.err != nil { return 0, c.err } return c.wbuf.Write(p) } func (c *testWSFakeNetConn) SetDeadline(t time.Time) error { if t.IsZero() { c.deadlineCleared = true } return nil } func (c *testWSFakeNetConn) Close() error { c.isClosed = true return nil } func (trw *testResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if trw.conn == nil { trw.conn = &testWSFakeNetConn{} } trw.conn.wsOpened = true if trw.brw == nil { trw.brw = bufio.NewReadWriter(bufio.NewReader(trw.conn), bufio.NewWriter(trw.conn)) } return trw.conn, trw.brw, trw.err } func testWSOptions() *Options { opts := DefaultOptions() opts.DisableShortFirstPing = true opts.Websocket.Host = "127.0.0.1" opts.Websocket.Port = -1 opts.NoSystemAccount = true var err error tc := &TLSConfigOpts{ CertFile: "./configs/certs/server.pem", KeyFile: "./configs/certs/key.pem", } opts.Websocket.TLSConfig, err = GenTLSConfig(tc) if err != nil { panic(err) } return opts } func testWSCreateValidReq() *http.Request { req := &http.Request{ Method: "GET", Host: "localhost", Proto: "HTTP/1.1", } req.Header = make(http.Header) req.Header.Set("Upgrade", "websocket") req.Header.Set("Connection", "Upgrade") req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") req.Header.Set("Sec-Websocket-Version", "13") return req } func TestWSCheckOrigin(t *testing.T) { notSameOrigin := false sameOrigin := true allowedListEmpty := []string{} someList := []string{"http://host1.com", "http://host2.com:1234"} for _, test := range []struct { name string sameOrigin bool origins []string reqHost string reqTLS bool origin string err string }{ {"any", notSameOrigin, allowedListEmpty, "", false, "http://any.host.com", ""}, {"same origin ok", sameOrigin, allowedListEmpty, "host.com", false, "http://host.com:80", ""}, {"same origin bad host", sameOrigin, allowedListEmpty, "host.com", false, "http://other.host.com", "not same origin"}, {"same origin bad port", sameOrigin, allowedListEmpty, "host.com", false, "http://host.com:81", "not same origin"}, {"same origin bad scheme", sameOrigin, allowedListEmpty, "host.com", true, "http://host.com", "not same origin"}, {"same origin bad uri", sameOrigin, allowedListEmpty, "host.com", false, "@@@://invalid:url:1234", "invalid URI"}, {"same origin bad url", sameOrigin, allowedListEmpty, "host.com", false, "http://invalid:url:1234", "too many colons"}, {"same origin bad req host", sameOrigin, allowedListEmpty, "invalid:url:1234", false, "http://host.com", "too many colons"}, {"no origin same origin ignored", sameOrigin, allowedListEmpty, "", false, "", ""}, {"no origin list ignored", sameOrigin, someList, "", false, "", ""}, {"no origin same origin and list ignored", sameOrigin, someList, "", false, "", ""}, {"allowed from list", notSameOrigin, someList, "", false, "http://host2.com:1234", ""}, {"allowed with different path", notSameOrigin, someList, "", false, "http://host1.com/some/path", ""}, {"list bad port", notSameOrigin, someList, "", false, "http://host1.com:1234", "not in the allowed list"}, {"list bad scheme", notSameOrigin, someList, "", false, "https://host2.com:1234", "not in the allowed list"}, } { t.Run(test.name, func(t *testing.T) { opts := DefaultOptions() opts.Websocket.SameOrigin = test.sameOrigin opts.Websocket.AllowedOrigins = test.origins s := &Server{opts: opts} s.wsSetOriginOptions(&opts.Websocket) req := testWSCreateValidReq() req.Host = test.reqHost if test.reqTLS { req.TLS = &tls.ConnectionState{} } if test.origin != "" { req.Header.Set("Origin", test.origin) } err := s.websocket.checkOrigin(req) if test.err == "" && err != nil { t.Fatalf("Unexpected error: %v", err) } else if test.err != "" && (err == nil || !strings.Contains(err.Error(), test.err)) { t.Fatalf("Expected error %q, got %v", test.err, err) } }) } } func TestWSUpgradeValidationErrors(t *testing.T) { for _, test := range []struct { name string setup func() (*Options, *testResponseWriter, *http.Request) err string status int }{ { "bad method", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Method = "POST" return opts, nil, req }, "must be GET", http.StatusMethodNotAllowed, }, { "no host", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Host = "" return opts, nil, req }, "'Host' missing in request", http.StatusBadRequest, }, { "invalid upgrade header", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Del("Upgrade") return opts, nil, req }, "invalid value for header 'Upgrade'", http.StatusBadRequest, }, { "invalid connection header", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Del("Connection") return opts, nil, req }, "invalid value for header 'Connection'", http.StatusBadRequest, }, { "no key", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Del("Sec-Websocket-Key") return opts, nil, req }, "key missing", http.StatusBadRequest, }, { "empty key", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Set("Sec-Websocket-Key", "") return opts, nil, req }, "key missing", http.StatusBadRequest, }, { "missing version", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Del("Sec-Websocket-Version") return opts, nil, req }, "invalid version", http.StatusBadRequest, }, { "wrong version", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() req := testWSCreateValidReq() req.Header.Set("Sec-Websocket-Version", "99") return opts, nil, req }, "invalid version", http.StatusBadRequest, }, { "origin", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() opts.Websocket.SameOrigin = true req := testWSCreateValidReq() req.Header.Set("Origin", "http://bad.host.com") return opts, nil, req }, "origin not allowed", http.StatusForbidden, }, { "hijack error", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() rw := &testResponseWriter{err: fmt.Errorf("on purpose")} req := testWSCreateValidReq() return opts, rw, req }, "on purpose", http.StatusInternalServerError, }, { "hijack buffered data", func() (*Options, *testResponseWriter, *http.Request) { opts := testWSOptions() buf := &bytes.Buffer{} buf.WriteString("some data") rw := &testResponseWriter{ conn: &testWSFakeNetConn{}, brw: bufio.NewReadWriter(bufio.NewReader(buf), bufio.NewWriter(nil)), } tmp := [1]byte{} io.ReadAtLeast(rw.brw, tmp[:1], 1) req := testWSCreateValidReq() return opts, rw, req }, "client sent data before handshake is complete", http.StatusBadRequest, }, } { t.Run(test.name, func(t *testing.T) { opts, rw, req := test.setup() if rw == nil { rw = &testResponseWriter{} } s := &Server{opts: opts} s.wsSetOriginOptions(&opts.Websocket) res, err := s.wsUpgrade(rw, req) if err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("Should get error %q, got %v", test.err, err) } if res != nil { t.Fatalf("Should not have returned a result, got %v", res) } expected := fmt.Sprintf("%v%s\n", test.status, http.StatusText(test.status)) if got := rw.buf.String(); got != expected { t.Fatalf("Expected %q got %q", expected, got) } // Check that if the connection was opened, it is now closed. if rw.conn != nil && rw.conn.wsOpened && !rw.conn.isClosed { t.Fatal("Connection was opened, but has not been closed") } }) } } func TestWSUpgradeResponseWriteError(t *testing.T) { opts := testWSOptions() s := &Server{opts: opts} expectedErr := errors.New("on purpose") rw := &testResponseWriter{ conn: &testWSFakeNetConn{err: expectedErr}, } req := testWSCreateValidReq() res, err := s.wsUpgrade(rw, req) if err != expectedErr { t.Fatalf("Should get error %q, got %v", expectedErr.Error(), err) } if res != nil { t.Fatalf("Should not have returned a result, got %v", res) } if !rw.conn.isClosed { t.Fatal("Connection should have been closed") } } func TestWSUpgradeConnDeadline(t *testing.T) { opts := testWSOptions() opts.Websocket.HandshakeTimeout = time.Second s := &Server{opts: opts} rw := &testResponseWriter{} req := testWSCreateValidReq() res, err := s.wsUpgrade(rw, req) if res == nil || err != nil { t.Fatalf("Unexpected error: %v", err) } if rw.conn.isClosed { t.Fatal("Connection should NOT have been closed") } if !rw.conn.deadlineCleared { t.Fatal("Connection deadline should have been cleared after handshake") } } func TestWSCompressNegotiation(t *testing.T) { // No compression on the server, but client asks opts := testWSOptions() s := &Server{opts: opts} rw := &testResponseWriter{} req := testWSCreateValidReq() req.Header.Set("Sec-Websocket-Extensions", "permessage-deflate") res, err := s.wsUpgrade(rw, req) if res == nil || err != nil { t.Fatalf("Unexpected error: %v", err) } // The http response should not contain "permessage-deflate" output := rw.conn.wbuf.String() if strings.Contains(output, "permessage-deflate") { t.Fatalf("Compression disabled in server so response to client should not contain extension, got %s", output) } // Option in the server and client, so compression should be negotiated. s.opts.Websocket.Compression = true rw = &testResponseWriter{} res, err = s.wsUpgrade(rw, req) if res == nil || err != nil { t.Fatalf("Unexpected error: %v", err) } // The http response should not contain "permessage-deflate" output = rw.conn.wbuf.String() if !strings.Contains(output, "permessage-deflate") { t.Fatalf("Compression in server and client request, so response should contain extension, got %s", output) } // Option in server but not asked by the client, so response should not contain "permessage-deflate" rw = &testResponseWriter{} req.Header.Del("Sec-Websocket-Extensions") res, err = s.wsUpgrade(rw, req) if res == nil || err != nil { t.Fatalf("Unexpected error: %v", err) } // The http response should not contain "permessage-deflate" output = rw.conn.wbuf.String() if strings.Contains(output, "permessage-deflate") { t.Fatalf("Compression in server but not in client, so response to client should not contain extension, got %s", output) } } func TestWSParseOptions(t *testing.T) { for _, test := range []struct { name string content string checkOpt func(*WebsocketOpts) error err string }{ // Negative tests {"bad type", "websocket: []", nil, "to be a map"}, {"bad listen", "websocket: { listen: [] }", nil, "port or host:port"}, {"bad port", `websocket: { port: "abc" }`, nil, "not int64"}, {"bad host", `websocket: { host: 123 }`, nil, "not string"}, {"bad advertise type", `websocket: { advertise: 123 }`, nil, "not string"}, {"bad tls", `websocket: { tls: 123 }`, nil, "not map[string]interface {}"}, {"bad same origin", `websocket: { same_origin: "abc" }`, nil, "not bool"}, {"bad allowed origins type", `websocket: { allowed_origins: {} }`, nil, "unsupported type"}, {"bad allowed origins values", `websocket: { allowed_origins: [ {} ] }`, nil, "unsupported type in array"}, {"bad handshake timeout type", `websocket: { handshake_timeout: [] }`, nil, "unsupported type"}, {"bad handshake timeout duration", `websocket: { handshake_timeout: "abc" }`, nil, "invalid duration"}, {"unknown field", `websocket: { this_does_not_exist: 123 }`, nil, "unknown"}, // Positive tests {"listen port only", `websocket { listen: 1234 }`, func(wo *WebsocketOpts) error { if wo.Port != 1234 { return fmt.Errorf("expected 1234, got %v", wo.Port) } return nil }, ""}, {"listen host and port", `websocket { listen: "localhost:1234" }`, func(wo *WebsocketOpts) error { if wo.Host != "localhost" || wo.Port != 1234 { return fmt.Errorf("expected localhost:1234, got %v:%v", wo.Host, wo.Port) } return nil }, ""}, {"host", `websocket { host: "localhost" }`, func(wo *WebsocketOpts) error { if wo.Host != "localhost" { return fmt.Errorf("expected localhost, got %v", wo.Host) } return nil }, ""}, {"port", `websocket { port: 1234 }`, func(wo *WebsocketOpts) error { if wo.Port != 1234 { return fmt.Errorf("expected 1234, got %v", wo.Port) } return nil }, ""}, {"advertise", `websocket { advertise: "host:1234" }`, func(wo *WebsocketOpts) error { if wo.Advertise != "host:1234" { return fmt.Errorf("expected %q, got %q", "host:1234", wo.Advertise) } return nil }, ""}, {"same origin", `websocket { same_origin: true }`, func(wo *WebsocketOpts) error { if !wo.SameOrigin { return fmt.Errorf("expected same_origin==true, got %v", wo.SameOrigin) } return nil }, ""}, {"allowed origins one only", `websocket { allowed_origins: "https://host.com/" }`, func(wo *WebsocketOpts) error { expected := []string{"https://host.com/"} if !reflect.DeepEqual(wo.AllowedOrigins, expected) { return fmt.Errorf("expected allowed origins to be %q, got %q", expected, wo.AllowedOrigins) } return nil }, ""}, {"allowed origins array", ` websocket { allowed_origins: [ "https://host1.com/" "https://host2.com/" ] } `, func(wo *WebsocketOpts) error { expected := []string{"https://host1.com/", "https://host2.com/"} if !reflect.DeepEqual(wo.AllowedOrigins, expected) { return fmt.Errorf("expected allowed origins to be %q, got %q", expected, wo.AllowedOrigins) } return nil }, ""}, {"handshake timeout in whole seconds", `websocket { handshake_timeout: 3 }`, func(wo *WebsocketOpts) error { if wo.HandshakeTimeout != 3*time.Second { return fmt.Errorf("expected handshake to be 3s, got %v", wo.HandshakeTimeout) } return nil }, ""}, {"handshake timeout n duration", `websocket { handshake_timeout: "4s" }`, func(wo *WebsocketOpts) error { if wo.HandshakeTimeout != 4*time.Second { return fmt.Errorf("expected handshake to be 4s, got %v", wo.HandshakeTimeout) } return nil }, ""}, {"tls config", ` websocket { tls { cert_file: "./configs/certs/server.pem" key_file: "./configs/certs/key.pem" } } `, func(wo *WebsocketOpts) error { if wo.TLSConfig == nil { return fmt.Errorf("TLSConfig should have been set") } return nil }, ""}, {"compression", ` websocket { compression: true } `, func(wo *WebsocketOpts) error { if !wo.Compression { return fmt.Errorf("Compression should have been set") } return nil }, ""}, {"jwt cookie", ` websocket { jwt_cookie: "jwtcookie" } `, func(wo *WebsocketOpts) error { if wo.JWTCookie != "jwtcookie" { return fmt.Errorf("Invalid JWTCookie value: %q", wo.JWTCookie) } return nil }, ""}, {"no auth user", ` websocket { no_auth_user: "noauthuser" } `, func(wo *WebsocketOpts) error { if wo.NoAuthUser != "noauthuser" { return fmt.Errorf("Invalid NoAuthUser value: %q", wo.NoAuthUser) } return nil }, ""}, {"auth block", ` websocket { authorization { user: "webuser" password: "pwd" token: "token" timeout: 2.0 } } `, func(wo *WebsocketOpts) error { if wo.Username != "webuser" || wo.Password != "pwd" || wo.Token != "token" || wo.AuthTimeout != 2.0 { return fmt.Errorf("Invalid auth block: %+v", wo) } return nil }, ""}, {"auth timeout as int", ` websocket { authorization { timeout: 2 } } `, func(wo *WebsocketOpts) error { if wo.AuthTimeout != 2.0 { return fmt.Errorf("Invalid auth timeout: %v", wo.AuthTimeout) } return nil }, ""}, } { t.Run(test.name, func(t *testing.T) { conf := createConfFile(t, []byte(test.content)) o, err := ProcessConfigFile(conf) if test.err != _EMPTY_ { if err == nil || !strings.Contains(err.Error(), test.err) { t.Fatalf("For content: %q, expected error about %q, got %v", test.content, test.err, err) } return } else if err != nil { t.Fatalf("Unexpected error for content %q: %v", test.content, err) } if err := test.checkOpt(&o.Websocket); err != nil { t.Fatalf("Incorrect option for content %q: %v", test.content, err.Error()) } }) } } func TestWSValidateOptions(t *testing.T) { nwso := DefaultOptions() wso := testWSOptions() for _, test := range []struct { name string getOpts func() *Options err string }{ {"websocket disabled", func() *Options { return nwso.Clone() }, ""}, {"no tls", func() *Options { o := wso.Clone(); o.Websocket.TLSConfig = nil; return o }, "requires TLS configuration"}, {"bad url in allowed list", func() *Options { o := wso.Clone() o.Websocket.AllowedOrigins = []string{"http://this:is:bad:url"} return o }, "unable to parse"}, {"missing trusted configuration", func() *Options { o := wso.Clone() o.Websocket.JWTCookie = "jwt" return o }, "keys configuration is required"}, {"websocket username not allowed if users specified", func() *Options { o := wso.Clone() o.Nkeys = []*NkeyUser{{Nkey: "abc"}} o.Websocket.Username = "b" o.Websocket.Password = "pwd" return o }, "websocket authentication username not compatible with presence of users/nkeys"}, {"websocket token not allowed if users specified", func() *Options { o := wso.Clone() o.Nkeys = []*NkeyUser{{Nkey: "abc"}} o.Websocket.Token = "mytoken" return o }, "websocket authentication token not compatible with presence of users/nkeys"}, } { t.Run(test.name, func(t *testing.T) { err := validateWebsocketOptions(test.getOpts()) if test.err == "" && err != nil { t.Fatalf("Unexpected error: %v", err) } else if test.err != "" && (err == nil || !strings.Contains(err.Error(), test.err)) { t.Fatalf("Expected error to contain %q, got %v", test.err, err) } }) } } func TestWSSetOriginOptions(t *testing.T) { o := testWSOptions() for _, test := range []struct { content string err string }{ {"@@@://host.com/", "invalid URI"}, {"http://this:is:bad:url/", "invalid port"}, } { t.Run(test.err, func(t *testing.T) { o.Websocket.AllowedOrigins = []string{test.content} s := &Server{} l := &captureErrorLogger{errCh: make(chan string, 1)} s.SetLogger(l, false, false) s.wsSetOriginOptions(&o.Websocket) select { case e := <-l.errCh: if !strings.Contains(e, test.err) { t.Fatalf("Unexpected error: %v", e) } case <-time.After(50 * time.Millisecond): t.Fatalf("Did not get the error") } }) } } type captureFatalLogger struct { DummyLogger fatalCh chan string } func (l *captureFatalLogger) Fatalf(format string, v ...any) { select { case l.fatalCh <- fmt.Sprintf(format, v...): default: } } func TestWSFailureToStartServer(t *testing.T) { // Create a listener to use a port l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error listening: %v", err) } defer l.Close() o := testWSOptions() // Make sure we don't have unnecessary listen ports opened. o.HTTPPort = 0 o.Cluster.Port = 0 o.Gateway.Name = "" o.Gateway.Port = 0 o.LeafNode.Port = 0 o.Websocket.Port = l.Addr().(*net.TCPAddr).Port s, err := NewServer(o) if err != nil { t.Fatalf("Error creating server: %v", err) } defer s.Shutdown() logger := &captureFatalLogger{fatalCh: make(chan string, 1)} s.SetLogger(logger, false, false) wg := sync.WaitGroup{} wg.Add(1) go func() { s.Start() wg.Done() }() select { case e := <-logger.fatalCh: if !strings.Contains(e, "Unable to listen") { t.Fatalf("Unexpected error: %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Should have reported a fatal error") } // Since this is a test and the process does not actually // exit on Fatal error, wait for the client port to be // ready so when we shutdown we don't leave the accept // loop hanging. checkFor(t, time.Second, 15*time.Millisecond, func() error { s.mu.Lock() ready := s.listener != nil s.mu.Unlock() if !ready { return fmt.Errorf("client accept loop not started yet") } return nil }) s.Shutdown() wg.Wait() } func TestWSAbnormalFailureOfWebServer(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() logger := &captureFatalLogger{fatalCh: make(chan string, 1)} s.SetLogger(logger, false, false) // Now close the WS listener to cause a WebServer error s.mu.Lock() s.websocket.listener.Close() s.mu.Unlock() select { case e := <-logger.fatalCh: if !strings.Contains(e, "websocket listener error") { t.Fatalf("Unexpected error: %v", e) } case <-time.After(2 * time.Second): t.Fatalf("Should have reported a fatal error") } } type testWSClientOptions struct { compress, web bool host string port int extraHeaders map[string][]string noTLS bool path string } func testNewWSClient(t testing.TB, o testWSClientOptions) (net.Conn, *bufio.Reader, []byte) { t.Helper() c, br, info, err := testNewWSClientWithError(t, o) if err != nil { t.Fatal(err) } return c, br, info } func testNewWSClientWithError(t testing.TB, o testWSClientOptions) (net.Conn, *bufio.Reader, []byte, error) { addr := fmt.Sprintf("%s:%d", o.host, o.port) wsc, err := net.Dial("tcp", addr) if err != nil { return nil, nil, nil, fmt.Errorf("Error creating ws connection: %v", err) } if !o.noTLS { wsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true}) wsc.SetDeadline(time.Now().Add(time.Second)) if err := wsc.(*tls.Conn).Handshake(); err != nil { return nil, nil, nil, fmt.Errorf("Error during handshake: %v", err) } wsc.SetDeadline(time.Time{}) } req := testWSCreateValidReq() if o.compress { req.Header.Set("Sec-Websocket-Extensions", "permessage-deflate") } if o.web { req.Header.Set("User-Agent", "Mozilla/5.0") } if len(o.extraHeaders) > 0 { for hdr, values := range o.extraHeaders { if len(values) == 0 { req.Header.Set(hdr, _EMPTY_) continue } req.Header.Set(hdr, values[0]) for i := 1; i < len(values); i++ { req.Header.Add(hdr, values[i]) } } } req.URL, _ = url.Parse("wss://" + addr + o.path) if err := req.Write(wsc); err != nil { return nil, nil, nil, fmt.Errorf("Error sending request: %v", err) } br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if err != nil { return nil, nil, nil, fmt.Errorf("Error reading response: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusSwitchingProtocols { return nil, nil, nil, fmt.Errorf("Expected response status %v, got %v", http.StatusSwitchingProtocols, resp.StatusCode) } var info []byte if o.path == mqttWSPath { if v := resp.Header[wsSecProto]; len(v) != 1 || v[0] != wsMQTTSecProtoVal { return nil, nil, nil, fmt.Errorf("No mqtt protocol in header: %v", resp.Header) } } else { // Wait for the INFO info = testWSReadFrame(t, br) if !bytes.HasPrefix(info, []byte("INFO {")) { return nil, nil, nil, fmt.Errorf("Expected INFO, got %s", info) } } return wsc, br, info, nil } type testClaimsOptions struct { nac *jwt.AccountClaims nuc *jwt.UserClaims connectRequest any dontSign bool expectAnswer string } func testWSWithClaims(t *testing.T, s *Server, o testWSClientOptions, tclm testClaimsOptions) (kp nkeys.KeyPair, conn net.Conn, rdr *bufio.Reader, auth_was_required bool) { t.Helper() okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() if tclm.nac == nil { tclm.nac = jwt.NewAccountClaims(apub) } else { tclm.nac.Subject = apub } ajwt, err := tclm.nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() if tclm.nuc == nil { tclm.nuc = jwt.NewUserClaims(pub) } else { tclm.nuc.Subject = pub } jwt, err := tclm.nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } addAccountToMemResolver(s, apub, ajwt) c, cr, l := testNewWSClient(t, o) var info struct { Nonce string `json:"nonce,omitempty"` AuthRequired bool `json:"auth_required,omitempty"` } if err := json.Unmarshal([]byte(l[5:]), &info); err != nil { t.Fatal(err) } if info.AuthRequired { cs := "" if tclm.connectRequest != nil { customReq, err := json.Marshal(tclm.connectRequest) if err != nil { t.Fatal(err) } // PING needed to flush the +OK/-ERR to us. cs = fmt.Sprintf("CONNECT %v\r\nPING\r\n", string(customReq)) } else if !tclm.dontSign { // Sign Nonce sigraw, _ := nkp.Sign([]byte(info.Nonce)) sig := base64.RawURLEncoding.EncodeToString(sigraw) cs = fmt.Sprintf("CONNECT {\"jwt\":%q,\"sig\":\"%s\",\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt, sig) } else { cs = fmt.Sprintf("CONNECT {\"jwt\":%q,\"verbose\":true,\"pedantic\":true}\r\nPING\r\n", jwt) } wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(cs)) c.Write(wsmsg) l = testWSReadFrame(t, cr) if !strings.HasPrefix(string(l), tclm.expectAnswer) { t.Fatalf("Expected %q, got %q", tclm.expectAnswer, l) } } return akp, c, cr, info.AuthRequired } func setupAddTrusted(o *Options) { kp, _ := nkeys.FromSeed(oSeed) pub, _ := kp.PublicKey() o.TrustedKeys = []string{pub} } func setupAddCookie(o *Options) { o.Websocket.JWTCookie = "jwt" } func testWSCreateClientGetInfo(t testing.TB, compress, web bool, host string, port int) (net.Conn, *bufio.Reader, []byte) { t.Helper() return testNewWSClient(t, testWSClientOptions{ compress: compress, web: web, host: host, port: port, }) } func testWSCreateClient(t testing.TB, compress, web bool, host string, port int) (net.Conn, *bufio.Reader) { wsc, br, _ := testWSCreateClientGetInfo(t, compress, web, host, port) // Send CONNECT and PING wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, []byte("CONNECT {\"verbose\":false,\"protocol\":1}\r\nPING\r\n")) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } // Wait for the PONG if msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected PONG, got %s", msg) } return wsc, br } func testWSReadFrame(t testing.TB, br *bufio.Reader) []byte { t.Helper() fh := [2]byte{} if _, err := io.ReadAtLeast(br, fh[:2], 2); err != nil { t.Fatalf("Error reading frame: %v", err) } fc := fh[0]&wsRsv1Bit != 0 sb := fh[1] size := 0 switch { case sb <= 125: size = int(sb) case sb == 126: tmp := [2]byte{} if _, err := io.ReadAtLeast(br, tmp[:2], 2); err != nil { t.Fatalf("Error reading frame: %v", err) } size = int(binary.BigEndian.Uint16(tmp[:2])) case sb == 127: tmp := [8]byte{} if _, err := io.ReadAtLeast(br, tmp[:8], 8); err != nil { t.Fatalf("Error reading frame: %v", err) } size = int(binary.BigEndian.Uint64(tmp[:8])) } buf := make([]byte, size) if _, err := io.ReadAtLeast(br, buf, size); err != nil { t.Fatalf("Error reading frame: %v", err) } if !fc { return buf } buf = append(buf, 0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff) dbr := bytes.NewBuffer(buf) d := flate.NewReader(dbr) uncompressed, err := io.ReadAll(d) if err != nil { t.Fatalf("Error reading frame: %v", err) } return uncompressed } func TestWSPubSub(t *testing.T) { for _, test := range []struct { name string compression bool }{ {"no compression", false}, {"compression", true}, } { t.Run(test.name, func(t *testing.T) { o := testWSOptions() if test.compression { o.Websocket.Compression = true } s := RunServer(o) defer s.Shutdown() // Create a regular client to subscribe nc := natsConnect(t, s.ClientURL()) defer nc.Close() nsub := natsSubSync(t, nc, "foo") checkExpectedSubs(t, 1, s) // Now create a WS client and send a message on "foo" wsc, br := testWSCreateClient(t, test.compression, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() // Send a WS message for "PUB foo 2\r\nok\r\n" wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("PUB foo 7\r\nfrom ws\r\n")) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } // Now check that message is received msg := natsNexMsg(t, nsub, time.Second) if string(msg.Data) != "from ws" { t.Fatalf("Expected message to be %q, got %q", "ok", string(msg.Data)) } // Now do reverse, create a subscription on WS client on bar wsmsg = testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("SUB bar 1\r\n")) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending subscription: %v", err) } // Wait for it to be registered on server checkExpectedSubs(t, 2, s) // Now publish from NATS connection and verify received on WS client natsPub(t, nc, "bar", []byte("from nats")) natsFlush(t, nc) // Check for the "from nats" message... // Set some deadline so we are not stuck forever on failure wsc.SetReadDeadline(time.Now().Add(10 * time.Second)) ok := 0 for { line, _, err := br.ReadLine() if err != nil { t.Fatalf("Error reading: %v", err) } // Note that this works even in compression test because those // texts are likely not to be compressed, but compression code is // still executed. if ok == 0 && bytes.Contains(line, []byte("MSG bar 1 9")) { ok = 1 continue } else if ok == 1 && bytes.Contains(line, []byte("from nats")) { break } } }) } } func TestWSTLSConnection(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() addr := fmt.Sprintf("%s:%d", o.Websocket.Host, o.Websocket.Port) for _, test := range []struct { name string useTLS bool status int }{ {"client uses TLS", true, http.StatusSwitchingProtocols}, {"client does not use TLS", false, http.StatusBadRequest}, } { t.Run(test.name, func(t *testing.T) { wsc, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer wsc.Close() if test.useTLS { wsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true}) if err := wsc.(*tls.Conn).Handshake(); err != nil { t.Fatalf("Error during handshake: %v", err) } } req := testWSCreateValidReq() var scheme string if test.useTLS { scheme = "s" } req.URL, _ = url.Parse("ws" + scheme + "://" + addr) if err := req.Write(wsc); err != nil { t.Fatalf("Error sending request: %v", err) } br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if err != nil { t.Fatalf("Error reading response: %v", err) } defer resp.Body.Close() if resp.StatusCode != test.status { t.Fatalf("Expected status %v, got %v", test.status, resp.StatusCode) } }) } } func TestWSTLSVerifyClientCert(t *testing.T) { o := testWSOptions() tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/server-cert.pem", KeyFile: "../test/configs/certs/server-key.pem", CaFile: "../test/configs/certs/ca.pem", Verify: true, } tlsc, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating tls config: %v", err) } o.Websocket.TLSConfig = tlsc s := RunServer(o) defer s.Shutdown() addr := fmt.Sprintf("%s:%d", o.Websocket.Host, o.Websocket.Port) for _, test := range []struct { name string provideCert bool }{ {"client provides cert", true}, {"client does not provide cert", false}, } { t.Run(test.name, func(t *testing.T) { wsc, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer wsc.Close() tlsc := &tls.Config{} if test.provideCert { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/client-cert.pem", KeyFile: "../test/configs/certs/client-key.pem", } var err error tlsc, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } } tlsc.InsecureSkipVerify = true wsc = tls.Client(wsc, tlsc) if err := wsc.(*tls.Conn).Handshake(); err != nil { t.Fatalf("Error during handshake: %v", err) } req := testWSCreateValidReq() req.URL, _ = url.Parse("wss://" + addr) if err := req.Write(wsc); err != nil { t.Fatalf("Error sending request: %v", err) } br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if resp != nil { resp.Body.Close() } if !test.provideCert { if err == nil { t.Fatal("Expected error, did not get one") } else if !strings.Contains(err.Error(), "bad certificate") && !strings.Contains(err.Error(), "certificate required") { t.Fatalf("Unexpected error: %v", err) } return } if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.StatusCode != http.StatusSwitchingProtocols { t.Fatalf("Expected status %v, got %v", http.StatusSwitchingProtocols, resp.StatusCode) } }) } } func testCreateAllowedConnectionTypes(list []string) map[string]struct{} { if len(list) == 0 { return nil } m := make(map[string]struct{}, len(list)) for _, l := range list { m[l] = struct{}{} } return m } func TestWSTLSVerifyAndMap(t *testing.T) { accName := "MyAccount" acc := NewAccount(accName) certUserName := "CN=example.com,OU=NATS.io" users := []*User{{Username: certUserName, Account: acc}} for _, test := range []struct { name string filtering bool provideCert bool }{ {"no filtering, client provides cert", false, true}, {"no filtering, client does not provide cert", false, false}, {"filtering, client provides cert", true, true}, {"filtering, client does not provide cert", true, false}, {"no users override, client provides cert", false, true}, {"no users override, client does not provide cert", false, false}, {"users override, client provides cert", true, true}, {"users override, client does not provide cert", true, false}, } { t.Run(test.name, func(t *testing.T) { o := testWSOptions() o.Accounts = []*Account{acc} o.Users = users if test.filtering { o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}) } tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/server.pem", KeyFile: "../test/configs/certs/tlsauth/server-key.pem", CaFile: "../test/configs/certs/tlsauth/ca.pem", Verify: true, } tlsc, err := GenTLSConfig(tc) if err != nil { t.Fatalf("Error creating tls config: %v", err) } o.Websocket.TLSConfig = tlsc o.Websocket.TLSMap = true s := RunServer(o) defer s.Shutdown() addr := fmt.Sprintf("%s:%d", o.Websocket.Host, o.Websocket.Port) wsc, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer wsc.Close() tlscc := &tls.Config{} if test.provideCert { tc := &TLSConfigOpts{ CertFile: "../test/configs/certs/tlsauth/client.pem", KeyFile: "../test/configs/certs/tlsauth/client-key.pem", } var err error tlscc, err = GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating tls config: %v", err) } } tlscc.InsecureSkipVerify = true wsc = tls.Client(wsc, tlscc) if err := wsc.(*tls.Conn).Handshake(); err != nil { t.Fatalf("Error during handshake: %v", err) } req := testWSCreateValidReq() req.URL, _ = url.Parse("wss://" + addr) if err := req.Write(wsc); err != nil { t.Fatalf("Error sending request: %v", err) } br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if resp != nil { resp.Body.Close() } if !test.provideCert { if err == nil { t.Fatal("Expected error, did not get one") } else if !strings.Contains(err.Error(), "bad certificate") && !strings.Contains(err.Error(), "certificate required") { t.Fatalf("Unexpected error: %v", err) } return } if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp.StatusCode != http.StatusSwitchingProtocols { t.Fatalf("Expected status %v, got %v", http.StatusSwitchingProtocols, resp.StatusCode) } // Wait for the INFO l := testWSReadFrame(t, br) if !bytes.HasPrefix(l, []byte("INFO {")) { t.Fatalf("Expected INFO, got %s", l) } var info serverInfo if err := json.Unmarshal(l[5:], &info); err != nil { t.Fatalf("Unable to unmarshal info: %v", err) } // Send CONNECT and PING wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("CONNECT {\"verbose\":false,\"protocol\":1}\r\nPING\r\n")) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } // Wait for the PONG if msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected PONG, got %s", msg) } var uname string var accname string c := s.getClient(info.CID) if c != nil { c.mu.Lock() uname = c.opts.Username if c.acc != nil { accname = c.acc.GetName() } c.mu.Unlock() } if uname != certUserName { t.Fatalf("Expected username %q, got %q", certUserName, uname) } if accname != accName { t.Fatalf("Expected account %q, got %v", accName, accname) } }) } } func TestWSHandshakeTimeout(t *testing.T) { o := testWSOptions() o.Websocket.HandshakeTimeout = time.Millisecond tc := &TLSConfigOpts{ CertFile: "./configs/certs/server.pem", KeyFile: "./configs/certs/key.pem", } o.Websocket.TLSConfig, _ = GenTLSConfig(tc) s := RunServer(o) defer s.Shutdown() logger := &captureErrorLogger{errCh: make(chan string, 1)} s.SetLogger(logger, false, false) addr := fmt.Sprintf("%s:%d", o.Websocket.Host, o.Websocket.Port) wsc, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer wsc.Close() // Delay the handshake wsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true}) time.Sleep(20 * time.Millisecond) // We expect error since the server should have cut us off if err := wsc.(*tls.Conn).Handshake(); err == nil { t.Fatal("Expected error during handshake") } // Check that server logs error select { case e := <-logger.errCh: // Check that log starts with "websocket: " if !strings.HasPrefix(e, "websocket: ") { t.Fatalf("Wrong log line start: %s", e) } if !strings.Contains(e, "timeout") { t.Fatalf("Unexpected error: %v", e) } case <-time.After(time.Second): t.Fatalf("Should have timed-out") } } func TestWSServerReportUpgradeFailure(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() logger := &captureErrorLogger{errCh: make(chan string, 1)} s.SetLogger(logger, false, false) addr := fmt.Sprintf("127.0.0.1:%d", o.Websocket.Port) req := testWSCreateValidReq() req.URL, _ = url.Parse("wss://" + addr) wsc, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Error creating ws connection: %v", err) } defer wsc.Close() wsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true}) if err := wsc.(*tls.Conn).Handshake(); err != nil { t.Fatalf("Error during handshake: %v", err) } // Remove a required field from the request to have it fail req.Header.Del("Connection") // Send the request if err := req.Write(wsc); err != nil { t.Fatalf("Error sending request: %v", err) } br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if err != nil { t.Fatalf("Error reading response: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusBadRequest { t.Fatalf("Expected status %v, got %v", http.StatusBadRequest, resp.StatusCode) } // Check that server logs error select { case e := <-logger.errCh: if !strings.Contains(e, "invalid value for header 'Connection'") { t.Fatalf("Unexpected error: %v", e) } // The client IP's local should be printed as a remote from server perspective. clientIP := wsc.LocalAddr().String() if !strings.HasPrefix(e, clientIP) { t.Fatalf("IP should have been logged, it was not: %v", e) } case <-time.After(time.Second): t.Fatalf("Should have timed-out") } } func TestWSCloseMsgSendOnConnectionClose(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() wsc, br := testWSCreateClient(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() checkClientsCount(t, s, 1) var c *client s.mu.Lock() for _, cli := range s.clients { c = cli break } s.mu.Unlock() c.closeConnection(ProtocolViolation) msg := testWSReadFrame(t, br) if len(msg) < 2 { t.Fatalf("Should have 2 bytes to represent the status, got %v", msg) } if sc := int(binary.BigEndian.Uint16(msg[:2])); sc != wsCloseStatusProtocolError { t.Fatalf("Expected status to be %v, got %v", wsCloseStatusProtocolError, sc) } expectedPayload := ProtocolViolation.String() if p := string(msg[2:]); p != expectedPayload { t.Fatalf("Expected payload to be %q, got %q", expectedPayload, p) } } func TestWSAdvertise(t *testing.T) { o := testWSOptions() o.Cluster.Port = 0 o.HTTPPort = 0 o.Websocket.Advertise = "xxx:host:yyy" s, err := NewServer(o) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer s.Shutdown() l := &captureFatalLogger{fatalCh: make(chan string, 1)} s.SetLogger(l, false, false) s.Start() select { case e := <-l.fatalCh: if !strings.Contains(e, "Unable to get websocket connect URLs") { t.Fatalf("Unexpected error: %q", e) } case <-time.After(time.Second): t.Fatal("Should have failed to start") } s.Shutdown() o1 := testWSOptions() o1.Websocket.Advertise = "host1:1234" s1 := RunServer(o1) defer s1.Shutdown() wsc, br := testWSCreateClient(t, false, false, o1.Websocket.Host, o1.Websocket.Port) defer wsc.Close() o2 := testWSOptions() o2.Websocket.Advertise = "host2:5678" o2.Routes = RoutesFromStr(fmt.Sprintf("nats://%s:%d", o1.Cluster.Host, o1.Cluster.Port)) s2 := RunServer(o2) defer s2.Shutdown() checkInfo := func(expected []string) { t.Helper() infob := testWSReadFrame(t, br) info := &Info{} json.Unmarshal(infob[5:], info) if n := len(info.ClientConnectURLs); n != len(expected) { t.Fatalf("Unexpected info: %+v", info) } good := 0 for _, u := range info.ClientConnectURLs { for _, eu := range expected { if u == eu { good++ } } } if good != len(expected) { t.Fatalf("Unexpected connect urls: %q", info.ClientConnectURLs) } } checkInfo([]string{"host1:1234", "host2:5678"}) // Now shutdown s2 and expect another INFO s2.Shutdown() checkInfo([]string{"host1:1234"}) // Restart with another advertise and check that it gets updated o2.Websocket.Advertise = "host3:9012" s2 = RunServer(o2) defer s2.Shutdown() checkInfo([]string{"host1:1234", "host3:9012"}) } func TestWSFrameOutbound(t *testing.T) { for _, test := range []struct { name string maskingWrite bool }{ {"no write masking", false}, {"write masking", true}, } { t.Run(test.name, func(t *testing.T) { c, _, _ := testWSSetupForRead() c.ws.maskwrite = test.maskingWrite getKey := func(buf []byte) []byte { return buf[len(buf)-4:] } var bufs net.Buffers bufs = append(bufs, []byte("this ")) bufs = append(bufs, []byte("is ")) bufs = append(bufs, []byte("a ")) bufs = append(bufs, []byte("set ")) bufs = append(bufs, []byte("of ")) bufs = append(bufs, []byte("buffers")) en := 2 for _, b := range bufs { en += len(b) } if test.maskingWrite { en += 4 } c.mu.Lock() c.out.nb = bufs res, n := c.collapsePtoNB() c.mu.Unlock() if n != int64(en) { t.Fatalf("Expected size to be %v, got %v", en, n) } if eb := 1 + len(bufs); eb != len(res) { t.Fatalf("Expected %v buffers, got %v", eb, len(res)) } var ob []byte for i := 1; i < len(res); i++ { ob = append(ob, res[i]...) } if test.maskingWrite { wsMaskBuf(getKey(res[0]), ob) } if !bytes.Equal(ob, []byte("this is a set of buffers")) { t.Fatalf("Unexpected outbound: %q", ob) } bufs = nil c.out.pb = 0 c.ws.fs = 0 c.ws.frames = nil c.ws.browser = true bufs = append(bufs, []byte("some smaller ")) bufs = append(bufs, []byte("buffers")) bufs = append(bufs, make([]byte, wsFrameSizeForBrowsers+10)) bufs = append(bufs, []byte("then some more")) en = 2 + len(bufs[0]) + len(bufs[1]) en += 4 + len(bufs[2]) - 10 en += 2 + len(bufs[3]) + 10 c.mu.Lock() c.out.nb = bufs res, n = c.collapsePtoNB() c.mu.Unlock() if test.maskingWrite { en += 3 * 4 } if n != int64(en) { t.Fatalf("Expected size to be %v, got %v", en, n) } if len(res) != 8 { t.Fatalf("Unexpected number of outbound buffers: %v", len(res)) } if len(res[4]) != wsFrameSizeForBrowsers { t.Fatalf("Big frame should have been limited to %v, got %v", wsFrameSizeForBrowsers, len(res[4])) } if len(res[6]) != 10 { t.Fatalf("Frame 6 should have the partial of 10 bytes, got %v", len(res[6])) } if test.maskingWrite { b := &bytes.Buffer{} key := getKey(res[0]) b.Write(res[1]) b.Write(res[2]) ud := b.Bytes() wsMaskBuf(key, ud) if string(ud) != "some smaller buffers" { t.Fatalf("Unexpected result: %q", ud) } b.Reset() key = getKey(res[3]) b.Write(res[4]) ud = b.Bytes() wsMaskBuf(key, ud) for i := 0; i < len(ud); i++ { if ud[i] != 0 { t.Fatalf("Unexpected result: %v", ud) } } b.Reset() key = getKey(res[5]) b.Write(res[6]) b.Write(res[7]) ud = b.Bytes() wsMaskBuf(key, ud) for i := 0; i < len(ud[:10]); i++ { if ud[i] != 0 { t.Fatalf("Unexpected result: %v", ud[:10]) } } if string(ud[10:]) != "then some more" { t.Fatalf("Unexpected result: %q", ud[10:]) } } bufs = nil c.out.pb = 0 c.ws.fs = 0 c.ws.frames = nil c.ws.browser = true bufs = append(bufs, []byte("some smaller ")) bufs = append(bufs, []byte("buffers")) // Have one of the exact max size bufs = append(bufs, make([]byte, wsFrameSizeForBrowsers)) bufs = append(bufs, []byte("then some more")) en = 2 + len(bufs[0]) + len(bufs[1]) en += 4 + len(bufs[2]) en += 2 + len(bufs[3]) c.mu.Lock() c.out.nb = bufs res, n = c.collapsePtoNB() c.mu.Unlock() if test.maskingWrite { en += 3 * 4 } if n != int64(en) { t.Fatalf("Expected size to be %v, got %v", en, n) } if len(res) != 7 { t.Fatalf("Unexpected number of outbound buffers: %v", len(res)) } if len(res[4]) != wsFrameSizeForBrowsers { t.Fatalf("Big frame should have been limited to %v, got %v", wsFrameSizeForBrowsers, len(res[4])) } if test.maskingWrite { key := getKey(res[5]) wsMaskBuf(key, res[6]) } if string(res[6]) != "then some more" { t.Fatalf("Frame 6 incorrect: %q", res[6]) } bufs = nil c.out.pb = 0 c.ws.fs = 0 c.ws.frames = nil c.ws.browser = true bufs = append(bufs, []byte("some smaller ")) bufs = append(bufs, []byte("buffers")) // Have one of the exact max size, and last in the list bufs = append(bufs, make([]byte, wsFrameSizeForBrowsers)) en = 2 + len(bufs[0]) + len(bufs[1]) en += 4 + len(bufs[2]) c.mu.Lock() c.out.nb = bufs res, n = c.collapsePtoNB() c.mu.Unlock() if test.maskingWrite { en += 2 * 4 } if n != int64(en) { t.Fatalf("Expected size to be %v, got %v", en, n) } if len(res) != 5 { t.Fatalf("Unexpected number of outbound buffers: %v", len(res)) } if len(res[4]) != wsFrameSizeForBrowsers { t.Fatalf("Big frame should have been limited to %v, got %v", wsFrameSizeForBrowsers, len(res[4])) } bufs = nil c.out.pb = 0 c.ws.fs = 0 c.ws.frames = nil c.ws.browser = true bufs = append(bufs, []byte("some smaller buffer")) bufs = append(bufs, make([]byte, wsFrameSizeForBrowsers-5)) bufs = append(bufs, []byte("then some more")) en = 2 + len(bufs[0]) en += 4 + len(bufs[1]) en += 2 + len(bufs[2]) c.mu.Lock() c.out.nb = bufs res, n = c.collapsePtoNB() c.mu.Unlock() if test.maskingWrite { en += 3 * 4 } if n != int64(en) { t.Fatalf("Expected size to be %v, got %v", en, n) } if len(res) != 6 { t.Fatalf("Unexpected number of outbound buffers: %v", len(res)) } if len(res[3]) != wsFrameSizeForBrowsers-5 { t.Fatalf("Big frame should have been limited to %v, got %v", wsFrameSizeForBrowsers, len(res[4])) } if test.maskingWrite { key := getKey(res[4]) wsMaskBuf(key, res[5]) } if string(res[5]) != "then some more" { t.Fatalf("Frame 6 incorrect %q", res[5]) } bufs = nil c.out.pb = 0 c.ws.fs = 0 c.ws.frames = nil c.ws.browser = true bufs = append(bufs, make([]byte, wsFrameSizeForBrowsers+100)) c.mu.Lock() c.out.nb = bufs res, _ = c.collapsePtoNB() c.mu.Unlock() if len(res) != 4 { t.Fatalf("Unexpected number of frames: %v", len(res)) } }) } } func TestWSWebrowserClient(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() wsc, br := testWSCreateClient(t, false, true, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() checkClientsCount(t, s, 1) var c *client s.mu.Lock() for _, cli := range s.clients { c = cli break } s.mu.Unlock() proto := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("SUB foo 1\r\nPING\r\n")) wsc.Write(proto) if res := testWSReadFrame(t, br); !bytes.Equal(res, []byte(pongProto)) { t.Fatalf("Expected PONG back") } c.mu.Lock() ok := c.isWebsocket() && c.ws.browser == true c.mu.Unlock() if !ok { t.Fatalf("Client is not marked as webrowser client") } nc := natsConnect(t, s.ClientURL()) defer nc.Close() // Send a big message and check that it is received in smaller frames psize := 204813 nc.Publish("foo", make([]byte, psize)) nc.Flush() rsize := psize + len(fmt.Sprintf("MSG foo %d\r\n\r\n", psize)) nframes := 0 for total := 0; total < rsize; nframes++ { res := testWSReadFrame(t, br) total += len(res) } if expected := psize / wsFrameSizeForBrowsers; expected > nframes { t.Fatalf("Expected %v frames, got %v", expected, nframes) } } type testWSWrappedConn struct { net.Conn mu sync.RWMutex buf *bytes.Buffer partial bool } func (wc *testWSWrappedConn) Write(p []byte) (int, error) { wc.mu.Lock() defer wc.mu.Unlock() var err error n := len(p) if wc.partial && n > 10 { n = 10 err = io.ErrShortWrite } p = p[:n] wc.buf.Write(p) wc.Conn.Write(p) return n, err } func TestWSCompressionBasic(t *testing.T) { payload := "This is the content of a message that will be compresseddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd." msgProto := fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", len(payload), payload) cbuf := &bytes.Buffer{} compressor, err := flate.NewWriter(cbuf, flate.BestSpeed) require_NoError(t, err) compressor.Write([]byte(msgProto)) compressor.Flush() compressed := cbuf.Bytes() // The last 4 bytes are dropped compressed = compressed[:len(compressed)-4] o := testWSOptions() o.Websocket.Compression = true s := RunServer(o) defer s.Shutdown() c, br := testWSCreateClient(t, true, false, o.Websocket.Host, o.Websocket.Port) defer c.Close() proto := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, []byte("SUB foo 1\r\nPING\r\n")) c.Write(proto) l := testWSReadFrame(t, br) if !bytes.Equal(l, []byte(pongProto)) { t.Fatalf("Expected PONG, got %q", l) } var wc *testWSWrappedConn s.mu.RLock() for _, c := range s.clients { c.mu.Lock() wc = &testWSWrappedConn{Conn: c.nc, buf: &bytes.Buffer{}} c.nc = wc c.mu.Unlock() } s.mu.RUnlock() nc := natsConnect(t, s.ClientURL()) defer nc.Close() natsPub(t, nc, "foo", []byte(payload)) res := &bytes.Buffer{} for total := 0; total < len(msgProto); { l := testWSReadFrame(t, br) n, _ := res.Write(l) total += n } if !bytes.Equal([]byte(msgProto), res.Bytes()) { t.Fatalf("Unexpected result: %q", res) } // Now check the wrapped connection buffer to check that data was actually compressed. wc.mu.RLock() res = wc.buf wc.mu.RUnlock() if bytes.Contains(res.Bytes(), []byte(payload)) { t.Fatalf("Looks like frame was not compressed: %q", res.Bytes()) } header := res.Bytes()[:2] body := res.Bytes()[2:] expectedB0 := byte(wsBinaryMessage) | wsFinalBit | wsRsv1Bit expectedPS := len(compressed) expectedB1 := byte(expectedPS) if b := header[0]; b != expectedB0 { t.Fatalf("Expected first byte to be %v, got %v", expectedB0, b) } if b := header[1]; b != expectedB1 { t.Fatalf("Expected second byte to be %v, got %v", expectedB1, b) } if len(body) != expectedPS { t.Fatalf("Expected payload length to be %v, got %v", expectedPS, len(body)) } if !bytes.Equal(body, compressed) { t.Fatalf("Unexpected compress body: %q", body) } wc.mu.Lock() wc.buf.Reset() wc.mu.Unlock() payload = "small" natsPub(t, nc, "foo", []byte(payload)) msgProto = fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", len(payload), payload) res = &bytes.Buffer{} for total := 0; total < len(msgProto); { l := testWSReadFrame(t, br) n, _ := res.Write(l) total += n } if !bytes.Equal([]byte(msgProto), res.Bytes()) { t.Fatalf("Unexpected result: %q", res) } wc.mu.RLock() res = wc.buf wc.mu.RUnlock() if !bytes.HasSuffix(res.Bytes(), []byte(msgProto)) { t.Fatalf("Looks like frame was compressed: %q", res.Bytes()) } } func TestWSCompressionWithPartialWrite(t *testing.T) { payload := "This is the content of a message that will be compresseddddddddddddddddddddd." msgProto := fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", len(payload), payload) o := testWSOptions() o.Websocket.Compression = true s := RunServer(o) defer s.Shutdown() c, br := testWSCreateClient(t, true, false, o.Websocket.Host, o.Websocket.Port) defer c.Close() proto := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, []byte("SUB foo 1\r\nPING\r\n")) c.Write(proto) l := testWSReadFrame(t, br) if !bytes.Equal(l, []byte(pongProto)) { t.Fatalf("Expected PONG, got %q", l) } pingPayload := []byte("my ping") pingFromWSClient := testWSCreateClientMsg(wsPingMessage, 1, true, false, pingPayload) var wc *testWSWrappedConn var ws *client s.mu.Lock() for _, c := range s.clients { ws = c c.mu.Lock() wc = &testWSWrappedConn{ Conn: c.nc, buf: &bytes.Buffer{}, } c.nc = wc c.mu.Unlock() break } s.mu.Unlock() wc.mu.Lock() wc.partial = true wc.mu.Unlock() nc := natsConnect(t, s.ClientURL()) defer nc.Close() expected := &bytes.Buffer{} for i := 0; i < 10; i++ { if i > 0 { time.Sleep(10 * time.Millisecond) } expected.Write([]byte(msgProto)) natsPub(t, nc, "foo", []byte(payload)) if i == 1 { c.Write(pingFromWSClient) } } var gotPingResponse bool res := &bytes.Buffer{} for total := 0; total < 10*len(msgProto); { l := testWSReadFrame(t, br) if bytes.Equal(l, pingPayload) { gotPingResponse = true } else { n, _ := res.Write(l) total += n } } if !bytes.Equal(expected.Bytes(), res.Bytes()) { t.Fatalf("Unexpected result: %q", res) } if !gotPingResponse { t.Fatal("Did not get the ping response") } checkFor(t, time.Second, 15*time.Millisecond, func() error { ws.mu.Lock() pb := ws.out.pb wf := ws.ws.frames fs := ws.ws.fs ws.mu.Unlock() if pb != 0 || len(wf) != 0 || fs != 0 { return fmt.Errorf("Expected pb, wf and fs to be 0, got %v, %v, %v", pb, wf, fs) } return nil }) } func TestWSCompressionFrameSizeLimit(t *testing.T) { for _, test := range []struct { name string maskWrite bool noLimit bool }{ {"no write masking", false, false}, {"write masking", true, false}, } { t.Run(test.name, func(t *testing.T) { opts := testWSOptions() opts.MaxPending = MAX_PENDING_SIZE s := &Server{opts: opts} c := &client{srv: s, ws: &websocket{compress: true, browser: true, nocompfrag: test.noLimit, maskwrite: test.maskWrite}} c.initClient() uncompressedPayload := make([]byte, 2*wsFrameSizeForBrowsers) for i := 0; i < len(uncompressedPayload); i++ { uncompressedPayload[i] = byte(rand.Intn(256)) } c.mu.Lock() c.out.nb = append(net.Buffers(nil), uncompressedPayload) nb, _ := c.collapsePtoNB() c.mu.Unlock() if test.noLimit && len(nb) != 2 { t.Fatalf("There should be only 2 buffers, the header and payload, got %v", len(nb)) } bb := &bytes.Buffer{} var key []byte for i, b := range nb { if !test.noLimit { // frame header buffer are always very small. The payload should not be more // than 10 bytes since that is what we passed as the limit. if len(b) > wsFrameSizeForBrowsers { t.Fatalf("Frame size too big: %v (%q)", len(b), b) } } if test.maskWrite { if i%2 == 0 { key = b[len(b)-4:] } else { wsMaskBuf(key, b) } } // Check frame headers for the proper formatting. if i%2 == 0 { // Only the first frame should have the compress bit set. if b[0]&wsRsv1Bit != 0 { if i > 0 { t.Fatalf("Compressed bit should not be in continuation frame") } } else if i == 0 { t.Fatalf("Compressed bit missing") } } else { if test.noLimit { // Since the payload is likely not well compressed, we are expecting // the length to be > wsFrameSizeForBrowsers if len(b) <= wsFrameSizeForBrowsers { t.Fatalf("Expected frame to be bigger, got %v", len(b)) } } // Collect the payload bb.Write(b) } } buf := bb.Bytes() buf = append(buf, 0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff) dbr := bytes.NewBuffer(buf) d := flate.NewReader(dbr) uncompressed, err := io.ReadAll(d) if err != nil { t.Fatalf("Error reading frame: %v", err) } if !bytes.Equal(uncompressed, uncompressedPayload) { t.Fatalf("Unexpected uncomressed data: %q", uncompressed) } }) } } func TestWSBasicAuth(t *testing.T) { for _, test := range []struct { name string opts func() *Options user string pass string err string }{ { "top level auth, no override, wrong u/p", func() *Options { o := testWSOptions() o.Username = "normal" o.Password = "client" return o }, "websocket", "client", "-ERR 'Authorization Violation'", }, { "top level auth, no override, correct u/p", func() *Options { o := testWSOptions() o.Username = "normal" o.Password = "client" return o }, "normal", "client", "", }, { "no top level auth, ws auth, wrong u/p", func() *Options { o := testWSOptions() o.Websocket.Username = "websocket" o.Websocket.Password = "client" return o }, "normal", "client", "-ERR 'Authorization Violation'", }, { "no top level auth, ws auth, correct u/p", func() *Options { o := testWSOptions() o.Websocket.Username = "websocket" o.Websocket.Password = "client" return o }, "websocket", "client", "", }, { "top level auth, ws override, wrong u/p", func() *Options { o := testWSOptions() o.Username = "normal" o.Password = "client" o.Websocket.Username = "websocket" o.Websocket.Password = "client" return o }, "normal", "client", "-ERR 'Authorization Violation'", }, { "top level auth, ws override, correct u/p", func() *Options { o := testWSOptions() o.Username = "normal" o.Password = "client" o.Websocket.Username = "websocket" o.Websocket.Password = "client" return o }, "websocket", "client", "", }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := RunServer(o) defer s.Shutdown() wsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() connectProto := fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"user\":\"%s\",\"pass\":\"%s\"}\r\nPING\r\n", test.user, test.pass) wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if test.err == "" && !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected to receive PONG, got %q", msg) } else if test.err != "" && !bytes.HasPrefix(msg, []byte(test.err)) { t.Fatalf("Expected to receive %q, got %q", test.err, msg) } }) } } func TestWSAuthTimeout(t *testing.T) { for _, test := range []struct { name string at float64 wat float64 err string }{ {"use top-level auth timeout", 10.0, 0.0, ""}, {"use websocket auth timeout", 10.0, 0.05, "-ERR 'Authentication Timeout'"}, } { t.Run(test.name, func(t *testing.T) { o := testWSOptions() o.AuthTimeout = test.at o.Websocket.Username = "websocket" o.Websocket.Password = "client" o.Websocket.AuthTimeout = test.wat s := RunServer(o) defer s.Shutdown() wsc, br, l := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() var info serverInfo json.Unmarshal([]byte(l[5:]), &info) // Make sure that we are told that auth is required. if !info.AuthRequired { t.Fatalf("Expected auth required, was not: %q", l) } start := time.Now() // Wait before sending connect time.Sleep(100 * time.Millisecond) connectProto := "CONNECT {\"verbose\":false,\"protocol\":1,\"user\":\"websocket\",\"pass\":\"client\"}\r\nPING\r\n" wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if test.err != "" && !bytes.HasPrefix(msg, []byte(test.err)) { t.Fatalf("Expected to receive %q error, got %q", test.err, msg) } else if test.err == "" && !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Unexpected error: %q", msg) } if dur := time.Since(start); dur > time.Second { t.Fatalf("Too long to get timeout error: %v", dur) } }) } } func TestWSTokenAuth(t *testing.T) { for _, test := range []struct { name string opts func() *Options token string err string }{ { "top level auth, no override, wrong token", func() *Options { o := testWSOptions() o.Authorization = "goodtoken" return o }, "badtoken", "-ERR 'Authorization Violation'", }, { "top level auth, no override, correct token", func() *Options { o := testWSOptions() o.Authorization = "goodtoken" return o }, "goodtoken", "", }, { "no top level auth, ws auth, wrong token", func() *Options { o := testWSOptions() o.Websocket.Token = "goodtoken" return o }, "badtoken", "-ERR 'Authorization Violation'", }, { "no top level auth, ws auth, correct token", func() *Options { o := testWSOptions() o.Websocket.Token = "goodtoken" return o }, "goodtoken", "", }, { "top level auth, ws override, wrong token", func() *Options { o := testWSOptions() o.Authorization = "clienttoken" o.Websocket.Token = "websockettoken" return o }, "clienttoken", "-ERR 'Authorization Violation'", }, { "top level auth, ws override, correct token", func() *Options { o := testWSOptions() o.Authorization = "clienttoken" o.Websocket.Token = "websockettoken" return o }, "websockettoken", "", }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := RunServer(o) defer s.Shutdown() wsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() connectProto := fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"auth_token\":\"%s\"}\r\nPING\r\n", test.token) wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if test.err == "" && !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected to receive PONG, got %q", msg) } else if test.err != "" && !bytes.HasPrefix(msg, []byte(test.err)) { t.Fatalf("Expected to receive %q, got %q", test.err, msg) } }) } } func TestWSBindToProperAccount(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: "127.0.0.1:-1" accounts { a { users [ {user: a, password: pwd, allowed_connection_types: ["%s", "%s"]} ] } b { users [ {user: b, password: pwd} ] } } websocket { listen: "127.0.0.1:-1" no_tls: true } `, jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeWebsocket)))) // on purpose use lower case to ensure that it is converted. s, o := RunServerWithConfig(conf) defer s.Shutdown() nc := natsConnect(t, fmt.Sprintf("nats://a:pwd@127.0.0.1:%d", o.Port)) defer nc.Close() sub := natsSubSync(t, nc, "foo") wsc, br, _ := testNewWSClient(t, testWSClientOptions{host: o.Websocket.Host, port: o.Websocket.Port, noTLS: true}) // Send CONNECT and PING wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"user\":\"%s\",\"pass\":\"%s\"}\r\nPING\r\n", "a", "pwd"))) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } // Wait for the PONG if msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected PONG, got %s", msg) } wsmsg = testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("PUB foo 7\r\nfrom ws\r\n")) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } natsNexMsg(t, sub, time.Second) } func TestWSUsersAuth(t *testing.T) { users := []*User{{Username: "user", Password: "pwd"}} for _, test := range []struct { name string opts func() *Options user string pass string err string }{ { "no filtering, wrong user", func() *Options { o := testWSOptions() o.Users = users return o }, "wronguser", "pwd", "-ERR 'Authorization Violation'", }, { "no filtering, correct user", func() *Options { o := testWSOptions() o.Users = users return o }, "user", "pwd", "", }, { "filering, user not allowed", func() *Options { o := testWSOptions() o.Users = users // Only allowed for regular clients o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard}) return o }, "user", "pwd", "-ERR 'Authorization Violation'", }, { "filtering, user allowed", func() *Options { o := testWSOptions() o.Users = users o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}) return o }, "user", "pwd", "", }, { "filtering, wrong password", func() *Options { o := testWSOptions() o.Users = users o.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}) return o }, "user", "badpassword", "-ERR 'Authorization Violation'", }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := RunServer(o) defer s.Shutdown() wsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() connectProto := fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"user\":\"%s\",\"pass\":\"%s\"}\r\nPING\r\n", test.user, test.pass) wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if test.err == "" && !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected to receive PONG, got %q", msg) } else if test.err != "" && !bytes.HasPrefix(msg, []byte(test.err)) { t.Fatalf("Expected to receive %q, got %q", test.err, msg) } }) } } func TestWSNoAuthUserValidation(t *testing.T) { o := testWSOptions() o.Users = []*User{{Username: "user", Password: "pwd"}} // Should fail because it is not part of o.Users. o.Websocket.NoAuthUser = "notfound" if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), "not present as user") { t.Fatalf("Expected error saying not present as user, got %v", err) } // Set a valid no auth user for global options, but still should fail because // of o.Websocket.NoAuthUser o.NoAuthUser = "user" o.Websocket.NoAuthUser = "notfound" if _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), "not present as user") { t.Fatalf("Expected error saying not present as user, got %v", err) } } func TestWSNoAuthUser(t *testing.T) { for _, test := range []struct { name string override bool useAuth bool expectedUser string expectedAcc string }{ {"no override, no user provided", false, false, "noauth", "normal"}, {"no override, user povided", false, true, "user", "normal"}, {"override, no user provided", true, false, "wsnoauth", "websocket"}, {"override, user provided", true, true, "wsuser", "websocket"}, } { t.Run(test.name, func(t *testing.T) { o := testWSOptions() normalAcc := NewAccount("normal") websocketAcc := NewAccount("websocket") o.Accounts = []*Account{normalAcc, websocketAcc} o.Users = []*User{ {Username: "noauth", Password: "pwd", Account: normalAcc}, {Username: "user", Password: "pwd", Account: normalAcc}, {Username: "wsnoauth", Password: "pwd", Account: websocketAcc}, {Username: "wsuser", Password: "pwd", Account: websocketAcc}, } o.NoAuthUser = "noauth" if test.override { o.Websocket.NoAuthUser = "wsnoauth" } s := RunServer(o) defer s.Shutdown() wsc, br, l := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() var info serverInfo json.Unmarshal([]byte(l[5:]), &info) var connectProto string if test.useAuth { connectProto = fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"user\":\"%s\",\"pass\":\"pwd\"}\r\nPING\r\n", test.expectedUser) } else { connectProto = "CONNECT {\"verbose\":false,\"protocol\":1}\r\nPING\r\n" } wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Unexpected error: %q", msg) } c := s.getClient(info.CID) c.mu.Lock() uname := c.opts.Username aname := c.acc.GetName() c.mu.Unlock() if uname != test.expectedUser { t.Fatalf("Expected selected user to be %q, got %q", test.expectedUser, uname) } if aname != test.expectedAcc { t.Fatalf("Expected selected account to be %q, got %q", test.expectedAcc, aname) } }) } } func TestWSNkeyAuth(t *testing.T) { nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() wsnkp, _ := nkeys.CreateUser() wspub, _ := wsnkp.PublicKey() badkp, _ := nkeys.CreateUser() badpub, _ := badkp.PublicKey() for _, test := range []struct { name string opts func() *Options nkey string kp nkeys.KeyPair err string }{ { "no filtering, wrong nkey", func() *Options { o := testWSOptions() o.Nkeys = []*NkeyUser{{Nkey: pub}} return o }, badpub, badkp, "-ERR 'Authorization Violation'", }, { "no filtering, correct nkey", func() *Options { o := testWSOptions() o.Nkeys = []*NkeyUser{{Nkey: pub}} return o }, pub, nkp, "", }, { "filtering, nkey not allowed", func() *Options { o := testWSOptions() o.Nkeys = []*NkeyUser{ { Nkey: pub, AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard}), }, { Nkey: wspub, AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeWebsocket}), }, } return o }, pub, nkp, "-ERR 'Authorization Violation'", }, { "filtering, correct nkey", func() *Options { o := testWSOptions() o.Nkeys = []*NkeyUser{ {Nkey: pub}, { Nkey: wspub, AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}), }, } return o }, wspub, wsnkp, "", }, { "filtering, wrong nkey", func() *Options { o := testWSOptions() o.Nkeys = []*NkeyUser{ { Nkey: wspub, AllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}), }, } return o }, badpub, badkp, "-ERR 'Authorization Violation'", }, } { t.Run(test.name, func(t *testing.T) { o := test.opts() s := RunServer(o) defer s.Shutdown() wsc, br, infoMsg := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port) defer wsc.Close() // Sign Nonce var info nonceInfo json.Unmarshal([]byte(infoMsg[5:]), &info) sigraw, _ := test.kp.Sign([]byte(info.Nonce)) sig := base64.RawURLEncoding.EncodeToString(sigraw) connectProto := fmt.Sprintf("CONNECT {\"verbose\":false,\"protocol\":1,\"nkey\":\"%s\",\"sig\":\"%s\"}\r\nPING\r\n", test.nkey, sig) wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto)) if _, err := wsc.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } msg := testWSReadFrame(t, br) if test.err == "" && !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected to receive PONG, got %q", msg) } else if test.err != "" && !bytes.HasPrefix(msg, []byte(test.err)) { t.Fatalf("Expected to receive %q, got %q", test.err, msg) } }) } } func TestWSJWTWithAllowedConnectionTypes(t *testing.T) { o := testWSOptions() setupAddTrusted(o) s := RunServer(o) buildMemAccResolver(s) defer s.Shutdown() for _, test := range []struct { name string connectionTypes []string expectedAnswer string }{ {"not allowed", []string{jwt.ConnectionTypeStandard}, "-ERR"}, {"allowed", []string{jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeWebsocket)}, "+OK"}, {"allowed with unknown", []string{jwt.ConnectionTypeWebsocket, "SomeNewType"}, "+OK"}, {"not allowed with unknown", []string{"SomeNewType"}, "-ERR"}, } { t.Run(test.name, func(t *testing.T) { nuc := newJWTTestUserClaims() nuc.AllowedConnectionTypes = test.connectionTypes claimOpt := testClaimsOptions{ nuc: nuc, expectAnswer: test.expectedAnswer, } _, c, _, _ := testWSWithClaims(t, s, testWSClientOptions{host: o.Websocket.Host, port: o.Websocket.Port}, claimOpt) c.Close() }) } } func TestWSJWTCookieUser(t *testing.T) { nucSigFunc := func() *jwt.UserClaims { return newJWTTestUserClaims() } nucBearerFunc := func() *jwt.UserClaims { ret := newJWTTestUserClaims() ret.BearerToken = true return ret } o := testWSOptions() setupAddTrusted(o) setupAddCookie(o) s := RunServer(o) buildMemAccResolver(s) defer s.Shutdown() genJwt := func(t *testing.T, nuc *jwt.UserClaims) string { okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } nkp, _ := nkeys.CreateUser() pub, _ := nkp.PublicKey() nuc.Subject = pub jwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } addAccountToMemResolver(s, apub, ajwt) return jwt } cliOpts := testWSClientOptions{ host: o.Websocket.Host, port: o.Websocket.Port, } for _, test := range []struct { name string nuc *jwt.UserClaims opts func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) expectAnswer string }{ { name: "protocol auth, non-bearer key, with signature", nuc: nucSigFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { return cliOpts, testClaimsOptions{nuc: claims} }, expectAnswer: "+OK", }, { name: "protocol auth, non-bearer key, w/o required signature", nuc: nucSigFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { return cliOpts, testClaimsOptions{nuc: claims, dontSign: true} }, expectAnswer: "-ERR", }, { name: "protocol auth, bearer key, w/o signature", nuc: nucBearerFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { return cliOpts, testClaimsOptions{nuc: claims, dontSign: true} }, expectAnswer: "+OK", }, { name: "cookie auth, non-bearer key, protocol auth fail", nuc: nucSigFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { co := cliOpts co.extraHeaders = map[string][]string{} co.extraHeaders["Cookie"] = []string{o.Websocket.JWTCookie + "=" + genJwt(t, claims)} return co, testClaimsOptions{connectRequest: struct{}{}} }, expectAnswer: "-ERR", }, { name: "cookie auth, bearer key, protocol auth success with implied cookie jwt", nuc: nucBearerFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { co := cliOpts co.extraHeaders = map[string][]string{} co.extraHeaders["Cookie"] = []string{o.Websocket.JWTCookie + "=" + genJwt(t, claims)} return co, testClaimsOptions{connectRequest: struct{}{}} }, expectAnswer: "+OK", }, { name: "cookie auth, non-bearer key, protocol auth success via override jwt in CONNECT opts", nuc: nucSigFunc(), opts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) { co := cliOpts co.extraHeaders = map[string][]string{} co.extraHeaders["Cookie"] = []string{o.Websocket.JWTCookie + "=" + genJwt(t, claims)} return co, testClaimsOptions{nuc: nucBearerFunc()} }, expectAnswer: "+OK", }, } { t.Run(test.name, func(t *testing.T) { cliOpt, claimOpt := test.opts(t, test.nuc) claimOpt.expectAnswer = test.expectAnswer _, c, _, _ := testWSWithClaims(t, s, cliOpt, claimOpt) c.Close() }) } s.Shutdown() } func TestWSReloadTLSConfig(t *testing.T) { tlsBlock := ` tls { cert_file: '%s' key_file: '%s' ca_file: '../test/configs/certs/ca.pem' verify: %v } ` template := ` listen: "127.0.0.1:-1" websocket { listen: "127.0.0.1:-1" %s no_tls: %v } ` conf := createConfFile(t, []byte(fmt.Sprintf(template, fmt.Sprintf(tlsBlock, "../test/configs/certs/server-noip.pem", "../test/configs/certs/server-key-noip.pem", false), false))) s, o := RunServerWithConfig(conf) defer s.Shutdown() addr := fmt.Sprintf("127.0.0.1:%d", o.Websocket.Port) check := func(tlsConfig *tls.Config, handshakeFail bool, errTxt string) { t.Helper() wsc, err := net.Dial("tcp", addr) require_NoError(t, err) defer wsc.Close() wsc = tls.Client(wsc, tlsConfig) err = wsc.(*tls.Conn).Handshake() if handshakeFail { require_True(t, err != nil) require_Contains(t, err.Error(), errTxt) return } require_NoError(t, err) req := testWSCreateValidReq() req.URL, _ = url.Parse(wsSchemePrefixTLS + "://" + addr) err = req.Write(wsc) require_NoError(t, err) br := bufio.NewReader(wsc) resp, err := http.ReadResponse(br, req) if errTxt == _EMPTY_ { require_NoError(t, err) } else { require_True(t, err != nil) require_Contains(t, err.Error(), errTxt) return } defer resp.Body.Close() l := testWSReadFrame(t, br) require_True(t, bytes.HasPrefix(l, []byte("INFO {"))) var info Info err = json.Unmarshal(l[5:], &info) require_NoError(t, err) require_True(t, info.TLSAvailable) require_True(t, info.TLSRequired) require_Equal[string](t, info.Host, "127.0.0.1") require_Equal[int](t, info.Port, o.Websocket.Port) } tc := &TLSConfigOpts{CaFile: "../test/configs/certs/ca.pem"} tlsConfig, err := GenTLSConfig(tc) require_NoError(t, err) tlsConfig.ServerName = "127.0.0.1" tlsConfig.RootCAs = tlsConfig.ClientCAs tlsConfig.ClientCAs = nil // Handshake should fail with error regarding SANs check(tlsConfig.Clone(), true, "SAN") // Replace certs with ones that allow IP. reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, fmt.Sprintf(tlsBlock, "../test/configs/certs/server-cert.pem", "../test/configs/certs/server-key.pem", false), false)) // Connection should succeed check(tlsConfig.Clone(), false, _EMPTY_) // Udpate config to require client cert. reloadUpdateConfig(t, s, conf, fmt.Sprintf(template, fmt.Sprintf(tlsBlock, "../test/configs/certs/server-cert.pem", "../test/configs/certs/server-key.pem", true), false)) // Connection should fail saying that a tls cert is required check(tlsConfig.Clone(), false, "required") // Add a client cert tc = &TLSConfigOpts{ CertFile: "../test/configs/certs/client-cert.pem", KeyFile: "../test/configs/certs/client-key.pem", } tlsConfig, err = GenTLSConfig(tc) require_NoError(t, err) tlsConfig.InsecureSkipVerify = true // Connection should succeed check(tlsConfig.Clone(), false, _EMPTY_) // Removing the tls{} block but with no_tls still false should fail changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, _EMPTY_, false))) err = s.Reload() require_True(t, err != nil) require_Contains(t, err.Error(), "TLS configuration") // We should still be able to connect a TLS client check(tlsConfig.Clone(), false, _EMPTY_) // Now remove the tls{} block and set no_tls: true and that should fail // since this is not supported. changeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, _EMPTY_, true))) err = s.Reload() require_True(t, err != nil) require_Contains(t, err.Error(), "not supported") // We should still be able to connect a TLS client check(tlsConfig.Clone(), false, _EMPTY_) } type captureClientConnectedLogger struct { DummyLogger ch chan string } func (l *captureClientConnectedLogger) Debugf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if !strings.Contains(msg, "Client connection created") { return } select { case l.ch <- msg: default: } } func TestWSXForwardedFor(t *testing.T) { o := testWSOptions() s := RunServer(o) defer s.Shutdown() l := &captureClientConnectedLogger{ch: make(chan string, 1)} s.SetLogger(l, true, false) for _, test := range []struct { name string headers func() map[string][]string useHdrValue bool expectedValue string }{ {"nil map", func() map[string][]string { return nil }, false, _EMPTY_}, {"empty map", func() map[string][]string { return make(map[string][]string) }, false, _EMPTY_}, {"header present empty value", func() map[string][]string { m := make(map[string][]string) m[wsXForwardedForHeader] = []string{} return m }, false, _EMPTY_}, {"header present invalid IP", func() map[string][]string { m := make(map[string][]string) m[wsXForwardedForHeader] = []string{"not a valid IP"} return m }, false, _EMPTY_}, {"header present one IP", func() map[string][]string { m := make(map[string][]string) m[wsXForwardedForHeader] = []string{"1.2.3.4"} return m }, true, "1.2.3.4"}, {"header present multiple IPs", func() map[string][]string { m := make(map[string][]string) m[wsXForwardedForHeader] = []string{"1.2.3.4", "5.6.7.8"} return m }, true, "1.2.3.4"}, {"header present IPv6", func() map[string][]string { m := make(map[string][]string) m[wsXForwardedForHeader] = []string{"::1"} return m }, true, "[::1]"}, } { t.Run(test.name, func(t *testing.T) { c, r, _ := testNewWSClient(t, testWSClientOptions{ host: o.Websocket.Host, port: o.Websocket.Port, extraHeaders: test.headers(), }) defer c.Close() // Send CONNECT and PING wsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte("CONNECT {\"verbose\":false,\"protocol\":1}\r\nPING\r\n")) if _, err := c.Write(wsmsg); err != nil { t.Fatalf("Error sending message: %v", err) } // Wait for the PONG if msg := testWSReadFrame(t, r); !bytes.HasPrefix(msg, []byte("PONG\r\n")) { t.Fatalf("Expected PONG, got %s", msg) } select { case d := <-l.ch: ipAndSlash := fmt.Sprintf("%s/", test.expectedValue) if test.useHdrValue { if !strings.HasPrefix(d, ipAndSlash) { t.Fatalf("Expected debug statement to start with: %q, got %q", ipAndSlash, d) } } else if strings.HasPrefix(d, ipAndSlash) { t.Fatalf("Unexpected debug statement: %q", d) } case <-time.After(time.Second): t.Fatal("Did not get connect debug statement") } }) } } type partialWriteConn struct { net.Conn } func (c *partialWriteConn) Write(b []byte) (int, error) { max := len(b) if max > 0 { max = rand.Intn(max) if max == 0 { max = 1 } } n, err := c.Conn.Write(b[:max]) if err == nil && max != len(b) { err = io.ErrShortWrite } return n, err } func TestWSWithPartialWrite(t *testing.T) { conf := createConfFile(t, []byte(` listen: "127.0.0.1:-1" websocket { listen: "127.0.0.1:-1" no_tls: true } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() nc1 := natsConnect(t, fmt.Sprintf("ws://127.0.0.1:%d", o.Websocket.Port)) defer nc1.Close() sub := natsSubSync(t, nc1, "foo") sub.SetPendingLimits(-1, -1) natsFlush(t, nc1) nc2 := natsConnect(t, fmt.Sprintf("ws://127.0.0.1:%d", o.Websocket.Port)) defer nc2.Close() // Replace websocket connections with ones that will produce short writes. s.mu.RLock() for _, c := range s.clients { c.mu.Lock() c.nc = &partialWriteConn{Conn: c.nc} c.mu.Unlock() } s.mu.RUnlock() var msgs [][]byte for i := 0; i < 100; i++ { msg := make([]byte, rand.Intn(10000)+10) for j := 0; j < len(msg); j++ { msg[j] = byte('A' + j%26) } msgs = append(msgs, msg) natsPub(t, nc2, "foo", msg) } for i := 0; i < 100; i++ { rmsg := natsNexMsg(t, sub, time.Second) if !bytes.Equal(msgs[i], rmsg.Data) { t.Fatalf("Expected message %q, got %q", msgs[i], rmsg.Data) } } } func testWSNoCorruptionWithFrameSizeLimit(t *testing.T, total int) { tmpl := ` listen: "127.0.0.1:-1" cluster { name: "local" port: -1 %s } websocket { listen: "127.0.0.1:-1" no_tls: true } ` conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() routes := fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port) conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, routes))) s2, o2 := RunServerWithConfig(conf2) defer s2.Shutdown() conf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, routes))) s3, o3 := RunServerWithConfig(conf3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) nc3 := natsConnect(t, fmt.Sprintf("ws://127.0.0.1:%d", o3.Websocket.Port)) defer nc3.Close() nc2 := natsConnect(t, fmt.Sprintf("ws://127.0.0.1:%d", o2.Websocket.Port)) defer nc2.Close() payload := make([]byte, 100000) for i := 0; i < len(payload); i++ { payload[i] = 'A' + byte(i%26) } errCh := make(chan error, 1) doneCh := make(chan struct{}, 1) count := int32(0) createSub := func(nc *nats.Conn) { sub := natsSub(t, nc, "foo", func(m *nats.Msg) { if !bytes.Equal(m.Data, payload) { stop := len(m.Data) if l := len(payload); l < stop { stop = l } start := 0 for i := 0; i < stop; i++ { if m.Data[i] != payload[i] { start = i break } } if stop-start > 20 { stop = start + 20 } select { case errCh <- fmt.Errorf("Invalid message: [%d bytes same]%s[...]", start, m.Data[start:stop]): default: } return } if n := atomic.AddInt32(&count, 1); int(n) == 2*total { doneCh <- struct{}{} } }) sub.SetPendingLimits(-1, -1) } createSub(nc2) createSub(nc3) checkSubInterest(t, s1, globalAccountName, "foo", time.Second) nc1 := natsConnect(t, fmt.Sprintf("ws://127.0.0.1:%d", o1.Websocket.Port)) defer nc1.Close() natsFlush(t, nc1) // Change websocket connections to force a max frame size. for _, s := range []*Server{s1, s2, s3} { s.mu.RLock() for _, c := range s.clients { c.mu.Lock() if c.ws != nil { c.ws.browser = true } c.mu.Unlock() } s.mu.RUnlock() } for i := 0; i < total; i++ { natsPub(t, nc1, "foo", payload) if i%100 == 0 { select { case err := <-errCh: t.Fatalf("Error: %v", err) default: } } } select { case err := <-errCh: t.Fatalf("Error: %v", err) case <-doneCh: return case <-time.After(10 * time.Second): t.Fatalf("Test timed out") } } func TestWSNoCorruptionWithFrameSizeLimit(t *testing.T) { testWSNoCorruptionWithFrameSizeLimit(t, 1000) } // ================================================================== // = Benchmark tests // ================================================================== const testWSBenchSubject = "a" var ch = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$#%^&*()") func sizedString(sz int) string { b := make([]byte, sz) for i := range b { b[i] = ch[rand.Intn(len(ch))] } return string(b) } func sizedStringForCompression(sz int) string { b := make([]byte, sz) c := byte(0) s := 0 for i := range b { if s%20 == 0 { c = ch[rand.Intn(len(ch))] } b[i] = c } return string(b) } func testWSFlushConn(b *testing.B, compress bool, c net.Conn, br *bufio.Reader) { buf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, []byte(pingProto)) c.Write(buf) c.SetReadDeadline(time.Now().Add(5 * time.Second)) res := testWSReadFrame(b, br) c.SetReadDeadline(time.Time{}) if !bytes.HasPrefix(res, []byte(pongProto)) { b.Fatalf("Failed read of PONG: %s\n", res) } } func wsBenchPub(b *testing.B, numPubs int, compress bool, payload string) { b.StopTimer() opts := testWSOptions() opts.Websocket.Compression = compress s := RunServer(opts) defer s.Shutdown() extra := 0 pubProto := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", testWSBenchSubject, len(payload), payload)) singleOpBuf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, pubProto) // Simulate client that would buffer messages before framing/sending. // Figure out how many we can fit in one frame based on b.N and length of pubProto const bufSize = 32768 tmpa := [bufSize]byte{} tmp := tmpa[:0] pb := 0 for i := 0; i < b.N; i++ { tmp = append(tmp, pubProto...) pb++ if len(tmp) >= bufSize { break } } sendBuf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, tmp) n := b.N / pb extra = b.N - (n * pb) wg := sync.WaitGroup{} wg.Add(numPubs) type pub struct { c net.Conn br *bufio.Reader bw *bufio.Writer } var pubs []pub for i := 0; i < numPubs; i++ { wsc, br := testWSCreateClient(b, compress, false, opts.Websocket.Host, opts.Websocket.Port) defer wsc.Close() bw := bufio.NewWriterSize(wsc, bufSize) pubs = append(pubs, pub{wsc, br, bw}) } // Average the amount of bytes sent by iteration avg := len(sendBuf) / pb if extra > 0 { avg += len(singleOpBuf) avg /= 2 } b.SetBytes(int64(numPubs * avg)) b.StartTimer() for i := 0; i < numPubs; i++ { p := pubs[i] go func(p pub) { defer wg.Done() for i := 0; i < n; i++ { p.bw.Write(sendBuf) } for i := 0; i < extra; i++ { p.bw.Write(singleOpBuf) } p.bw.Flush() testWSFlushConn(b, compress, p.c, p.br) }(p) } wg.Wait() b.StopTimer() } func Benchmark_WS_Pubx1_CN_____0b(b *testing.B) { wsBenchPub(b, 1, false, "") } func Benchmark_WS_Pubx1_CY_____0b(b *testing.B) { wsBenchPub(b, 1, true, "") } func Benchmark_WS_Pubx1_CN___128b(b *testing.B) { s := sizedString(128) wsBenchPub(b, 1, false, s) } func Benchmark_WS_Pubx1_CY___128b(b *testing.B) { s := sizedStringForCompression(128) wsBenchPub(b, 1, true, s) } func Benchmark_WS_Pubx1_CN__1024b(b *testing.B) { s := sizedString(1024) wsBenchPub(b, 1, false, s) } func Benchmark_WS_Pubx1_CY__1024b(b *testing.B) { s := sizedStringForCompression(1024) wsBenchPub(b, 1, true, s) } func Benchmark_WS_Pubx1_CN__4096b(b *testing.B) { s := sizedString(4 * 1024) wsBenchPub(b, 1, false, s) } func Benchmark_WS_Pubx1_CY__4096b(b *testing.B) { s := sizedStringForCompression(4 * 1024) wsBenchPub(b, 1, true, s) } func Benchmark_WS_Pubx1_CN__8192b(b *testing.B) { s := sizedString(8 * 1024) wsBenchPub(b, 1, false, s) } func Benchmark_WS_Pubx1_CY__8192b(b *testing.B) { s := sizedStringForCompression(8 * 1024) wsBenchPub(b, 1, true, s) } func Benchmark_WS_Pubx1_CN_32768b(b *testing.B) { s := sizedString(32 * 1024) wsBenchPub(b, 1, false, s) } func Benchmark_WS_Pubx1_CY_32768b(b *testing.B) { s := sizedStringForCompression(32 * 1024) wsBenchPub(b, 1, true, s) } func Benchmark_WS_Pubx5_CN_____0b(b *testing.B) { wsBenchPub(b, 5, false, "") } func Benchmark_WS_Pubx5_CY_____0b(b *testing.B) { wsBenchPub(b, 5, true, "") } func Benchmark_WS_Pubx5_CN___128b(b *testing.B) { s := sizedString(128) wsBenchPub(b, 5, false, s) } func Benchmark_WS_Pubx5_CY___128b(b *testing.B) { s := sizedStringForCompression(128) wsBenchPub(b, 5, true, s) } func Benchmark_WS_Pubx5_CN__1024b(b *testing.B) { s := sizedString(1024) wsBenchPub(b, 5, false, s) } func Benchmark_WS_Pubx5_CY__1024b(b *testing.B) { s := sizedStringForCompression(1024) wsBenchPub(b, 5, true, s) } func Benchmark_WS_Pubx5_CN__4096b(b *testing.B) { s := sizedString(4 * 1024) wsBenchPub(b, 5, false, s) } func Benchmark_WS_Pubx5_CY__4096b(b *testing.B) { s := sizedStringForCompression(4 * 1024) wsBenchPub(b, 5, true, s) } func Benchmark_WS_Pubx5_CN__8192b(b *testing.B) { s := sizedString(8 * 1024) wsBenchPub(b, 5, false, s) } func Benchmark_WS_Pubx5_CY__8192b(b *testing.B) { s := sizedStringForCompression(8 * 1024) wsBenchPub(b, 5, true, s) } func Benchmark_WS_Pubx5_CN_32768b(b *testing.B) { s := sizedString(32 * 1024) wsBenchPub(b, 5, false, s) } func Benchmark_WS_Pubx5_CY_32768b(b *testing.B) { s := sizedStringForCompression(32 * 1024) wsBenchPub(b, 5, true, s) } func wsBenchSub(b *testing.B, numSubs int, compress bool, payload string) { b.StopTimer() opts := testWSOptions() opts.Websocket.Compression = compress s := RunServer(opts) defer s.Shutdown() var subs []*bufio.Reader for i := 0; i < numSubs; i++ { wsc, br := testWSCreateClient(b, compress, false, opts.Websocket.Host, opts.Websocket.Port) defer wsc.Close() subProto := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, []byte(fmt.Sprintf("SUB %s 1\r\nPING\r\n", testWSBenchSubject))) wsc.Write(subProto) // Waiting for PONG testWSReadFrame(b, br) subs = append(subs, br) } wg := sync.WaitGroup{} wg.Add(numSubs) // Use regular NATS client to publish messages nc := natsConnect(b, s.ClientURL()) defer nc.Close() b.StartTimer() for i := 0; i < numSubs; i++ { br := subs[i] go func(br *bufio.Reader) { defer wg.Done() for count := 0; count < b.N; { msgs := testWSReadFrame(b, br) count += bytes.Count(msgs, []byte("MSG ")) } }(br) } for i := 0; i < b.N; i++ { natsPub(b, nc, testWSBenchSubject, []byte(payload)) } wg.Wait() b.StopTimer() } func Benchmark_WS_Subx1_CN_____0b(b *testing.B) { wsBenchSub(b, 1, false, "") } func Benchmark_WS_Subx1_CY_____0b(b *testing.B) { wsBenchSub(b, 1, true, "") } func Benchmark_WS_Subx1_CN___128b(b *testing.B) { s := sizedString(128) wsBenchSub(b, 1, false, s) } func Benchmark_WS_Subx1_CY___128b(b *testing.B) { s := sizedStringForCompression(128) wsBenchSub(b, 1, true, s) } func Benchmark_WS_Subx1_CN__1024b(b *testing.B) { s := sizedString(1024) wsBenchSub(b, 1, false, s) } func Benchmark_WS_Subx1_CY__1024b(b *testing.B) { s := sizedStringForCompression(1024) wsBenchSub(b, 1, true, s) } func Benchmark_WS_Subx1_CN__4096b(b *testing.B) { s := sizedString(4096) wsBenchSub(b, 1, false, s) } func Benchmark_WS_Subx1_CY__4096b(b *testing.B) { s := sizedStringForCompression(4096) wsBenchSub(b, 1, true, s) } func Benchmark_WS_Subx1_CN__8192b(b *testing.B) { s := sizedString(8192) wsBenchSub(b, 1, false, s) } func Benchmark_WS_Subx1_CY__8192b(b *testing.B) { s := sizedStringForCompression(8192) wsBenchSub(b, 1, true, s) } func Benchmark_WS_Subx1_CN_32768b(b *testing.B) { s := sizedString(32768) wsBenchSub(b, 1, false, s) } func Benchmark_WS_Subx1_CY_32768b(b *testing.B) { s := sizedStringForCompression(32768) wsBenchSub(b, 1, true, s) } func Benchmark_WS_Subx5_CN_____0b(b *testing.B) { wsBenchSub(b, 5, false, "") } func Benchmark_WS_Subx5_CY_____0b(b *testing.B) { wsBenchSub(b, 5, true, "") } func Benchmark_WS_Subx5_CN___128b(b *testing.B) { s := sizedString(128) wsBenchSub(b, 5, false, s) } func Benchmark_WS_Subx5_CY___128b(b *testing.B) { s := sizedStringForCompression(128) wsBenchSub(b, 5, true, s) } func Benchmark_WS_Subx5_CN__1024b(b *testing.B) { s := sizedString(1024) wsBenchSub(b, 5, false, s) } func Benchmark_WS_Subx5_CY__1024b(b *testing.B) { s := sizedStringForCompression(1024) wsBenchSub(b, 5, true, s) } func Benchmark_WS_Subx5_CN__4096b(b *testing.B) { s := sizedString(4096) wsBenchSub(b, 5, false, s) } func Benchmark_WS_Subx5_CY__4096b(b *testing.B) { s := sizedStringForCompression(4096) wsBenchSub(b, 5, true, s) } func Benchmark_WS_Subx5_CN__8192b(b *testing.B) { s := sizedString(8192) wsBenchSub(b, 5, false, s) } func Benchmark_WS_Subx5_CY__8192b(b *testing.B) { s := sizedStringForCompression(8192) wsBenchSub(b, 5, true, s) } func Benchmark_WS_Subx5_CN_32768b(b *testing.B) { s := sizedString(32768) wsBenchSub(b, 5, false, s) } func Benchmark_WS_Subx5_CY_32768b(b *testing.B) { s := sizedStringForCompression(32768) wsBenchSub(b, 5, true, s) } nats-server-2.10.27/test/000077500000000000000000000000001477524627100151325ustar00rootroot00000000000000nats-server-2.10.27/test/accounts_cycles_test.go000066400000000000000000000353771477524627100217200ustar00rootroot00000000000000// Copyright 2020-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "strconv" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) func TestAccountCycleService(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: help } ] imports [ { service { subject: help, account: B } } ] } B { exports [ { service: help } ] imports [ { service { subject: help, account: A } } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } conf = createConfFile(t, []byte(` accounts { A { exports [ { service: * } ] imports [ { service { subject: help, account: B } } ] } B { exports [ { service: help } ] imports [ { service { subject: *, account: A } } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } conf = createConfFile(t, []byte(` accounts { A { exports [ { service: * } ] imports [ { service { subject: help, account: B } } ] } B { exports [ { service: help } ] imports [ { service { subject: help, account: C } } ] } C { exports [ { service: * } ] imports [ { service { subject: *, account: A } } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } } func TestAccountCycleStream(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { stream: strm } ] imports [ { stream { subject: strm, account: B } } ] } B { exports [ { stream: strm } ] imports [ { stream { subject: strm, account: A } } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cyclic import, got none") } } func TestAccountCycleStreamWithMapping(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { stream: * } ] imports [ { stream { subject: bar, account: B } } ] } B { exports [ { stream: bar } ] imports [ { stream { subject: foo, account: A }, to: bar } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cyclic import, got none") } } func TestAccountCycleNonCycleStreamWithMapping(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { stream: foo } ] imports [ { stream { subject: bar, account: B } } ] } B { exports [ { stream: bar } ] imports [ { stream { subject: baz, account: C }, to: bar } ] } C { exports [ { stream: baz } ] imports [ { stream { subject: foo, account: A }, to: bar } ] } } `)) if _, err := server.ProcessConfigFile(conf); err != nil { t.Fatalf("Expected no error but got %s", err) } } func TestAccountCycleServiceCycleWithMapping(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: a } ] imports [ { service { subject: b, account: B }, to: a } ] } B { exports [ { service: b } ] imports [ { service { subject: a, account: A }, to: b } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } } func TestAccountCycleServiceNonCycle(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: * } ] imports [ { service { subject: help, account: B } } ] } B { exports [ { service: help } ] imports [ { service { subject: nohelp, account: C } } ] } C { exports [ { service: * } ] imports [ { service { subject: *, account: A } } ] } } `)) if _, err := server.ProcessConfigFile(conf); err != nil { t.Fatalf("Expected no error but got %s", err) } } func TestAccountCycleServiceNonCycleChain(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: help } ] imports [ { service { subject: help, account: B } } ] } B { exports [ { service: help } ] imports [ { service { subject: help, account: C } } ] } C { exports [ { service: help } ] imports [ { service { subject: help, account: D } } ] } D { exports [ { service: help } ] } } `)) if _, err := server.ProcessConfigFile(conf); err != nil { t.Fatalf("Expected no error but got %s", err) } } // bug: https://github.com/nats-io/nats-server/issues/1769 func TestServiceImportReplyMatchCycle(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts { A { users: [{user: d, pass: x}] imports [ {service: {account: B, subject: ">" }}] } B { users: [{user: x, pass: x}] exports [ { service: ">" } ] } } no_auth_user: d `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc1 := clientConnectToServerWithUP(t, opts, "x", "x") defer nc1.Close() msg := []byte("HELLO") nc1.Subscribe("foo", func(m *nats.Msg) { m.Respond(msg) }) nc2 := clientConnectToServer(t, s) defer nc2.Close() resp, err := nc2.Request("foo", nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || string(resp.Data) != string(msg) { t.Fatalf("Wrong or empty response") } } func TestServiceImportReplyMatchCycleMultiHops(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts { A { users: [{user: d, pass: x}] imports [ {service: {account: B, subject: ">" }}] } B { exports [ { service: ">" } ] imports [ {service: {account: C, subject: ">" }}] } C { users: [{user: x, pass: x}] exports [ { service: ">" } ] } } no_auth_user: d `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc1 := clientConnectToServerWithUP(t, opts, "x", "x") defer nc1.Close() msg := []byte("HELLO") nc1.Subscribe("foo", func(m *nats.Msg) { m.Respond(msg) }) nc2 := clientConnectToServer(t, s) defer nc2.Close() resp, err := nc2.Request("foo", nil, time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || string(resp.Data) != string(msg) { t.Fatalf("Wrong or empty response") } } // Go's stack are infinite sans memory, but not call depth. However its good to limit. func TestAccountCycleDepthLimit(t *testing.T) { var last *server.Account chainLen := server.MaxAccountCycleSearchDepth + 1 // Services for i := 1; i <= chainLen; i++ { acc := server.NewAccount(fmt.Sprintf("ACC-%d", i)) if err := acc.AddServiceExport("*", nil); err != nil { t.Fatalf("Error adding service export to '*': %v", err) } if last != nil { err := acc.AddServiceImport(last, "foo", "foo") switch i { case chainLen: if err != server.ErrCycleSearchDepth { t.Fatalf("Expected last import to fail with '%v', but got '%v'", server.ErrCycleSearchDepth, err) } default: if err != nil { t.Fatalf("Error adding service import to 'foo': %v", err) } } } last = acc } last = nil // Streams for i := 1; i <= chainLen; i++ { acc := server.NewAccount(fmt.Sprintf("ACC-%d", i)) if err := acc.AddStreamExport("foo", nil); err != nil { t.Fatalf("Error adding stream export to '*': %v", err) } if last != nil { err := acc.AddStreamImport(last, "foo", "") switch i { case chainLen: if err != server.ErrCycleSearchDepth { t.Fatalf("Expected last import to fail with '%v', but got '%v'", server.ErrCycleSearchDepth, err) } default: if err != nil { t.Fatalf("Error adding stream import to 'foo': %v", err) } } } last = acc } } // Test token and partition subject mapping within an account func TestAccountSubjectMapping(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 mappings = { "foo.*.*" : "foo.$1.{{wildcard(2)}}.{{partition(10,1,2)}}" } `)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() nc1 := clientConnectToServer(t, s) defer nc1.Close() numMessages := 100 subjectsReceived := make(chan string) msg := []byte("HELLO") sub1, err := nc1.Subscribe("foo.*.*.*", func(m *nats.Msg) { subjectsReceived <- m.Subject }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub1.AutoUnsubscribe(numMessages * 2) nc1.Flush() nc2 := clientConnectToServer(t, s) defer nc2.Close() // publish numMessages with an increasing id (should map to partition numbers with the range of 10 partitions) - twice for j := 0; j < 2; j++ { for i := 0; i < numMessages; i++ { err = nc2.Publish(fmt.Sprintf("foo.%d.%d", i, numMessages-i), msg) if err != nil { t.Fatalf("Unexpected error: %v", err) } } } // verify all the partition numbers are in the expected range partitionsReceived := make([]int, numMessages) for i := 0; i < numMessages; i++ { var subject string select { case subject = <-subjectsReceived: case <-time.After(5 * time.Second): t.Fatal("Timed out waiting for messages") } sTokens := strings.Split(subject, ".") if err != nil { t.Fatalf("Unexpected error: %v", err) } t1, _ := strconv.Atoi(sTokens[1]) t2, _ := strconv.Atoi(sTokens[2]) partitionsReceived[i], err = strconv.Atoi(sTokens[3]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if partitionsReceived[i] > 9 || partitionsReceived[i] < 0 || t1 != i || t2 != numMessages-i { t.Fatalf("Error received unexpected %d.%d to partition %d", t1, t2, partitionsReceived[i]) } } // verify hashing is deterministic by checking it produces the same exact result twice for i := 0; i < numMessages; i++ { subject := <-subjectsReceived partitionNumber, err := strconv.Atoi(strings.Split(subject, ".")[3]) if err != nil { t.Fatalf("Unexpected error: %v", err) } if partitionsReceived[i] != partitionNumber { t.Fatalf("Error: same id mapped to two different partitions") } } } // test token subject mapping within an account // Alice imports from Bob with subject mapping func TestAccountImportSubjectMapping(t *testing.T) { conf := createConfFile(t, []byte(` port: -1 accounts { A { users: [{user: a, pass: x}] imports [ {stream: {account: B, subject: "foo.*.*"}, to : "foo.$1.{{wildcard(2)}}"}] } B { users: [{user: b, pass x}] exports [ { stream: ">" } ] } } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() ncA := clientConnectToServerWithUP(t, opts, "a", "x") defer ncA.Close() numMessages := 100 subjectsReceived := make(chan string) msg := []byte("HELLO") sub1, err := ncA.Subscribe("foo.*.*", func(m *nats.Msg) { subjectsReceived <- m.Subject }) if err != nil { t.Fatalf("Unexpected error: %v", err) } sub1.AutoUnsubscribe(numMessages) ncA.Flush() ncB := clientConnectToServerWithUP(t, opts, "b", "x") defer ncB.Close() // publish numMessages with an increasing id for i := 0; i < numMessages; i++ { err = ncB.Publish(fmt.Sprintf("foo.%d.%d", i, numMessages-i), msg) if err != nil { t.Fatalf("Unexpected error: %v", err) } } for i := 0; i < numMessages; i++ { var subject string select { case subject = <-subjectsReceived: case <-time.After(1 * time.Second): t.Fatal("Timed out waiting for messages") } sTokens := strings.Split(subject, ".") if err != nil { t.Fatalf("Unexpected error: %v", err) } t1, _ := strconv.Atoi(sTokens[1]) t2, _ := strconv.Atoi(sTokens[2]) if t1 != i || t2 != numMessages-i { t.Fatalf("Error received unexpected %d.%d", t1, t2) } } } // bug: https://github.com/nats-io/nats-server/issues/1789 func TestAccountCycleWithRenaming(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: * } ] imports [ { service { subject: foo, account: B }, to: foo } ] } B { exports [ { service: foo } ] imports [ { service { subject: *, account: A }, to: "$1" } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } conf = createConfFile(t, []byte(` accounts { A { exports [ { stream: * } ] imports [ { stream { subject: foo, account: B }, to: foo } ] } B { exports [ { stream: foo } ] imports [ { stream { subject: *, account: A }, to: "$1" } ] } } `)) if _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) { t.Fatalf("Expected an error on cycle service import, got none") } } // https://github.com/nats-io/nats-server/issues/5752 func TestAccountCycleFalsePositiveSubjectMapping(t *testing.T) { conf := createConfFile(t, []byte(` accounts { A { exports [ { service: "a.*" } ] imports [ { service { subject: "a.*", account: B }, to: "b.*" } ] } B { exports [ { service: "a.*" } ] imports [ { service { subject: "a.foo", account: A }, to: "c.foo" } ] } } `)) if _, err := server.ProcessConfigFile(conf); err != nil { t.Fatalf("Expected no errors on cycle service import, got error") } } func clientConnectToServer(t *testing.T, s *server.Server) *nats.Conn { t.Helper() nc, err := nats.Connect(s.ClientURL(), nats.Name("JS-TEST"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1)) if err != nil { t.Fatalf("Failed to create client: %v", err) } return nc } func clientConnectToServerWithUP(t *testing.T, opts *server.Options, user, pass string) *nats.Conn { curl := fmt.Sprintf("nats://%s:%s@%s:%d", user, pass, opts.Host, opts.Port) nc, err := nats.Connect(curl, nats.Name("JS-UP-TEST"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1)) if err != nil { t.Fatalf("Failed to create client: %v", err) } return nc } nats-server-2.10.27/test/auth_test.go000066400000000000000000000150561477524627100174700ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "fmt" "net" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) func doAuthConnect(t tLogger, c net.Conn, token, user, pass string) { cs := fmt.Sprintf("CONNECT {\"verbose\":true,\"auth_token\":\"%s\",\"user\":\"%s\",\"pass\":\"%s\"}\r\n", token, user, pass) sendProto(t, c, cs) } func testInfoForAuth(t tLogger, infojs []byte) bool { var sinfo server.Info err := json.Unmarshal(infojs, &sinfo) if err != nil { t.Fatalf("Could not unmarshal INFO json: %v\n", err) } return sinfo.AuthRequired } func expectAuthRequired(t tLogger, c net.Conn) { buf := expectResult(t, c, infoRe) infojs := infoRe.FindAllSubmatch(buf, 1)[0][1] if !testInfoForAuth(t, infojs) { t.Fatalf("Expected server to require authorization: '%s'", infojs) } } //////////////////////////////////////////////////////////// // The authorization token version //////////////////////////////////////////////////////////// const AUTH_PORT = 10422 const AUTH_TOKEN = "_YZZ22_" func runAuthServerWithToken() *server.Server { opts := DefaultTestOptions opts.Port = AUTH_PORT opts.Authorization = AUTH_TOKEN return RunServer(&opts) } func TestNoAuthClient(t *testing.T) { s := runAuthServerWithToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "", "") expectResult(t, c, errRe) } func TestAuthClientBadToken(t *testing.T) { s := runAuthServerWithToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "ZZZ", "", "") expectResult(t, c, errRe) } func TestAuthClientNoConnect(t *testing.T) { s := runAuthServerWithToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) // This is timing dependent.. time.Sleep(server.AUTH_TIMEOUT) expectResult(t, c, errRe) } func TestAuthClientGoodConnect(t *testing.T) { s := runAuthServerWithToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, AUTH_TOKEN, "", "") expectResult(t, c, okRe) } func TestAuthClientFailOnEverythingElse(t *testing.T) { s := runAuthServerWithToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) sendProto(t, c, "PUB foo 2\r\nok\r\n") expectResult(t, c, errRe) } //////////////////////////////////////////////////////////// // The username/password version //////////////////////////////////////////////////////////// const AUTH_USER = "derek" const AUTH_PASS = "foobar" func runAuthServerWithUserPass() *server.Server { opts := DefaultTestOptions opts.Port = AUTH_PORT opts.Username = AUTH_USER opts.Password = AUTH_PASS return RunServer(&opts) } func TestNoUserOrPasswordClient(t *testing.T) { s := runAuthServerWithUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "", "") expectResult(t, c, errRe) } func TestBadUserClient(t *testing.T) { s := runAuthServerWithUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "derekzz", AUTH_PASS) expectResult(t, c, errRe) } func TestBadPasswordClient(t *testing.T) { s := runAuthServerWithUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", AUTH_USER, "ZZ") expectResult(t, c, errRe) } func TestPasswordClientGoodConnect(t *testing.T) { s := runAuthServerWithUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", AUTH_USER, AUTH_PASS) expectResult(t, c, okRe) } //////////////////////////////////////////////////////////// // The bcrypt username/password version //////////////////////////////////////////////////////////// // Generated with nats server passwd (Cost 4 because of cost of --race, default is 11) const BCRYPT_AUTH_PASS = "IW@$6v(y1(t@fhPDvf!5^%" const BCRYPT_AUTH_HASH = "$2a$04$Q.CgCP2Sl9pkcTXEZHazaeMwPaAkSHk7AI51HkyMt5iJQQyUA4qxq" func runAuthServerWithBcryptUserPass() *server.Server { opts := DefaultTestOptions opts.Port = AUTH_PORT opts.Username = AUTH_USER opts.Password = BCRYPT_AUTH_HASH return RunServer(&opts) } func TestBadBcryptPassword(t *testing.T) { s := runAuthServerWithBcryptUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", AUTH_USER, BCRYPT_AUTH_HASH) expectResult(t, c, errRe) } func TestGoodBcryptPassword(t *testing.T) { s := runAuthServerWithBcryptUserPass() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", AUTH_USER, BCRYPT_AUTH_PASS) expectResult(t, c, okRe) } //////////////////////////////////////////////////////////// // The bcrypt authorization token version //////////////////////////////////////////////////////////// const BCRYPT_AUTH_TOKEN = "0uhJOSr3GW7xvHvtd^K6pa" const BCRYPT_AUTH_TOKEN_HASH = "$2a$04$u5ZClXpcjHgpfc61Ee0VKuwI1K3vTC4zq7SjphjnlHMeb1Llkb5Y6" func runAuthServerWithBcryptToken() *server.Server { opts := DefaultTestOptions opts.Port = AUTH_PORT opts.Authorization = BCRYPT_AUTH_TOKEN_HASH return RunServer(&opts) } func TestBadBcryptToken(t *testing.T) { s := runAuthServerWithBcryptToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, BCRYPT_AUTH_TOKEN_HASH, "", "") expectResult(t, c, errRe) } func TestGoodBcryptToken(t *testing.T) { s := runAuthServerWithBcryptToken() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", AUTH_PORT) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, BCRYPT_AUTH_TOKEN, "", "") expectResult(t, c, okRe) } nats-server-2.10.27/test/bench_results.txt000066400000000000000000000071121477524627100205340ustar00rootroot000000000000002017 iMac Pro 3Ghz (Turbo 4Ghz) 10-Core Skylake OSX High Sierra 10.13.2 =================== Go version go1.9.2 =================== Benchmark_____Pub0b_Payload-20 30000000 55.1 ns/op 199.78 MB/s Benchmark_____Pub8b_Payload-20 30000000 55.8 ns/op 340.21 MB/s Benchmark____Pub32b_Payload-20 20000000 63.4 ns/op 694.34 MB/s Benchmark___Pub128B_Payload-20 20000000 79.8 ns/op 1766.47 MB/s Benchmark___Pub256B_Payload-20 20000000 98.1 ns/op 2741.51 MB/s Benchmark_____Pub1K_Payload-20 5000000 283 ns/op 3660.72 MB/s Benchmark_____Pub4K_Payload-20 1000000 1395 ns/op 2945.30 MB/s Benchmark_____Pub8K_Payload-20 500000 2846 ns/op 2882.35 MB/s Benchmark_AuthPub0b_Payload-20 10000000 126 ns/op 86.82 MB/s Benchmark____________PubSub-20 10000000 135 ns/op Benchmark____PubSubTwoConns-20 10000000 136 ns/op Benchmark____PubTwoQueueSub-20 10000000 152 ns/op Benchmark___PubFourQueueSub-20 10000000 152 ns/op Benchmark__PubEightQueueSub-20 10000000 152 ns/op Benchmark___RoutedPubSub_0b-20 5000000 385 ns/op Benchmark___RoutedPubSub_1K-20 1000000 1076 ns/op Benchmark_RoutedPubSub_100K-20 20000 78501 ns/op 2015 iMac5k 4Ghz i7 Haswell OSX El Capitan 10.11.3 =================== Go version go1.6 =================== Benchmark____PubNo_Payload-8 20000000 88.6 ns/op 124.11 MB/s Benchmark____Pub8b_Payload-8 20000000 89.8 ns/op 211.63 MB/s Benchmark___Pub32b_Payload-8 20000000 97.3 ns/op 452.20 MB/s Benchmark__Pub256B_Payload-8 10000000 129 ns/op 2078.43 MB/s Benchmark____Pub1K_Payload-8 5000000 216 ns/op 4791.00 MB/s Benchmark____Pub4K_Payload-8 1000000 1123 ns/op 3657.53 MB/s Benchmark____Pub8K_Payload-8 500000 2309 ns/op 3553.09 MB/s Benchmark___________PubSub-8 10000000 210 ns/op Benchmark___PubSubTwoConns-8 10000000 205 ns/op Benchmark___PubTwoQueueSub-8 10000000 231 ns/op Benchmark__PubFourQueueSub-8 10000000 233 ns/op Benchmark_PubEightQueueSub-8 5000000 231 ns/op OSX Yosemite 10.10.5 =================== Go version go1.4.2 =================== Benchmark___PubNo_Payload 10000000 133 ns/op 82.44 MB/s Benchmark___Pub8b_Payload 10000000 135 ns/op 140.27 MB/s Benchmark__Pub32b_Payload 10000000 147 ns/op 297.56 MB/s Benchmark_Pub256B_Payload 10000000 211 ns/op 1273.82 MB/s Benchmark___Pub1K_Payload 3000000 447 ns/op 2321.55 MB/s Benchmark___Pub4K_Payload 1000000 1677 ns/op 2450.43 MB/s Benchmark___Pub8K_Payload 300000 3670 ns/op 2235.80 MB/s Benchmark__________PubSub 5000000 263 ns/op Benchmark__PubSubTwoConns 5000000 268 ns/op Benchmark__PubTwoQueueSub 2000000 936 ns/op Benchmark_PubFourQueueSub 1000000 1103 ns/op =================== Go version go1.5.0 =================== Benchmark___PubNo_Payload-8 10000000 122 ns/op 89.94 MB/s Benchmark___Pub8b_Payload-8 10000000 124 ns/op 152.72 MB/s Benchmark__Pub32b_Payload-8 10000000 135 ns/op 325.73 MB/s Benchmark_Pub256B_Payload-8 10000000 159 ns/op 1685.78 MB/s Benchmark___Pub1K_Payload-8 5000000 256 ns/op 4047.90 MB/s Benchmark___Pub4K_Payload-8 1000000 1164 ns/op 3530.77 MB/s Benchmark___Pub8K_Payload-8 500000 2444 ns/op 3357.34 MB/s Benchmark__________PubSub-8 5000000 254 ns/op Benchmark__PubSubTwoConns-8 5000000 245 ns/op Benchmark__PubTwoQueueSub-8 2000000 845 ns/op Benchmark_PubFourQueueSub-8 1000000 1004 ns/op nats-server-2.10.27/test/bench_test.go000066400000000000000000001100671477524627100176040ustar00rootroot00000000000000// Copyright 2012-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Please note that these tests will stress a system and they need generous // amounts of CPU, Memory and network sockets. Make sure the 'open files' // setting for your platform is at least 8192. On linux and MacOSX you can // do this via 'ulimit -n 8192' // package test import ( "bufio" "fmt" "math/rand" "net" "net/url" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) const PERF_PORT = 8422 // For Go routine based server. func runBenchServer() *server.Server { opts := DefaultTestOptions opts.Port = PERF_PORT opts.DisableShortFirstPing = true return RunServer(&opts) } const defaultRecBufSize = 32768 const defaultSendBufSize = 32768 func flushConnection(b *testing.B, c net.Conn) { buf := make([]byte, 32) c.Write([]byte("PING\r\n")) c.SetReadDeadline(time.Now().Add(5 * time.Second)) n, err := c.Read(buf) c.SetReadDeadline(time.Time{}) if err != nil { b.Fatalf("Failed read: %v\n", err) } if n != 6 && buf[0] != 'P' && buf[1] != 'O' { b.Fatalf("Failed read of PONG: %s\n", buf) } } func benchPub(b *testing.B, subject, payload string) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) b.SetBytes(int64(len(sendOp))) b.StartTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } bw.Flush() flushConnection(b, c) b.StopTimer() c.Close() s.Shutdown() } var ch = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$#%^&*()") func sizedBytes(sz int) []byte { b := make([]byte, sz) for i := range b { b[i] = ch[rand.Intn(len(ch))] } return b } func sizedString(sz int) string { return string(sizedBytes(sz)) } // Publish subject for pub benchmarks. var psub = "a" func Benchmark______Pub0b_Payload(b *testing.B) { benchPub(b, psub, "") } func Benchmark______Pub8b_Payload(b *testing.B) { b.StopTimer() s := sizedString(8) benchPub(b, psub, s) } func Benchmark_____Pub32b_Payload(b *testing.B) { b.StopTimer() s := sizedString(32) benchPub(b, psub, s) } func Benchmark____Pub128B_Payload(b *testing.B) { b.StopTimer() s := sizedString(128) benchPub(b, psub, s) } func Benchmark____Pub256B_Payload(b *testing.B) { b.StopTimer() s := sizedString(256) benchPub(b, psub, s) } func Benchmark______Pub1K_Payload(b *testing.B) { b.StopTimer() s := sizedString(1024) benchPub(b, psub, s) } func Benchmark______Pub4K_Payload(b *testing.B) { b.StopTimer() s := sizedString(4 * 1024) benchPub(b, psub, s) } func Benchmark______Pub8K_Payload(b *testing.B) { b.StopTimer() s := sizedString(8 * 1024) benchPub(b, psub, s) } func Benchmark_____Pub32K_Payload(b *testing.B) { b.StopTimer() s := sizedString(32 * 1024) benchPub(b, psub, s) } func drainConnection(b *testing.B, c net.Conn, ch chan bool, expected int) { buf := make([]byte, defaultRecBufSize) bytes := 0 for { c.SetReadDeadline(time.Now().Add(30 * time.Second)) n, err := c.Read(buf) if err != nil { b.Errorf("Error on read: %v\n", err) break } bytes += n if bytes >= expected { break } } if bytes != expected { b.Errorf("Did not receive all bytes: %d vs %d\n", bytes, expected) } ch <- true } // Benchmark the authorization code path. func Benchmark__AuthPub0b_Payload(b *testing.B) { b.StopTimer() srv, opts := RunServerWithConfig("./configs/authorization.conf") opts.DisableShortFirstPing = true defer srv.Shutdown() c := createClientConn(b, opts.Host, opts.Port) defer c.Close() expectAuthRequired(b, c) cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench", DefaultPass) sendProto(b, c, cs) bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB a 0\r\n\r\n") b.SetBytes(int64(len(sendOp))) b.StartTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } bw.Flush() flushConnection(b, c) b.StopTimer() } func Benchmark_____________PubSub(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) sendProto(b, c, "SUB foo 1\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Errorf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() s.Shutdown() } func Benchmark_____PubSubTwoConns(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) bw := bufio.NewWriterSize(c, defaultSendBufSize) c2 := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c2) sendProto(b, c2, "SUB foo 1\r\n") flushConnection(b, c2) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c2, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() c2.Close() s.Shutdown() } func benchDefaultOptionsForAccounts() *server.Options { o := DefaultTestOptions o.Host = "127.0.0.1" o.Port = -1 o.Cluster.Host = o.Host o.Cluster.Port = -1 o.DisableShortFirstPing = true fooAcc := server.NewAccount("$foo") fooAcc.AddStreamExport("foo", nil) barAcc := server.NewAccount("$bar") barAcc.AddStreamImport(fooAcc, "foo", "") o.Accounts = []*server.Account{fooAcc, barAcc} return &o } func createClientWithAccount(b *testing.B, account, host string, port int) net.Conn { c := createClientConn(b, host, port) checkInfoMsg(b, c) cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"account\":%q}\r\n", false, false, false, account) sendProto(b, c, cs) return c } func benchOptionsForServiceImports() *server.Options { o := DefaultTestOptions o.Host = "127.0.0.1" o.Port = -1 o.DisableShortFirstPing = true foo := server.NewAccount("$foo") bar := server.NewAccount("$bar") o.Accounts = []*server.Account{foo, bar} return &o } func addServiceImports(b *testing.B, s *server.Server) { // Add a bunch of service exports with wildcards, similar to JS. var exports = []string{ server.JSApiAccountInfo, server.JSApiTemplateCreate, server.JSApiTemplates, server.JSApiTemplateInfo, server.JSApiTemplateDelete, server.JSApiStreamCreate, server.JSApiStreamUpdate, server.JSApiStreams, server.JSApiStreamInfo, server.JSApiStreamDelete, server.JSApiStreamPurge, server.JSApiMsgDelete, server.JSApiConsumerCreate, server.JSApiConsumers, server.JSApiConsumerInfo, server.JSApiConsumerDelete, } foo, _ := s.LookupAccount("$foo") bar, _ := s.LookupAccount("$bar") for _, export := range exports { if err := bar.AddServiceExport(export, nil); err != nil { b.Fatalf("Could not add service export: %v", err) } if err := foo.AddServiceImport(bar, export, ""); err != nil { b.Fatalf("Could not add service import: %v", err) } } } func Benchmark__PubServiceImports(b *testing.B) { o := benchOptionsForServiceImports() s := RunServer(o) defer s.Shutdown() addServiceImports(b, s) c := createClientWithAccount(b, "$foo", o.Host, o.Port) defer c.Close() bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") b.ResetTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } b.StopTimer() } func Benchmark___PubSubAccsImport(b *testing.B) { o := benchDefaultOptionsForAccounts() s := RunServer(o) defer s.Shutdown() pub := createClientWithAccount(b, "$foo", o.Host, o.Port) defer pub.Close() sub := createClientWithAccount(b, "$bar", o.Host, o.Port) defer sub.Close() sendProto(b, sub, "SUB foo 1\r\n") flushConnection(b, sub) ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, sub, ch, expected) bw := bufio.NewWriterSize(pub, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") b.ResetTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() } func Benchmark_____PubTwoQueueSub(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) sendProto(b, c, "SUB foo group1 1\r\n") sendProto(b, c, "SUB foo group1 2\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Fatalf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() s.Shutdown() } func Benchmark____PubFourQueueSub(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) sendProto(b, c, "SUB foo group1 1\r\n") sendProto(b, c, "SUB foo group1 2\r\n") sendProto(b, c, "SUB foo group1 3\r\n") sendProto(b, c, "SUB foo group1 4\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Fatalf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() s.Shutdown() } func Benchmark___PubEightQueueSub(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) sendProto(b, c, "SUB foo group1 1\r\n") sendProto(b, c, "SUB foo group1 2\r\n") sendProto(b, c, "SUB foo group1 3\r\n") sendProto(b, c, "SUB foo group1 4\r\n") sendProto(b, c, "SUB foo group1 5\r\n") sendProto(b, c, "SUB foo group1 6\r\n") sendProto(b, c, "SUB foo group1 7\r\n") sendProto(b, c, "SUB foo group1 8\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Fatalf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() s.Shutdown() } func Benchmark_PubSub512kTwoConns(b *testing.B) { b.StopTimer() s := runBenchServer() c := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c) bw := bufio.NewWriterSize(c, defaultSendBufSize) c2 := createClientConn(b, "127.0.0.1", PERF_PORT) doDefaultConnect(b, c2) sendProto(b, c2, "SUB foo 1\r\n") flushConnection(b, c2) sz := 1024 * 512 payload := sizedString(sz) sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", sz, payload)) ch := make(chan bool) expected := len(fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", sz, payload)) * b.N go drainConnection(b, c2, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { bw.Write(sendOp) } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() c.Close() c2.Close() s.Shutdown() } func Benchmark__DenyMsgNoWCPubSub(b *testing.B) { s, opts := RunServerWithConfig("./configs/authorization.conf") opts.DisableShortFirstPing = true defer s.Shutdown() c := createClientConn(b, opts.Host, opts.Port) defer c.Close() expectAuthRequired(b, c) cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"pedantic\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench-deny", DefaultPass) sendProto(b, c, cs) sendProto(b, c, "SUB foo 1\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Errorf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch // To not count defer cleanup of client and server. b.StopTimer() } func Benchmark_DenyMsgYesWCPubSub(b *testing.B) { s, opts := RunServerWithConfig("./configs/authorization.conf") opts.DisableShortFirstPing = true defer s.Shutdown() c := createClientConn(b, opts.Host, opts.Port) defer c.Close() expectAuthRequired(b, c) cs := fmt.Sprintf("CONNECT {\"verbose\":false,\"pedantic\":false,\"user\":\"%s\",\"pass\":\"%s\"}\r\n", "bench-deny", DefaultPass) sendProto(b, c, cs) sendProto(b, c, "SUB * 1\r\n") bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte("PUB foo 2\r\nok\r\n") ch := make(chan bool) expected := len("MSG foo 1 2\r\nok\r\n") * b.N go drainConnection(b, c, ch, expected) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Errorf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch // To not count defer cleanup of client and server. b.StopTimer() } func routePubSub(b *testing.B, size int) { b.StopTimer() s1, o1 := RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 := RunServerWithConfig("./configs/srv_b.conf") o2.DisableShortFirstPing = true defer s2.Shutdown() sub := createClientConn(b, o1.Host, o1.Port) doDefaultConnect(b, sub) sendProto(b, sub, "SUB foo 1\r\n") flushConnection(b, sub) payload := sizedString(size) pub := createClientConn(b, o2.Host, o2.Port) doDefaultConnect(b, pub) bw := bufio.NewWriterSize(pub, defaultSendBufSize) ch := make(chan bool) sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) expected := len(fmt.Sprintf("MSG foo 1 %d\r\n%s\r\n", len(payload), payload)) * b.N go drainConnection(b, sub, ch, expected) b.StartTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() pub.Close() sub.Close() } func Benchmark____RoutedPubSub_0b(b *testing.B) { routePubSub(b, 2) } func Benchmark____RoutedPubSub_1K(b *testing.B) { routePubSub(b, 1024) } func Benchmark__RoutedPubSub_100K(b *testing.B) { routePubSub(b, 100*1024) } func routeQueue(b *testing.B, numQueueSubs, size int) { s1, o1 := RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 := RunServerWithConfig("./configs/srv_b.conf") o2.DisableShortFirstPing = true defer s2.Shutdown() sub := createClientConn(b, o1.Host, o1.Port) defer sub.Close() doDefaultConnect(b, sub) for i := 0; i < numQueueSubs; i++ { sendProto(b, sub, fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) } flushConnection(b, sub) payload := sizedString(size) pub := createClientConn(b, o2.Host, o2.Port) defer pub.Close() doDefaultConnect(b, pub) bw := bufio.NewWriterSize(pub, defaultSendBufSize) ch := make(chan bool) sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) expected := len(fmt.Sprintf("MSG foo 100 %d\r\n%s\r\n", len(payload), payload)) * b.N go drainConnection(b, sub, ch, expected) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) } // Wait for connection to be drained <-ch b.StopTimer() } func Benchmark____Routed2QueueSub(b *testing.B) { routeQueue(b, 2, 2) } func Benchmark____Routed4QueueSub(b *testing.B) { routeQueue(b, 4, 2) } func Benchmark____Routed8QueueSub(b *testing.B) { routeQueue(b, 8, 2) } func Benchmark___Routed16QueueSub(b *testing.B) { routeQueue(b, 16, 2) } func doS2CompressBench(b *testing.B, compress string) { b.StopTimer() conf1 := createConfFile(b, []byte(fmt.Sprintf(` port: -1 cluster { name: "local" port: -1 pool_size: -1 compression: %s } `, compress))) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(b, []byte(fmt.Sprintf(` port: -1 cluster { name: "local" port: -1 pool_size: -1 compression: %s routes: ["nats://127.0.0.1:%d"] } `, compress, o1.Cluster.Port))) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(b, s1, s2) nc2, err := nats.Connect(s2.ClientURL()) if err != nil { b.Fatalf("Error on connect: %v", err) } defer nc2.Close() ch := make(chan struct{}, 1) var count int nc2.Subscribe("foo", func(_ *nats.Msg) { if count++; count == b.N { select { case ch <- struct{}{}: default: } } }) checkSubInterest(b, s1, "$G", "foo", time.Second) nc1, err := nats.Connect(s1.ClientURL()) if err != nil { b.Fatalf("Error on connect: %v", err) } defer nc1.Close() // This one is easily compressible. payload1 := make([]byte, 128) // Make it random so that compression code has more to do. payload2 := make([]byte, 256) for i := 0; i < len(payload); i++ { payload2[i] = byte(rand.Intn(26) + 'A') } b.StartTimer() for i := 0; i < b.N; i++ { if i%2 == 0 { nc1.Publish("foo", payload1) } else { nc1.Publish("foo", payload2) } } select { case <-ch: return case <-time.After(10 * time.Second): b.Fatal("Timeout waiting to receive all messages") } } func Benchmark____________RouteCompressOff(b *testing.B) { doS2CompressBench(b, server.CompressionOff) } func Benchmark_RouteCompressS2Uncompressed(b *testing.B) { doS2CompressBench(b, server.CompressionS2Uncompressed) } func Benchmark_________RouteCompressS2Fast(b *testing.B) { doS2CompressBench(b, server.CompressionS2Fast) } func Benchmark_______RouteCompressS2Better(b *testing.B) { doS2CompressBench(b, server.CompressionS2Better) } func Benchmark_________RouteCompressS2Best(b *testing.B) { doS2CompressBench(b, server.CompressionS2Best) } func doFanout(b *testing.B, numServers, numConnections, subsPerConnection int, subject, payload string) { var s1, s2 *server.Server var o1, o2 *server.Options switch numServers { case 1: s1, o1 = RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 = s1, o1 case 2: s1, o1 = RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 = RunServerWithConfig("./configs/srv_b.conf") o2.DisableShortFirstPing = true defer s2.Shutdown() default: b.Fatalf("%d servers not supported for this test\n", numServers) } // To get a consistent length sid in MSG sent to us for drainConnection. var sidFloor int switch { case subsPerConnection <= 100: sidFloor = 100 case subsPerConnection <= 1000: sidFloor = 1000 case subsPerConnection <= 10000: sidFloor = 10000 default: b.Fatalf("Unsupported SubsPerConnection argument of %d\n", subsPerConnection) } msgOp := fmt.Sprintf("MSG %s %d %d\r\n%s\r\n", subject, sidFloor, len(payload), payload) expected := len(msgOp) * subsPerConnection * b.N // Client connections and subscriptions. clients := make([]chan bool, 0, numConnections) for i := 0; i < numConnections; i++ { c := createClientConn(b, o2.Host, o2.Port) doDefaultConnect(b, c) defer c.Close() ch := make(chan bool) clients = append(clients, ch) for s := 0; s < subsPerConnection; s++ { subOp := fmt.Sprintf("SUB %s %d\r\n", subject, sidFloor+s) sendProto(b, c, subOp) } flushConnection(b, c) go drainConnection(b, c, ch, expected) } // Publish Connection c := createClientConn(b, o1.Host, o1.Port) doDefaultConnect(b, c) flushConnection(b, c) bw := bufio.NewWriterSize(c, defaultSendBufSize) sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) b.SetBytes(int64(len(sendOp) + (len(msgOp) * numConnections * subsPerConnection))) b.ResetTimer() for i := 0; i < b.N; i++ { _, err := bw.Write(sendOp) if err != nil { b.Fatalf("Received error on PUB write: %v\n", err) } } err := bw.Flush() if err != nil { b.Fatalf("Received error on FLUSH write: %v\n", err) } // Wait for connections to be drained for i := 0; i < numConnections; i++ { <-clients[i] } b.StopTimer() } var sub = "x" var payload = "12345678" func Benchmark______FanOut_8x1x10(b *testing.B) { doFanout(b, 1, 1, 10, sub, payload) } func Benchmark_____FanOut_8x1x100(b *testing.B) { doFanout(b, 1, 1, 100, sub, payload) } func Benchmark____FanOut_8x10x100(b *testing.B) { doFanout(b, 1, 10, 100, sub, payload) } func Benchmark___FanOut_8x10x1000(b *testing.B) { doFanout(b, 1, 10, 1000, sub, payload) } func Benchmark___FanOut_8x100x100(b *testing.B) { doFanout(b, 1, 100, 100, sub, payload) } func Benchmark__FanOut_8x100x1000(b *testing.B) { doFanout(b, 1, 100, 1000, sub, payload) } func Benchmark__FanOut_8x10x10000(b *testing.B) { doFanout(b, 1, 10, 10000, sub, payload) } func Benchmark___FanOut_8x500x100(b *testing.B) { doFanout(b, 1, 500, 100, sub, payload) } func Benchmark___FanOut_128x1x100(b *testing.B) { doFanout(b, 1, 1, 100, sub, sizedString(128)) } func Benchmark__FanOut_128x10x100(b *testing.B) { doFanout(b, 1, 10, 100, sub, sizedString(128)) } func Benchmark_FanOut_128x10x1000(b *testing.B) { doFanout(b, 1, 10, 1000, sub, sizedString(128)) } func Benchmark_FanOut_128x100x100(b *testing.B) { doFanout(b, 1, 100, 100, sub, sizedString(128)) } func BenchmarkFanOut_128x100x1000(b *testing.B) { doFanout(b, 1, 100, 1000, sub, sizedString(128)) } func BenchmarkFanOut_128x10x10000(b *testing.B) { doFanout(b, 1, 10, 10000, sub, sizedString(128)) } func BenchmarkFanOut__128x500x100(b *testing.B) { doFanout(b, 1, 500, 100, sub, sizedString(128)) } func Benchmark_FanOut_512x100x100(b *testing.B) { doFanout(b, 1, 100, 100, sub, sizedString(512)) } func Benchmark__FanOut_512x100x1k(b *testing.B) { doFanout(b, 1, 100, 1000, sub, sizedString(512)) } func Benchmark____FanOut_1kx10x1k(b *testing.B) { doFanout(b, 1, 10, 1000, sub, sizedString(1024)) } func Benchmark__FanOut_1kx100x100(b *testing.B) { doFanout(b, 1, 100, 100, sub, sizedString(1024)) } func Benchmark_____RFanOut_8x1x10(b *testing.B) { doFanout(b, 2, 1, 10, sub, payload) } func Benchmark____RFanOut_8x1x100(b *testing.B) { doFanout(b, 2, 1, 100, sub, payload) } func Benchmark___RFanOut_8x10x100(b *testing.B) { doFanout(b, 2, 10, 100, sub, payload) } func Benchmark__RFanOut_8x10x1000(b *testing.B) { doFanout(b, 2, 10, 1000, sub, payload) } func Benchmark__RFanOut_8x100x100(b *testing.B) { doFanout(b, 2, 100, 100, sub, payload) } func Benchmark_RFanOut_8x100x1000(b *testing.B) { doFanout(b, 2, 100, 1000, sub, payload) } func Benchmark_RFanOut_8x10x10000(b *testing.B) { doFanout(b, 2, 10, 10000, sub, payload) } func Benchmark_RFanOut_1kx10x1000(b *testing.B) { doFanout(b, 2, 10, 1000, sub, sizedString(1024)) } func doFanIn(b *testing.B, numServers, numPublishers, numSubscribers int, subject, payload string) { b.Helper() if b.N < numPublishers { return } // Don't check for number of subscribers being lower than the number of publishers. // We also use this bench to show the performance impact of increased number of publishers, // and for those tests, the number of publishers will start at 1 and increase to 10, // while the number of subscribers will always be 3. if numSubscribers > 10 { b.Fatalf("numSubscribers should be <= 10") } var s1, s2 *server.Server var o1, o2 *server.Options switch numServers { case 1: s1, o1 = RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 = s1, o1 case 2: s1, o1 = RunServerWithConfig("./configs/srv_a.conf") o1.DisableShortFirstPing = true defer s1.Shutdown() s2, o2 = RunServerWithConfig("./configs/srv_b.conf") o2.DisableShortFirstPing = true defer s2.Shutdown() default: b.Fatalf("%d servers not supported for this test\n", numServers) } msgOp := fmt.Sprintf("MSG %s %d %d\r\n%s\r\n", subject, 9, len(payload), payload) l := b.N / numPublishers expected := len(msgOp) * l * numPublishers // Client connections and subscriptions. For fan in these are smaller then numPublishers. clients := make([]chan bool, 0, numSubscribers) for i := 0; i < numSubscribers; i++ { c := createClientConn(b, o2.Host, o2.Port) doDefaultConnect(b, c) defer c.Close() ch := make(chan bool) clients = append(clients, ch) subOp := fmt.Sprintf("SUB %s %d\r\n", subject, i) sendProto(b, c, subOp) flushConnection(b, c) go drainConnection(b, c, ch, expected) } sendOp := []byte(fmt.Sprintf("PUB %s %d\r\n%s\r\n", subject, len(payload), payload)) startCh := make(chan bool) pubLoop := func(c net.Conn, ch chan bool) { bw := bufio.NewWriterSize(c, defaultSendBufSize) // Signal we are ready close(ch) // Wait to start up actual sends. <-startCh for i := 0; i < l; i++ { _, err := bw.Write(sendOp) if err != nil { b.Errorf("Received error on PUB write: %v\n", err) return } } err := bw.Flush() if err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) return } } // Publish Connections SPINUP for i := 0; i < numPublishers; i++ { c := createClientConn(b, o1.Host, o1.Port) doDefaultConnect(b, c) flushConnection(b, c) ch := make(chan bool) go pubLoop(c, ch) <-ch } b.SetBytes(int64(len(sendOp) + len(msgOp))) b.ResetTimer() // Closing this will start all publishers at once (roughly) close(startCh) // Wait for connections to be drained for i := 0; i < len(clients); i++ { <-clients[i] } b.StopTimer() } func Benchmark_____FanIn_1kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(1024)) } func Benchmark_____FanIn_4kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(4096)) } func Benchmark_____FanIn_8kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(8192)) } func Benchmark____FanIn_16kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(16384)) } func Benchmark____FanIn_64kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(65536)) } func Benchmark___FanIn_128kx100x1(b *testing.B) { doFanIn(b, 1, 100, 1, sub, sizedString(65536*2)) } func Benchmark___BumpPubCount_1x3(b *testing.B) { doFanIn(b, 1, 1, 3, sub, sizedString(128)) } func Benchmark___BumpPubCount_2x3(b *testing.B) { doFanIn(b, 1, 2, 3, sub, sizedString(128)) } func Benchmark___BumpPubCount_5x3(b *testing.B) { doFanIn(b, 1, 5, 3, sub, sizedString(128)) } func Benchmark__BumpPubCount_10x3(b *testing.B) { doFanIn(b, 1, 10, 3, sub, sizedString(128)) } func testDefaultBenchOptionsForGateway(name string) *server.Options { opts := testDefaultOptionsForGateway(name) opts.DisableShortFirstPing = true return opts } func gatewaysBench(b *testing.B, optimisticMode bool, payload string, numPublishers int, subInterest bool) { b.Helper() if b.N < numPublishers { return } ob := testDefaultBenchOptionsForGateway("B") sb := RunServer(ob) defer sb.Shutdown() server.SetGatewaysSolicitDelay(10 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() gwbURL, err := url.Parse(fmt.Sprintf("nats://%s:%d", ob.Gateway.Host, ob.Gateway.Port)) if err != nil { b.Fatalf("Error parsing url: %v", err) } oa := testDefaultBenchOptionsForGateway("A") oa.Gateway.Gateways = []*server.RemoteGatewayOpts{ { Name: "B", URLs: []*url.URL{gwbURL}, }, } sa := RunServer(oa) defer sa.Shutdown() sub := createClientConn(b, ob.Host, ob.Port) defer sub.Close() doDefaultConnect(b, sub) sendProto(b, sub, "SUB end.test 1\r\n") if subInterest { sendProto(b, sub, "SUB foo 2\r\n") } flushConnection(b, sub) // If not optimisticMode, make B switch GW connection // to interest mode only if !optimisticMode { pub := createClientConn(b, oa.Host, oa.Port) doDefaultConnect(b, pub) // has to be more that defaultGatewayMaxRUnsubBeforeSwitch for i := 0; i < 2000; i++ { sendProto(b, pub, fmt.Sprintf("PUB reject.me.%d 2\r\nok\r\n", i+1)) } flushConnection(b, pub) pub.Close() } ch := make(chan bool) var msgOp string var expected int l := b.N / numPublishers if subInterest { msgOp = fmt.Sprintf("MSG foo 2 %d\r\n%s\r\n", len(payload), payload) expected = len(msgOp) * l * numPublishers } // Last message sent to end.test lastMsg := "MSG end.test 1 2\r\nok\r\n" expected += len(lastMsg) * numPublishers go drainConnection(b, sub, ch, expected) sendOp := []byte(fmt.Sprintf("PUB foo %d\r\n%s\r\n", len(payload), payload)) startCh := make(chan bool) lastMsgSendOp := []byte("PUB end.test 2\r\nok\r\n") pubLoop := func(c net.Conn, ch chan bool) { bw := bufio.NewWriterSize(c, defaultSendBufSize) // Signal we are ready close(ch) // Wait to start up actual sends. <-startCh for i := 0; i < l; i++ { if _, err := bw.Write(sendOp); err != nil { b.Errorf("Received error on PUB write: %v\n", err) return } } if _, err := bw.Write(lastMsgSendOp); err != nil { b.Errorf("Received error on PUB write: %v\n", err) return } if err := bw.Flush(); err != nil { b.Errorf("Received error on FLUSH write: %v\n", err) return } flushConnection(b, c) } // Publish Connections SPINUP for i := 0; i < numPublishers; i++ { c := createClientConn(b, oa.Host, oa.Port) defer c.Close() doDefaultConnect(b, c) flushConnection(b, c) ch := make(chan bool) go pubLoop(c, ch) <-ch } // To report the number of bytes: // from publisher to server on cluster A: numBytes := len(sendOp) if subInterest { // from server in cluster A to server on cluster B: // RMSG $G foo \r\n numBytes += len("RMSG $G foo xxxx ") + len(payload) + 2 // From server in cluster B to sub: numBytes += len(msgOp) } b.SetBytes(int64(numBytes)) b.ResetTimer() // Closing this will start all publishers at once (roughly) close(startCh) // Wait for end of test <-ch b.StopTimer() } func Benchmark____GWs_Opt_1kx01x0(b *testing.B) { gatewaysBench(b, true, sizedString(1024), 1, false) } func Benchmark____GWs_Opt_2kx01x0(b *testing.B) { gatewaysBench(b, true, sizedString(2048), 1, false) } func Benchmark____GWs_Opt_4kx01x0(b *testing.B) { gatewaysBench(b, true, sizedString(4096), 1, false) } func Benchmark____GWs_Opt_1kx10x0(b *testing.B) { gatewaysBench(b, true, sizedString(1024), 10, false) } func Benchmark____GWs_Opt_2kx10x0(b *testing.B) { gatewaysBench(b, true, sizedString(2048), 10, false) } func Benchmark____GWs_Opt_4kx10x0(b *testing.B) { gatewaysBench(b, true, sizedString(4096), 10, false) } func Benchmark____GWs_Opt_1kx01x1(b *testing.B) { gatewaysBench(b, true, sizedString(1024), 1, true) } func Benchmark____GWs_Opt_2kx01x1(b *testing.B) { gatewaysBench(b, true, sizedString(2048), 1, true) } func Benchmark____GWs_Opt_4kx01x1(b *testing.B) { gatewaysBench(b, true, sizedString(4096), 1, true) } func Benchmark____GWs_Opt_1kx10x1(b *testing.B) { gatewaysBench(b, true, sizedString(1024), 10, true) } func Benchmark____GWs_Opt_2kx10x1(b *testing.B) { gatewaysBench(b, true, sizedString(2048), 10, true) } func Benchmark____GWs_Opt_4kx10x1(b *testing.B) { gatewaysBench(b, true, sizedString(4096), 10, true) } func Benchmark____GWs_Int_1kx01x0(b *testing.B) { gatewaysBench(b, false, sizedString(1024), 1, false) } func Benchmark____GWs_Int_2kx01x0(b *testing.B) { gatewaysBench(b, false, sizedString(2048), 1, false) } func Benchmark____GWs_Int_4kx01x0(b *testing.B) { gatewaysBench(b, false, sizedString(4096), 1, false) } func Benchmark____GWs_Int_1kx10x0(b *testing.B) { gatewaysBench(b, false, sizedString(1024), 10, false) } func Benchmark____GWs_Int_2kx10x0(b *testing.B) { gatewaysBench(b, false, sizedString(2048), 10, false) } func Benchmark____GWs_Int_4kx10x0(b *testing.B) { gatewaysBench(b, false, sizedString(4096), 10, false) } func Benchmark____GWs_Int_1kx01x1(b *testing.B) { gatewaysBench(b, false, sizedString(1024), 1, true) } func Benchmark____GWs_Int_2kx01x1(b *testing.B) { gatewaysBench(b, false, sizedString(2048), 1, true) } func Benchmark____GWs_Int_4kx01x1(b *testing.B) { gatewaysBench(b, false, sizedString(4096), 1, true) } func Benchmark____GWs_Int_1kx10x1(b *testing.B) { gatewaysBench(b, false, sizedString(1024), 10, true) } func Benchmark____GWs_Int_2kx10x1(b *testing.B) { gatewaysBench(b, false, sizedString(2048), 10, true) } func Benchmark____GWs_Int_4kx10x1(b *testing.B) { gatewaysBench(b, false, sizedString(4096), 10, true) } // This bench only sends the requests to verify impact // of reply mapping in GW code. func gatewaySendRequestsBench(b *testing.B, singleReplySub bool) { server.SetGatewaysSolicitDelay(10 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ob := testDefaultBenchOptionsForGateway("B") sb := RunServer(ob) defer sb.Shutdown() gwbURL, err := url.Parse(fmt.Sprintf("nats://%s:%d", ob.Gateway.Host, ob.Gateway.Port)) if err != nil { b.Fatalf("Error parsing url: %v", err) } oa := testDefaultBenchOptionsForGateway("A") oa.Gateway.Gateways = []*server.RemoteGatewayOpts{ { Name: "B", URLs: []*url.URL{gwbURL}, }, } sa := RunServer(oa) defer sa.Shutdown() sub := createClientConn(b, ob.Host, ob.Port) defer sub.Close() doDefaultConnect(b, sub) sendProto(b, sub, "SUB foo 1\r\n") flushConnection(b, sub) lenMsg := len("MSG foo reply.xxxxxxxxxx 1 2\r\nok\r\n") expected := b.N * lenMsg ch := make(chan bool, 1) go drainConnection(b, sub, ch, expected) c := createClientConn(b, oa.Host, oa.Port) defer c.Close() doDefaultConnect(b, c) flushConnection(b, c) // From pub to server in cluster A: numBytes := len("PUB foo reply.0123456789 2\r\nok\r\n") if !singleReplySub { // Add the preceding SUB numBytes += len("SUB reply.0123456789 0123456789\r\n") // And UNSUB... numBytes += len("UNSUB 0123456789\r\n") } // From server in cluster A to cluster B numBytes += len("RMSG $G foo reply.0123456789 2\r\nok\r\n") // If mapping of reply... if !singleReplySub { // the mapping uses about 24 more bytes. So add them // for RMSG from server to server. numBytes += 24 } // From server in cluster B to sub numBytes += lenMsg b.SetBytes(int64(numBytes)) bw := bufio.NewWriterSize(c, defaultSendBufSize) var subStr string b.ResetTimer() for i := 0; i < b.N; i++ { if !singleReplySub { subStr = fmt.Sprintf("SUB reply.%010d %010d\r\n", i+1, i+1) } bw.Write([]byte(fmt.Sprintf("%sPUB foo reply.%010d 2\r\nok\r\n", subStr, i+1))) // Simulate that we are doing actual request/reply and therefore // unsub'ing the subs on the reply subject. if !singleReplySub && i > 1000 { bw.Write([]byte(fmt.Sprintf("UNSUB %010d\r\n", (i - 1000)))) } } bw.Flush() flushConnection(b, c) <-ch } func Benchmark__GWs_Reqs_1_SubAll(b *testing.B) { gatewaySendRequestsBench(b, true) } func Benchmark__GWs_Reqs_1SubEach(b *testing.B) { gatewaySendRequestsBench(b, false) } nats-server-2.10.27/test/client_auth_test.go000066400000000000000000000044431477524627100210240ustar00rootroot00000000000000// Copyright 2016-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "os" "testing" "github.com/nats-io/nats.go" ) func TestMultipleUserAuth(t *testing.T) { srv, opts := RunServerWithConfig("./configs/multi_user.conf") defer srv.Shutdown() if opts.Users == nil { t.Fatal("Expected a user array that is not nil") } if len(opts.Users) != 2 { t.Fatal("Expected a user array that had 2 users") } // Test first user url := fmt.Sprintf("nats://%s:%s@%s:%d/", opts.Users[0].Username, opts.Users[0].Password, opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Expected a successful connect, got %v\n", err) } defer nc.Close() if !nc.AuthRequired() { t.Fatal("Expected auth to be required for the server") } // Test second user url = fmt.Sprintf("nats://%s:%s@%s:%d/", opts.Users[1].Username, opts.Users[1].Password, opts.Host, opts.Port) nc, err = nats.Connect(url) if err != nil { t.Fatalf("Expected a successful connect, got %v\n", err) } defer nc.Close() } // Resolves to "test" const testToken = "$2a$05$3sSWEVA1eMCbV0hWavDjXOx.ClBjI6u1CuUdLqf22cbJjXsnzz8/." func TestTokenInConfig(t *testing.T) { content := ` listen: 127.0.0.1:4567 authorization={ token: ` + testToken + ` timeout: 5 }` confFile := createConfFile(t, []byte(content)) if err := os.WriteFile(confFile, []byte(content), 0666); err != nil { t.Fatalf("Error writing config file: %v", err) } s, opts := RunServerWithConfig(confFile) defer s.Shutdown() url := fmt.Sprintf("nats://test@%s:%d/", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Expected a successful connect, got %v\n", err) } defer nc.Close() if !nc.AuthRequired() { t.Fatal("Expected auth to be required for the server") } } nats-server-2.10.27/test/client_cluster_test.go000066400000000000000000000277121477524627100215500ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "math/rand" "strconv" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats.go" ) func TestServerRestartReSliceIssue(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) // msg to send.. msg := []byte("Hello World") servers := []string{urlA, urlB} opts := nats.GetDefaultOptions() opts.Timeout = (5 * time.Second) opts.ReconnectWait = (50 * time.Millisecond) opts.MaxReconnect = 1000 numClients := 20 reconnects := int32(0) reconnectsDone := make(chan bool, numClients) opts.ReconnectedCB = func(nc *nats.Conn) { atomic.AddInt32(&reconnects, 1) reconnectsDone <- true } clients := make([]*nats.Conn, numClients) // Create 20 random clients. // Half connected to A and half to B.. for i := 0; i < numClients; i++ { opts.Url = servers[i%2] nc, err := opts.Connect() if err != nil { t.Fatalf("Failed to create connection: %v\n", err) } clients[i] = nc defer nc.Close() // Create 10 subscriptions each.. for x := 0; x < 10; x++ { subject := fmt.Sprintf("foo.%d", (rand.Int()%50)+1) nc.Subscribe(subject, func(m *nats.Msg) { // Just eat it.. }) } // Pick one subject to send to.. subject := fmt.Sprintf("foo.%d", (rand.Int()%50)+1) go func() { time.Sleep(10 * time.Millisecond) for i := 1; i <= 100; i++ { if err := nc.Publish(subject, msg); err != nil { return } if i%10 == 0 { time.Sleep(time.Millisecond) } } }() } // Wait for a short bit.. time.Sleep(20 * time.Millisecond) // Restart SrvB srvB.Shutdown() srvB = RunServer(optsB) defer srvB.Shutdown() // Check that all expected clients have reconnected done := false for i := 0; i < numClients/2 && !done; i++ { select { case <-reconnectsDone: done = true case <-time.After(3 * time.Second): t.Fatalf("Expected %d reconnects, got %d\n", numClients/2, reconnects) } } // Since srvB was restarted, its defer Shutdown() was last, so will // exectue first, which would cause clients that have reconnected to // it to try to reconnect (causing delays on Windows). So let's // explicitly close them here. // NOTE: With fix of NATS GO client (reconnect loop yields to Close()), // this change would not be required, however, it still speeeds up // the test, from more than 7s to less than one. for i := 0; i < numClients; i++ { nc := clients[i] nc.Close() } } // This will test queue subscriber semantics across a cluster in the presence // of server restarts. func TestServerRestartAndQueueSubs(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) // Client options opts := nats.GetDefaultOptions() opts.Timeout = 5 * time.Second opts.ReconnectWait = 20 * time.Millisecond opts.MaxReconnect = 1000 opts.NoRandomize = true // Allow us to block on a reconnect completion. reconnectsDone := make(chan bool) opts.ReconnectedCB = func(nc *nats.Conn) { reconnectsDone <- true } // Helper to wait on a reconnect. waitOnReconnect := func() { t.Helper() select { case <-reconnectsDone: case <-time.After(2 * time.Second): t.Fatalf("Expected a reconnect, timedout!\n") } } // Create two clients.. opts.Servers = []string{urlA, urlB} c1, err := opts.Connect() if err != nil { t.Fatalf("Failed to create connection for c1: %v\n", err) } defer c1.Close() opts.Servers = []string{urlB, urlA} c2, err := opts.Connect() if err != nil { t.Fatalf("Failed to create connection for c2: %v\n", err) } defer c2.Close() // Flusher helper function. flush := func() { // Wait for processing. c1.Flush() c2.Flush() // Wait for a short bit for cluster propagation. time.Sleep(50 * time.Millisecond) } // To hold queue results. results := make(map[int]int) var mu sync.Mutex // This corresponds to the subsriptions below. const ExpectedMsgCount = 3 // Make sure we got what we needed, 1 msg only and all seqnos accounted for.. checkResults := func(numSent int) { mu.Lock() defer mu.Unlock() for i := 0; i < numSent; i++ { if results[i] != ExpectedMsgCount { t.Fatalf("Received incorrect number of messages, [%d] vs [%d] for seq: %d\n", results[i], ExpectedMsgCount, i) } } // Auto reset results map results = make(map[int]int) } subj := "foo.bar" qgroup := "workers" cb := func(msg *nats.Msg) { mu.Lock() defer mu.Unlock() seqno, _ := strconv.Atoi(string(msg.Data)) results[seqno] = results[seqno] + 1 } // Create queue subscribers c1.QueueSubscribe(subj, qgroup, cb) c2.QueueSubscribe(subj, qgroup, cb) // Do a wildcard subscription. c1.Subscribe("foo.*", cb) c2.Subscribe("foo.*", cb) // Wait for processing. flush() sendAndCheckMsgs := func(numToSend int) { for i := 0; i < numToSend; i++ { if i%2 == 0 { c1.Publish(subj, []byte(strconv.Itoa(i))) } else { c2.Publish(subj, []byte(strconv.Itoa(i))) } } // Wait for processing. flush() // Check Results checkResults(numToSend) } //////////////////////////////////////////////////////////////////////////// // Base Test //////////////////////////////////////////////////////////////////////////// // Make sure subscriptions are propagated in the cluster if err := checkExpectedSubs(4, srvA, srvB); err != nil { t.Fatalf("%v", err) } // Now send 10 messages, from each client.. sendAndCheckMsgs(10) //////////////////////////////////////////////////////////////////////////// // Now restart SrvA and srvB, re-run test //////////////////////////////////////////////////////////////////////////// srvA.Shutdown() // Wait for client on A to reconnect to B. waitOnReconnect() srvA = RunServer(optsA) defer srvA.Shutdown() srvB.Shutdown() // Now both clients should reconnect to A. waitOnReconnect() waitOnReconnect() srvB = RunServer(optsB) defer srvB.Shutdown() // Make sure the cluster is reformed checkClusterFormed(t, srvA, srvB) // Make sure subscriptions are propagated in the cluster // Clients will be connected to srvA, so that will be 4, // but srvB will only have 2 now since we coaelsce. if err := checkExpectedSubs(4, srvA); err != nil { t.Fatalf("%v", err) } if err := checkExpectedSubs(2, srvB); err != nil { t.Fatalf("%v", err) } // Now send another 10 messages, from each client.. sendAndCheckMsgs(10) // Since servers are restarted after all client's close defer calls, // their defer Shutdown() are last, and so will be executed first, // which would cause clients to try to reconnect on exit, causing // delays on Windows. So let's explicitly close them here. c1.Close() c2.Close() } // This will test request semantics across a route func TestRequestsAcrossRoutes(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Failed to create connection for nc1: %v\n", err) } defer nc1.Close() nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Failed to create connection for nc2: %v\n", err) } defer nc2.Close() response := []byte("I will help you") // Connect responder to srvA nc1.Subscribe("foo-req", func(m *nats.Msg) { nc1.Publish(m.Reply, response) }) // Make sure the route and the subscription are propagated. nc1.Flush() if err = checkExpectedSubs(1, srvA, srvB); err != nil { t.Fatal(err.Error()) } for i := 0; i < 100; i++ { if _, err = nc2.Request("foo-req", []byte(strconv.Itoa(i)), 250*time.Millisecond); err != nil { t.Fatalf("Received an error on Request test [%d]: %s", i, err) } } } // This will test request semantics across a route to queues func TestRequestsAcrossRoutesToQueues(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Failed to create connection for nc1: %v\n", err) } defer nc1.Close() nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Failed to create connection for nc2: %v\n", err) } defer nc2.Close() response := []byte("I will help you") // Connect one responder to srvA nc1.QueueSubscribe("foo-req", "booboo", func(m *nats.Msg) { nc1.Publish(m.Reply, response) }) // Make sure the route and the subscription are propagated. nc1.Flush() // Connect the other responder to srvB nc2.QueueSubscribe("foo-req", "booboo", func(m *nats.Msg) { nc2.Publish(m.Reply, response) }) if err = checkExpectedSubs(2, srvA, srvB); err != nil { t.Fatal(err.Error()) } for i := 0; i < 100; i++ { if _, err = nc2.Request("foo-req", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil { t.Fatalf("Received an error on Request test [%d]: %s", i, err) } } for i := 0; i < 100; i++ { if _, err = nc1.Request("foo-req", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil { t.Fatalf("Received an error on Request test [%d]: %s", i, err) } } } // This is in response to Issue #1144 // https://github.com/nats-io/nats-server/issues/1144 func TestQueueDistributionAcrossRoutes(t *testing.T) { srvA, srvB, _, _ := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) urlA := srvA.ClientURL() urlB := srvB.ClientURL() nc1, err := nats.Connect(urlA) if err != nil { t.Fatalf("Failed to create connection for nc1: %v\n", err) } defer nc1.Close() nc2, err := nats.Connect(urlB) if err != nil { t.Fatalf("Failed to create connection for nc2: %v\n", err) } defer nc2.Close() var qsubs []*nats.Subscription // Connect queue subscriptions as mentioned in the issue. 2(A) - 6(B) - 4(A) for i := 0; i < 2; i++ { sub, _ := nc1.QueueSubscribeSync("foo", "bar") qsubs = append(qsubs, sub) } nc1.Flush() for i := 0; i < 6; i++ { sub, _ := nc2.QueueSubscribeSync("foo", "bar") qsubs = append(qsubs, sub) } nc2.Flush() for i := 0; i < 4; i++ { sub, _ := nc1.QueueSubscribeSync("foo", "bar") qsubs = append(qsubs, sub) } nc1.Flush() if err := checkExpectedSubs(7, srvA, srvB); err != nil { t.Fatalf("%v", err) } send := 10000 for i := 0; i < send; i++ { nc2.Publish("foo", nil) } nc2.Flush() tp := func() int { var total int for i := 0; i < len(qsubs); i++ { pending, _, _ := qsubs[i].Pending() total += pending } return total } checkFor(t, time.Second, 10*time.Millisecond, func() error { if total := tp(); total != send { return fmt.Errorf("Number of total received %d", total) } return nil }) // The bug is essentially that when we deliver across a route, we // prefer locals, but if we randomize to a block of bounce backs, then // we walk to the end and find the same local for all the remote options. // So what you will see in this case is a large value at #9 (2+6, next one local). avg := send / len(qsubs) for i := 0; i < len(qsubs); i++ { total, _, _ := qsubs[i].Pending() if total > avg+(avg*3/10) { if i == 8 { t.Fatalf("Qsub in 8th position gets majority of the messages (prior 6 spots) in this test") } t.Fatalf("Received too high, %d vs %d", total, avg) } } } nats-server-2.10.27/test/cluster_test.go000066400000000000000000000417441477524627100202130ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "errors" "fmt" "math/rand" "runtime" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) // Helper function to check that a cluster is formed func checkClusterFormed(t testing.TB, servers ...*server.Server) { t.Helper() expectedNumRoutes := len(servers) - 1 checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { for _, s := range servers { if numRoutes := s.NumRoutes(); numRoutes != expectedNumRoutes { return fmt.Errorf("Expected %d routes for server %q, got %d", expectedNumRoutes, s.ID(), numRoutes) } } return nil }) } func checkNumRoutes(t *testing.T, s *server.Server, expected int) { t.Helper() checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { if nr := s.NumRoutes(); nr != expected { return fmt.Errorf("Expected %v routes, got %v", expected, nr) } return nil }) } // Helper function to check that a server (or list of servers) have the // expected number of subscriptions. func checkExpectedSubs(expected int, servers ...*server.Server) error { var err string maxTime := time.Now().Add(10 * time.Second) for time.Now().Before(maxTime) { err = "" for _, s := range servers { if numSubs := int(s.NumSubscriptions()); numSubs != expected { err = fmt.Sprintf("Expected %d subscriptions for server %q, got %d", expected, s.ID(), numSubs) break } } if err != "" { time.Sleep(10 * time.Millisecond) } else { break } } if err != "" { return errors.New(err) } return nil } func checkSubInterest(t testing.TB, s *server.Server, accName, subject string, timeout time.Duration) { t.Helper() checkFor(t, timeout, 15*time.Millisecond, func() error { acc, err := s.LookupAccount(accName) if err != nil { return fmt.Errorf("error looking up account %q: %v", accName, err) } if acc.SubscriptionInterest(subject) { return nil } return fmt.Errorf("no subscription interest for account %q on %q", accName, subject) }) } func checkNoSubInterest(t *testing.T, s *server.Server, accName, subject string, timeout time.Duration) { t.Helper() acc, err := s.LookupAccount(accName) if err != nil { t.Fatalf("error looking up account %q: %v", accName, err) } start := time.Now() for time.Now().Before(start.Add(timeout)) { if acc.SubscriptionInterest(subject) { t.Fatalf("Did not expect interest for %q", subject) } time.Sleep(5 * time.Millisecond) } } func runThreeServers(t *testing.T) (srvA, srvB, srvC *server.Server, optsA, optsB, optsC *server.Options) { srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") srvC, optsC = RunServerWithConfig("./configs/srv_c.conf") checkClusterFormed(t, srvA, srvB, srvC) return } func runServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) { srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") checkClusterFormed(t, srvA, srvB) return } func TestProperServerWithRoutesShutdown(t *testing.T) { before := runtime.NumGoroutine() srvA, srvB, _, _ := runServers(t) srvA.Shutdown() srvB.Shutdown() time.Sleep(100 * time.Millisecond) after := runtime.NumGoroutine() delta := after - before // There may be some finalizers or IO, but in general more than // 2 as a delta represents a problem. if delta > 2 { t.Fatalf("Expected same number of goroutines, %d vs %d\n", before, after) } } func TestDoubleRouteConfig(t *testing.T) { srvA, srvB, _, _ := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() } func TestBasicClusterPubSub(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupConn(t, clientA) sendA("SUB foo 22\r\n") sendA("PING\r\n") expectA(pongRe) if err := checkExpectedSubs(1, srvA, srvB); err != nil { t.Fatalf("%v", err) } sendB, expectB := setupConn(t, clientB) sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) expectMsgs := expectMsgsCommand(t, expectA) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "22", "", "2", "ok") } func TestClusterQueueSubs(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupConn(t, clientA) sendB, expectB := setupConn(t, clientB) expectMsgsA := expectMsgsCommand(t, expectA) expectMsgsB := expectMsgsCommand(t, expectB) // Capture sids for checking later. qg1SidsA := []string{"1", "2", "3"} // Three queue subscribers for _, sid := range qg1SidsA { sendA(fmt.Sprintf("SUB foo qg1 %s\r\n", sid)) } sendA("PING\r\n") expectA(pongRe) // Make sure the subs have propagated to srvB before continuing // New cluster proto this will only be 1. if err := checkExpectedSubs(1, srvB); err != nil { t.Fatalf("%v", err) } sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) // Make sure we get only 1. matches := expectMsgsA(1) checkMsg(t, matches[0], "foo", "", "", "2", "ok") // Capture sids for checking later. pSids := []string{"4", "5", "6"} // Create 3 normal subscribers for _, sid := range pSids { sendA(fmt.Sprintf("SUB foo %s\r\n", sid)) } // Create a FWC Subscriber pSids = append(pSids, "7") sendA("SUB > 7\r\n") sendA("PING\r\n") expectA(pongRe) // Make sure the subs have propagated to srvB before continuing // Normal foo and the queue group will be one a piece, so 2 + wc == 3 if err := checkExpectedSubs(3, srvB); err != nil { t.Fatalf("%v", err) } // Send to B sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) // Give plenty of time for the messages to flush, so that we don't // accidentally only read some of them. time.Sleep(time.Millisecond * 250) // Should receive 5. matches = expectMsgsA(5) checkForQueueSid(t, matches, qg1SidsA) checkForPubSids(t, matches, pSids) // Send to A sendA("PUB foo 2\r\nok\r\n") // Give plenty of time for the messages to flush, so that we don't // accidentally only read some of them. time.Sleep(time.Millisecond * 250) // Should receive 5. matches = expectMsgsA(5) checkForQueueSid(t, matches, qg1SidsA) checkForPubSids(t, matches, pSids) // Now add queue subscribers to B qg2SidsB := []string{"1", "2", "3"} for _, sid := range qg2SidsB { sendB(fmt.Sprintf("SUB foo qg2 %s\r\n", sid)) } sendB("PING\r\n") expectB(pongRe) // Make sure the subs have propagated to srvA before continuing // This will be all the subs on A and just 1 from B that gets coalesced. if err := checkExpectedSubs(len(qg1SidsA)+len(pSids)+1, srvA); err != nil { t.Fatalf("%v", err) } // Send to B sendB("PUB foo 2\r\nok\r\n") // Give plenty of time for the messages to flush, so that we don't // accidentally only read some of them. time.Sleep(time.Millisecond * 250) // Should receive 1 from B. matches = expectMsgsB(1) checkForQueueSid(t, matches, qg2SidsB) // Should receive 5 still from A. matches = expectMsgsA(5) checkForQueueSid(t, matches, qg1SidsA) checkForPubSids(t, matches, pSids) // Now drop queue subscribers from A for _, sid := range qg1SidsA { sendA(fmt.Sprintf("UNSUB %s\r\n", sid)) } sendA("PING\r\n") expectA(pongRe) // Make sure the subs have propagated to srvB before continuing if err := checkExpectedSubs(1+1+len(qg2SidsB), srvB); err != nil { t.Fatalf("%v", err) } // Send to B sendB("PUB foo 2\r\nok\r\n") // Should receive 1 from B. matches = expectMsgsB(1) checkForQueueSid(t, matches, qg2SidsB) sendB("PING\r\n") expectB(pongRe) // Should receive 4 now. matches = expectMsgsA(4) checkForPubSids(t, matches, pSids) // Send to A sendA("PUB foo 2\r\nok\r\n") // Give plenty of time for the messages to flush, so that we don't // accidentally only read some of them. time.Sleep(time.Millisecond * 250) // Should receive 4 now. matches = expectMsgsA(4) checkForPubSids(t, matches, pSids) } // Issue #22 func TestClusterDoubleMsgs(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA1 := createClientConn(t, optsA.Host, optsA.Port) defer clientA1.Close() clientA2 := createClientConn(t, optsA.Host, optsA.Port) defer clientA2.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA1, expectA1 := setupConn(t, clientA1) sendA2, expectA2 := setupConn(t, clientA2) sendB, expectB := setupConn(t, clientB) expectMsgsA1 := expectMsgsCommand(t, expectA1) expectMsgsA2 := expectMsgsCommand(t, expectA2) // Capture sids for checking later. qg1SidsA := []string{"1", "2", "3"} // Three queue subscribers for _, sid := range qg1SidsA { sendA1(fmt.Sprintf("SUB foo qg1 %s\r\n", sid)) } sendA1("PING\r\n") expectA1(pongRe) // Make sure the subs have propagated to srvB before continuing if err := checkExpectedSubs(1, srvB); err != nil { t.Fatalf("%v", err) } sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) // Make sure we get only 1. matches := expectMsgsA1(1) checkMsg(t, matches[0], "foo", "", "", "2", "ok") checkForQueueSid(t, matches, qg1SidsA) // Add a FWC subscriber on A2 sendA2("SUB > 1\r\n") sendA2("SUB foo 2\r\n") sendA2("PING\r\n") expectA2(pongRe) pSids := []string{"1", "2"} // Make sure the subs have propagated to srvB before continuing if err := checkExpectedSubs(1+2, srvB); err != nil { t.Fatalf("%v", err) } sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) matches = expectMsgsA1(1) checkMsg(t, matches[0], "foo", "", "", "2", "ok") checkForQueueSid(t, matches, qg1SidsA) matches = expectMsgsA2(2) checkMsg(t, matches[0], "foo", "", "", "2", "ok") checkForPubSids(t, matches, pSids) // Close ClientA1 clientA1.Close() sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) time.Sleep(10 * time.Millisecond) matches = expectMsgsA2(2) checkMsg(t, matches[0], "foo", "", "", "2", "ok") checkForPubSids(t, matches, pSids) } // This will test that we drop remote sids correctly. func TestClusterDropsRemoteSids(t *testing.T) { srvA, srvB, optsA, _ := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConn(t, clientA) // Add a subscription sendA("SUB foo 1\r\n") sendA("PING\r\n") expectA(pongRe) // Wait for propagation. time.Sleep(100 * time.Millisecond) if sc := srvA.NumSubscriptions(); sc != 1 { t.Fatalf("Expected one subscription for srvA, got %d\n", sc) } if sc := srvB.NumSubscriptions(); sc != 1 { t.Fatalf("Expected one subscription for srvB, got %d\n", sc) } // Add another subscription sendA("SUB bar 2\r\n") sendA("PING\r\n") expectA(pongRe) // Wait for propagation. time.Sleep(100 * time.Millisecond) if sc := srvA.NumSubscriptions(); sc != 2 { t.Fatalf("Expected two subscriptions for srvA, got %d\n", sc) } if sc := srvB.NumSubscriptions(); sc != 2 { t.Fatalf("Expected two subscriptions for srvB, got %d\n", sc) } // unsubscription sendA("UNSUB 1\r\n") sendA("PING\r\n") expectA(pongRe) // Wait for propagation. time.Sleep(100 * time.Millisecond) if sc := srvA.NumSubscriptions(); sc != 1 { t.Fatalf("Expected one subscription for srvA, got %d\n", sc) } if sc := srvB.NumSubscriptions(); sc != 1 { t.Fatalf("Expected one subscription for srvB, got %d\n", sc) } // Close the client and make sure we remove subscription state. clientA.Close() // Wait for propagation. time.Sleep(100 * time.Millisecond) if sc := srvA.NumSubscriptions(); sc != 0 { t.Fatalf("Expected no subscriptions for srvA, got %d\n", sc) } if sc := srvB.NumSubscriptions(); sc != 0 { t.Fatalf("Expected no subscriptions for srvB, got %d\n", sc) } } // This will test that we drop remote sids correctly. func TestAutoUnsubscribePropagation(t *testing.T) { srvA, srvB, optsA, _ := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConn(t, clientA) expectMsgs := expectMsgsCommand(t, expectA) // We will create subscriptions that will auto-unsubscribe and make sure // we are not accumulating orphan subscriptions on the other side. for i := 1; i <= 100; i++ { sub := fmt.Sprintf("SUB foo %d\r\n", i) auto := fmt.Sprintf("UNSUB %d 1\r\n", i) sendA(sub) sendA(auto) // This will trip the auto-unsubscribe sendA("PUB foo 2\r\nok\r\n") expectMsgs(1) } sendA("PING\r\n") expectA(pongRe) time.Sleep(50 * time.Millisecond) // Make sure number of subscriptions on B is correct if subs := srvB.NumSubscriptions(); subs != 0 { t.Fatalf("Expected no subscriptions on remote server, got %d\n", subs) } } func TestAutoUnsubscribePropagationOnClientDisconnect(t *testing.T) { srvA, srvB, optsA, _ := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() cluster := []*server.Server{srvA, srvB} clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConn(t, clientA) // No subscriptions. Ready to test. if err := checkExpectedSubs(0, cluster...); err != nil { t.Fatalf("%v", err) } sendA("SUB foo 1\r\n") sendA("UNSUB 1 1\r\n") sendA("PING\r\n") expectA(pongRe) // Waiting cluster subs propagation if err := checkExpectedSubs(1, cluster...); err != nil { t.Fatalf("%v", err) } clientA.Close() // No subs should be on the cluster when all clients is disconnected if err := checkExpectedSubs(0, cluster...); err != nil { t.Fatalf("%v", err) } } func TestClusterNameOption(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { name: MyCluster listen: 127.0.0.1:-1 } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() si := checkInfoMsg(t, c) if si.Cluster != "MyCluster" { t.Fatalf("Expected a cluster name of %q, got %q", "MyCluster", si.Cluster) } } func TestEphemeralClusterName(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:-1 } `)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() si := checkInfoMsg(t, c) if si.Cluster == "" { t.Fatalf("Expected an ephemeral cluster name to be set") } } type captureErrLogger struct { dummyLogger ch chan string } func (c *captureErrLogger) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) select { case c.ch <- msg: default: } } func TestClusterNameConflictsDropRoutes(t *testing.T) { ll := &captureErrLogger{ch: make(chan string, 4)} conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { name: MyCluster33 listen: 127.0.0.1:5244 } `)) s1, _ := RunServerWithConfig(conf) defer s1.Shutdown() s1.SetLogger(ll, false, false) conf2 := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { name: MyCluster22 listen: 127.0.0.1:-1 routes = [nats-route://127.0.0.1:5244] } `)) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() s2.SetLogger(ll, false, false) select { case msg := <-ll.ch: if !strings.Contains(msg, "Rejecting connection") || !strings.Contains(msg, "does not match") { t.Fatalf("Got bad error about cluster name mismatch") } case <-time.After(time.Second): t.Fatalf("Expected an error, timed out") } } func TestClusterNameDynamicNegotiation(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster {listen: 127.0.0.1:5244} `)) seed, _ := RunServerWithConfig(conf) defer seed.Shutdown() oconf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 cluster { listen: 127.0.0.1:-1 routes = [nats-route://127.0.0.1:5244] } `)) // Create a random number of additional servers, up to 20. numServers := rand.Intn(20) + 1 servers := make([]*server.Server, 0, numServers+1) servers = append(servers, seed) for i := 0; i < numServers; i++ { s, _ := RunServerWithConfig(oconf) defer s.Shutdown() servers = append(servers, s) } // If this passes we should have all the same name. checkClusterFormed(t, servers...) clusterName := seed.ClusterName() for _, s := range servers { if s.ClusterName() != clusterName { t.Fatalf("Expected the cluster names to all be the same as %q, got %q", clusterName, s.ClusterName()) } } } nats-server-2.10.27/test/cluster_tls_test.go000066400000000000000000000102611477524627100210630ustar00rootroot00000000000000// Copyright 2013-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "os" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) func runTLSServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) { srvA, optsA = RunServerWithConfig("./configs/srv_a_tls.conf") srvB, optsB = RunServerWithConfig("./configs/srv_b_tls.conf") checkClusterFormed(t, srvA, srvB) return } func TestTLSClusterConfig(t *testing.T) { srvA, srvB, _, _ := runTLSServers(t) defer srvA.Shutdown() defer srvB.Shutdown() } func TestBasicTLSClusterPubSub(t *testing.T) { srvA, srvB, optsA, optsB := runTLSServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupConn(t, clientA) sendA("SUB foo 22\r\n") sendA("PING\r\n") expectA(pongRe) if err := checkExpectedSubs(1, srvA, srvB); err != nil { t.Fatalf("%v", err) } sendB, expectB := setupConn(t, clientB) sendB("PUB foo 2\r\nok\r\n") sendB("PING\r\n") expectB(pongRe) expectMsgs := expectMsgsCommand(t, expectA) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "22", "", "2", "ok") } type captureTLSError struct { dummyLogger ch chan struct{} } func (c *captureTLSError) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "handshake error") { select { case c.ch <- struct{}{}: default: } } } type captureClusterTLSInsecureLogger struct { dummyLogger ch chan struct{} } func (c *captureClusterTLSInsecureLogger) Warnf(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "solicited routes will not be verified") { select { case c.ch <- struct{}{}: default: } } } func TestClusterTLSInsecure(t *testing.T) { confA := createConfFile(t, []byte(` port: -1 cluster { name: "xyz" listen: "127.0.0.1:-1" pool_size: -1 compression: "disabled" tls { cert_file: "./configs/certs/server-noip.pem" key_file: "./configs/certs/server-key-noip.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 } } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() l := &captureTLSError{ch: make(chan struct{}, 1)} srvA.SetLogger(l, false, false) bConfigTemplate := ` port: -1 cluster { name: "xyz" listen: "127.0.0.1:-1" pool_size: -1 compression: "disabled" tls { cert_file: "./configs/certs/server-noip.pem" key_file: "./configs/certs/server-key-noip.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 %s } routes [ "nats://%s:%d" ] } ` confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, "", optsA.Cluster.Host, optsA.Cluster.Port))) srvB, _ := RunServerWithConfig(confB) defer srvB.Shutdown() // We should get errors select { case <-l.ch: case <-time.After(2 * time.Second): t.Fatalf("Did not get handshake error") } // Set a logger that will capture the warning wl := &captureClusterTLSInsecureLogger{ch: make(chan struct{}, 1)} srvB.SetLogger(wl, false, false) // Need to add "insecure: true" and reload if err := os.WriteFile(confB, []byte(fmt.Sprintf(bConfigTemplate, "insecure: true", optsA.Cluster.Host, optsA.Cluster.Port)), 0666); err != nil { t.Fatalf("Error rewriting file: %v", err) } if err := srvB.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } checkClusterFormed(t, srvA, srvB) // Make sure we have the tracing select { case <-wl.ch: case <-time.After(2 * time.Second): t.Fatalf("Did not get warning about using cluster's insecure setting") } } nats-server-2.10.27/test/configs/000077500000000000000000000000001477524627100165625ustar00rootroot00000000000000nats-server-2.10.27/test/configs/auth_seed.conf000066400000000000000000000003131477524627100213670ustar00rootroot00000000000000# Cluster Seed Node listen: 127.0.0.1:5222 http: 8222 cluster { listen: 127.0.0.1:4248 name: xyz authorization { user: ruser password: T0PS3cr3T! timeout: 1 } } no_sys_acc: true nats-server-2.10.27/test/configs/authorization.conf000066400000000000000000000013521477524627100223320ustar00rootroot00000000000000listen: 127.0.0.1:2442 authorization { # Authorizations include "auths.conf" # Just foo for testing PASS: $2a$04$P/.bd.7unw9Ew7yWJqXsl.f4oNRLQGvadEL2YnqQXbbb.IVQajRdK # Users listed with permissions. users = [ {user: alice, password: $PASS, permissions: $ADMIN} {user: bob, password: $PASS, permissions: $REQUESTOR} {user: bench, password: $PASS, permissions: $BENCH} {user: joe, password: $PASS} {user: ns, password: $PASS, permissions: $NEW_STYLE} {user: ns-pub, password: $PASS, permissions: $NS_PUB} {user: bench-deny, password: $PASS, permissions: $BENCH_DENY} {user: svca, password: $PASS, permissions: $MY_SERVICE} {user: svcb, password: $PASS, permissions: $MY_STREAM_SERVICE} ] } nats-server-2.10.27/test/configs/auths.conf000066400000000000000000000024471477524627100205640ustar00rootroot00000000000000# Our role based permissions. # Admin can do anything. ADMIN = { publish = ">" subscribe = ">" } # Can do requests on req.foo or req.bar, and subscribe to anything # that is a response, e.g. _INBOX.* # # Notice that authorization filters can be singletons or arrays. REQUESTOR = { publish = ["req.foo", "req.bar"] subscribe = "_INBOX.*" } # Default permissions if none presented. e.g. Joe below. DEFAULT_PERMISSIONS = { publish = "SANDBOX.*" subscribe = ["PUBLIC.>", "_INBOX.>"] } # This is to benchmark pub performance. BENCH = { publish = "a" } # New Style Permissions NEW_STYLE = { publish = { allow = "*.*" deny = ["SYS.*", "bar.baz", "foo.*"] } subscribe = { allow = ["foo.*", "SYS.TEST.>"] deny = ["foo.baz", "SYS.*"] } } NS_PUB = { publish = "foo.baz" subscribe = "foo.baz" } BENCH_DENY = { subscribe = { allow = ["foo", "*"] deny = "foo.bar" } } # This is for services where you only want # responses to reply subjects to be allowed. MY_SERVICE = { subscribe = "my.service.req" publish_allow_responses: true } # This is a more detailed example where responses # could be streams and you want to set the TTL # and maximum allowed. MY_STREAM_SERVICE = { subscribe = "my.service.req" allow_responses = {max: 10, ttl: "50ms"} } nats-server-2.10.27/test/configs/certs/000077500000000000000000000000001477524627100177025ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ca.pem000066400000000000000000000031531477524627100207720ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEkDCCA3igAwIBAgIUSZwW7btc9EUbrMWtjHpbM0C2bSEwDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmlj YXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAqilVqyY8rmCpTwAsLF7DEtWEq37KbljBWVjmlp2Wo6TgMd3b537t 6iO8+SbI8KH75i63RcxV3Uzt1/L9Yb6enDXF52A/U5ugmDhaa+Vsoo2HBTbCczmp qndp7znllQqn7wNLv6aGSvaeIUeYS5Dmlh3kt7Vqbn4YRANkOUTDYGSpMv7jYKSu 1ee05Rco3H674zdwToYto8L8V7nVMrky42qZnGrJTaze+Cm9tmaIyHCwUq362CxS dkmaEuWx11MOIFZvL80n7ci6pveDxe5MIfwMC3/oGn7mbsSqidPMcTtjw6ey5NEu Z0UrC/2lL1FtF4gnVMKUSaEhU2oKjj0ZAQIDAQABo4IBHjCCARowHQYDVR0OBBYE FP7Pfz4u7sSt6ltviEVsx4hIFIs6MIGuBgNVHSMEgaYwgaOAFP7Pfz4u7sSt6ltv iEVsx4hIFIs6oXWkczBxMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5p YTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEpMCcGA1UEAwwg Q2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMjItMDgtMjeCFEmcFu27XPRFG6zFrYx6 WzNAtm0hMAwGA1UdEwQFMAMBAf8wOgYJYIZIAYb4QgENBC0WK25hdHMuaW8gbmF0 cy1zZXJ2ZXIgdGVzdC1zdWl0ZSB0cmFuc2llbnQgQ0EwDQYJKoZIhvcNAQELBQAD ggEBAHDCHLQklYZlnzHDaSwxgGSiPUrCf2zhk2DNIYSDyBgdzrIapmaVYQRrCBtA j/4jVFesgw5WDoe4TKsyha0QeVwJDIN8qg2pvpbmD8nOtLApfl0P966vcucxDwqO dQWrIgNsaUdHdwdo0OfvAlTfG0v/y2X0kbL7h/el5W9kWpxM/rfbX4IHseZL2sLq FH69SN3FhMbdIm1ldrcLBQVz8vJAGI+6B9hSSFQWljssE0JfAX+8VW/foJgMSx7A vBTq58rLkAko56Jlzqh/4QT+ckayg9I73v1Q5/44jP1mHw35s5ZrzpDQt2sVv4l5 lwRPJFXMwe64flUs9sM+/vqJaIY= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/client-cert.pem000066400000000000000000000126411477524627100226220ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 38:4c:16:24:9b:04:1c:b3:db:e0:4c:3c:ed:b7:40:7d:68:b5:fa:1f Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:ac:9c:3e:9d:3b:7a:12:56:85:78:ca:df:9c:fc: 0c:7e:5e:f2:4f:22:33:46:81:38:53:d7:a7:25:8f: d7:ee:16:13:e2:67:49:88:f6:94:99:f0:a9:a6:db: fe:7a:17:c9:e3:df:31:73:71:38:70:3a:96:1e:99: 7b:5d:07:e3:63:e4:e8:bf:99:f7:3d:5c:27:f5:b7: 37:29:da:ee:82:80:00:d4:c8:d3:1b:36:0d:8b:d3: 8a:9b:8e:12:a1:4d:0c:c5:22:f8:56:3b:6a:1a:fb: e9:3d:08:1e:13:7f:55:6e:2e:65:93:9a:90:54:03: 6d:0d:e6:44:d6:f7:c0:d7:d8:e1:c7:1e:c2:9b:a3: 6e:88:f1:7c:58:08:a2:9f:13:cc:5b:b9:11:2c:1d: 23:6f:3a:ae:47:9a:0f:6a:ce:e5:80:34:09:e6:e3: fd:76:4a:cf:5a:18:bb:9c:c5:c1:74:49:67:77:1b: ba:28:86:31:a6:fc:12:af:4a:85:1b:73:5b:f4:d6: 42:ff:0c:1c:49:e7:31:f2:5a:2a:1e:cd:87:cb:22: ff:70:1c:48:ed:ba:e0:be:f0:bc:9e:e0:dc:59:db: a5:74:25:58:b3:61:04:f6:33:28:6b:07:25:60:0f: 72:93:16:6c:9f:b0:ad:4a:18:f7:9e:29:1e:b7:61: 34:17 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: 1F:14:EF:2B:53:AB:28:4A:93:42:98:AE:85:06:0F:B4:7D:DC:36:AE X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io Netscape Cert Type: SSL Client X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 60:43:0b:c6:11:0b:96:ae:03:dc:77:26:9a:4a:bd:6a:d7:03: ec:43:16:2d:ba:8c:e5:50:fa:57:a9:1f:2f:a4:15:c3:a8:13: b9:d3:59:2a:97:7c:ae:ce:a9:f8:44:e4:97:ee:7d:09:dc:74: 38:80:94:cf:47:e0:84:52:2a:91:44:8a:85:55:da:42:6a:f1: 91:1a:6e:5a:63:e6:0b:61:3c:0d:b0:aa:17:b8:77:94:32:20: 4d:20:8f:84:56:64:ae:ef:d8:8d:42:b5:52:4d:b0:1c:46:97: bc:4c:77:8c:3f:a3:73:43:87:27:71:62:e7:fe:02:de:a1:27: 77:be:86:29:8f:62:a1:d9:e7:ea:61:33:73:f4:1f:0a:12:14: 68:eb:7d:8c:71:5b:42:e7:48:10:c9:df:30:3b:5b:eb:69:29: b6:95:bc:09:fc:01:b0:be:fc:9f:ee:c4:f3:df:a0:01:c5:68: 20:f5:2f:f8:e7:1c:a5:4c:a8:a8:a2:20:a1:d2:0f:f6:f6:c4: 0d:f5:26:fd:ea:8b:b5:06:a9:9e:17:35:47:f7:fd:6e:78:3d: 5f:7a:87:ed:21:b2:4e:e9:6a:d1:d9:ed:0e:cf:43:61:83:7c: fe:0d:b1:ad:ff:fa:2d:2b:36:9d:99:9c:20:48:21:0d:36:c8: dd:b6:0a:d8 -----BEGIN CERTIFICATE----- MIIE5zCCA8+gAwIBAgIUOEwWJJsEHLPb4Ew87bdAfWi1+h8wDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKycPp07ehJWhXjK35z8 DH5e8k8iM0aBOFPXpyWP1+4WE+JnSYj2lJnwqabb/noXyePfMXNxOHA6lh6Ze10H 42Pk6L+Z9z1cJ/W3Nyna7oKAANTI0xs2DYvTipuOEqFNDMUi+FY7ahr76T0IHhN/ VW4uZZOakFQDbQ3mRNb3wNfY4ccewpujbojxfFgIop8TzFu5ESwdI286rkeaD2rO 5YA0Cebj/XZKz1oYu5zFwXRJZ3cbuiiGMab8Eq9KhRtzW/TWQv8MHEnnMfJaKh7N h8si/3AcSO264L7wvJ7g3FnbpXQlWLNhBPYzKGsHJWAPcpMWbJ+wrUoY954pHrdh NBcCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU HxTvK1OrKEqTQpiuhQYPtH3cNq4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA AAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAGBDC8YR C5auA9x3JppKvWrXA+xDFi26jOVQ+lepHy+kFcOoE7nTWSqXfK7OqfhE5JfufQnc dDiAlM9H4IRSKpFEioVV2kJq8ZEablpj5gthPA2wqhe4d5QyIE0gj4RWZK7v2I1C tVJNsBxGl7xMd4w/o3NDhydxYuf+At6hJ3e+himPYqHZ5+phM3P0HwoSFGjrfYxx W0LnSBDJ3zA7W+tpKbaVvAn8AbC+/J/uxPPfoAHFaCD1L/jnHKVMqKiiIKHSD/b2 xA31Jv3qi7UGqZ4XNUf3/W54PV96h+0hsk7patHZ7Q7PQ2GDfP4Nsa3/+i0rNp2Z nCBIIQ02yN22Ctg= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/client-id-auth-cert.pem000066400000000000000000000126411477524627100241530ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 74:4b:fa:47:3b:79:81:c8:07:a4:c8:d9:70:df:d2:98:bc:a1:27:76 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:e4:2f:ec:ba:66:87:36:a8:09:ab:66:9b:cc:77: 24:50:e0:29:4d:c9:98:c8:9f:08:1c:a5:66:0a:b6: 83:38:8a:3a:f3:3c:16:64:fe:13:b6:02:bd:50:64: 7d:bb:19:a5:ec:e1:1c:2f:1c:30:be:cd:07:c4:33: 1f:ae:e2:7f:c5:11:f0:d3:1d:2a:4b:28:66:c1:a0: 09:52:77:64:75:b7:86:a9:63:cf:a4:29:c8:ad:87: 3d:79:71:6b:19:34:86:23:80:ed:ed:e6:eb:29:c4: 6f:2c:9a:c8:ef:da:6a:4e:e5:c9:ce:ef:48:87:26: 11:d3:f2:2c:4f:d2:a8:b6:26:b1:1d:48:49:9f:f5: 5a:d4:8a:30:a8:67:1c:8a:b9:3a:13:5d:04:82:e4: 89:aa:4b:ec:13:59:f7:92:32:33:8f:28:df:50:98: 65:92:0a:35:cc:c5:04:e3:86:ac:69:cc:6a:84:30: 91:af:af:44:4a:e8:9f:26:0a:f6:1b:78:f6:90:81: b1:eb:11:e6:85:1e:42:0e:09:15:e5:e3:36:86:b2: 46:38:b7:81:c2:c4:16:72:b9:a7:55:89:03:78:28: e0:2e:29:17:ce:58:2f:16:80:34:b2:5e:02:00:11: 01:2c:35:ac:bb:0e:2d:41:00:6d:0c:85:94:1a:f6: 7a:8b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: 86:48:10:75:95:44:01:FC:76:A0:96:E9:35:52:55:A6:38:8C:5F:37 X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io Netscape Cert Type: SSL Client X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 0e:37:24:36:db:25:73:c2:8d:71:e0:44:05:c1:2f:ac:e7:a4: 20:26:2a:15:69:20:2f:ea:4c:fb:0c:74:4b:29:e0:92:0f:12: 8a:a0:4a:73:78:ba:81:eb:4d:91:1c:3f:8e:58:cb:3f:30:77: 6e:95:5d:a6:a6:22:b3:f9:20:0a:75:93:bc:95:6c:b3:1c:c2: 6a:72:9a:a6:62:1f:ed:a1:df:9c:2f:d7:28:02:08:49:07:c5: 3b:5b:58:0f:f6:7b:b5:39:de:05:c7:be:c7:89:bf:62:77:56: eb:96:b9:74:d6:bf:ce:20:8d:ef:0c:08:bb:ec:0d:b2:55:6d: cb:94:83:88:89:a5:47:c4:95:91:72:10:c5:3c:6a:be:e5:86: bb:be:96:74:fd:67:d2:08:55:4e:df:e9:96:c4:62:58:93:6a: 0c:2c:68:a1:93:b9:61:38:98:eb:c4:4a:1c:3f:9f:9d:c2:3c: b0:eb:62:7f:a0:aa:e9:53:a3:f7:55:5d:43:1d:5a:63:8b:97: 32:4c:dc:b8:11:0b:65:35:de:2b:f9:af:8e:28:8c:6f:dd:4d: 13:3e:b9:f8:7b:7f:27:0d:43:a9:da:b6:59:10:d5:9c:a6:97: 79:97:19:94:f0:8d:0e:13:f7:c1:ee:50:f2:4c:81:6a:e6:0f: d7:42:c3:67 -----BEGIN CERTIFICATE----- MIIE5zCCA8+gAwIBAgIUdEv6Rzt5gcgHpMjZcN/SmLyhJ3YwDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOQv7LpmhzaoCatmm8x3 JFDgKU3JmMifCBylZgq2gziKOvM8FmT+E7YCvVBkfbsZpezhHC8cML7NB8QzH67i f8UR8NMdKksoZsGgCVJ3ZHW3hqljz6QpyK2HPXlxaxk0hiOA7e3m6ynEbyyayO/a ak7lyc7vSIcmEdPyLE/SqLYmsR1ISZ/1WtSKMKhnHIq5OhNdBILkiapL7BNZ95Iy M48o31CYZZIKNczFBOOGrGnMaoQwka+vREronyYK9ht49pCBsesR5oUeQg4JFeXj NoayRji3gcLEFnK5p1WJA3go4C4pF85YLxaANLJeAgARASw1rLsOLUEAbQyFlBr2 eosCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU hkgQdZVEAfx2oJbpNVJVpjiMXzcwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA AAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAA43JDbb JXPCjXHgRAXBL6znpCAmKhVpIC/qTPsMdEsp4JIPEoqgSnN4uoHrTZEcP45Yyz8w d26VXaamIrP5IAp1k7yVbLMcwmpymqZiH+2h35wv1ygCCEkHxTtbWA/2e7U53gXH vseJv2J3VuuWuXTWv84gje8MCLvsDbJVbcuUg4iJpUfElZFyEMU8ar7lhru+lnT9 Z9IIVU7f6ZbEYliTagwsaKGTuWE4mOvEShw/n53CPLDrYn+gqulTo/dVXUMdWmOL lzJM3LgRC2U13iv5r44ojG/dTRM+ufh7fycNQ6natlkQ1Zyml3mXGZTwjQ4T98Hu UPJMgWrmD9dCw2c= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/client-id-auth-key.pem000066400000000000000000000032541477524627100240060ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDkL+y6Zoc2qAmr ZpvMdyRQ4ClNyZjInwgcpWYKtoM4ijrzPBZk/hO2Ar1QZH27GaXs4RwvHDC+zQfE Mx+u4n/FEfDTHSpLKGbBoAlSd2R1t4apY8+kKcithz15cWsZNIYjgO3t5uspxG8s msjv2mpO5cnO70iHJhHT8ixP0qi2JrEdSEmf9VrUijCoZxyKuToTXQSC5ImqS+wT WfeSMjOPKN9QmGWSCjXMxQTjhqxpzGqEMJGvr0RK6J8mCvYbePaQgbHrEeaFHkIO CRXl4zaGskY4t4HCxBZyuadViQN4KOAuKRfOWC8WgDSyXgIAEQEsNay7Di1BAG0M hZQa9nqLAgMBAAECggEBAI3c3aflJhrszVYqLKIpUAKX2hXqR3oypLBqg84VOe9k wNGHgcS30TlO6rOYRjKT93wVV5hSRlvYzANGZWQsnJLAXKBjeW/QZlHqVOFYKLSm rKmSy/ybnY+EjMt7n8HDzcE03rcQ4RLOdO+eK14yw/TZF2X3jXe4S00hOjtGFG1o 5BvIaN1HMnrV4G5WsbD2n4gE17lLGLlmn1FhbZaI3/uJgSvM/RgRzzSOSZL+XOzo pyoMAm9DTI8U020qMuU23KEnjY2Wg68bMNMF38H7z73W+whRVQTyHq/mw0HhD4K7 P15eWaBUytmRsAkeoybO1hEfGQbUYVdPOtqrvxCCp8ECgYEA83Jcmeiv5DRaH8Sk /K+O0LM1CDfdWoM/w+Z2frooKqjHKsSa8B8fqI/2AIafgLorksVb2WA8nzzhBvT+ cIO9NiFbFkFVeLbrGc7supzPVjT1ZAFj7Qj9T0FmrMK937lnOmRNp6DU9t0hEJuF 4wUly0smzVZiBiFRfXsPzANX2T8CgYEA7/Qg401pp99rQfjgCC70n/jVgI1cOFKh pAdJHU+64GiozP4adMN/fd3fqqG3XnvFG8mRViwgL57iiIvxbbhx5pXUIZZO5LKC xwuMBzFhX8f4JAR7KANetOIsNF/ML7wTNvGhlUgc3CYXc/Fa1Wh78Xf4vhSd04KA OdkYuuVk37UCgYEAhCb4jbP6h27D3arpxSGn7TLa/vMUfiXxX26jtHdphn7IXzcK xH6guOgtKmvp+f8V2D66dW4AepCZtyUXWgypkdDZmWMt+rGRPhlN+J9XDf0BmKAI lovitjtSeUXdvKzwlIoOfYiZEslHQbSrIWmR8qGBFsZlv94mVm+PS7gk4BECgYEA ts6t054hlXSAGYXK4FEtq0Z/Ge4YSQyi+v1V9Y/NlrQFjg81Bqn+Ul0bzrpfogr3 cyEQqa76Ym1QtqivKWEw3XReZaxGtLNPMOeaKcy0G62UXZRQY36Vw4bgGJK3U9Kc bOqPqNSEsDARBBLnmdh9PMyi4+V/DCnLGMdNsO2c+VUCgYEAqz79qg3S+GON2woM Npe72bNBJC9UCciA4XpWkFB0f4AiPtFU0VW8fjZ4ZT6rd4sxvl727Y/FYgPcl3Dg r0zWWZAsaZ4LQRFk7EG9kMYe3Qsc/1211c5fg5dwmJwEybgW1gVvx/cb9qnbBifo hyWhzlSGwd/pB4skREo/fbNM3hA= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/client-key.pem000066400000000000000000000032541477524627100224550ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCsnD6dO3oSVoV4 yt+c/Ax+XvJPIjNGgThT16clj9fuFhPiZ0mI9pSZ8Kmm2/56F8nj3zFzcThwOpYe mXtdB+Nj5Oi/mfc9XCf1tzcp2u6CgADUyNMbNg2L04qbjhKhTQzFIvhWO2oa++k9 CB4Tf1VuLmWTmpBUA20N5kTW98DX2OHHHsKbo26I8XxYCKKfE8xbuREsHSNvOq5H mg9qzuWANAnm4/12Ss9aGLucxcF0SWd3G7oohjGm/BKvSoUbc1v01kL/DBxJ5zHy WioezYfLIv9wHEjtuuC+8Lye4NxZ26V0JVizYQT2MyhrByVgD3KTFmyfsK1KGPee KR63YTQXAgMBAAECggEBAKc6FHt2NPTxOAxn2C6aDmycBftesfiblnu8EWaVrmgu oYMV+CsmYZ+mhmZu+mNFCsam5JzoUvp/+BKbNeZSjx2nl0qRmvOqhdhLcbkuLybl ZmjAS64wNv2Bq+a6xRfaswWGtLuugkS0TCph4+mV0qmVb7mJ5ExQqWXu8kCl9QHn uKacp1wVFok9rmEI+byL1+Z01feKrkf/hcF6dk62U7zHNPajViJFTDww7hiHyfUH 6qsxIe1UWSNKtE61haEHkzqbDIDAy79jX4t3JobLToeVNCbJ7BSPf2IQSPJxELVL sidIJhndEjsbDR2CLpIF/EjsiSIaP7jh2zC9fxFpgSkCgYEA1qH0PH1JD5FqRV/p n9COYa6EifvSymGo4u/2FHgtX7wNSIQvqAVXenrQs41mz9E65womeqFXT/AZglaM 1PEjjwcFlDuLvUEYYJNgdXrIC515ZXS6TdvJ0JpQJLx28GzZ7h31tZXfwn68C3/i UGEHp+nN1BfBBQnsqvmGFFvHZFUCgYEAzeDlZHHijBlgHU+kGzKm7atJfAGsrv6/ tw7CIMEsL+z/y7pl3nwDLdZF+mLIvGuKlwIRajEzbYcEuVymCyG2/SmPMQEUf6j+ C1OmorX9CW8OwHmVCajkIgKn0ICFsF9iFv6aYZmm1kG48AIuYiQ7HOvY/MlilqFs 1p8sw6ZpQrsCgYEAj7Z9fQs+omfxymYAXnwc+hcKtAGkENL3bIzULryRVSrrkgTA jDaXbnFR0Qf7MWedkxnezfm+Js5TpkwhnGuiLaC8AZclaCFwGypTShZeYDifEmno XT2vkjfhNdfjo/Ser6vr3BxwaSDG9MQ6Wyu9HpeUtFD7c05D4++T8YnKpskCgYEA pCkcoIAStcWSFy0m3K0B3+dBvAiVyh/FfNDeyEFf24Mt4CPsEIBwBH+j4ugbyeoy YwC6JCPBLyeHA8q1d5DVmX4m+Fs1HioBD8UOzRUyA/CzIZSQ21f5OIlHiIDCmQUl cNJpBUQAfT2AmpgSphzfqcsBhWeLHjLvVx8rEYLC0fsCgYAiHdPZ3C0f7rWZP93N gY4DuldiO4d+KVsWAdBxeNgPznisUI7/ZZ/9NvCxGvA5NynyZr0qlpiKzVvtFJG8 1ZPUuFFRMAaWn9h5C+CwMPgk65tFC6lw/el0hpmcocSXVdiJEbkV0rnv9iGh0CYX HMACGrYlyZdDYM0CH/JAM+K/QQ== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/000077500000000000000000000000001477524627100206465ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp/ca-cert.pem000066400000000000000000000024621477524627100226730ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDqDCCApCgAwIBAgIBATANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMG0xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEVMBMGA1UEAwwMbG9jYWxob3N0IGNhMIIB IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwy+fupDc9MZldhetmGqPJtuM sp5VV6W9amlzkTck15B9Vc3laC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjM FpkjkLIVdFkbcWiErXYWDBHdA6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJ gljFl2ymHaQEhta/87tSvPULZ7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXp i8kHqPCkImlv/s7Jw/QRIknV/kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF 6O/4LvOC8UCbu50eW7OudppN8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQID AQABo1MwUTAdBgNVHQ4EFgQU7upCnRG44j5THcgKd28H4ESXBFkwHwYDVR0jBBgw FoAU7upCnRG44j5THcgKd28H4ESXBFkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG 9w0BAQsFAAOCAQEAfBHCa8sm0e767+oIZj3JIRi9MWN24hB9i4lVjDrwdOMaapMC YLLj5urqIgjOULjdsxBMzdNgNgH1vPenRYUUvIQcq7tk1q8DpfvmHEg2DHajpTAC DroutE5fYtlmFPSQ5UGG1if237osd6pDarVhGAdxex4YhwM+y+OXgpLqk6oC85oI fatf+hcovwFOlNeOTUqNZW6fEC+iFdH5g4+dtlx2LAJLpW57+5z25iTH7z16nUwB Vi76fezpaGA3xwkP/NMujgD4MbpVpF22a0YdK5fjUjXFwRI4Vu1zAjyJFhVuOWCS yT9yNzidtD5pho+Iv3JMzu54VWSq7nSUoPmKHQ== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/ca-key.pem000066400000000000000000000032131477524627100225210ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAwy+fupDc9MZldhetmGqPJtuMsp5VV6W9amlzkTck15B9Vc3l aC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjMFpkjkLIVdFkbcWiErXYWDBHd A6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJgljFl2ymHaQEhta/87tSvPUL Z7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXpi8kHqPCkImlv/s7Jw/QRIknV /kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF6O/4LvOC8UCbu50eW7OudppN 8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQIDAQABAoIBAC5RXtYnCkgLzIf5 lnhU0SOndfvtFtN1wT0/SO1s6JE++H8kHQxcBl7svShdMdnk4axnn9mHF//HnZu+ HlT9dbjE4al7LbVojS7O9nQzGUkQfKrgklILqoyR0AkZ05s3KQT2v/eCPFDaGK6w iuCiGZBkYy1LY5hLcCAYi/pze5mar1m6S2ZqvfpZWjlrlZOVi73cZOq91Es2g/iU TYaiR4HGHJ0McXFNjL6q7DxCBkRLWb6i+Xy+9+84XFZAVzRBAo1EaFJBPg0p88EC VKxQt4X0jgWYgLRABAqoQ+DkNEEoaCEQuUV67aIcIyV++ddn1FsegPtGYo38z7mr M+fzUAECgYEA/XL53y3eNnZ/U90gfr1HYIvyo6WShXEoIFw/s9QJnd5/ZkxcQFWr wUtMDNyjKFPnTTSLPr+vc3CGYqh8wFNxfids9KP95bMpVN8XGoTI99QqYK2tIgFl T8q46igOTcrg0f49ecqtQjL9F/dnZqzMlh5nJLMVGqRpNoZz9YpiS0kCgYEAxSaH uvaW7WxiBC+xf2vvgf585Wn9jh6QDX+MQjA7Ao7Tk4ZjDwAPMJkYd475BWp+DHSl b74x/nbRHwdLAfkxvYEsv4KrPR0yzdzGrXVATWcOP2bftEGPYVQnlgjjzMpLVXdt QErRS1vVnkavJLsxOHw++qiiyGENQk8LR2yOTnkCgYBBzy30dluBtskfBIbggdNb zVrmhSKDhbtOk8VyszcAB/r6nA9EITqkySFpIY039nlTwbX6SBmNlwU97tPduIz6 ndAbwc02bIvp3reICjyIpU2PpukSsFwXGONk4Zu9NVWlESfzTN4qF0VCiNoPfgTt Yd2UWO+86D3ti4HmmtUlCQKBgDRFYvdPKfUJJ3O0sXr3QylUMAkjcPadY9QwXR+v afXjqHUUzG7NtTlNXg9U+PFWqtTimHpoExlEp21yoZCEYYu9FAAyxPQPKckrIAId dE8RY9WrkORZ/Ynwpg5BjSRe/lpKr8y8CYHRd3Hfi9BRUVuIlaofzAkUsk9CZdsq DREBAoGBAKfvd2PpW9B8Wu248h/zxkK4M58XQUubkFTvUm6ErH0GWFBwfq8b8Q5Y 7/KnzZ3BefqQcNQKyoM06bTDT/YjlPLZdZtgN8UulGZQjBbLV9EIZJwRj+AUwMQV X+U8NAibD6yFqnOoJ0P5r8rlKPMg9+BkdZfnbhaCXb/KygYxqGcH -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/client-cert.pem000066400000000000000000000025531477524627100235670ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIID0jCCArqgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IGNsaWVu dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMn1VyxBY4AkODPmOxK5 VG3F2qQ+0jNFeikwcgJPHvFamqn3cA5AIJIUVmMtBiUfjnperHVKeuPfmW1bJw4E ne3V2eccDySoAR/BTX4kw0SPtIO3hnHyhOLX4bY4/Xw5OWgw2HMEwEwuoWxd+jpc GGzXY49J9gRKqxJFXR9tXD6T+1ABZPynqrTm3SYYCJoWq/C6feTSkf13HvnTnf8k fWcFum1Y5FegAObqbPqJwA0TGiuXSFkqw5oV0uAZzRQ7zqB6V8MB3W1U1pw86F1h 09EN78PrW1yXX1LZrLKwlqPTVh53Y1HuT+mwJkdQjFbOGXwh3x7rmp8+A3QLD4pR 5tsCAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI KwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA AAAAAAGCCWxvY2FsaG9zdIIQY2xpZW50LmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF AAOCAQEAWie6Pz2iJP6F9HfVH7anKVHeIXecwXJj4iLgEONaIcOyMcLPU4cthx1S OdvKAh+D9tT2PhVaIeDyYTUgFg/aaZUqI/W3odRH5HwQmE2YJDfXQusRtdFDTAUV XDqFkkNoJo4w3OQmlnQGm6QVReedyQ3jMTvqDRV+pa8gx6aH64jhP9fQRS4WkpYX d0HjWarV9/GzCP/+vGVZhwrhRG9p4F2ZCsflBzTx0YMGdo+vLDCSjwMbIT9t0T6/ mt07Q70QSk8M3QAClrqarvLk+5z5XSZjtM06s/Z6opyqK2X8KYcOYX4WQyNFbOpy 0YHy3iqmx/Ii0Zn5XZUXzAVGyJk5Yg== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/client-key.pem000066400000000000000000000032171477524627100234200ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAyfVXLEFjgCQ4M+Y7ErlUbcXapD7SM0V6KTByAk8e8Vqaqfdw DkAgkhRWYy0GJR+Oel6sdUp649+ZbVsnDgSd7dXZ5xwPJKgBH8FNfiTDRI+0g7eG cfKE4tfhtjj9fDk5aDDYcwTATC6hbF36OlwYbNdjj0n2BEqrEkVdH21cPpP7UAFk /KeqtObdJhgImhar8Lp95NKR/Xce+dOd/yR9ZwW6bVjkV6AA5ups+onADRMaK5dI WSrDmhXS4BnNFDvOoHpXwwHdbVTWnDzoXWHT0Q3vw+tbXJdfUtmssrCWo9NWHndj Ue5P6bAmR1CMVs4ZfCHfHuuanz4DdAsPilHm2wIDAQABAoIBAHu5DodxI5i8F6ZL 1EK7QOri+/gE+FcqqBUVtbKOcCFh5UBc8sv4Izv6s5WcXphdhbaXy0UrtK9nKyIg ZoOi9nFewlhgCzLkrZObo3K06N9Wvjq3MukZrqkdogw1S61PjUi0K9YCwh+prYCq 7gHUq636IecFY27ro3PVBKCdKZa3jwNIg4Oy5H0dsLgA1ma+tJaZMxDQZGyXvPqD wGXdkjdIefo7WQHi3yyujSs3Bu76ChmkGQfma48clWiu4HNlWI3Uc8aLBwDWVmjF S4T8s9HNw+nDWNHvnD45dkgKyTNAGA4rmD01OtuiC44GfZkRtupXcI0xA7d0lzd+ yvMBlfkCgYEA/9zpUgL2Uw24H28pDXiV6Lc0PmCmHVLPQxQpDzsudX/ukaUl0tMC tOL9575sluXOXDSkuoMX5nDX/8C6fPNSbi4AzrqDxQ7Dc05W41MvlJ9DdIzlozEm LFJ4RfXccxcVCXRrgNDNHpkMKQkk3Z+zmlZeNbbhWKs2CgZGrvyFnW8CgYEAyhEJ Z1aRC6XrP+GbMCiN7w2X6ZcDNm89o8jXR/94hTDGvyyQ4XIs0245XjHNYVP/ghzi iFpIRlph4QpKZebWiqgkwlsNsPKQmiBn5uqobo3VsUt+rFIhaX50GyM5cxSbSRkl /i9Z7ZZdj/dcsviURthKZiBn2uw+aROLCgKm71UCgYEA102h2K09cm4c/fagaQGL xCRGBid2IT7Jwfx5AKQgWCerLUv3JA0EPgq09gm7fs8qc1SpOXmO5w8V89TOGM74 ElcLvuocb/oYZjMJ0ojxhPLv5Geb5VM6eBl9tAFL3F0UCry4qdEKijDnlrBnIUd9 7uW2qSSXQ/Huq0jUufMszGkCgYEAnayLnPJkviUbG77svLh4gHgn+SNYY2qMO7il nE3R+oRkIZsh9nmEVvtkkobkDzVfZGUrs2BXk2ZFiDfic/+bm5i3Dl3EojW09j+h NAQZqCLPA8i4MLjpz4rYCLEEzDLhNToFdoH2dzllCsjnsdPcyCdQbr6Mq7y6un2A ejA1mP0CgYABUHeLemQTvsciXVAy7ZggDYmFIVPvxwYejAXWAeUdq2FEFWIqmywy gTISeMrWpaCpwRr97Rez5bLgYe1Crqujd03uzYzoMtuiE0027XE6CaqeDGEk7jWQ bnbanxoy7Ax6yjcEbqyaaG4ZbVXu5EbsCMPFE3mws3AqyF1/54YmbQ== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/000077500000000000000000000000001477524627100224515ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp/desgsign/ca-cert.pem000066400000000000000000000027251477524627100245000ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL BQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy YW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z dCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI hvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw MzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g RnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI b3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq hkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC AQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM W5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk cM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa DPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu NIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3 Nqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz mJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB u0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b XkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F gF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp pZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh rU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH tCztUKY= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/ca-chain-cert.pem000066400000000000000000000061141477524627100255540ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM b2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw EwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z dDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV UzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM CkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc MBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et aW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo jeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q ttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa AyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq NEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP 1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G A1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j BBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH AwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA AAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI hvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp JE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx W7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy /i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr y6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+ Tp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL BQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy YW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z dCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI hvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw MzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g RnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI b3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq hkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC AQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM W5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk cM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa DPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu NIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3 Nqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz mJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB u0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b XkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F gF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp pZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh rU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH tCztUKY= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/ca-interm-cert.pem000066400000000000000000000031671477524627100257750ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM b2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw EwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z dDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV UzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM CkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc MBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et aW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo jeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q ttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa AyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq NEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP 1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G A1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j BBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH AwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA AAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI hvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp JE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx W7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy /i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr y6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+ Tp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/ca-interm-key.pem000066400000000000000000000032171477524627100256240ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAuWOx+D080xh+YKANDjP7tWx+pQQcH63/o2QAKQFCLo2M0qFf EPUA98jorll4IyiN5JOaOYgMHtKG72hTwNvksUHF7s8T51selfT9IDMGRblTIKhf r3UweFvZVutL3iq20G2XRT8OC0LB9XEdZ/UYGGwmqw9ScEtEifEH8ExeXOHDwVJL oHk+QeYTImCJShoDI3rSwQ5SzBrwgPMeUf+YO5gNQkoIciB6dUfs3ypll+Qrrxsl 9KM0eg9ZPexY9io0TI+C8uqxvV7BNICwpZMAR5+GAT8PNWmzETDrXdW9b+f9OX2N rLgilDfxy4vweE/V3ukXmSmD60IMiMrrOMIKbwIDAQABAoIBAHrcuHCk3O+llvOg fqaACvvkaFYiUCUqonX9ayHBxMOnacGZ0rAhPz/39US+5KCgWtE2hQpFwIqYyXZW dNMmp/xVc8DdmfpE6BNHpo21Yx3IQXAuRiO5DaNmc7ZoAAK/2iJtzsfEjyElX/ey vGNOCmb9He9WGzNynnywsasuYYc5sr6+2b5AN0yMxCggJY9tdyxAK5Tgh0ahMumf gmNNziSqwRhJ8V9natT7TzFqheQe8u6NuC1uSZ7ARqHsf9v9nfjq3jscBjkcQexx kuBSXS43DoksMangyDRK7epbQfQf0KmmrScdlCy3vJquKWG4wsSaob+Wsyg/XvOa k/bCsuECgYEA55wfpvuzB4J5S9nes7D9Fd3mGi/HBjEflWFXIv0uns4zljDTEd1k /Ye13UfvIyVcKMiLzmVsGpf6sESdDo6Fgd/g6gjOfeGerc3t/MZLYsXVukheABhF YJX+4xvwVp7QxEXcftSbZjcoTL4HdTYDeKyfn0KftTm3C26MHJhron8CgYEAzOmI q8rqW/6CbGb12zxDNKnKgANS0MNFeLUKdJqwrX8LxYTXmXypdlwOJ66D1pAvhblR j7A7QBItbkvr0Qwv56a9fa2F2Nuot5VmJyHz6mL9cRGrhIZL6aFIVZ3C1iqhQKoS /6mAxPSWjD3u7eCFBwKeZwmWdSx3zM9VaefDwBECgYEAwWDycJqRJUEEA5faQNAS z/IhEFY552qWg0Pt3DHmfgOOwOTtJmpiyuhHqYVJHmAwLYEccezusNmaHxh6xc+r mv+RK/bEagg6U8Wv4jCyerrRs6J+kbeyHW2/jmIibkBV8Lqf2mmrglGlXUYAthWu GlCPSgr3i/mvYmUfqTR+EgMCgYEAvpX7GyWpIpUug1qkExwSufmuMbBlp2vnwqRI LDnwV/4RWc37pXNwPnjSZZAIaVlEChFaTdWw1h/SB4MvuwilycSo/CqXkiKD4vRe xcjrj7YwWakAqUsrcgojOBZ6sC2IO5e0AfyKmyWOnLPB9Zfcwq9p2xXszeDlMCYr IEyDIbECgYBT8aN5dimyIduEk4XTQ377l5i/Ng9EVOaxsuX4MhHUUScaWogo18Fq eggKOwFXD08J0xVk0WORdkyt01/EEwhUmUUPolya8HZK2rpikpp0eIX68qSCMB6q IEuHf42ip+4d8IbQp7i8SFh8LNaxkUPdJi92fFuwbQ4WeNRiL4CoVg== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/ca-key.pem000066400000000000000000000032131477524627100243240ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAwr1jzxA05UDI3jcRJNBUIUmQGemAC6n/WgUAC/vuku5XRaXV RU7yAoxbmTWpNqFVJleRf6AvS6hF7uftdlvjSIhzNrV6YR0wpwg7FJD+POPzGKTO IF1mu2Rwz8ySmQMKD/jnC49S6ri6aNefuwndfCkqZHzs9M1xJVEbtgKPxWrN78oG EIOZM9oM+zQYZYGqUYePHwuYb1Zg8jFtX3CLlJhRDgjDDb3EShkFxM93ViW5oDMX 06xiwW40i5piOH9elkVvEYD30F+hi0grX9f4I7U4SQJNx9LYNHA/MZX26Pf3IYSZ 39w6OLc2qPnkOCWmGNt0ZHzsJMqLCKcpovBfMwIDAQABAoIBAANogyWtl+9nOdzB w/gL+Vz3x6DceFQ77m8p5T1QdHV33GkopNnors2inTvKN6LwIehg4fgE2q8NS+QJ hRtsMkcjtDiB5plqhF54A2ixvPFd3/RPdhgU6mZfNKY+Y1ZwOi3bYlfOc1oOT5zk ITSJ0KmKouZNVWxXaCKKD90YoGGJHCxZGPkAQlX/GT3h7g8h6QUYovgKwn788EVi n78ldLq8AeqSiLZNK76DpnniKtaIwshslUkqPDW22EWk8YM6uvTNQQMuZzV1oBlQ Ta5rJizOuWcIi+Mu2/QAhSVjhMSjXMyLoIrO5WkHpj9LzRwys9SQM7A1njZ9Xatl iWWPmKECgYEA6pm1qgMG84ZhsFTxojYkrR/bs7LuLCSAIUfEoyypXJTrWNIJv5eb nZ/HL5o90KQHYN5T/LpZJfe/Etz+4npS6ahTXM71o51EpIN9DhNzJNduPSDaBqHP LiomPg+ipgPJ3ziFu4m2dK2y2TW/u9IO8+6Ah/2ozkXX4/kf+DFq+GMCgYEA1IDf KybhAgWImgf6m3GoZMDQOQLUUcn1cAagUUe2eNUa24lXx67ZmpJZpoHqixYylhfx Bp8fmMD3q/OGe+j427OlVcomzlDTqnahJ9m2KFfn2tkovfJKYqZqx8r7F7kuqlgM r9DOAkR5s50tHz+EuQZqC1VZ796YcqnuIVoWbvECgYAhzJdxsRH1T+0PHI3bkvVh w+9BSowp6/BR2ycnYy3bWtE1cL1azxrqcLSf1RcG0jsF58It7SMe5zyuGQzX0EvV whyQiHi3Y9cZ5J/FwWObcTY+tFb1EabpvcTYuCP0yyLweBI8XLDeyo+z35yKEM96 sWfvL8p8PW/HNoM2nNgOhQKBgQCkxHPForSue5nqTKt84Yi+7l3FBrOX4y8iOJEP 1LngQOQ9OuXMF3/0AOvwViWEyKZaiJ/DEZhPObgQJJeu9foXZ9iXh1HFgRhNwQO9 wWojJ93Ha7/SX85bZUvANFuyjkxnmjPkEtPZIDz5DrLQ2tBPInEQ7pH4kjDEH4xb Yd2pEQKBgATSX2zyg3POs3958Ls/4oegfnVclFeUf8zQpwGtsDxP6NtpVti9YGgM rO63dfeCuAfapxyX+I6iNw5pbDWK3TC8STjWnhhzAUZnZSoIGRB8q0yKCK8TZSMG 3rMw+2LuO+br5tu/Hq+d6YlHbhdYW8JJFEF6QfNT31ltGE5+Nr6D -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/server-01-cert.pem000066400000000000000000000031431477524627100256340ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEiTCCA3GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM b2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa BgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu dGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB rDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp c2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy dXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMS5sb2NhbGhvc3QxIjAgBgkq hkiG9w0BCQEWE3NlcnZlci0wMUBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDF2MDDwVEZqC/dje8+soPRapPa9C4V8hmHuCQS9sg27HZv bJlywJMJE0PKm+612JoVB9p5tsXlV29gKcIK7zLEhesGlhiW9d2/7MOrPBSwqJfo IeabwlUfUTZ2M3XWGvU5Z65cK+cK7f0FABUzwTqb4/M3UR41cRvNtSK95nwl48Kb x4Dew4MIqvIN/H1qmjkNM8lsDHrCV2IyyYc1W1zDqZVU1JdpjO7tBBL6pkwlM31k 8xjMr+hiVA64N+fESCauf4xArnqcHdhAkfnX+8DWCfSLFjP1lBZZTw1ByLEBBeNr kHt22RhO1mvBUHnML78k7TEFIWeKuNmOv1RGksfzAgMBAAGjgbMwgbAwDAYDVR0T AQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB hhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG AQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl ci0wMS5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAngLW TZx4PVsbAQ+saPRJErZU4JfA8Csx1MpdOkmvqbTwpIvnkgE6mER6DA31z+Fml2J1 q+TpGh83mZTaH2yivF1i7/EU4h/lDhCXOns+yPto1t8GbIGNRTvG9VwJjhjgbILm 5l5cZQY7NeIryU4LmbELJCwPoHt52HnK0pfbhjCbD3Q6ugEx7xzESeqIwVS4gP5M so/S+lvlQi1YWn1JJrj+SM8E5k/7x1kjTQZ/sHXAsUTwdtE6Qk/xbT5dbZYuhhPU TF6sSfQpIO3Ju5Z+MTbcaCbJmqSe/y6J21e0Di1ve1BORKAy833F4leyPX8Ziv1x iB8gZs4dMh88tJHqgw== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/server-01-key.pem000066400000000000000000000032171477524627100254710ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAxdjAw8FRGagv3Y3vPrKD0WqT2vQuFfIZh7gkEvbINux2b2yZ csCTCRNDypvutdiaFQfaebbF5VdvYCnCCu8yxIXrBpYYlvXdv+zDqzwUsKiX6CHm m8JVH1E2djN11hr1OWeuXCvnCu39BQAVM8E6m+PzN1EeNXEbzbUiveZ8JePCm8eA 3sODCKryDfx9apo5DTPJbAx6wldiMsmHNVtcw6mVVNSXaYzu7QQS+qZMJTN9ZPMY zK/oYlQOuDfnxEgmrn+MQK56nB3YQJH51/vA1gn0ixYz9ZQWWU8NQcixAQXja5B7 dtkYTtZrwVB5zC+/JO0xBSFnirjZjr9URpLH8wIDAQABAoIBAA/+b8bjb9z1Hbl5 aefVopZhWUaQCtG3Wp0AI9psnM7j2sczLkx6iOho6EgOxwNoWTxuABlqzyC1KsnJ Vmhv4djFQrSrmZ1Kjvye9Up4duu5FV6srunUkfEQLajsjBAc6coUOaI14l2d06B9 2zjt3AESMm08X4bOeALvK7nSUiL13CAzKfv/ufM+bqi3nXxzNCBhtd51sHDaYqr7 nbhFMrX1frhRb0a5k0Q0UPf3V8pUf04difa0MZJ8tQTW8HYLrEVjOE8NPp5yGQj0 UX0d4XqS5LVDQl6Y1DAOQ7IsDq91gGKa8sk48HVvnwRjez0igAtbTq5I2DTydFDf D19fpCECgYEA8eyHbdojrawzQM9P5QADRTLN+r2rk8Tv0cW2CdJTmxZiU6HhVqxU NrP5G9FpCkn0jOikzhGdJalTgd3TKWMbZS9gPdezpj1viIT7v7NsK7Gnc8EcF+v8 4H+Rj7jsxiu6ryNoNPmXZSsIyfGIvbOE9SwUki1Cc2NVRYf0g6aLfXsCgYEA0Vuw 2DRg7FCP9Ck08eburI163OEKE2sNIc2z4O6I0gy/u/U86Y/bLBrpRypiSEAf1r/p yEEzzFqSNf7UNCLumdKk4LXUk3FMPygMZY/D7s1j/MVW/6HWeMeMy7IKKqOclTT1 R1nNjbjRkorBRyHDkYqDrAGLuQUleWLrqSzIyekCgYBjxm5wcvgmB5A32YiU5LV0 k6h6EkGyNxXFiWozkMgkfU3eOjRqf7ZXvVAvFeXhdXDjsItP5dnPD4++TtNpDVPe HnTt7IlONaZLQrVlccVaG/H4/prsjsqDeHl7MgSNErnyw4KV3p5+/gmo5/HCc0iR qTVuuDXgywX/IDxLE6QSAwKBgQChDDWXu4MrhjWWjvRJeWn0lskCjKJhmaH6dPCA gT4CxwffIKGA0ca5wOHeer4r8hgL7Il8IJwmAS4kFylKCe0dqypmKbmiyi7rDnWq 0tLYKmtWEMAB2Y7QTkECmKy1bDKRnLFp96zl7lxYrCBOBa0ZkkID2RSQeWMAY5YB B5BVsQKBgQDRIIKN0UhGqHZRcHb26c5lE5Rs/8nX2nyFgxKlwno6Dk5I0Cdo9CDh z8f5klwm6Rgmsvr41crTg4IOimvCYGu94vyJTOKo+ln8Ky+02mdReomCVKoFdIB/ WR+1aFvk1+r9aRr7MyPcHUGuIdUDJBnWv0pfNH1TUyvjpE3rMoaIqg== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/server-02-cert.pem000066400000000000000000000031431477524627100256350ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEiTCCA3GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM b2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa BgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu dGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB rDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp c2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy dXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMi5sb2NhbGhvc3QxIjAgBgkq hkiG9w0BCQEWE3NlcnZlci0wMkBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDWzAyE9A/EzJv5TcQlM9o4C/UOIQD4Rzs8S80x4ckGyF00 8xF5FMmGzGMka4a9FNPDBzbBM41hlhJXcDhxRfvQodzvpOUXqi8Siha2SsyFzhD5 o3exxV8ETkHMtST1nW/bwuYqQbDf3PLpVV5qZgv4jIByyGE4nCPDqHMKX0h1xXIQ E3fv2qTcs5j72c3jcTlINFJ9aqTaFgSvvGwg+0KLrOjs1Og3408zTIHYSjTScFHd heDLHjfmIx8oqXj4+g3F+5BgLNvO2E6if9YYGOFjxzRCPOrRw4JB8nveI1gZhTD9 +OmUkIMoNLX/+4Kbq+a47/kqrXv5GxLxwola8do3AgMBAAGjgbMwgbAwDAYDVR0T AQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB hhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG AQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl ci0wMi5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAH9jr P5EepmTiJzxYsrGE9q5BqhspBYqomgCbLWH3vDX50MwmxZ4ETe1icpHbXiddDA1k dfBxn/1HXEWXxwHCTgoRsSzufgacHdk/LgXCxkFB0JBZ9c+ewOFdUQihxFUx1rMt Sz0QnO40aopTk7PN8YlinihAvil/JC8fN54T0myCSjtE54dtZq2wwFiiorxR6hyi hLL4zYKi/kCFa6DG0qmR6Yl2VsKiq3yp+6QxZ8w5srXTeeOoTOEyUkYKCTND7U1G 5vyCdMFNIDWHSB35I+7e0D1QFaH5XQlFztUfmLoOMSMLbOuzG2cFO6J+Y0AgTP5h pZ5mw1ixmVdNmkSf3g== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/desgsign/server-02-key.pem000066400000000000000000000032171477524627100254720ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpgIBAAKCAQEA1swMhPQPxMyb+U3EJTPaOAv1DiEA+Ec7PEvNMeHJBshdNPMR eRTJhsxjJGuGvRTTwwc2wTONYZYSV3A4cUX70KHc76TlF6ovEooWtkrMhc4Q+aN3 scVfBE5BzLUk9Z1v28LmKkGw39zy6VVeamYL+IyAcshhOJwjw6hzCl9IdcVyEBN3 79qk3LOY+9nN43E5SDRSfWqk2hYEr7xsIPtCi6zo7NToN+NPM0yB2Eo00nBR3YXg yx435iMfKKl4+PoNxfuQYCzbzthOon/WGBjhY8c0Qjzq0cOCQfJ73iNYGYUw/fjp lJCDKDS1//uCm6vmuO/5Kq17+RsS8cKJWvHaNwIDAQABAoIBAQChTG2CUbydroDa P9mxjjSqreACEiqmRudmqg2XDdRl8YR8xKM0Z7XGMimpKc3uo3s6E2q8vrfGtmlj m/PmrAUjcMl2dD9M/BGJPIU9swHO4SXCMU0rA+oMU6/5x3XPs8BSKROqW9Y05bjM G71g7OzEYs36ZBsN5cK9pPtuqIrDVHfPQI+Y3eP7wBnLscK/AqUoapbc6oQqarl4 9rdHY4IHsab3xTxt/riSooia17R6xrrSpDcg+tdx8CphjD53zLN2Y8jHpl6LVJEN rRFud68sDEtdU5EOj0WiQIke0Z8yBOgAt2YKqrU+O+Wuh2ad9pBhhp7oLvaRi0C8 s+URKM2BAoGBAO4LtDgpWyMC3FfMDuSGT7hwp5+TtKM9Js2mZsQYZbJRX3BVK/G1 +/LhZzpqcKqGAarZ/yz1cLUEprbEXX7+TFbPKKpBsEwTOQXxlvhmjSYxyHVfBCbB +o43EolRey0Zdnnawk2OpqOVWSQafrp5JyJumyvfwES3Oc2iT8288bXhAoGBAOb/ cwLtk5I9Bx0BFUDFNgn8g8QXfWJ0axyCUQE2UGn/tiXodpkrvFF2QrpEQ+OG+iKB 0d6I9lLNThw/7uKvecYN1HliWnJ9dzUe59nguwG+PNZJgSmBhnPs4Wg/gsG6h7K1 JZ3VPeyNaSRK+pCn8sWkcQjumImZR/PHKzsDdOMXAoGBANp18UoNYi9qY69LfWtq e+Unth30HzYkW+UlznAud75DgZQFBlRIkFWhWOw6XPWSEBus/stS4MGv6BQZeDig xoxwh8BgkpvulEmJIuUKsIUZ8P8OWS/8m6ZCkodlOOb11E4WXnVw8it0V5+TlTQr Fag627tTGA+4G5tFV3nX6ffhAoGBANloGOXrlhVvzK8WotsYATk66QT4mrC8I0ds uzKp3Ns2qUdaV6znhdEhvcGzmDWfhvJNPqn0O+lIgziBT6MYRkMKJyyrTbctsLFV Sh88rKUCWB3Shnb7CgE3NBq6k6Ujmq2uYh3/Yc2udgOLcfINr6cmkqA2d2gh1J9y l5RuN2e3AoGBAN9FEe+OLRbq+qnMHkEoozoshxgEqv/M/BgXvoX8kelcp1zyBaWX t5yelDn7UjANhJNBsripzvaLilMxracm1Oy0fCtRJ3dvwm6djUelhSPBV9HJSEN8 t2Sv1z45SNq3l2cz8pVvoQDNmUbh0QCQnScrLN8L3vBJSibAJgSdml51 -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/gen.sh000077500000000000000000000207231477524627100217620ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail # gen.sh generates certificates used in OCSP tests. It generates a CA, client # certs, and a few different types of server certs with different OCSP # settings. This requires OpenSSL, not LibreSSL. # # usage: ./gen.sh ################################################################################ # Setup CA ################################################################################ mkdir -p ./demoCA/newcerts rm -f demoCA/index.txt touch demoCA/index.txt echo "01" > demoCA/serial prefix="ca" openssl genrsa -out ${prefix}-key.pem openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ -config <(echo " [ req ] prompt = no distinguished_name = req_distinguished_name string_mask = utf8only utf8 = yes x509_extensions = v3_ca [ req_distinguished_name ] C = US ST = CA L = San Francisco O = Synadia OU = nats.io CN = localhost ca [ v3_ca ] subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer basicConstraints = critical,CA:true ") openssl ca -batch -keyfile ${prefix}-key.pem -selfsign -notext \ -config <(echo " [ ca ] default_ca = ca_default [ ca_default ] dir = ./demoCA database = ./demoCA/index.txt new_certs_dir = ./demoCA/newcerts serial = ./demoCA/serial default_md = default policy = policy_anything x509_extensions = v3_ca default_md = sha256 default_enddate = 20291014135726Z copy_extensions = copy [ policy_anything ] countryName = optional stateOrProvinceName = optional localityName = optional organizationName = optional organizationalUnitName = optional commonName = supplied emailAddress = optional [ v3_ca ] subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer basicConstraints = critical,CA:true ") \ -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem ################################################################################ # Client cert ################################################################################ prefix="client" openssl genrsa -out ${prefix}-key.pem openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ -config <(echo " [ req ] prompt = no distinguished_name = req_distinguished_name req_extensions = v3_req string_mask = utf8only utf8 = yes [ req_distinguished_name ] C = US ST = CA L = San Francisco O = Synadia OU = nats.io CN = localhost client [ v3_req ] subjectAltName = @alt_names [ alt_names ] IP.1 = 127.0.0.1 IP.2 = 0:0:0:0:0:0:0:1 DNS.1 = localhost DNS.2 = client.localhost ") openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ -config <(echo " [ ca ] default_ca = ca_default [ ca_default ] dir = ./demoCA database = ./demoCA/index.txt new_certs_dir = ./demoCA/newcerts serial = ./demoCA/serial default_md = default policy = policy_anything x509_extensions = ext_ca default_md = sha256 default_enddate = 20291014135726Z copy_extensions = copy [ policy_anything ] countryName = optional stateOrProvinceName = optional localityName = optional organizationName = optional organizationalUnitName = optional commonName = supplied emailAddress = optional [ ext_ca ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = serverAuth, clientAuth ") \ -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem ################################################################################ # Server cert ################################################################################ prefix="server" openssl genrsa -out ${prefix}-key.pem openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ -config <(echo " [ req ] prompt = no distinguished_name = req_distinguished_name req_extensions = v3_req string_mask = utf8only utf8 = yes [ req_distinguished_name ] C = US ST = CA L = San Francisco O = Synadia OU = nats.io CN = localhost server [ v3_req ] subjectAltName = @alt_names [ alt_names ] IP.1 = 127.0.0.1 IP.2 = 0:0:0:0:0:0:0:1 DNS.1 = localhost DNS.2 = server.localhost ") openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ -config <(echo " [ ca ] default_ca = ca_default [ ca_default ] dir = ./demoCA database = ./demoCA/index.txt new_certs_dir = ./demoCA/newcerts serial = ./demoCA/serial default_md = default policy = policy_anything x509_extensions = ext_ca default_md = sha256 default_enddate = 20291014135726Z copy_extensions = copy [ policy_anything ] countryName = optional stateOrProvinceName = optional localityName = optional organizationName = optional organizationalUnitName = optional commonName = supplied emailAddress = optional [ ext_ca ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = serverAuth, clientAuth ") \ -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem ################################################################################ # Server cert (tlsfeature) ################################################################################ prefix="server-status-request" openssl genrsa -out ${prefix}-key.pem openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ -config <(echo " [ req ] prompt = no distinguished_name = req_distinguished_name req_extensions = v3_req string_mask = utf8only utf8 = yes [ req_distinguished_name ] C = US ST = CA L = San Francisco O = Synadia OU = nats.io CN = localhost server status request [ v3_req ] subjectAltName = @alt_names [ alt_names ] IP.1 = 127.0.0.1 IP.2 = 0:0:0:0:0:0:0:1 DNS.1 = localhost DNS.2 = server-status-request.localhost ") openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ -config <(echo " [ ca ] default_ca = ca_default [ ca_default ] dir = ./demoCA database = ./demoCA/index.txt new_certs_dir = ./demoCA/newcerts serial = ./demoCA/serial default_md = default policy = policy_anything x509_extensions = ext_ca default_md = sha256 default_enddate = 20291014135726Z copy_extensions = copy [ policy_anything ] countryName = optional stateOrProvinceName = optional localityName = optional organizationName = optional organizationalUnitName = optional commonName = supplied emailAddress = optional [ ext_ca ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment tlsfeature = status_request extendedKeyUsage = serverAuth, clientAuth ") \ -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem ################################################################################ # Server cert (authorityInfoAccess and tlsfeature) ################################################################################ for n in {01..08}; do prefix="server-status-request-url-${n}" openssl genrsa -out ${prefix}-key.pem openssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \ -config <(echo " [ req ] prompt = no distinguished_name = req_distinguished_name req_extensions = v3_req string_mask = utf8only utf8 = yes [ req_distinguished_name ] C = US ST = CA L = San Francisco O = Synadia OU = nats.io CN = localhost ${prefix} [ v3_req ] subjectAltName = @alt_names [ alt_names ] IP.1 = 127.0.0.1 IP.2 = 0:0:0:0:0:0:0:1 DNS.1 = localhost DNS.2 = ${prefix}.localhost ") openssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \ -config <(echo " [ ca ] default_ca = ca_default [ ca_default ] dir = ./demoCA database = ./demoCA/index.txt new_certs_dir = ./demoCA/newcerts serial = ./demoCA/serial default_md = default policy = policy_anything x509_extensions = ext_ca default_md = sha256 default_enddate = 20291014135726Z copy_extensions = copy [ policy_anything ] countryName = optional stateOrProvinceName = optional localityName = optional organizationName = optional organizationalUnitName = optional commonName = supplied emailAddress = optional [ ext_ca ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment authorityInfoAccess = OCSP;URI:http://127.0.0.1:8888 tlsfeature = status_request extendedKeyUsage = serverAuth, clientAuth ") \ -out ${prefix}-cert.pem -infiles ${prefix}-csr.pem done ################################################################################ # Clean up ################################################################################ rm -f *-csr.pem rm -rf ./demoCA nats-server-2.10.27/test/configs/certs/ocsp/server-cert.pem000066400000000000000000000025531477524627100236170ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIID0jCCArqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu YWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IHNlcnZl cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN/v4lwsEQqEr8gqG6Xu UjNfvUBN/enc/26FqtsAF6ms0r4oHcyR3RZQGQj+Z0RF3Wu0Kq9692gk7FD/QulE hYJTjq6lEwvETuUHbkNmIAppNJW1JvgLsTOfm38VorBVU5PUMbrcfsVsFijXVACj 9VMZ23So4dxtlvnqrd5/fVx0Pql5EjY87bJEKH5Zngy1v+AR5kybZaorOX9T4/Nl e0P184GwGs15hKAokoQMPm9uIhG527JMyhQh5J/2wooY2DBZ9jDt5FVXNpb0C+nr M+AULk5QHQsobTtmC3RSNHiNw5B5w+gmauhGziurq8gcx0DctqAslKFBkCLkL9fc F30CAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI KwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA AAAAAAGCCWxvY2FsaG9zdIIQc2VydmVyLmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF AAOCAQEATM/K671w3aHt665HBMawzMIZZPq/ZoBfEUkSUW9KdnQHgTxatHcZonsL aFn4XZBYQ0Pqkz7H1w39mHdvpURQ5ZMnsmn4jH3LECsOtQ4ztrLk2fhLSoMQBVdb UjdYhrM8AuILKRCzOBNsDm/ZB/vPSlmYhnaEBUjO0t+I/A0X1z5eDcYPLl578kfJ WjlvRluWr7Uku1DaZUy7TByYvUuOjP4c33DAnbZ5Sldx18repZ20REASxsCpa/CW tptxVfUvLcGRHIY0FxOn+5Pfm1QDo2uh6yVYHgsOCh1qW8FHfJvgnrMlvvXniKXu 5H6A5GeyCkIVvAENDfl1cN9LaV5eQg== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-key.pem000066400000000000000000000032171477524627100234500ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA3+/iXCwRCoSvyCobpe5SM1+9QE396dz/boWq2wAXqazSvigd zJHdFlAZCP5nREXda7Qqr3r3aCTsUP9C6USFglOOrqUTC8RO5QduQ2YgCmk0lbUm +AuxM5+bfxWisFVTk9Qxutx+xWwWKNdUAKP1UxnbdKjh3G2W+eqt3n99XHQ+qXkS NjztskQoflmeDLW/4BHmTJtlqis5f1Pj82V7Q/XzgbAazXmEoCiShAw+b24iEbnb skzKFCHkn/bCihjYMFn2MO3kVVc2lvQL6esz4BQuTlAdCyhtO2YLdFI0eI3DkHnD 6CZq6EbOK6uryBzHQNy2oCyUoUGQIuQv19wXfQIDAQABAoIBAC5rWl/C3rFwecOj PuHxeeaeVOuMfzLIFcbCPH1zEnSgl3rFdA/guJSUe+wKWDulw9U8npCLi9dxt+6+ Sw9xnb87NNts6nrI8ZW2KZwdAk1GK5lQ2MgYHF5YGnKIeJXWyiFdngVfCYRA+IL0 x7vuQL8+H+iZzV/U87PQesQhZ0oPh03n/4OtSo1bnPOoiPLAC+GVVpSSKoh13beb GpIRXhGNXonGPWnj1t/oBHQwLDfDndnljkRaWTzxeCZGcQ1wYucf6jne5dI1E21q vrdyK8GL4SqzHC2SHnJO24aKam/Xr7YqOM2T+XsVywKxE4I1icT2YhQAgRuN8Nlg 7CELdAECgYEA+CEhY7JYfmRb/+PIvogRr67uaYtxYygvDcYfse5O/V/HOzotIXIT t99deD8XOFpWhJHdDzDD/TStb/rmwu5kavMQzmQafyesu7okk1NtB+fZtMLQfF9/ 0+bW9uJQFyc1EVHKgyya3UidXTcmKcExKQZT+jdgFz614zndcOKl0H0CgYEA5wpO WIjjxVqke5ucrVSRsm37l0BZYeea2vxLAzUV4txVw3hcOgJo+FhEGFy6EjNSV2OR y589gZDEPZ/LNeaj7nbNUqf2xZV7MAXBgUCtj7fJqMbVhbXZB0w0hIknDL+td2WZ bocuVExTwRwXyuSCgUFo0Bz++L0cR3JK6kNBEwECgYEAyOfKaTbWgEAyXZbJy7vQ 1jcFw1+sh2TZ9IUe1KroOi962XHZaOM9I/wvalVrL621r9GK8+nARxyH8cttXRg5 Jn94dCSJb7toGPg29TLvbR9FHx8+P/XzQlf+ZhgIUTbluQhIuL09Bz7sa7VjqRtL +rOs+0QrAac9DqajretV5uECgYEAgaXe4P+wEQcUVei0uu9B8waUsAOEJNR6qXf6 AArCBVPvLIlV95dyoCmnzKP8Jkp2YmOVZNYvBY3fEVWiCtUqGJ7CCSgH6kg/oGsa cxWAT62qk/M/zpCFAPtaXSU5rIXDKcTxnHxvGw7Z0PuavlgMg8vYrTAYRCyaud0A /QRQeAECgYBuRDM+mk5EDbsi6MOD24x6krRDHr5/Ch5xCQXK2FhU8zcQ8P24UixU Re71LBsYLBHkhB/slofGdBvgeiVHwJyVWA9c3+kb+IwSilNRLV7IwxrfLg3xSVBu 0KEwWLSXlJmPnGWObpBmz62HrfquyMME4srQrNfW1q+Qm8OlZIeInw== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-cert.pem000066400000000000000000000026601477524627100266250ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEBjCCAu6gAwIBAgIBBDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGAMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKDAmBgNVBAMMH2xvY2FsaG9zdCBzZXJ2 ZXIgc3RhdHVzIHJlcXVlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB AQDViockMMFbHDOsiD+JLalZc1g3rd10xbXwHFHsgFQUdr+g6guGnnQhKMHEsQWf bGmn5G+K0QG2jxpz2IpfXA2UpMIrYLhIcf5faEbUgmwd8zaNXNs0I6sZZk8GrKYw PLHhXam94FUfVwsecJ/V+sK7limOB/T1AuPamBhgqIgubz40N3Qmkh9J/rkhq2b1 ffgr2v6qKjQ8bIyPPDh9OB10KffjqoaN8ogXuE3hZdQTniRWW1nT38NqQdJg6T+N cEheH0H4pYUSx/TdF6AHXMxWFD6lX9nLU9UGpXLAHY63rVJCv2KIMMsCGBUPKkKK TujdFu2KHh52CstdasQ3gpnRAgMBAAGjgZwwgZkwCQYDVR0TBAIwADALBgNVHQ8E BAMCBeAwEQYIKwYBBQUHARgEBTADAgEFMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr BgEFBQcDAjBNBgNVHREERjBEhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2Nh bGhvc3SCH3NlcnZlci1zdGF0dXMtcmVxdWVzdC5sb2NhbGhvc3QwDQYJKoZIhvcN AQELBQADggEBAIObz14Fkbn0Cp6qo1Sbvv8owyKxPULnV3i6qx9abThlAusbMvjt I4MXVmgn+mNV8raivSauqKKSp0QWKVrToMUIYurJOHqDBf7idL/g6ZP9u0RIcxE2 b4gK54xdfh2gnc4BS+5LITB0bS96zFouz5gfz+pj7Yoe8dteylSDeG7rlO6g6qgI Y2EkS9eayCTfr99joCfvHhuJxqWFQq++OAPujbMSC03CKb87Jg7jk3WKq8RzFgut tuTtJjPGvDgv15axrQk1zktIzzfMG3//gOAVH185AUbIA5tsiUmXY+q3mHxJejPS sr1dXay2Kw2scEgWeYiu4X7SrJg2C9ksCtw= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-key.pem000066400000000000000000000032171477524627100264570ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA1YqHJDDBWxwzrIg/iS2pWXNYN63ddMW18BxR7IBUFHa/oOoL hp50ISjBxLEFn2xpp+RvitEBto8ac9iKX1wNlKTCK2C4SHH+X2hG1IJsHfM2jVzb NCOrGWZPBqymMDyx4V2pveBVH1cLHnCf1frCu5Ypjgf09QLj2pgYYKiILm8+NDd0 JpIfSf65Iatm9X34K9r+qio0PGyMjzw4fTgddCn346qGjfKIF7hN4WXUE54kVltZ 09/DakHSYOk/jXBIXh9B+KWFEsf03RegB1zMVhQ+pV/Zy1PVBqVywB2Ot61SQr9i iDDLAhgVDypCik7o3Rbtih4edgrLXWrEN4KZ0QIDAQABAoIBAQCS0tXKr28y+vgY p/Gvm2K2a1V7qyL6eDp5Zq95gl7NVzy5IlNcczR73C8m02R/UiZvVuTOuJiJ8mkI wBmcKYn46zPKaY0r2p1A8kzJLwexrBmVJwRdHC03oJ2zhCAcSI8x2pmsQUOl8c5m Vp+/m9Qq8LuPua7Wi+8ozKzuQNKpwXcdPlD8jXbQNVmjbFAzSnFukxzHDxkEV2MO 9seOf0b+2ByoBsAEKW6whdHX7qEJ68OjAbv6lYu0dJmAA3v9rXdKk1tYKorvGRms sRcSgvjGol+SwM7daa1V+Pa4c8DszSfieocCWJt1wde4BEMqmdgZVRXxjJVHzEPa YkZsLTCxAoGBAPfUARuZlH6yfVi2kK3rpjjqToAzSTQW3GRiEAb1/UeE3XKUor/N bJxHwB8Zxkk06Qfjc1Y7MP45qTfDUwk6pLJSCYsdX/c1rtKPHKTGKrg7qy6vYqNN sNv/E2C8spTCe0FD3hOFnRPBFtZrfRuWl8r7eI/uiTzES0E2QX/s1u/tAoGBANyV GJSyvs7vZHXTjaTd5prfpcGZFgdB0DivGjFISfTJ4cqFlxb7doDUh0//xIhoKMP9 BnWpV7c+FN5sOReB+wkbqiWqRaflm8BhEJmHddYfpz3kgBBrfOHAA6reeG1kBJ5h NqSagt3VMmmkHSrGtuzeZAHF/owH/cAqgEIaE2z1AoGAIgWrxUM3PJF6XcRqZkX0 gtm/vx/LS+hbhzhjJOF+TOQzlnhLQ3OLFoVPHbXnH8OwvhF+kvb7SdtWnL4m1xyC 4awbfUqiEwj+oA9fikteL/6ZCIaxTuPqhLkmyt/80ClGzHXptdpg7wZSAUuuWCw7 WHVfXrsLghkcj65IHazA8R0CgYEA0mQPPv1CS4RAULYIewD/zCaXJhHK1f8rCHXT SMBHcgkAqLdExjHw9K5BpccxgF3AzDbRa3aq6Gd5ZjDZP5fFhglx/1zp9VtmdFbg i1+NwD8OuFTy5TZwta38kYSCXuwwD5Rvlw6c1dNcszBKdZt1rHXt81cTFCMnH5wq G2JdtuUCgYBtUJVhPpUn23Lnmc/UEH+kVHAx+YytfOngUbJoPBe8HgNUnB4Gczii uQtKfJMTEXz/e45tAmNTaA2MwTAgbD2sH1gJDnALqUTxoWfFW4qTwl5ZB4CVue2X imQj/a/PallX0KENE+0VCQRhSwdhO60tMHJhK1KEycorHo+57QHJrA== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-01-cert.pem000066400000000000000000000030111477524627100276320ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBBTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9MT5rT 8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr/BqU BQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIWzgxr UzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUrWFwN jngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxFfzwH 4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxLmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAPyxQodHHHRAW9t9A3jR1LE9FkEzN FmzFb3sZpYIzw/wTmD0KvbW3D8YVe/lj5e/LxhtHzun+AnFa/JXoK0t5uLzItges zGEyOGFj3zP+E9Chr1fr2X8sHJjqnUrknC9dQPoiBKlNRCWTP2eeKC+yODgEmgzS fEdcuuqGuM7MFbf3eVwtkWNCv1R14GZN4cuSeQn3xA8/8aLcGEHV2N1GLVxK55x0 hbB1hjTaq/t9a66ByIiaeEex832H1MRRkPMhEQxAIx8HZFQJSX+yf9CtTJTc18Z+ I/1utSjX9VW+4YW/vN7kvYgQ3YQUbMSGjqkROe/hb12TCTSbSt6O7P3JTw== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-01-key.pem000066400000000000000000000032171477524627100274750ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9M T5rT8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr /BqUBQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIW zgxrUzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUr WFwNjngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxF fzwH4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABAoIBADIx0seANENROg0n GiQiCQu+cqiyAqtju812R8gE3Ay6UYVH943f8NOeGO5qgrp0Ip/OyoYc5w7lmIFR 4hbMdlDQTWAC+cfyUX11uvuuym9mUGL61WW6RJhsQERe8Ni28v5DV9EPx4/RC/E4 JGEztJeZPUSgmfwUhl8SxoDOgPRH18ilJ2kkBAC1UrS3jFTTSiIENXfpCkXcaBQN +DFstygHragvXXIRjpSAYo1hv/YNJHAvdsPDGfEYvL0G5MTwLtFVBMZVzjQqkriZ pTYke2epm1cNY2+PrW3yrZhgaM+dZkVAaRVqsAKbP6L6z48PlqSOIKJtceOKYVMT 1awxrUECgYEA8P3qOt7R6ICu9taxW+pVxScLr4UUtpvERXigUjbTCW2VTXBS18oe 2MwihcRtRa5DI+4qCNC+mIRIKWKLqTyqiL733fVykEaXJQGnUtbHe1TmwT5TipQ3 cc9XJ+8di7BsygVGrw6S3Yu20HjSbg8pTF3CqpHfGYd8hUl5Fb5bBCsCgYEA7RLv em1T/KlfrsT2jNp7aIH/frhPZu701UTHEPw6xbr/wXrMRQ25rKB0BrjGhA2IDmHO C3qXc/6J6uyl+Q7UnioPJ5uPIWQO9zmOqzUeaqpZi4ZDzKbZOcvDXGEh75SAcA2F G8rWgUsSG1MJrvRyc6LrMFflzFboNlM1RFKZuGkCgYAPcMVcJkCeu/f52sGcWQRL 0HegNE+ib5KAvPzQp7CXzwyc3JCCQPH/A/1GQtKZf1PP7oeE7xL3JKqW9DhsjHWE s2+gKgTAF7XBajy1Qwue7E6onPyvSVXQFe/IoDptY81kmDfyKWXhif0ZmFQrNw93 VB6P3S7FLPgedhmq+fpM5wKBgQDQxmYi2Hin/riGfmnpLNaU+DZ57/Hs4e62ibUM 3jMbwkxMpMmS3j1vKnZDHSGlVQ1GLRd4wGL0AWqPegvHwym2h6+E39cyPp13Vjav yz2YDaCB4KGAbkbKQQkwJ1HtXEenZiCckwpIHkY5zeyvKBJKh2S61A1I4BfrD2fP XTaGUQKBgQCNb+1O1VNwkxAXYTSo8VDoRAcN+RagRGRPIWk38zAirN7+RZ6LzWQe 8co0SD2Ex8eI5U7bHGObFIzNKgWNkZFX1CkyQMV96GWMfwdaIygOpXG9kbyaJBLo /oZMeSRMIN+OSMOsE7G+93y23sU5s/fI1sXn+U89n1frpnyVO2d9Zw== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-02-cert.pem000066400000000000000000000030111477524627100276330ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBBjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3bLi/ F1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1ZojJbB bt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUhGlWP rNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB8rIC SUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmCUUmm Bc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyLmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAjbMrXjneMWFNr4IHuVsjso86Ay83 6E8ltbjOeqsyWI6uyWzaJvBlec2Gyr4rib+wgFOAGFDAGPB/hvyveas6wg4JUbQ/ P3e8a+0Ls6eTYz6ijrwUGFxVsEu2VQTF7oy7SEbfabsC2g9sQSiaWJ0Js1txIhaH bqR2bzRDorgQ3HoLjiKbVGNvifg6qjLE2M8AhJ7FhnOpUpewsanBn4p0BCjfG8Ne v2EaxdJxK+kHiWX4D7ybOeIKRzX9E0HMmCh1KyEkV+qGsOrYq3ErWPcoPODOT/f3 CLrfKmdHHGdxCc+NTxxSyc/CMZceMnV8yplX2ixPY7o1gFQyT3CvJ40Vnw== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-02-key.pem000066400000000000000000000032131477524627100274720ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3 bLi/F1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1Zo jJbBbt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUh GlWPrNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB 8rICSUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmC UUmmBc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABAoIBAC1hE0T0P+H7VgEl rxyJbZU1iyJ7ExYmI2616wTx63JZ82U73D4qG249i03xOlOiimQpyHH3ps9h+d3O zPtx0914Orb6DdEeILoXBdbLWc/BqJUtrQVU3Eg+wsVQPvd32m+A5N+io8WIFDa0 X4VOAOge8+yi89cNkSbtsDFsfnrUXQf/CzeNKutEHWqC3GmKtfxMyEVT+KbpV82b ghoe+MrhILyMoMppTz1QOoo2ESBHzLz3WxhsJFMqhoECFmatNsq7etQ2fw3h5B6G oOStbFB7AqYALuwYNdL4UkLA6k0hifM8ew/iNNSl9E9s+GYfVJfAE7lneI9PDwJ9 9OLGNWECgYEA4qNCay5zV1MuJFdEFbZV8v7Nd+B7tIHE/kA14+rxNkhhMee8zLsc Ve79UWWtVLhDsZSTkdUw/4r+DF7/TEl1q423aKD5Ar2jyPpW8rWOelB9GZ9R6Eui RutJclSWSBO40epOpIRIQjgFzN+gDyfvM1rQo3ZJ7dZy6bPaEHudlH0CgYEA10Wc JKEFd3xk62uYNv+nBZ0AucrGqlYLsaqUiEFRVEJkVng/le3DNNJt/kIvi9vhunQK ksZCHjupsyfMfYS2sgkzzRfyZsgZH5DTNexX1dE9ijngfh1EO9rPQN+nzN60fEIy XWBiLx0G/b1YqhqSHyiNtmQlNs/9PXDzZtF0aAkCgYBPngjkPFI6uDb6f0mk0wRI tCicV1k95WS96PLFmbCZYD382SsjRQxESAvnv29v1re5N7fLwHhRHZP35puLQjc7 SDLJZ9tykgpqvT80ToG6CHKaQLT2hTOB9IA90OdmdL81xzlPZEU6NhIbkefZyy6l /N5UvmZkTjTaUttldPe7xQKBgEiL9vEJKtR3oXMNEYEHavwjSwlR4t2oncxEFkZM 7OGedj4FzDf0pqJ4gAT9vRQ/B7VUQfPwyHtz097CfNGYFhGttD1b4p7stDrFDcjQ W1F9cGXS12ro5gPd25abSOtr6hsuG9cIEk6aU67TTrwUtEW33vomibwNH+TC2eQ+ liIxAoGAd+hpl56dAFSEFgaHn/cq+nKgrT/wkJjjIukAL7X9GCM/+qsXQbKjCh+B pSi8G8ClSOoLLW0Of/j4uMrD3yNW/3Mww6Rkrl8Pemlno4+mt7ZJW0jGImM1tiFk QBwTY2HA9w7HhaOn2eDJ/pG1FSdbCeDBTuNMrAtVKsHbJFvWTdw= -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-03-cert.pem000066400000000000000000000030111477524627100276340ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBBzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatSMQdn RvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbNXDzY 9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqnuk8Q R/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbLLVYy dXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBIoYTK FpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzLmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAM0XD1w1gEzD/1AubpOF5r0zvRGl3 Cr9ucLLAXA7/7kq0KsL3FKDU4ybVRZauZVEQYVKXAJGu8mGHM0VIRXtvEjriGw8o 8acKXoMWfKVriAzaKPzHDgPfc1Cq6ejsLrsFMge4BqSua7OVmMNedshEU+Z7bvRD w+ikh+S0DxWcZxFVnKQqn9WSGvlCF+n0RG0yVjOHt2tLygfzcUVAifAuVs3ktyeH enX54T/drMIUaUBRjwlMOBXwehRBfOnPpx+RZ/W3IIpf4Pi2XfTEAHVXiDSPz3vM l+Kqp4ntmJII1XJfoC2+NS7sR2OGDJoIFlrpmetFWhUkSMojPxoeKieR9g== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-03-key.pem000066400000000000000000000032131477524627100274730ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatS MQdnRvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbN XDzY9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqn uk8QR/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbL LVYydXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBI oYTKFpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABAoIBADTGqzfcQ0cB3tEn ni4bmKU83NM3WwjL495OpGpKgoRkwdijLoEnwHgrgt9b2dQxsCK0bSpMhCaWfV42 lx09CR8awM4d2+refsSc578CompMoMFCOB2ww8Q1iM/D6aaiqaZUY2VqLOE1u23M ZGtJlqY0LB/jETkRi8KX4dyY7YwUvHE2aDIkz0BKxWudmSbVNT5C0Ykujrm8EJpr KtRPUoBtw1+gGzEjuIRLxKJbWb5NS8+WA9ePQsS/10u9/qxH6VZ2SHJ2BzBaTHJr 7z0VLtu7VqwXz+N8/N/K6roSlJWihRufPwNcaPagBzWVCURyodyWcSr65VnJB7t4 9tcnnJECgYEA9a8sIAsmNMb+Kb3AESpxmdFxgX/GJrP8EPis+PlFqTx6EM6J9dbd e6r9hXAB9tvVouEEi56hRwfv3f0OEhlwdYMK5ar3T2mKrzFoOAT53YwULPj4moH0 M6hta8B+YBzkWeOC/S6NRzGEkB1ORFI25nZeUtqJ3WvAxDXumxEerDMCgYEA0ksl GLCbxCefXT9fPySAUafdnyNtBFJPeP3ykSoXrBqbXmdc+1hRUS/weZz7xVtazMrt Ueg52+wtMhj81RKZbW8wpfrU4tAQcb02DyJFzGwvZT0F4ChCTuNJn3HBtCxNCm9K 4k83BV+4JOrH5VQXncaOMz3U9R/ROqkpJ6mT9eUCgYB/D7d8YwfBZ+Du6Ym51v+l k6JmXRS304HbrSCYKyMR4YsnwgmrsRIe8VHofwMlDpCwijt1kfbK65nsbwGl7q4w uDMckI0S7vygmqbRwhEPuXB9yc2Y6vjG9qaQgZ9aSdb3fiiylC7Q0RVEC0P9JZ/r FPC3XPrMHvw0/ceHPxVj7QKBgBfGGQaGiMWPOSwAixMHXF5e3OYtyhhP+d4Sz4Zr vMtfIqt0ggWEQYUtJ0GIZsoz/rriQhwdZdUgCSf9vS+Vb7T4egZ5qfGOVh2Vp3Pk iwGGRYFreauSwZRLi7oz0RM+YuNIG64kCHNaE+ZQiJK7hAP5O4A9gELJ+wxnrVhn imulAoGAOtDcYdA2gb7rKHTWwVhMEmFtztPGDOzQOMsKJjilRpab88syGtH/Eycd 1nD11MsLGQjrxaZo4eFWEDnomtMVsCixXzr2mmciDw3ZdRdlBrEUfeUjTSBtxSWA DSx94pwGGVaT+C+cDaFAOms9k3B96AwjomWnbH94w5aufOVy+ak= -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-04-cert.pem000066400000000000000000000030111477524627100276350ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBCDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauKVv2x m+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2Dm/C3 dAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE6XQw G7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZs6Cp LmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfvBK1I gbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0Lmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAGWF7FwDv6NE0oEBsjmOoLXNw9qyJ TEiTfrWgHfJ2XFDIbpj8dmquX3TmSneWTFRS8LUwR73zl/JJ3Op6Q5ISBpwSXCh4 8V9XWOB02fhJN1NhXOqxPgNH4EM2m+x/C1wdpZuOrmZjuH3uAW4lxi3lXS+H2AQu ixD5dXgzrR8l7LIo6RAibqUs2aNG393ck19BF5ghAL+iZtZs0klTrsAyQnzHIGMP Za2DnK9yjir0M7n4AXspTvtvFj8Zr3WzCAwKW4w025iGxrbJ7Nu4ED9p65s/KZr0 u8ES5UVaVM1v+7Y+fLzt0v7EX5bSx4okWbNbcMJ3e14JqUGoGMhB8o7IPw== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-04-key.pem000066400000000000000000000032171477524627100275000ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauK Vv2xm+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2D m/C3dAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE 6XQwG7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZ s6CpLmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfv BK1IgbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABAoIBABJ6PETHyBGKd+gk 4MbhpYus9lVv4IjqEWdOLYcL/rqeRIsTzZZNWB1f1caWEkdyaivtpLXhUZfSAxtl ZtiyRh2Fa76rOkozhib+pqMdlNJgWejJQlUdwsUDf/cJGUXTfhwIDxsN68cMtON9 I2ATt4sash2NvR4eLeL16Lz7tC8u6ANGleI2PjG+17qtTSnSoy5wO10M2aK+SrBn V0C0pT1BqapSTM6NkZ9Q/Z8gPH38Q5Eqc1iabYh0tHop6DLgudh+Au+z9q9NITgL fkEYxBIcBZ+1b/6HT5XPd1j5sh+VbG7y/xtoIU6YCguWkcxem/JFJWSgSVTyfK57 Bg284rECgYEA59RnZbyEC2VALPrHAnW7+Ik6LM6fhRED2MlIy7swzrjoT5s5u9OS li83xgVsSRhwWCf3UoDTyG7ArIF87hwaPXbH8PfZEsV2K8P0ZyaMIz2vQcotT9+A QmbXbKiXSg6MoSSJic+MrnCspF4c2YpCRnc3h285wUwH9ZvOVfkc6GsCgYEAw5z7 XpV9iO0eA5dfIeHsTwbFz6AcbPkTPZ5nf4zn+o6HaAV7oyV6f7bF0JYf26guvvg4 oSYAU9VLRQrw/zzXAyJ16mnJBdlqO6dZU1G7HI6zPZlovuJvze2Q+tNUBZGD6Ouf 0PhWzQtxL+LcmpCul4sDCP7mUCdG7aiClNg5LncCgYBRL8JSD5XSg8/YsK4W/3In lK8p1+ZnbFEDj2IN7u0lx/2bO0oZq/tO2xRWJboJUySsyrpDS1hffeG9x88fd/n8 gmd8zN9ZQouiwgoKQaacBNMmYA5ERYoeNvPEWro4tiWrnScJewTSZiUfntHNoSya js+Ef4AjYGP9MGYvF4F3JwKBgQCEl7kPwNbp5IhuuMFkTyaF3rpg5U+/UgoHv4K6 Q8HO5aPFD8phqPri7PojTE9l7hdZnRmNqhuYt8CgmS0IZa380vQIpBH95ASNUP9M ad8iGVxHyd/lW8mbVYfrbSnL6Hn4fRbEaEE2FaZwZh9QqfeegzzYcSeedzEt0QZK bFI+OwKBgQCDqh0JwrL1WLXlor4N45EImHbeMbFzIZ4ARQuGptQ/m66MKN0cdMG2 qCysRlj7eac6rEJo4gsQ/ASR0NLPZAxDnHk63PP15ObgALDUppa7fQgTpUgNKgD6 LqWoCGvCm5etr5WUu621mlZoZpi4IWPU5zJfSoxNwGOJKy0ExNws5A== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-05-cert.pem000066400000000000000000000030111477524627100276360ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBCTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67ZOxt pZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5UOZEA MvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Ynoi+V u1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD0DF0 mmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGAKlld +ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1Lmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAkmk/YJoaCGip+h+kUjmUQHT929TP Ix3Jd/nA1LT1QQ7b8rP+jImw51CSLbWwNGihVRKkOOR/kkKZLgWdsa77jySKvqnf 2QX4YcpaSzXitctiVoJAXhyJwEo4y6sRJS27IwFOh+X3gHzPdz7y5KYRfj6OHutC ArqD6Ohy5dRg1Ixwf2+2go7I1pZIEfzhFHzFUylFmh9ko5EyTQ7cQw8I/D3UI9TB 3TCPUrlSNGOHuREsGMtF7An5Idxytx6Rsx45sJq89tB1eipS//Pk/obRGCZYIb2v ZmsOSYeppa+97tr2j3ox1cXgXlIvt7ANroZzVHrPhiOMCztrlkFhlqUibA== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-05-key.pem000066400000000000000000000032131477524627100274750ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67 ZOxtpZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5U OZEAMvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Yn oi+Vu1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD 0DF0mmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGA Klld+ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABAoIBAH6YfgjRrNiYXZQ5 haM5fV7nO68jKB0xur/FV1ouPSExXQHDhY0sFFRqJzN6HaCkApgoEIo45P70nveT /jtrtPoBqnM7jVAD8m704CH0qglhfyLXa28uxxhAAJhjxokF/weHt2nbL9UxqhUB poxGWHxVhbQwGV2fYJ9vEexn/wp7ok1/gU7E/TVuKAal/PGMjOgkWijDUWHFRzZi EbaEfXBhLROOX7P3+rCjd7pfPw8oc6ktGYBpQXJmSS6jiDGypPm5XGjvb5lU0wTM hVEUlKIiZgwtdUieM3vwENoX8z5s3PXkJouGGWhUN63o3HTMD7WppGmYitSTO93b 1u8uRgECgYEA204MDJkLZt10r95+d5N9tnD2gjENJHQzoaziKOOBoTDaTP2+69CI NpAYOKXcbxY7oOp1a0FbQEcmbF9QsMMFFPzm/+9kn8ekrPIxPmO6o1DB8PTcWOBD br7x3OgyNtClRa+91D++/CkxBLgZz11uo/VW4UDl5KApuMEzGs4FpKcCgYEAwOV8 RgmZRZMbS03VOvJsZVhybHVRnyQqtvsm8o7dE1URvHwxon3o9sryGLbtWLh43DcJ Og87B1CchuiydB9QAof32nVZmAcW0snxmg1/fhDiDO0Nmggm69097INwdypQJNGV Bo3c7T3uExHQgFVjjyA2hNj8lnLz9H8PJhQOWokCgYB2em3MCIq+Yfn+YYeVdls+ Sz0CDRn6jcSvHcV5LaAEw23MlnGk2J4eTC6pvAGhCjPgtYoGDeMMkOPTPJNNS67d LdxmfKCyKZtvy9CK0josrR553O+GHHKRzBrCq7clIgeH5G/70QyPEnnnAMcA3os9 jPgI7ZTFtTmVFNtVKeSmlQKBgGJr9TfLKABe6dtHcFfuX65qLZ/3UFkx2/WXOi0j BdwaWL1iZPTy544cyOAhXgMZxNkf375XDRhTYjpXus4TOADCnY1CuR8f5t5Qmcyv vHB3bs+HmJwSMsHAFht6iUTUOyY+JZq9gY8vPS1PtqH0b6MUnZy1FlQxfRYwSmt0 akp5AoGAJMvbFeQtX+1v8ronYwSZ1RK5RrvhUp9myP+4naeG9RvvlmmhRf7y+ohe IRXrvANTbbOTabygbLvuIVCmBus6fbtkH6ez4cLn+CnUpHCbpzrrXKLT2T9r9tl+ IIgVtavl+Ags+X5XvmP5eGViFZ4oqSECM5Se1oqBX52ljZ/FQmU= -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-06-cert.pem000066400000000000000000000030111477524627100276370ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBCjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJnWZVQ /Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K5Yez G3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lYirw+ hhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/LgL2Pa z2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicEtGLd w5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2Lmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAYGlcBAP3Hdc7sruYdxXk8FYFIBrh vqLLElpR2qQZHfnXBTDjRf4ZyHbrEPsv7JWDlQMXI3U5SBReiqvv4LYgQ3W6PvZ/ W3nKUlClu5L64E7ZGGQvGh2lyNWEdG21dlpves3Sb5UNtZdAuOBBJihHcJsUJGSE P2+lKy4O6lTTba0H9wt0uOiqwUVkwHMF6rqirncRLihXaLcAOFb+NSuFDOMMEX7D bAZBFDkk38Y7fBlWQ3sR2EvWpXp1uzg0ug8zl+JbFnlDswGSom1Y/NXBzIDYx87c BlPrhizNUKrCYT1OFz+a6VIYTNM01vyNZ2QEMYgcL4Ug3wYeb88HxEEeMA== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-06-key.pem000066400000000000000000000032171477524627100275020ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJn WZVQ/Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K 5YezG3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lY irw+hhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/Lg L2Paz2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicE tGLdw5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABAoIBADCLDXk6X4AUmNer Zj3NxYC+6nzaQb7eaoXW/vd9lf4kBvQ/tovdg75w1ri2BcO8AI71HucvA5qaAcF9 Ah5G/ToLVnDeUvWUMzGJIAMonMaSm3INAYGqYPos7PsLOEbc9DLs6aUo9qKwe5bC pOjoeT1xDSjvCRFoQSHM1GIqziwiBSYl+30wB6PtE/ww2HtLm1i7z3u8AOB4MHYY hpDm8r0VMys290goZGSUgz4OD7lXV3Is3F4DO1KJAcP06XDhJhUhIn54xhGJsj85 24p7hYx1D1gFnbsCpWN2MPj1ZcXgYHR0brW2kEvf4stq5uYqDuQyT1VtvP/31jtW kmTZBIkCgYEAzY9fsDT8c6DU+Pk2kAiDyKWfEFwSVy0FY1Zs1H0DzL0go7szQGAY B3bzuuHhvHV450zwLVBec3H8iooi/2H50nnQg6IAjSjvu1qWjmuK6zS/jMcJ1ONX r7mscyPmeYgnEG4k1AcKnbBVQ0mCYw2u1nJH/jxpavp8BXjfAdhAfqcCgYEAyOc9 uKZk2WwC+sHXC/gUfjiorFql2KVqJUG/K7hL4Yv/no6HPnI8ksHF9TUFtOGMRR36 bDd+DwFuaix8IZJw0M8XU7koOrh81iDifbfkljaY35qMlqrVdKIa+fUUh8hgzBWj DLRj0PoymzguaSoOrnjj+lNhR2hZs0tDayPsVc0CgYEAyJhLFpb+bI/chQIdfrzw bVNCQCK5Ox0SDvh9x3YfySIeweiigQXFLTOlcQ+Qy+oPDBiGoJG3Og1YFpHqyTEB K6X4Fxx2UUpLd5dVKLJFpHWbH3P0Yi1gmnkkkk/MT3jQNjZl9grRD0TOT5ViiesG XPq0yqKFdQHvGR/078XqWi0CgYAjs2P2cistr9H/uX57dARAQdVHe3xJOWvZXwuX u1JQrE4qYO2LnUVCVwjUgC2ZbRM+HQupO1s9U2XJnEoatSkzEfn1OMv7U1lru8BD 0u/d+anE4nPlOkpgRYZBsNmLH4KEAbxNv6iVNEDV8G/e5EhjnLv2eeaph9OY1uEO Iv4WaQKBgQDElaNA6WQXsZAH1u1LLC6EWAKAJ/SyJzhMzrs+51MXXB7dIFlpM0BO DEmXkXSUO5aZQ+fsBgvDJlQhp6yheoLb9K+eCLgELh40sChg9jDFKdVRDShJFh5O UZYOrHYUSdy/gSzhaF47c+MZrNwwMfE6fbLSOtQrGYS4w5Hh5i+RHA== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-07-cert.pem000066400000000000000000000030111477524627100276400ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBCzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnroEDT/ /ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaNHlkg iar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbMyyCh MwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1niQB h80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam3dHv t8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3Lmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAUcf3kUOmWALEcw/2DPQWzY7nn0mv RGlgQP4xl3W0MLXg2IywAqpOsrdR8sx9sNBA7WYtUT0fdR4gs1PWbddQeJqIwjLZ kOxjNwKVCp4DWiKStYA6bIdzNnxXvMKD4FHAUMBo7jsEGzvln0IfutNeE8WKloAB nezVDTefi+kbB8npk93yag3HXQldAKjUt8VZwebJKu1TvCSLiq4BXXV2DjGIrmTb 0Zhdbar1HeEhf1IsnUyxfuCS+eXrVmF6XesRiaWux7Y79SGD37bBAH+dMwEuKH7+ 025W+IXGls4RvwfoXjVe2GNT9G5aLzzKwY2SE+A3GeeAqf8WCeCAA10nCg== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-07-key.pem000066400000000000000000000032171477524627100275030ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnro EDT//ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaN Hlkgiar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbM yyChMwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1 niQBh80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam 3dHvt8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABAoIBAC13dR9Ay8eTtWXs grfx3F0ynyvRQxbiecRa4IM0guDdZMOMr5DzRvQRx+GiVdX17GLcCEugbM67KmrS btA/+YrYXVXtSfDoELsD5oi+jz/yxg0v0bEM3clkBKoxB91Im1+/v4CEw2EnBtDb T7h4l3kfMFtpQa2xDebnPw4YdllVGcKEwG5xqMe/EqMg+QpXbqsjvC4lCTJutdHD NpdHiz/GVq/UdrsM2Wsw1nlBokbt92Gn0LMx/R9kiegwRr0b2Mf1hxIQ/7lStAwB ZIaWPqQn/nzCZrdn3cx3yzqKDmYKeXSME6SNsKEY465wO1iq0iTdPTdDN40bFZrb 8g31GAECgYEAz2NCn+p+12G9EoeHm1ozdahaF6MZfTp/5cgJwPWeY5TYy3qFFCvE XPqaOo3EpdEYWjmNBgBqFrlgkuUpOsZekMEyAm6YmG+EPDK9XFuRNTBRaPaDp8bK RMYh3oDToDzzYVig7MJKi1a05rbo/oGogqex0kMEdToAFIcL56PL588CgYEAz2Lz sx/hTw0YjaviL+6aSVomttJE4gwrUcMheWUgGQDOK12wSsNqW1UWXdKtLmjqlwbz y1fxVsCLIdDFdvp7397+xkC9PNPoF78Nq9Kbj6BiC870dGGm/V6o9Bw1FZpDwT0h aJNecMCHP7PUeGtw0U7x3eBBXB/BCQHa+/iXXoECgYBjDm85QOE68RVFL3UHMAta TIJTvfSjyvhiAa0e/HKd7++pKSk9XDZbjttx0ls2NGxkVA7W6gXCOuM05r5Ns4hh rdW8MaUzUjigJEAsSBRQinaoIu75iUr9lIGC7JeodtRtD881pwvCCDU56e3Z+oZJ ed3Gp3oOoBh/tY1rI+J2IwKBgQCEtMw/NkpF/JpWpcOyalueTqrxWDIt+B2MT3JP LS/R8Br90xDpdozLbvJGDXc2eHqNNCyzVU4g9krR07dYQEceZaLgmDLABtXAxEfq SHW0/atf5Qm2o3ppLbatppMthK6QrB6BvbO0MO8bC2cNu6rQtVS+Zy9L1SIAMoi/ rj6mgQKBgQDDx62JtXHSfaiIOEkpfZW1kOvzYdkjwYT92mJgrpso9ZDr/FhJuAj+ NxPcO/e6Iie3pijTKbLpJQdE6n2srJ9GEdLPdWUC94ecwmKBA/m0GIO94hm4oIka +B5U5rkTWxn9xUtOfX4O4MBXpSvgf2a2ZrD34JNftYwzbB7pCkRxGg== -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-08-cert.pem000066400000000000000000000030111477524627100276410ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIERzCCAy+gAwIBAgIBDDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe Fw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL MAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5 bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2 ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5Uz/kP F5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4NjancD RmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Qnc9m umtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2gwrUV W6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU7zcJ u0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw CwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov LzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA AAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4Lmxv Y2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAgIB/wKMd38ZiSGP+Pj3t/h7H3uDV onmoqGw5+By0B555k3g8AHaGzRw2DTWz+1ncxkDZlZvt/GNDX+gg3B/MxA/HKlQp nf6Ctbo4MFkHtw2DlLPZSo4KHn5TZnBw26R6SApCIJspGBTjCd71l2i50LsaLC9S fX5OWl2Abgz5xjX7iGBGbR/4S5tWOc/JmSQJtawLGF1w6JDyCHIjWlmz0EjzKu5d w0+8BGzR8s116TA9PNn/BMf7lxD3bEM2Vh0wchikliFNO5Bdtzz8CCakvBrHhUK4 Pu28NEST4868VmxhBpn0Im3/3QFfaKuQN2AeAZAojdLorqXwzDZ8wc4XHA== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp/server-status-request-url-08-key.pem000066400000000000000000000032131477524627100275000ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5U z/kPF5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4Nj ancDRmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Q nc9mumtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2g wrUVW6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU 7zcJu0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABAoIBAEW39QFlJpc43DtP prlKx0SJxFfXyfwWR/s4sFaClStyLHagrG8vAERXSb2dlNLdcRZma0SDyiMVIdpO lc3En876s/afC4mqjJ7td6g3irhc72x8jVNNimLKeZhqT+Lb20/RBNj4fqe/yvo7 S42yR6ytn7YeBVcSOoV1y6NP7t3AW2L84snfXKBqddOmwoaodV13IDBAEsmwjhVy XkwRhiuPGCA+A/9x8YWqC+/BJMXQ0zEaAtE9tPnebWbbY8aHhCEhEpo36uG5Wypk lDS6Bg/w34IBbuUho0NFFVLXkM0/yGVS8UQW8eqEEWLJq6H2t8xwDWNNbXhUgl95 B85B7bECgYEA60ZLMjYC3BWl+EDlxxxv7DT3FkhFAblWH5IRfCc4ilMJaQxPdwYj n2Zx/jjrpbk86AiuI4YdE1jM4FIxrkAAqiD91/uTSxX9U81jujuR7GDvBxY73SLY JvinO+6jpyyDrik03EZh2vZYr5VlwK0qTt588GeA5JGatVxfF6ID3JcCgYEAyG3a 4+/qHW804XkfLrcM5esmBwPateleY1IeQPgv5W3ODlb5uS5C+8eLToM5MCkxvUaN 92siQk/o5+yrNuVUB3Ao4et2iBGpASC2UZ5i5v0mctYOafFz2t1rhPyISI2ivbSe q272qY7P2qtPMMeyHe3xF33nHcc3JnSiu62C1ukCgYEAttYPxdt3aXVhX4V/i7ar u9KFWkmbZvWS2kH3WJZaOBTDsWEhuAuLT2qbl8bASi+kB2YHfg/RNKHDxWfat+GB IrU0z83v72ANWDy7DZURl7LUzpsWtolHlTGTNN4FS/sp8gSP0cbYcQMUdI8TXiK0 SEpaqbCl1/rXUa2RMJp1ic0CgYBJX64ac+IJFIUPZf/8YhbetM8fElIm9mAPjCh3 MGSYYTJmHYEeQclT0yE0hOWStAH5gLLIOPg5vndNMF8doaa5cl4FFuY1ugFc4FTe XFVoqpRAxgxQzIvVO+n4rVpW9UL2oADWUbELMbT2IHDFMtYKDumL6BOL2zpdYaWR f1u+wQKBgDk9d7b/D53ThOnW+0vYMjo4YMxwg1dcdOBB6p1TrpEBjoqZSPOaY4Ht OC/cYzvxh5H7YOsfCYzTEea+y3r5lpmpQi6EfvNpbCVsdKVCHFLivFb9/6mirdJq rUMrTPqCM+Q6aI2/GAwFc//3ItktChuwOxJtH0w/AirvFYR/NHaP -----END RSA PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/000077500000000000000000000000001477524627100216615ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/000077500000000000000000000000001477524627100231765ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/caocsp/000077500000000000000000000000001477524627100244465ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem000066400000000000000000000115261477524627100274430ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 17:37:00:a1:ce:35:e0:84:dd:e9:30:0c:a7:12:b9:50:88:9c:16:07 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:02:58 2023 GMT Not After : Apr 28 19:02:58 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=CA OCSP Responder Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:d6:10:15:61:34:1e:97:0d:c6:c2:7d:f2:0f:9a: 35:56:54:7a:9b:9e:a3:0b:ff:31:0d:db:49:4b:98: e0:64:3a:3c:7f:4f:4b:d0:a8:01:80:c9:68:4e:76: 3b:be:7b:d9:56:8d:d4:fd:bf:e1:6f:d0:5c:88:07: 3f:05:a8:83:b3:7e:0b:ba:e0:36:f6:1c:e0:75:fd: be:38:26:33:1b:42:96:4e:62:0b:88:36:ef:cc:14: e3:97:86:dd:c2:78:d3:05:b7:4d:cd:2b:52:f2:11: 16:d2:7e:8f:f3:47:8c:f9:0f:1e:cd:5e:f7:a4:1c: 62:34:03:70:74:89:6b:bc:75:e3:30:82:c1:5b:67: f3:d1:ca:81:13:10:d8:c5:d8:20:05:6d:d1:e7:51: 19:ac:03:96:2a:a1:21:ff:88:2e:d2:e9:67:79:cf: ef:17:b5:2b:7c:10:1f:5e:79:3e:08:98:7f:42:bb: 8a:13:17:2d:9a:1a:8d:ff:36:c2:e9:c0:07:ea:cb: 4f:72:35:f7:f2:d9:86:d2:ab:6b:70:2b:57:82:c8: 02:93:aa:04:aa:00:3a:53:23:3d:61:82:32:0e:68: 33:7e:5f:03:52:c9:53:db:e3:26:46:8a:ab:e0:e5: 54:57:0d:e3:e3:24:b8:d9:69:92:0a:fb:bd:51:25: 89:fd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: D2:00:A2:C3:AA:00:76:1C:E7:67:37:96:89:77:38:69:C5:1B:5E:45 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:FALSE X509v3 Key Usage: critical Digital Signature X509v3 Extended Key Usage: critical OCSP Signing X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b0:36:29:84:91:de:14:e5:db:bf:55:fc:d8:0a:81:b5:df:84: e4:5c:ae:e2:3c:1d:05:09:8a:85:7a:9e:f4:82:61:1b:7b:8a: 0f:1d:e3:ad:b0:60:45:12:2e:38:6d:9c:95:d2:42:fe:2e:1a: d2:a5:2c:82:40:1e:6c:4b:35:d1:3c:a6:4c:1c:73:c9:d0:32: e9:47:c9:9a:fa:d0:1a:ef:86:c7:1e:49:ca:62:f1:81:9d:4e: 38:35:56:1b:53:fe:4a:f4:4c:91:31:8f:32:70:64:ee:91:f7: 4e:fe:ab:c5:1e:84:d1:43:cd:af:f6:5d:2a:b1:4f:b1:f4:1f: 5a:9d:33:7a:48:94:c8:88:23:e5:b9:c8:a1:4d:51:4c:d5:3b: 5f:f7:e8:e5:e1:53:a6:de:c8:95:14:32:e0:52:db:43:d6:c9: 2f:7f:96:07:fb:87:0a:f0:53:3d:ce:e1:56:6f:dc:0e:84:f3: e2:ef:dc:17:0f:59:1f:1a:70:d5:7f:08:36:3d:7e:8e:f8:1f: 55:47:9a:96:1b:11:25:d9:27:7f:bf:e1:65:e5:16:ca:d9:bc: 6f:5c:5e:a6:4c:d0:7a:24:8d:42:c4:dc:b5:4a:75:4a:7c:88: da:21:5e:27:e1:0c:36:64:69:10:58:81:3d:cd:74:df:50:85: c2:71:fe:43 -----BEGIN CERTIFICATE----- MIIEGzCCAwOgAwIBAgIUFzcAoc414ITd6TAMpxK5UIicFgcwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDI1OFoXDTMzMDQyODE5MDI1OFowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFD QSBPQ1NQIFJlc3BvbmRlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANYQFWE0HpcNxsJ98g+aNVZUepueowv/MQ3bSUuY4GQ6PH9PS9CoAYDJaE52O757 2VaN1P2/4W/QXIgHPwWog7N+C7rgNvYc4HX9vjgmMxtClk5iC4g278wU45eG3cJ4 0wW3Tc0rUvIRFtJ+j/NHjPkPHs1e96QcYjQDcHSJa7x14zCCwVtn89HKgRMQ2MXY IAVt0edRGawDliqhIf+ILtLpZ3nP7xe1K3wQH155PgiYf0K7ihMXLZoajf82wunA B+rLT3I19/LZhtKra3ArV4LIApOqBKoAOlMjPWGCMg5oM35fA1LJU9vjJkaKq+Dl VFcN4+MkuNlpkgr7vVElif0CAwEAAaOB4jCB3zAdBgNVHQ4EFgQU0gCiw6oAdhzn ZzeWiXc4acUbXkUwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwDAYD VR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUH AwkwMwYDVR0fBCwwKjAooCagJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3Rf Y3JsLmRlcjAyBggrBgEFBQcBAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcu MC4wLjE6ODg4OC8wDQYJKoZIhvcNAQELBQADggEBALA2KYSR3hTl279V/NgKgbXf hORcruI8HQUJioV6nvSCYRt7ig8d462wYEUSLjhtnJXSQv4uGtKlLIJAHmxLNdE8 pkwcc8nQMulHyZr60BrvhsceScpi8YGdTjg1VhtT/kr0TJExjzJwZO6R907+q8Ue hNFDza/2XSqxT7H0H1qdM3pIlMiII+W5yKFNUUzVO1/36OXhU6beyJUUMuBS20PW yS9/lgf7hwrwUz3O4VZv3A6E8+Lv3BcPWR8acNV/CDY9fo74H1VHmpYbESXZJ3+/ 4WXlFsrZvG9cXqZM0HokjULE3LVKdUp8iNohXifhDDZkaRBYgT3NdN9QhcJx/kM= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/caocsp/private/000077500000000000000000000000001477524627100261205ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem000066400000000000000000000032541477524627100316230ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWEBVhNB6XDcbC ffIPmjVWVHqbnqML/zEN20lLmOBkOjx/T0vQqAGAyWhOdju+e9lWjdT9v+Fv0FyI Bz8FqIOzfgu64Db2HOB1/b44JjMbQpZOYguINu/MFOOXht3CeNMFt03NK1LyERbS fo/zR4z5Dx7NXvekHGI0A3B0iWu8deMwgsFbZ/PRyoETENjF2CAFbdHnURmsA5Yq oSH/iC7S6Wd5z+8XtSt8EB9eeT4ImH9Cu4oTFy2aGo3/NsLpwAfqy09yNffy2YbS q2twK1eCyAKTqgSqADpTIz1hgjIOaDN+XwNSyVPb4yZGiqvg5VRXDePjJLjZaZIK +71RJYn9AgMBAAECggEACnoECdtaqervMOKoH7Jc7Oo6i/ZCJZqqRHLYjf4f8VfW USbI35/xXuO8mqZ3uxVlqDJN29NxzZ6lgLTWFUlPlM/U9CL4HaiBJdUy452e/7UN FS4AQXzq1JKrJuXfYZ63OT7k7Gcz6owCkW/HTNFSKXhfeg6tURdgiQooDVQSdUk6 xX4gVEK3skozRXf4mrjTaNnFCOk2+sZdqrRn19ZAUGRisv6ECf8/wQlh3+ySfPYV u+BHQqzntToYP0HUZAO6rezcTVayW25E+AaOqNdNmSqcOX218ohVCzwzFpzIk8LW jYLyGQBhHHcw+RHJeitcHrDuTTpOZFznQxzHiGH3AwKBgQD91TNHx9Y9jUBkISSi XylSiZEAOjPl4VrhRfI5OUx1l3XTqB/e3xBYLwxEpXjs7m3qyXhCe+rVuIwSjLzc mLCspPZw/fxdRefWW5B+v1HbHxC3/lBOhqaDfLL6x4A/q3n/itG9X0GjpfvRkdJY GYOJea/2rJuMsFs3atX160p4cwKBgQDX4/VXJWxxWUJbObwoxxABG9VTZdI6Dsqr 8tgg+7NPqw3PAo5W+XLsGZCSWQfJTD49AHcHBon5IfEDa5srfKsOXFXoiNEdCjIG zJ9mNtGMokXOWLKgxMoqHz+WnqWgxi9D7QwWWNq5hWnACJUqeqelRMzoNkmr96DX NloqHREHzwKBgQC0jKnlLOfe8FIU5t5AAKBL7T4Og1fW8+zIwBADVBZmrk1JOBUz Wkct8okvauQQ46ebkaLQ54OqcZJwv1q3LoS8yLnitUaEseyuNIMbJMr8qaQiu+oz cOOQM2q7ppw6raYhdoSpxs/Rr4bnEmoj8EH3z26ybyRVdjvrtzppqetWsQKBgQCa YogGA9siy6PqPMVTm9bUFCVfeEb4Aa/pesYYACbgaAB98uP7SnNmZ3m9TjGFQCKZ 2QVFXuW35Q/HVGIonQRuRpWgroZr7+iKeDXdEIKVwU2OHFvRICk6KhJ9EYJ8EH2o Y5HrQStY1BElpH2XXRMZ2rN1s6zHb1Pz0whzaUnOfQKBgQCpfJYh1Yzpryb0hkfa MAL2Rsw+mpYeJ27Bmv+taW5iEVMQr2AEYNJhQx1SjNZOml2mqY6un4UPwhUwqAqg SOgWNQGD5g6xoM6Hom+nZG03QacCYUOaD6xDmVKTY0LnzVBwspfvrIgLKgZ7IWBx KlqvY5FJ+NXg3wHLNwgGzkgVPg== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/000077500000000000000000000000001477524627100245355ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/System_bundle.pem000066400000000000000000000236501477524627100300630ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0: 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7: 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f: 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3: 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76: 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66: 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd: 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98: a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08: c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77: 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a: 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d: e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93: 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46: 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c: eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba: de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63: be:59 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:System@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c: 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64: e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71: f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55: a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca: 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c: 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3: 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59: ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02: 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5: a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8: 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30: a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad: fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca: 95:a0:88:62 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA oyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS fhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ 6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F EVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw apMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re VMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h u+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz 5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j grlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth Wf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS U4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl nYT/mVDKFU/UMMiFypWgiGI= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/System_cert.pem000066400000000000000000000122211477524627100275370ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0: 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7: 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f: 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3: 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76: 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66: 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd: 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98: a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08: c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77: 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a: 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d: e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93: 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46: 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c: eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba: de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63: be:59 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:System@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c: 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64: e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71: f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55: a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca: 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c: 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3: 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59: ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02: 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5: a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8: 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30: a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad: fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca: 95:a0:88:62 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA oyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS fhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ 6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F EVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw apMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re VMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h u+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz 5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j grlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth Wf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS U4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl nYT/mVDKFU/UMMiFypWgiGI= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem000066400000000000000000000236501477524627100276770ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69: 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c: 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f: b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd: 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26: c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d: be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28: ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3: e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15: 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08: 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77: 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21: fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19: 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f: 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44: c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3: 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8: c5:ef Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:UserA1@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2: fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9: 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96: 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee: 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40: 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2: 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43: 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a: 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a: 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53: 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c: b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb: d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c: 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81: aa:42:9f:53 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA tOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC pt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP VPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz Qxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX ihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO CRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z OUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb cS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW X/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB ChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a AC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v seLqxt4dsJfJNh2FgapCn1M= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem000066400000000000000000000122211477524627100273530ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69: 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c: 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f: b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd: 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26: c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d: be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28: ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3: e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15: 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08: 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77: 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21: fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19: 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f: 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44: c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3: 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8: c5:ef Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:UserA1@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2: fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9: 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96: 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee: 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40: 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2: 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43: 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a: 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a: 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53: 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c: b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb: d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c: 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81: aa:42:9f:53 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA tOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC pt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP VPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz Qxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX ihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO CRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z OUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb cS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW X/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB ChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a AC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v seLqxt4dsJfJNh2FgapCn1M= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem000066400000000000000000000236501477524627100277000ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e: 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac: 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7: a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9: 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02: 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d: 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c: 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a: e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35: 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91: 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71: c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1: 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0: 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2: 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2: d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99: b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8: fd:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:UserA2@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5: 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1: 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f: 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff: 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a: 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac: 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1: cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e: ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16: da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92: 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91: 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2: 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01: 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78: a4:b7:fd:c2 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA pnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4 IXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q 1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055 L88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+ bsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5 qOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl I8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em PaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3 qcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+ bsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB NaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp HTG1T5fg6jht4JhLeKS3/cI= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_cert.pem000066400000000000000000000122211477524627100273540ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:37:36 2023 GMT Not After : Apr 28 19:37:36 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e: 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac: 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7: a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9: 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02: 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d: 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c: 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a: e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35: 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91: 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71: c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1: 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0: 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2: 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2: d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99: b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8: fd:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: email:UserA2@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5: 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1: 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f: 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff: 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a: 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac: 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1: cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e: ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16: da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92: 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91: 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2: 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01: 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78: a4:b7:fd:c2 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA pnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4 IXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q 1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055 L88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+ bsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5 qOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl I8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em PaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3 qcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+ bsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB NaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp HTG1T5fg6jht4JhLeKS3/cI= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/certfile.pem000066400000000000000000000225431477524627100270430ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/private/000077500000000000000000000000001477524627100262075ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/private/System_keypair.pem000066400000000000000000000032501477524627100317220ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjIS90NMEbQZC2 TkFy4D+eSZRV7AJM3RSAuD3Gx0e7pVnDNYaJFwjO/nHmL5zB29J+FCTaYTA6526x 4yE4gbxH37J/H2C+PcXtdgOU48SzPr/4Q7rCVLy7ZlmYo/mq4xDow4jcGhhv3ZDr b6NL1K80XEMg1VvnmKV8e6kVhrsov7rgu/ccCMQm68GsBR90TwURV+ASdxeeid2l OO7Pz2e+DF5qSnRhIXmOwyjx4gYALeo6beKmJf0ti/WCNpGKIfBqkxnWdgj9ze6Q qanPmTBxRlfq+8VlT3yGXJ3XtMMnPOsn3bxVdh8lDctvQ5qfut5UwZADnuUN2c2E 1Fh0Y75ZAgMBAAECggEAGJh8EGwU0pB56nbVmOW1Sd8jsanGNgMeYIMG83Xf+6uk Y1GqcXiK4DTOhQuYOcV0UQSmAtQlAriawNDzVRMAiaCxh8e6HSzwrws8YoJOCc2U AbFqkvrWQvYdW62bive1+LZkp/T6SsGQJGNebmRIr18a0vRAaWSjTOfTOFbqWKwD 640JDw2KmJmba6JtOaEL4QWrvbugTNwh3OEHugBVTCiRTdruVpCpLSxW1yZEpwB2 BmxQxHvbtIjiOmHuNrsh21jzi7IEx+TFawJ0EV6Wm9XCbjX62XETraILg5bWVGv7 X+TIDE2JBCC9GZMm9Qj1EfCojRmKfxopv7sA1yBYRQKBgQC5K5NzQzk14G64tyvW 61BteydWlBzFbgiMjYUq9wqgf2WbDVONBUB5x3MOArOmYuJOy0Xbt+pF3pF8wrkl hMt/hZqKzDtdDWo3+PnNFgcWmB+T76Jei9khIk3D9ENcGaNwGAS3l8sxqXNLVVBJ u5qHKeKFreXSra7xlXOuw5IMbQKBgQDhh1QqpQMloXOPCQWWzPtrEa8BRGNMFQTU yZFHeetQjjX5opxxMbbXU/wNz/dRgdfe2VLVo9e4dtQbzCKenuwWeivuDxd4YOsF Von9XDOzVWoXuP01MxDcU+sRoBLwWbpCWMe7r4Ny98C78+/5GssvkFUo+hd2vPo6 U20pVZfuHQKBgGYD1eZooLpH/XgSojpzxgmrEc8nJnq21krpJPa4x8gIp+e2fdNx k0YEViTf5C3EyL10S/Zy6sS3jBvaA7rh4GNPLgdN4V6wp1ZS+vy8KAeQo8US/rds AKG6jnFovzucfGijMuYa4L1ph7V3ORaGHupcbwoK9lUNjxZVqjgcUvg5AoGBANOU zpWjcaxgJ7XNVP0BGe59DJ43tqCuJ3YqFK3l56oPgPvOXs6jQVIKbLHYpcJF+mwL nvbnW36nnJ7niKMfnYYI4CXa6r34zwSXB6Y2Vhqsy3euCX9bhTnvUN2cO6hZxbBw 8hFWvA+j96FdXYlqZa0dz4c9+b1f1bHaitL4hizRAoGAVlH2lJr6s+ZmzkDY7D+Y 6YKyjXaxhHBIqB2oLK1KuxGMiQnRADs9UOC4x2PQPOfemjVTJ3eN3rwxdqSh+Y2v K+RejHBJzbd4JIv0QRxpPAm9sezaNEHa7ss387cLZEBEYUI9HkIuPunKX+2lHITn WpVRyzYjVkFUUcRe3DyTlh8= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem000066400000000000000000000032501477524627100315360ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC06yLixLp/M6pX qxPxaQmYKDx9p+JBKigv+YWhbJTuCutNAUwofJ0FTdgQf7fPE8Km3hEMlziXzW0R /RZ2wOtat3sXE0WdSwBPJsWxm2eTLNbVMzfhUB17Db6My70pmY9U9n4EhIIqKO5x Po1fcrJqd2tHPrpNs+KWFHEKHiYWj2wbByqsFYkeiGPDgTuR6fNDG/DsCCSWRich KlYlLLbM2QJwd53kfESMkwSFowkKjvXnIfq9Vii3UiAJ7JrE1NeKGU56EOmyEDZo zs54i3k/b3A7dW1wWTrJhaj4I9SrRMKu9RxuOBHhX8yP4kP1sw4JF7PG7kf7OcRY YrrjqMXvAgMBAAECggEATFRQOaCKlpQzsB0rotSQCbQgIVutZ5Tjs6nwqTRoeS3+ LFT5zrMUhGJdYEiiQimyHDjgtJEwfUtcUxSWX6/xHCsBMbEd08kK7loLWm2Ye02V rgmX7+WfKoWX+UsUGfMBt/TvIfTN/f+a6ghcGQMJJ0YO6tYaQCI+3NbvAjfKFgXi nWWZA+ipjh+Nu3YhVAy/uMInMi0qGWmpomU1yS+04E3OQksKYc3OmER7zFwbmNbF 0LanWlLURUeHIS1BY+V4yXw6yBJaCDUpVA37mfLqRQshGtGjmWLtMt/AuSFokwHd yewoORlpVkZnE4Igv1JDggFdEI5lZ4PTmOjEXfntYQKBgQDH6sBr24OMUceNWyvf k03pqUaoiJkivAcUI/krfY7mdSLkiqs+UPuRrikGbvKT3R+iJVbTB4dXzGG6nzBc es7xwvzDGNHHXe0KAFhyIXwNMZmLGTmsNVfnKPAQ1BfKG9MtD5ck2gI1L1DkpaRz X57YONvG05HYmY7TaV2VOKK1iQKBgQDnq/zW6P9WHpIHBjZRN0V4yFl1dMfn2VwZ c3QWBd+kTwVBBhlJlqYeRIt4kmwExPnd3OX8Y7N18RttIc+k4dZgTA4w8G3xzvgk 0sHgf3EBbrkUuS23BJ2IPIb4LmWckH6+KJkvBrlZOfoLBj8uxQwz/wmWA2IoQgKv CvDNr6G5twKBgECWSgZOjAhoX1T+0ITRvUkxJB/MydSb9JmAKb7wOJuh2l0Fo99l IHFnV9+5Nmuo89BZydwxwXsPD7/QMDqgfn1C5pBNU3Damnsxs2FkCgTlMlrrEmPd dAG9ixmUu/7S0H3tXIJOYIo4OCU2kpOnn9TxQafRsHvO2ILatp5ABukpAoGBALgP KJ4GF3bwaswx302/P+6qHoj28yv8wPNnir9Eg14jeeUjV0vj6K77fmOY0UEozeu6 6O4QuC/oEwYtaq9wzcVMJ6oyGueWrAd1eptGJR4iPeF9DhjuDcqDbCgZlJlDI68o yitWiEOfkEzZ9bDO1NcqtQ7+OSoK597yLkb8Vt0ZAoGAN2dHPkTiNFlbzCefv/EP A4xQUAUiwfQ9ZlhMtD9Tlea8cMAD901rxy52YrgCvBPxw3HmKG2H0NOpa7BwrgA8 uODxi6xBRExRhvaqZe1aP1xn4XKw2VVsMlIlJQj2Wmuxeknfm9R1sfRD797c4nuN ntLUOPAWtDkLoJLrTd9EqFk= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem000066400000000000000000000032501477524627100315370ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmfECAK0QAMxHG wp1nPoeOfkDT9dMntn0YPMCGrJY6rdjDy6tyXky3JEXax6jMdLghdWKegYiWVG7b +YwvTJcNziFCL5JXfzQrAkNMIq4UyvyyLNBnDlLgbWGWpjvMT2rW70WcdJIlbAoQ YhsiKxFr0VJN2o3DSuZ0pxse74r0logCDbdXNZ+j/6Istw4nTnkvzwzxkQ6/Adei cSy3Dkt+UJGJccIXqsspgJ7XK/ozQeiC0TqXPWzeZpu06hrrlL5uwGbodz1y1Vyl 6Ks7M/SzwiZJvAhVzxa2EiKR/sFastd34/RHvMR3a/V/w+hImbmo6rGu5sw6EvpN L18PqP2NAgMBAAECggEAHsf4UPou326RydLvsUgRXhofuFDKEpyd8l5BJmVAfWbp HgJJF6Mxwea196ZUokCuTplae33tmAXSXV99OL2LbCUBZzBOeVjud0k60hfTYcrJ /9NjULqIPjBbC7R+d97zHPwuPagb4UlhbvgElkOqO+n+sqBG96WgiE7hJ84YPfJO Y+Is9vRbESkMVK1TH5PxfDE0Yu/i2vm/Fv+Ekgqe+GiZgbDw+L98D7ZX1xb8W2Ix WnM2Skd22pit1ftpixuHbdHcPX2NNdhFKy8r/ZYK0SCLwkb8yJhDLQF8Q01YXd6q FHtuE+MGXsr7dkcqYtc2QvigJdHs72WCjZwcpA+vgQKBgQC9qt3AIXPjPckhTEEK 97tg0zqFVPHyhiy23qsKJ/egMIhYESQngLOPcQ0Q/bG5OJqe5sx31rmKQ368QUSX lIPG9WrRxCh3BTo7nOOEmAh4uGnKtvDJbTRP56fPQhkKlDywua8vKs0moUdcact7 jjXYxXSPGEqHQrjPkuurPJvc7QKBgQDgtd4/kYGM9R2ltSLWm/TZnE6LM1EtBWrA HNAYV7WxxKdUvTtxBIXDKer0RAbDKHIoZ6HI3lon5siuBVtIFoq2VLxS1jm3rEJv qV6USxxDnEkbLla6Jzmd5eqFPZErWfNqmdXP1sqC8fs5q1PUJXNtEIdJulXQnHP2 5lJxq8ovIQKBgQC1HBerg0YZ08HfHeVuB6jRiGH1N2vhXeYMqQtCI2/9ctp+3b9c STUs35LOirHOYBKlcVYFiPCa6mB2ewx4gcRjk61wqJLLNB6rFeDbmCFexRmgDJhY fwLY2igPbNpkk7BwQJ7bt082eAKgaBV54g3g9IucqGFiT4ASFgUb+kAK8QKBgFYJ rJgAWW8kJv7clQNA4YY0j+pCctFfIpl+LrszUhFHr54Fem3ygljQgvKV3VT59oO7 8jkb0b83YR0oVeQLJX9cgGLjPWQzI5jna5wyChdlDqTGoFRUUn4/mwT7JstHfKkT T8dtgUqT5lIVZFp1IHXg/zveiZ7/WHNvip+VXCuhAoGAE3aA/rPBYEHJFLHaSgcR E+ggLP3HjQPN6347NXBZSaWSXqXtdLDlgnnysRnEd+JHSDI3lNuFa6d6nY7x7Mc0 Bn54Tf3KLLHcyUrwQTCjY230212gYGqWXMgeaTPJRtl4K0PchWzKzZ1m9RQAZHOQ OaBsh0IA+LCDTmsPsbzh6U4= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/000077500000000000000000000000001477524627100245365ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem000066400000000000000000000236501477524627100277010ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:40:31 2023 GMT Not After : Apr 28 19:40:31 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a: c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00: 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28: 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5: 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0: 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6: d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1: df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3: ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7: ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da: af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76: b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd: 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4: 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f: 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c: 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4: ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4: 66:6b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: email:UserB1@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8: d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1: 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29: 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77: 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c: eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65: 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c: e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73: bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1: 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f: 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3: fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5: c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86: 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6: 5a:e4:81:74 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA uhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF 7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT fzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl dwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T /LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK S9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ Y2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6 S12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22 PQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN c7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2 gygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko 4iDBH6wH/Kzg9e56xlrkgXQ= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem000066400000000000000000000122211477524627100273550ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:40:31 2023 GMT Not After : Apr 28 19:40:31 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a: c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00: 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28: 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5: 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0: 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6: d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1: df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3: ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7: ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da: af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76: b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd: 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4: 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f: 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c: 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4: ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4: 66:6b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: email:UserB1@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8: d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1: 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29: 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77: 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c: eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65: 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c: e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73: bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1: 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f: 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3: fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5: c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86: 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6: 5a:e4:81:74 -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA uhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF 7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT fzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl dwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T /LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK S9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ Y2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6 S12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22 PQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN c7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2 gygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko 4iDBH6wH/Kzg9e56xlrkgXQ= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_bundle.pem000066400000000000000000000236501477524627100277020ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:40:31 2023 GMT Not After : Apr 28 19:40:31 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea: d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa: e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1: 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0: 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e: 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e: 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9: 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a: 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84: 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05: ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38: e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66: d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c: 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b: 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86: 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6: 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03: a5:5b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: email:UserB2@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f: 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef: 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a: a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7: 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab: 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc: 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4: 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51: 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8: ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0: 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78: 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48: 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42: 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b: 13:5f:cf:4c -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA sh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb QmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/ JB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R qeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ ilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl 6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW ZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL GhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA v3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF UW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr mHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO oFsGb/GFJ4xrIN/gaxNfz0w= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_cert.pem000066400000000000000000000122211477524627100273560ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:40:31 2023 GMT Not After : Apr 28 19:40:31 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea: d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa: e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1: 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0: 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e: 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e: 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9: 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a: 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84: 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05: ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38: e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66: d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c: 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b: 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86: 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6: 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03: a5:5b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, S/MIME X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Client Authentication, E-mail Protection X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: email:UserB2@user.net Signature Algorithm: sha256WithRSAEncryption Signature Value: 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f: 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef: 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a: a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7: 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab: 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc: 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4: 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51: 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8: ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0: 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78: 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48: 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42: 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b: 13:5f:cf:4c -----BEGIN CERTIFICATE----- MIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP MA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA sh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb QmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/ JB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R qeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ ilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl 6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW ZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG A1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs aHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI KwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4 LzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB AH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL GhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA v3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF UW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr mHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO oFsGb/GFJ4xrIN/gaxNfz0w= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/certfile.pem000066400000000000000000000225431477524627100270440ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/private/000077500000000000000000000000001477524627100262105ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem000066400000000000000000000032501477524627100315400ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6GWWrPy7yepPq BuuimsW5IGYudBuUWkMcjCJyAHktIBjjSjWm34pYM3MsKCDn2YXs9YGuRERVZmXW tXhxxNjCe0wtixi2hvxQwH62bvd2wDBsZwlTLYeY1tTYs6mARZN/Mz9BKnDz4d+g hWRLJeSR6ebIw6A+s++XH66dRIQ1JiZODHodx++2Ro2CuLAY+yV3BCCM2q/6nqKw Z7amW9eVpTw+drQ3SkiYNJad0v82avQqzYWz43F0D+Al8QbLnVP8tF3EjXoLvRbu XFghrUk0n54bbfZHUh+gdAD+PE1fTFojStVM/z9CXYXf9jsyxMpL0J1LnoamZES4 riQa9GZrAgMBAAECggEAAVnSLX+MajFnQj3our9FMrZSGx4bbAhQz9dndUWc1HT4 d4AgPFpAfqpof6vVycHx2jSnILhuseJykSGzwoHgynrVpI82T6f9EzhRmkLbK1Y5 6t6jC9uwXDvv37RgYcW02o1avD8VdHtN+qXtO4Db22P1p7zeA6LzSscmmLjf4QcY 15O5DFUsVD6jfjI+edTKY4OgqblwD/t5EqApBI/KhAypSRD/NDzKdtHZO+K3eJW0 apznw5wrzPVX1xk4p+1LnM5nLBRnwECqRyzlmxjX3rJr7tVVWqOkTHs807wK+7AW o9rujmS/J8I86BtZdj938VGVyuyqhJndANF8rOh6nQKBgQD09ZFmj/SMIeJIa2Xj MiK1JMU1rcr2h8NxYhQqZV/sj8TD+Sm/ljCDDClqyo5wAvBdIkFO689sIDEFT1W1 vUOnE8xa4kkoSf4TVADiGAt4aLHiPiRAoX0aPqgBSy9IcXg7p/iG5qFLp72CNEFg 3vM5vgjX+xio42Hqdo6+ruE1pwKBgQDCfK4KpR2BAv6dbuGNF8qZHWkgBDpSSlug WMEZe6c9l44EAIHgJNr4nBviVZTZAHD+H5qSC8STQ6Y4ccOZYnG4dGxAztKYnX9Z T6R+zOkisK+Zhq9noj8veBwS6F2fGTL7cagBkj2q3SveagGtutkV6kOKUw5uu8dI GnSxaiNpnQKBgQDrzURlVWgUST3ZdsECvq1YcIgCj0TUooYKLF67HREE2LSR7dU5 XytdyyRHb6tDuiCFlscFYMwwCqEFuoQISaPJPq62QiQoS2nwUynyezD3fNjXr/gX 2xxhWjVB4Y0nkEssKhp8SaC1AkjUANd6l8PNLti2iDkJwrDsEaqBdjjG+wKBgAVM Eg12K9SMuVSeZYRLRphfBbL6ioAdSFuYr0G7bXWvAA452U+6kUA+OEA05oX2jh1N zQ73RRZhvFBDQPmXhdNpUF1/hJrlh0dudOODP0JTn6TF11cyQxhO5CzbqVkg/ZN9 p/7K9eUGeyBmsL8DnNAM/mPxGS6I7MeY+N6wLmC9AoGBAPL97OOwtkfCqCXBzIua eNFIPvW8cKEM1ggxUPGar36TuaKnDt8bdGL3/ZEAD28XMGCbUwo99WrW0J4r9b95 Rrs1FzUW9iVIqB+4W35lMfSbFOC/2GsSUf95ANT4wihu2QbVQU7iqjXw+w8ZN9Vx Qkiwv6M/K0lzm6Q1H1pb7urx -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB2_keypair.pem000066400000000000000000000032501477524627100315410ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyHZKDvg9AXLg0 k2Yo6tOFHuxm45fQ/qctLInEquD/YqKLGRmKH7upJC+ooRaVp1tCZS8DJxKsRPsv 4JsZUjKn24PQGtY217dADoXGp3Vc0XGpmdPaK3D5np0LqDW8PH8kHrUugzEHyZtK DqMyNr2mLFV5+HFmaiqP+flnsAYh5CoCRLY5hBh6AF40NvRhDRGp4gy4Be1nl7wp 52msSG77eOk7OOPbCcsiD5pXHMwG8fdEZtABxMEUZSnlzxkmc8mKXCslqdHGPthN 9fNnxyO5eyv1lyiJgZmdgkUhJ/TKhgIiLyZLYYrLdvuxe0xCtiXoPsurLGCno4L7 7wVZA6VbAgMBAAECggEACzGbuulEMPd1DwetATtNZTbHBOoMe3vVj0A7dEiIXokG zc2tl10Td26EVEBFvTpI5YiaqwzElYNMTo2M7TjizvTynZyGusPnisl6SoWoh0U5 2HWIAHkSKCAww1RbGL+HbEuO5Wy3R7FMC0C6PuQPP3Bo+swVnqn1s6wf88U/zWml Nthu0uQSj+pxW4tK/p7IoUVBnSqKExODDLG4LpO3meSaZIr36wC6bJZ8w8lZfRBy DkPJu9NNknL6qSoVGozLzgtg1//yCkU+LX0OcDgTNeup5DlA08jglQY8p3Xo3FPn evofoPvDnku4H1gCXT/djERRSlPdcGPEcy7xMQx12QKBgQDqdoL8hkp/DUzoKZyM u2Vud5E1jULal3SmRB1XFzqxEiFsAT6UBH2feVweBOKTjLBqIuC+teQ+JgC5TsYP CGbclQG/XBTYzOPfn3bBJWS4j7Jd68uXDQvkM9+RroFVaCXn75UGWEMqcbtgTNyU wUrAVgfTtz07iHf2oUy+IreW7wKBgQDCegdlOojhn4juC+B5ROJHXzwI1qEznpJa ftI7RERUbDFRIaucwvI6y95nduIRORO1bzpBhHZzJDPNBhZZya9wkaLElXktgi1Z IwF6eb3m/FtOxx7DtI9daCVsuZsoPEw08NJq6UYQqeauaJ3LM5rDSMX0DN3V//2m 7tULbZn4VQKBgQCT4dwMWsdyC3mOlXBgc3IuksvL8yVPqmew1xWKcORb+wuJi99k jNCPXYR0irA+UGaVCxqmLyOe72lVeBIEOVBnoLRRdkrP06uGyJWmjWdR4ZCnHKp0 w43UicNhp6d7rwz5lWtxbQowIzwEKXaXfLMhTSHyr4i3nAPOUz6MTmltkQKBgB6z ePtoFDfaIZnC0jsSvs4ZoLace3JUtDIJF1M34bmaIub188uZkvfpO0EGKYYihpP7 7SxupuxiaLMTJPAjwMh6lUGHf0vJ4zLRLeiR04Llj9yN3rNyi7dpO49AddgSPM2W vwEVtnPm/n3GEjMEAIiXsnhml5azBO4XghZ9xPLJAoGBALctm1sK8MdawZ+cnc1i 4P3VP2/nzGeODF29MbJefYrg0hlKHZSfKsWMKg3Dk9jDUplwsVjK5oBgN1vg/zOV ysTtyn1q/RBbe96lYkPHzdYPWDD5Rg80/t0n6jItTOQr6QCshDLrMB3bruIQz7V9 6PPhzvdQu3v3e07wrKDa1F3t -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate1/000077500000000000000000000000001477524627100257315ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem000066400000000000000000000114271477524627100322110ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate1/private/000077500000000000000000000000001477524627100274035ustar00rootroot00000000000000intermediate1_keypair.pem000066400000000000000000000032441477524627100343110ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate1/private-----BEGIN PRIVATE KEY----- MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQC8xoQtwqtdBddl qOIVdNjy8VURRZOWTKXcy0T19BR+RgJZ6K54WWkhWPcWOLnCwmDYdquhOboLowMX 5KHLXRoMYnEkZLAA8G9MrwhijNxP4NfUVSzbNvypqtdYJ+SZy9wp2eo1FssuvgSy glj05VwH2xKO4zyaXpBLxaPUIZZf4Y/3y57b4BCgbKIeMBdsMp97Q6Sf02szGxjN pK0zSKOYsCvIInQXcdjxZCFV4TO8f3Rfpaaim1gv2+3HweU2LoYmrcb+uACFbnzt /UrGoNmyP069+ghSyF0xE4a9P+x62DoV4nGv7ACIfqbo4Z2rV1qKH/jiTSlYU3kl 8J7ZGEAnAgMBAAECgf9Now4nMXr/fdU8+hNvCMPnuMbV5ewWCN2bzEa04K1D09BI Tmm78MCVGwGoRoeNJBr5fdTPMMoJ/yVrG+W34iSvzqgnT4rJ/KqlA6CTwsiPyFay RgxRQHCpVuLwp8ClyQ0wu26XQlrgJ480trAoUQdj6pC3V+ICdk90R/j0RW5JtsSu e0ML3jNA9C4OgKlt2ia/MLqriaHXOf30EPONvtyqyKeGUFL7Un4eYKh4euRFEEMb MKngNonefDCIdYA1wVFa3wT8bNBbpuHl3ghkokv6VpdHIVn9wC1l6HY5nPRjgmo7 sguRI1bRa2TFkOIVwZjCJTyfANyQw14pRS6rxIkCgYEAwzSYHRpJlPHAD7wi3tls bw7cBF9Q1P9PYKmVD9fAjx6eOjzDVOCdpGDijEkYoQoX1yYK3JaS8Vvp8V1wZ5Uh HTTr6Y5uS6CPh37wGTJc9XhXdJpeN67fEOBZGU04FUlASVFeCiV3Ga6YX0HQ/yKd VSc2JMX9mzxZjwhKRHmCEr0CgYEA95FFAxPxPNzYU3yHIdBlQWB1Z6AUFn+D4FgF xeFOGmul1E+0PnPH78IlYanMjhhJ1nkc6X71rdX4ylonB/x6ldUpghWW9VWrqRGG 76S010aaZgOinwVE7+eeoelsIuma2W0QDwWrUT+RAsJBvZpGx1keo1qZEAaocs9V R2lvHrMCgYBNMTMl7wtB9wd4MXGoploW4M1ofTi9wehl1Sm5BhyDfBwd84FawygT pKxxxUYUCKW80rJg4Lpi73HnnIeirnpVzmOsDELZbTjU4AGaNSxFdb0/wvuXEXPs fIs/UiXnZPwjAiYp5P7gDQb8RE6dVdbZoZPrns/W31qbETAtO8+QEQKBgQDgA710 yYjSz+uXr+j/OflFrSjPedRzfzMvv7aJlhP8aEgH049/q3jRhNYah3Enaub1gWYe Ctn4UNPtFqKW4WlzRw1mPm741Gqec9Or6VgSLDrt8IAocLYud2HdlMBa3xNVhxCu 5yxcOq7W1jxyerVtEUFeA07ZZ4zpRp8eHVOFbQKBgGJGU7xoJWO9P17SUGNfmSEF 6VIYFX6orA1Fi/kAJiqiFf98T4jnUWnL8LXVckt9FNw6KQqBCB6JuKXBFVkG2Bkr f5IIhziTuDVpdLQSf0Z2i59TspgYjiKs4WEN3N0HGtCXfbyPO6Tt08d4icxL5Myt W84T6Uof3+QQaqQnGvBE -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate2/000077500000000000000000000000001477524627100257325ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem000066400000000000000000000114271477524627100322130ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate2/private/000077500000000000000000000000001477524627100274045ustar00rootroot00000000000000intermediate2_keypair.pem000066400000000000000000000032501477524627100343100ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/intermediate2/private-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDaX/8d940anprz K2iPwQwzBkEAyT7kGuHgcGr1L63f8+mZ7cXXqpMTN/9HqvPFife3rTpH5ZxOn4zi Qe2kfJ2IMq71ioSfDBigs/6O3CqIavUvnIaS+ntus1p4Z1MLIWwNbIAaDh7uBsTS 5yTG5XS+Hi4XVSvlnwugWMz+v1M399yViPR3plm0uHyiS7dqZ6qE3Cnx+deJBU0L 84stUplX7W8Rnq8oo2FEwuxuf589C9z3GW0UiqW4tikCNJC0lsHLp0JGl8+NWf0X saYnp3uKR2/6AyQcEiXuNNZc2kWYIzDhSMma3zeqG3Bssg+VOdZtPiUgqAcsSFcM mVLLiQhBAgMBAAECggEABFew29ByrKMXSzsjgOpSmwgmjkSyPLiBIeSyZ85LK58u 18oH62Y/tvvf1nXCk7zO4YbvGANrpI+dLlmnx2PYAR+a5ZSb1wrXSYjSyNX9fYl8 9zWqYm1bO4QTCj5pwximzKyJ7pq1yD93tgb1LwRcmjRA7+NYdGBBi66AYxd8aOo6 QB7JoME+hzYAWB+foCOAPGAxYe7EFCPkPEyz08oxRCvDua0xa0+tWkU77MhUSCu+ /uSq/Og9C9TfzCX0W91TNDnq8VeXbLDJoPNzgfSWIeYxSw/X5dUkYU8N2LuPLQOO 84Xv5UqU9YV22TEjg22YAL8/GMZ160K1xzXnQb1LPQKBgQDs/jOBp9NFiFlcNbJ8 MKdfv+sktQR7onGehOaz/dEFEKiHNO8UmSlkAk+aZoTmYXKgIT4uytKRSOfZUWSl kY64sKJ7KTvVq/Dzm4KsyH8VgYYQ3OrNbqSCSK7DiOiKJxQ+Jhm2+a+io16B8ZbM RXLoaQ5+8oET6BgM5R6IMe4iFQKBgQDr44q7I7nhQdNz4h7OJ4HZ8X8WmQG7YpSX EMLb5sX5wymfPi5uUDTcthUw7dpleO9js5iT93fB6+rd5yyiDPIes/dWjqULprvR zIIr0u+cyt1TRxrNSa6dz/dJO3t/g/fTPKeM9j7ON4RvEGW4LPA+PbEUU0Q6xfSq OZ0sZSXUfQKBgQDh8+r/rxbLsJgiRkAKEAlETSLQOJYxmkthq6yZ52ElxyAm6N0Z cn34EAv9VclYLYiwC4HR8yaXxj7m/6dKBGFizWXcrw+RRQHSAW6xdedUhc1gvoBP pTHL1ahqXVn4fhHav1C9F4nRMpmkosX3tC8+Twu3FVbjt+FWSgy2JYS5kQKBgD5B 6u6jaj7Skc2HA5xjfvkXrPQ44+UiCpeoW9WQHfZilQyra7O/xYPvJr6oODkJ5xzI XN/Is7nh2zY/+l62zfxegUw+D794fR/NOxn37TfTrwB4xtEhvk12gwy3/0tTeEgv PQWORFtG+dQaXs5yReIXhDIaG+rrLjzzQdFizM49AoGBAOulUGVDBpUFDVJn8S5r bqss/PXW+5xj5g8b9/tzBuyfL0NJ9p3q6EWlELPRTX3zXuVRYjSe9cBUm5FXPxP2 s1TsGUILjSw21dOtodahvXRDN3Uw2ALQy1MTDy8xLhr9Le+e6xF1T2muzg0vDT6L VXAYfY5NPUOiPaYAj792oZk/ -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/misc/000077500000000000000000000000001477524627100241315ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem000066400000000000000000000237231477524627100324210ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:33:37 2023 GMT Not After : Apr 28 19:33:37 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: 82:5f Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: 96:a5:e5:d9 -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ 57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem000066400000000000000000000341721477524627100307630ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem000066400000000000000000000341721477524627100307640ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem000066400000000000000000000341721477524627100307650ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp1/000077500000000000000000000000001477524627100242235ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem000066400000000000000000000232301477524627100273040ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:28:39 2023 GMT Not After : Apr 28 19:28:39 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9: 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1: 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d: 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79: 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45: f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30: 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17: f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94: f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91: 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6: 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6: 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef: 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4: a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56: fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08: 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3: 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d: 38:21 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13 X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE X509v3 Key Usage: critical Digital Signature X509v3 Extended Key Usage: critical OCSP Signing X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32: 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d: bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3: 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60: c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b: 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24: 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3: a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb: ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc: 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b: 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f: e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21: 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9: 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e: cf:28:5c:69 -----BEGIN CERTIFICATE----- MIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX MBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw ggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp PEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj MCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8 Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd /l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux Vt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte UGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY Go6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG CCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4 ODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF BQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB AEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf c4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S 9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ u++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3 adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9 nOaFOvjJTDP96aqIjs8oXGk= -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_cert.pem000066400000000000000000000116011477524627100267670ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:28:39 2023 GMT Not After : Apr 28 19:28:39 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9: 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1: 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d: 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79: 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45: f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30: 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17: f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94: f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91: 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6: 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6: 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef: 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4: a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56: fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08: 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3: 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d: 38:21 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13 X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE X509v3 Key Usage: critical Digital Signature X509v3 Extended Key Usage: critical OCSP Signing X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32: 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d: bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3: 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60: c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b: 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24: 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3: a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb: ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc: 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b: 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f: e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21: 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9: 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e: cf:28:5c:69 -----BEGIN CERTIFICATE----- MIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX MBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw ggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp PEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj MCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8 Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd /l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux Vt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte UGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY Go6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG CCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4 ODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF BQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB AEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf c4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S 9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ u++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3 adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9 nOaFOvjJTDP96aqIjs8oXGk= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/000077500000000000000000000000001477524627100256755ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem000066400000000000000000000032501477524627100311510ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjDMrrgOuhDh5x m9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEw eWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX 7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lW bvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+C T7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvz a9qqHTghAgMBAAECggEAE++sPxPuG6zzhX4hakvYCiAo6GtQAGBi6CjetTsmRwti DnKoyCMeTUQwXZ+4X5SvP35f1urSPAozSIdMR3qoSqSsqjQy+G8DIyWyHejmgBwe uhxYcRbC7Ct29k8m9ykb7bO1WtqDZf/hYkvbXbKFFXKM2/IuOcPnuZ8xe+z7IPsQ ODHnrQs45wQyi2i2/+AbvEJjb3bb3oS8MfoZfvO8F06ejTOmv/ATZSxX0T6ppCPj HdmKqKDXlYQNA/LQeM4cs2FaQH170R1vGHppDjcs2ezqElB7/HKfKWeEn0Eytu9E eWw9tZteisnzfqEvDMgOM2eWwAzfIhXSQYMWlVBicQKBgQC6MPaLd4r82BBMj7qx ChdBxB7LXptvx/q3SrMjZ6GKmrGdXMbsos50XexajktBqkXfUMa8hGqmlciN5xL1 +w//p7oSzb3VorOyHVXZpc8p79eUeX8ONcwySOYwO+CpqFBBDlvPn1OuPnlUL1pv IgCMT66flWJxRklDMIJsHr+iWQKBgQDgLq3I2cj4q+3121ECPXKLt+VCHUY0aygc tl6lvQw61UnmyLQ+k53/MmyPGGCxIFr18DsoKeWYwt3kWTW0MCDrQuO6PZkB268v gdsmN3nhAKiR0gUwJDrFjpPWr0GAhw9LE7HqpvkQ3fG5YSnXTUibhm6smHg7dzVL ER+QJ+Y7CQKBgHIDN4WRjy9jEx/+x0BPwIwKDx1TcnURjQoeGPHuLHJWZbrJrBoN W8TQGsIc7iJopN6pdPjNUQ1vHN8gB3FO6q4PRBbtm3gtaEICSqa7LM8uSeFmQJIw CTklgKc6k0jwgyxDIZ9SnghNwzf0wzjYJmPFC1Y3QI/CjWwyUTrp3UkJAoGBANHc IKcS6MWQ/RPYGP+F0kLlBWJc0Smk3knylquES3yPybyXSdQCkDcjVuilo25soXn1 RwuUHPBiCyIGOPXS0B4r4c6odyF8K4THhQVDjX6KBUNsXZrxb2scy1x/d0wAItrf NwA5CpM1kWE+idKY8E1XDSfZG0Rfla4N+4QRNb8xAoGAQrVe80TpPpzDH846xaPF BAhjKz7dRrUQ1n7ZI6yw8I5bU2ky9jSF3PRsDismM23aNbYhIgjcYv0tG0LMlWLV 2eIrU9OoA7aepDPozyhEkENaWqYXX/T8AjD+Kaw7XJnt/NX8eS0RF2qgDA/HEwWw uf1ecRqpjZ9cxNGLZ+/pOkM= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp2/000077500000000000000000000000001477524627100242245ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem000066400000000000000000000232361477524627100273140ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:29:28 2023 GMT Not After : Apr 28 19:29:28 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2: 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33: 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9: 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95: ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44: 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5: c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2: c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0: 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e: 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5: 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7: d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0: af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5: 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a: 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd: 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69: be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be: 7e:89 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE X509v3 Key Usage: critical Digital Signature X509v3 Extended Key Usage: critical OCSP Signing X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33: 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76: fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd: c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05: 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba: 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78: e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d: 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c: 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5: a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad: 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09: f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52: 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf: ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77: 4d:ef:b5:43 -----BEGIN CERTIFICATE----- MIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ MBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP ADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN 06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B f4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO bEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3 zlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l 56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU 5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi V3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww CgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy ODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG AQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC AQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN /ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7 lpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv sjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0 8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8 snFsePS65Lm3HxFl1E93Te+1Qw== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_cert.pem000066400000000000000000000116071477524627100267770ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: May 1 19:29:28 2023 GMT Not After : Apr 28 19:29:28 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2: 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33: 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9: 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95: ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44: 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5: c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2: c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0: 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e: 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5: 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7: d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0: af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5: 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a: 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd: 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69: be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be: 7e:89 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15 X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE X509v3 Key Usage: critical Digital Signature X509v3 Extended Key Usage: critical OCSP Signing X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33: 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76: fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd: c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05: 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba: 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78: e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d: 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c: 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5: a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad: 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09: f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52: 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf: ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77: 4d:ef:b5:43 -----BEGIN CERTIFICATE----- MIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ MBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP ADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN 06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B f4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO bEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3 zlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l 56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU 5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi V3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww CgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy ODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG AQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC AQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN /ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7 lpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv sjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0 8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8 snFsePS65Lm3HxFl1E93Te+1Qw== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/000077500000000000000000000000001477524627100256765ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem000066400000000000000000000032501477524627100311530ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4mD0DTV6yZl5R O/k98noka3BcL3oFslF3YkXnM3V32zFvLRMyzdOgA4Tu+SuBneXJuuIlyacYK/3x la3TRpDZe385LYW0cHxyRJn7358iTIF3Nbv+QX+G9ccpU3zu1MwJVPrMsU1LwsfH PhoTWWY2Ma5gG2oFsFtklneddMxCbhPRIYOUjmxM2EJXlBf/JtTRL2RYtUcaIjhp v8BanMOIAQod99hoiHxXXUTEcdBmjRw54K/o985RYHwdt9XntT5qpStGw0657969 pr7iZnmpag3BsudeA53e3UG5yYAsvW0fCV9OJees/yNHj190ab6BQlzmGvdlH+uh 0Glvvn6JAgMBAAECggEAIlCyruV4ICPljqZefASSbijG12w/+8UdXdsX8ZXgVWqa 8vbnJb+bgpiE4sPRMaQ/rlOebLXi6RxsdbeEe80XakaJ7QAoZdWvXLKiCW+VrpOY UafcjbRxV45i+qy5gdBvKaDxipG/M8E+0CwcPtKUrKhpqRYPjIUvSDCshcnLmuF3 zztB/4VyVEUUaM0pEqSZhxSyraRmGARvF1iOSu1npe3AzWTrrjrSkbk6fi4GyECL If0EQ1ZD+ZXQ6tcGDyNtmPox7lPMZOgwLJZ5zISXZ6QBjn0JvSzE+e4z0IFinLgx q5yBz2BhJEN8OBcs3J2N/ivQetWil64YbrbK6WbocQKBgQD/b4uHOuJVVifjIf6/ kJ0UHhki4Q2Fj164royDigyyzaZmMzrlReZ5rAQLk8wGqw2hI+9gYoYBYqHm71kd WrwLS1TVZJ6x8TBh0sYOG2CPndqIjWFx9Wjjf1xNknwYdIoEdAAKZ/M1E71V0tZb +Ampl+lHPnKqYRSCd7gbYBU/TQKBgQC5AKGJusjRRRRWQqQ0RdJuxusZrtAAUd7l wOGMC0zVQSSvUuegFtWEaZUbByhCARtYp8o4rT6Fw9yOvMaMNcfd8tV5nYVHDsrw MurPhPitgI0/LdVvkAOO4fgPZHIXV9GbUDGq4uqB61daBSLQg1JjtzG8GvlGiYZl mKOWEXjWLQKBgQC3nHHaehxZpT20yin5f7U50czVwppaqE05Sdcdcq1gFe2Hx0mN pypdyaV6wPnGzUxVyaP3T7rt4f1pKCGRtTg4kiTf450jYbEakEzntQw7EAgXYjFq njKQXWt3I1XqqlLPkqa41DIBtDfEKnMF1wzzCIyaNqxsBq6cffwsSWvcfQKBgF/y UNUCd0X5Yqu+EjU+BP4I0kNWo2+XBlf36cHc1nM/Psxi3dfsH751l6wV0S4yLsGS +9DbILL1On0YsIxlFAwq9cYGCOoqZNugPKF1oBcztY2PssMSWJYQ4brx6C3tELtR IwEygFby/DGmukCT6vXmO7gH8UJA7t/gAu9Ajn/dAoGAI/Ejqb7HborIvCw/p+kB JkPIhTUuT5XonDm8h6KHWUESPikS7SMeRM/4V+AL/Y5MiiCBfjh3tCOup/16x6GQ 4z6FvcIaYusxKup+afQaDyv1Phv5/mr74liLhC5Qp9EGU2FZrMZwG3EZjSn/0IE+ dBJeWNtNHiFPcyTzYMMhDBw= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/root/000077500000000000000000000000001477524627100241615ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/root/private/000077500000000000000000000000001477524627100256335ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/root/private/root_keypair.pem000066400000000000000000000032541477524627100310510ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDiIWuf70i53iL7 WzcJaMe1kldSJO+FAOhxhU0PW4zG508Z9uMLcKNBfnHUD9b98hrKqleRdpqygmJg zvIALtS8WNNgMEKmKLJQe1gBn/sKZbBA1nzit9qNGdmlUdJGfhRGq/rfzv6ECJhj Rh1NindXZ9oWizIMfEHipezufSAo6wNf9eYF2IuWeG+uKZpQ99yWMYaBsXjo6+9d u+1C7JTGVEbsBW8bDDYkxqgGflxWuEM7EfQGCgUVGTsfyGcx6ztbKhUKe/lr5BDu RL4Z2NtEAfo6VvVsTvNgquTNsq13B0Xv8df1+lKEXANOcuCpkcXZ1gqEM5gx8gJb PxAVZXbXAgMBAAECggEAW6YC6i+/cIFs+SW3cStT4a29kU/h+axsCPJnUIWg0U6X WyUaUR0mNZmrRbDjyEmS/Te7xPtmaFn6yFSndVaFpw5zIQV+RbyxxHexK/tscgLT w/uKYxLz04M6GExIpoRb8Gash3/r3JRlOrsEjlRD2RuAoulob/H+e/8Wv3PcEGio R8jwCj5DEnWiMxDzgtxsVgR4OeRYqg3zKjWrLALEYoRbFTVncCVA40OnmGJZ3+E5 +3OOX6p9y/nY36888345yuwiCOTdNwQVaCXnLDZlAIVpB8QmjXVB35RSs+r2H5SF p/KRbZ/JNKdNrbTKfJyvbnIpyTAtJB9OkhyiR9AegQKBgQDkKAplyZ6ChT3l53nn 4ngFi/nSTfrfJepmA5lVJk1Wxk0a4W++HxJkdKY2sUP7WuQ1xaPdcHxKzfp2HQE5 L95jObU5dtY64QD4q0xqOw1ISDQi1euqZEmZziupEgPcMtw4sAVhHohzvTWo6a8o fGMSkLTd+2303xgBCZo2I/hZVwKBgQD9uha6pQmCg4Oi2i/38Vm3ByArnQvgkELC eGBBJrCE8pSm+ToPtgL+gSuA8PlWzIYlIf0UVOJFyws8GkUFm6n0nUlN0NmK8Rhm Bg4IvasxdRgtySJzZO7ipAqGIaWJIBi1Vj4/rnAVggkadbQgyw+eCZNc5Pg3D9MV TJ7d/xHegQKBgQCprGVfITuyUSihKy3rlu4vIdPd5IQnI2lYCGElg+CMIdkBnpmd SDpDXsSlc9rcuNFyc9LTQW4Nq3USFavtPX4jSK1PWOMk0mQIiku/zL6p/JhZN8GU 7BQYP80UZQNd5K0Fs1Gs0ioj+JhJT9AlSavcCKWZV/yD2M1fKCb5EHMG7QKBgQDV SvtSeeytp8sgOtU6VMz7fOUBZOsYI43Ll5ArFNAtYxOt7jNuA68urf2ZTnn9Cr/2 NUVgMx9oVpEiPF8roLlV5mc6IEjQcW72TT69AF0KnYnu63enlADxy78BFQXoaW/7 +P0pYYXdvsvST4JWUv3U9+3GmMFE4GutKxUeQA+QgQKBgQCauejVixhfKcmkM9nn MGLSOUuFyd9HpQk3efxylphFNjpohk+k3fVKXBhmE4BDXbSlYUmMemm27tuQ/I6Z bWOjGl57ZbCgJ7LdXLanJhyJJ6cSmkX8+oD+fwPMrD8yaAfh37MdTnriZKIDMXp2 7HtfLcz0evmbW06b/dReyvcqyQ== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/root/root_cert.pem000066400000000000000000000111141477524627100266620ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 18:57:57 2023 GMT Not After : Apr 28 18:57:57 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68: c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b: 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f: d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce: f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b: 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19: d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08: 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c: 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8: 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1: 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05: 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11: f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a: 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01: fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07: 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91: c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65: 76:d7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Signature Algorithm: sha256WithRSAEncryption Signature Value: 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5: ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60: d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59: 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed: 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a: f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d: 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7: 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28: 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58: af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f: 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95: 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95: 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82: d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8: dc:29:ac:17 -----BEGIN CERTIFICATE----- MIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4 NTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS b290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i +1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi YM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY Y0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv XbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ 7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC Wz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220 4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD AQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3 LjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5 Xfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr UFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I 4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y VPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb hpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp MElNbuoFHtjcKawX -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/000077500000000000000000000000001477524627100245655ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem000066400000000000000000000237231477524627100310170ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:33:37 2023 GMT Not After : Apr 28 19:33:37 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: 82:5f Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: 96:a5:e5:d9 -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ 57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem000066400000000000000000000122741477524627100305020ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:33:37 2023 GMT Not After : Apr 28 19:33:37 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51: 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c: e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5: 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2: 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1: 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5: 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2: 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13: 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25: c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd: 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07: 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8: e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26: b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1: 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5: 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5: fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76: 82:5f Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6: ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec: dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf: 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d: f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df: 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72: 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50: 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3: b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce: 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06: b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa: d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40: 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9: cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c: 96:a5:e5:d9 -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT pV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga OURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS ZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB NRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd A29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm A683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ 57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf hnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi DTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N vk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ y/ax19PH5nUVAP/H2VRjJxlclqXl2Q== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem000066400000000000000000000237231477524627100310200ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:33:53 2023 GMT Not After : Apr 28 19:33:53 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e: bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb: bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9: ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66: 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6: a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4: 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31: 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff: 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5: 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3: 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e: 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18: 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55: 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86: 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f: 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51: 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4: b1:e9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6 X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4: 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa: ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b: 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5: 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41: 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4: 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1: d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95: df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f: b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85: aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08: 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6: f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31: 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49: fc:f5:61:1c -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3 qeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4 71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq P5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS hQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ OaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5 XEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu Mp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB OhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P tlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z tSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx eTjjUIregZeAnkZxXJ/l3gxJ/PVhHA== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:15 2023 GMT Not After : Apr 28 19:01:15 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74: d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4: 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38: b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4: a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af: 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa: d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be: 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e: 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0: 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3: 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22: 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5: a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad: c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f: 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8: 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57: 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18: 40:27 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b: a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e: d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28: 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3: 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74: c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33: 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e: 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce: 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71: be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce: 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba: 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4: ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d: 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9: 91:5c:43:a9 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC YNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL 3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy n3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu hiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX Woof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2 +bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp LtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr uyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN zvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg zoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX KSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem000066400000000000000000000122741477524627100305030ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1 Validity Not Before: May 1 19:33:53 2023 GMT Not After : Apr 28 19:33:53 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e: bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb: bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9: ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66: 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6: a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4: 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31: 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff: 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5: 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3: 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e: 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18: 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55: 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86: 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f: 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51: 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4: b1:e9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6 X509v3 Authority Key Identifier: B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:18888/intermediate1_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:18888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4: 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa: ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b: 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5: 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41: 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4: 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1: d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95: df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f: b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85: aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08: 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6: f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31: 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49: fc:f5:61:1c -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe Fw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3 qeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4 71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq P5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS hQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ OaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5 XEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga jp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 MTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu Mp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB OhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P tlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z tSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx eTjjUIregZeAnkZxXJ/l3gxJ/PVhHA== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/private/000077500000000000000000000000001477524627100262375ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem000066400000000000000000000032501477524627100326550ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvJlxQwPpitf09 wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRTpV0v0nHYr+MmlbQnFEZ/ 4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5TgaOURy1Rkgh/6QTzuX8n29 V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiSZFgEzKOdz9TSvSCri51V 3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFBNRkJwaDcJrnIZvqHZyJu DKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6QdA29Npfs7wAtzp6uPtBAl jmlCdoJfAgMBAAECggEAQLRoOEECfwMCehUUKs20XAl41WQ/7QiQvm4+GXwQgjyV hkccCGkI7H5TJK+bfHY/LrDTtsZpVmKMJORJvfcvFkBg08lakVFmWWy3L1pFjlcy DoWGxJzgYVPf5PgxDEcjUDxNU9yhhGHGB/Pa5oZwg7Iqw9kJ2XixPBx5RpjxkXYw tR8V3IaKq0YRI5lpUfuaofmJnHJnWCMTmawWMxWuTlzlbDDZTHQs8aTDUnwZ26kD 6tYB2Tp3aP3zUE8MQZwOEyhRH1WQeS3kcIWh4UnPyA09g0aTb6YK8qacnTL2CixF VJpLDtlkQk0TCo06AZkcvWkPTQyFXnVsgkG8rRUlEQKBgQDrTHyf6merJAohUeBV 5IIfoKHWbGc1DXSdmHtCSN9wFGkhCYtfCZ7YaSLjFF7GOvd6mfHJVnIp3aFONqM7 dk/MZDsAvogO6lU+zgQc+EcKk+e6zyfsUYghy/R3+QKsYtd4SyNDq6cl80MUujjG pE2b41O57sNCVZgywCCGXvt/ZwKBgQC+jyufgKRIptM+OOhHlKUaxkTDaMHA1KKY iFPLuLgWmyCYHQq2D6uoCRGnEguEnXtbtOz6SYlMMNfeHtX0SATkdCGae/bh5ibG uQoWwRMkRkAgl1gyAh7h669pDUiD2gh0q56cS8El7Jgze7NRF4hUyY2mWc5nGhVR 7rHKlOCiSQKBgHBiWevvg5BkaEo91w5vVA9TI7lMkYbvZFGZcNXaBI590TCsZFsC N1JZ9QXMxu+bXnS6bpehqGmCp/a5dgGCot6WyO+0ETw+hHS45ZIIq7XLqxS4uPLQ hlrOFXfwAWzg0NVt3ewGYpFnvRR7VX7bHw5j56uY9L4ML+OdjGthlnHlAoGAZAm7 R/f7xtw1h7POVU22w3CUxtUm6jl2xobDHu7xTYTQvqp4Zg2h+wwPxVqWy171VLaN tfOG7YWyvbwIbD6mutwwi+5KNFtjve2EW1+u0dtDbRimx1IPrmDRbF/50qZSzBUQ plKqqmMjn9tvzsGA46oP/+WjksLBsIqTsZsotmkCgYAn8Ap+e6ZNX2uM8Kg7LB+T hBNGczNOGQX8SpfCeH9eV4VzfpEHn8Fxk+lcI2WpYkandQ8ju2s0mT5OoQ2VjxGT eql9jMd8MQZTx/aWridt5qG3hsFcx9GILlcXTUqyRH0SFAU7xDO5HzzKP3tiW6BN YE3GakolPPymOR9q69sT0Q== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem000066400000000000000000000032501477524627100326560ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsSM6nsq16aAFV P4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3qeyK3K8nBQz1wBmxLA1W Znt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW471JyEQKBYfI2YyUYMX/H kYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbqP5+r++eXPHLqs4OrSYis qUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJShQLgY/SgVYDJZousT4s2 9FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQOaL1UQaSPK3MWZ0KgVAm MAHp9LHpAgMBAAECggEABbGGbngQX9Un+x8FQesSPgHnGM92wtY6J5Gwn2qhJy6M VYwwFZ3Nz5pBPbrOY9SRGhPihrdixKOWgWppA8ne0WB4JC26HnGZnFAbAQRVqPbQ duhd4ILpOpzpkh1K6b+vvU0addXpsUlHJjYZmdy+9tPBkhtwz1xDCFGShrguR0Pa WTudsee4skdGfw6wMyHEfM4IXXuSfb1hIse1xlnZMPXMMi3ebCqpOy4IzJ4ML7sF RySdrdAHcWJqOQjPkDTOPCXpthBn3iQ8Fa7Znd0GGLZvdRbq3p10H5LNhMg+LBc7 oRVQ67qAfQKPHKQMSsR4x2fWo8/hw/QEi3cj6CohYQKBgQDtjDBm7VfbLojZeCMx +32EZ0bLUTob5qInTKpDbdKcYmxP857LRAglaGu+pkOTnHi6lOjJYSiBDd1+vWm/ 1lgMUjKerI0l5ol5yRHWNDFyeQoh10TqEUbIUqB8E5Vi4gl0DlpnsfEm899rlfhP dmi1rNpc/C7ZK8Zpt7l4eLbqYQKBgQC5qs+K01WwjtrjoqkEwKqjy7ASrbBuZ56u wOe+iO7pYVP4/VdAvOsfEYCWfjhoETYGKob9ZZlo3StpQ5Ku5CigpWQVSCvJhO2T KQe75DfXXxaqoPmlNcqAFpqY383Sm+1r3a815sg83XhQAu7GdCyTrLocBLM9SFWX fVbojv/EiQKBgBlOpCFzC7cYIBA7ElTS3C5s6kfi4XPzgDb7nfANFTD/81YZOEOj fdKuazwmbnCdbOdD0gESTsRg+8Xy2/9KEJtPboElFOyCwQauey385X+ykXfFfVwK dyYEV4CgfXvJZQRuOwdtF6n0tUq68XdVwBYK0kCxxTPxy/ObVTEWezZBAoGAPPX2 evB0vCnLeN5ZfHP+ExW31AovWbCwC1TPQmIXf40zUxdrZJgi4uqOO9tpjdHI2TFx bRXEzwd/T2qeaMoFBOoI+Gvf5KS+lIjuPyTpqM9R0+hSz4nf2TqSvAsPu0zzIW2C L8J8kG9vJ2YvG/3c/QfDe5uXdlGfuMOwm18IX3ECgYAelsVWNSm9YR2H7S6dlgjm 8U1IZO342Ab5qlrIWn9Vr/X9MRsAASlVVESeNeQTToBoub5pakUrpz9nNQy29+TX xYju72RsCjKywKXWZrCAdHfY+wJJWVo5XkdDZJVl2AYrnP3C07S9aKIjhpGHwz7n jbbCEkHZREMbQJCQjuKT1w== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/000077500000000000000000000000001477524627100245665ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem000066400000000000000000000237231477524627100310220ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: Aug 2 22:15:27 2023 GMT Not After : Jul 30 22:15:27 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76: ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b: c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2: dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a: 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab: 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f: 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78: 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed: 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2: 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35: 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca: 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c: 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e: 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d: 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71: 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b: f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7: 0b:73 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62: 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43: 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c: 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51: 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7: 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9: 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00: ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae: 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5: 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a: 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16: cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a: 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6: df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74: 30:be:fb:f7 -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y 8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508 umx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz TBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3 rfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC 26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM nvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv 9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3 kP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ VKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E Lv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG 3xz8gjuuH/clyN/AxdGNUZR0ML779w== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem000066400000000000000000000122741477524627100305050ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: Aug 2 22:15:27 2023 GMT Not After : Jul 30 22:15:27 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76: ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b: c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2: dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a: 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab: 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f: 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78: 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed: 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2: 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35: 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca: 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c: 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e: 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d: 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71: 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b: f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7: 0b:73 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62: 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43: 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c: 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51: 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7: 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9: 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00: ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae: 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5: 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a: 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16: cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a: 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6: df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74: 30:be:fb:f7 -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y 8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508 umx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz TBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3 rfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC 26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM nvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv 9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3 kP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ VKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E Lv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG 3xz8gjuuH/clyN/AxdGNUZR0ML779w== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_bundle.pem000066400000000000000000000237231477524627100310230ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: Aug 2 22:15:38 2023 GMT Not After : Jul 30 22:15:38 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af: 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3: 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52: 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80: 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53: 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48: 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f: dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35: be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82: 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc: 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54: cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f: 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d: 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42: 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27: 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5: 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a: 5a:71 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6: 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95: 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84: 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d: 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b: 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d: 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e: 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25: 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d: 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8: 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3: 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a: 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f: 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10: 7d:2e:97:de -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK UohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr VbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo aJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir zOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1 +YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh QuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO s7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr Hi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/ HkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS VKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP bPF0Y4sRt/Npt9bgcrId4RUQfS6X3g== -----END CERTIFICATE----- Certificate: Data: Version: 3 (0x2) Serial Number: 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA Validity Not Before: May 1 19:01:43 2023 GMT Not After : Apr 28 19:01:43 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1: 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f: ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa: f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41: ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3: fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3: 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06: c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f: 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6: 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9: d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e: af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19: 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7: 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47: 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23: 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39: d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89: 08:41 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Authority Key Identifier: C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1 X509v3 Basic Constraints: critical CA:TRUE, pathlen:0 X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:8888/root_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:8888/ Signature Algorithm: sha256WithRSAEncryption Signature Value: 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3: dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01: 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63: 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a: 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5: 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9: 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd: ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16: 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c: e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d: 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5: 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2: 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d: 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86: f1:5e:60:55 -----BEGIN CERTIFICATE----- MIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5 MDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB MQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ bnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB ANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ 97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh bA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn qoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW wcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5 1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A PckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag JIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB AQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI hvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08 ATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG m+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2 J3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc jRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz EOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_cert.pem000066400000000000000000000122741477524627100305060ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2 Validity Not Before: Aug 2 22:15:38 2023 GMT Not After : Jul 30 22:15:38 2033 GMT Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4 Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af: 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3: 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52: 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80: 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53: 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48: 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f: dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35: be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82: 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc: 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54: cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f: 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d: 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42: 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27: 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5: 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a: 5a:71 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE X509v3 Authority Key Identifier: 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC X509v3 Basic Constraints: critical CA:FALSE Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: critical Digital Signature, Non Repudiation, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, TLS Web Client Authentication X509v3 CRL Distribution Points: Full Name: URI:http://127.0.0.1:28888/intermediate2_crl.der Authority Information Access: OCSP - URI:http://127.0.0.1:28888/ X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption Signature Value: 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6: 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95: 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84: 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d: 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b: 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d: 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e: 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25: 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d: 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8: 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3: 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a: 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f: 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10: 7d:2e:97:de -----BEGIN CERTIFICATE----- MIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL BQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx ETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe Fw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU MBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK UohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr VbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo aJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir zOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1 +YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh QuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft FawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD AgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg MKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl cjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6 Mjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF AAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO s7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr Hi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/ HkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS VKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP bPF0Y4sRt/Npt9bgcrId4RUQfS6X3g== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/private/000077500000000000000000000000001477524627100262405ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem000066400000000000000000000032501477524627100326600ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaPNt2yRkPe+bT 7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y8t1XQoqrXb03tvFLUfCz ajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508umx2gkcl63ki9AlseDym 70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZzTBeS0Y2Bm0y4NU2R3d/T haafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3rfrYaYveLkpRWTJLPQYh BGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC26Upm/leOAo5qThx7EBA tdxpxwtzAgMBAAECggEALjBPYLE0SgjGxWyQj6hI1cyeGy0/xNa2wE9kxmT6WPEH 6grVkdiCVGBSJIZKdpk8wbjes1Kby/yL4o7Kk5u+xkilIZzVpmEZWF/Ii9TlN7gj Jja+ZGIOjkrWoZsKZCr7d4WezzLZp5wSPcOndrGVa1wdjQ02cvORjNyJi28uX9gd 8uBK5AIXS1lbkt/v+8mrBPgZUttz6gxhlHwxKs6JWWlIpGemNddE39UxuGDGHmVA aw/gH/G4LNXtbAIPq5zDtFbfCKnQVgU1ppWILehoFqIs8JLtz4LPuvIxeztzKff4 DU31rs14Zati5ykq9CVqY/d+4nKdstwhRPcPfsvgYQKBgQDBNVPn73A7fRoURpzV sdJPA4RDbrbiZj0x/cAskuzzx/mmJUuNyuJxGizJU0ebT3VxtdCR2LqpgGEQEaKS wYmMlSJ4NccugWgRl7/of5d5oY2m6f4W4YaNp4RebdVhNPJ4wSbeW7pH+2OKr2xd my+m1WJUvRBbPq5kV2BdHNw62QKBgQDMXTqaOjsC9jpOOIjsUHmV55MbMmwK8For H6e3Dn1ZO0Tpcg33GMLO5wHwzH6dlT2JVJAOdr5HqZgdIqjt30ACZsdf2VkutH94 OvZmEAbwI9A+TAoxE8QlLYyz/qjJSGopJRU0x+KqEORxBmjO6LVV1GL9VVdoYrlH Z7mrJ+7RKwKBgQC87LyDS2rfgNEDipjJjPwtLy8iERzb/UVRoONNss3pA15mzIk4 uW77UbEBnGGkyOn6quKr+tVr8ZD3+YaTIpSx1xLBoTSHkRqGOXD6k+k2knbFBIHl NdowoeGZxKSmTPPciGLNg7x/rp4Des3oKltKM9XXLpjT4FL+40HjStk+4QKBgQC8 71AXd9BIy7VZzaCgwUG3GhIBadtDPbRO/AQFFAtE7KuoGz7X+/dWa3F62sQQEgKD LT/Fb3g5LoyoGvwMdoJp9fVLItj1egAC+pgEAbs4VhPXFFuzxa9oI7VaTwxikmU7 RsJVOprOWbGo4KES8Ud8Y09lIHof0m2ymy2nE9MRYwKBgDn86ZcbBr6sBXgc6PEM rq4JXBCX8O17id9rJO37PkhPsOKpNf7YbQwHlHjwkUq5+g7Ec/LbeZ/tssEBY0ab zUXwgWFMUKJVTEZUFwl2aTBqW8+LSu1TgzGMx2H/sxrvS4ElxC04jpPWUQstcuRH y3yIz1HsmlMEg7qCiQ4maZE3 -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer4_keypair.pem000066400000000000000000000032501477524627100326610ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDV/fs/QsfKAjdy bnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHKUohNizh/eM7tqkiw3Fdi gHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsrVbOQmJjmpuvKj+z4T9xN fnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqoaJ74Szmaj9XrvFloyfCl 6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5IrzOmdwKiSDXaPrvY/JDjp W/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1+YzvxZ+xjao8t8xHtuWF 4nP1ilpxAgMBAAECggEABmE7dr39Ep3ZDRdz0QwaNY5O6p8Dvy7llQTdZCsaDAPQ NJsC46w87LgoNVnbUDOGwE8n3TBS2ToCfXBu6joc5V2jkS10LOR7x+0+wpCtEdhL RFyEKP51u+yaXf8Aut5/zX2bwUbj9d28p89NnMV4AIo7Dau0pKXcDlW1Qk+LztyI hKFN6hrSFqAurmSt/pu3oo9kI9WJkrCxoj+VjQdVi420uAYOFR22aFaHrzpuHouW 4IzFbLhVF+c33xSbs1OEIpZSFzNucWYEKSwEREcyFgIXfWpDaXjoqWcrvXkeqyo9 vGytQ3YaEsZPzfzgcViwa30g7WAA7kO9RuwcCPK4wQKBgQDpVmbVnmTlRwFbtdkD 4rjd5vtAB3nfsl0Ex11nU8+Oo0kZWeg8mm+Gba4vjEKfVyojbjFmm0ytQG0OGEK7 UQ13mE1wueMn5qEVX9nTXIxVwcS7+rQAUrC5a6SSg81WIWzeclkqNc1J1EVC7jtl zqy3PtC94g4tV68urpD86RRxUQKBgQDqxpWscN1u7GeuYf8rSPhPcoZTupqyrV3L h+w7jUt5O/vfNPOYIXVfo2u05jiK0mTvLf5tVjYoQDF+x6odA2oBH2yz1ED0DZsf 2AhdtCSrMbxazcl/5fPrIIa1GRBp6y5i0ddX8T19twr/PVoYGRqkU4xoN+KoOKz+ HLFUUgQPIQKBgG5N9v0DDMVKRL0bAQUSN7xGxf1ly1pRUiHBMUl4WEUgsZy3YM7N Xu1YiiBWGOSEaxomrFnKDnxUWXlxRJKSZWBk8i7Y4SZqozmcfzeop3qeyCbpBBCn Bn4RAdJ1VitiT7n0qmwG1Q4St89FGXUuN33Exx8MbxFGQz05LrcwZAaRAoGAVFez PZfudQMI3GToPqygSCpkh3/qQ3Z008Go5FwGWS9rdOyY9nZOrGURNJPgjD65dBOZ 672lByDIpzsjqfioBG89pf0CuKqKqA38M22cHsRnXle/o+sAjd/JhRXUB7ktmOK5 8iYAaUFw+fEYhL/ACnjZYDdzfeueekvkiN5OBwECgYB90hQJ2lw5s6GFJd+9T5xS OMngfLAWDvW8+0hvtWCTLAVpMDWRGhGmvj532jWfkgqnvUemyF541RkV0Hy5K1Xl 0icXtpuZ+REh7NCXFJlEiOd+69OEdu78s5Zy8V1zCkEsgxzl2q6PkBDWfxepgdRC LbwiAF8h2mxCwvvHbaBiKA== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/000077500000000000000000000000001477524627100206505ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/rdns/ca.key000066400000000000000000000032501477524627100217450ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDd8u+Lbq5qjxj0 D8YdouHVlsnH+8UH1spKRr53wRqdPX7qC9eyBDMJ8ltUVAfQcKjXqP3EN22rAm+1 L8vSXcdqxO2pGStXbfrdvDIeiMuJJ2zpz4m3g5T98AcNFUxyUQet4rLHtQGyzK+S sTSOKCg7hULMvnXMLjUJTBsDgSYPNxLw3jlbppTaEVhna74d0aRNBkHVJvO8BJsT J39iz2SrhvGzGfhK2Jbov0QPC44hIV28AFrwKbJI8gB7kD9BWe4hNfqoECLP2K/Z qYw27Ybr9nUVC3ZSnTudAueHwsT+Pa5qbBoNw4QGwyCc7y9nolIOosi7RK1LxxOS 477I9ORrAgMBAAECggEAB2C0csOWm6cvEqnVkZWhASsWgQXO+mA68DYizbNHEbQC HICRRnMaSTnbrFVwvw6HpkeRS+z1wcn+cZzpnxAL+XIbXlQeWzUmMim31skATwpW 9fy2nLmMgdZxPY+YL3KfGTogbEAJkup36Z3nW9Rc5gGbg1fPbZ6Zl7oGadroq3e+ 0mKdUzKNw46VVUpoHIq5ty/5mGyiew0E6IJEv3tD8MtFZ9r29zyDF6thsj3k1K+l elRc6hgvP74RI4vJ6yNGWgJRoLK4mHAxhS6Zew3aVr/5P9P0WRnj2WD65geUgaFv hCcJzqkGQPUl2CGRkspIUErpJkAZRyJyjuw/XU91aQKBgQD9dy6nqqfTpf/+dWzc VltdhlEuGfXN1Yfg9A4w+o3+goeFwt0dim6plX/iSLbohylXbpPcugx+OMIw0xHs ZsT8L1NIUSiXIri+VIU+5iU5X+tQUlCXnTy1MhqRSI0BD02Ac/+MMp36RmYXgghd MIiMN11wDbeGsCzjDecziALi0wKBgQDgKxPmUniPe5eBF3QrpDpBnaxZEz1/e0Hq xKcN7ClCbfPSvyIGJhB2NJR597vfaHCwsp7ReUK1RrvmRBaBg8bFffCbEfFSyB5c pZ6r9BXBhPTB9NMsnMn/e3Al8sVcYWbZdgkkxuVE5DdWZS3zXscmTzls/RPvDlOK yLI+GaiJCQKBgHxjGtBJnabuVhzoP446CwhwvDIlp01mNxKccsJyk3CNWji4ko7G utwac/H2wVyHyDASIho2a58d957CVo9vN8iS6QoaWhMhhQxSqjld9HKdsftvCgH9 tict+X/G3PviKKSbSJPi20hReBSdScGB6eD14rL06FX+62haEFZnrxLlAoGAYrK0 zUjsagg5mY3xCTICsFcyxflRr2peiHZTMy8Sr3vnyZd77Icf4lueL+FiZ6f9Td8n TAV+2H+vRWAfJKEXiGa95BjPTupra9FD/mO2nIDknu0jeVYDHiiEJUExok4EUaTF u9qSoXV14+UiiS/msThaiWEYQL3nDIj26Z60dLkCgYBBrZN/M8NwT111KFGXgYzm CxFCu1WPCRUXIKVJjbj9reJgfHMhj58NFpqtVSS4KNeoc6ZGOfkAT/Nb1/H001TF AK3CBTGzFQiqNmvmWP+0O6JaqVXPgHXXRCdXz9CnJmyIeJqvQbQEZPlIC4cTOi3U Yr+e93q7SOS6wbf446ij5A== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/ca.pem000066400000000000000000000021271477524627100217400ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDBzCCAe+gAwIBAgIUC6fLO8KV763IBygl6/0w7EKectswDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD ggEPADCCAQoCggEBAN3y74turmqPGPQPxh2i4dWWycf7xQfWykpGvnfBGp09fuoL 17IEMwnyW1RUB9BwqNeo/cQ3basCb7Uvy9Jdx2rE7akZK1dt+t28Mh6Iy4knbOnP ibeDlP3wBw0VTHJRB63isse1AbLMr5KxNI4oKDuFQsy+dcwuNQlMGwOBJg83EvDe OVumlNoRWGdrvh3RpE0GQdUm87wEmxMnf2LPZKuG8bMZ+ErYlui/RA8LjiEhXbwA WvApskjyAHuQP0FZ7iE1+qgQIs/Yr9mpjDbthuv2dRULdlKdO50C54fCxP49rmps Gg3DhAbDIJzvL2eiUg6iyLtErUvHE5Ljvsj05GsCAwEAAaNTMFEwHQYDVR0OBBYE FHMfJfiA8Nb10G8rKcO/am9Ogo6tMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/ am9Ogo6tMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABMr45yW SDHQ+zOW6MXCn2CAqagr9TaRjkqYmaaNh0uDBI+oaFKi8WHCi7jt1iSb2f8wp6dv jQ97osGsAsU5KgyyliweIaTftd58oegpmADwpQTpVa2RIz6o4z2FKnDm6ZtH5fm+ RRS+FpcS81s1m6e3gJ3Ie4nIqQRrBcvKpQEgMgiwJH4v2rIB0RvkTztA2EeVyyH/ IjbrbO6Rc7EpEJNbsVWHcKt0tMNx9F1qicgscrcEgAPG77yz1bP4jKSqhXt6OhM9 lhXFq+EbzAhbPzkgQuxghHxJuDNZDafKvf1cl/b7jEykLEiKo3s1oto+9yzRKZ2y uKJ+WESZ0V6XOU4= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-a.key000066400000000000000000000032501477524627100230560ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC2yUu/qqBD+rKF q2f2ISuw7KZlpne5NN4RXhAK1fe1b6TtZi/hDeDaBo0A5ttFn+lmb99DWwQh5UZO OrjueeZLjhkMIBqmVY0Z8wu0BudYFWrjbEEFz2oAN4UXxu6PvKdcHsJQHbMjVvxS j7GauUafZ1MZmcxRizQDW0+bD0NDKERR4JropXC+9Cths9MG8VKXGzYEBX1g9j2r pRZ064ZDUEM3ImZI5fAdeUPZuutg2J9swgXvppVpXOMQPGR6TUZsQZC8Z39zm3OK vsyqKE519gpC5ZNTMm9xaD+Dd8tyv3OhMnVZvIEBvtJysKS076cGqxwIeFNZ5zky iNzZ6A2jAgMBAAECggEATs2pDSiEMNqf5bYYu4ngesYEFRX4Ts5GoSky9GD8Cupn qfvgz3+6vXSOjOF6jmFbcFDx6emO9ggRT//MI/BFA7TvAoXCx4xhqVrZ5FMHoTeW nhbZT6Z09kP5at9WkXyNkM8oCf9L8IP7g6K0uNaA5o9WydY/OravhXTz9I9Xeddi 6Cgjr42PVqpdlN/CvQQ+ISYGSYzg6r/o8gaIg1zypJPm3DqZ+TFBxalv7d0FoH54 eagk+blA6kUGvJ/CX5Fr19hliycDNfxT/uofdVKEUXSWlPQgzvGk6Pb0B/Jco0C5 BDuvA97LigY4AR5FXZa2wmYhdlBontjixiZNFgeqHQKBgQDn4XriO8ZNkERW4e2V aLrkRH5nd3IqRX9aUn7Qz1xXV+nE9+984Mwk0xtTkwtmmiII8v+2cfz1J7I+HVB9 NRFGtcKX2REEM6wp+UPJi8HFx1uOr3Mi/JM8i+1/vq9YPXR4DJ0CBJwydTQo0zIz jJrqKPUiEAtrAmT4YCz/hdrARQKBgQDJzIdWWCv8LeyFBdJJHikjSy1PETFMoeRj FEtdi3Oqninn99ovV43fgVXK2C38OaiO0X7N1OwXtJhWaBU5TkmlEbXe7eNlayFc RFfEmg3RFItFEU348oyNuhqDU7RRG6FapMOX6fAIhaF0Ox+qLlwsW+uNbjxcq/5Q JIhLji+4xwKBgFk9V9OVeZ4ENajSTmM/6R1MlvEu8Qr/sCGMui2WgE1FEjyxxsNi qG5LqKqjXrHgA9U539vRRq761gg9s+pb8Agsj3VtHrVlRY7p1YLNfIenT788Gq2O UwPsPjz5n/XvwNEq2YobSBnLLYXAsgNQy5XuqViwoBRQ2ZcVPHwigauZAoGASzFu 7HmGSj1CeH6m0J/wbDpC88dQO2HHnDOfsYeY3eZ5bvonzqQtNS8YLLFI0Ucay7y8 Jy5DmsGsUeYOon/NaTikMCEsLkow8BR3L8bHhzTEgEPmbfDS9qx13KF/+wj0orXq O9zrmAFTG+A7+Em2BsFpkBWXYGCmLm6uapjcp1kCgYA1cGEgpOXGv3HR21ItrpWh wx/iEqybIdYU5bOr1OxZJE6PgWD3k0DcPAFGbfJX8I5AEQQHhbufv+QMwfdm4irt /EMWE+5MzmCOPKQzb3jLhvdqAKie2rnDFuFvlhT2woDxQaY5gaS3Tay380WZ3+rW tHA+wi2+w8REX4KLuaY25A== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-a.pem000066400000000000000000000024561477524627100230560ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcIwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is ZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtslLv6qg Q/qyhatn9iErsOymZaZ3uTTeEV4QCtX3tW+k7WYv4Q3g2gaNAObbRZ/pZm/fQ1sE IeVGTjq47nnmS44ZDCAaplWNGfMLtAbnWBVq42xBBc9qADeFF8buj7ynXB7CUB2z I1b8Uo+xmrlGn2dTGZnMUYs0A1tPmw9DQyhEUeCa6KVwvvQrYbPTBvFSlxs2BAV9 YPY9q6UWdOuGQ1BDNyJmSOXwHXlD2brrYNifbMIF76aVaVzjEDxkek1GbEGQvGd/ c5tzir7MqihOdfYKQuWTUzJvcWg/g3fLcr9zoTJ1WbyBAb7ScrCktO+nBqscCHhT Wec5Mojc2egNowIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w bGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFCkA7CbJMs70HE5rQxHm qHL/UgkOMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3 DQEBCwUAA4IBAQC87KaD2UuvQWIbSPOKy2iPc5bHrsmzxtPzT219NqzI4JZ9QeAz btvLlMxgACreCb6wzYxaOjbU+O2LqO7mq/M4UyY1+wrAwYn+c3rVcimuPa2bGKin Le+aBr/4yXAahqH4DW5K4x3x/7c7wyNlj9MHUQDl9A/JHsit68hw1YY/1ALPaOpg 9L5K52gGHVXrxb8In8OkJEoM38G5Zstiuh1umsWNWBp6Nd/FlNr+XbvNuFsWLC/V lTaHxFt9WRQNINc8RpK4YCHRdaOiEqAGexDkHgNaLHeHEwCvP3Zhj7Zpp+svlmm8 cLL2A5wl0T0OEehT9wxKaG5cyP7jfYfJmENW -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-b.key000066400000000000000000000032501477524627100230570ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSHSxxgyq3Yd4H LpzmpRceCQ8y4qqsysiZ2C/9HhwcvvoUXRJvLB5VzYb1702AZNgbeffrz8RRGcaa YuPRXwkitRGb/MdmC/AFi2aALk1M4EKzGhw5IjyL6tfk+qahxm0IACJVQW9W7Ts8 O6odqnqenvXLzvTNn0UWWv7jZonOpqnR36CoxIj9WGNXIlwDtcTwCoLfd9V+kUzX PybIVrAL13tLPypZ7fpyAkwd/4qYAZ59HbRsQDNvYItkwe4JQiFt4PQ94+eGDXtk DqttKPi4uUUgSGqRzTNkvkM+KAghRD45XN2J1ugw16GGUt4sQKPViTDh06yQ/OTR X7CyuvmHAgMBAAECggEAGBhPXe8QSXpYJ1Oxm4OIHKImU4wJzYntqjLQAgNfbu55 qDcqI4SYL/fDR9W6+gPWgoQgqbKoTTKbA15eyMZQumGwVMaOqUWqL0CWrxFgPPYk v/w5C1ZBQjxaqMATts6V90popYU8+ud2mQgwkHlZ2XHaNxaTHTdsCU3DGBnMqN1f tq3glwkHZqUxKsGXwWqSKsDwEUXpcs2ww0fbuzpdZ5IokPX9e9TdFkw10rKXboRn owcTxHI8kuTVMFg9ht4G39V/LVpqrPsHGz4Zj0tJ8PkwxowcwLMcxg2KVsHRWzC+ i5UVlzjWVK/Gu1mMMCfySJLBi4GaO88hZzK8acdu2QKBgQDwgN5zkgOELSulfGDQ x6spyjbEdN9dn/cwCpolMJMg3mdcKP/XjPTTEe1dsWT7mTa9qxjWgcxOzr1A3h4B ZPJTXgOQyt6rPYw2WffEh6+5H28+629VlhQary7Sp5rZAXiJoHgzHpb9sURwliTL 62GzaCOxJd1AI3qd8atjgI/JmQKBgQDfpwcGmcpAhQMA/owdowCoJhQ7Jc5h3+RD 55WO6sFFxigxa+gSt6NVw3ef5TB/zKyovPGceunaMyVOeKA3RSNYNX3gKKhpugeg wv465jGw84jPj7b1LMc7ydWHKWQ4M6JLPxlSM9lIV0Zdb+gNu64hCNMGOQljcCuh a0Fi7XgQHwKBgES1XFgCF/UT/ospjHM5B+JJffoENagGwlS8QqtrRC04vKisoeLP x0XhskF5I5NpgyUOk2r87AR7tb+pdUMKttwwWK5t9s1PKRaB/3HrHb2yiFKealE2 LgotkB/oeCmU7P3MlWptS8+wbBAKJPsJBQF/N+stGRdwy2ACIeesW+fJAoGAbG6w i1S4qDtUz4CaMiw7P0rm+B5GR1GjfACllBrhBZEjH4Lvi1OZTxVrxAv8TZnQBCdH BNTa2D1/0uHM278becLFeo57yHa3CAxB8hB0+xO3uto0m/3Pwn8ClXtN6amu/8hh 4Gbe66HUrax5116s97sthJDWqm1R2fsEQpBba4UCgYEAitrHGT/Kr/91Ttccc7Ww Eee/xEJR8iMFU010zkX9MAu1JExXOaMWVJRWlzSIJFhDjEovaq9qxdWPsoDT1C/7 JIs7RGDVWX2a6uun5Oz9JJJ9KmNBzOAicTBBrWMT67TGWZhVYjEN9PvRyX/kMssP yqRcbXt4Unw93950LphbVOw= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-b.pem000066400000000000000000000023611477524627100230520ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDejCCAmKgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcMwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQHDAtM b3MgQW5nZWxlczENMAsGA1UECgwETkFUUzENMAsGA1UECwwETkFUUzESMBAGA1UE AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0h0s cYMqt2HeBy6c5qUXHgkPMuKqrMrImdgv/R4cHL76FF0SbyweVc2G9e9NgGTYG3n3 68/EURnGmmLj0V8JIrURm/zHZgvwBYtmgC5NTOBCsxocOSI8i+rX5PqmocZtCAAi VUFvVu07PDuqHap6np71y870zZ9FFlr+42aJzqap0d+gqMSI/VhjVyJcA7XE8AqC 33fVfpFM1z8myFawC9d7Sz8qWe36cgJMHf+KmAGefR20bEAzb2CLZMHuCUIhbeD0 PePnhg17ZA6rbSj4uLlFIEhqkc0zZL5DPigIIUQ+OVzdidboMNehhlLeLECj1Ykw 4dOskPzk0V+wsrr5hwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4 YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFBN5/1B/G8UTcl5O tqEHRzrAJ4IHMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqG SIb3DQEBCwUAA4IBAQAioN2YIK8yKkyn3x/UH+YhgLJj9CZfkCPa2uJTBaXCmZcv KBPfjs4bQ/c6omLzbGnIVEDHjUEzwxUf40cVQbciPXvrVxB5PIcUY/e8RAgFux7j YnP4F4fM4DC1yOA5AIqWZGX66GVnw7rslxz5Pko6QKNaHuVgwHHEizN0d8hHexdH rGHtX3tsFmI7GOwsVgLJNV3VcpT+W8ZdviHtbjL2gR3N/KpXSU3FHmDC56Zi8HyA iUICVuCo06LCEq5J8M8f5dBEMtLJ31gDX1c/arLJuS6VS/+XrC7lKNT571SXPa9x gRmY0EtzyYut/yfG2qXWV9Xi0DbrZUKZVIUcqd8y -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-c.key000066400000000000000000000032501477524627100230600ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDG0eLmkpj4R7o7 Phx+EUuInMKkHD2yoYVIDc3+ipvj/3bptxWVcRZoWroFLrUroSGnPBI2stXHGFti 5qkCCjmsWLsido8luDsLnr9kBQafBoV+tj+u/ZkJSeTJ+bVXaGS9GR4rGvYPgwRD dyuebRSrqW/GB5Chp0X4JqxNGrX+SrtU4fz23eBpg1+Tv9Td3xhlw4FdVAphSYS9 NYLg7fNpIVwXLcvyXgLya2IOvItHGM1TyWq4jsOrgU23z640GzTUEIic34KkgF78 Be3ZN+0oEiIrojpUb9I0k52NhHL6Jjxu01/IIgan4n7h/tr3O44YOeNfpRtn7RSA CelkiXUpAgMBAAECggEAAwbQ7cek3Sq3srfYVob237vbLq+2ChEcEU5nKLfSywlp iwHxfyvl4RgjJXm3jP7P9KwdGYi8E7goM/zk7RYGGLyPug+UpZ4EsBbuPPHipEBv fyVRGjMxcDXCw4cHSVmv0ZJ0ph1Wn5fT/x7s+BAbCh3zM1A21cR7yTJK2dr36Ei+ CgUEvVJQh3rult/4JJ5E74oXq6rXRqAjO7i/DXImLiV7YePjixhy4YGpYH8T1Da9 CdhA53Ec2wqcEfaIZC+fMtz+4XQnCOJJEjJGfGApGadu52sqA6dB0I0D+DWy3H84 V4MPrJMvllK1GH3pk5zqm+tWlt62R7hc9O4a7OOVgQKBgQD9ydnmPv5Zf03UAzT3 VLuwvM+5H1EthWV0CsjPf9M5QGfAm5tuxrqWRfZ4mPX8YRJFc0RXBy5WD+UkBwDo GvmmD96t6FJ+xsXTz0Oq58cUAWiyuqPldbe+NdOIHasam31jBss/jrL61YHU4eec tCtTZtJQsvQ/X3wjWSF4fOmaOQKBgQDIjWlmxnxC+AhdrtlLYVhg87NoVgAOAZcf MCnJT9xMbhfmZko75vUdT4OnTXVjtxfqLP8yZE6lgb7GaTum3YLhhfZjnRs3buGk y+6mKlNNhfrOFCw8tgVzaeGQi+dE0WyBHqBpiKxU6EixY/28oV1+einxu9agZCNy USU03LdycQKBgQCkSdwGAfdzhkainaTXC4IpCkFKLKzHVBh2A1k1giEBaeEAPXtk Pb+h8g54yURMKabDULgHwn1LdyS1qtb6aiP8TT+wwrlMmm6MDBtY8ovcNoFJWisp KohU2NFjxxjxs4B673X5Ye4ZFMfkQI1H0xZM+j3hwcb9k1lwJI3XSr2KeQKBgEUA AkeN+qq/04EH61L4BwQ0VIGNNS+cdHYSiA3vIAhbyHVItDmv2J1hAhbJm53XHK9B E/wubrCa1xxEkHV0uNcG1CKppveHerLMRyt8XHTLp+LHJgEtTurKfwTQXZ1bwE0c UGx+zWvZD0mY8W+4xQYC2fOFgO5mBZwLNxXc6nDBAoGAHJaD/ZSSjtAqQnBYN7Wt /QOpWx3UrZ4xAKM6KPp0jAKNHgFlH1j7KXIT+fC54TdYw6K9TOzMOG83BFNBs1vf GTRc7NbMW5fW5NqLEwqBK6lWChqXMwjmz/W7qxRBPqtsmAkubgUO1fBw/XReut7C XOa0n2b/bfIb5jCCFAH/Gw8= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-c.pem000066400000000000000000000024561477524627100230600ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcQwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzMxFDASBgoJkiaJk/Is ZAEZFgRmb280MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxtHi5pKY +Ee6Oz4cfhFLiJzCpBw9sqGFSA3N/oqb4/926bcVlXEWaFq6BS61K6EhpzwSNrLV xxhbYuapAgo5rFi7InaPJbg7C56/ZAUGnwaFfrY/rv2ZCUnkyfm1V2hkvRkeKxr2 D4MEQ3crnm0Uq6lvxgeQoadF+CasTRq1/kq7VOH89t3gaYNfk7/U3d8YZcOBXVQK YUmEvTWC4O3zaSFcFy3L8l4C8mtiDryLRxjNU8lquI7Dq4FNt8+uNBs01BCInN+C pIBe/AXt2TftKBIiK6I6VG/SNJOdjYRy+iY8btNfyCIGp+J+4f7a9zuOGDnjX6Ub Z+0UgAnpZIl1KQIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w bGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFNZnXD/At0QIFoqsEUNn hd0XqkJWMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3 DQEBCwUAA4IBAQCiIEr4+IiHo4PLK41l6Yuo6QKETZg5tNKluqkT7r5ulYaTss+C O7nBIZaVAK3h5+uRTzrL2mEUyme4tYBUaIxqVTnjDgIbeKvg3Z0k07Zld7+eeE/B vnPk3c4aZaKshcbKoA+tZUeHk+BYZ28YkDH70OYsyVhemvTVDDy2EZGdgeaWJAbx RcQ7iVAIR8SEgJy9PWZMAFChNxa2N7Q1AcEqnU9UV0+XOHVHe+PYgK7SfuNOxM6x rP2NnkLenDoZ7IXEvl2WYVlnJpIZiNTRz94nkTz5MqjCuWS+YGYlPz76gpGPJ5NT VEfAZXP0EW8NWJ1XEIM6+u6c1SWnSJfTt6im -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-d.key000066400000000000000000000032501477524627100230610ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCxkPAQkDuzNvqJ /iVOuucGi7n2N3VI0CXBoAIUTlyWnf73m2VOjeXUCBL95so3uGBd1Q+R9BwEnPH3 w8P7zEn007tryzskGywjEqCn2WYW3TKBGuFhuTc5sFE4IesCjglRwebsCn78V7kE Rnz9/hRgULEIRPjWZkdwnxO03S/4n/lzMxCkZdLjyIWpr99B9l5Z3dAFlMchRqKK 7+IJGorNjEIkVLQIHdWFUAnI2ZBh0Xkuebqxg061dhEnvo8aBi/gqcc/8IVfnUbX mmQM4nPwRFzOY1/+hfCKOIR1MAQZ7ydXK1C0tuoR3rSvUEPgT7MbAoi8PZIaoLCB bFDsQ0LjAgMBAAECggEAGDmobvIRRbWPVTVDB2yQvqq1z8V9hFYVg0bzvFZ746FM RaFsnwd/9BS36+5JPMne/TOSFhJiTNomruwFTdHvbcNXSMZnhq5dsAPyFC72R9Hy JSgILhTJV98EwwOUSaXG2Ax8F0xP9S3N/wB+PAl749oOamI9OsRQedPSKfTv/wYw zPNhr4MgnnD34VDnkoYTFIU/n9OKeSgh8okx3PFaPkfYKjaOoIFHA4j1lMMVa7kl q27MrL/vpWF0MUu88ip3lDmZQC9t6BovoIidwRhNEYo4Qf+igcwcIce2MY77gg3Q IvWpsG7r/jyr6KFRONvyJWpyIgAq0Y6aRb4cNzkDxQKBgQDWrOdLVg6xw+I8BVrM 9gDJyfeF8zEXu6Mi3zrypyOOd11ZqlVFaeq/sT2s7HCVEZOiqTgWE0ueBry6FGX3 DkzGpTo1+kd5HiKawViPHnIIVTV/F1Hifrx8bDHRRPVqyh/iTV2c4S8Jdf90zgZJ vW2wErGTrpTVvgnb5GhZBZIQbQKBgQDTv082UDFD3fpA98t7DptqmYn8hpdAa4Yy MFMHBNbNntB5G4RDdDut0eWA01MKNRjKrxBw+YjuOgTIRx5oVre73y83932rGSiQ g1/y4PidwPVEAnVbbLqvvHkkKTbBbHENSh2pWCikJhAGKhVI665bAc84o8o4ikN9 IQ20TUqujwKBgBt1rHRPgrusYcDpsm6rPPiS1A2XSP1PLBPm2AR36q1riU5R4mxp mvSAOHJpIBGBWRAicyEnwLFrDTMELvLGKn3yXprO89uDRkvjVW+hJlb4h0pFclz1 dyi3MjwhhP7u8dshqErL5xdft5h6TgWarHAsQ/ivCSy6DUrKUaqpcsCVAoGASq8A Nkzkg2ZonL+JYlbNlDShSZMDB/Kku1D2B9S6Gn34U67T4KK/ZdhRVTWz5TbDDsHe T6qDlFqcUzNaUzy6wyW91sSQ62cNOqNLlTOqHKHxH9KqJ9vaoJ8eLxXmPSSNXz2T 5qW0d+kA39u59CVEMs8ZkoWajoSdtyWQWtakD5sCgYAUgWP7Z0CbUeNyaH811Aef /P7inGoQEqTYrmnyAFmRyBnmq3C3OjTRiz7+R7cERpgVVmLt4xpNWjmTbcngfvau c2YViXlsFx9s/xXFQBtppmdH1WeZBIOZbN6EaA5q0oHGOXAW5SmZR3sGKeG5Mz/m VDNDxg7Fspw4ZdHu6IeOCg== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-d.pem000066400000000000000000000024661477524627100230620ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDrTCCApWgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcUwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL TG9zIEFuZ2VsZXMxDTALBgNVBAsMBE5BVFMxDTALBgNVBAoMBE5BVFMxFjAUBgNV BAMMDSouZXhhbXBsZS5jb20xFzAVBgoJkiaJk/IsZAEZFgdleGFtcGxlMRMwEQYK CZImiZPyLGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA sZDwEJA7szb6if4lTrrnBou59jd1SNAlwaACFE5clp3+95tlTo3l1AgS/ebKN7hg XdUPkfQcBJzx98PD+8xJ9NO7a8s7JBssIxKgp9lmFt0ygRrhYbk3ObBROCHrAo4J UcHm7Ap+/Fe5BEZ8/f4UYFCxCET41mZHcJ8TtN0v+J/5czMQpGXS48iFqa/fQfZe Wd3QBZTHIUaiiu/iCRqKzYxCJFS0CB3VhVAJyNmQYdF5Lnm6sYNOtXYRJ76PGgYv 4KnHP/CFX51G15pkDOJz8ERczmNf/oXwijiEdTAEGe8nVytQtLbqEd60r1BD4E+z GwKIvD2SGqCwgWxQ7ENC4wIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SC C2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFM4b9lmfPrfM tHVxnMbxR8oyE0ZtMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0G CSqGSIb3DQEBCwUAA4IBAQB3O3Q+oTfAVitx8K4+ZzXILszfblsGO4kvwvxV6EUB BUsat6mjawFTGhB1TdBR0CflA/nCjTZpoXNY+nCoJmr/Lxk1V+UNn5zhFTTfhZjU PA+++5QcYxXz3ukCEqTHLPZKvv2xB24xQUmsdwLlX3VGE1VqBfaz/2x6jFTVT3lz 4W5JPltgFSVOLqQC1T0MY8L0h4eL+JpynHPjMkEwb0U4RPaSBhiRSjHD8dgoZ8Ft ZbV6WCJeBYey0edKNwujuLl+McVpx6DIy8BPGPEcuo20Sy1BP3JAoN1Wu5J3zLBA AXwUZ6r7jAjU5v1nx6ijwtv3qavBVdclOlW0aDu3+Y3z -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-e.key000066400000000000000000000032501477524627100230620ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDRoOr56ToxIMnw palVuFJwDnPotA0toHmQe2+iGUtsOaDQW5USKynV7YrP2DkBTzai+KyoA6cW6b3M i8FIivJk9hkxnWp7CuSn0dFZVoyMwSuyelw+ff+8Cn6hNmg9s0BgQklk6pe5LsGh Rwun6CNPnlJapYycwaH3e7W4fiFrs5njz4yrYQTQZPA6fBybtcT5NatPMWBzCHl9 HoBzLWEOphN7D+Pa5Cx0zwqHf8h5/NJUWXnNO89/Kca9oGZOm6t2+Dk0OYa+nw1s d1CWlzSYobilQiEEiYLnPgRLoNiJt7hDZqE4ubVcIDptN/SfiOr5cg6aao2Vdb0P iNcKn531AgMBAAECggEAJj6R8AMtYy4kuXs3wBRPMfNpESDcMKFQ3Fuwu6WavYVs AedbdIOcb9OVGiNMDyGZCbWNpl94OfVt86Xhnu7+lw3Q/dzGwujzW2yroX1pSfqq ErdBIgORxwgxHw9SiEMuMQGW60rjYF0X5WkVRs5StuFfC/udAjJPbtvKhPy58zBu C2ZtcZFmf4KuwzCkZT8K8zvEFcOOXNIhtKMdPRSsbO7EpWn+J3SA+mG3nKo36swe 6EptIgFo4Z0dSzS1xK50iNOw10dKgxOSJpDjg6vH/cGduUxGvGWW0kiV1vD0ndUU U1qyQSwlfFfFUk9WrtxoWFCXgZWyhoSONQmG9eRlgQKBgQDtLYVNIKK4n/TEiz9s BtkFmh+ohQ6vjorjkzrj88Szxy9MivCD5vvU2Q61V15H9255RxgRPZ1lczGP++N2 IckY+iC4vmeEyKOyFX8XYaJILEXPRkoyj6/PMyS8eWciZ08QW/bhOEDSiIlFrZ+4 C/0KfMaT3MHRFRNV+jlza2RzNQKBgQDiQ7W+xtCqsvSXbVjPrC4dmxXlo1B/O4P3 KsdiNoGpzu1I9nfKNKlfwfnzI5vtk7dS85kZPuCQ18fOQ7aO6w3VReg91zoEDSAV GSJLVHeiRr+XOBCV4Cpy3PloYD+m2f4niT/Ru6CfEmHq6IoouWCqSobZYlkIHtAe Rfjg1b8XwQKBgQDU8e7iDbArZWok4QTjX86QKg3MhxJlauymYPUZ3y63XtnHtmM1 ejiWpP+Ar1217aChgMRKDnD3uhJAvR4/XRwpscGZPVLCNEZMyOIfwwgqFp3i+K/7 b2ig9sE/+xwvFmQ1QuyIz7HblLzy820YmKLrPJYqAaV+rJZ8tjnIuB4rFQKBgF69 PS9JbfvbfKCfD229SX3p/uwtSrpLgEEQ73VHH1mrpB1F3fiTvkuzG+ZbhaGflUYt a3BvrHXZc/cA0ULcVulzIQAry0YA/Or4oCxsjL8s4nH97qvitcNslR3IFz7Uzh9d z5QQ/Ps+JjPG9HqCzF/hXr9M97x3smrKn5/8v8vBAoGBAIRJPSik2ILc8V/1FnPa aOfcBJwcuKdRJ4yVWiwe6meQCLWBva19rH8qx/D7h+r96a8OlSDZuYltM/UF5rV5 lEKUs7uH9Xqw8QMMI17fQ//aRx265PJvdLtoO302IUQiC5qZx9YZKSLIzetVEapD SfKVumS7pEPZgZI6WD5YoahG -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-e.pem000066400000000000000000000024461477524627100230610ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDnzCCAoegAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcYwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owgYYxETAPBgNVBAMMCEpvaG4gRG9lMQ8wDQYDVQQDDAYxMjM0NTYx DTALBgNVBAMMBGpkb2UxDjAMBgNVBAsMBVVzZXJzMRYwFAYDVQQLDA1Pcmdhbmlj IFVuaXRzMRQwEgYKCZImiZPyLGQBGRYEYWNtZTETMBEGCgmSJomT8ixkARkWA2Nv bTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANGg6vnpOjEgyfClqVW4 UnAOc+i0DS2geZB7b6IZS2w5oNBblRIrKdXtis/YOQFPNqL4rKgDpxbpvcyLwUiK 8mT2GTGdansK5KfR0VlWjIzBK7J6XD59/7wKfqE2aD2zQGBCSWTql7kuwaFHC6fo I0+eUlqljJzBofd7tbh+IWuzmePPjKthBNBk8Dp8HJu1xPk1q08xYHMIeX0egHMt YQ6mE3sP49rkLHTPCod/yHn80lRZec07z38pxr2gZk6bq3b4OTQ5hr6fDWx3UJaX NJihuKVCIQSJguc+BEug2Im3uENmoTi5tVwgOm039J+I6vlyDppqjZV1vQ+I1wqf nfUCAwEAAaN2MHQwMgYDVR0RBCswKYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIP d3d3LmV4YW1wbGUuY29tMB0GA1UdDgQWBBToYUZkIK4kXJRs9Af7V10od3oJiDAf BgNVHSMEGDAWgBRzHyX4gPDW9dBvKynDv2pvToKOrTANBgkqhkiG9w0BAQsFAAOC AQEAKMs8gDYeTAITUMJ2cfQqU2t1I7DGIt4eQ/bJ1LWOhC6eUXpUrwjjNo4pq95P cct0haleycD4vjZF4/Jv81oSIUsGzQ8r3LZhJlPnzbZzaA4i4Tpxnw8JRE5iiF+z lm0Cl783Nh8voVKE1uJrSx+pMTH9Ihwu7vBklNI192zLPLPtDFSZ497oLVt1e8Wt 2urYXF4/Wb1pyGL49gp/eOAEpw2j6pyxGSK/QyXyvvPnDLqNXJCOTyx7YQRvZnBy h9vTtHg4HSKmgrOHA/taW051lh2PQbi3pbi+ik3rk68QY/Y7b3fKjc/6lvXPRoh+ 94valHgOVFJaSG+Qil0A8i6bzQ== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/client-f.key000066400000000000000000000032501477524627100230630ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCgF4OXW1j7wJKD Kq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDwT7p/TL1FpGtK ykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGqRSWzbGcXt5GR LpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPfynWmmHKf80/F kIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587a0DilwR1Pcr2 q7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrYK74q1uMUJogc 45EXMhGfAgMBAAECggEAHC+XpQXZuFVS1lIlAF5zW9Mv+k20pB80moRdZro/h6mV QFoKAq6SMvBofRQ7pqq746N959zIdCL7GutbXF6sU6rYxzFBk6FhzDdJZnI7Qc4z PpE0JKa2/WOjpodEPvsZvNs4xIrA6d2RfPiw9Wl7tR2M/AQCrC8Bvse5sRq/+dP4 U24eetvPJDTYLk3PFnZJPShAUEOcKUaL/73nIwCFbC8ez3ae9EIt4CO4ljHORp6h g2zJFei7ie6VLHAZx4WUeGTVlhgF2bYjsjM1h254ZTIfNgqGs/DaOwzO2kpC5b+m TE3Tuq1r0b7uMEnsCJb4gj39RYOqThZ9IG3I1oXKiQKBgQDMp67wfZm4jKeNfwr2 gUxR7Btn0XqB6aMBofAgQiUuq9z8lRr8PTM1wPnw/ZKN6mYunIhXHGP+t8628hJx FAy0fSy2sy9ucxipcDlyPBalSjtj1i1jqmZJtvmal7zfi0S/wgBn/J0X0lw7927Z ai1VPVoHOeTouV4PhcJPHkqOuQKBgQDIQa+N4q4kCvmoMJSIHT9/vUV60uOO09QN 4WEtSUdrZVYnSYvO52f11bh1ZLRiZayEfov3vJ9LsLhnil5pNhEvxCZ3v+z5Fqgi PRxY8+PnWleMC4QhCQOywpUbC5HqvqyEhGVVYt/CKbD0BKJoK/iVSkVUIhQuvh32 W1x3wcG3FwKBgQCC9iL6fkVqVYe7Ajsvt1nxape2/dPZYnsPRmN1IR7OGOiXMYtN MpScp1rKHlo0OnUdCsoshFxw2YqMg6fNeLkQnrGIUG3fzgNQGiKIuW78Yt5SavIs vxQpw89CYCtbGbHqy+iaooqcfd7L+PCUbF+KFnKQATo8urI4WK4ZFxc68QKBgBZs 3Kbr6opYvEpsXkW72L+KR3yQnzEAYa/IPGSg3yGUsIgnwUNDQK4T59slktmt/xq7 PRtaARCt0oitwxLPHi+WLKSeVoAyXFOxOjpv3WasBD/Hjl1QsBxVk/L6YbXC8njI hryAHQSWPJ2m4zT9L5IwRgE867usSJis9HbouLOzAoGBAIZtLLPx8KYgjiVx8QPx 4uggZ7iblh1aI5ZO0XbsmWQfVDJ7HKpLFc9vd+nX2++Mv0vm3yv2WslMVliFvjXl tgZNx10zn2usg+Cqy5728ldTXnXKA9q2KZOfhVPHv3sTao9bji5tywjnvt4ZLp+j ngI3derKCSC10cXUiDEs3dXI -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/client-f.pem000066400000000000000000000023651477524627100230620ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDfDCCAmSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfccwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owZDETMBEGCgmSJomT8ixkARkWA29yZzEXMBUGCgmSJomT8ixkARkW B09wZW5TU0wxITAMBgNVBAoMBXVzZXJzMBEGCgmSJomT8ixkARkWA0RFVjERMA8G A1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg F4OXW1j7wJKDKq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDw T7p/TL1FpGtKykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGq RSWzbGcXt5GRLpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPf ynWmmHKf80/FkIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587 a0DilwR1Pcr2q7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrY K74q1uMUJogc45EXMhGfAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIIL ZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQU1Fzej0nnnMw9 Aai0AC5g2XsdiDcwHwYDVR0jBBgwFoAUcx8l+IDw1vXQbyspw79qb06Cjq0wDQYJ KoZIhvcNAQELBQADggEBAKqH6YEyYgW5CnhHNOlbvB7kCNrQN/nAITWJILkJRtch hMhsODV49fj593Rp3vTJXJ+fRCwJeF5pCfHIyUaEC9FM8IERzK7yZUW3h6X/KfwW vHdRabjsuNuXk92wvslZK82jjosDBvdLv0pnApVv5OPnSXDUr/kGFnXhe7OQgLAb OE5o3jRHWGCiSa1CnXrY4fzjDNR/MRgk7N7nQ29cY5fC//3jtBlYlcY7smjH81OF 9kFgQK1Mf83cN3zFLkVejLpquVM1CX210Z0GV7hEg6zr0jS8v1YRnCQ9xp+Euhd+ TIU2SuidadW4ww/yCWsBfBuoX8ijByvG8sYyQ/jXL3s= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/rdns/server.key000066400000000000000000000032501477524627100226700ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjc0LulDTiYpXX tTTXW8ubZueU3abq16rixbIbpQ+hZlpGhPOMDQbBEJdFuurt6GiwUJYvPonsuLxM VhwDBGwKAs8Sw0vjwIHoPNBo8oKa7lW6Be2vv1+VBiDOd8ua7xnrLBmpKgADP40a J+VAgSSCAVBA19tG05CJbxGnGHBgXqNwMamR+DhQYp1JV8JqivGA4UqOAchHOyp9 ucFItKJ6EeB/qq1O53KnEcB5WqHYGy06/tJPQ/jsdQxakSaVMrKvrhTY4XkSMPAA sOcrX/BFpWkloNthICYP2ROzV2HYTDfQgGt1SU4wdNseOsRfs0TQRKMa29gx7iQI pH3D2i/FAgMBAAECggEAArqksjOWy3Xebx7XLkZZ2LMn6DgoNpEZyRYIcoY3w9ff 54AGSf8YrwYOJIupqwW/o7wAufK3HlT6TeL9Lj63aBZiazuIC5o2ARDs4Oe1rw0O MOAy9wWAvM4Ao7nj1CcsXV0b45NwMx2vRzHF1eUiHSqU8rQ39M+JAQZx+sKBc9Zx 2oiL2Dy2q6YBqgJA24dDk2S2bez32ZWBdyJ+/kMzanhsHZb4a8StuxMjX201vhmG QzV9PR+oi2dn9Kee/GX4yZUVDZ9A1jOdusk0+SnAPJDMi2JEp5kj898GXW22PZCw mQ8iEqJCmsB5zWUiYiF0IkmTeSkukZoX3YaTts9jYQKBgQDRBnGrFimcM9QJVbeR ObGix2TxmgKm8qkLuUb4LX/AgLHCNAy5SdodMOFHdV+zZlrqwY3d54yiFwix5ujD AaJcAhOKQGX0aTEz3Vd4XIj9Y/uMwhU9XG5zTbbT0QlVWLDy93KJYeVa9ateJsg5 DBcF025ApLQv/Z6bT2R7VIjdYQKBgQDILtFktkfvQS/uMcJDX35Vw8VDsJWc2Aas oBQBfHwFM7mzfoofmHnUFh/901uSXvj3YVgNHnSQx7+sJQhViO3RFHSNHwyTs7aV RErs/vseFpYf9SorASygkOR9bH4z+KIw2GpSMUIWq5c8E1xe7lkZUuHJhQht8Vzv dVwHBfUo5QKBgQDHwObD30wokIcj3Jyu1nnh27emA85hCSlvoMInziN2LgayVteK Av/EQcAocAzi/wAHtK0E0ZFeHbEbglYSde9ZCkruJhjI9/YjYHWE+rmXngL5Q0jU +Q48dpov1maa/0UrDDqS+9EZmgkI0vspOVqPIL0OXdgsvKzkM30NN34MQQKBgEzn N4giQsZWW6x9Ly0kzWrzV6AmgYOMthuxL55WjWqOMYQUU8nJkNv6V/XyMZasp6aZ nfMERTGtmyPt4iLCBOKyogfo6rL/cmArqUEcv6oScT/7tmRpAhT5NN4+RMmAdgaf zVCgHcyJxQLOSUkq9c21uonpcpCSDersQhtsnX9FAoGBAMLm40fsn5hqeOSHfF7W ZEX0eKSBPuCCWTtUi5c7pPmyj0B8+9o3wiP6YHcs91ysfAWgnJqBlZkPxkwC5zvT AlS1GNsCCQzZT8c60QiOESTMWa+398AGtFRYVNjbamGcQOW3lprfeXiIQCAyQ2d1 GNZICja01bMZsLidFoyUIH95 -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/rdns/server.pem000066400000000000000000000022071477524627100226620ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDLDCCAhSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcEwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx MzIyMzAyM1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF AAOCAQ8AMIIBCgKCAQEAo3NC7pQ04mKV17U011vLm2bnlN2m6teq4sWyG6UPoWZa RoTzjA0GwRCXRbrq7ehosFCWLz6J7Li8TFYcAwRsCgLPEsNL48CB6DzQaPKCmu5V ugXtr79flQYgznfLmu8Z6ywZqSoAAz+NGiflQIEkggFQQNfbRtOQiW8RpxhwYF6j cDGpkfg4UGKdSVfCaorxgOFKjgHIRzsqfbnBSLSiehHgf6qtTudypxHAeVqh2Bst Ov7ST0P47HUMWpEmlTKyr64U2OF5EjDwALDnK1/wRaVpJaDbYSAmD9kTs1dh2Ew3 0IBrdUlOMHTbHjrEX7NE0ESjGtvYMe4kCKR9w9ovxQIDAQABo3YwdDAyBgNVHREE KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD VR0OBBYEFBTXnZytLW93IszwKUGgXXh7OdNtMB8GA1UdIwQYMBaAFHMfJfiA8Nb1 0G8rKcO/am9Ogo6tMA0GCSqGSIb3DQEBCwUAA4IBAQDCrwa/UTnBOoJ/X2FIfaWP g0llr0OSQx3l2RweWte6O90VB9AWTgstYRVErmYyXV70lYNp+HOPpxak1DEGr+P8 4REsMjX+odz9+UGOq/n5N+0VAfLTsQ9CG5EnHsfzwamgL/Ax3czrzgvmP4lz0tvp 07le9YUWkuG9UsUhN4/qe65LVweg8AfhihiijlcQe3WnrB7WvyyOZO81lBnLIIar qq62NSNxPWNa/TGX/og+E5HwTPcMNeMOsok5D5TQtx1zNX+2Zj7i2Py3ScTCCY/G 4U1nO4k+APE/3BKuvnrz8ZJ1UglT0lrVEkVqzW7TNCJpDz37A9QIQVLpw8oOr+Yc -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/regenerate_rdns_svid.sh000077500000000000000000000155741477524627100244510ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail # # regenerate_rnds_svid: just remake the certs in the rdns & svid dirs. # # We're getting the hard requirements down in scripts, can integrate all into # one all-singing all-dancing script later, so that anyone can regenerate # without having to read test source code. # progname="$(basename "$0" .sh)" note() { printf >&2 '%s: %s\n' "$progname" "$*"; } warn() { note "$@"; } die() { warn "$@"; exit 1; } readonly CERT_SUBDIR='test/configs/certs' readonly RDNS_SUBDIR='rdns' readonly SVID_SUBDIR='svid' # WARNING: # This data is hard-coded into tests such as TestTLSClientAuthWithRDNSequence # so do not "fix" it without editing the tests too! readonly COMMON_SUB_COUNTRY=US readonly COMMON_SUB_STATE=CA readonly COMMON_SUB_LOCALITY='Los Angeles' readonly COMMON_SUB_ORG=NATS readonly COMMON_SUB_ORGUNIT=NATS readonly COMMON_SUBJECT_OOU="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT" readonly COMMON_SUBJECT_OUO="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/OU=$COMMON_SUB_ORGUNIT/O=$COMMON_SUB_ORG" readonly RDNS_COMMON_SAN='subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com' readonly CA_KEYFILE=ca.key CA_CERTFILE=ca.pem CA_NAME='NATS CA' CA_SERIAL_FILE='ca.srl' readonly RSA_SIZE=2048 readonly DIGEST_ALG=sha256 readonly CERT_DURATION=$((2 * 365)) REPO_TOP="$(git rev-parse --show-toplevel)" CERT_ABSOLUTE_DIR="$REPO_TOP/$CERT_SUBDIR" readonly REPO_TOP CERT_ABSOLUTE_DIR okay=true for cmd in openssl ; do if command -v "$cmd" >/dev/null 2>&1; then continue fi okay=false warn "missing command: $cmd" done $okay || die "missing necessary commands" make_keyfile() { local keyfile="${1:?need a keyfile to create}" (umask 077; openssl genrsa "$RSA_SIZE" > "$keyfile") } ensure_keyfile() { local keyfile="${1:?need a keyfile to create}" local description="${2:?need a description}" if [ -f "$keyfile" ]; then note "reusing EXISTING $description file: $keyfile" return 0 fi note "creating NEW $description file: $keyfile" make_keyfile "$keyfile" } o_req_newkey() { openssl req -newkey "rsa:$RSA_SIZE" -nodes "$@"; } o_x509_casign() { local extfile_contents="$1" shift openssl x509 -req -days "$CERT_DURATION" \ -CA "$CA_CERTFILE" -CAkey "$CA_KEYFILE" -CAcreateserial \ -sha256 \ -extfile <(printf '%s\n' "$extfile_contents") \ "$@" } o_new_cafile() { local keyfile="$1" cacertfile="$2" shift 2 openssl req -x509 -new -key "$CA_KEYFILE" -out "$CA_CERTFILE" -outform pem \ -days "$CERT_DURATION" -subj "/CN=$CA_NAME" \ "$@" # We want these: # -addext subjectKeyIdentifier=hash \ # -addext authorityKeyIdentifier=keyid:always,issuer \ # -addext basicConstraints=critical,CA:true \ # but even without an extensions section, those seem to have been included anyway, # resulting in a doubling when I created them, and Go cert parsing did not like # the doubled X509v3 extensions data, leading to a panic. # So, removed. } # ######################################################################## # RDNS note "Working on rdns files" cd "$CERT_ABSOLUTE_DIR/$RDNS_SUBDIR" || die "unable to chdir($CERT_ABSOLUTE_DIR/$RDNS_SUBDIR)" note "creating: CA" # This one is kept in-git, so we don't force-recreate. # TBD: should we delete, as we do in the parent dir? ensure_keyfile "$CA_KEYFILE" "rdns CA key" o_new_cafile "$CA_KEYFILE" "$CA_CERTFILE" make_rdns_client_pair() { local client_id="$1" local subject="$2" shift 2 local prefix="client-$client_id" note "creating: $prefix" rm -fv -- "$prefix.key" "$prefix.csr" "$prefix.pem" # TBD: preserve the .key if it already exists? # For now, just using the same ultimate command as was documented in the tls_test.go comments, # so that we minimize moving parts to debug. Key preservation is a future optimization. # The "$@" goes into the req invocation to let us specify -multivalue-rdn o_req_newkey -keyout "$prefix.key" -out "$prefix.csr" "$@" -subj "$subject" -addext extendedKeyUsage=clientAuth o_x509_casign "$RDNS_COMMON_SAN" -in "$prefix.csr" -out "$prefix.pem" rm -v -- "$prefix.csr" echo >&2 } make_svid_rsa_pair() { local prefix="$1" local subject="$2" local san_addition="$3" shift 3 note "creating: $prefix" rm -fv -- "$prefix.key" "$prefix.csr" "$prefix.pem" # TBD: preserve the .key if it already exists? # For now, just using the same ultimate command as was documented in the tls_test.go comments, # so that we minimize moving parts to debug. Key preservation is a future optimization. o_req_newkey -keyout "$prefix.key" -out "$prefix.csr" -subj "$subject" -addext extendedKeyUsage=clientAuth o_x509_casign "$RDNS_COMMON_SAN${san_addition:+,}${san_addition:-}" -in "$prefix.csr" -out "$prefix.pem" rm -v -- "$prefix.csr" echo >&2 } # KEEP DN STRINGS HERE MATCHING THOSE IN tls_test.go SO THAT IT's COPY/PASTE! # C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 make_rdns_client_pair a "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2" # C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost make_rdns_client_pair b "$COMMON_SUBJECT_OOU/CN=localhost" # C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 make_rdns_client_pair c "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo3/DC=foo4" # C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com make_rdns_client_pair d "$COMMON_SUBJECT_OUO/CN=*.example.com/DC=example/DC=com" # OpenSSL: -subj "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" make_rdns_client_pair e "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" # OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" # WITH -multivalue-rdn for the -req make_rdns_client_pair f "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn note "creating: server" rm -fv -- "server.key" "server.csr" "server.pem" o_req_newkey -keyout "server.key" -out "server.csr" -subj "/CN=localhost" o_x509_casign "$RDNS_COMMON_SAN" -in "server.csr" -out "server.pem" rm -v -- "server.csr" echo >&2 rm -rfv "$CA_SERIAL_FILE" # ######################################################################## # SVID note "Working on svid files" cd "$CERT_ABSOLUTE_DIR/$SVID_SUBDIR" || die "unable to chdir($CERT_ABSOLUTE_DIR/$SVID_SUBDIR)" note "creating: CA" ensure_keyfile "$CA_KEYFILE" "svid CA key" o_new_cafile "$CA_KEYFILE" "$CA_CERTFILE" make_svid_rsa_pair client-a "$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2" 'URI:spiffe://localhost/my-nats-service/user-a' make_svid_rsa_pair client-b "/C=US/O=SPIRE" 'URI:spiffe://localhost/my-nats-service/user-b' make_svid_rsa_pair server "/CN=localhost" '' rm -rfv "$CA_SERIAL_FILE" # FIXME: svid-user-a and svid-user-b are ECC certs, but not expiring at the # same time as the rest and with differing requirements, so not coding that up # now. nats-server-2.10.27/test/configs/certs/regenerate_top.sh000077500000000000000000000161311477524627100232460ustar00rootroot00000000000000#!/bin/sh set -eu # # regenerate_top: just remake the certs in this top-dir # we don't (currently) handle any sub-dirs # progname="$(basename "$0" .sh)" note() { printf >&2 '%s: %s\n' "$progname" "$*"; } warn() { note "$@"; } die() { warn "$@"; exit 1; } readonly COMMON_SUB_COUNTRY=US readonly COMMON_SUB_STATE=California readonly COMMON_SUB_ORG=Synadia readonly COMMON_SUB_ORGUNIT=nats.io readonly COMMON_SUBJECT="/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT" readonly TEMP_CONFIG=openssl.cnf readonly TEMP_CA_KEY_REL=ca-key.pem readonly CA_FILE=ca.pem CA_NAME="Certificate Authority $(date +%Y-%m-%d)" readonly CA_NAME readonly RSA_SIZE=2048 readonly DIGEST_ALG=sha256 readonly CERT_DURATION=$((10 * 365)) okay=true for cmd in openssl ; do if command -v "$cmd" >/dev/null 2>&1; then continue fi okay=false warn "missing command: $cmd" done $okay || die "missing necessary commands" delete_list="" trap 'if test -n "$delete_list"; then rm -rfv $delete_list; fi' EXIT add_delete() { delete_list="${delete_list:-}${delete_list:+ }$*" } # Issuer: C = US, ST = CA, O = Synadia, OU = nats.io, CN = localhost, emailAddress = derek@nats.io CA_DIR="$(mktemp -d)" add_delete "$CA_DIR" mkdir "$CA_DIR/copies" touch "$CA_DIR/index.txt" readonly CA_DIR readonly CA_KEY="$CA_DIR/$TEMP_CA_KEY_REL" COMMON_X509V3=' basicConstraints = CA:FALSE nsComment = "nats.io nats-server test-suite certificate" subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always subjectAltName = ${ENV::SUBJECTALTNAME} ' cat > "$TEMP_CONFIG" < "$keyfile") } ensure_keyfile() { local keyfile="${1:?need a keyfile to create}" local description="${2:?need a description}" if [ -f "$keyfile" ]; then note "reusing EXISTING $description file: $keyfile" return 0 fi note "creating NEW $description file: $keyfile" make_keyfile "$keyfile" } o_req() { openssl req -config "$TEMP_CONFIG" "$@"; } sign_csr() { local san="${1:?need subjectAltName}" shift env SUBJECTALTNAME="$san" openssl ca -config "$TEMP_CONFIG" -policy policy_anything -batch "$@" } make_keyfile "$CA_KEY" o_req -x509 -new -key "$CA_KEY" -out "$CA_FILE" -outform PEM -days "$CERT_DURATION" -subj "$COMMON_SUBJECT/CN=$CA_NAME" -extensions v3_ca echo readonly CLIENT_KEY=client-key.pem BASE=client-cert ensure_keyfile "$CLIENT_KEY" "client key" o_req -new -key "$CLIENT_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" add_delete "$BASE.csr" sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_client echo readonly CLIENT_ID_AUTH_KEY=client-id-auth-key.pem BASE=client-id-auth-cert ensure_keyfile "$CLIENT_ID_AUTH_KEY" "client id auth key" o_req -new -key "$CLIENT_ID_AUTH_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" add_delete "$BASE.csr" sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_client echo readonly SERVER_KEY=server-key.pem BASE=server-cert ensure_keyfile "$SERVER_KEY" "server key" o_req -new -key "$SERVER_KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" add_delete "$BASE.csr" sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server echo readonly SK_IPONLY=server-key-iponly.pem BASE=server-iponly ensure_keyfile "$SK_IPONLY" "server key, IP-only" # Be careful not to put something verifiable that's not an IP into the CN field, for verifiers which check CN o_req -new -key "$SK_IPONLY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=ip-only-localhost" add_delete "$BASE.csr" sign_csr "IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server echo readonly SK_NOIP=server-key-noip.pem BASE=server-noip ensure_keyfile "$SK_NOIP" "server key, no IPs" o_req -new -key "$SK_NOIP" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" add_delete "$BASE.csr" sign_csr "DNS:localhost" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server for SRV in srva srvb; do echo KEY="${SRV}-key.pem" BASE="${SRV}-cert" ensure_keyfile "$KEY" "server key, variant $SRV" o_req -new -key "$KEY" -out "$BASE.csr" -subj "$COMMON_SUBJECT/CN=localhost" add_delete "$BASE.csr" sign_csr "DNS:localhost, IP:127.0.0.1, IP:::1" -in "$BASE.csr" -out "$BASE.pem" -extensions nats_server done nats-server-2.10.27/test/configs/certs/sans/000077500000000000000000000000001477524627100206465ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/sans/ca.pem000066400000000000000000000021271477524627100217360ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDBzCCAe+gAwIBAgIUc56AEpW980qCeFPAyYjDMQmgD4MwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy NDA3NTg1OVowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD ggEPADCCAQoCggEBALcG8Uka9njBYwtVEachD6ZzpiEZV8CVHU4lfXv2e0VD6D/z Q8DTWc7TRMy+15u/h8ZSNSQm2mNVDUA5SbJPoeka6LqXl47ZMhY9fwIa860KxphQ ODdwu75v6jjh2IZz4BW3PGjeVAN6pJfejmn07y/lGa+jDvKPl0BEee+Vj2APpDu0 JRtnNoLvJryuddQq9AaagPXP5wsvS5mRVqjM3w9dvvxGJUZW+nEKDX0cEw3nv1h4 K57Z/fIHOk9cTH2b960QG/MjadJi/8tvVduBRt6ZZj9M5xQ4dg5+28lJbVq4eoAi lN4XIbah1tg2q0zWBprYieVLWEgT6680rFMrI2cCAwEAAaNTMFEwHQYDVR0OBBYE FK442m9BmE189lFzd57Hbz/b95psMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57H bz/b95psMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEx7HfRg QoInmdQwN8IKVB21RU7gdkFT6DxmS/ZEW3SFXctrkPftyToJ3PoFlp7rYooKCI8e MPOo2GeSt8Ocv2A9qgV+X5euZzKDrOW53kvk6BpvqiadfmOjw/aa4HwjSDh3lUwP hgQWriNYK7NBXINIxHXT6tRv9NvIvR3UeJ2DBujwAF5rBmTIBI9b5N1oQnE2jeY/ 2LsV8rcnWRcYEoyhb55AJ5aD640saGJ0StmdwRK9nOFzg02xUP4z8QuentlwPVVo DhCGGbMLY9fneXjrD4jEUjAHn4CWuPLdn/agIrC96qv+PZYUl9hJoDJnshuPczyR UQFYI1rGa9JuwRw= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/sans/client-key.pem000066400000000000000000000032541477524627100234210ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkjw4sZLkZJPTP /c4gIPXr0IacaCvNJne8fivl1FnVg0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJm sjkCGVSsNTSY3yqZCM2mLIgQmaSCuKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVT kJd2qDALXh6cHI8esCigBAZsnUS90mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PA SIakuGFvCWgv6uS57BOgn+c3134fCYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfy vkXIt51WoMm04gNymiVYYZxbNpe9M33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG 8J8Rzl39AgMBAAECggEARvhc0FAeYb5l/kezP6q8chL4GY0HPNabC36qJt1/X5s6 s5rM23JjAYz4XfSG9P5mTGI6JEvclgRazsS8ivhOoIKM6IMzeP2Qze65+V2cBbNO FhIl9RKOkPADjfwgaLdhHISBOzBilb/NxFr7jS7AbGc3XgZIMqFEnBtj20oiVEZu l9BAoiDlAiMKt18AMGxsI6Kr5Px/vuNhQpxjKDw+47wbhHP7EW+6ywVFn+Nb9kV+ 8QC/14GZ9EbDPfKhlU+NqTIAKZbvI+NvAyETB8ezEnEkEcSr3FaWhA4WpHauhijx qAJSBVsVinoFQVRHwDjZGDJ+M2QkFk24Jm+X3VQySQKBgQD7qFMRYF7+KbXBp8uK YpW3pyM+28oHaeVjKckCtFuAIJS02yYo2neANPeVIthurBwl4pRSZchNiqW70nSW aYPw+4JCprYCkfNZFFP6X5FzIlmGRCpgjuWzXHFgkSRnOg+GckcrMu1VcvVYuosC KqQWkXSbupvrQTK1f8XZQTnR1wKBgQDogLG++I1+wLUFQmPw6c+RrTwj9i1SUEt6 50pUp+9/KKDXDQTyjV/XZILmaLcn4jcqCXDIvEddkEUngkrazS6quwQtB4wpQgYZ SSQ/eWsRCvA0VUI+6tHFThPGg1SZREXorhflm7YYa2nsLF118d1pHqKO/gUhBJX+ Pv5Pzdy8SwKBgQC5E7PzxZJXDcnUIFk67wH4zPzIz8+m4CSJZ0Ojr4zTkCKNV9aa mQDl0w56KeROkFkrK5W1e5FyJZN5rG995x/X7MCB0CVvgnMbgi24puxLZmm8qwkX dkBMRqJDSLsjB7o/QHBCvvN4slDp7lcpQr7msha2KOlefNaUUOHqw2OIhQKBgQC5 4go5kCYv7InNRqL4fTYCVen7JlpdsOxnunrm68zCcQ0GYdZOxVCWuDkfVSD5thY8 eYe+NSkpWKqxR63o+JoSzaotBhe89JhDpwJf7Qb4fTJF5NQt0Tcc86tDzsPYNYle 2bEpVTBknZv1whKGtXQ7Es8MW3JmT3BL8LkJvKB5uQKBgQCIUUm3zKTXo4fxfR84 485La1x9ThnOM7aDgLhKJLg0hQTMPSozsKS2wbOIbg45BS9BL/HGkjqHXryY35Ee wphElPZGLtw4bd7xjSQykXjsCHJIg1lxnEG0sdTaVBD3iQnx4L2pCg8fY0iUC1m4 KnOwsB6IfaV1EnouUebes3ydjQ== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/sans/client.pem000066400000000000000000000022141477524627100226260ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDLjCCAhagAwIBAgIUTOgYUed1dyp+Gan755liL7b29wEwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQDkjw4sZLkZJPTP/c4gIPXr0IacaCvNJne8fivl1FnV g0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJmsjkCGVSsNTSY3yqZCM2mLIgQmaSC uKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVTkJd2qDALXh6cHI8esCigBAZsnUS9 0mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PASIakuGFvCWgv6uS57BOgn+c3134f CYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfyvkXIt51WoMm04gNymiVYYZxbNpe9 M33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG8J8Rzl39AgMBAAGjdjB0MDIGA1Ud EQQrMCmCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAd BgNVHQ4EFgQUiUGJiS99EVH6/iPsQESqZSfXxCIwHwYDVR0jBBgwFoAUrjjab0GY TXz2UXN3nsdvP9v3mmwwDQYJKoZIhvcNAQELBQADggEBAAugrGg7+6LX7UZ6YhCM P/CGBnxIMODeuikNf3OAEjwNRJjdcy8OyHKd9pSwEzNZ0jZCN127UDPHQzZGw5JE KCe3B4a0Lh0G5BQVOOTcUJM9et6aFtBDvNbKhiiwi11d5aAEQX7k3ugPPaEjP9si SM1b8UBcaO3mGSnveMVjBiInzSkJdc8kPaAsssqvledmQa9RyM1y3ZEcvixbF+6d pfQG61UXR3rnGEUAoHWmnKzJ+o/kFlThd2jtKT8T4qp2Ws1ga43V5kacFqyWcpfy 6dK1lfr3Mx4qHMlnFp7O9L4ciFLtLiv4ua9fi4zH8zS7EALkwQKxhaDHo6Afi1mI ASw= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/sans/dev-email-key.pem000066400000000000000000000032541477524627100240060ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCoFArKsSQLasob BGo4+d2WUxjW3rp7qMZ2M92RvefInqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM 56/QtNx6fapKltO8FSOs7dSWLQquvOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9 F22R+MNLhCYacSytogTz9ZoCLLuqfaAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT 7loxW9+VDXN1wKP3U+ziHJVPX2+438OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQ qEsY+mcjh/+JZk2xbhScSB/bsZFFIIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mS yH1rT3x9AgMBAAECggEAGk4GsVm/XyUPq+JJQJEEZ7gqGUjC7UMJCmtMGDjdK2X/ s8thZKjpaZEO8MfcsGQSRGutAo5XT4c8BQImnzaLEgWUTvejfBpf2rrfDEDQ3O+W bINSaYYVIk1gX7hMwFZBVaCK48d5YLOMSW8u+Aszf2BXeUhwml0Swt6Tg5ceKMye fZa9geVZhC61Gtxbk+L4P4cnKLAvlxjFSn3iAS0YNGGsIVIHmpFQ5l0JdMLUF/4h C6enmgzs9fxnL67Wy54MoICH4qEekW09AHbDI2byKgBNEmmm/xsQlRGoHdwo3QFV aCDmul7V9LX1DMs3aamzr/+jw0/4bixZvRN9zGgH6QKBgQDraDBtrY2Fk+uFNkPL zf29H3+W0bibfOKSoyTkvYHYxK44sYAN/ly7BRkH5CaYiTNQWJ26lVwi8nanSnNV Nl7Hevx1C9Yn++guQtnUNOBvz1CSuRQ0L52US/YF4NKmnVN3Ny4DO99+bHXoWESh 9mMlerG2Axr/FKf8E8bdG81RuQKBgQC2yBBGAJolYaGZyDMgu3CjXuJP1hPuePtC uXa2hp/uTf0D0bUg2vxDJjoZVSBFmpSoKaybsUip8gI3GFjJXGmu+No3gX8bRDaq H42+I507I1wbYqaWtVW8HmveAK3qi5ed886p7JGYZW9olTj6sfRlvoWa+P5A/9uz AUTOlMly5QKBgQDNVNfJEvS6aseoLij8f/SvHeZgWxW3KjtGxF7N1i6IMSX19X1I t3GS/2NR6sNvkVzc3C7YLKdtJCgyy1HGJeKOBMxoG6b0wVlH4K+31Vder2oMULs6 ub2tOISjo/KZueivt8W+tF7BG0HNJBDZZNweOOMBa7wEerP7wBRZkIKKoQKBgQCt +JpUproRHm4b2wue+glp1iP97TsnXgt5JOGzNUwAHEbYXb/St9wnZbki531CArG/ pXre7czFxM0K96d6cPU+TyoUbrM2lqSZJFNbSLac1TkT77+z7oDd/u6YbXkbpyX2 d1qbLcoejV2O44lKRBrkxISSTrBh2aWZKXn+Tmu3aQKBgQCu7PIEgAfX78QrQWje 2owrnV/3DQCtBOh1PyQ5a4LgxsSuvO+qRdsK07tYVxKLc5TOq4DJcmbIzrkEptfO WqMUoRk6BT7EfE7AIm2W3K9ohy6U7Jsej7YxhqYvMZllrksqluKbz51K/q1xDDFa kkwCvAfMIRx8Z8+2lZkc6yjGug== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/sans/dev-email.pem000066400000000000000000000022401477524627100232120ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDPDCCAiSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29wAwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQCoFArKsSQLasobBGo4+d2WUxjW3rp7qMZ2M92RvefI nqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM56/QtNx6fapKltO8FSOs7dSWLQqu vOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9F22R+MNLhCYacSytogTz9ZoCLLuq faAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT7loxW9+VDXN1wKP3U+ziHJVPX2+4 38OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQqEsY+mcjh/+JZk2xbhScSB/bsZFF IIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mSyH1rT3x9AgMBAAGjgYMwgYAwPgYD VR0RBDcwNYIMYXBwLm5hdHMuZGV2gRJhZG1pbkBhcHAubmF0cy5kZXaBEXJvb3RA YXBwLm5hdHMuZGV2MB0GA1UdDgQWBBQfZUwRFVCxJAU4eNYLvovRmH6WezAfBgNV HSMEGDAWgBSuONpvQZhNfPZRc3eex28/2/eabDANBgkqhkiG9w0BAQsFAAOCAQEA dVs5iZ1SPtR7fGQoNnMocd2GECOMZQ5cQz27GtLb8jKVbblBjWYNECUpHA44gWhx YhT4gEBww0w7FeQEnWyNMj8vidEm/Fkz4c01zDqwCd5Lh4UyQlWRwppHnV9+j9Xa Ye2T89L9txLFzkaHZDhbZJy5/SDs2lRy+6IE1vxmiMZdVIzVzHhLQUIBfC1gh0Yc qfn2RVbulW11CUgSNXj7G5TPkHw/RG/vs3AsDdgXaCKfwttTirrpFl4OvGKH+xSV XMykd7Ck3ScD3oPpp3oY/Nq8iHBB8EgB6SZSSVVL9zl/4jo1s5IBr2ekAhSv9HGj 29VtK+VVixHazzkGdvS/OA== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/sans/dev-key.pem000066400000000000000000000032501477524627100227150ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdqfE5jKdpe3cJ 1Pd2mwBtfaDnaGsfOxyy7VsIFMGdekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/Y GzlntFaAZ7cZMaeCuce6HsE/vofOmMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8 lRCQLVqHwkuhjlcr8r0A9CWfx0bc2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD2 7X42IX3tUU5sljVbDd3TFKSyS4l2EebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoH ftyCbvG/Rxd8LlAeWqgqkdKXUj8gsL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq 6bvzPXoDAgMBAAECggEAOHkfRQhbDN7jCBwG2a5yjEV0BX9y41hmkraPJUleApzD HbFraIXW+zXcWjcVSUDbLat5CpamZWHnd2Zk/P7s/whnXrhY8mu+HS4X9U+3UGqN NfINHIrlcZqSak5hQVXeA8uL0v9zsFiU9ckFCkechkUzlvYnxj+nsHhDI8Sa3s95 5POvYg1VrxfTM7yq8jo9hNoCKd16ghxPXHLHpimZ32qnmje0GqGHvQD1aL83fcaY SjSI0OhVf+rkKy8uy63xp20rKj+zCMmLIuV9AiFapiznp5ui7+WAH6oEjMjCzCKH SzE4arFgYOjHDF8ElMDhzyOrFg50uGA5Cl1kfn9x2QKBgQDXNSqqZQNCFU5Ah7K5 wLSedQkHmofN8moH44YXLlK2zfEknXsmu56qoGBRlIVPwYqkXTUg9uDiv7IzE6Fh eADCB+hiwnDPw6vB17aLJc+uE+XtLYWjjPMpOHyR4HAbgq/s3GGzAxqvtPOLrc26 gzJReRpN/0/7sodj271LitbCuwKBgQC7jH4SiLuRZWobOrFNDtPBpPBvA56/YNWe 0gvHfWYCPwt14aF2mkTo+wvY6bzSgy+KmRjCXSyQD9/5GOUtSCIrM5rQvqI3iYMw 8kkJMDB6b7GEustNydGgpEVl5KqfwUzy+asOnjlqVO3FZSwvJlLNYVtgjRzGWsZX 1pcL/0BlWQKBgQC3u5yBfVwavzXfwfrEu9F4t1LOOxBWs+/ybD2+7L2RdKG8MPdD SktcQS/6dmKahRJo4WrKifvVmvP4x3mwTVPYVAgCyR6nQtcQ16nxgoaciEB1Dbha uaugNamkoYkU865e+ogu3Sebe1GynuBVrEz5YfsjPCZ9LR6KEsC6Df7soQKBgQCo aNrYh8FuKRPjiYumN1c8/oAMH8MP9MM+Dz0WkGrmP3hqDQaw+oxAbRXRXOn6WmR5 X0pVVddrMWYcRxeb1rcf9gHhyhzeI/QTIq0kvAn8F4nfNuDSZBSB3KCYg4IXXDtd v4Wz5a6G5eZwp43KdO5LkE2+YFhjYSXTwFT4J+fKWQKBgCyoF/Za1xghz/+dmH/K vZtARsJgZjWIknsBdTwPEcqZjV7OBe7IKRutPgVnzHGt+3gWLqXQtz9uLxAMw/8B k1JQuEKEeuAltA2bE6vp8NXP7kC38oR/9Z7aHKru9+8nrrG/dVK28PrSymdd4keB J+vQW0oEllTH7M7Wgyy3r4O6 -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/sans/dev.pem000066400000000000000000000021731477524627100221320ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDIzCCAgugAwIBAgIUTOgYUed1dyp+Gan755liL7b29v8wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQCdqfE5jKdpe3cJ1Pd2mwBtfaDnaGsfOxyy7VsIFMGd ekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/YGzlntFaAZ7cZMaeCuce6HsE/vofO mMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8lRCQLVqHwkuhjlcr8r0A9CWfx0bc 2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD27X42IX3tUU5sljVbDd3TFKSyS4l2 EebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoHftyCbvG/Rxd8LlAeWqgqkdKXUj8g sL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq6bvzPXoDAgMBAAGjazBpMCcGA1Ud EQQgMB6CDGFwcC5uYXRzLmRldoIOKi5hcHAubmF0cy5kZXYwHQYDVR0OBBYEFDvM qcu0rwkeE32c0A2VgPLmf4eDMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57Hbz/b 95psMA0GCSqGSIb3DQEBCwUAA4IBAQBxVdfxsnSvL3WTuZroo+inViR8T7RV2Dys 8NiwvO1o+uNATmzV5v2iLux2INEXBgt+71OqzcPPiTvjHcBkh6aTCk2N54mYRK6Q L3d7wfeYJhHUfeC/idM5c8UQVHkKa7Em+6D/IoWnzen6QTs2CtxJ+UE4Ie56DTHQ 1qc96IqNUKVAq61tZe9u89sCsn9S5r/cyBa8pTMzfjZ6FHJFC/MZHu8InfNJ13MB q5edoCH761+MbyXz/JyrU2hTcno0z1ZSS6VteehDPdnDPjysoZPCjn1jP7G4rZ3W eevtdw2p2SP+jdFt5Hy6tvy4sXtj79C4MHVbsAVw6hkE5PNoNRM3 -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/sans/prod-key.pem000066400000000000000000000032501477524627100231030ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQhGgjBMhK3sIJ IAKwXPUd/QrxbHQ+fGkXsHm9EIMzm9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0Aiu V+tHoilSzEZhfTRsVz+CPDBotyW8rK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6 Y0+klARJFsOp2kDhCyjW2Vz+M4pJIMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtn U4uypTXqKAYUQX88TS4mCBgxPYh7XZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUck jPJkJNBaLcGgAJq3Msd7t2kXulmIJTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyF Le6T6NopAgMBAAECggEADzlRlJfzazL5Gfs0Mw3waAZ68246jPrenegItV70o1Ka QB2cKDHDTxi8fWHi5oB/2zhJLPrHxuS7s4XwA8iBSZ4oN9MxPgU/Oeoc0zJm/w/g 9a2jh8xhUYpGwkiiSthjIZO7EEGiJOG5AlNQo+iaKP9wXPD2Sm/7BPe1AErBSo4o IyXhnx3e32sGAQekR/1AqAl2elXsFeO/nTYOBABd1Tv3RQW7JFtSiDKIIQV50Vhl 2kdFqRlqfEt0JJeRB/hJZloQrKNOR5JUymn497kvmHeLOzNMq7+65naY86BLy+pu B6uBGgo3AZibfB8R95YpPTj+hn5YsGzYztMKRZ5PUQKBgQDwjXDqTMRdtoWVblvF 89PY2bF4dvyo1WW5FWN7kbjjOmwz+rhsc3ct8IDhxdTh84gxv6+nQfNaXCmPl7LP Y4Qe6PC1GlhgC/XP0CSb4YhqIlk8Ja1yLOVttT/u77Fd9WRLYHWQqWBMVJvPg+9q pdXsfIvjQg3cYWrM8l7wzKuJEQKBgQDd6FJ0H9bTFYnMc60BHSujd/xTlxEgkS/2 c+cFvVJ+Z8+kmNRxLWGeuj/pFh1Lg2o1dP5WH+1oT+gq50MHNyAqLP2+mY/iZTaD xp6+ChAZpg6FLxKfMr6vv8rfA/6259RjqrGKzVT7ZD9yHAkG86n+ZHvXxBIkVEOF WslsBhz/mQKBgAKYVAaDC39DT8+b4CuThM6LEqoNBo6/tpg3jdowaUEySSaKZuDs ofWB6bIF01UYrnhy6+8u+/QWmHAvH8Oy/CLlOmuJvDhVCLLUOQDhfvo+Ip5Ofb+E knkLgoQOW+h6lDln8fy8hwhrxT6I2tVYsqUZdtzdwo6FEoThfHq2iocxAoGAS0Ri uzqVbhFvJObTdn5Db/XhoySpTYeRYiGb6Yl2sqNZfbXM0PqYkvMPfGrg0t0nKCyf m+zGZMw9rle0l3zuLwAFrSF++UZlQDsdWs4k+d4mLKxzv4XBwfDmydAcFEP7+TZL dA4DViWBk53Ivg/lyACjNOMvWB0wrcnGahqEuXkCgYAh47V2/4fEHLWwbsnvWL8T 6C56/8RUBTvlF2vFtRA/THmYV34pGoM0EWLQ8th4mQQ4mPo6JcQ6L6K1zjw87RJ2 zvlWhCNZdOUFtrsoVwuRsP3U8PW+qYiDasbNHJoy5UC5jTI5byluM802Vw+9uQ+l wHleMjtW69bGbRbHWDOVvw== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/sans/prod.pem000066400000000000000000000021771477524627100223240ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDJTCCAg2gAwIBAgIUTOgYUed1dyp+Gan755liL7b29v4wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy NDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQDQhGgjBMhK3sIJIAKwXPUd/QrxbHQ+fGkXsHm9EIMz m9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0AiuV+tHoilSzEZhfTRsVz+CPDBotyW8 rK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6Y0+klARJFsOp2kDhCyjW2Vz+M4pJ IMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtnU4uypTXqKAYUQX88TS4mCBgxPYh7 XZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUckjPJkJNBaLcGgAJq3Msd7t2kXulmI JTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyFLe6T6NopAgMBAAGjbTBrMCkGA1Ud EQQiMCCCDWFwcC5uYXRzLnByb2SCDyouYXBwLm5hdHMucHJvZDAdBgNVHQ4EFgQU Knk97WhPStd4LcWf52kZxmG4PVowHwYDVR0jBBgwFoAUrjjab0GYTXz2UXN3nsdv P9v3mmwwDQYJKoZIhvcNAQELBQADggEBACiIPya5Mgy2y4+PrBv6KbYBvpSMA0+g 6OCq3pX4+okIGBoytjL+0DJchEAVGHxf6qnMWSQKnBFgjCHbi805JgJu819PDWFr lyWsH/YuWP/Df9zl6KiXbrLUBBnGT+bxaqc/FVEsJrhB+29kIrDmxdcL/CcmihwF pAeYMDs7zjn3W8BQiBG+odapzP7VHsA0L/HpWOZrvJnSIU4vAujnn6SYRyk5y1T1 7o7nSKo7Fh4YjteTUGzOKFKDS+Mk3SxXA+TIqskV0ZWgZD4KebN69VZWRXgDwTpT E9jt46Yzmk3RD1NsrfwmgNSGet1+mOzjOLzzXjlxl7qmpYvbyx8U+9c= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/sans/server-key.pem000066400000000000000000000032501477524627100234450ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLLgBYDBSeNemh 8b9yPmGiaJkSkJtaXL7ESe1h/aNpdCRrkja7XdHqQ20a2H9X94nXuzvFbpJw8VGT q8ifGet3+++AJvEP+LfK7HSIvnyewKXZ+Oqf5AJg1YIb3k31VGzGEjjjnW7F/Gw4 2qKyZ7+CEnIFbo4lLFBoTGvzJo2OwDVKTCby9E428te7rH1+F0OOANSG+SLzl5Vj bidr4OryTPjO2qJSs22rVnvjZdcG38Z1ams6s4fxwQWvuD9qBBrb0GGtvVP2cYBe XfOTOydS5IxOerb0EbbijudXVh+xH3dRgvfeJZWTqegCktT6OEa1pe7Q3BoMLtk0 HJO77RtrAgMBAAECggEAMe91YC5f5t5jNE0A+2f/gPXqIRjvRY8jx4RP1lSLFADa bqG51+TgEY5Ow1lQOuN4uk+nCgf5784veXav0QKCz7NYkot/lahQK4xfu7ftUusQ F/lIWNGmD/yJhKXnTRNZxHh4COEJd5lpU7PQZr5+383+vf0E7HfuryORKGpiQiHl F1ZOYFG2GjFZychMTe+ggQISJqX8U2bE1mIhZNSkBvStt9LDIyWsvbud3j5DHPv/ nX3EEymKwDiMrkNc+HTodJ3Ci9cSVGJ6igAODeu5wwddn9rpP4obOvJW5B2NuNXR OirU9o4Hd4cPcUqE8FHloEmIKmALqCQgH3gu/+euUQKBgQDoR4IMFRe4d+G/FJLE MUnQ1H1ipHOaZcLl0bBFeEtkpuqC5MDoCDJuQCeNrohSrE0dh4z15lq8G79eucYH E4vbmsyXAm+KgpYZ5S0wvMDoPaXxOSUJe95eUqeGDl3vcexSOXph7/syxXft7MpA JGFirKH16WiKn4u30siaNK8feQKBgQDf7b1pEdUr+ajhrJ0+LwC+/Q12nBgrULvf N1rOBXrXX+yI0esk0/i6EWThTZP0iyh/CpxZQyfsqQXBG94spJLGa/C3dcOWohQ4 tTIYZN9uSozO8OXU+WEHM41kdOByItIpFVzJ2VQ711NizZxchsVUPexUxW99rxuy JYPe891lAwKBgDX1JoiX/cKkVpSEuvNIX9VbByV6/j/Hk9a/NytsHldrTt7JNOax wMVLseR/vrs1u1Q4wM3+jOVVrMbccNwD9mE1PIF312FdpHCvhCfmMQnCCGJY2/mw BJ/0o2XIwJl25WbUY1GM5dWNeaaTcwgja1v4rkbpyZcqKNKy7FamFmhBAoGADTg+ z14jJJZ3luMW4V3rAFt2GQN8FcqNXM8qyDDgoKhkEWu7IxB0hZ+TQg3PZ0dz25Qn yMlDKSCNW2omwqHwnFAxBNOp+VGWOgbQb0o9OoiiKXFlUB5s0P0FIuIXggV2PMCr Gqt+4o2CubmDup6mNaP2OMbzHwtgajL2xwnEjzsCgYEAhEQwa0t+x2+/vZ1NOQW7 hp7vM2/DyAZnqgAdCwXnlmVayxR4r9OSDZmz5vqJWamQ0UIWYnTkvLN962hYdIC1 eIQ3yEjxIgZj9Kb+6vnHI81kkjg8LK/XG87HFBnL9juNgdbNQVIyMI84FM2jm+AP xReBzjjjrY3NrcJ8B6a4AUA= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/sans/server.pem000066400000000000000000000022071477524627100226600ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDLDCCAhSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29v0wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy NDA3NTg1OVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF AAOCAQ8AMIIBCgKCAQEAyy4AWAwUnjXpofG/cj5homiZEpCbWly+xEntYf2jaXQk a5I2u13R6kNtGth/V/eJ17s7xW6ScPFRk6vInxnrd/vvgCbxD/i3yux0iL58nsCl 2fjqn+QCYNWCG95N9VRsxhI4451uxfxsONqisme/ghJyBW6OJSxQaExr8yaNjsA1 Skwm8vRONvLXu6x9fhdDjgDUhvki85eVY24na+Dq8kz4ztqiUrNtq1Z742XXBt/G dWprOrOH8cEFr7g/agQa29Bhrb1T9nGAXl3zkzsnUuSMTnq29BG24o7nV1YfsR93 UYL33iWVk6noApLU+jhGtaXu0NwaDC7ZNByTu+0bawIDAQABo3YwdDAyBgNVHREE KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD VR0OBBYEFG5wB9htqpiQze/q1+9rPaq1HlcpMB8GA1UdIwQYMBaAFK442m9BmE18 9lFzd57Hbz/b95psMA0GCSqGSIb3DQEBCwUAA4IBAQA1tni0pG6LtlStmEUmn9xJ mcWlWgZtmUbZD7aE5P+SFLt02TY9ea7IoMy3Rv1yRihxLgURNB9Y8aON9IrraVRf U+EizfG9goQQINs652TNMoxjNEwbF19TzgY7k445gkcl8aZ5fGJJhxP7UGBC5xYL 1Cx0uINiqs0fb3pPrcvhxD+I5tHDwJY7maW0fks/JD1M738hl5zpV1POMvarDFW0 L4LUDweUuNeylOWyCBv6WFV7avSsMmoQmUvUZoPqp2gk0A17pDeuufI84fHhEghU UigXcElvZkaoThbE3YA0Hq8UJHSzAe8x0bDSRz4WfNzeqq1RtdZ+H1uPoMIJbZ4D -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/server-cert.pem000066400000000000000000000130141477524627100226450ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 1d:d9:1f:06:dd:fd:90:26:4e:27:ea:2e:01:4b:31:e6:d2:49:31:1f Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:e6:fb:47:65:cd:c9:a2:2d:af:8b:cd:d5:6a:79: 54:3c:07:5f:eb:5a:71:2b:2b:e5:6f:be:31:fb:16: 65:68:76:0e:59:e7:e4:57:ca:88:e9:77:d6:41:ad: 57:7a:42:b2:d2:54:c4:0f:7c:5b:c1:bc:61:97:e3: 22:3a:3e:1e:4a:5d:47:9f:6b:7d:6f:34:e3:8c:86: 9d:85:19:29:9a:11:58:44:4c:a1:90:d3:14:61:e1: 57:da:01:ea:ce:3f:90:ae:9e:5d:13:6d:2c:89:ca: 39:15:6b:b6:9e:32:d7:2a:4c:48:85:2f:b0:1e:d8: 4b:62:32:14:eb:32:b6:29:04:34:3c:af:39:b6:8b: 52:32:4d:bf:43:5f:9b:fb:0d:43:a6:ad:2c:a7:41: 29:55:c9:70:b3:b5:15:46:34:bf:e4:1e:52:2d:a4: 49:2e:d5:21:ed:fc:00:f7:a2:0b:bc:12:0a:90:64: 50:7c:c5:14:70:f5:fb:9b:62:08:78:43:49:31:f3: 47:b8:93:d4:2d:4c:a9:dc:17:70:76:34:66:ff:65: c1:39:67:e9:a6:1c:80:6a:f0:9d:b3:28:c8:a3:3a: b7:5d:de:6e:53:6d:09:b3:0d:b1:13:10:e8:ec:e0: bd:5e:a1:94:4b:70:bf:dc:bd:8b:b9:82:65:dd:af: 81:7b Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: 2B:8C:A3:8B:DB:DB:5C:CE:18:DB:F6:A8:31:4E:C2:3E:EE:D3:40:7E X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 54:49:34:2b:38:d1:aa:3b:43:60:4c:3f:6a:f8:74:ca:49:53: a1:af:12:d3:a8:17:90:7b:9d:a3:69:13:6e:da:2c:b7:61:31: ac:eb:00:93:92:fc:0c:10:d4:18:a0:16:61:94:4b:42:cb:eb: 7a:f6:80:c6:45:c0:9c:09:aa:a9:48:e8:36:e3:c5:be:36:e0: e9:78:2a:bb:ab:64:9b:20:eb:e6:0f:63:2b:59:c3:58:0b:3a: 84:15:04:c1:7e:12:03:1b:09:25:8d:4c:03:e8:18:26:c0:6c: b7:90:b1:fd:bc:f1:cf:d0:d5:4a:03:15:71:0c:7d:c1:76:87: 92:f1:3e:bc:75:51:5a:c4:36:a4:ff:91:98:df:33:5d:a7:38: de:50:29:fd:0f:c8:55:e6:8f:24:c2:2e:98:ab:d9:5d:65:2f: 50:cc:25:f6:84:f2:21:2e:5e:76:d0:86:1e:69:8b:cb:8a:3a: 2d:79:21:5e:e7:f7:2d:06:18:a1:13:cb:01:c3:46:91:2a:de: b4:82:d7:c3:62:6f:08:a1:d5:90:19:30:9d:64:8e:e4:f8:ba: 4f:2f:ba:13:b4:a3:9f:d1:d5:77:64:8a:3e:eb:53:c5:47:ac: ab:3e:0e:7a:9b:a6:f4:48:25:66:eb:c7:4c:f9:50:24:eb:71: e0:75:ae:e6 -----BEGIN CERTIFICATE----- MIIE+TCCA+GgAwIBAgIUHdkfBt39kCZOJ+ouAUsx5tJJMR8wDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOb7R2XNyaItr4vN1Wp5 VDwHX+tacSsr5W++MfsWZWh2Dlnn5FfKiOl31kGtV3pCstJUxA98W8G8YZfjIjo+ HkpdR59rfW8044yGnYUZKZoRWERMoZDTFGHhV9oB6s4/kK6eXRNtLInKORVrtp4y 1ypMSIUvsB7YS2IyFOsytikENDyvObaLUjJNv0Nfm/sNQ6atLKdBKVXJcLO1FUY0 v+QeUi2kSS7VIe38APeiC7wSCpBkUHzFFHD1+5tiCHhDSTHzR7iT1C1MqdwXcHY0 Zv9lwTln6aYcgGrwnbMoyKM6t13eblNtCbMNsRMQ6OzgvV6hlEtwv9y9i7mCZd2v gXsCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU K4yji9vbXM4Y2/aoMU7CPu7TQH4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI hvcNAQELBQADggEBAFRJNCs40ao7Q2BMP2r4dMpJU6GvEtOoF5B7naNpE27aLLdh MazrAJOS/AwQ1BigFmGUS0LL63r2gMZFwJwJqqlI6Dbjxb424Ol4KrurZJsg6+YP YytZw1gLOoQVBMF+EgMbCSWNTAPoGCbAbLeQsf288c/Q1UoDFXEMfcF2h5LxPrx1 UVrENqT/kZjfM12nON5QKf0PyFXmjyTCLpir2V1lL1DMJfaE8iEuXnbQhh5pi8uK Oi15IV7n9y0GGKETywHDRpEq3rSC18Nibwih1ZAZMJ1kjuT4uk8vuhO0o5/R1Xdk ij7rU8VHrKs+DnqbpvRIJWbrx0z5UCTrceB1ruY= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/server-iponly.pem000066400000000000000000000130011477524627100232160ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 21:8e:d4:f9:10:fc:32:5f:46:45:23:53:14:03:f0:b2:dc:a1:d7:e5 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=ip-only-localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:bd:98:dc:14:49:92:43:1a:73:36:75:6d:c1:33: 73:b3:e7:f6:8a:e2:03:f7:24:93:eb:36:da:8f:64: c6:56:19:b7:92:d2:f1:df:d3:4f:7e:78:ce:db:dc: 6e:d4:5c:58:a4:a7:79:6b:49:ff:30:37:d9:71:92: 21:d7:91:72:8d:1a:21:75:0f:80:23:f0:2f:0e:6a: 1a:62:9b:eb:1e:74:df:df:61:8a:05:a8:31:b7:d4: 97:ee:c6:60:1b:6f:b1:85:9b:ac:3d:cd:a1:9e:a8: b5:56:84:ce:9f:8c:64:a5:57:41:75:81:c6:00:c8: d8:3d:ef:85:9c:78:04:49:5a:94:29:af:27:0a:be: 56:60:6b:a0:52:06:53:f5:cf:ce:47:9c:cf:50:8a: 1d:92:3a:21:d1:24:ed:81:e8:85:97:8c:41:46:96: da:02:d7:b7:f5:ab:92:4c:ab:0a:e2:19:9f:b8:21: 07:5c:f7:d1:7e:6e:49:70:a9:c4:45:c3:b2:10:ab: ca:50:e3:c9:ea:6f:b5:19:64:12:32:0e:93:59:f8: 8e:fc:a2:bc:33:f3:25:e4:59:98:6d:0e:97:76:b6: da:9e:50:cf:4d:a1:6b:62:ce:47:84:1d:f7:82:8a: da:d0:90:e7:a6:1f:43:25:61:57:c3:15:7e:e0:b8: 5c:17 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: 0A:AC:82:F2:35:73:89:FB:49:07:EB:9C:87:72:D9:91:26:DF:84:8A X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 87:00:2d:ec:ea:ab:13:3e:21:6c:4a:3e:f1:9b:c3:ab:c5:81: d3:34:2d:cb:f1:e7:98:96:13:a3:7f:29:00:83:70:20:81:56: a1:31:b7:d9:fa:35:02:f4:6f:32:8a:58:cf:68:39:e1:f9:fb: 50:20:60:85:f1:bd:22:73:e6:ac:b6:36:38:75:86:de:8a:cf: e5:41:df:0f:bd:61:94:3d:3a:59:81:50:c5:44:39:47:d9:e8: 56:2a:b4:2c:59:b7:89:f5:5d:a7:28:15:6e:98:1e:47:47:7d: a6:35:65:de:b7:06:9e:04:67:98:04:f9:57:54:82:2a:dc:e2: 6a:76:df:06:05:b4:61:84:ec:7d:5e:46:34:ea:14:b7:14:16: fe:51:0e:65:52:c6:74:96:8e:30:3a:b9:c2:d5:05:2d:a8:08: 11:ba:7e:65:c1:89:c3:61:ec:d6:06:e0:7e:ac:11:ee:18:89: 67:59:1d:6b:dc:cc:1e:48:c5:79:5b:8f:18:7e:e4:c4:9f:4f: 3d:a9:11:60:28:77:49:e7:23:4d:fa:49:8d:6b:5a:52:57:9b: bf:a7:d7:da:13:45:64:dd:d3:2f:4c:14:eb:79:46:8f:fd:41: ab:88:17:d4:8e:77:45:8c:7f:cb:e5:d8:19:7c:1b:1e:cd:21: 32:43:a3:c9 -----BEGIN CERTIFICATE----- MIIE9jCCA96gAwIBAgIUIY7U+RD8Ml9GRSNTFAPwstyh1+UwDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowYjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xGjAYBgNVBAMMEWlwLW9ubHkt bG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvZjcFEmS QxpzNnVtwTNzs+f2iuID9yST6zbaj2TGVhm3ktLx39NPfnjO29xu1FxYpKd5a0n/ MDfZcZIh15FyjRohdQ+AI/AvDmoaYpvrHnTf32GKBagxt9SX7sZgG2+xhZusPc2h nqi1VoTOn4xkpVdBdYHGAMjYPe+FnHgESVqUKa8nCr5WYGugUgZT9c/OR5zPUIod kjoh0STtgeiFl4xBRpbaAte39auSTKsK4hmfuCEHXPfRfm5JcKnERcOyEKvKUOPJ 6m+1GWQSMg6TWfiO/KK8M/Ml5FmYbQ6XdrbanlDPTaFrYs5HhB33gora0JDnph9D JWFXwxV+4LhcFwIDAQABo4IBkzCCAY8wCQYDVR0TBAIwADA5BglghkgBhvhCAQ0E LBYqbmF0cy5pbyBuYXRzLXNlcnZlciB0ZXN0LXN1aXRlIGNlcnRpZmljYXRlMB0G A1UdDgQWBBQKrILyNXOJ+0kH65yHctmRJt+EijCBrgYDVR0jBIGmMIGjgBT+z38+ Lu7Erepbb4hFbMeISBSLOqF1pHMwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh bGlmb3JuaWExEDAOBgNVBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAn BgNVBAMMIENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3ghRJnBbtu1z0 RRusxa2MelszQLZtITAhBgNVHREEGjAYhwR/AAABhxAAAAAAAAAAAAAAAAAAAAAB MBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYB BQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcN AQELBQADggEBAIcALezqqxM+IWxKPvGbw6vFgdM0Lcvx55iWE6N/KQCDcCCBVqEx t9n6NQL0bzKKWM9oOeH5+1AgYIXxvSJz5qy2Njh1ht6Kz+VB3w+9YZQ9OlmBUMVE OUfZ6FYqtCxZt4n1XacoFW6YHkdHfaY1Zd63Bp4EZ5gE+VdUgirc4mp23wYFtGGE 7H1eRjTqFLcUFv5RDmVSxnSWjjA6ucLVBS2oCBG6fmXBicNh7NYG4H6sEe4YiWdZ HWvczB5IxXlbjxh+5MSfTz2pEWAod0nnI036SY1rWlJXm7+n19oTRWTd0y9MFOt5 Ro/9QauIF9SOd0WMf8vl2Bl8Gx7NITJDo8k= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/server-key-iponly.pem000066400000000000000000000032501477524627100240110ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9mNwUSZJDGnM2 dW3BM3Oz5/aK4gP3JJPrNtqPZMZWGbeS0vHf009+eM7b3G7UXFikp3lrSf8wN9lx kiHXkXKNGiF1D4Aj8C8Oahpim+sedN/fYYoFqDG31JfuxmAbb7GFm6w9zaGeqLVW hM6fjGSlV0F1gcYAyNg974WceARJWpQprycKvlZga6BSBlP1z85HnM9Qih2SOiHR JO2B6IWXjEFGltoC17f1q5JMqwriGZ+4IQdc99F+bklwqcRFw7IQq8pQ48nqb7UZ ZBIyDpNZ+I78orwz8yXkWZhtDpd2ttqeUM9NoWtizkeEHfeCitrQkOemH0MlYVfD FX7guFwXAgMBAAECggEAE6qdeYVAJLHDrax0nAvIPqsbCcD0BFjI9ycTeLhNUnUM c7Bp4nu6zTWez3OIE4MYtsugbp6YV9oTNhKgbAnsRfKl8cyP0CqD1wzue7gMpXYe Gr+1X2zY62aj8+Kj6XSmh2NkdGy2DQ0W8kiIXkhj0DrC0XuKnF45AAOualKQr0MG JRFXi/AEEDkyWwfmqteD60HBzdweBqzYvsIyA8cQAgI4LguRKHKmLexqNDL0h0K+ nPbQl5pg7es3kOaQ7DMICqd+E11btSXBJop95MLVhPXkzAyktIdfc1M7LsBNb7Dw ja5G+wmmGG0KxCbqMVKCACxs63lWakIxFlrsxi+Q4QKBgQDwrM4f/j289+PBKlED UCXHsGBgE1vNZMAinfAazWAImTAwWoY0FDrMBBPJGx+Azke5teI82o+Sq4jTL44Y qvmqblHb9XEQLnRxSdl5WP794o1K9pP7nEvIpL+RngRIC1qt9BEGd5Pmn8FcF4BT Zo9sdP8Hh6EJLGb33iYxUrTNTwKBgQDJq3GclCnVukbaGbCU2w54EDTJoghIY+W4 kXeGL1VbSXD/NnyuhH/mua+UwBM7XSFJXKV2vfIhI1Qe9VW+7p4vyiLFp3atSb86 AirajUPrCJNX1X/sm7fIss9U1ko/JruXSh1DxzEZvBQa52SwYH5bM6LbE6NJRuhS xMY2ipyiuQKBgQCm0gCl6GH+w4wYbi5tL3agbT7AGWr+eSE8XWD6EvTHwPbH7Vcs bgE7PHBCawxxCYppzQqdx5jQvxk92K6Tpp8bZRBUeFIAN1L624dkNy236PqqxTNZ qcJVtuwaEP9CuKwH+y553xSjPISYQqnuJR6wvH+xRm92nlJY6KBse7lavQKBgCEV NeMIz0AXec4HjtcshFgf2HkHUrKFaMb5XhEuLKN4DchgKN38MHsqFOqjA8SmR3Kg dyheipzzDbayammS/XI7h67DBQ3yXiNm/Z6ys+SXmIw9IuoutVyAMNDrAm0PrpBo ARsAT0a4etfbA8KHYdMWSm4D77JypmQFkbqazI1JAoGBAKZTn6KQaBcZ5JrSya17 sClZPlSsYhIW8EbxqDgNwDippcNrLuJCsWiwMNW+glQoDIcGAo557PwAXrridH89 5mjFCVggHeso3T6/nsBB86Q8B1hq5E8AszhEKH6IeKd5tFNZkK83eWjAlbIUrEBH xXZFMGnYPPS7korkEzzuGJ+7 -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/server-key-noip.pem000066400000000000000000000032501477524627100234440ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCt8Ic/MmaHejGb ylQKrqYayiXVxfxJayEL3qcVyJw8zUEdMiV3aHuD6F0Uei4L6kGRpCDsIBcPy41M G4ig0ndGZX7RoOZMS8aMOaGzWzRXyKEQDBNUOnSQezu62kFigfXctXNsgzj0oVKr vcKVPnn/r6Su39YR2SkguLQV4zKTXDbOVrQBAqFFMaOhHuq4xAEEVxFE9FXq4q5o CHCFwFv/ur/ei7yhxgOiL4rrnrd5OmdqsHDT6AinEiTVu1eIcjfI5i7bh+AqcRos kJyIKQx1KITWf3UtUAg2K8/zujNyHnoH2yDamDs5hpZM4kpCYRqbC2dNbRPRn0Df EseNnVBpAgMBAAECggEAcmiqXRwmqmfqZ4Ge4+Pap/ZdCo6OkjAf7XHHTyHD+o47 jRul3zPfQnU9fDGdRgMQm95sNUQqRx5pUy0tIjMtdyVdVD9UG80fzK4/uPx9olv5 7Nc0g4trjnkwYYgbx9KZyFGlmTN67BWMjiBj88zDbDW4ybm7UcQYNEipU1g8tQW1 tUwcZ1oahXfzO75vcMqDVlS2IE0s0AD9sh+AaJIwxV9kSLNjlSwkpsH6PBKKB/3r WvG2p6Og1whdQ54PGADUVSx1yWFyXQDeygqLmryEWaHJQz1jt7bvaaAMy2PTdwVf A5LVG3VHkoQOBv8imtpCbU2J7zAk9ypDuRUlpa8h/QKBgQDdCCCbV02BhrqDYchm ojB95Vx8KtvQdXhvsxShxyuIktuB7W+NnheBmLY0TNcYSQyzithCUBhtmyaC5S4f dHmT52e7HS0xaL9r9BhAQrtWReMcplKB1IIXtdYXEY3qOjZMxX3seJo0iBWS3hMH EG6tC6tlr5ZXOKJOrBMGuMgplwKBgQDJdSYkC3AX2p+4BNf3hgQyzotuSVSbx/zu 0ZHhi8Wp7yF49c8+9+ahO9AMrVM0ZSh2buznfF46FNC/C55M7a9Rn60sFQQ16b5L rJTzlPoUGTnPLt8C3TdMIFg/5cAW6ZgZWNlU3aVU0W34NVh/H2m/M72tGrk250zs YhZ8/RGV/wKBgQCKlMfs3YXoyhIywaImR1Zj+ORNrYl4X86NKhirffbbgEhEZBvn DNHsHVVP4UWTImnmQA1rNlC6l+ZDd3G9owd/Jj0xYg+txOEPzFFQKQbQBq1ojxd3 80dFmmqKuCTkUG8vHzvegZcdjJ0KIlaHvVPHB2QFM1vtf8Kz1MtxEXXeLQKBgDn0 Bm3WEH/8N3gzhIFDP0/yVO/8DmfmByAYj5PHpqw1C3cFl4HwxJrbXwVWkxn+g75W OLZ684xX0pky2W4d7hJYEfQdc6GixUh1tD/COpKvkw7D2Am146N1po1zJWgx+LxJ 7/NW86nLuYvupK+lNMF5O/ZhOqjNrzZNHVUFZBq3AoGAPwixh7/ZMX6mmm8foImh qibytx72gl1jhHWSaX3rwrSOO9dxO2rlI7LOZQrarU632Y9KMkP3HNbBHPRkA4MI 6I9wqawRzGjcpeXIMlPzOHDHYLyrTpEzo8nrSNk/cM8P4RxE12FqySzQIkiN06J7 AxJ7hVqtX6wZIoqoOa9aK1E= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/server-key.pem000066400000000000000000000032441477524627100225040ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDm+0dlzcmiLa+L zdVqeVQ8B1/rWnErK+VvvjH7FmVodg5Z5+RXyojpd9ZBrVd6QrLSVMQPfFvBvGGX 4yI6Ph5KXUefa31vNOOMhp2FGSmaEVhETKGQ0xRh4VfaAerOP5Cunl0TbSyJyjkV a7aeMtcqTEiFL7Ae2EtiMhTrMrYpBDQ8rzm2i1IyTb9DX5v7DUOmrSynQSlVyXCz tRVGNL/kHlItpEku1SHt/AD3ogu8EgqQZFB8xRRw9fubYgh4Q0kx80e4k9QtTKnc F3B2NGb/ZcE5Z+mmHIBq8J2zKMijOrdd3m5TbQmzDbETEOjs4L1eoZRLcL/cvYu5 gmXdr4F7AgMBAAECggEBAK4sr3MiEbjcsHJAvXyzjwRRH1Bu+8VtLW7swe2vvrpd w4aiKXrV/BXpSsRtvPgxkXyvdMSkpuBZeFI7cVTwAJFc86RQPt77x9bwr5ltFwTZ rXCbRH3b3ZPNhByds3zhS+2Q92itu5cPyanQdn2mor9/lHPyOOGZgobCcynELL6R wRElkeDyf5ODuWEd7ADC5IFyZuwb3azNVexIK+0yqnMmv+QzEW3hsycFmFGAeB7v MIMjb2BhLrRr6Y5Nh+k58yM5DCf9h/OJhDpeXwLkxyK4BFg+aZffEbUX0wHDMR7f /nMv1g6cKvDWiLU8xLzez4t2qNIBNdxw5ZSLyQRRolECgYEA+ySTKrBAqI0Uwn8H sUFH95WhWUXryeRyGyQsnWAjZGF1+d67sSY2un2W6gfZrxRgiNLWEFq9AaUs0MuH 6syF4Xwx/aZgU/gvsGtkgzuKw1bgvekT9pS/+opmHRCZyQAFEHj0IEpzyB6rW1u/ LdlR3ShEENnmXilFv/uF/uXP5tMCgYEA63LiT0w46aGPA/E+aLRWU10c1eZ7KdhR c3En6zfgIxgFs8J38oLdkOR0CF6T53DSuvGR/OprVKdlnUhhDxBgT1oQjK2GlhPx JV5uMvarJDJxAwsF+7T4H2QtZ00BtEfpyp790+TlypSG1jo/BnSMmX2uEbV722lY hzINLY49obkCgYBEpN2YyG4T4+PtuXznxRkfogVk+kiVeVx68KtFJLbnw//UGT4i EHjbBmLOevDT+vTb0QzzkWmh3nzeYRM4aUiatjCPzP79VJPsW54whIDMHZ32KpPr TQMgPt3kSdpO5zN7KiRIAzGcXE2n/e7GYGUQ1uWr2XMu/4byD5SzdCscQwJ/Ymii LoKtRvk/zWYHr7uwWSeR5dVvpQ3E/XtONAImrIRd3cRqXfJUqTrTRKxDJXkCmyBc 5FkWg0t0LUkTSDiQCJqcUDA3EINFR1kwthxja72pfpwc5Be/nV9BmuuUysVD8myB qw8A/KsXsHKn5QrRuVXOa5hvLEXbuqYw29mX6QKBgDGDzIzpR9uPtBCqzWJmc+IJ z4m/1NFlEz0N0QNwZ/TlhyT60ytJNcmW8qkgOSTHG7RDueEIzjQ8LKJYH7kXjfcF 6AJczUG5PQo9cdJKo9JP3e1037P/58JpLcLe8xxQ4ce03zZpzhsxR2G/tz8DstJs b8jpnLyqfGrcV2feUtIZ -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/server-noip.pem000066400000000000000000000126721477524627100226660ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 1d:5c:7c:59:0c:cd:27:83:dd:97:64:53:b0:44:3c:b4:5b:d4:fc:d1 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:ad:f0:87:3f:32:66:87:7a:31:9b:ca:54:0a:ae: a6:1a:ca:25:d5:c5:fc:49:6b:21:0b:de:a7:15:c8: 9c:3c:cd:41:1d:32:25:77:68:7b:83:e8:5d:14:7a: 2e:0b:ea:41:91:a4:20:ec:20:17:0f:cb:8d:4c:1b: 88:a0:d2:77:46:65:7e:d1:a0:e6:4c:4b:c6:8c:39: a1:b3:5b:34:57:c8:a1:10:0c:13:54:3a:74:90:7b: 3b:ba:da:41:62:81:f5:dc:b5:73:6c:83:38:f4:a1: 52:ab:bd:c2:95:3e:79:ff:af:a4:ae:df:d6:11:d9: 29:20:b8:b4:15:e3:32:93:5c:36:ce:56:b4:01:02: a1:45:31:a3:a1:1e:ea:b8:c4:01:04:57:11:44:f4: 55:ea:e2:ae:68:08:70:85:c0:5b:ff:ba:bf:de:8b: bc:a1:c6:03:a2:2f:8a:eb:9e:b7:79:3a:67:6a:b0: 70:d3:e8:08:a7:12:24:d5:bb:57:88:72:37:c8:e6: 2e:db:87:e0:2a:71:1a:2c:90:9c:88:29:0c:75:28: 84:d6:7f:75:2d:50:08:36:2b:cf:f3:ba:33:72:1e: 7a:07:db:20:da:98:3b:39:86:96:4c:e2:4a:42:61: 1a:9b:0b:67:4d:6d:13:d1:9f:40:df:12:c7:8d:9d: 50:69 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: C9:AA:3C:08:39:7E:C1:42:C0:3D:B7:2F:84:21:E7:8A:30:E7:C7:B1 X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 9b:63:ae:ec:56:ec:0c:7a:d5:88:d1:0a:0a:81:29:37:4f:a6: 08:b8:78:78:23:af:5b:b7:65:61:d7:64:2a:c9:e7:a6:d2:b1: cb:36:bf:23:2e:2d:48:85:7f:16:0f:64:af:03:db:5d:0e:a7: 14:c5:f6:04:b2:6b:92:27:ba:cb:d2:13:25:a2:15:b0:8e:4a: 2d:eb:41:18:09:b1:68:d5:0f:6b:56:da:86:ed:4a:7a:29:30: 09:77:63:a4:64:3d:e3:2e:d7:6f:1a:8c:96:c9:cb:81:fe:a3: 6d:35:e3:09:ea:9b:2e:da:8c:8e:c8:c9:69:b1:83:e7:6f:2d: 5f:a1:ac:32:ae:29:57:a9:5c:9b:7d:f0:fd:47:3c:f3:6a:d0: eb:77:8d:70:06:a2:74:3d:d6:37:1e:7b:e7:d9:e4:33:c9:9d: ad:fa:24:c6:4d:e2:2c:c9:25:cb:75:be:8d:e9:83:7e:ad:db: 53:9e:97:be:d5:7f:83:90:fc:75:1d:02:29:b7:99:18:a3:39: 25:a2:54:b7:21:7d:be:0b:4c:ea:ff:80:b9:4b:5e:21:ed:25: ad:d4:62:52:59:79:83:32:df:30:a1:64:68:05:cc:35:ad:8b: d3:66:6b:b1:31:b7:b3:b2:d8:0f:5b:96:40:ef:57:1d:7f:b0: b0:f4:e9:db -----BEGIN CERTIFICATE----- MIIE4TCCA8mgAwIBAgIUHVx8WQzNJ4Pdl2RTsEQ8tFvU/NEwDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3whz8yZod6MZvKVAqu phrKJdXF/ElrIQvepxXInDzNQR0yJXdoe4PoXRR6LgvqQZGkIOwgFw/LjUwbiKDS d0ZlftGg5kxLxow5obNbNFfIoRAME1Q6dJB7O7raQWKB9dy1c2yDOPShUqu9wpU+ ef+vpK7f1hHZKSC4tBXjMpNcNs5WtAECoUUxo6Ee6rjEAQRXEUT0VerirmgIcIXA W/+6v96LvKHGA6Iviuuet3k6Z2qwcNPoCKcSJNW7V4hyN8jmLtuH4CpxGiyQnIgp DHUohNZ/dS1QCDYrz/O6M3IeegfbINqYOzmGlkziSkJhGpsLZ01tE9GfQN8Sx42d UGkCAwEAAaOCAYYwggGCMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU yao8CDl+wULAPbcvhCHnijDnx7Ewga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MBEGCWCGSAGG+EIBAQQEAwIGwDAL BgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYB BAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAJtjruxW7Ax61YjR CgqBKTdPpgi4eHgjr1u3ZWHXZCrJ56bSscs2vyMuLUiFfxYPZK8D210OpxTF9gSy a5InusvSEyWiFbCOSi3rQRgJsWjVD2tW2obtSnopMAl3Y6RkPeMu128ajJbJy4H+ o2014wnqmy7ajI7IyWmxg+dvLV+hrDKuKVepXJt98P1HPPNq0Ot3jXAGonQ91jce e+fZ5DPJna36JMZN4izJJct1vo3pg36t21Oel77Vf4OQ/HUdAim3mRijOSWiVLch fb4LTOr/gLlLXiHtJa3UYlJZeYMy3zChZGgFzDWti9Nma7Ext7Oy2A9blkDvVx1/ sLD06ds= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/srva-cert.pem000066400000000000000000000130141477524627100223120ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 0a:7d:45:32:e2:24:e3:81:f9:5a:64:e1:f2:50:92:d8:e2:3e:14:8e Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:9f:b5:d0:95:c7:8c:ee:16:13:fc:d6:90:8c:75: a9:3b:2c:ab:ff:b6:b6:ae:39:8f:c3:d4:75:7d:01: ba:0b:f0:db:8a:72:2d:41:49:e1:f1:b5:f6:5c:32: af:b0:46:89:ee:33:5d:84:47:69:a6:4b:de:3f:8b: a4:29:4c:98:20:6b:94:29:bc:4b:cb:84:73:d1:6d: 35:15:63:f8:55:a7:50:b4:2e:3e:e5:61:8a:c4:a5: fb:7a:88:aa:14:b2:06:36:8f:93:3b:8c:29:d2:ee: f6:8f:e7:83:68:33:59:17:d6:1f:b4:05:4d:02:ff: f3:a0:1f:15:2c:ad:8a:33:13:95:95:d7:b6:54:6f: ac:cb:98:8c:fb:69:c7:a6:cc:77:5f:20:08:e7:8b: 3b:79:5d:7d:63:2e:6e:cf:ce:b6:04:d0:0a:c5:19: 6d:ed:aa:45:7d:32:c3:53:ba:5d:d6:5c:e5:30:e0: c7:3c:2f:88:9e:0d:33:8a:c8:30:40:af:48:a6:aa: 4d:68:81:55:4f:43:cc:f2:79:3a:c2:23:0d:fe:09: dc:c0:85:94:cf:c0:f4:e2:7e:c2:64:7a:55:41:f1: 3f:3d:e3:46:4a:ef:55:33:d2:ea:72:56:25:15:6c: 47:1b:4d:c5:da:d3:6a:cb:62:db:d4:7c:86:64:81: 86:61 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: C7:6D:8A:C9:67:13:C9:63:AB:01:87:58:33:DC:FB:24:AC:61:65:4E X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption a1:3f:db:97:03:ca:e3:34:36:01:8a:d9:34:1e:1e:9e:d5:bf: 7b:9f:37:cf:61:2a:8e:fa:c3:cb:68:ca:f3:4e:04:78:8d:5a: 0c:c0:5b:24:fe:56:a5:c0:7a:b1:bd:63:1e:ff:bf:72:9a:fa: 08:33:4b:e0:2b:35:8b:03:c4:23:7f:54:3a:8a:f2:a9:28:d7: a8:7f:10:96:93:a3:b5:82:59:f1:b6:05:3b:58:cd:a1:8d:a9: 77:fa:71:36:f7:68:c9:50:46:4c:0f:c3:f8:f8:90:0c:69:eb: 53:e6:d0:5d:32:2f:d3:f7:b0:8d:75:68:59:37:d7:de:ee:da: 42:99:1c:24:03:28:c4:42:8c:17:e4:ed:5a:d8:70:32:01:fb: f8:57:f8:1a:2d:bd:f6:e1:35:12:28:65:ef:00:05:1b:e2:50: d3:1e:3c:25:34:56:5a:86:8c:30:9d:e0:f4:79:d8:09:7d:74: 0d:70:3a:2c:67:2a:d6:d2:6b:ed:d4:bd:1d:f2:d6:02:74:2a: 75:b9:8b:6b:a4:c1:c7:d8:42:58:ad:85:d1:0c:0f:14:88:12: 57:34:60:9f:18:32:f2:75:77:69:30:42:65:3a:f7:c1:16:49: 76:21:ea:cd:95:b1:29:d1:66:e2:90:e9:86:c4:7b:2d:aa:2b: 00:6e:59:f6 -----BEGIN CERTIFICATE----- MIIE+TCCA+GgAwIBAgIUCn1FMuIk44H5WmTh8lCS2OI+FI4wDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ+10JXHjO4WE/zWkIx1 qTssq/+2tq45j8PUdX0Bugvw24pyLUFJ4fG19lwyr7BGie4zXYRHaaZL3j+LpClM mCBrlCm8S8uEc9FtNRVj+FWnULQuPuVhisSl+3qIqhSyBjaPkzuMKdLu9o/ng2gz WRfWH7QFTQL/86AfFSytijMTlZXXtlRvrMuYjPtpx6bMd18gCOeLO3ldfWMubs/O tgTQCsUZbe2qRX0yw1O6XdZc5TDgxzwviJ4NM4rIMECvSKaqTWiBVU9DzPJ5OsIj Df4J3MCFlM/A9OJ+wmR6VUHxPz3jRkrvVTPS6nJWJRVsRxtNxdrTasti29R8hmSB hmECAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU x22KyWcTyWOrAYdYM9z7JKxhZU4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI hvcNAQELBQADggEBAKE/25cDyuM0NgGK2TQeHp7Vv3ufN89hKo76w8toyvNOBHiN WgzAWyT+VqXAerG9Yx7/v3Ka+ggzS+ArNYsDxCN/VDqK8qko16h/EJaTo7WCWfG2 BTtYzaGNqXf6cTb3aMlQRkwPw/j4kAxp61Pm0F0yL9P3sI11aFk3197u2kKZHCQD KMRCjBfk7VrYcDIB+/hX+BotvfbhNRIoZe8ABRviUNMePCU0VlqGjDCd4PR52Al9 dA1wOixnKtbSa+3UvR3y1gJ0KnW5i2ukwcfYQlithdEMDxSIElc0YJ8YMvJ1d2kw QmU698EWSXYh6s2VsSnRZuKQ6YbEey2qKwBuWfY= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/srva-key.pem000066400000000000000000000032501477524627100221460ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCftdCVx4zuFhP8 1pCMdak7LKv/trauOY/D1HV9AboL8NuKci1BSeHxtfZcMq+wRonuM12ER2mmS94/ i6QpTJgga5QpvEvLhHPRbTUVY/hVp1C0Lj7lYYrEpft6iKoUsgY2j5M7jCnS7vaP 54NoM1kX1h+0BU0C//OgHxUsrYozE5WV17ZUb6zLmIz7acemzHdfIAjnizt5XX1j Lm7PzrYE0ArFGW3tqkV9MsNTul3WXOUw4Mc8L4ieDTOKyDBAr0imqk1ogVVPQ8zy eTrCIw3+CdzAhZTPwPTifsJkelVB8T8940ZK71Uz0upyViUVbEcbTcXa02rLYtvU fIZkgYZhAgMBAAECggEAAcqJN1TQmM74lGpXnqunJTnACBMlg1iz6w9T7f359y5R WAElzsikVxCiXbBX1hNEcr5yuwwN/EZ3jKDlS21He32ZYMpy0yp+Hggqgz3myBcj SZIHNI1egAwYgOxNdAP3G9+KWC2fPnVdoImJKL8CrcjB/EPe9Dyon9AZbbHYjfAS wOCy06KTCvwJrNO1mYFSftEOe1S3TVGHp3KX96YWxR/yIgXNC+5bJWzf+X8Ca0i7 YHsgoF1CqoLg7sq+nMo1xzGEapy2bkx8DRdxigIXBBNnKQdVVO7qjmUOw8EdJZ9/ luZ+/lb6Oc9RT6+V3B97kpYP7Z+frItGZWuMtifLAQKBgQDP1eI2w/VW6CW0ODjQ O1ivNzK1CK4L9FNA07tnhfzEOlF/NMqfBz7pYyn+1ObxMcXzFAZAmh5E/Q8b8QEP 4b51IUqGL6FSQlKSlWJlszD7QhUM6nXFmvDo1g3pBZ9w8EW12cL+FM/KBpXKB0aF S8wcekbvGkgqI6SQEvkcRrM6EQKBgQDEuNXn8B7YTlkw7eXu/F87ZQkWb1bes8VI aQzVDH/ywu5Cbou8iJYsdfpZFwnJ4ojIl0eBleqXpgb+SiEx8q5zmR/Tt0NDIjfP lVdlcf3oLxjz6LKwHWPnXmz2WEeYez2/lkBc0XmOYrAF8jy2h1RX4kkvVGEBNNKE ooiX2Kq3UQKBgFPQQCqvubezZMkZxyeKV0hJrUQ+XNnrTMue1zt3WCjWD9mJs5CQ PnmmCOCAgSWRWe+eCqtwu3Y8yyQMe84ozkK0Czaz7I2cu2nrnkO/EKiyzIRp5Nas lix/bhqjrtm/u40LZUxLnkQthah0aKcGmyh80zBHXATKXOnRDO6vQEXRAoGAKmuB FMaeoB4k9XwXPPWGw5QB5aiUnuk8/WRJtBdB7+NA5WpZD4qUd3npRhTFs5b5z+dv L+5X6+ONVoA9sDuYiqzWRB0cj/ls94JImkN+cPbW9qGTBo8P3/BopqZaqfLJWerY bittftqhAmc92E5OTbUp+t7DHDN7wcYalkPObEECgYB0sNTdWVl78AayFqI/5Bg0 FfZYRTdVyx6qTPsQHqmPGHNn+ZyPIzVJJyfT3Fp8cToJDyYZtp5lUcnsSb2Q3j/C riF99UpDJfetEFJATlSWjgWkMgQVHm97qIZtr7XLzM2gWJ1/fE0IxIx3ZFGUi19x l6nGMJ23s64br4XiyNqHIw== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/srvb-cert.pem000066400000000000000000000130141477524627100223130ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 20:98:70:f3:55:1c:7e:a3:15:0c:64:c1:a2:8e:56:4d:4a:77:70:e8 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27 Validity Not Before: Aug 27 20:23:02 2022 GMT Not After : Aug 24 20:23:02 2032 GMT Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:bc:15:41:85:42:e2:da:58:c4:d4:2b:8a:92:88: cf:a8:81:5b:03:5d:8b:d5:44:32:07:ae:ef:a2:ff: 0f:29:14:dd:bf:7e:6b:37:e4:31:e1:94:7e:5e:be: 74:40:68:ce:3b:d5:bc:00:a6:42:fc:0f:e1:c5:8d: 4f:09:32:fc:af:6a:17:c8:92:3d:b0:2d:bd:ce:de: 7f:d8:bd:9e:5b:84:f8:b6:8c:a3:7e:98:37:f1:37: 1e:aa:cd:87:92:3f:33:34:17:0e:a1:78:3b:1a:88: d6:73:c2:ce:41:71:81:ea:83:2d:f7:e5:e6:32:f1: 18:33:6b:ad:9d:ec:0a:3e:8e:74:a6:b7:7b:4b:22: cb:0e:6b:57:14:cc:4d:c0:27:a1:38:30:df:35:e6: fe:49:8d:4f:18:4d:be:5d:df:46:d0:4e:3d:00:93: f8:e7:48:4c:6f:fb:13:c0:fb:6c:8a:9e:53:66:24: 3f:c3:93:66:57:6a:f6:66:88:94:b7:46:e1:e8:d6: 8a:04:02:92:e1:c5:c9:b1:71:f0:d2:1e:4a:69:8b: 0c:ac:a1:01:89:5f:1f:ac:79:da:8d:ad:bf:03:81: b6:8c:70:5c:1e:74:f5:a4:d8:d1:8a:41:d7:bf:27: 51:d1:fc:75:b9:34:73:03:ae:10:2e:ab:78:a2:af: e7:99 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: nats.io nats-server test-suite certificate X509v3 Subject Key Identifier: 5F:CD:62:19:69:22:48:0B:79:B2:8F:53:D7:EA:C6:9B:C2:5E:58:94 X509v3 Authority Key Identifier: keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27 serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21 X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 Netscape Cert Type: SSL Client, SSL Server X509v3 Key Usage: Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication Signature Algorithm: sha256WithRSAEncryption 08:6c:2f:cd:9d:b4:44:42:51:29:c2:55:92:e9:52:69:af:ae: c6:c4:4b:81:7a:a9:7c:81:f9:dd:27:fe:c1:e2:9b:59:92:38: 1f:f4:d3:04:b7:ed:59:78:da:09:33:c0:c0:1d:60:79:07:78: 15:aa:50:c4:f8:7c:47:96:a7:76:3f:ac:5f:eb:7d:8b:05:81: 83:f7:be:7d:1e:6f:05:d4:bb:7b:ee:14:a8:af:e2:22:82:bf: 25:7c:f2:30:38:98:68:43:c0:4a:08:80:11:81:be:c1:7e:43: 22:cb:85:c3:b6:da:37:86:c5:7f:5b:8c:d9:b4:61:3a:08:c0: 12:a0:a1:ea:2f:cd:58:da:41:8c:1e:10:1f:54:19:c2:6a:2e: 3a:d9:0e:f6:7c:34:95:72:9a:41:ed:1e:79:84:1f:8e:89:8c: 43:84:9d:32:42:e9:b5:db:1a:ab:2e:1a:f3:d1:b0:2b:38:fc: e3:c7:4f:1b:6f:9c:2e:3f:83:7a:fc:22:23:aa:64:6c:6f:c9: 26:81:81:5b:f1:8c:ca:0a:ba:2d:91:f7:2f:ba:d4:7f:dc:53: e2:f6:1f:38:50:6e:6b:87:10:b6:02:af:56:f3:83:9f:3e:3d: 83:e6:59:cf:3b:f8:bd:21:0a:8c:90:d4:26:74:70:1c:b1:cc: a6:3a:27:e1 -----BEGIN CERTIFICATE----- MIIE+TCCA+GgAwIBAgIUIJhw81UcfqMVDGTBoo5WTUp3cOgwDQYJKoZIhvcNAQEL BQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM B1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl IEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw MjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV BAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALwVQYVC4tpYxNQripKI z6iBWwNdi9VEMgeu76L/DykU3b9+azfkMeGUfl6+dEBozjvVvACmQvwP4cWNTwky /K9qF8iSPbAtvc7ef9i9nluE+LaMo36YN/E3HqrNh5I/MzQXDqF4OxqI1nPCzkFx geqDLffl5jLxGDNrrZ3sCj6OdKa3e0siyw5rVxTMTcAnoTgw3zXm/kmNTxhNvl3f RtBOPQCT+OdITG/7E8D7bIqeU2YkP8OTZldq9maIlLdG4ejWigQCkuHFybFx8NIe SmmLDKyhAYlfH6x52o2tvwOBtoxwXB509aTY0YpB178nUdH8dbk0cwOuEC6reKKv 55kCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu aW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU X81iGWkiSAt5so9T1+rGm8JeWJQwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I RWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb M0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA AAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI KwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI hvcNAQELBQADggEBAAhsL82dtERCUSnCVZLpUmmvrsbES4F6qXyB+d0n/sHim1mS OB/00wS37Vl42gkzwMAdYHkHeBWqUMT4fEeWp3Y/rF/rfYsFgYP3vn0ebwXUu3vu FKiv4iKCvyV88jA4mGhDwEoIgBGBvsF+QyLLhcO22jeGxX9bjNm0YToIwBKgoeov zVjaQYweEB9UGcJqLjrZDvZ8NJVymkHtHnmEH46JjEOEnTJC6bXbGqsuGvPRsCs4 /OPHTxtvnC4/g3r8IiOqZGxvySaBgVvxjMoKui2R9y+61H/cU+L2HzhQbmuHELYC r1bzg58+PYPmWc87+L0hCoyQ1CZ0cByxzKY6J+E= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/srvb-key.pem000066400000000000000000000032541477524627100221530ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC8FUGFQuLaWMTU K4qSiM+ogVsDXYvVRDIHru+i/w8pFN2/fms35DHhlH5evnRAaM471bwApkL8D+HF jU8JMvyvahfIkj2wLb3O3n/YvZ5bhPi2jKN+mDfxNx6qzYeSPzM0Fw6heDsaiNZz ws5BcYHqgy335eYy8Rgza62d7Ao+jnSmt3tLIssOa1cUzE3AJ6E4MN815v5JjU8Y Tb5d30bQTj0Ak/jnSExv+xPA+2yKnlNmJD/Dk2ZXavZmiJS3RuHo1ooEApLhxcmx cfDSHkppiwysoQGJXx+sedqNrb8DgbaMcFwedPWk2NGKQde/J1HR/HW5NHMDrhAu q3iir+eZAgMBAAECggEAfFfVJEPO9Clhnx9WEoBOQQmb4QK0Un8uUDQQC4NyQ2ef Hl12htHfRFJGpV5mPrrLNXWdQBSrDjd1vIQqb1t9AH80UA6Wp+XlqWEhxmm9yqWD RDSAVk5OFbOqG6ObcE1GDbb8njV9ZZJ3rLpqX4uqJx8ogc5EvQL1S/FxK4mEUEv8 bq4Iswp590+KwnaYtkXEoieUbE1QC0y6z9BEpEe3WkNVGAmnuJ6wCiE2pW6yzK94 NxGvnsejgBVDZZprWbLdIztHDJ4+VDDctkxnqsx3cb2o+mK7CCP6C7v9MAvgrMzH hEDu2WZC6T6GrVysYXSNmlWYPTsPh89F5Dyrp5vq8QKBgQDwTnMq4SBbrq3tsJEV ChX3pqQ9icyaT/yLI1oWcrdJKpyMdDVxOBPjB04i7pUZFNKqHS/aKjozKmXHVFNy X9wy5D0der7gqHmMeEV4B7UeO31E2B6nftlntt+V8U3Nu6r9mi3QCbH5x5vBjDH+ ulx4Qma+CeBL+bP1Z8Vm8a6qNQKBgQDIXbY3jm3Gl2hE2+Ci7JceloRlwp3CYlHS Kejhxx+secKLbbeXUp2A2uj6lz9DX/Gx+qw8zbhoodh6FkGJVrZchatUxci3P9rs 6c2rVMzcPGSt67tFtGGm04cbZgD83QDj0xPtlolZlrGV55PpUFJkS2AThcitMYkx oNOZf3hUVQKBgQDK3n/NoFbv74z4D8q7h1M6Sl13ckDfuU0LraGdN+VSCH42+Ngx J9VqIT2usdn/XZvnb1J5jJKrpUxdNexor1K3SAXsQDaqdLCAjPygs2kNB83ec9GV N7qUG0ewTxGO8LMO+71XVwgARapRgGiokm4EPJJ+Nto7aaeqMpvtJ3V8TQKBgQCW tZ5M1Y4OtW4RdU3XXpwgvtihbiyZvLiMm0d3I/7YDdHebE2Ove6IsKBTdRZ5yB4V YFbTeW2/DhW1Zy4ubpHGsCh6IDbkV1yFnSs9NxDMi4SzKT/4o0BcankSi7wsdtOn iepQaE2xnF7BQaoI8Yi3tdZzeMrEMyFdufnPWIka7QKBgQC9nCUtLl3FCaeq6PMv KrNQRu5dPbpy5SJI1SSXKCUU9VaA6XMKrNeQHkgsnsj6KZ9Rl+yNe31L/GHNRiSj 4O/vChYMk1bXgESenvNafSMbl6agwi/Y1GFF6CM6/vDas4tB7bsFQPh+cnb9uIvq 5IFkU1NTCSSydk03li94ZF81Bw== -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/000077500000000000000000000000001477524627100206475ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/svid/ca.key000066400000000000000000000032501477524627100217440ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDXYllrRiGR1WPO rT07fN7bY5q8Pnsb1zaO13ZlTdQpiNsEa9AkOL7NaAZFZJ9+KdhQYQgbAUb5Cvyw UIrydTvDrqzghQLaHn9VEHx7xJWJRaCYSqB7MBZCsc7ov/4ntIQO4H/aYhBdcXzj ELpqeMSSLpnH9+9KmZU4ArxU0dqx+2YzUoCDXszsD8541797ksrxrZnLkuNxDmtZ ULn55pY/MT7Tq0gkUA+ReDpHeElQuf9L40ODdueSpteR4mDNPQmLoTE5qwTsyhjF Bght+121Chu3tRwprJqxEjwe/YB6rAFVnlgWwLumfYer/jI8UNoSsiVlj8ToMR11 kIVF6NGTAgMBAAECggEAVJJoJSCxDsfY/G7207R493FZcWAhS/HKGPWOpkGUtMhp o4dH/+kONq6Jg9b6b3oz13/6RExQl3qSdRLzNDgHQxOEC1b/IzGFcxvZnWm4A+JE qHfpIuXjX8XcxnSY8fNGRWcunMwTu/VTgf9wC12nZfUJakU2/zZmKsucMrwCIQ8q rYBoHlVVs/R+3Hj6FYj5yFeoCsWyPswFd07llu6/h8ADY+P1xnOeilJ8pdNR6vaQ wZQhBc0m414XRp2OzpKAX0+7sATSUhhGnU5VdlCcP6whHKmva2U3RusIN1Qy7UIA 35sNxMsSzOvozwZXbWMpWaUU69uLtGjeX6qhh+bk9QKBgQD6XyViXijKHN56xFx9 4inTPlqOrgtuZDm+9pQ/TFaZJfK4GoG/8QyUOXmLAnxbbTAAKWXc/zHXhVr71wsW 4pgsUxypWfND9yQ7dYfJCR7PhkYqdRm3WG+JOss9aL+FULdqjD/4tw4Y5EHc2vpO plQ0p0/xJtLAlg0aqjAonjDUJwKBgQDcOdrrklov3IvKw7LgKYmvDRjLGEyMm3qs 0fPgWDG7BzZ8nfDQ7o2Pglf/ypuLYTQyZV4DXAXdiQGjkM/SzStsSV0hmEFfaLtX JPSbk6gjW1+9+sxq1xJfxvu9tlUeLoRwA0Ck0rwGn3y3OjcQCCVL+5jkqmmERihM +3IY8+/etQKBgG4yZHToNpHGpRFpzb/GRFxqrFbyOavLxzLKurMleVQMxMjNOeBu SvgOV/WcEXn4E9FuZxwe4iQW2NXRb1sSPqH7rrjHE3ANnc/hyfLs5be3RLi5M6lj 7it7SirsKKeXwDLQKfPhNf40sNxxMQxe3t8R6Pid+tKy+G+NCodGIP2vAoGBALWE sl8tGE0O2eJc/5koY6X3g5ITCzVyCeFchElMKGFiBpUckZuEpQW+h838L8zoZ9+K Eb1vIh6BiOpKuCGmnr34klzYZDnB13hyq3N4d0P9UqMCj28YqrprBqBOI0ZnTR4O cD/qurQyzNcxkqUSMbu5O0Ju+93c/ebF8juDBRlpAoGAM14TPQSUz4oiMVB5VWAj 75OK2rFQLC+mMx2T7u8UqpHVb+A6lUZcj0aN6TgRQxQo63bRXaEc/13BTMKmmqze rM7bpj4+b08Fr0yhw7fjO2b/q+zn8Gagk6UWIptpY/8WC4OSgaSjMC6ZErPWqRJs cJXNOa3YaoQcl8SfLW45RNg= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/ca.pem000066400000000000000000000021271477524627100217370ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDBzCCAe+gAwIBAgIUHba56kOIMIRpaKPovfEccmejyJcwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx MzIyMzAzNlowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD ggEPADCCAQoCggEBANdiWWtGIZHVY86tPTt83ttjmrw+exvXNo7XdmVN1CmI2wRr 0CQ4vs1oBkVkn34p2FBhCBsBRvkK/LBQivJ1O8OurOCFAtoef1UQfHvElYlFoJhK oHswFkKxzui//ie0hA7gf9piEF1xfOMQump4xJIumcf370qZlTgCvFTR2rH7ZjNS gINezOwPznjXv3uSyvGtmcuS43EOa1lQufnmlj8xPtOrSCRQD5F4Okd4SVC5/0vj Q4N255Km15HiYM09CYuhMTmrBOzKGMUGCG37XbUKG7e1HCmsmrESPB79gHqsAVWe WBbAu6Z9h6v+MjxQ2hKyJWWPxOgxHXWQhUXo0ZMCAwEAAaNTMFEwHQYDVR0OBBYE FCcUWYVGF5cPrQMR/VNkapjuXMRbMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNk apjuXMRbMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBALdP3pUX SkBo3xJ1m7KskL6Xtic9eDxu9aJKYUH4qw16Aoyj+tZm5R1UvyaEsHMG+hFOZRRp tArbOknL4RZG7Tx/ATTiSQl6ozmkZ8GRpKY6VsN5c6Y1e9sbdXeGgfitWMUkQQhD Ftqps2TM6IfJBdj4BhVGAjG45tUhOTfMjZliaNZexVyR6XjuhDHcDDpcdaFgDfDA uAYDDbfhO7MIKxplzrPcI7e9zDdb9mgB8UExNltxrH9w3J1cG4VpKxFtXYs5FO4d swXWTr2cY4sTT/bhVJET+YnHyyZgg+Ear4n7eujA9klUHEMwPy/QDCn8u/091yew NnOmSiP2A+dWQB4= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/svid/client-a.key000066400000000000000000000032501477524627100230550ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUuqRV+zUywU/g 8Q5z6i+mJxzHCXjM1XPKu3gmWHtTTfRbQtvGdFg5mWHL8rgvEkeYkK+yn6NmGFPP z8vJDLyVzz70YQpi1Qv2Awe0bHTgwlCXAKUlHfEapDh3068hEonz2Fp8n652ArNd oaKHJCUZtKIfm10cs+nX+cuiv+trfJUVYxPyrQhu7lhWhNRwDWedI3pc7382mwze SwFft0Mi1AyK7JhVMRM6ld91MXuJlLNfHKEejrumkt6dIP3o7cJJGxW4IFjDs4gh IK9cd6JMeYQXs9Bc03hKazCKCWITrPPiz0BD7beiqFUNcPgOlzqveBnqsXyRLEjE iQ9a9QwDAgMBAAECggEAEC8994FOU7HyRacrDHVyg/aDoivSB8WT7n7MtKzc7BQd QC/GshIyZj+JAzmWzhXY0Vhaq5RwTTQz1/cOf61cJEKY4N0NKp4byW9LpQLxtmJO 2hp0M4AjnuOPUDLVPu3BMgDcL+2XKDkOWGULcQCqQjbZjEOgM/cRaQhCHvlwM+er JqZ0CN6zPrcQTVlwEZLiL+aQ3igZwXaEPBq3VmuEW6rjpq7n7AE19GCmtwzo9PXW BxZFKMMDxsiiqTY/UyM0Sfci+oGYmDMwVMFh5wDdsyAv6ulF8g0U8YTV4w8uBj0I BRsSS8P6nwUIWGS9pz/1mgnsESPQtj9cs2dDUp5P0QKBgQDRCdVvlObEeka/rlIM to9J2iThHPecXHHfD5S5AxC6/fU+n/X2k3Q6GzlYIkFdYQzic8oTtQaODtib9eFL XHDCNyj4x3Gb7UPBl/f9uqIseUhpAJk6GRiUBgQlCJGp4btMVNx1bhoU+nWgysEo eGvC/B8WB+04MLGURn1k3TTnawKBgQC2JE/dLa7bYxKH1K7h5HxwDjrdYQGFfWIb 9/uFyZa+1NAWYj0TD3vy/pox7PI/VMMa6oWl2C6Mv7aSLlFTwcZz3YKPv8iB7WHR F12/s5xcKNID37J4Fo9EFq2751OqKyhW0aU65a7W/mjJY1loLo0fZVmuc1ZGPcDG v39nwJ1LyQKBgQCY8Ub2Qs2tB4HoENT7dD7WXAMLqbQI9SJjf/TigLmm9tSOo17E sBIjyXNlZnrIpY9VyD4buSE4ougdBAN4rgPaNZ2Gl/Ypjak7ZcMOI5UwUPSHzuZN Obtf4h76MZDR6NSu/o/mY0419yJFKNO2lpTRrsHXzqjjqczjncd1GtvwTQKBgCVV Ug1axMFxECZJQMNKavso34Xq0T4EUiy7apGknUJmKnS8FBqpDNXku8RJ9elnPUpy EYzJv9jXnttdpQvO5xduqsk+HSIMfwU1jeCyNvVo5IsoudMFhL4HC/s80hhKGk2Z UQ2+cFTe9ql4tKW3DPQHeRdZyrBBDhduvvWMNHGJAoGAD2x8zNEhBqOuBn5prtsC WPsjXRIjhXIlQbWwda30an74Bw5Z6OWgZr8nUjBtHn7RkeS8PqTE/UzMTLqD1+TD PvnA8GZ2WWdxhdqy04qkYS5SnkSSLeU0hUYZkLrjZJv5QdTnhBBVG/kDDI7TO8NA gKMQVjTCSStdZm02D6EqEfw= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/client-a.pem000066400000000000000000000025531477524627100230530ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIID1DCCArygAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe4wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx MzIyMzAzNlowgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL TG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV BAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is ZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlLqkVfs1 MsFP4PEOc+ovpiccxwl4zNVzyrt4Jlh7U030W0LbxnRYOZlhy/K4LxJHmJCvsp+j ZhhTz8/LyQy8lc8+9GEKYtUL9gMHtGx04MJQlwClJR3xGqQ4d9OvIRKJ89hafJ+u dgKzXaGihyQlGbSiH5tdHLPp1/nLor/ra3yVFWMT8q0Ibu5YVoTUcA1nnSN6XO9/ NpsM3ksBX7dDItQMiuyYVTETOpXfdTF7iZSzXxyhHo67ppLenSD96O3CSRsVuCBY w7OIISCvXHeiTHmEF7PQXNN4SmswigliE6zz4s9AQ+23oqhVDXD4Dpc6r3gZ6rF8 kSxIxIkPWvUMAwIDAQABo4GiMIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhh bXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215 LW5hdHMtc2VydmljZS91c2VyLWEwHQYDVR0OBBYEFDFDpnqTvAQQBUIBMzip3XCT FdsTMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEB CwUAA4IBAQA8lg93kYHKkc4iF+mB8xHtyT1Pmy8CoxbDGy98cac1ny9s9K5kROTZ rE9p2FaC00jf5T/+si5diXa0EnIjWCk0uRlrW5PL85UbsqHJkWav7zCk7cDa9YrR nk4LMABf40RP3SOM4yxjjoVU13jFwIZ5M69Hce3CJ6ZkSoWRzWlKf6ECAbX/JIcJ u8wLBXbJaAvc1CQiZpiSWDvItg+sUn1dU1nmk2o1YmyBPN6uVF874f3Ihazb9wmJ dkWO48O+FaahjR6GzfjEx7w765ATqvTVYAgGZ55ps65VwfjcQbP1jYO4Sb3Aeeub yapE1595oBWoWdwtEeOTJrzSIEhkhyiC -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/svid/client-b.key000066400000000000000000000032501477524627100230560ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7bOmV7/qGy+1L +cgBawm55lxS2w8QQr/lZrq4k/plQC2UDRvGPDaIihbvbP+pfiAgfSiWd/s+Qhdr oOL/gkAJo/iD8HYu9FN+Y4vPlJyoEguNJzOyRe7U9YnW+Gsefwr60PoeiKfXK5B8 J5KahSQSka3hgW+yp4ECzAqpjobGUpzNoFVoQ0o9gBsvBWVOCJ02h5kcwsLFSSbh h76FPiedKHtA3VAj+vrg7rYT4wHpmSAdJZ7uNvbjbB4Jv35ebTnlkywBcPJi3YGt UI8O8HTDP1K8mlTitzvqHGMDtiT6OQJpffuJhOJ+PnXd4nOwKN+AAMQ8RxPx7lv5 n2oYmq0vAgMBAAECggEAQNdYJIBGivXoHZ3HnP37+m2AyUVx+60sGGAm/w5q1TaT ziGO11EjiK9FagWqmWfCE5tZPF079XkGAz1A6yaTowIwSu0Y5fuvGw2zY9hFLjw3 9Nh4IRfinZho3I9K99z53nYH4iJi9qTisqE6Le0b4rS1lEukaglbL62Jjsl5RQje EogelhVsVh1Lv2GxZw2GUZHogTuWK3ZW3objq6HE0oReS9vUDwSL/rwuOcrGFdnb XkVD9Z+1eD8QD4Jp0rP0Abc8hkotb/alStbHTl3MzBEwVZXyc13HFDB4loOfhJU/ bgPBgeFh1xXCR15+ouZsbsPkRZ5kwLSsbskUpkXeSQKBgQDe60LcdI0w7edfl3bl pEOyA4Volh5net4dpQhLZFbjFAoTG1OA0Z9hBoZwoRLi1xkuMdLUzJOH6d9sBni2 qn2sbHVKXTtb7Ie+RMb3XMpN34tlpqH2+vnrVkxkrPwWluhynVZm4AsefMHRAza2 K3JBl2M+IAzgsfLUYNj4xWh35QKBgQDXPT5xjdr+NOADocrlf80w2iVnvLah0yVb FgVenkX1u5ZUiv9calJagfaVQNm4VwxBXGr1/3+PoPdKNWFjnlPO8R1E3Eia1Iu+ FberWhEpAJuctT+lcPS8sFDh4r5edgyOXE3ikuI4tDRsQErGATfWkIfmuKUThW2u A8YZWmTXgwKBgQC0VUu0fhEOSRcIzMYhhh5BbvnIntCf09TF1JD+NIDZmIeB53RI 3Mt7OUUe7h8SJQI4zZOKFKkwpTDZ+e5Dj261QtQrwXpYpeM2N/thds4t6y6goAmc fHIbQKfyNcKFH7WRqtiPxqUYF8LPLDNhcYovp6FgS5s/VpVj26kKruARRQKBgAP6 Krw4bdKCsZz/kVs6SFVdwpEvKITG/Qq41nKQJCTNSZLYWq2ffVQU0LSjVSKV4TEX /xX6maXSyJ8Da4BGva+2Pt813tWVldt2VUWCTYzfaQC1TK7G7o7KB0SVRwVB0yuS QxIruqhnlAxoB7mu7hQ16Xz27n302jj78t4nEHP3AoGBAJMEOropyzzX3sYp4E9c u+VPyqcEMXfFimh2il0hijgB17dSW2UXeoQ85gevbPfGURL7a549xar1cNgEfQO0 T82PecgRs83SEcgHT4mntytGY/lsiowAQZ2rXtlyG4ADPwbQGV3UXqV1Bm7yBT38 YJJKJNKjyMHA8auKGG37WDhu -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/client-b.pem000066400000000000000000000023211477524627100230450ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDYjCCAkqgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe8wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx MzIyMzAzNlowHTELMAkGA1UEBhMCVVMxDjAMBgNVBAoMBVNQSVJFMIIBIjANBgkq hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu2zple/6hsvtS/nIAWsJueZcUtsPEEK/ 5Wa6uJP6ZUAtlA0bxjw2iIoW72z/qX4gIH0olnf7PkIXa6Di/4JACaP4g/B2LvRT fmOLz5ScqBILjSczskXu1PWJ1vhrHn8K+tD6Hoin1yuQfCeSmoUkEpGt4YFvsqeB AswKqY6GxlKczaBVaENKPYAbLwVlTgidNoeZHMLCxUkm4Ye+hT4nnSh7QN1QI/r6 4O62E+MB6ZkgHSWe7jb242weCb9+Xm055ZMsAXDyYt2BrVCPDvB0wz9SvJpU4rc7 6hxjA7Yk+jkCaX37iYTifj513eJzsCjfgADEPEcT8e5b+Z9qGJqtLwIDAQABo4Gi MIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFt cGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215LW5hdHMtc2VydmljZS91c2Vy LWIwHQYDVR0OBBYEFAiaPOQsBb3+rAOlSevDy5kp7zCnMB8GA1UdIwQYMBaAFCcU WYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQBq7BxhZMCTl6Xe i3GcPlOLp9JPP8bcyYZVB6nAf38eErpwYSPvp4I3IFU+KyZgtDIAb6Oy04BNV9eT BlpX300ZbylO/TLCrlMIJDYLIt5NciVe8IxsE//uLXFq5wZpcCcL9aQA2g0wW8hi pkK1dQd3W1ryR/LwKiy0fcZJw/EFskoqq6vPJATIFvH/O0OxdKP9T24YVGgLBzwj xzUqVlU0CuP0snx6x4F3Oha6kJGwc90RlXszh2ELhO+o4sk4wtfAlEYaM+H3kbAK lRe8FENyGprOjlxjy6N1noJW9Mgisx9kNcsCzW8auGY6l8Vs3wW3vO3n4ZUnq9qc t95/tstH -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/svid/server.key000066400000000000000000000032501477524627100226670ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmmTq1fDtsMKVU kc9jWb/JlOXU2i7s5uPzETulgrQRhHCWE6gACW3VYhTCecSf6TyLaFeTkwVsqv+r teOld0gRyA/NdCN0fY04OGfr7eg3dDyK+ovGInP1rh+6UrMl2Y+Iy5n/lymFPFkd EoqaKjXv5npGQScvrvTYN8hFu6ww3EdPI3rAYwolujGadbXttprLRIZvZQuYI30J LydCVsFMwfm0lnegEHtr/lx2UbxXeyf9nIOle7UsPAdR7oRXomf5FCoXUuWajv3Q W9NZEM4E0O2SbH/RO12MmmeXH0mwQuvrZAvgEtc8ZDMzmRZ+AOyCXxp6F7ME9SrB pQqS7ljPAgMBAAECggEAFHvephXiUIkiquVKtJKCbEysfaFeB67T9Dz4Mn+BmG3K UgmUrHx8TrJCjy7dqDPLR/DKY9C96ACWcwxoMx4ZdGY+2yoeeeI+DN2SmYWk+byG /SeVLAVscJa6O/STJmI7rbwq8Swq5MDozBRNimsKOrcB7cVxBgHjBPrG2xwaLd+U zExqarE89tfvtFVUQobyeOimcSEerMIkOngwtRxbhvF956VpvVEaJDsm1rHqeiZA rKoky01IKMTXcf6g1bERKKXc/nPbMMOMYDD0FbQM5bK7OqSuN1+XFgLbIQOgShWv OhlENsa8TttAUq6dst+bv+zxnXTLNhV1INEdJWgSQQKBgQDfUktQoMFRCfwnFCNJ 6r4jR108hLH7e8/2AfEJWAW8iwfvnpf4/fW7kqttcZ0FwLOGDb3TKJbXx0Jlz3Fr bkD3BIb3+19ludqI+hZanoSUqXBt/iTWRJpOGwpoLSIGNCR01GePMCva5MvETPj5 nh7la+6AXngvxx8cR7RJMcxAMQKBgQC++hLaJ+cec3YpJGAAIFnXRgtFyzKn+ToE TvcbJEtHn1HO1MLJhd3E04A0BufjglxJ16Kpc95252O8GyveIKm408vbXjIY3Fxa y0zlnUB3UNdZ21HdZU3vr8T0C2s1c6tKTStuxUjtzfu1LXElJp/MxfodnyL0UE4L OiOmza7o/wKBgQC17uY6rmwvaDILaCWDn4D9XrM/jV8uYOTAJz7F/PbXbnCtrwdi KwtZn5iXFdr1h9L5YBKIh2W0LroSFVjyS6UTWJXYSuqiBjJaK1uwwbmDHzLf+Q7r nIIVJYRp8HLqmOomX7Bqf7UKpCC5MHgZa88B2tc3rz68tbzLnkLq6m3a0QKBgQCL 8MVVtH2oE6VXGn96ODp9A6eI+g/hhD/eHlr2OY8sKXOR4tflQcGy0+SSuyi6zrB/ r5JTJ0Oxz3aGxUZM25RiilhMI+cDzp2iLmznqGYvyD41av+/AtesNhlVVvS1U1AF /yw/XDNHhnPWXDkXpwjbzYqsoArGCf5WPcEYHon30wKBgDhFrftmpc8AgXoCDaAN Ymb23qURm8m/+FTc+49xX3F/Lk2stW36M1Q2d0Liu30DTj1Yzkeb62ROycCdMN4l SG87HwV0A2Fs4lAKhaYhd505+3keNg+tAr5puySRaysaT6S9uw9QsoskA8+AZzq4 mzR28luIuvUmtSP+UBrQ6hyW -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/server.pem000066400000000000000000000022071477524627100226610ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDLDCCAhSgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe0wDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx MzIyMzAzNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF AAOCAQ8AMIIBCgKCAQEAppk6tXw7bDClVJHPY1m/yZTl1Nou7Obj8xE7pYK0EYRw lhOoAAlt1WIUwnnEn+k8i2hXk5MFbKr/q7XjpXdIEcgPzXQjdH2NODhn6+3oN3Q8 ivqLxiJz9a4fulKzJdmPiMuZ/5cphTxZHRKKmio17+Z6RkEnL6702DfIRbusMNxH TyN6wGMKJboxmnW17baay0SGb2ULmCN9CS8nQlbBTMH5tJZ3oBB7a/5cdlG8V3sn /ZyDpXu1LDwHUe6EV6Jn+RQqF1Llmo790FvTWRDOBNDtkmx/0TtdjJpnlx9JsELr 62QL4BLXPGQzM5kWfgDsgl8aehezBPUqwaUKku5YzwIDAQABo3YwdDAyBgNVHREE KzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD VR0OBBYEFO2EJhb54LzOxx6W7lqphByWFT8pMB8GA1UdIwQYMBaAFCcUWYVGF5cP rQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQA86URTSVJU6k6VjcnT9Fzh fnid+OV2NPoKzczw4pTc7aGjkZxtCD1ENlYhHlcni0ZFMIRtLiDARjwhBkVJ5S84 1NS5l4J86ymazkSFZ27m8y0UeSDuPxZJFA/yBAmt/BoKRNMAAmonepdx73JpbiGE yMD9RU5qI2E6BGo0B2khRYuY+POPFGPueVbqg3qR+LJPlxp8OIet9HGagEcUK7lG PeFNKSUCfmuHHD/QO/gmG4ZM9/qB7M3McYh4/+CIihEmhfVK9Odo6Fs0t5MQdcEo v6++7DlnpwRnmgC8GtEBMK5XJAILb6cI11TearTphFoP7xpvz0VHp3Gy9mA5cdwx -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/svid/svid-user-a.key000066400000000000000000000003611477524627100235200ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgST6YP9hyfw/Vmoxo MFp6MJFZu4xaYK3OweYcANEFTkmhRANCAAQCY7xD5sWZDVSRmBu2l4sjJYzpGVqg d7M8I6LnFjkhkJFc0h9n8jPud8POip9BfXJyLBzmtW+CfZC84zlFSknN -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/svid-user-a.pem000066400000000000000000000034231477524627100235130ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIICAzCCAamgAwIBAgIRAJXUSiQv6UVx+RHn17gl6xswCgYIKoZIzj0EAwIwMDEL MAkGA1UEBhMCVVMxDTALBgNVBAoTBE5BVFMxEjAQBgNVBAMTCWxvY2FsaG9zdDAe Fw0yMDA1MjcxODI3MTRaFw0yNTA1MDkyMTA1MTFaMB0xCzAJBgNVBAYTAlVTMQ4w DAYDVQQKEwVTUElSRTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABAJjvEPmxZkN VJGYG7aXiyMljOkZWqB3szwjoucWOSGQkVzSH2fyM+53w86Kn0F9cnIsHOa1b4J9 kLzjOUVKSc2jgbYwgbMwDgYDVR0PAQH/BAQDAgOoMB0GA1UdJQQWMBQGCCsGAQUF BwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBS9gO5XK6fzpkTH DuhPV7lB2AAbNjAfBgNVHSMEGDAWgBSeUg2uZMN8Eio3bHcxv7zJIzclhzA0BgNV HREELTArhilzcGlmZmU6Ly9sb2NhbGhvc3QvbXktbmF0cy1zZXJ2aWNlL3VzZXIt YTAKBggqhkjOPQQDAgNIADBFAiA2TvD3xhOCvn9E2QF42o7gTjqGicTeNInKTEKe A6AMzgIhAKdpmH5367YqHijKhtfklnM7g8WhdPhn38xWL7jG+5+a -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw ajELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv cyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD DAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp EHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD fBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw HQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB AQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm pOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK dXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5 UQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw K8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6 rnloUNEr2o9DlZp8EvK2I4dU -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/svid/svid-user-b.key000066400000000000000000000003611477524627100235210ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEiQo4GXKbViodiF2 LltOkXLauMoyKJu01c/FUoGpnXahRANCAASiSiVhimnedxcnXY1ffLWV6Ez9XIkq 3pXxtk6q6jvDfn3OPPjIB47OH4KCqNaMoIsKxwK/mtOEETb0/gFqeQWa -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/svid/svid-user-b.pem000066400000000000000000000034231477524627100235140ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIICAjCCAaigAwIBAgIQUGwbDXAjCmdvfiGGjS/+PzAKBggqhkjOPQQDAjAwMQsw CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MB4X DTIwMDUyNzE4MjkxMFoXDTI1MDUwOTIxMDUxMVowHTELMAkGA1UEBhMCVVMxDjAM BgNVBAoTBVNQSVJFMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEokolYYpp3ncX J12NX3y1lehM/VyJKt6V8bZOquo7w359zjz4yAeOzh+CgqjWjKCLCscCv5rThBE2 9P4BankFmqOBtjCBszAOBgNVHQ8BAf8EBAMCA6gwHQYDVR0lBBYwFAYIKwYBBQUH AwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFOZVW2w2T+3afeJU JMuZg6Q8FXc/MB8GA1UdIwQYMBaAFJ5SDa5kw3wSKjdsdzG/vMkjNyWHMDQGA1Ud EQQtMCuGKXNwaWZmZTovL2xvY2FsaG9zdC9teS1uYXRzLXNlcnZpY2UvdXNlci1i MAoGCCqGSM49BAMCA0gAMEUCIQD81ueLXy2MerMclzKoMnP9VDjOLuHVHf7RkLYb OdqBigIgH0XT2q5pVmDQgCBP2bKaWZndvXlb5kkPw17XcSD2cKs= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw ajELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv cyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD DAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw CQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp EHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD fBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw HQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB AQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm pOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK dXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5 UQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw K8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6 rnloUNEr2o9DlZp8EvK2I4dU -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/tlsauth/000077500000000000000000000000001477524627100213665ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/tlsauth/ca.pem000066400000000000000000000021231477524627100224520ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDBTCCAe2gAwIBAgIUYm7HlYXiXY8u67v2vKjU5LUCHDUwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MjRaFw0zNDAxMzEy MTQ4MjRaMBIxEDAOBgNVBAMMB05BVFMgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB DwAwggEKAoIBAQC9GEoJ84UWZTsnovuz3StEtri3NdKlOqzNCGvurd3rZgop/jZB RtJXQz9ZxdKx1ARpDV1m3CrQe63UbBRXJxA2XGfmBQ/BPo3IPEXJOsNEs9x5RsSL RiJ2re7jbWKeQv/ucQPdmLJumAp+TGAzdOM/AnDaVTSLPoARp/Va8Frs7iFfPpuJ tObvux4qnb/hxS2z39MWjyeM0dVOmjGwx9opxcE0hNI5ZutkoNxpmRayZqJSe85V BSPGsuBwgncvA2GWTNIGFfN2oxQhSuI8yM7+l/0+BHFWfm2G7/09tWDvFnWTSpTQ VISM3+6Wh91c6qSd0wsIb8q6jADAD6H8yhT/AgMBAAGjUzBRMB0GA1UdDgQWBBQk vMZGyNfVHU3oTUASSfYiT4arNTAfBgNVHSMEGDAWgBQkvMZGyNfVHU3oTUASSfYi T4arNTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBxN8yaacok 1lyg7eTEv7/Sn4VGkfbuWm9ZQ3DPLKJhbe7uVxNChRrR8nKshwlGv1Wa8i0c0lJ3 O+Uw24gjfhl/8zhFyxh4ijllj6/FVBNqsTvnGQqtKJP4h8kScUIH21mQ6JQAvh4e RY1sjPwZp+6vvogSrgQQ32jBaa8vfzcL2wECvnT1YqePVZYuRqEBjIvyG0ALlmE9 DqZ8gH+W8E5IVulLVJxnYArCT1dW5AyM2fBETLB3PAWvSBkaCBl6QR+hLuyeR4vT m6Qx9EKr8MgIpiH7psnx8C9eF5j5HiwHfhwAdWD9W2tRzTxSZP2LJ9E+qjaohdLf 6NYxXL8AHa17 -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/tlsauth/certstore/000077500000000000000000000000001477524627100234005ustar00rootroot00000000000000nats-server-2.10.27/test/configs/certs/tlsauth/certstore/ca.p12000066400000000000000000000020531477524627100243070ustar00rootroot000000000000000‚'0‚Ý *†H†÷  ‚΂Ê0‚Æ0‚ *†H†÷  ‚³0‚¯0‚¨ *†H†÷ 0W *†H†÷  0J0) *†H†÷  0fÉÕq<äÇ0 *†H†÷  0 `†He*†Ÿ)ƒùÃÞv‡¼m ·´€‚@?^ƒ Ãà‰q{5ã—þ¬ ­ê7}ùØЗ\£á¦¥;û/ëa“Ý^÷Õß»*XŽÃèh’Þ­ð– ŠDnoÄôŸ¶µ¡Ll +NG±]þ•fÒ+Þå¿IŠË¥&X0 Ií’ÝÙ$º·O×Ìö”xÌEˆ° Q)­ÈwT›z!ÅÆЈ ê8"xVÿ1^÷ÊÜEìÖ $@¼îw­M:V7Bå$'Sè3gæx{³7 öâ%bLÀJÑ«¡gãPö!´ \Ÿ¡Ñú_Ý›%Šë­Â®án0úŽ[wÛ•ŸÇõÕÊ":èv7gz7VŽÚŠ70÷Íà ¨më^®ÙɤÎQÒΓ¥U{i‘8/W‹bèÐ\ö*kÝ=mÏÎ_£•9uzíÃâhf6*m¤åiKȬæÏGýÓçÕlž>ˆCê³U4mõgá n©ddGé!³£OÇ-„†Ñ”­V"ÞJ'‰6äS)ýÅ:ú[a¶8ƒßä8è)v§f#‡ÓÐÑt÷ÅÌCü£ç]n!M¡K!v@ù%Æk®övÑAí½Ì­ƒPdõô C¹Ù%œˆ&ŸÿfÌLƒ›„QåÙÑ~[ž%eÕÆ&*6‹QW`o•¡ÆÚ`Âæprù!ŠÌÉ»Ò ÑBٸ͞Ûû9¾’6øÑ¡¦~ã悯f¿‹õ²îè½ð€/M hícVúÿ· pG{¨³b÷¹c§T¸‚œy~’²Å™ÿ —'4Ip¤¯»ÀlÚ ‚¨½q«>¯ÝN péOÞˆp»¨Š¸Ó4’ O75XýÞ>-Ž‹ÝÙiá03M´Ú š“–c2rŒ\*(Œžý¾µ« m(´œLsŠ¼>Âí‘6}§ð vGÛ¸u¬ê<3Åy¼zÍåHÁj 7Ÿhça1Jz "G^§2”žØ9?±Æ,©Ñ&G¸ªGüáÂHÍ‘Á®vò›ÂÇŸ#û#å€äS“0A010  `†He –#œapKKº ¨N}2¤o½`…E0åÆÕxÕ$Æ(©öHnats-server-2.10.27/test/configs/certs/tlsauth/certstore/client.p12000066400000000000000000000050231477524627100252020ustar00rootroot000000000000000‚ 0‚ Å *†H†÷  ‚ ¶‚ ²0‚ ®0‚" *†H†÷  ‚0‚0‚ *†H†÷ 0W *†H†÷  0J0) *†H†÷  0U^Tl0 *†H†÷  0 `†He*ªûŸìWšgõU;ÝRÕ°R€‚ ÀDC‚³WLÑWOØ<Y\Ü°§ÍÇiÖd• 0)ÊI`©¶‡¡ Ãý¤ûï-rÛeC•ˆ¢Úåìvœ!um¨³H‹Á½Õ³w¼X˜yÈN- é[=À!mÂmvà¾)LÖBb¸èkÅÄ^FÌ—µf6å¤XZWþ©¾‹ˆ›R´R™[äÌÛ}P::Rl¹·`©—ÉÔªä9]5ÐZ…Ý‹K§v›KGð¤Ä~1E¢úš#~vQIWC6Bþ¼%ÊÛ0¸i(\^ßãÌÙÃd^ò: O>µzµ‰¡'â\º’G¿Gþ0ñ«s© м5Ÿç—3p®H"lH­ X‡Æþnÿ =™•0e•`*˜ÉàqsI^¾Ó»þO1ˬ—Ê<“ó6¿öœÃMN§¤ê[NxÑݪ“'–«ØÙm¦õª gæ¯kÅýËðºfh™‡"gû¤¡Y¶ÖÝç)æIüY§\O¾¨|¬Ëïpì¡;{Œäöy($?TÛvä¦á’ ðÕÍtÔîS‹Cw]KhDæŒHô ÔÁós®ëîà ᆾ–å2Q‹¡òÓòyÈl-ÉQVM„Í4·xpçE5=ÔôcªŸhAØ3§Qü RèªL‹*ìFÕÊ*(€+â8¹sKf=¾ _d[›àÿmEnB„Ò‹Ò°£_”œy5Mï¢ÙF!”'ÂÕK̺6qY±«1©ÜŠd„Ÿ8VDv_4Ìw¸™ ×R%þü®Ãò!x Z1?…±é8½Êhà3íˆf®Èúv†÷¶ÔäHAúxR ­Ø¹Ò¡«²ã–$˜ÁÃâV5n‹ðCï9«zˆ;lŠ/~ü¨ò´ˆÊš Sà;½¢" E¢8BDŽuÆàÇãm׋“obú[¾ÐL½JmR£¬ˆ€È?J¦¢SxÕ¿·Í3„µP¹¡ˆ?ð¼è:Î×4Ió˜ᩧP—mßþž`;¦÷'³ÚV­åp*$@'‘Ô-ÉûV#Y…øÑþ‰ó±ì}÷ƽ1p9c"¿¹–ñþÒ˜|¶lPz¥T÷ÑÕׇ–T´cª| ì‘ÍÕòî ™GàlëÒZ¤ä<½TTüª@ì® 0‚„ *†H†÷  ‚u‚q0‚m0‚i *†H†÷   ‚10‚-0W *†H†÷  0J0) *†H†÷  0´.¦gŽ€Ê0 *†H†÷  0 `†He*;öÜ»…iœ/Þæ…!‚Ѓ<ì®qÖ/V*¢ežÜ‚è‰õi¶2Ùš°mE‰~‡‘¢rÙPs‘ ]ÃÙ˜;”Ò½€çæs$ð½ñ_bòS‡÷Â3{>Ñ«?Ü\<‡3Cüžß+’tÚÃë–DÌ$¢iéÙgoÙ)s›2ðv*ËÆô~8€`ôìÉÄ|F©}]–d Ây4éEŸPq4ÖÑôdŸBÛeÛ Ù°égÿ&D‚]E3ýïƒ/9èá#/šf¹âwåôzÄHx"íЋ§Ïpà6ùÊEVX}·îžR×%ðT†äÅc¯‚¶ÌD³&€!¤aS´UlgTEJß–å¨E?T*[ãϳPÿ»ð™÷8‘!2º[) /À/t¨‰ò¼:c.íûƒ­!›9ÓuÆ~Q#-DŽ@Ÿ®ãNÁ?é&늞Q“tÐ^wµ=²[i.M|Å«KгË„p·»VªÌJê.™tx ;uSõT/…8:7€bAÐUdœÿZO­ùõ±H¥ôjïW­O¬7‹¤Ü<ðˆ87_ʼðµ™Îý9NöëZ ¡Úë•;( võ Wð¢·5O‘œ??<‹WÐDzÞÇÔƒ¶ó¸Q;ÞÊuÈùyׂw´©×¦œëP±6hdªËy|ïÍ)’M+¦-pøHnмÉqmx¼-é–bz¯ð­þÑ«=µý¡’TƒGÃÞ·Þ×bOV¶‚—ìŒjðU»L¥€„C¾ÊˆëéA,¨»/3ª‰gð4Y‘ÌSa”õ¿[õáXó+TäÄ,¶Éñ;ز|µGÂÁÆÐÏäÂœ2~oòpé[EV·“ä§8ð™¹øÒ(78U6kN|µ&‘­o_—oŠ§áüÈ̓`¿"W¯l2S<¯—F°»!3Æh¸5þ ×帑ŸÔ” [­NÆû4“U ^Ö#ÞƳµŽ7tÈi]63×Lr_Û8rÑS‡Æ;û‘E¦å£Wi2´ @U­˜]n)™ùÍûÆ® ÈȾ¦eMlÂlE8˜xÚ¥—Ö´#†DIn ®{ØM ÀFyÚ‹€5…º¶FÏ’vu˜0>t¤Ž£ pF9¾©ëW ¶ÚîË S¶O>‹£„ÁdR#Ì‚œcoÞíêI·“'¹µˆUê«L$¢I0ôb;5öüsîËËúÃ;èóÞe¶)†"¶Ý¨Ê–8¡œ:?ؘžÏHÿŸä'Ï$¿”ÂBN! ÿ$’PýÙëZáo‡¼ *,{º×ág¼ÛÆ—{q¬–l|1¹É3>>;+Û÷q½än«žÉŠmfžÝöo:`.à,€°­z´Æ›û»;…KVûŠ†n0ŒÑw¾ w´±0ÏØÞy™yÛ«ä°2·:£3ýø- ›AÌoN La~µÔ@×ä×ˉUï°¶7ŠªxÛÛsé$_¶¡\€;?w§Œ=Ô3³£3ôr2 ± }læÊ1›÷qMRàPyëNmn:ÈKôúø4JÖwÄuná¡T{´ÿ/v+ =Î7K ðëÂë„;d"­ýªßŨ҈ ‹ 4Q?Ä ܬå\j/.¯*Å=28ÑôÑàœU°+oÒlmÕ…²qóaysÈ’0{î]D™ÒHK8Ö¾”r™»k†ŒŽqD;{Pa·#´Ðà¿Î8ñæjÔß~³ OSõçïp«Ÿû£ý“éF‡x;úw±Ö¿ªCÉ… $á›îŒÂŽ–ý•e]ÙGð³ñnsl°l©Jxª„leX…§'Wˆ».¥ÜÂ&™½ëIÑ3¡PïjzØЫÒõŽMoÅ°eÅ5 2ÇéÜÏ¿‹£ËÓç=FÃÞÁ¦Â€¹*¯è¼/)ñÃx𸄠/ ê^g0ÿqТç·n):–Áé pùè·Ì¸ÔÈ&ÓÚØ™›®±ëJ¹6$Kùg¡ÄÑ.o ½o¾rã:’ŒÜìÞGKÑ{wxA¡þm…x’Í`l¹š ðÒå¨Þþ¥àóòtžtîôo½(á×!ëŒMLrºÔ AWËXT±ÚV0‚„ *†H†÷  ‚u‚q0‚m0‚i *†H†÷   ‚10‚-0W *†H†÷  0J0) *†H†÷  0bVËó7H*0 *†H†÷  0 `†He*]¼ûøv؉˧s»¡÷‚дٖæ!k™@Vrðsº1Ó?ÃÝÉC—zÝñ^qÚ)”æ6dÛiü0-ß«aP1"ÝLjõ×k*9Ôs9~b³¡úw–mIŸ‚³HápݤšEè#Ó·e¹~®¸î45Ö='ç±?A|\ÿÄeòš„$Oõ5òJ³ÖVן Ó%óªÐèñ¦p¼ÆáÙ‹»J=‘i$‚°ßeÏURÇÎñEp­e8èlJ*Ÿç¿ K´Ÿ ±ÿ¨}?Ä#èçŸW×lŽc¯*`-aÌò¬M3ýXª¶Á«UÍÙZN€JœlB1Ê®õ챓Bl1øÑíp€’ @wŒÒø¼Æ´Ðo DÕ:õ‘fÁW5Üž7W/Ï]™[ l˜ßòÂå˜ô"MC­yÚ,ýßÄqT;F€\!òÚL\>ê¥×J 76!-0®âÁijܸŽ¶…j„eþWÓ;ž®‡Q¬2KŠÇw¬—\/”Úeø,ªè†ý³T•>­ÚR{ÍÆXCsˆßûçlÒÑ\õ¾¹›ïª˜iØë×1fj€ø5%lT¿ïJ.ÑÖ§U…Éàò¨ÐÝ•UKÐ 6³|”~|‰´B«u76Ïã눲÷û³¯ÓL+„lâïø,]–-C¢X9)Õ¿‹-œx»}@ãOAW‚¥2ÀÝ é©Áê…-‹]¥oÌ85#©"7R ß7óc唹LRÅ–ÓÏ…A¾?ÄÒ~(~Œ;»ã¶'4)è¥ä6¨Z9ÿMÑ• ÆÛµÚã’”y멼¿º!„a…ñŠ*aðžü˜Çñ'nD¢vaXöðwûáL¸”#9.ˆZíÿiûõÀù n?mYîë.ÜtËÍ^¥¸Ëjãz Î=3ßù~bÞ5YÎâÉ¥o·ë¹ÉÓˆï€'–cEÑx·n¥C'ç9n…Ÿ 3ɹɾ17ÎVìºT7r;µ!1ó=ªÜVZþå©ŽßdªNÁÆŠÐí{^ º>.?É!ΓDVc‘¸ú 7勞ò^µŸd­Ÿmjß½Ÿa&’gL¬]c?_—ƒ~{ð|U ¼ÄÝ£|ž{5›²Ýö ¿Uì0…È×›{¢š‚í•‘º®ÁL`aw éF[ê\r²ðÊ"òk§Š &ôæð^ò³Ò4U=È·G6á”[a£{1 Xº2Æ6^µ[žle‡•óQ"h÷© >É¢ŽÀxWècnÒ=úq`òS×&RÖOFB’°.áAi³šïæPûn9c›7}Ëäs—÷ÝæÅ´’®ÿŸjŠ®öñ.SUNw´;)¼ä"½õK{EüÿÉ¥ P§†8Î"ƒBU§Æ–Eø4WÙjN“¥1z¢åbµø13¹˜äáªÎºóñ‘œG÷.Ø×#ã’m⦕>›]kpKáIu“ÂcÃä ÷bÚŽ€ìþDU&‹‘Òû8ˆ ôÚ+¿é]+ãkg :]РޱÑ&„¸™r $Ü«M§ÔDÈ‘@}ƒ‚ÖP¯Ó _. —­(ÙÒ; a¹Ôà¢çhqº§„„T´;û®Ú_J\Bš:ÐÇC/ŒB¬ñ•ªfnÃ÷Û¾y8—ŸÀT§ã-¼ˆ€™9ß|Æ2Ì1+ùK¥8uô^-śȶ9¤B±,ÀLöu•2H»tµ§œuëJüÛé)$Ú×ßÒC‹xãÜ /oU†µx¥6h °MЗ(«õVñø¬Cj»­ð8ñ&š2C:ÑRBÝP$»á³î„Ž­#$ÓøÍ oiDò—;Ñ"ëÒŸÎ:™¿1å.…>vX×…ŽHßI¸(}.@U®åÑ™•Õ £†r˜¾õ³ìÿ£B&†üEî¨#`¨?ÓÈÚŸ5“úiH#u«Ÿ†d”¼la qºéµ™Þ¦á¹ü”V·†˜{H@h`,âˆ&r/ZZÉnµ# m˜^Pi’<ÿÿ¤tÆÜÛ²³tÙíK/Bú…ì‘ýfÚWÍ™DÆiWýeÿ0ÍøÿÎøÞη»ÓW[©©¡á)’!40(˜BçâË•Õ‡Þ\p_¹qŒp¥¦Y®îQ ˜ä#U`½;ˆ>K…­}…Eª”¸?Õ[5IÊek#ߺXV”î~q”keåÃz6°ÌE Üõ%-ø²’xÕS†Ã]ÈœÓÀ˜á—wF_‹¨øEœiݺ ȺýìCG’þ8ž˜®î !MÚ°ué¿8Vý߬XmY‘¯–¥²2éø˜âF­ÔM'{qœt»ô´|¶yox–G›ÉÐnÝmœ£GòÁü{&Fæi¥úG+6!»˜Á¬‹Zç¦ÒXîžz8KRƒŸðŸöŸ½,÷Ž‰­õi=t¦Nôʼ­ð›>W¡ý„¾Øâî'n²˜¥LŽån›ýÍ·MÇýjˆ¦´ÆÒ{¹÷b©‡œþÝèdâ¬g/×W˜´Âª¿±¥‹êø\6o¹%às–¸;ÔEOh™%6)s4Ι°!tbgãc;Á·]oN'y•§² ßð¯¿`ƒ}ïî–4áÃ~O1ke¾-JP”‡Ÿ×6œ—/ˆYgÎ]}§ÚfæÊÒH¸V¹¢]–(+D¥$—çÓ8£ÍC¦.ìÉs×êøXŸcN9—Lä äôýúß3¨îâ ܠܪþ´MÞ·æ‘Wm¶nêk‚µzüç.Í©îHÝðÇ-¸g‘p6¢žO«³%àXçuâßD-™'9Ä}Y;„«=ý8¢‚ˆS ðGHÍÊ«G®†”Ú¢°¥Ð뢈<¢Äq…|§Þ+s鞉ˆ}”Ð;™õêü ²è|„^§…²kl3Þå<âo&ÌÜçÁ€š6Ù4@þ$Ï°€›á¤7ék’ŠMŲˆ¬(±ptå}¢ Note: set the PKCS12 bundle password to `s3cr3t` as required by provisioning scripts ## Cert Store Provisioning Scripts Windows cert store supports p12/pfx bundle for certificate-with-key import. Windows cert store tests will execute a Powershell script to import relevant PKCS12 bundle into the Windows store before the test. Equivalent to: `powershell.exe -command "& '..\test\configs\certs\tlsauth\certstore\import--p12.ps1'"` The `delete-cert-from-store.ps1` script deletes imported certificates from the Windows store (if present) that can cause side-effects and impact the validity of different use tests. > Note: Tests are configured for "current user" store context. Execute tests with appropriate Windows permissions > (e.g. as Admin) if adding tests with "local machine" store context specified.nats-server-2.10.27/test/configs/certs/tlsauth/certstore/server.p12000066400000000000000000000050431477524627100252340ustar00rootroot000000000000000‚ 0‚ Õ *†H†÷  ‚ Æ‚ Â0‚ ¾0‚2 *†H†÷  ‚#0‚0‚ *†H†÷ 0W *†H†÷  0J0) *†H†÷  0™J‹Ûˆ!£¡0 *†H†÷  0 `†He*|Á˜:à·iªëíåKW€‚°/‚ó˜ro¤ OÌ‚%áÇz!$J"bnºóŒ7®[bIBØeT Øw, Ç‚JyíÏ­@.Þ`Ø Ð¿¾½îû„çßûMCû×:>´ê ~Ë[2AÒ®éyqUnàL¿h@P¾?Ÿ—UÚzeg#œkQ]µäÖnú©íb¢‚û|Þò8 æJ™‰Lº%äfž¯Oç iÀz`ø"Ʒר¾ "î°¹•„n`1ãûáÖpÈš1Õ®Ú¢‡âP³¶²•tQC~9‘ì{=õº1fY˾vÇ:}Ñ]ãMÊLBD¡u[c)¤¦¥5I!O ¨Ã2ƒó}½gSc,ï•|Á9x&Pó{§_¥o±$×½®þ*ÁO¼_Êcz„¿Š³ÊdEbN§6‡£ o¶æ ¼BÛZûçòñìbÃilê™,¶N"Ãcd#¼$ÿy 6¯£#© àç[b ø£^9Ëó£Zȶ)F¹¹ŸªL:Hø¹jhrD ­&p#Üó›{Ú³’ûÖÓ-éñ†ôV–pÉã—Ý ÄüFtÓuü¥CZroJàÎ6<›Í:·ÓíNùs¹ü¦§ä*).k_6Òåt“Ï“òÞ•¶Má‚;¿¾h¢‡ýd¨0&¸<4Å:Œ›Sýe± $qi^¬å³t0še½L+dyF ná‡îÝ2ÙŒ˜ð#›p‘2Ä'»û!¥7d ¥iÁÀ\Ôɫ-7Ái'÷{ºÀ€TŽòüqô4¼\ÛN; ó¶çÓáÙ>‰‰­µ¡%Ù¦¡5 ë–Ãv1!úÒùct¼:7Ìæ9x’´ÿíˆZ_Ø´F bUE#ág?K |”ý}Ðû¥ŠòÙê Pý{TœµŽ‹¹æ|Çv†'aYéñ’™^^7e?¯T¡(©GÝ*DAHŸNýbۻυn Ø #ßú¡žt°SBãw} ¿¹I;v;:VvOQ®³ÁÙ/Ó¹†p¢R6~¨â 6:‚ÝM¸:LÝY(6.õzµa"ÂÉï\£ÎŽºu{ô]^*AÖŠ…bÙŠ:§ ã]”>ºÈÏ¢b é…cÚ¶¢4d±Ò¾ü±,nƒ)¯/üœ‘¸w’ mäÙîÞ„ŠÌ—a ¥TêÑ~ÿ|—@3sýÔwêÒCf®D ÃÑiøEœ—D°b «°Cú€?mc#Ce ãw? ¾"þ¾‹©S2ÿGoÈÇÊ[øq¶BÆ&G±0‚„ *†H†÷  ‚u‚q0‚m0‚i *†H†÷   ‚10‚-0W *†H†÷  0J0) *†H†÷  09 À±Ø#0 *†H†÷  0 `†He*âWÔŠ #B#Y!ãUÇ’/‚Љ†»ã4%(wÇÌmBÄ“¿(`/'¤ú‚ÓÖ¨+Þ?œÍ Õ~}øòÉ€q.÷˜ a° jÕP&}*Jk]¡ X§}JŠÇvD¶åW…e°© $ò7æüÊòD!ÀóCkÎùFžÔcx Ñ¨Ñò/¬B”B&Ç| •ÏV§F˜Æ&4v¤¢nÁ§ë}H®p²äG1.œ`Tç0Áj<“£Ìæ‡í³}-ÖÂȈëÉ^žñÛñbïªÉÊ´ù¡DòÓšéÎ×q{(õC™GŸ 'òG­þ*p+nêàÕþ‡sgå§E7lÇÆð@SˆÕPGä1M3ï§ÇÉŒ+ \@P€ÝÅžnÏ ;k˜Á€—Éžcõ¤ëË?^ð“FXN¤·©ŸÕÚúՈ﫻AÜ Ñ? yùRÖcH›& Û9/Þ/¥ÖúÔhpuV6.qºz>‰ÅÎqÞZç/±Lì’,X‡õ}¼¿`Ø_Gp.÷ñsÎèð+ó²P…¶þ¡÷±þP"Mã…"®½Î本]’–ÿ–|+Jú1þtM«Éán?]Q©¡,5 ¤0ÝNk’3ñî¡ÇÒž¡OÜ?¯À©q<}þ5ú9Óé Ú¦v®tD€¡;?×õy¤d>Š­6â7¦Ý%2°³eŒS*Ú#³÷º ˜Û6؈¡`w[.*'FÌAÎ&U¢®¡%a™S5WÀD´º7÷ñ_h61ø+ï~ßG¥¶x¸¯‘Mà„þ¡>G~èÖÙgfÁ E·!ôˆ´2Á:²ÅÃÆv2æÌŠÉ+‰3{6À* -]Ê›ìC—Ä1Ø2¼ì—ð†I‹ÇyØ{=jɾÈ#éJ¦p .æÁ†Ö\qK¬Œìcücu×ÁZ¯ ÷Ý¡|úÀ òçL˜6=%šæ––_^Ns5§à(÷ÖJ›S/wÞe8G'qXJ¾°·ô÷%L=Rµ°]¿qÜÔ’àÄQ/¤÷™„(áõQEÇ@=õNv¨%ëxJJìòŽ#QÖgBË’¾ö!—QwyõŽð{•¯>D&v/Uù ¹”¤ÇrÞ½“±ÎÉþ‚Á·°g¯£[ªéN]†j¹º1»º •›€14è'ßȈ^{[¬Õù›ô¸(Dñë1º¸Ì“o²íŸ~Ž6‚lé;UÏPî­ÉêÓ«#­ÌËE-¬òп")Çã!Xz_«H…:¦-Ä‹û÷è2D< Eì›—©çÓô¾i:¤¼¸TRï™;µž”&´¼¡ú¹¯_Õš¡ü¹p^ô¥AAù~Á—å":lp«_S…œê—¼8ÀFµô|94v¾vçܽA¯ÿ¤æÆíßO͆ô§?"– Óoô'$[ !"6Ñc÷óÞÇ‹ËÀÌ"i³°ìZÎóê’ðioÂÚ<…|ËÁNqïÊ%¿@þ¿À˜’çZ ¹rƦÇšÀ,îÏï™Å»W_ÛpW'þñ7Aï¬Jî‚@~[ŠlÙö÷œëÿÌ„ÃØ'ÓkV‡€WEc®°— ®:˜“Ÿ{#3m˽g7PæÑLŸ¼ã¿›,8\°A®6㤒©¨È»Üñ’¢CÇÏÛÓžú®t|iûj1%0# *†H†÷  1ßØ¡<‘IT€Z‚dw¦…‹IA0A010  `†He @¯ôžbê€En¦ùf¼÷ÓV ¤©É€74Åò1!¿á>& " ¦¤nats-server-2.10.27/test/configs/certs/tlsauth/client-key.pem000066400000000000000000000032501477524627100241350ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjAaejJ5wqwsst 7T71LQCdsbRpctZU2ZbTIOalEvHNikkdB2gIqxlmswxITFVtNp2IvVhcd4KaNGdu a4uUc7BifG6XwFtmw54SPTlbFtZbr1y4mjF6+Wm3tV0XnMtM18wvkRm7wbYG5fz8 BfWl4tbCDF8lnuC3YHFwVJst+o1HoZ4YNutDFJqMXkHuxEPsI5PmS3XHHpttPQGr vKnoYR0y0Cf4y3LCfQCQ+bFe2JS335WmJy2KVm+dpN+L2ASO8VvV/YTltSy8f4HQ 3Dd/VjZtri7ANP61Xrrx4OJS6IRkPo3uxaR306oKyF9a4irX1KY+Vi1gpYOX9foc mVumCWCrAgMBAAECggEARk+F7CG/QlCQjEhb0ixtqheHPr7KhYHvhTUZV5XC2Aow fEWAEdEfnUVY5GyMopWewOcPUJ86JeK5xI69/7QhHnIWz/0oT7zMF4jyDwDcSGLt RzE3a5heid/Aflli9cvVZqUbaPnm1rXoeBrn+PxN7xigB92uh1qhw7ay0tPSkdUL sIA2fkouwAYIE5TDhKk4lF8Xg6KlYJEesFDGTVsaGL8/RH7VEH/Z7EssiHF60m4l Oj6Vg7gxMB1UWQFw2/LDMrDw0bLVdAoOC0M7NLXlFdoeKPREvFpIqw8BWMF3NxRV OpUiLB/tv6A7nqevaa8KoWGUlCVY2DkCkgCmg+K8pQKBgQDVtb3nEJrOR496BaLn pHKV+pEQjlVfcy36CoTa2PblbjkgSyA6wL9Itpkw4OF/Q8dop0qlbqZM9gUdjEhB 6DfVINNMGx4SkWDtBvPwouCRIWRvZUXdVuZMgDpUY+5n2nEkUHbmSUEtPbhHn5dp 9Tn1dsjNjweDTx2Q6nDuKUlSVwKBgQDDQ1fyyp6qI6opeFkfOsZ78Yk2lM6tRSkI JqwdOwZoSptZfiaoAQtmwU3zxioFgIv4FjbPCosJyej0Egv7oyMucZHPGhP6kZDJ LxZ0Jp0kDYYFCmUjOEpDf0mSmFEwmyPDb8052To3EuA4anEQd1KURG/0yKE4efwP 0m3ml9R3zQKBgAh1cxjMPXRgvLsVsgb9KVPqYQeIurRWeMFm3S9UWyFlpXkzwAjT TD7yi0m1/Pbuldv8kyXNJWPycO1keg+xw1P6QqLGiAAwJOf82Hbz23OjILiQB53l LKRmhuiENBGEQeowDSS8TYoe4UZkeLfG7w5aL0SDnsaBwSfVP7cNh0ttAoGAEJDO DVMTUuvjq9EB/pxF6o37Th4hyqFrcb2WLIStbnul4lnJfcdY6EbODjhpqD3XohyA WeBTG2l90fcV/StB+Na5wBA+Uau31Nmh1gjQnBZpoFPZcLt90WwjGcTCXpVK23HI v3emcLWxQBgHr5Xv85Q6y1GaG+h9cfowSLfo1qECgYEAmqGyMa9OZ44+zni3nq2A JM2t4B5QJdG36R/2m4J7odUnmsaRVxw6JxaoF0CctFxGs/zdumwuhYeTfq6Lj/ly N1kJbvJJpguN1h4sOOLEk646mZzK2h5bMxsmPU2kNZFf1EUlB+Ro8i1TDvGY26Qs qgjd6RlSZFBmSfVS47ihEQA= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/tlsauth/client.pem000066400000000000000000000022401477524627100233450ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDPjCCAiagAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipYwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ5MzRaFw0zNDAxMzEy MTQ5MzRaMCgxEDAOBgNVBAsMB05BVFMuaW8xFDASBgNVBAMMC2V4YW1wbGUuY29t MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAowGnoyecKsLLLe0+9S0A nbG0aXLWVNmW0yDmpRLxzYpJHQdoCKsZZrMMSExVbTadiL1YXHeCmjRnbmuLlHOw Ynxul8BbZsOeEj05WxbWW69cuJoxevlpt7VdF5zLTNfML5EZu8G2BuX8/AX1peLW wgxfJZ7gt2BxcFSbLfqNR6GeGDbrQxSajF5B7sRD7COT5kt1xx6bbT0Bq7yp6GEd MtAn+Mtywn0AkPmxXtiUt9+VpictilZvnaTfi9gEjvFb1f2E5bUsvH+B0Nw3f1Y2 ba4uwDT+tV668eDiUuiEZD6N7sWkd9OqCshfWuIq19SmPlYtYKWDl/X6HJlbpglg qwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93 d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFA5GJsoT3uf5GCyL5KSFoyEuRVltMB8G A1UdIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IB AQAmBOQOp3CR5/P27RUonsgf5MCzblt6tBPJBMiZwB8sJyl9fXCTkzdqD0rJQb7q 5s7p00rUXdeh13oRjFcoFuopMDCk3kBYnKJHXRREJNLHfp6CPUMKlg0GJUZ6v04A V7gVuhvmynHrmlbibMwbgZtZMnRU3x8JjawEUsEhoj3O2Qfen3sNfaOBlnwVUCBQ ygSHQ0Pto1kQS+1Pc5DCwnOZ/qh5lORPdO1MNKqeu8HiiSJfuaCrQQM9zm72CHHY F755qy8OvWjwK0H9rCFBYSrAnYk/pTvXIeBsgNRlURS/qv1rqIEvAbXhRnw7oyvl P4bYY4pcpk32Ir2mFQFRQnSh -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/tlsauth/client2-key.pem000066400000000000000000000032501477524627100242170ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMsHsIYdxGt8ws eoxGQQs4K+AfvznwuW+Qbnx+eGTS+OSaLqL9u81JdXxheeAVsVjRtz5qP3X9Vu75 gSQU+AVhFp689VNR41vaOsKvX2dzVCaWCEwv66CekAop89vyB6fm14uSr9IusixH u6vfjuze2FZE5xcw5mSYMysqKTbM4DMjXFJXFg5mw1YC9zkh3zCWUZdmidafwOB6 MiJkVZsAQjAwc5oaD4I/ubCdmWr8COYbbwUl32k+t0eQfj9cvd4+R7uBYVNWu1BN g0RqNZZlIM4EVonc1J+eN7oGo57jLrwf4jCOkcg2wd6X62CuuQHhwddQek4KGthM JUIH7Q9TAgMBAAECggEAE80u2cy9xomZUuQ4FcPNFg4IjImvTT5jMJG/sWxsNIyn cNL6KZm1blnTQorLxs11TjRv8U9aVrvGOpTnrK+htZa+nIEPImjgRehRVS3hkCKf 6Pu8gxZEX5KHqS9SI8Ph1k8bzYD80E+kQPxC0Em/WH+NOPUyJSTkrmSk1FtQVdle KFk0+dRm10guI/rcInPUjpY6Cv9nSiAhanRPAtAANUvwYxVEMKzx5ns/JnQfAYAK gYTopXAT61VqW1jn4opjD+OqparjbK362q/spggHGPhSamXq8y/N9JadSx+5591u o0oZm/UxefwP1CkE4Q6Pv9n3yiZOwVaOiLLVzmr6wQKBgQD4kC953if336Ga+BdB Hj83qUk5U2Avp6qWLptd6z/VQXDwlL2oQaaXP83T6XfpXTMwDabUYkpo1JZrFOJl s9eZfxIDa0eH6evphqp6s/V6EhZkI9lL59ptZlK1hv1glmFDfn7WB1Yo7d8GgUx/ S7CKoBK2WaX8YNsW9fIunadJeQKBgQDS0EDR46KiQBrjvNK4UxdfERl2af+6Sf+J BRlfj+YA9/zKEL3jZGWVa5rgrD8BJrTsa86QSSACsxQx5J9vPogKgogmghd9RQVd tHvIV0C2C6zCmZIvNuzn5+wMV7bWdoABPgbuZGTpc/LCwm//1EbwdNCTom8ssE19 NTIBb7t4KwKBgQDXOqKheAhLzkz1D1WzgSlkXSWWieeD3D8OBBVsYcPIOP4+k80V 4KML3KexkzvNynIEbg3DYcjktQ/6cP8I6Y0K0MkcRMyPl7I7Z+w+i41Hwlm5JIGI BJ9Sk4OSw+yqsgxOkT3qvjeRAUhZLaS7pSKdJraNR1s/Ce8sFpM6YjD0oQKBgAUc gYXVRBs0/LHq4R0Q/q8SZhCl70pgAu8ajYvwnD4HxTxM/Z2m0IO38TBjXL+1ZYuZ 7Y84BquqFeJDzc3PsVK36X8thk5GPyQPfTTVUL9ZNx4cxRuZ9FKHIAUIl2lJxD7D dz2Od5fldMxeFIMabYHlAy2hMZrex3IyuPyp7dyzAoGAKCqmXStl7QzV6ovrjyJ1 oMP3BcQhTe1CvOEW7pGce/IGxqB0p7gICIH4UUDEVpitgvGLkOGScsd1rS86Fn+/ RUlQQdgRIwzrlF1USV1kJ/SUI+7BbhiXUeRUmKuzPLgLSq8K3xEW0z21juSvq64W gtiiGjt3D3I47v7ecjJ9dIA= -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/tlsauth/client2.pem000066400000000000000000000022341477524627100234320ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDOzCCAiOgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipcwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTUwMzlaFw0zNDAxMzEy MTUwMzlaMCUxDTALBgNVBAsMBENOQ0YxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIB IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzLB7CGHcRrfMLHqMRkELOCvg H7858LlvkG58fnhk0vjkmi6i/bvNSXV8YXngFbFY0bc+aj91/Vbu+YEkFPgFYRae vPVTUeNb2jrCr19nc1QmlghML+ugnpAKKfPb8gen5teLkq/SLrIsR7ur347s3thW ROcXMOZkmDMrKik2zOAzI1xSVxYOZsNWAvc5Id8wllGXZonWn8DgejIiZFWbAEIw MHOaGg+CP7mwnZlq/AjmG28FJd9pPrdHkH4/XL3ePke7gWFTVrtQTYNEajWWZSDO BFaJ3NSfnje6BqOe4y68H+IwjpHINsHel+tgrrkB4cHXUHpOChrYTCVCB+0PUwID AQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cu ZXhhbXBsZS5jb20wHQYDVR0OBBYEFEbKWjY1gwvxJusV+M5wUn+7MKR5MB8GA1Ud IwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IBAQAo 873zVMG6tfoPRUZ/kEJcPEIaLmaolALyLEx3sZHe/B8hszuuMecBEfa02HwTSlzq fKrkME95LGE9D+4hxyPEqPTqruESShUvBFQIoTQxePAhhUG9icF4gqUpYvRHXMiR xIyfH3/KojDlBXRfDOaoXEXshiXfcYqbeh2qFdaoN24Vyh6lkNa2K3SUDAtKVFiF jjBtNXuH/IJ3EWbs5AOOy98QtBMlT7kmummJVeaRR4QUfnzFrlj5nMSIopoxDm4N QeoQSC+63fce6ZJLGQqEFQNR6howBcDQ/8fMR/oLsJ1Hr9VshIsu3kGTk4RUqHI9 ipGt1UTvVf/uMUzA3yoC -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/tlsauth/server-key.pem000066400000000000000000000032501477524627100241650ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAgrLiRyYcP54+ AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1Puv8 J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTCk/tV kuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0+pM+ 5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbhC/Gp OX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKOiZC5 xYXBxO4DAgMBAAECggEAA+ILzXOfVneHPHuO4acuRvoIG/yoT1OgHcWoPl16ywBX BBUPYh9Yqm/lEQrDDwRCpuX54HEEVO+WRY/gpEHv/I6z8k3FTgWu66h6qhZGW/eo U02yRzdFBz9AV/xzWpvUEvUq3Uk/GkBvRxKyiN7RPMODpXktXxKTo9Kg/XTYnjIb 2RWzKdoeptX6q/RRdPkBXxYV5XIaZLXPy+VaGgI7QcwApmCcin20c5smtLmGE7TV 4hh6wxPwGEiDkfMQGnqWd+VdzbnZjZF2/4ZVCNAbJJaFffTxnoC0uGjE3ImAztmT TgmFbdfO/5qHTkJmfZaAbmo461k1O2Vd/p6LCN1mhQKBgQDokSJxOMuYurf1STAb om6E5lGXAYR/2TsBxzfWPvUlZDzaSZ7IU+/zpv9GTATc0mvdIpnT2kZvRoz7nbbR LFGjc2d1ITRBuDEX+qDkX1puFyhoYPyaz0AuC3AMqg28e3Z1I8nq3Op1TUKAMr9d ZdOfRUcL65OM+YHPZ197AVwC9wKBgQDT6FgGceqBr3XJDikjU+kJfyrZrcE7sfrE 3RIV6uMj6Myw4WpUpuH7hwqQ0xAir8UKHG6SDScymXtrJCoG1xJmGGDyoLt80VsF fXVhzL8rhIl48TVWptRUiIeZ2dXvianLkLPlLhVUcTThpjjAAbnNYhod1bHygX12 w+Bd8lkeVQKBgBYLS7x3qbS8Xht96HV2HAu02R77IdgMey9b9sr0BMCak7oNKGPM sP3jYmcDZaKYv2iikvolwm9hvJNNC7sf/E0F71SG5TEliGHBe+apsySkRUw/hTIX WvoCU4ifxdWLzlqkHcuJTR/5RshoBwOPV1PNeUKD/eRq8gb6wW4jXtlZAoGBAK2o C3MEqcQrUSA53ZaY7jGdKDWJQgC0oyfvbyHNAuVro0sU/3lt5WWmTg9PGDsExjm6 ARbpdoTt6Ilt8o72c5p9Qf2zoNHyE2CVZruF+egkzi/xo99mCj1YQZ/gN4T80MwE wpf+wvYXa9m7yWf4QhbA3VwzwodUfMf2T4lN0KCdAoGBAM04ecwBXCNIXzTgJDeK zxfI95lphcNywdhUNw6KJPDg4ozPu72XrVToAtl7l6+2RPygNhSj117002AAgz30 l0cfZoogXmIhZt+R39/2e9y31k+GDdEmNtPLsvbLPQ+Ttnab87pUABn4UC1XXGhF Vt/jfLEw8ZDjB+aO4kFyZPIQ -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/tlsauth/server-no-ou-key.pem000066400000000000000000000032501477524627100252200ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP GCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt +6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ XgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ Wxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8 CrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT dKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX dDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed /3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M ah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV Fuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz CVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ u3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z 3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/ VNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti H/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG HQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq zOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c iGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu 0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7 ua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2 JR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn XlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf rBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6 WokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A JegHloScSH3BcemRFfK31+ba -----END PRIVATE KEY----- nats-server-2.10.27/test/configs/certs/tlsauth/server-no-ou.pem000066400000000000000000000022071477524627100244330ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy MTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD ggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8 lhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck aNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels fn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/ 1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C Jc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw KYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud DgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o TUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn n86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN BAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak 3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL LPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt h2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w== -----END CERTIFICATE----- nats-server-2.10.27/test/configs/certs/tlsauth/server.pem000066400000000000000000000022541477524627100234020ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDRjCCAi6gAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipgwDQYJKoZIhvcNAQEL BQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMjA0NTdaFw0zNDAxMzEy MjA0NTdaMDAxGjAYBgNVBAsMEU5BVFMuaW8gT3BlcmF0b3JzMRIwEAYDVQQDDAls b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAgrLiRyYc P54+AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1 Puv8J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTC k/tVkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0 +pM+5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbh C/GpOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKO iZC5xYXBxO4DAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBs ZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQUcpTmScu/KZHhs0KbdxtN GXndXBEwHwYDVR0jBBgwFoAUJLzGRsjX1R1N6E1AEkn2Ik+GqzUwDQYJKoZIhvcN AQELBQADggEBAKdqG82z3JBim/hiGf4LZT90ZHhw7ngPT9HUV4jYRIk0ngJ37ogK KCYW0UBCkugdf0elxcggjAsJZGlz+hW2j8MynEqJ9UU7jPPp4AKJqZHy5x49Y1iL kFlJE5a3LFJUaVG4JeYMqTL2zDtoj+hk7QPPoz88moDUbOHg3HccObHlISelVPON K/kvnJ2NfXImYkh7MusRxVuB4LcRRi5rwT0pOdtSPBCeSH96BOeCHTriPHGecgc4 71tgSaELXPM1YnaM2WmXoGU1MZ7Dx6c2q97FI+SWgKfm7B1GQGyAghgKxlRyhfNj UvCrbaZDInrMWpMo3+upIBWpHzfmJVvUcYI= -----END CERTIFICATE----- nats-server-2.10.27/test/configs/cluster.conf000066400000000000000000000007471477524627100211220ustar00rootroot00000000000000# Cluster config file listen: 127.0.0.1:5242 cluster { listen: 127.0.0.1:5244 name: xyz authorization { user: route_user password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://foo:bar@127.0.0.1:5245 nats-route://foo:bar@127.0.0.1:5246 ] } no_sys_acc: true nats-server-2.10.27/test/configs/jetstream/000077500000000000000000000000001477524627100205605ustar00rootroot00000000000000nats-server-2.10.27/test/configs/jetstream/restore_bad_stream/000077500000000000000000000000001477524627100244245ustar00rootroot00000000000000nats-server-2.10.27/test/configs/jetstream/restore_bad_stream/backup.json000066400000000000000000000012621477524627100265650ustar00rootroot00000000000000{ "config": { "name": "TEST", "subjects": [ "foo" ], "retention": "limits", "max_consumers": -1, "max_msgs_per_subject": -1, "max_msgs": -1, "max_bytes": -1, "max_age": 0, "max_msg_size": -1, "storage": "file", "discard": "old", "num_replicas": 1, "duplicate_window": 120000000000, "sealed": false, "deny_delete": false, "deny_purge": false, "allow_rollup_hdrs": false }, "state": { "messages": 10, "bytes": 381, "first_seq": 1, "first_ts": "2022-03-07T23:59:01.710801Z", "last_seq": 10, "last_ts": "2022-03-07T23:59:01.712378Z", "num_subjects": 1, "consumer_count": 1 } }nats-server-2.10.27/test/configs/jetstream/restore_bad_stream/stream.tar.s2000077500000000000000000000030661477524627100267620ustar00rootroot00000000000000ÿS2sTwO( kV€˜TPaxHeaders.0/meta.infî 0 0 0033 $011641 xî‡ VMustar00=îCnh27 mtime=1646697580.762314 îÌmî±ðîó2‹00006yð>`055114211516154012435 î”!‘ nats~ ‰¼åfJîIðy{"Created":"2022-03-07T23:58:59.492218Z","name":"TEST","subjects":["foo"],"retention":"limits","max_consumers":-1,"max_msg byteage":0)_per_ q&ðTmsg_size":-1,"discard":"old","storage":"file","num_replicas":1,"duplicate_window":120-Æt00,"sealed":false,"deny_delete6 purg@allow_rollup_hdrs }îöIŒFsumî  -T>øR7hîó28~020 54î&Úfoomsg#1—éqßSnR& % hJ36&42µ­=Æ–(d& Ðý5 3­ îÜú 0Y9 4ÙwºìvTk È< 5ÞÓ쎊(q= ÐJ> 6œÒM&`FHµ ^@ 7Ã@„k8½ŽÕA6 48·‘fxÒ€P& (¿D09ÞwV%’ûƒ'@%xF6L10îc Gobs/test6î ( Æ340î'•îó ü ~267:420î'j(<9:11.834611Z","N (","durable_(€test","deliver_policy":"all","acknone"Ù' /Ð'replay(instan.V"ÿ'"bar"}IYîÿ•Fî$Î3qîó2š~F( 4216b<5c1f135f3b268d8bìVo.datî Æ 2707kîó 2;‚1 3471b×î·8../../fail1.txtî0000644ð7>0030440716010120d îùîU_XThis should not be here½îÂ_î£ú" ‰ fail2î² 31012306bî>úî=‚nats-server-2.10.27/test/configs/multi_accounts.conf000066400000000000000000000007571477524627100224730ustar00rootroot00000000000000listen: 127.0.0.1:4033 http: 127.0.0.1:8033 password = "s3cr3t!" accounts: { engineering: { users = [ {user: alice, password: $password} {user: bob, password: $password} ] } legal: { users = [ {user: john, password: $password} {user: mary, password: $password} ] } finance: { users = [ {user: peter, password: $password} {user: paul, password: $password} ] } } no_sys_acc: true nats-server-2.10.27/test/configs/multi_user.conf000066400000000000000000000002311477524627100216150ustar00rootroot00000000000000listen: 127.0.0.1:4233 http: 127.0.0.1:8233 authorization { users = [ {user: alice, password: foo} {user: bob, password: bar} ] } nats-server-2.10.27/test/configs/new_cluster.conf000066400000000000000000000006051477524627100217640ustar00rootroot00000000000000# New Cluster config file listen: 127.0.0.1:5343 cluster { listen: 127.0.0.1:5344 name: xyz # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://127.0.0.1:5345 nats-route://127.0.0.1:5346 ] } no_sys_acc: true nats-server-2.10.27/test/configs/nkeys/000077500000000000000000000000001477524627100177135ustar00rootroot00000000000000nats-server-2.10.27/test/configs/nkeys/op.jwt000066400000000000000000000015621477524627100210630ustar00rootroot00000000000000-----BEGIN TEST OPERATOR JWT----- eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw ------END TEST OPERATOR JWT------nats-server-2.10.27/test/configs/nkeys/sigkeys.txt000066400000000000000000000012731477524627100221350ustar00rootroot00000000000000 ######################################################## # TESTS ONLY # ######################################################## # These are the public signing keys -----BEGIN SIGNING KEYS----- ODSKR7MYFQZ5MMAJ6FPMEETCTE3RIHOFLTYPJRMAVVN4OLV2YYAMHCAC ODSKACSRBWP537DZDRVJ657JOIGOPOQ6KG7T4HN6OK4F6IECGXDAHNP2 ODSKI36LZB44OY5IVCR6P52FZJZYMYWZVWNUDTLEZ5TK2PN3OEMRTABR ------END SIGNING KEYS------ # These are the seeds. ----BEGIN SIGNING SEEDS----- SOAO7RDW6CLJORHHBS4DPYYIIIAASEIUJ5WWS5FMWLNTFHUCKQ5CAC45AA SOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ SOACSMP662P2BZDKVF6WCB6FIQYORADDWWWEAI55QY24CQRTY4METUING4 ------END SIGING SEEDS------ nats-server-2.10.27/test/configs/nkeys/test.seed000066400000000000000000000004551477524627100215400ustar00rootroot00000000000000######################################################## # TESTS ONLY # ######################################################## -----BEGIN TEST OPERATOR SEED----- SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU ------END TEST OPERATOR SEED------ nats-server-2.10.27/test/configs/operator.conf000066400000000000000000000007061477524627100212670ustar00rootroot00000000000000# Server that loads an operator JWT listen: 127.0.0.1:22222 # Can be an array of filenames as well. # Key can be operator, operators, roots, root, root_operators, root_operator operator = "./configs/nkeys/op.jwt" # This is for account resolution. # Can be MEMORY (Testing) or can be URL(url). # The resolver will append the account name to url for retrieval. # E.g. # resolver = URL("https://api.synadia.com/ngs/v1/accounts/jwt") # resolver = MEMORY nats-server-2.10.27/test/configs/operator_inline.conf000066400000000000000000000022171477524627100226240ustar00rootroot00000000000000# Server that loads an operator JWT listen: 127.0.0.1:22222 # This example is a single inline JWT. operator = "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw" # This is for account resolution. # Can be MEMORY (Testing) or can be URL(url). # The resolver will append the account name to url for retrieval. # E.g. # resolver = URL("https://api.synadia.com/ngs/v1/accounts/jwt") # resolver = MEMORY nats-server-2.10.27/test/configs/override.conf000066400000000000000000000001471477524627100212520ustar00rootroot00000000000000# Config file to test overrides to client listen: 127.0.0.1:5224 # maximum payload max_payload: 2222 nats-server-2.10.27/test/configs/resolver_preload.conf000066400000000000000000000032051477524627100230000ustar00rootroot00000000000000# Server that loads an operator JWT listen: 127.0.0.1:22222 # Can be an array of filenames as well. # Key can be operator, operators, roots, root, root_operators, root_operator operator = "./configs/nkeys/op.jwt" system_account = "AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG" # This is for account resolution. # Can be MEMORY (Testing) or can be URL(url). # The resolver will append the account name to url for retrieval. # E.g. # resolver = URL("https://api.synadia.com/ngs/v1/accounts/jwt") # resolver = MEMORY # This is a map that can preload keys:jwts into a memory resolver. resolver_preload = { AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ" ADM2CIIL3RWXBA6T2HW3FODNCQQOUJEHHQD6FKCPVAMHDNTTSMO73ROX: "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJCMk0zTFRMT1ZNRk03REY3U0M3SE9RTzNXUzI2RFhMTURINk0zRzY3RzRXRFdTWExPNlVBIiwiaWF0IjoxNTQzOTU4NzI0LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFETTJDSUlMM1JXWEJBNlQySFczRk9ETkNRUU9VSkVISFFENkZLQ1BWQU1IRE5UVFNNTzczUk9YIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.pvvPmBei_IFEbspHGN5FkWJoSfHk8BVeJCCVODTgul8-xUU8p1fidvsg3sgMvrXqXtmL8SFc68jGQd0nGtk5Dw" } nats-server-2.10.27/test/configs/seed.conf000066400000000000000000000001741477524627100203530ustar00rootroot00000000000000# Cluster Seed Node listen: 127.0.0.1:5222 http: 8222 cluster { listen: 127.0.0.1:4248 name: xyz } no_sys_acc: true nats-server-2.10.27/test/configs/srv_a.conf000066400000000000000000000007001477524627100205400ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:5222 cluster { listen: 127.0.0.1:5244 name: xyz authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:5246 ] } no_sys_acc: true nats-server-2.10.27/test/configs/srv_a_leaf.conf000066400000000000000000000007261477524627100215370ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:5222 leafnodes { listen: 127.0.0.1:5223 } cluster { listen: 127.0.0.1:5244 name: xyz authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:5246 ] } nats-server-2.10.27/test/configs/srv_a_perms.conf000066400000000000000000000011161477524627100217500ustar00rootroot00000000000000# Cluster Server A with Permissions listen: 127.0.0.1:5222 http: 127.0.0.1:5223 cluster { listen: 127.0.0.1:5244 name: xyz authorization { user: ruser password: top_secret timeout: 0.5 } permissions { import: "foo" export: { allow: "*" deny: ["foo", "nats"] } } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:5246 ] } no_sys_acc: true nats-server-2.10.27/test/configs/srv_a_tls.conf000066400000000000000000000013031477524627100214220ustar00rootroot00000000000000# Cluster Server A listen: 127.0.0.1:5222 cluster { listen: 127.0.0.1:5244 name: xyz tls { # Route cert cert_file: "./configs/certs/srva-cert.pem" # Private key key_file: "./configs/certs/srva-key.pem" # Specified time for handshake to complete timeout: 2 # Optional certificate authority verifying connected routes # Required when we have self-signed CA, etc. ca_file: "./configs/certs/ca.pem" } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://127.0.0.1:5246 ] } no_sys_acc: true nats-server-2.10.27/test/configs/srv_b.conf000066400000000000000000000007251477524627100205500ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:5224 http: 127.0.0.1:5225 cluster { listen: 127.0.0.1:5246 name: xyz authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:5244 ] } no_sys_acc: true nats-server-2.10.27/test/configs/srv_b_tls.conf000066400000000000000000000013031477524627100214230ustar00rootroot00000000000000# Cluster Server B listen: 127.0.0.1:5224 cluster { listen: 127.0.0.1:5246 name: xyz tls { # Route cert cert_file: "./configs/certs/srvb-cert.pem" # Private key key_file: "./configs/certs/srvb-key.pem" # Specified time for handshake to complete timeout: 2 # Optional certificate authority verifying connected routes # Required when we have self-signed CA, etc. ca_file: "./configs/certs/ca.pem" } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://127.0.0.1:5244 ] } no_sys_acc: true nats-server-2.10.27/test/configs/srv_c.conf000066400000000000000000000007001477524627100205420ustar00rootroot00000000000000# Cluster Server C listen: 127.0.0.1:5226 cluster { listen: 127.0.0.1:5248 name: xyz authorization { user: ruser password: top_secret timeout: 0.5 } # Routes are actively solicited and connected to from this server. # Other servers can connect to us if they supply the correct credentials # in their routes definitions from above. routes = [ nats-route://ruser:top_secret@127.0.0.1:5244 ] } no_sys_acc: true nats-server-2.10.27/test/configs/tls.conf000066400000000000000000000005141477524627100202330ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:5443 https: 11522 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 } authorization { user: derek password: monkey timeout: 1 } nats-server-2.10.27/test/configs/tls_cert_cn.conf000066400000000000000000000010751477524627100217330ustar00rootroot00000000000000 listen: localhost:9334 tls { cert_file = "./configs/certs/tlsauth/server.pem" key_file = "./configs/certs/tlsauth/server-key.pem" ca_file = "./configs/certs/tlsauth/ca.pem" verify = true verify_and_map = true } authorization { # Default permissions permissions { publish { allow = ["public.>"] } subscribe { allow = ["public.>"] } } users [ { user = "CN=example.com,OU=NATS.io" } { user = "CN=example.com,OU=CNCF", permissions = { publish { allow = [">"] } subscribe { allow = [">"] } } } ] } nats-server-2.10.27/test/configs/tls_cert_cn_gateways.conf000066400000000000000000000004451477524627100236370ustar00rootroot00000000000000 gateway { tls { cert_file = "./configs/certs/tlsauth/server.pem" key_file = "./configs/certs/tlsauth/server-key.pem" ca_file = "./configs/certs/tlsauth/ca.pem" verify_and_map = true timeout = 2 } authorization { user = "CN=localhost,OU=NATS.io Operators" } } nats-server-2.10.27/test/configs/tls_cert_cn_gateways_invalid_auth.conf000066400000000000000000000004341477524627100263640ustar00rootroot00000000000000 gateway { tls { cert_file = "./configs/certs/tlsauth/server-no-ou.pem" key_file = "./configs/certs/tlsauth/server-no-ou-key.pem" ca_file = "./configs/certs/tlsauth/ca.pem" verify_and_map = true timeout = 2 } authorization { user = "CN=localhost" } } nats-server-2.10.27/test/configs/tls_cert_cn_routes.conf000066400000000000000000000006241477524627100233330ustar00rootroot00000000000000 cluster { tls { cert_file = "./configs/certs/tlsauth/server.pem" key_file = "./configs/certs/tlsauth/server-key.pem" ca_file = "./configs/certs/tlsauth/ca.pem" verify_and_map = true timeout = 2 } permissions { publish { allow = ["public.>"] } subscribe { allow = ["public.>"] } } authorization { user = "CN=localhost,OU=NATS.io Operators" } } nats-server-2.10.27/test/configs/tls_cert_cn_routes_invalid_auth.conf000066400000000000000000000006421477524627100260620ustar00rootroot00000000000000 cluster { tls { cert_file = "./configs/certs/tlsauth/server-no-ou.pem" key_file = "./configs/certs/tlsauth/server-no-ou-key.pem" ca_file = "./configs/certs/tlsauth/ca.pem" verify_and_map = true timeout = 2 } no_advertise = true permissions { publish { allow = ["public.>"] } subscribe { allow = ["public.>"] } } authorization { user = "CN=localhost" } } nats-server-2.10.27/test/configs/tls_cert_id.conf000066400000000000000000000011561477524627100217270ustar00rootroot00000000000000# TLS config file # We require client certs and pull the user from the cert itself. listen: 127.0.0.1:9333 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 # Optional certificate authority for clients ca_file: "./configs/certs/ca.pem" # Require a client certificate and map user id from certificate verify_and_map: true } # User authenticated from above in certificate. authorization { users = [ {user: derek@nats.io, permissions: { publish:"foo" }} ] } nats-server-2.10.27/test/configs/tls_cert_san_auth.conf000066400000000000000000000015331477524627100231340ustar00rootroot00000000000000 listen: localhost:9335 tls { cert_file = "./configs/certs/sans/server.pem" key_file = "./configs/certs/sans/server-key.pem" ca_file = "./configs/certs/sans/ca.pem" verify = true verify_and_map = true } authorization { # Default permissions permissions { publish { allow = ["public.>"] } subscribe { allow = ["public.>"] } } users [ # CN used by default if there are no SANs { user = "CN=www.nats.io" } # All permissions { user = "app.nats.prod", permissions = { publish { allow = [">"] } subscribe { allow = [">"] } } } # Dev certs are isolated to own sandbox but can # also publish to public. { user = "app.nats.dev", permissions = { publish { allow = ["public.>", "sandbox.>"] } subscribe { allow = ["public.>", "sandbox.>"] } } } ] } nats-server-2.10.27/test/configs/tls_cert_san_emails.conf000066400000000000000000000015421477524627100234450ustar00rootroot00000000000000 listen: localhost:9336 tls { cert_file = "./configs/certs/sans/server.pem" key_file = "./configs/certs/sans/server-key.pem" ca_file = "./configs/certs/sans/ca.pem" verify = true verify_and_map = true } authorization { # Default permissions permissions { publish { allow = ["public.>"] } subscribe { allow = ["public.>"] } } users [ # CN used by default if there are no SANs { user = "CN=www.nats.io" } # All permissions { user = "*.app.nats.prod", permissions = { publish { allow = [">"] } subscribe { allow = [">"] } } } # Dev certs are isolated to own sandbox but can # also publish to public. { user = "root@app.nats.dev", permissions = { publish { allow = ["public.>", "sandbox.>"] } subscribe { allow = ["public.>", "sandbox.>"] } } } ] } nats-server-2.10.27/test/configs/tls_curve_pref.conf000066400000000000000000000005621477524627100224560ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:5443 https: 11522 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 curve_preferences: [ "CurveP256" ] } authorization { user: derek password: boo timeout: 1 } nats-server-2.10.27/test/configs/tls_mixed.conf000066400000000000000000000005041477524627100214200ustar00rootroot00000000000000# Allow TLS and non TLS on same port. listen: 127.0.0.1:-1 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 } # This allows non tls traffic on same port. allow_non_tls: true nats-server-2.10.27/test/configs/tlsverify.conf000066400000000000000000000006001477524627100214540ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:5443 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 # Optional certificate authority for clients ca_file: "./configs/certs/ca.pem" # Require a client certificate verify: true } nats-server-2.10.27/test/configs/tlsverify_noca.conf000066400000000000000000000006401477524627100224600ustar00rootroot00000000000000# Simple TLS config file listen: 127.0.0.1:5443 tls { # Server cert cert_file: "./configs/certs/server-cert.pem" # Server private key key_file: "./configs/certs/server-key.pem" # Specified time for handshake to complete timeout: 2 # Require a client certificate verify: true # Omit the client CA, this is to verify that # the server is really trying to verify the # client certificate. } nats-server-2.10.27/test/fanout_test.go000066400000000000000000000066121477524627100200210ustar00rootroot00000000000000// Copyright 2018-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !race && !skipnoracetests // +build !race,!skipnoracetests package test import ( "fmt" "strconv" "sync" "testing" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) // IMPORTANT: Tests in this file are not executed when running with the -race flag. // The test name should be prefixed with TestNoRace so we can run only // those tests: go test -run=TestNoRace ... // As we look to improve high fanout situations make sure we // have a test that checks ordering for all subscriptions from a single subscriber. func TestNoRaceHighFanoutOrdering(t *testing.T) { opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT} s := RunServer(opts) defer s.Shutdown() url := fmt.Sprintf("nats://%s", s.Addr()) const ( nconns = 100 nsubs = 100 npubs = 500 ) // make unique subj := nats.NewInbox() var wg sync.WaitGroup wg.Add(nconns * nsubs) for i := 0; i < nconns; i++ { nc, err := nats.Connect(url) if err != nil { t.Fatalf("Expected a successful connect on %d, got %v\n", i, err) } nc.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, e error) { t.Fatalf("Got an error %v for %+v\n", s, err) }) for y := 0; y < nsubs; y++ { expected := 0 nc.Subscribe(subj, func(msg *nats.Msg) { n, _ := strconv.Atoi(string(msg.Data)) if n != expected { t.Fatalf("Expected %d but received %d\n", expected, n) } expected++ if expected >= npubs { wg.Done() } }) } nc.Flush() defer nc.Close() } nc, _ := nats.Connect(url) for i := 0; i < npubs; i++ { nc.Publish(subj, []byte(strconv.Itoa(i))) } defer nc.Close() wg.Wait() } func TestNoRaceRouteFormTimeWithHighSubscriptions(t *testing.T) { srvA, optsA := RunServerWithConfig("./configs/srv_a.conf") defer srvA.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConn(t, clientA) // Now add lots of subscriptions. These will need to be forwarded // to new routes when they are added. subsTotal := 100000 for i := 0; i < subsTotal; i++ { subject := fmt.Sprintf("FOO.BAR.BAZ.%d", i) sendA(fmt.Sprintf("SUB %s %d\r\n", subject, i)) } sendA("PING\r\n") expectA(pongRe) srvB, _ := RunServerWithConfig("./configs/srv_b.conf") defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) // Now wait for all subscriptions to be processed. if err := checkExpectedSubs(subsTotal, srvB); err != nil { // Make sure we are not a slow consumer // Check for slow consumer status if srvA.NumSlowConsumers() > 0 { t.Fatal("Did not receive all subscriptions due to slow consumer") } else { t.Fatalf("%v", err) } } // Just double check the slow consumer status. if srvA.NumSlowConsumers() > 0 { t.Fatalf("Received a slow consumer notification: %d", srvA.NumSlowConsumers()) } } nats-server-2.10.27/test/gateway_test.go000066400000000000000000000566351477524627100202000ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bufio" "bytes" "crypto/tls" "encoding/json" "fmt" "net" "net/url" "regexp" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) func testDefaultOptionsForGateway(name string) *server.Options { o := DefaultTestOptions o.Port = -1 o.Cluster.Name = name o.Gateway.Name = name o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 return &o } func runGatewayServer(o *server.Options) *server.Server { s := RunServer(o) return s } func createGatewayConn(t testing.TB, host string, port int) net.Conn { t.Helper() return createClientConn(t, host, port) } func setupGatewayConn(t testing.TB, c net.Conn, org, dst string) (sendFun, expectFun) { t.Helper() dstInfo := checkInfoMsg(t, c) if dstInfo.Gateway != dst { t.Fatalf("Expected to connect to %q, got %q", dst, dstInfo.Gateway) } cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"gateway\":%q}\r\n", false, false, false, org) sendProto(t, c, cs) sendProto(t, c, fmt.Sprintf("INFO {\"gateway\":%q}\r\n", org)) return sendCommand(t, c), expectCommand(t, c) } func expectNumberOfProtos(t *testing.T, expFn expectFun, proto *regexp.Regexp, expected int, ignore ...*regexp.Regexp) { t.Helper() buf := []byte(nil) for count := 0; count != expected; { buf = append(buf, expFn(anyRe)...) for _, skip := range ignore { buf = skip.ReplaceAll(buf, []byte(``)) } count += len(proto.FindAllSubmatch(buf, -1)) if count > expected { t.Fatalf("Expected %v matches, got %v", expected, count) } buf = proto.ReplaceAll(buf, []byte(``)) } if len(buf) != 0 { t.Fatalf("did not consume everything, left with: %q", buf) } } func TestGatewayAccountInterest(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) // Sending a bunch of messages. On the first, "B" will send an A- // protocol. for i := 0; i < 100; i++ { gASend("RMSG $foo foo 2\r\nok\r\n") } // We expect single A- followed by PONG. If "B" was sending more // this expect call would fail. gAExpect(aunsubRe) gASend("PING\r\n") gAExpect(pongRe) // Start gateway C that connects to B gC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect := setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) // Send more messages, C should get A-, but A should not (already // got it). for i := 0; i < 100; i++ { gCSend("RMSG $foo foo 2\r\nok\r\n") } gCExpect(aunsubRe) gCSend("PING\r\n") gCExpect(pongRe) expectNothing(t, gA) // Restart one of the gateway, and resend a message, verify // that it receives A- (things get cleared on reconnect) gC.Close() gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) gCSend("RMSG $foo foo 2\r\nok\r\n") gCExpect(aunsubRe) gCSend("PING\r\n") gCExpect(pongRe) expectNothing(t, gA) // Close again and re-create, but this time don't send anything. gC.Close() gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) // Now register the $foo account on B and create a subscription, // A should receive an A+ because B knows that it previously sent // an A-, but since it did not send one to C, C should not receive // the A+. client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConnWithAccount(t, sb, client, "$foo") clientSend("SUB not.used 1234567\r\nPING\r\n") clientExpect(pongRe) gAExpect(asubRe) expectNothing(t, gC) } func TestGatewaySubjectInterest(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") fooAcc := server.NewAccount("$foo") ob.Accounts = []*server.Account{fooAcc} ob.Users = []*server.User{{Username: "ivan", Password: "password", Account: fooAcc}} sb := runGatewayServer(ob) defer sb.Shutdown() // Create a client on B client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConnWithUserPass(t, client, "ivan", "password") // Since we want to test RS+/-, we need to have at // least a subscription on B so that sending from A does // not result in A- clientSend("SUB not.used 1234567\r\nPING\r\n") clientExpect(pongRe) gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) for i := 0; i < 100; i++ { gASend("RMSG $foo foo 2\r\nok\r\n") } // We expect single RS- followed by PONG. If "B" was sending more // this expect call would fail. gAExpect(runsubRe) gASend("PING\r\n") gAExpect(pongRe) // Start gateway C that connects to B gC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect := setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) // Send more messages, C should get RS-, but A should not (already // got it). for i := 0; i < 100; i++ { gCSend("RMSG $foo foo 2\r\nok\r\n") } gCExpect(runsubRe) gCSend("PING\r\n") gCExpect(pongRe) expectNothing(t, gA) // Restart one of the gateway, and resend a message, verify // that it receives RS- (things get cleared on reconnect) gC.Close() gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) gCSend("RMSG $foo foo 2\r\nok\r\n") gCExpect(runsubRe) gCSend("PING\r\n") gCExpect(pongRe) expectNothing(t, gA) // Close again and re-create, but this time don't send anything. gC.Close() gC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gC.Close() gCSend, gCExpect = setupGatewayConn(t, gC, "C", "B") gCSend("PING\r\n") gCExpect(pongRe) // Now register a subscription on foo for account $foo on B. // A should receive a RS+ because B knows that it previously // sent a RS-, but since it did not send one to C, C should // not receive the RS+. clientSend("SUB foo 1\r\nSUB foo 2\r\n") // Also subscribe to subject that was not used before, // so there should be no RS+ for this one. clientSend("SUB bar 3\r\nPING\r\n") clientExpect(pongRe) gAExpect(rsubRe) expectNothing(t, gC) // Check that we get only one protocol expectNothing(t, gA) // Unsubscribe the 2 subs on foo, expect to receive nothing. clientSend("UNSUB 1\r\nUNSUB 2\r\nPING\r\n") clientExpect(pongRe) expectNothing(t, gC) expectNothing(t, gA) gC.Close() // Send on foo, should get an RS- gASend("RMSG $foo foo 2\r\nok\r\n") gAExpect(runsubRe) // Subscribe on foo, should get an RS+ that removes the no-interest clientSend("SUB foo 4\r\nPING\r\n") clientExpect(pongRe) gAExpect(rsubRe) // Send on bar, message should be received. gASend("RMSG $foo bar 2\r\nok\r\n") clientExpect(msgRe) // Unsub foo and bar clientSend("UNSUB 3\r\nUNSUB 4\r\nPING\r\n") clientExpect(pongRe) expectNothing(t, gA) // Send on both foo and bar expect RS- gASend("RMSG $foo foo 2\r\nok\r\n") gAExpect(runsubRe) gASend("RMSG $foo bar 2\r\nok\r\n") gAExpect(runsubRe) // Now have client create sub on "*", this should cause RS+ on * // The remote will have cleared its no-interest on foo and bar // and this receiving side is supposed to be doing the same. clientSend("SUB * 5\r\nPING\r\n") clientExpect(pongRe) buf := gAExpect(rsubRe) if !bytes.Contains(buf, []byte("$foo *")) { t.Fatalf("Expected RS+ on %q, got %q", "*", buf) } // Check that the remote has cleared by sending from the client // on foo and bar clientSend("PUB foo 2\r\nok\r\n") clientExpect(msgRe) clientSend("PUB bar 2\r\nok\r\n") clientExpect(msgRe) // Check that A can send too and does not receive an RS- gASend("RMSG $foo foo 2\r\nok\r\n") expectNothing(t, gA) clientExpect(msgRe) gASend("RMSG $foo bar 2\r\nok\r\n") expectNothing(t, gA) clientExpect(msgRe) } func TestGatewayQueue(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") fooAcc := server.NewAccount("$foo") ob.Accounts = []*server.Account{fooAcc} ob.Users = []*server.User{{Username: "ivan", Password: "password", Account: fooAcc}} sb := runGatewayServer(ob) defer sb.Shutdown() gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConnWithUserPass(t, client, "ivan", "password") // Create one queue sub on foo.* for group bar. clientSend("SUB foo.* bar 1\r\nPING\r\n") clientExpect(pongRe) // Expect RS+ gAExpect(rsubRe) // Add another queue sub on same group clientSend("SUB foo.* bar 2\r\nPING\r\n") clientExpect(pongRe) // Should not receive another RS+ for that one expectNothing(t, gA) // However, if subject is different, we can expect to receive another RS+ clientSend("SUB foo.> bar 3\r\nPING\r\n") clientExpect(pongRe) gAExpect(rsubRe) // Unsub one of the foo.* qsub, no RS- should be received clientSend("UNSUB 1\r\nPING\r\n") clientExpect(pongRe) expectNothing(t, gA) // Remove the other one, now we should get the RS- clientSend("UNSUB 2\r\nPING\r\n") clientExpect(pongRe) gAExpect(runsubRe) // Remove last one clientSend("UNSUB 3\r\nPING\r\n") clientExpect(pongRe) gAExpect(runsubRe) // Create some queues and check that interest is sent // when GW reconnects. clientSend("SUB foo bar 4\r\n") gAExpect(rsubRe) clientSend("SUB foo baz 5\r\n") gAExpect(rsubRe) clientSend("SUB foo bat 6\r\n") gAExpect(rsubRe) // There is already one on foo/bar, so nothing sent clientSend("SUB foo bar 7\r\n") expectNothing(t, gA) // Add regular sub that should not cause RS+ clientSend("SUB foo 8\r\n") expectNothing(t, gA) // Recreate gA gA.Close() gA = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect = setupGatewayConn(t, gA, "A", "B") // A should receive 3 RS+ expectNumberOfProtos(t, gAExpect, rsubRe, 3) // Nothing more expectNothing(t, gA) gASend("PING\r\n") gAExpect(pongRe) // Have A send a message on subject that has no sub gASend("RMSG $foo new.subject 2\r\nok\r\n") gAExpect(runsubRe) // Now create a queue sub and check that we do not receive // an RS+ without the queue name. clientSend("SUB new.* queue 9\r\nPING\r\n") clientExpect(pongRe) buf := gAExpect(rsubRe) if !bytes.Contains(buf, []byte("new.* queue")) { t.Fatalf("Should have receives RS+ for new.* for queue, did not: %v", buf) } // Check for no other RS+. A should still keep an RS- for plain // sub on new.subject expectNothing(t, gA) // Send message, expected to be received by client gASend("RMSG $foo new.subject | queue 2\r\nok\r\n") clientExpect(msgRe) // Unsubscribe the queue sub clientSend("UNSUB 9\r\nPING\r\n") clientExpect(pongRe) // A should receive RS- for this queue sub buf = gAExpect(runsubRe) if !bytes.Contains(buf, []byte("new.* queue")) { t.Fatalf("Should have receives RS- for new.* for queue, did not: %v", buf) } expectNothing(t, gA) } func TestGatewaySendAllSubs(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() // Create a client on B client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) // Since we want to test RS+/-, we need to have at // least a subscription on B so that sending from A does // not result in A- clientSend("SUB not.used 1234567\r\nPING\r\n") clientExpect(pongRe) gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) // Bombard B with messages on different subjects. // TODO(ik): Adapt if/when we change the conditions for the // switch. for i := 0; i < 1010; i++ { gASend(fmt.Sprintf("RMSG $G foo.%d 2\r\nok\r\n", i)) if i < 1000 { gAExpect(runsubRe) } } // Expect an INFO + RS+ $G not.used + INFO buf := bufio.NewReader(gA) for i := 0; i < 3; i++ { line, _, err := buf.ReadLine() if err != nil { t.Fatalf("Error reading: %v", err) } switch i { case 0: case 2: if !bytes.HasPrefix(line, []byte("INFO {")) { t.Fatalf("Expected INFO, got: %s", line) } case 1: if !bytes.HasPrefix(line, []byte("RS+ ")) { t.Fatalf("Expected RS+, got: %s", line) } } } // After this point, any new sub or unsub on B should be // sent to A. clientSend("SUB foo 1\r\n") gAExpect(rsubRe) clientSend("UNSUB 1\r\n") gAExpect(runsubRe) } func TestGatewayNoPanicOnBadProtocol(t *testing.T) { ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() for _, test := range []struct { name string proto string }{ {"sub", "SUB > 1\r\n"}, {"unsub", "UNSUB 1\r\n"}, {"rsub", "RS+ $foo foo 2\r\n"}, {"runsub", "RS- $foo foo 2\r\n"}, {"pub", "PUB foo 2\r\nok\r\n"}, {"msg", "MSG foo 2\r\nok\r\n"}, {"rmsg", "RMSG $foo foo 2\r\nok\r\n"}, {"anything", "xxxx\r\n"}, } { t.Run(test.name, func(t *testing.T) { // Create raw tcp connection to gateway port client := createClientConn(t, ob.Gateway.Host, ob.Gateway.Port) defer client.Close() clientSend := sendCommand(t, client) clientSend(test.proto) }) } // Server should not have crashed. client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) clientSend("PING\r\n") clientExpect(pongRe) } func TestGatewayNoAccUnsubAfterQSub(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) // Simulate a client connecting to A and publishing a message // so we get an A- from B since there is no interest. gASend("RMSG $G foo 2\r\nok\r\n") gAExpect(runsubRe) // Now create client on B and create queue sub. client := createClientConn(t, ob.Host, ob.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) clientSend("SUB bar queue 1\r\nPING\r\n") clientExpect(pongRe) // A should receive an RS+ for this queue sub. gAExpect(rsubRe) // On B, create a plain sub now. We should get nothing. clientSend("SUB baz 2\r\nPING\r\n") clientExpect(pongRe) expectNothing(t, gA) } func TestGatewayErrorOnRSentFromOutbound(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") sb := runGatewayServer(ob) defer sb.Shutdown() for _, test := range []struct { name string proto string }{ {"RS+", "RS+"}, {"RS-", "RS-"}, } { t.Run(test.name, func(t *testing.T) { gA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) gASend(fmt.Sprintf("%s foo bar\r\n", test.proto)) expectDisconnect(t, gA) }) } } func TestGatewaySystemConnectionAllowedToPublishOnGWPrefix(t *testing.T) { sc := createSuperCluster(t, 2, 2) defer sc.shutdown() o := sc.clusters[1].opts[1] url := fmt.Sprintf("nats://sys:pass@%s:%d", o.Host, o.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() reply := nats.NewInbox() sub, err := nc.SubscribeSync(reply) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := nc.PublishRequest("$SYS.REQ.SERVER.PING", reply, nil); err != nil { t.Fatalf("Failed to send request: %v", err) } for i := 0; i < 4; i++ { if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Expected to get a response, got %v", err) } } } func TestGatewayTLSMixedIPAndDNS(t *testing.T) { server.SetGatewaysSolicitDelay(5 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() // Run this test extra times to make sure not flaky since it // on solicit time. for i := 0; i < 10; i++ { t.Run("", func(t *testing.T) { confA1 := createConfFile(t, []byte(` listen: 127.0.0.1:-1 server_name: A1 gateway { name: "A" listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-iponly.pem" key_file: "./configs/certs/server-key-iponly.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 } } cluster { listen: "127.0.0.1:-1" }`)) srvA1, optsA1 := RunServerWithConfig(confA1) defer srvA1.Shutdown() confA2Template := ` listen: 127.0.0.1:-1 server_name: A2 gateway { name: "A" listen: "localhost:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 } } cluster { listen: "127.0.0.1:-1" routes [ "nats://%s:%d" ] }` confA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template, optsA1.Cluster.Host, optsA1.Cluster.Port))) srvA2, optsA2 := RunServerWithConfig(confA2) defer srvA2.Shutdown() checkClusterFormed(t, srvA1, srvA2) // Create a GW connection to cluster "A". Don't use the helper since we need verification etc. o := DefaultTestOptions o.Port = -1 o.ServerName = "B1" o.Gateway.Name = "B" o.Gateway.Host = "127.0.0.1" o.Gateway.Port = -1 tc := &server.TLSConfigOpts{} tc.CertFile = "./configs/certs/server-cert.pem" tc.KeyFile = "./configs/certs/server-key.pem" tc.CaFile = "./configs/certs/ca.pem" tc.Timeout = 2.0 tlsConfig, err := server.GenTLSConfig(tc) if err != nil { t.Fatalf("Error generating TLS config: %v", err) } tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert tlsConfig.RootCAs = tlsConfig.ClientCAs o.Gateway.TLSConfig = tlsConfig.Clone() rurl, _ := url.Parse(fmt.Sprintf("nats://%s:%d", optsA2.Gateway.Host, optsA2.Gateway.Port)) remote := &server.RemoteGatewayOpts{Name: "A", URLs: []*url.URL{rurl}} remote.TLSConfig = tlsConfig.Clone() o.Gateway.Gateways = []*server.RemoteGatewayOpts{remote} srvB := RunServer(&o) defer srvB.Shutdown() waitForOutboundGateways(t, srvB, 1, 10*time.Second) waitForOutboundGateways(t, srvA1, 1, 10*time.Second) waitForOutboundGateways(t, srvA2, 1, 10*time.Second) // Now kill off srvA2 and force serverB to connect to srvA1. srvA2.Shutdown() // Make sure this works. waitForOutboundGateways(t, srvB, 1, 30*time.Second) }) } } func TestGatewayAdvertiseInCluster(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob1 := testDefaultOptionsForGateway("B") ob1.Cluster.Name = "B" ob1.Cluster.Host = "127.0.0.1" ob1.Cluster.Port = -1 sb1 := runGatewayServer(ob1) defer sb1.Shutdown() gA := createGatewayConn(t, ob1.Gateway.Host, ob1.Gateway.Port) defer gA.Close() gASend, gAExpect := setupGatewayConn(t, gA, "A", "B") gASend("PING\r\n") gAExpect(pongRe) ob2 := testDefaultOptionsForGateway("B") ob2.Cluster.Name = "B" ob2.Cluster.Host = "127.0.0.1" ob2.Cluster.Port = -1 ob2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) ob2.Gateway.Advertise = "srvB:7222" sb2 := runGatewayServer(ob2) defer sb2.Shutdown() checkClusterFormed(t, sb1, sb2) buf := gAExpect(infoRe) si := &server.Info{} json.Unmarshal(buf[5:], si) var ok bool for _, u := range si.GatewayURLs { if u == "srvB:7222" { ok = true break } } if !ok { t.Fatalf("Url srvB:7222 was not found: %q", si.GatewayURLs) } ob3 := testDefaultOptionsForGateway("B") ob3.Cluster.Name = "B" ob3.Cluster.Host = "127.0.0.1" ob3.Cluster.Port = -1 ob3.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", ob1.Cluster.Port)) ob3.Gateway.Advertise = "srvB:7222" sb3 := runGatewayServer(ob3) defer sb3.Shutdown() checkClusterFormed(t, sb1, sb2, sb3) // Since it is the save srvB:7222 url, we should not get an update. expectNothing(t, gA) // Now shutdown sb2 and make sure that we are not getting an update // with srvB:7222 missing. sb2.Shutdown() expectNothing(t, gA) } func TestGatewayAuthTimeout(t *testing.T) { for _, test := range []struct { name string setAuth bool // wait time.Duration }{ {"auth not explicitly set", false, 2500 * time.Millisecond}, {"auth set", true, 500 * time.Millisecond}, } { t.Run(test.name, func(t *testing.T) { ob := testDefaultOptionsForGateway("B") if test.setAuth { ob.Gateway.AuthTimeout = 0.25 } sb := RunServer(ob) defer sb.Shutdown() sa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer sa.Close() gAExpect := expectCommand(t, sa) dstInfo := checkInfoMsg(t, sa) if dstInfo.Gateway != "B" { t.Fatalf("Expected to connect to %q, got %q", "B", dstInfo.Gateway) } // Don't send our CONNECT and we should be disconnected due to auth timeout. time.Sleep(test.wait) gAExpect(errRe) expectDisconnect(t, sa) }) } } func TestGatewayFirstPingGoesAfterConnect(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) ob := testDefaultOptionsForGateway("B") // For this test, we want the first ping to NOT be disabled. ob.DisableShortFirstPing = false // Also, for this test increase auth_timeout so that it does not disconnect // while checking... ob.Gateway.AuthTimeout = 10.0 sb := RunServer(ob) defer sb.Shutdown() sa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port) defer sa.Close() gASend, gAExpect := sendCommand(t, sa), expectCommand(t, sa) dstInfo := checkInfoMsg(t, sa) if dstInfo.Gateway != "B" { t.Fatalf("Expected to connect to %q, got %q", "B", dstInfo.Gateway) } // Wait and we should not be receiving a PING from server B until we send // a CONNECT. We need to wait for more than the initial PING, so cannot // use expectNothing() helper here. buf := make([]byte, 256) sa.SetReadDeadline(time.Now().Add(2 * time.Second)) if n, err := sa.Read(buf); err == nil { t.Fatalf("Expected nothing, got %s", buf[:n]) } // Now send connect and INFO cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"gateway\":%q}\r\n", false, false, false, "A") gASend(cs) gASend(fmt.Sprintf("INFO {\"gateway\":%q}\r\n", "A")) // We should get the first PING gAExpect(pingRe) } nats-server-2.10.27/test/gosrv_test.go000066400000000000000000000033431477524627100176630ustar00rootroot00000000000000// Copyright 2012-2019 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "net" "runtime" "testing" "time" ) func TestSimpleGoServerShutdown(t *testing.T) { base := runtime.NumGoroutine() opts := DefaultTestOptions opts.Port = -1 s := RunServer(&opts) s.Shutdown() checkFor(t, time.Second, 100*time.Millisecond, func() error { delta := (runtime.NumGoroutine() - base) if delta > 1 { return fmt.Errorf("%d go routines still exist post Shutdown()", delta) } return nil }) } func TestGoServerShutdownWithClients(t *testing.T) { base := runtime.NumGoroutine() opts := DefaultTestOptions opts.Port = -1 s := RunServer(&opts) addr := s.Addr().(*net.TCPAddr) for i := 0; i < 50; i++ { createClientConn(t, "127.0.0.1", addr.Port) } s.Shutdown() // Wait longer for client connections time.Sleep(1 * time.Second) delta := (runtime.NumGoroutine() - base) // There may be some finalizers or IO, but in general more than // 2 as a delta represents a problem. if delta > 2 { t.Fatalf("%d Go routines still exist post Shutdown()", delta) } } func TestGoServerMultiShutdown(t *testing.T) { opts := DefaultTestOptions opts.Port = -1 s := RunServer(&opts) s.Shutdown() s.Shutdown() } nats-server-2.10.27/test/leafnode_test.go000066400000000000000000003540601477524627100203050ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bytes" "crypto/tls" "crypto/x509" "encoding/json" "fmt" "math/rand" "net" "net/url" "os" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/nats-io/nuid" ) func createLeafConn(t tLogger, host string, port int) net.Conn { return createClientConn(t, host, port) } func testDefaultOptionsForLeafNodes() *server.Options { o := DefaultTestOptions o.Port = -1 o.LeafNode.Host = o.Host o.LeafNode.Port = -1 o.NoSystemAccount = true return &o } func runLeafServer() (*server.Server, *server.Options) { o := testDefaultOptionsForLeafNodes() return RunServer(o), o } func runLeafServerOnPort(port int) (*server.Server, *server.Options) { o := testDefaultOptionsForLeafNodes() o.LeafNode.Port = port return RunServer(o), o } func runSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) { return runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) } func runSolicitLeafServerToURL(surl string) (*server.Server, *server.Options) { o := DefaultTestOptions o.Port = -1 o.NoSystemAccount = true rurl, _ := url.Parse(surl) o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} o.LeafNode.ReconnectInterval = 100 * time.Millisecond return RunServer(&o), &o } func TestLeafNodeInfo(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info := checkInfoMsg(t, lc) if !info.AuthRequired { t.Fatalf("AuthRequired should always be true for leaf nodes") } // By default headers should be true. if !info.Headers { t.Fatalf("Expected to have headers on by default") } sendProto(t, lc, "CONNECT {}\r\n") checkLeafNodeConnected(t, s) // Now close connection, make sure we are doing the right accounting in the server. lc.Close() checkLeafNodeConnections(t, s, 0) } func TestLeafNodeSplitBuffer(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() nc, err := nats.Connect(s.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.QueueSubscribe("foo", "bar", func(m *nats.Msg) { m.Respond([]byte("ok")) }) nc.Flush() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() sendProto(t, lc, "CONNECT {}\r\n") checkLeafNodeConnected(t, s) leafSend, leafExpect := setupLeaf(t, lc, 2) leafSend("LS+ reply\r\nPING\r\n") leafExpect(pongRe) leafSend("LMSG foo ") time.Sleep(time.Millisecond) leafSend("+ reply bar 2\r\n") time.Sleep(time.Millisecond) leafSend("OK\r") time.Sleep(time.Millisecond) leafSend("\n") leafExpect(lmsgRe) } func TestNumLeafNodes(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() createNewLeafNode := func() net.Conn { t.Helper() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) checkInfoMsg(t, lc) sendProto(t, lc, "CONNECT {}\r\n") return lc } checkLeafNodeConnections(t, s, 0) lc1 := createNewLeafNode() defer lc1.Close() checkLeafNodeConnections(t, s, 1) lc2 := createNewLeafNode() defer lc2.Close() checkLeafNodeConnections(t, s, 2) // Now test remove works. lc1.Close() checkLeafNodeConnections(t, s, 1) lc2.Close() checkLeafNodeConnections(t, s, 0) } func TestLeafNodeRequiresConnect(t *testing.T) { opts := testDefaultOptionsForLeafNodes() opts.LeafNode.AuthTimeout = 0.001 s := RunServer(opts) defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info := checkInfoMsg(t, lc) if !info.AuthRequired { t.Fatalf("Expected AuthRequired to force CONNECT") } if info.TLSRequired { t.Fatalf("Expected TLSRequired to be false") } if info.TLSVerify { t.Fatalf("Expected TLSVerify to be false") } // Now wait and make sure we get disconnected. errBuf := expectResult(t, lc, errRe) if !strings.Contains(string(errBuf), "Authentication Timeout") { t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) } expectDisconnect(t, lc) } func setupLeaf(t *testing.T, lc net.Conn, expectedSubs int) (sendFun, expectFun) { t.Helper() send, expect := setupConn(t, lc) // A loop detection subscription is sent, so consume this here, along // with the ones that caller expect on setup. expectNumberOfProtos(t, expect, lsubRe, expectedSubs, infoStartRe, pingRe) return send, expect } func TestLeafNodeSendsSubsAfterConnect(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("SUB foo 1\r\n") send("SUB bar 2\r\n") send("SUB foo baz 3\r\n") send("SUB foo baz 4\r\n") send("SUB bar 5\r\n") send("PING\r\n") expect(pongRe) lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This should compress down to 1 for foo, 1 for bar, and 1 for foo [baz] // and one for the loop detection subject. setupLeaf(t, lc, 4) } func TestLeafNodeSendsSubsOngoing(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) send("SUB foo 1\r\n") leafExpect(lsubRe) // Check queues send updates each time. // TODO(dlc) - If we decide to suppress this with a timer approach this test will break. send("SUB foo bar 2\r\n") leafExpect(lsubRe) send("SUB foo bar 3\r\n") leafExpect(lsubRe) send("SUB foo bar 4\r\n") leafExpect(lsubRe) // Now check more normal subs do nothing. send("SUB foo 5\r\n") expectNothing(t, lc) // Check going back down does nothing til we hit 0. send("UNSUB 5\r\n") expectNothing(t, lc) send("UNSUB 1\r\n") leafExpect(lunsubRe) // Queues going down should always send updates. send("UNSUB 2\r\n") leafExpect(lsubRe) send("UNSUB 3\r\n") leafExpect(lsubRe) send("UNSUB 4\r\n") leafExpect(lunsubRe) } func TestLeafNodeSubs(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) leafSend("LS+ foo\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) send("PUB foo 2\r\nOK\r\n") matches := lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "foo", "", "2", "OK") // Second sub should not change delivery leafSend("LS+ foo\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB foo 3\r\nOK!\r\n") matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "foo", "", "3", "OK!") // Now add in a queue sub with weight 4. leafSend("LS+ foo bar 4\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB foo 4\r\nOKOK\r\n") matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "foo", "| bar", "4", "OKOK") // Now add in a queue sub with weight 4. leafSend("LS+ foo baz 2\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB foo 5\r\nHELLO\r\n") matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "foo", "| bar baz", "5", "HELLO") // Test Unsub leafSend("LS- foo\r\n") leafSend("LS- foo bar\r\n") leafSend("LS- foo baz\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB foo 5\r\nHELLO\r\n") expectNothing(t, lc) } func TestLeafNodeMsgDelivery(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) // Now send from leaf side. leafSend("LMSG foo 2\r\nOK\r\n") expectNothing(t, lc) matches := msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkMsg(t, matches[0], "foo", "1", "", "2", "OK") send("UNSUB 1\r\nPING\r\n") expect(pongRe) leafExpect(lunsubRe) send("SUB foo bar 2\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) // Now send again from leaf side. This is targeted so this should // not be delivered. leafSend("LMSG foo 2\r\nOK\r\n") expectNothing(t, lc) expectNothing(t, c) // Now send targeted, and we should receive it. leafSend("LMSG foo | bar 2\r\nOK\r\n") expectNothing(t, lc) matches = msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkMsg(t, matches[0], "foo", "2", "", "2", "OK") // Check reply + queues leafSend("LMSG foo + myreply bar 2\r\nOK\r\n") expectNothing(t, lc) matches = msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkMsg(t, matches[0], "foo", "2", "myreply", "2", "OK") } func TestLeafNodeAndRoutes(t *testing.T) { optsA := LoadConfig("./configs/srv_a_leaf.conf") optsA.DisableShortFirstPing = true optsB := LoadConfig("./configs/srv_b.conf") optsB.DisableShortFirstPing = true srvA := RunServer(optsA) defer srvA.Shutdown() srvB := RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) lc := createLeafConn(t, optsA.LeafNode.Host, optsA.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 6) leafSend("PING\r\n") leafExpect(pongRe) c := createClientConn(t, optsB.Host, optsB.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) send("SUB foo 2\r\nPING\r\n") expect(pongRe) expectNothing(t, lc) send("UNSUB 2\r\nPING\r\n") expect(pongRe) expectNothing(t, lc) send("UNSUB 1\r\nPING\r\n") expect(pongRe) leafExpect(lunsubRe) // Now put it back and test msg flow. send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) leafSend("LMSG foo + myreply bar 2\r\nOK\r\n") expectNothing(t, lc) matches := msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkMsg(t, matches[0], "foo", "1", "myreply", "2", "OK") // Now check reverse. leafSend("LS+ bar\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB bar 2\r\nOK\r\n") matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "bar", "", "2", "OK") } // Helper function to check that a leaf node has connected to our server. func checkLeafNodeConnected(t *testing.T, s *server.Server) { t.Helper() checkLeafNodeConnections(t, s, 1) } func checkLeafNodeConnections(t *testing.T, s *server.Server, expected int) { t.Helper() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if nln := s.NumLeafNodes(); nln != expected { return fmt.Errorf("Expected a connected leafnode for server %q, got %d", s.ID(), nln) } return nil }) } func TestLeafNodeSolicit(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() sl, _ := runSolicitLeafServer(opts) defer sl.Shutdown() checkLeafNodeConnected(t, s) // Now test reconnect. s.Shutdown() // Need to restart it on the same port. s, _ = runLeafServerOnPort(opts.LeafNode.Port) defer s.Shutdown() checkLeafNodeConnected(t, s) } func TestLeafNodeNoEcho(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) // We should not echo back to ourselves. Set up 'foo' subscriptions // on both sides and send message across the leafnode connection. It // should not come back. send("SUB foo 1\r\n") leafExpect(lsubRe) leafSend("LS+ foo\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) leafSend("LMSG foo 2\r\nOK\r\n") expectNothing(t, lc) } func TestLeafNodeHeaderSupport(t *testing.T) { srvA, optsA := runLeafServer() defer srvA.Shutdown() srvB, optsB := runSolicitLeafServer(optsA) defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupHeaderConn(t, clientA) sendA("SUB foo bar 22\r\n") sendA("SUB bar 11\r\n") sendA("PING\r\n") expectA(pongRe) if err := checkExpectedSubs(3, srvB); err != nil { t.Fatalf("%v", err) } sendB, expectB := setupHeaderConn(t, clientB) // Can not have \r\n in payload fyi for regex. // With reply sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) expectHeaderMsgs := expectHeaderMsgsCommand(t, expectA) matches := expectHeaderMsgs(1) checkHmsg(t, matches[0], "foo", "22", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") // Without reply sendB("HPUB foo 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) matches = expectHeaderMsgs(1) checkHmsg(t, matches[0], "foo", "22", "", "12", "14", "K1:V1,K2:V2 ", "ok") // Without queues or reply sendB("HPUB bar 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) matches = expectHeaderMsgs(1) checkHmsg(t, matches[0], "bar", "11", "", "12", "14", "K1:V1,K2:V2 ", "ok") // Without queues but with reply sendB("HPUB bar reply 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) matches = expectHeaderMsgs(1) checkHmsg(t, matches[0], "bar", "11", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") } // Used to setup clusters of clusters for tests. type cluster struct { servers []*server.Server opts []*server.Options name string t *testing.T } func testDefaultClusterOptionsForLeafNodes() *server.Options { o := DefaultTestOptions o.Port = -1 o.Cluster.Host = o.Host o.Cluster.Port = -1 o.Gateway.Host = o.Host o.Gateway.Port = -1 o.LeafNode.Host = o.Host o.LeafNode.Port = -1 return &o } func (c *cluster) shutdown() { if c == nil { return } for i, s := range c.servers { if cf := c.opts[i].ConfigFile; cf != "" { os.RemoveAll(cf) } if sd := s.StoreDir(); sd != "" { os.RemoveAll(sd) } s.Shutdown() } } func shutdownCluster(c *cluster) { c.shutdown() } func (c *cluster) totalSubs() int { totalSubs := 0 for _, s := range c.servers { totalSubs += int(s.NumSubscriptions()) } return totalSubs } // Wait for the expected number of outbound gateways, or fails. func waitForOutboundGateways(t *testing.T, s *server.Server, expected int, timeout time.Duration) { t.Helper() if timeout < 2*time.Second { timeout = 2 * time.Second } checkFor(t, timeout, 15*time.Millisecond, func() error { if n := s.NumOutboundGateways(); n != expected { return fmt.Errorf("Expected %v outbound gateway(s), got %v (ulimit -n too low?)", expected, n) } return nil }) } // Creates a full cluster with numServers and given name and makes sure its well formed. // Will have Gateways and Leaf Node connections active. func createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster { t.Helper() return createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...) } // Creates a cluster and optionally additional accounts and users. // Will have Gateways and Leaf Node connections active. func createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster { t.Helper() if clusterName == "" || numServers < 1 { t.Fatalf("Bad params") } // Setup some accounts and users. // $SYS is always the system account. And we have default FOO and BAR accounts, as well // as DLC and NGS which do a service import. createAccountsAndUsers := func() ([]*server.Account, []*server.User) { if !doAccounts { return []*server.Account{server.NewAccount("$SYS")}, nil } sys := server.NewAccount("$SYS") ngs := server.NewAccount("NGS") dlc := server.NewAccount("DLC") foo := server.NewAccount("FOO") bar := server.NewAccount("BAR") accounts := []*server.Account{sys, foo, bar, ngs, dlc} ngs.AddServiceExport("ngs.usage.*", nil) dlc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.dlc") // Setup users users := []*server.User{ {Username: "dlc", Password: "pass", Permissions: nil, Account: dlc}, {Username: "ngs", Password: "pass", Permissions: nil, Account: ngs}, {Username: "foo", Password: "pass", Permissions: nil, Account: foo}, {Username: "bar", Password: "pass", Permissions: nil, Account: bar}, {Username: "sys", Password: "pass", Permissions: nil, Account: sys}, } return accounts, users } bindGlobal := func(s *server.Server) { ngs, err := s.LookupAccount("NGS") if err != nil { return } // Bind global to service import gacc, _ := s.LookupAccount("$G") gacc.AddServiceImport(ngs, "ngs.usage", "ngs.usage.$G") } // If we are going to connect to another cluster set that up now for options. var gws []*server.RemoteGatewayOpts for _, c := range connectTo { // Gateways autodiscover here too, so just need one address from the set. gwAddr := fmt.Sprintf("nats-gw://%s:%d", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port) gwurl, _ := url.Parse(gwAddr) gws = append(gws, &server.RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}}) } // Make the GWs form faster for the tests. server.SetGatewaysSolicitDelay(gwSolicit) defer server.ResetGatewaysSolicitDelay() // Create seed first. o := testDefaultClusterOptionsForLeafNodes() o.Gateway.Name = clusterName o.Gateway.Gateways = gws // All of these need system accounts. o.Accounts, o.Users = createAccountsAndUsers() o.SystemAccount = "$SYS" o.ServerName = fmt.Sprintf("%s1", clusterName) // Run the server s := RunServer(o) bindGlobal(s) c := &cluster{servers: make([]*server.Server, 0, numServers), opts: make([]*server.Options, 0, numServers), name: clusterName} c.servers = append(c.servers, s) c.opts = append(c.opts, o) // For connecting to seed server above. routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) rurl, _ := url.Parse(routeAddr) routes := []*url.URL{rurl} for i := 1; i < numServers; i++ { o := testDefaultClusterOptionsForLeafNodes() o.Gateway.Name = clusterName o.Gateway.Gateways = gws o.Routes = routes // All of these need system accounts. o.Accounts, o.Users = createAccountsAndUsers() o.SystemAccount = "$SYS" o.ServerName = fmt.Sprintf("%s%d", clusterName, i+1) s := RunServer(o) bindGlobal(s) c.servers = append(c.servers, s) c.opts = append(c.opts, o) } checkClusterFormed(t, c.servers...) if waitOnGWs { // Wait on gateway connections if we were asked to connect to other gateways. if numGWs := len(connectTo); numGWs > 0 { for _, s := range c.servers { waitForOutboundGateways(t, s, numGWs, 2*time.Second) } } } c.t = t return c } func TestLeafNodeGatewayRequiresSystemAccount(t *testing.T) { o := testDefaultClusterOptionsForLeafNodes() o.Gateway.Name = "CLUSTER-A" _, err := server.NewServer(o) if err == nil { t.Fatalf("Expected an error with no system account defined") } } func TestLeafNodeGatewaySendsSystemEvent(t *testing.T) { server.SetGatewaysSolicitDelay(50 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 1) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 1, ca) defer shutdownCluster(cb) // Create client on a server in cluster A opts := ca.opts[0] c := createClientConn(t, opts.Host, opts.Port) defer c.Close() // Listen for the leaf node event. send, expect := setupConnWithAccount(t, ca.servers[0], c, "$SYS") send("SUB $SYS.ACCOUNT.$G.LEAFNODE.CONNECT 1\r\nPING\r\n") expect(pongRe) opts = cb.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This is for our global responses since we are setting up GWs above. leafSend, leafExpect := setupLeaf(t, lc, 8) leafSend("PING\r\n") leafExpect(pongRe) matches := rawMsgRe.FindAllSubmatch(expect(rawMsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } m := matches[0] if string(m[subIndex]) != "$SYS.ACCOUNT.$G.LEAFNODE.CONNECT" { t.Fatalf("Got wrong subject for leaf node event, got %q, wanted %q", m[subIndex], "$SYS.ACCOUNT.$G.LEAFNODE.CONNECT") } } func TestLeafNodeGatewayInterestPropagation(t *testing.T) { server.SetGatewaysSolicitDelay(10 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) sl1, sl1Opts := runSolicitLeafServer(ca.opts[1]) defer sl1.Shutdown() c := createClientConn(t, sl1Opts.Host, sl1Opts.Port) defer c.Close() send, expect := setupConn(t, c) send("SUB foo 1\r\n") send("PING\r\n") expect(pongRe) // Now we will create a new leaf node on cluster B, expect to get the // interest for "foo". opts := cb.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() _, leafExpect := setupConn(t, lc) buf := leafExpect(infoStartRe) buf = infoStartRe.ReplaceAll(buf, []byte(nil)) foundFoo := false for count := 0; count < 10; { // skip first time if we still have data (buf from above may already have some left) if count != 0 || len(buf) == 0 { buf = append(buf, leafExpect(anyRe)...) } count += len(lsubRe.FindAllSubmatch(buf, -1)) if count > 10 { t.Fatalf("Expected %v matches, got %v (buf=%s)", 10, count, buf) } if strings.Contains(string(buf), "foo") { foundFoo = true } buf = lsubRe.ReplaceAll(buf, []byte(nil)) } if len(buf) != 0 { t.Fatalf("did not consume everything, left with: %q", buf) } if !foundFoo { t.Fatalf("Expected interest for 'foo' as 'LS+ foo\\r\\n', got %q", buf) } } func TestLeafNodeAuthSystemEventNoCrash(t *testing.T) { ca := createClusterWithName(t, "A", 1) defer shutdownCluster(ca) opts := ca.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend := sendCommand(t, lc) leafSend("LS+ foo\r\n") checkInfoMsg(t, lc) } func TestLeafNodeWithRouteAndGateway(t *testing.T) { server.SetGatewaysSolicitDelay(50 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) // Create client on a server in cluster A opts := ca.opts[0] c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) // Create a leaf node connection on a server in cluster B opts = cb.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This is for our global responses since we are setting up GWs above. leafSend, leafExpect := setupLeaf(t, lc, 8) leafSend("PING\r\n") leafExpect(pongRe) // Make sure we see interest graph propagation on the leaf node // connection. This is required since leaf nodes only send data // in the presence of interest. send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) send("SUB foo 2\r\nPING\r\n") expect(pongRe) expectNothing(t, lc) send("UNSUB 2\r\n") expectNothing(t, lc) send("UNSUB 1\r\n") leafExpect(lunsubRe) // Now put it back and test msg flow. send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) leafSend("LMSG foo 2\r\nOK\r\n") expectNothing(t, lc) matches := msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkMsg(t, matches[0], "foo", "1", "", "2", "OK") // Now check reverse. leafSend("LS+ bar\r\n") expectNothing(t, lc) leafSend("PING\r\n") leafExpect(pongRe) send("PUB bar 2\r\nOK\r\n") matches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkLmsg(t, matches[0], "bar", "", "2", "OK") } // This will test that we propagate interest only mode after a leafnode // has been established and a new server joins a remote cluster. func TestLeafNodeWithGatewaysAndStaggeredStart(t *testing.T) { ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) // Create the leafnode on a server in cluster A. opts := ca.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 8) leafSend("PING\r\n") leafExpect(pongRe) // Now setup the cluster B. cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) // Create client on a server in cluster B opts = cb.opts[0] c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) // Make sure we see interest graph propagation on the leaf node // connection. This is required since leaf nodes only send data // in the presence of interest. send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) } // This will test that we propagate interest only mode after a leafnode // has been established and a server is restarted.. func TestLeafNodeWithGatewaysServerRestart(t *testing.T) { ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) // Now setup the cluster B. cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) // Create the leafnode on a server in cluster B. opts := cb.opts[1] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 8) leafSend("PING\r\n") leafExpect(pongRe) // Create client on a server in cluster A opts = ca.opts[1] c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) // Make sure we see interest graph propagation on the leaf node // connection. This is required since leaf nodes only send data // in the presence of interest. send("SUB foo 1\r\nPING\r\n") expect(pongRe) leafExpect(lsubRe) // Close old leaf connection and simulate a reconnect. lc.Close() // Shutdown and recreate B and the leafnode connection to it. shutdownCluster(cb) // Create new cluster with longer solicit and don't wait for GW connect. cb = createClusterEx(t, false, 500*time.Millisecond, false, "B", 1, ca) defer shutdownCluster(cb) opts = cb.opts[0] lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() _, leafExpect = setupLeaf(t, lc, 8) // Now wait on GW solicit to fire time.Sleep(500 * time.Millisecond) // We should see the interest for 'foo' here. leafExpect(lsubRe) } func TestLeafNodeLocalizedDQ(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() sl, slOpts := runSolicitLeafServer(opts) defer sl.Shutdown() checkLeafNodeConnected(t, s) c := createClientConn(t, slOpts.Host, slOpts.Port) defer c.Close() send, expect := setupConn(t, c) send("SUB foo bar 1\r\n") send("SUB foo bar 2\r\n") send("SUB foo bar 3\r\n") send("SUB foo bar 4\r\n") send("PING\r\n") expect(pongRe) // Now create another client on the main leaf server. sc := createClientConn(t, opts.Host, opts.Port) defer sc.Close() sendL, expectL := setupConn(t, sc) sendL("SUB foo bar 11\r\n") sendL("SUB foo bar 12\r\n") sendL("SUB foo bar 13\r\n") sendL("SUB foo bar 14\r\n") sendL("PING\r\n") expectL(pongRe) for i := 0; i < 10; i++ { send("PUB foo 2\r\nOK\r\n") } expectNothing(t, sc) matches := msgRe.FindAllSubmatch(expect(msgRe), -1) if len(matches) != 10 { t.Fatalf("Expected 10 msgs, got %d", len(matches)) } for i := 0; i < 10; i++ { checkMsg(t, matches[i], "foo", "", "", "2", "OK") } } func TestLeafNodeBasicAuth(t *testing.T) { content := ` listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" authorization { user: "derek" password: "s3cr3t!" timeout: 2.2 } } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This should fail since we want u/p setupConn(t, lc) errBuf := expectResult(t, lc, errRe) if !strings.Contains(string(errBuf), "Authorization Violation") { t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) } expectDisconnect(t, lc) // Try bad password as well. lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This should fail since we want u/p setupConnWithUserPass(t, lc, "derek", "badpassword") errBuf = expectResult(t, lc, errRe) if !strings.Contains(string(errBuf), "Authorization Violation") { t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) } expectDisconnect(t, lc) // This one should work. lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupConnWithUserPass(t, lc, "derek", "s3cr3t!") leafExpect(infoRe) leafExpect(lsubRe) leafSend("PING\r\n") expectResult(t, lc, pongRe) checkLeafNodeConnected(t, s) } func runTLSSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) { o := DefaultTestOptions o.Port = -1 rurl, _ := url.Parse(fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port)) remote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}} remote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} host, _, _ := net.SplitHostPort(lso.LeafNode.Host) remote.TLSConfig.ServerName = host remote.TLSConfig.InsecureSkipVerify = true remote.Compression.Mode = server.CompressionOff o.LeafNode.Remotes = []*server.RemoteLeafOpts{remote} o.LeafNode.Compression.Mode = server.CompressionOff return RunServer(&o), &o } func TestLeafNodeTLS(t *testing.T) { content := ` listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" timeout: 0.1 } } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info := checkInfoMsg(t, lc) if !info.TLSRequired { t.Fatalf("Expected TLSRequired to be true") } if info.TLSVerify { t.Fatalf("Expected TLSVerify to be false") } // We should get a disconnect here since we have not upgraded to TLS. expectDisconnect(t, lc) // This should work ok. sl, _ := runTLSSolicitLeafServer(opts) defer sl.Shutdown() checkLeafNodeConnected(t, s) } func TestLeafNodeTLSConnCloseEarly(t *testing.T) { content := ` listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" timeout: 2.0 } } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() lc, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", opts.LeafNode.Port)) if err != nil { t.Fatalf("Unable to connect: %v", err) } // Then close right away lc.Close() // Check server does not crash... time.Sleep(250 * time.Millisecond) if s.ID() == "" { t.Fatalf("should not happen") } } type captureLeafNodeErrLogger struct { dummyLogger ch chan string } func (c *captureLeafNodeErrLogger) Errorf(format string, v ...any) { msg := fmt.Sprintf(format, v...) select { case c.ch <- msg: default: } } func TestLeafNodeTLSMixIP(t *testing.T) { content := ` listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" authorization { user: dlc pass: monkey } tls { cert_file: "./configs/certs/server-noip.pem" key_file: "./configs/certs/server-key-noip.pem" ca_file: "./configs/certs/ca.pem" } } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() slContent := ` listen: "127.0.0.1:-1" leafnodes { reconnect: 1 remotes: [ { url: [tls://127.0.0.1:%d, "tls://localhost:%d"] tls { ca_file: "./configs/certs/ca.pem" } } ] } ` slconf := createConfFile(t, []byte(fmt.Sprintf(slContent, opts.LeafNode.Port, opts.LeafNode.Port))) // This will fail but we want to make sure in the correct way, not with // TLS issue because we used an IP for serverName. sl, _ := RunServerWithConfig(slconf) defer sl.Shutdown() ll := &captureLeafNodeErrLogger{ch: make(chan string, 2)} sl.SetLogger(ll, false, false) // We may or may not get an error depending on timing. For the handshake bug // we would always get it, so make sure if we have anything it is not that. select { case msg := <-ll.ch: if strings.Contains(msg, "TLS handshake error") && strings.Contains(msg, "doesn't contain any IP SANs") { t.Fatalf("Got bad error about TLS handshake") } default: } } func runLeafNodeOperatorServer(t *testing.T) (*server.Server, *server.Options, string) { t.Helper() content := ` port: -1 server_name: OP operator = "./configs/nkeys/op.jwt" resolver = MEMORY listen: "127.0.0.1:-1" leafnodes { listen: "127.0.0.1:-1" } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) return s, opts, conf } func genCredsFile(t *testing.T, jwt string, seed []byte) string { creds := ` -----BEGIN NATS USER JWT----- %s ------END NATS USER JWT------ ************************* IMPORTANT ************************* NKEY Seed printed below can be used to sign and prove identity. NKEYs are sensitive and should be treated as secrets. -----BEGIN USER NKEY SEED----- %s ------END USER NKEY SEED------ ************************************************************* ` return createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), "\t\t", "", -1))) } func runSolicitWithCredentials(t *testing.T, opts *server.Options, creds string) (*server.Server, *server.Options, string) { content := ` port: -1 leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:%d credentials: '%s' } ] } ` config := fmt.Sprintf(content, opts.LeafNode.Port, creds) conf := createConfFile(t, []byte(config)) s, opts := RunServerWithConfig(conf) return s, opts, conf } func TestLeafNodeOperatorModel(t *testing.T) { s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Make sure we get disconnected without proper credentials etc. lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This should fail since we want user jwt, signed nonce etc. setupConn(t, lc) errBuf := expectResult(t, lc, errRe) if !strings.Contains(string(errBuf), "Authorization Violation") { t.Fatalf("Authentication Timeout response incorrect: %q", errBuf) } expectDisconnect(t, lc) // Setup account and a user that will be used by the remote leaf node server. // createAccount automatically registers with resolver etc.. _, akp := createAccount(t, s) kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s) } func TestLeafNodeUserPermsForConnection(t *testing.T) { s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Setup account and a user that will be used by the remote leaf node server. // createAccount automatically registers with resolver etc.. acc, akp := createAccount(t, s) kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) nuc.Permissions.Pub.Allow.Add("foo.>") nuc.Permissions.Pub.Allow.Add("baz.>") nuc.Permissions.Sub.Allow.Add("foo.>") // we would be immediately disconnected if that would not work nuc.Permissions.Sub.Deny.Add("$SYS.>") ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) content := ` port: -1 leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:%d credentials: '%s' deny_import: "foo.33" deny_export: "foo.33" } ] } ` config := fmt.Sprintf(content, opts.LeafNode.Port, mycreds) lnconf := createConfFile(t, []byte(config)) sl, _ := RunServerWithConfig(lnconf) defer sl.Shutdown() checkLeafNodeConnected(t, s) // Create credentials for a normal unrestricted user that we will connect to the op server. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Create a user on the leafnode server that solicited. nc2, err := nats.Connect(sl.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Make sure subscriptions properly do or do not make it to the hub. nc2.SubscribeSync("bar") checkNoSubInterest(t, s, acc.GetName(), "bar", 20*time.Millisecond) // This one should. nc2.SubscribeSync("foo.22") checkSubInterest(t, s, acc.GetName(), "foo.22", 100*time.Millisecond) // Capture everything. sub, _ := nc.SubscribeSync(">") nc.Flush() checkSubInterest(t, sl, "$G", ">", 100*time.Millisecond) // Now check local pubs are not forwarded. nc2.Publish("baz.22", nil) m, err := sub.NextMsg(1 * time.Second) if err != nil || m.Subject != "baz.22" { t.Fatalf("Expected to received this message") } nc2.Publish("bar.22", nil) if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Did not expect to receive this message") } // Check local overrides work. nc2.Publish("foo.33", nil) if _, err := sub.NextMsg(100 * time.Millisecond); err == nil { t.Fatalf("Did not expect to receive this message") } // This would trigger the sub interest below. sub.Unsubscribe() nc.Flush() nc2.SubscribeSync("foo.33") checkNoSubInterest(t, s, acc.GetName(), "foo.33", 20*time.Millisecond) } func TestLeafNodeMultipleAccounts(t *testing.T) { // So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have // the $G global account and no auth. Make sure things work properly here. s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Setup the two accounts for this server. a, akp1 := createAccount(t, s) kp1, _ := nkeys.CreateUser() pub1, _ := kp1.PublicKey() nuc1 := jwt.NewUserClaims(pub1) ujwt1, err := nuc1.Encode(akp1) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } // Create second account. createAccount(t, s) // Create the leaf node server using the first account. seed, _ := kp1.Seed() mycreds := genCredsFile(t, ujwt1, seed) sl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s) // To connect to main server. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc1, err := nats.Connect(url, createUserCreds(t, s, akp1)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() // This is a client connected to the leaf node with no auth, // binding to account1 via leafnode connection. // To connect to leafnode server. lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) ncl, err := nats.Connect(lurl) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() lsub, _ := ncl.SubscribeSync("foo.test") // Wait for the subs to propagate. LDS + foo.test checkSubInterest(t, s, a.GetName(), "foo.test", 2*time.Second) // Now send from nc1 with account 1, should be received by our leafnode subscriber. nc1.Publish("foo.test", nil) _, err = lsub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } } func TestLeafNodeOperatorAndPermissions(t *testing.T) { s, opts, conf := runLeafNodeOperatorServer(t) defer os.Remove(conf) defer s.Shutdown() acc, akp := createAccount(t, s) kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() // Create SRV user, with no limitations srvnuc := jwt.NewUserClaims(pub) srvujwt, err := srvnuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() srvcreds := genCredsFile(t, srvujwt, seed) defer os.Remove(srvcreds) // Create connection for SRV srvnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(srvcreds)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer srvnc.Close() // Create on the server "s" a subscription on "*" and on "foo". // We check that the subscription on "*" will be able to receive // messages since LEAF has publish permissions on "foo", so msg // should be received. srvsubStar, err := srvnc.SubscribeSync("*") if err != nil { t.Fatalf("Error on subscribe: %v", err) } srvsubFoo, err := srvnc.SubscribeSync("foo") if err != nil { t.Fatalf("Error on subscribe: %v", err) } srvnc.Flush() // Create LEAF user, with pub perms on "foo" and sub perms on "bar" leafnuc := jwt.NewUserClaims(pub) leafnuc.Permissions.Pub.Allow.Add("foo") leafnuc.Permissions.Sub.Allow.Add("bar") leafnuc.Permissions.Sub.Allow.Add("baz") leafujwt, err := leafnuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } leafcreds := genCredsFile(t, leafujwt, seed) defer os.Remove(leafcreds) content := ` port: -1 server_name: LN leafnodes { remotes = [ { url: nats-leaf://127.0.0.1:%d credentials: '%s' } ] } ` config := fmt.Sprintf(content, opts.LeafNode.Port, leafcreds) lnconf := createConfFile(t, []byte(config)) defer os.Remove(lnconf) sl, _ := RunServerWithConfig(lnconf) defer sl.Shutdown() checkLeafNodeConnected(t, s) // Check that interest makes it to "sl" server. // This helper does not check for wildcard interest... checkSubInterest(t, sl, "$G", "foo", time.Second) // Create connection for LEAF and subscribe on "bar" leafnc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(leafcreds)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer leafnc.Close() leafsub, err := leafnc.SubscribeSync("bar") if err != nil { t.Fatalf("Error on subscribe: %v", err) } // To check that we can pull in 'baz'. leafsubpwc, err := leafnc.SubscribeSync("*") if err != nil { t.Fatalf("Error on subscribe: %v", err) } leafnc.Flush() // Make sure the interest on "bar" from "sl" server makes it to the "s" server. checkSubInterest(t, s, acc.GetName(), "bar", time.Second) // Check for local interest too. checkSubInterest(t, sl, "$G", "bar", time.Second) // Now that we know that "s" has received interest on "bar", create // the sub on "bar" locally on "s" srvsub, err := srvnc.SubscribeSync("bar") if err != nil { t.Fatalf("Error on subscribe: %v", err) } srvnc.Publish("bar", []byte("hello")) if _, err := srvsub.NextMsg(time.Second); err != nil { t.Fatalf("SRV did not get message: %v", err) } if _, err := leafsub.NextMsg(time.Second); err != nil { t.Fatalf("LEAF did not get message: %v", err) } if _, err := leafsubpwc.NextMsg(time.Second); err != nil { t.Fatalf("LEAF did not get message: %v", err) } // The leafnode has a sub on '*', that should pull in a publish to 'baz'. srvnc.Publish("baz", []byte("hello")) if _, err := leafsubpwc.NextMsg(time.Second); err != nil { t.Fatalf("LEAF did not get message: %v", err) } // User LEAF user on "sl" server, publish on "foo" leafnc.Publish("foo", []byte("hello")) // The user SRV on "s" receives it because the LN connection // is allowed to publish on "foo". if _, err := srvsubFoo.NextMsg(time.Second); err != nil { t.Fatalf("SRV did not get message: %v", err) } // The wildcard subscription should get it too. if _, err := srvsubStar.NextMsg(time.Second); err != nil { t.Fatalf("SRV did not get message: %v", err) } // However, even when using an unrestricted user connects to "sl" and // publishes on "bar", the user SRV on "s" should not receive it because // the LN connection is not allowed to publish (send msg over) on "bar". nc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(srvcreds)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.Publish("bar", []byte("should not received")) if _, err := srvsub.NextMsg(250 * time.Millisecond); err == nil { t.Fatal("Should not have received message") } } func TestLeafNodeSignerUser(t *testing.T) { s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Setup the two accounts for this server. _, akp1 := createAccount(t, s) apk1, _ := akp1.PublicKey() // add a signing key to the account akp2, err := nkeys.CreateAccount() if err != nil { t.Fatal(err) } apk2, _ := akp2.PublicKey() token, err := s.AccountResolver().Fetch(apk1) if err != nil { t.Fatal(err) } ac, err := jwt.DecodeAccountClaims(token) if err != nil { t.Fatal(err) } ac.SigningKeys.Add(apk2) okp, _ := nkeys.FromSeed(oSeed) token, err = ac.Encode(okp) if err != nil { t.Fatal(err) } // update the resolver account, _ := s.LookupAccount(apk1) err = s.AccountResolver().Store(apk1, token) if err != nil { t.Fatal(err) } s.UpdateAccountClaims(account, ac) tt, err := s.AccountResolver().Fetch(apk1) if err != nil { t.Fatal(err) } ac2, err := jwt.DecodeAccountClaims(tt) if err != nil { t.Fatal(err) } if len(ac2.SigningKeys) != 1 { t.Fatal("signing key is not added") } if _, ok := ac2.SigningKeys[apk2]; !ok { t.Fatal("signing key is not added") } // create an user signed by the signing key kp1, _ := nkeys.CreateUser() pub1, _ := kp1.PublicKey() nuc1 := jwt.NewUserClaims(pub1) nuc1.IssuerAccount = apk1 ujwt1, err := nuc1.Encode(akp2) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } // Create the leaf node server using the first account. seed, _ := kp1.Seed() mycreds := genCredsFile(t, ujwt1, seed) sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s) } func TestLeafNodeExportsImports(t *testing.T) { // So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have // the $G global account and no auth. Make sure things work properly here. s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Setup the two accounts for this server. okp, _ := nkeys.FromSeed(oSeed) // Create second account with exports acc2, akp2 := createAccount(t, s) akp2Pub, _ := akp2.PublicKey() akp2AC := jwt.NewAccountClaims(akp2Pub) streamExport := &jwt.Export{Subject: "foo.stream", Type: jwt.Stream} serviceExport := &jwt.Export{Subject: "req.echo", Type: jwt.Service} akp2AC.Exports.Add(streamExport, serviceExport) akp2ACJWT, err := akp2AC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } s.UpdateAccountClaims(acc2, akp2AC) // Now create the first account and add on the imports. This will be what is used in the leafnode. acc1, akp1 := createAccount(t, s) akp1Pub, _ := akp1.PublicKey() akp1AC := jwt.NewAccountClaims(akp1Pub) streamImport := &jwt.Import{Account: akp2Pub, Subject: "foo.stream", To: "import", Type: jwt.Stream} serviceImport := &jwt.Import{Account: akp2Pub, Subject: "import.request", To: "req.echo", Type: jwt.Service} akp1AC.Imports.Add(streamImport, serviceImport) akp1ACJWT, err := akp1AC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } s.UpdateAccountClaims(acc1, akp1AC) // Create the user will we use to connect the leafnode. kp1, _ := nkeys.CreateUser() pub1, _ := kp1.PublicKey() nuc1 := jwt.NewUserClaims(pub1) ujwt1, err := nuc1.Encode(akp1) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } // Create the leaf node server using the first account. seed, _ := kp1.Seed() mycreds := genCredsFile(t, ujwt1, seed) sl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s) // To connect to main server. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) // Imported nc1, err := nats.Connect(url, createUserCreds(t, s, akp1)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() // Exported nc2, err := nats.Connect(url, createUserCreds(t, s, akp2)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Leaf node connection. lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) ncl, err := nats.Connect(lurl) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() // So everything should be setup here. So let's test streams first. lsub, _ := ncl.SubscribeSync("import.foo.stream") // Wait for all subs to propagate. checkSubInterest(t, s, acc1.GetName(), "import.foo.stream", time.Second) // Pub to other account with export on original subject. nc2.Publish("foo.stream", nil) _, err = lsub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // Services // Create listener on nc2 nc2.Subscribe("req.echo", func(msg *nats.Msg) { nc2.Publish(msg.Reply, []byte("WORKED")) }) nc2.Flush() // Now send the request on the leaf node client. if _, err := ncl.Request("import.request", []byte("fingers crossed"), 500*time.Millisecond); err != nil { t.Fatalf("Did not receive response: %v", err) } } func TestLeafNodeExportImportComplexSetup(t *testing.T) { content := ` port: -1 operator = "./configs/nkeys/op.jwt" resolver = MEMORY cluster { port: -1 name: xyz } leafnodes { listen: "127.0.0.1:-1" } ` conf := createConfFile(t, []byte(content)) s1, s1Opts := RunServerWithConfig(conf) defer s1.Shutdown() content = fmt.Sprintf(` port: -1 operator = "./configs/nkeys/op.jwt" resolver = MEMORY cluster { port: -1 name: xyz routes: ["nats://%s:%d"] } leafnodes { listen: "127.0.0.1:-1" } `, s1Opts.Cluster.Host, s1Opts.Cluster.Port) conf = createConfFile(t, []byte(content)) s2, s2Opts := RunServerWithConfig(conf) defer s2.Shutdown() // Setup the two accounts for this server. okp, _ := nkeys.FromSeed(oSeed) // Create second account with exports acc2, akp2 := createAccount(t, s1) akp2Pub, _ := akp2.PublicKey() akp2AC := jwt.NewAccountClaims(akp2Pub) streamExport := &jwt.Export{Subject: "foo.stream", Type: jwt.Stream} serviceExport := &jwt.Export{Subject: "req.echo", Type: jwt.Service} akp2AC.Exports.Add(streamExport, serviceExport) akp2ACJWT, err := akp2AC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s1.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } s1.UpdateAccountClaims(acc2, akp2AC) // Now create the first account and add on the imports. This will be what is used in the leafnode. acc1, akp1 := createAccount(t, s1) akp1Pub, _ := akp1.PublicKey() akp1AC := jwt.NewAccountClaims(akp1Pub) streamImport := &jwt.Import{Account: akp2Pub, Subject: "foo.stream", To: "import", Type: jwt.Stream} serviceImport := &jwt.Import{Account: akp2Pub, Subject: "import.request", To: "req.echo", Type: jwt.Service} akp1AC.Imports.Add(streamImport, serviceImport) akp1ACJWT, err := akp1AC.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s1.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } s1.UpdateAccountClaims(acc1, akp1AC) if err := s2.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } // Just make sure that account object registered in S2 is not acc2 if a, err := s2.LookupAccount(acc2.Name); err != nil || a == acc2 { t.Fatalf("Lookup account error: %v - accounts are same: %v", err, a == acc2) } if err := s2.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } // Just make sure that account object registered in S2 is not acc1 if a, err := s2.LookupAccount(acc1.Name); err != nil || a == acc1 { t.Fatalf("Lookup account error: %v - accounts are same: %v", err, a == acc1) } // Create the user will we use to connect the leafnode. kp1, _ := nkeys.CreateUser() pub1, _ := kp1.PublicKey() nuc1 := jwt.NewUserClaims(pub1) ujwt1, err := nuc1.Encode(akp1) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } // Create the leaf node server using the first account. seed, _ := kp1.Seed() mycreds := genCredsFile(t, ujwt1, seed) sl, lopts, _ := runSolicitWithCredentials(t, s1Opts, mycreds) defer sl.Shutdown() checkLeafNodeConnected(t, s1) // Url to server s2 s2URL := fmt.Sprintf("nats://%s:%d", s2Opts.Host, s2Opts.Port) // Imported nc1, err := nats.Connect(s2URL, createUserCreds(t, s2, akp1)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() // Exported nc2, err := nats.Connect(s2URL, createUserCreds(t, s2, akp2)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Leaf node connection. lurl := fmt.Sprintf("nats://%s:%d", lopts.Host, lopts.Port) ncl, err := nats.Connect(lurl) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() // So everything should be setup here. So let's test streams first. lsub, _ := ncl.SubscribeSync("import.foo.stream") // Wait for the sub to propagate to s2. LDS + subject above. checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { if acc1.RoutedSubs() != 6 { return fmt.Errorf("Still no routed subscription: %d", acc1.RoutedSubs()) } return nil }) // Pub to other account with export on original subject. nc2.Publish("foo.stream", nil) if _, err = lsub.NextMsg(1 * time.Second); err != nil { t.Fatalf("Did not receive stream message: %s", err) } // Services // Create listener on nc2 (which connects to s2) gotIt := int32(0) nc2.Subscribe("req.echo", func(msg *nats.Msg) { atomic.AddInt32(&gotIt, 1) nc2.Publish(msg.Reply, []byte("WORKED")) }) nc2.Flush() // Wait for it to make it across. time.Sleep(250 * time.Millisecond) // Now send the request on the leaf node client. if _, err := ncl.Request("import.request", []byte("fingers crossed"), 5500*time.Millisecond); err != nil { if atomic.LoadInt32(&gotIt) == 0 { t.Fatalf("Request was not received") } t.Fatalf("Did not receive response: %v", err) } } func TestLeafNodeInfoURLs(t *testing.T) { for _, test := range []struct { name string useAdvertise bool }{ { "without advertise", false, }, { "with advertise", true, }, } { t.Run(test.name, func(t *testing.T) { opts := testDefaultOptionsForLeafNodes() opts.Cluster.Name = "A" opts.Cluster.Port = -1 opts.LeafNode.Host = "127.0.0.1" if test.useAdvertise { opts.LeafNode.Advertise = "me:1" } s1 := RunServer(opts) defer s1.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info := checkInfoMsg(t, lc) if sz := len(info.LeafNodeURLs); sz != 1 { t.Fatalf("Expected LeafNodeURLs array to be size 1, got %v", sz) } var s1LNURL string if test.useAdvertise { s1LNURL = "me:1" } else { s1LNURL = net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(opts.LeafNode.Port)) } if url := info.LeafNodeURLs[0]; url != s1LNURL { t.Fatalf("Expected URL to be %s, got %s", s1LNURL, url) } lc.Close() opts2 := testDefaultOptionsForLeafNodes() opts2.Cluster.Name = "A" opts2.Cluster.Port = -1 opts2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", opts.Cluster.Host, opts.Cluster.Port)) opts2.LeafNode.Host = "127.0.0.1" if test.useAdvertise { opts2.LeafNode.Advertise = "me:2" } s2 := RunServer(opts2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info = checkInfoMsg(t, lc) if sz := len(info.LeafNodeURLs); sz != 2 { t.Fatalf("Expected LeafNodeURLs array to be size 2, got %v", sz) } var s2LNURL string if test.useAdvertise { s2LNURL = "me:2" } else { s2LNURL = net.JoinHostPort(opts2.LeafNode.Host, strconv.Itoa(opts2.LeafNode.Port)) } var ok [2]int for _, url := range info.LeafNodeURLs { if url == s1LNURL { ok[0]++ } else if url == s2LNURL { ok[1]++ } } for i, res := range ok { if res != 1 { t.Fatalf("URL from server %v was found %v times", i+1, res) } } lc.Close() // Remove s2, and wait for route to be lost on s1. s2.Shutdown() checkNumRoutes(t, s1, 0) // Now check that s1 returns only itself in the URLs array. lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info = checkInfoMsg(t, lc) if sz := len(info.LeafNodeURLs); sz != 1 { t.Fatalf("Expected LeafNodeURLs array to be size 1, got %v", sz) } if url := info.LeafNodeURLs[0]; url != s1LNURL { t.Fatalf("Expected URL to be %s, got %s", s1LNURL, url) } lc.Close() s1.Shutdown() // Now we need a configuration where both s1 and s2 have a route // to each other, so we need explicit configuration. We are trying // to get S1->S2 and S2->S1 so one of the route is dropped. This // should not affect the number of URLs reported in INFO. opts.Cluster.Port = 5223 opts.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:5224", opts2.Host)) s1, _ = server.NewServer(opts) defer s1.Shutdown() opts2.Cluster.Port = 5224 opts2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:5223", opts.Host)) s2, _ = server.NewServer(opts2) defer s2.Shutdown() // Start this way to increase chance of having the two connect // to each other at the same time. This will cause one of the // route to be dropped. wg := &sync.WaitGroup{} wg.Add(2) go func() { s1.Start() wg.Done() }() go func() { s2.Start() wg.Done() }() checkClusterFormed(t, s1, s2) lc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() info = checkInfoMsg(t, lc) if sz := len(info.LeafNodeURLs); sz != 2 { t.Fatalf("Expected LeafNodeURLs array to be size 2, got %v", sz) } ok[0], ok[1] = 0, 0 for _, url := range info.LeafNodeURLs { if url == s1LNURL { ok[0]++ } else if url == s2LNURL { ok[1]++ } } for i, res := range ok { if res != 1 { t.Fatalf("URL from server %v was found %v times", i+1, res) } } }) } } func TestLeafNodeFailover(t *testing.T) { server.SetGatewaysSolicitDelay(50 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 2) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 1, ca) defer shutdownCluster(cb) // Start a server that creates LeafNode connection to first // server in cluster A. s, opts := runSolicitLeafServer(ca.opts[0]) defer s.Shutdown() // Shutdown that server on A. ca.servers[0].Shutdown() // Make sure that s reconnects its LN connection checkLeafNodeConnected(t, ca.servers[1]) // Verify that LeafNode info protocol is sent to the server `s` // with list of new servers. To do that, we will restart // ca[0] but with a different LN listen port. ca.opts[0].Port = -1 ca.opts[0].Cluster.Port = -1 ca.opts[0].Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", ca.opts[1].Cluster.Host, ca.opts[1].Cluster.Port)) ca.opts[0].LeafNode.Port = -1 newa0 := RunServer(ca.opts[0]) defer newa0.Shutdown() checkClusterFormed(t, newa0, ca.servers[1]) // Shutdown the server the LN is currently connected to. It should // reconnect to newa0. ca.servers[1].Shutdown() checkLeafNodeConnected(t, newa0) // Now shutdown newa0 and make sure `s` does not reconnect // to server in gateway. newa0.Shutdown() // Wait for more than the reconnect attempts. time.Sleep(opts.LeafNode.ReconnectInterval + 50*time.Millisecond) checkLeafNodeConnections(t, cb.servers[0], 0) } func TestLeafNodeAdvertise(t *testing.T) { // Create a dummy listener which will we use for the advertise address. // We will then stop the server and the test will be a success if // this listener accepts a connection. ch := make(chan struct{}, 1) l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Error starting listener: %v", err) } defer l.Close() wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() c, _ := l.Accept() if c != nil { c.Close() } l.Close() ch <- struct{}{} }() port := l.Addr().(*net.TCPAddr).Port o2 := testDefaultOptionsForLeafNodes() o2.LeafNode.Advertise = fmt.Sprintf("127.0.0.1:%d", port) o2.Cluster.Name = "A" o2.Cluster.Port = -1 s2 := RunServer(o2) defer s2.Shutdown() o1 := testDefaultOptionsForLeafNodes() o1.Cluster.Name = "A" o1.Cluster.Port = -1 o1.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o2.Cluster.Port)) s1 := RunServer(o1) defer s1.Shutdown() checkClusterFormed(t, s1, s2) // Start a server that connects to s1. It should be made aware // of s2 (and its advertise address). s, _ := runSolicitLeafServer(o1) defer s.Shutdown() // Wait for leaf node connection to be established on s1. checkLeafNodeConnected(t, s1) // Shutdown s1. The listener that we created should be the one // receiving the connection from s. s1.Shutdown() select { case <-ch: case <-time.After(5 * time.Second): t.Fatalf("Server did not reconnect to advertised address") } wg.Wait() } func TestLeafNodeConnectionLimitsSingleServer(t *testing.T) { s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // Setup account and a user that will be used by the remote leaf node server. // createAccount automatically registers with resolver etc.. acc, akp := createAccount(t, s) // Now update with limits for lead node connections. const maxleafs = 2 apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) nac.Limits.LeafNodeConn = maxleafs s.UpdateAccountClaims(acc, nac) // Make sure we have the limits updated in acc. if mleafs := acc.MaxActiveLeafNodes(); mleafs != maxleafs { t.Fatalf("Expected to have max leafnodes of %d, got %d", maxleafs, mleafs) } // Create the user credentials for the leadnode connection. kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) checkAccConnectionCounts := func(t *testing.T, expected int) { t.Helper() checkFor(t, 2*time.Second, 15*time.Millisecond, func() error { // Make sure we are accounting properly here. if nln := acc.NumLeafNodes(); nln != expected { return fmt.Errorf("expected %v leaf node, got %d", expected, nln) } // clients and leafnodes counted together. if nc := acc.NumConnections(); nc != expected { return fmt.Errorf("expected %v for total connections, got %d", expected, nc) } return nil }) } checkAccNLF := func(n int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { if nln := acc.NumLeafNodes(); nln != n { return fmt.Errorf("Expected %d leaf node, got %d", n, nln) } return nil }) } sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() checkLeafNodeConnections(t, s, 1) // Make sure we are accounting properly here. checkAccNLF(1) // clients and leafnodes counted together. if nc := acc.NumConnections(); nc != 1 { t.Fatalf("Expected 1 for total connections, got %d", nc) } s2, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer s2.Shutdown() checkLeafNodeConnections(t, s, 2) checkAccConnectionCounts(t, 2) s2.Shutdown() checkLeafNodeConnections(t, s, 1) checkAccConnectionCounts(t, 1) // Make sure we are accounting properly here. checkAccNLF(1) // clients and leafnodes counted together. if nc := acc.NumConnections(); nc != 1 { t.Fatalf("Expected 1 for total connections, got %d", nc) } // Now add back the second one as #3. s3, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer s3.Shutdown() checkLeafNodeConnections(t, s, 2) checkAccConnectionCounts(t, 2) // Once we are here we should not be able to create anymore. Limit == 2. s4, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer s4.Shutdown() checkAccConnectionCounts(t, 2) // Make sure s4 has 0 still. We need checkFor because it is possible // that when we check we have actually the connection registered for // a short period before it is closed due to limit. checkFor(t, time.Second, 15*time.Millisecond, func() error { if nln := s4.NumLeafNodes(); nln != 0 { return fmt.Errorf("Expected no leafnodes accounted for in s4, got %d", nln) } return nil }) // Make sure this is still 2. checkLeafNodeConnections(t, s, 2) } func TestLeafNodeConnectionLimitsCluster(t *testing.T) { content := ` port: -1 operator = "./configs/nkeys/op.jwt" system_account = "AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG" resolver = MEMORY cluster { port: -1 name: xyz } leafnodes { listen: "127.0.0.1:-1" } resolver_preload = { AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ" } ` conf := createConfFile(t, []byte(content)) s1, s1Opts := RunServerWithConfig(conf) defer s1.Shutdown() content = fmt.Sprintf(` port: -1 operator = "./configs/nkeys/op.jwt" system_account = "AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG" resolver = MEMORY cluster { port: -1 name: xyz routes: ["nats://%s:%d"] } leafnodes { listen: "127.0.0.1:-1" } resolver_preload = { AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : "eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ" } `, s1Opts.Cluster.Host, s1Opts.Cluster.Port) conf = createConfFile(t, []byte(content)) s2, s2Opts := RunServerWithConfig(conf) defer s2.Shutdown() // Setup the two accounts for this server. okp, _ := nkeys.FromSeed(oSeed) // Setup account and a user that will be used by the remote leaf node server. // createAccount automatically registers with resolver etc.. acc, akp := createAccount(t, s1) // Now update with limits for lead node connections. const maxleafs = 10 apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) nac.Limits.LeafNodeConn = maxleafs ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s1.AccountResolver().Store(apub, ajwt); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } s1.UpdateAccountClaims(acc, nac) if err := s2.AccountResolver().Store(apub, ajwt); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } // Make sure that account object registered in S2 is not acc2 acc2, err := s2.LookupAccount(acc.Name) if err != nil || acc == acc2 { t.Fatalf("Lookup account error: %v - accounts are same: %v", err, acc == acc2) } // Create the user credentials for the leadnode connection. kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) loop := maxleafs / 2 // Now create maxleafs/2 leaf node servers on each operator server. for i := 0; i < loop; i++ { sl1, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds) defer sl1.Shutdown() sl2, _, _ := runSolicitWithCredentials(t, s2Opts, mycreds) defer sl2.Shutdown() } checkLeafNodeConnections(t, s1, loop) checkLeafNodeConnections(t, s2, loop) // Now check that we have the remotes registered. This will prove we are sending // and processing the leaf node connect events properly etc. checkAccLFCount := func(acc *server.Account, remote bool, n int) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { str := "" var nln int if remote { nln = acc.NumRemoteLeafNodes() str = "remote" } else { nln = acc.NumLeafNodes() } if nln != n { return fmt.Errorf("number of expected %sleaf nodes is %v, got %v", str, n, nln) } return nil }) } checkAccLFCount(acc, true, loop) checkAccLFCount(acc2, true, loop) // Now that we are here we should not be allowed anymore leaf nodes. l, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds) defer l.Shutdown() checkAccLFCount(acc, false, maxleafs) // Should still be at loop size. checkLeafNodeConnections(t, s1, loop) l, _, _ = runSolicitWithCredentials(t, s2Opts, mycreds) defer l.Shutdown() checkAccLFCount(acc2, false, maxleafs) // Should still be at loop size. checkLeafNodeConnections(t, s2, loop) } func TestLeafNodeSwitchGatewayToInterestModeOnly(t *testing.T) { server.SetGatewaysSolicitDelay(50 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) // Create client on a server in cluster A opts := ca.opts[0] c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) // Send a message from this client on "foo" so that B // registers a no-interest for account "$G" send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) // Create a leaf node connection on a server in cluster B opts = cb.opts[0] lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() // This is for our global responses since we are setting up GWs above. leafSend, leafExpect := setupLeaf(t, lc, 8) leafSend("PING\r\n") leafExpect(pongRe) } // The MSG proto for routes and gateways is RMSG, and we have an // optimization that a scratch buffer has RMSG and when doing a // client we just start at scratch[1]. For leaf nodes its LMSG and we // rewrite scratch[0], but never reset it which causes protocol // errors when used with routes or gateways after use to send // to a leafnode. // We will create a server with a leafnode connection and a route // and a gateway connection. // route connections to simulate. func TestLeafNodeResetsMSGProto(t *testing.T) { server.GatewayDoNotForceInterestOnlyMode(true) defer server.GatewayDoNotForceInterestOnlyMode(false) opts := testDefaultOptionsForLeafNodes() opts.Cluster.Name = "xyz" opts.Cluster.Host = opts.Host opts.Cluster.Port = -1 opts.Gateway.Name = "xyz" opts.Gateway.Host = opts.Host opts.Gateway.Port = -1 opts.Accounts = []*server.Account{server.NewAccount("$SYS")} opts.SystemAccount = "$SYS" s := RunServer(opts) defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupConn(t, lc) // To avoid possible INFO when switching to interest mode only, // delay start of gateway. time.Sleep(500 * time.Millisecond) gw := createGatewayConn(t, opts.Gateway.Host, opts.Gateway.Port) defer gw.Close() gwSend, gwExpect := setupGatewayConn(t, gw, "A", "xyz") gwSend("PING\r\n") gwExpect(pongRe) // This is for our global responses since we are setting up GWs above. leafExpect(lsubRe) // Now setup interest in the leaf node for 'foo'. leafSend("LS+ foo\r\nPING\r\n") leafExpect(pongRe) // Send msg from the gateway. gwSend("RMSG $G foo 2\r\nok\r\nPING\r\n") gwExpect(pongRe) leafExpect(lmsgRe) // At this point the gw inside our main server's scratch buffer is LMSG. When we do // same with a connected route with interest it should fail. rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() checkInfoMsg(t, rc) routeSend, routeExpect := setupRouteEx(t, rc, opts, "RC") routeSend("RS+ $G foo\r\nPING\r\n") routeExpect(pongRe) // This is for the route interest we just created. leafExpect(lsubRe) // Send msg from the gateway. gwSend("RMSG $G foo 2\r\nok\r\nPING\r\n") gwExpect(pongRe) leafExpect(lmsgRe) // Now make sure we get it on route. This will fail with the proto bug. routeExpect(rmsgRe) } // We need to make sure that as a remote server we also send our local subs on connect. func TestLeafNodeSendsRemoteSubsOnConnect(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() sl, slOpts := runSolicitLeafServer(opts) defer sl.Shutdown() checkLeafNodeConnected(t, s) s.Shutdown() c := createClientConn(t, slOpts.Host, slOpts.Port) defer c.Close() send, expect := setupConn(t, c) send("SUB foo 1\r\n") send("PING\r\n") expect(pongRe) // Need to restart it on the same port. s, _ = runLeafServerOnPort(opts.LeafNode.Port) defer s.Shutdown() checkLeafNodeConnected(t, s) lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() setupLeaf(t, lc, 3) } func TestLeafNodeServiceImportLikeNGS(t *testing.T) { gwSolicit := 10 * time.Millisecond ca := createClusterEx(t, true, gwSolicit, true, "A", 3) defer shutdownCluster(ca) cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) defer shutdownCluster(cb) // Hang a responder off of cluster A. opts := ca.opts[0] url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Create a queue subscriber to send results nc.QueueSubscribe("ngs.usage.*", "ngs", func(m *nats.Msg) { m.Respond([]byte("22")) }) nc.Flush() // Now create a leafnode server on B. opts = cb.opts[1] sl, slOpts := runSolicitLeafServer(opts) defer sl.Shutdown() checkLeafNodeConnected(t, sl) // Create a normal direct connect client on B. url = fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port) nc2, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() if _, err := nc2.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { t.Fatalf("Did not receive response: %v", err) } // Now create a client on the leafnode. url = fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port) ncl, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() if _, err := ncl.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { t.Fatalf("Did not receive response: %v", err) } } func TestLeafNodeServiceImportResponderOnLeaf(t *testing.T) { gwSolicit := 10 * time.Millisecond ca := createClusterEx(t, true, gwSolicit, true, "A", 3) defer shutdownCluster(ca) // Now create a leafnode server on A that will bind to the NGS account. opts := ca.opts[1] sl, slOpts := runSolicitLeafServerToURL(fmt.Sprintf("nats-leaf://ngs:pass@%s:%d", opts.LeafNode.Host, opts.LeafNode.Port)) defer sl.Shutdown() checkLeafNodeConnected(t, sl) // Now create a client on the leafnode. ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() // Create a queue subscriber to send results ncl.QueueSubscribe("ngs.usage.*", "ngs", func(m *nats.Msg) { m.Respond([]byte("22")) }) ncl.Flush() // Create a normal direct connect client on A. Needs to be same server as leafnode. opts = ca.opts[1] nc, err := nats.Connect(fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() if _, err := nc.Request("ngs.usage", []byte("fingers crossed"), 500*time.Millisecond); err != nil { t.Fatalf("Did not receive response: %v", err) } } func TestLeafNodeSendsAccountingEvents(t *testing.T) { s, opts, _ := runLeafNodeOperatorServer(t) defer s.Shutdown() // System account acc, akp := createAccount(t, s) if err := s.SetSystemAccount(acc.Name); err != nil { t.Fatalf("Expected this succeed, got %v", err) } // Leafnode Account lacc, lakp := createAccount(t, s) // Create a system account user and connect a client to listen for the events. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Watch only for our leaf node account. cSub, _ := nc.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.CONNECT", lacc.Name)) dSub, _ := nc.SubscribeSync(fmt.Sprintf("$SYS.ACCOUNT.%s.DISCONNECT", lacc.Name)) nc.Flush() // Now create creds for the leafnode and connect the server. kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(lakp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } seed, _ := kp.Seed() mycreds := genCredsFile(t, ujwt, seed) sl, _, _ := runSolicitWithCredentials(t, opts, mycreds) defer sl.Shutdown() // Wait for connect event msg, err := cSub.NextMsg(time.Second) if err != nil { t.Fatalf("Error waiting for account connect event: %v", err) } m := server.ConnectEventMsg{} if err := json.Unmarshal(msg.Data, &m); err != nil { t.Fatal("Did not get correctly formatted event") } // Shutdown leafnode to generate disconnect event. sl.Shutdown() msg, err = dSub.NextMsg(time.Second) if err != nil { t.Fatalf("Error waiting for account disconnect event: %v", err) } dm := server.DisconnectEventMsg{} if err := json.Unmarshal(msg.Data, &dm); err != nil { t.Fatal("Did not get correctly formatted event") } } func TestLeafNodeDistributedQueueAcrossGWs(t *testing.T) { gwSolicit := 10 * time.Millisecond ca := createClusterEx(t, true, gwSolicit, true, "A", 3) defer shutdownCluster(ca) cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) defer shutdownCluster(cb) cc := createClusterEx(t, true, gwSolicit, true, "C", 3, ca, cb) defer shutdownCluster(cc) // Create queue subscribers createQS := func(c *cluster) *nats.Conn { t.Helper() opts := c.opts[rand.Intn(len(c.opts))] url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.QueueSubscribe("ngs.usage.*", "dq", func(m *nats.Msg) { m.Respond([]byte(c.name)) }) nc.Flush() return nc } ncA := createQS(ca) defer ncA.Close() ncB := createQS(cb) defer ncB.Close() ncC := createQS(cc) defer ncC.Close() connectAndRequest := func(url, clusterName string, nreqs int) { t.Helper() nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() for i := 0; i < nreqs; i++ { m, err := nc.Request("ngs.usage", nil, 500*time.Millisecond) if err != nil { t.Fatalf("Did not receive a response: %v", err) } if string(m.Data) != clusterName { t.Fatalf("Expected to prefer %q, but got response from %q", clusterName, m.Data) } } } checkClientDQ := func(c *cluster, nreqs int) { t.Helper() // Pick one at random. opts := c.opts[rand.Intn(len(c.opts))] url := fmt.Sprintf("nats://dlc:pass@%s:%d", opts.Host, opts.Port) connectAndRequest(url, c.name, nreqs) } // First check that this works with direct connected clients. checkClientDQ(ca, 100) checkClientDQ(cb, 100) checkClientDQ(cc, 100) createLNS := func(c *cluster) (*server.Server, *server.Options) { t.Helper() // Pick one at random. s, opts := runSolicitLeafServer(c.opts[rand.Intn(len(c.servers))]) checkLeafNodeConnected(t, s) return s, opts } checkLeafDQ := func(opts *server.Options, clusterName string, nreqs int) { t.Helper() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) connectAndRequest(url, clusterName, nreqs) } // Test leafnodes to all clusters. for _, c := range []*cluster{ca, cb, cc} { // Now create a leafnode on cluster. sl, slOpts := createLNS(c) defer sl.Shutdown() // Now connect to the leafnode server and run test. checkLeafDQ(slOpts, c.name, 100) } } func TestLeafNodeDistributedQueueEvenly(t *testing.T) { gwSolicit := 10 * time.Millisecond ca := createClusterEx(t, true, gwSolicit, true, "A", 3) defer shutdownCluster(ca) cb := createClusterEx(t, true, gwSolicit, true, "B", 3, ca) defer shutdownCluster(cb) // Create queue subscribers createQS := func(c *cluster) *nats.Conn { t.Helper() opts := c.opts[rand.Intn(len(c.opts))] url := fmt.Sprintf("nats://ngs:pass@%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } cid, _ := nc.GetClientID() response := []byte(fmt.Sprintf("%s:%d:%d", c.name, opts.Port, cid)) nc.QueueSubscribe("ngs.usage.*", "dq", func(m *nats.Msg) { m.Respond(response) }) nc.Flush() return nc } ncA1 := createQS(ca) defer ncA1.Close() ncA2 := createQS(ca) defer ncA2.Close() ncA3 := createQS(ca) defer ncA3.Close() resp := make(map[string]int) connectAndRequest := func(url, clusterName string, nreqs int) { t.Helper() nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() for i := 0; i < nreqs; i++ { m, err := nc.Request("ngs.usage", nil, 500*time.Millisecond) if err != nil { t.Fatalf("Did not receive a response: %v", err) } if string(m.Data[0]) != clusterName { t.Fatalf("Expected to prefer %q, but got response from %q", clusterName, m.Data[0]) } resp[string(m.Data)]++ } } createLNS := func(c *cluster) (*server.Server, *server.Options) { t.Helper() // Pick one at random. copts := c.opts[rand.Intn(len(c.servers))] s, opts := runSolicitLeafServer(copts) checkLeafNodeConnected(t, s) return s, opts } checkLeafDQ := func(opts *server.Options, clusterName string, nreqs int) { t.Helper() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) connectAndRequest(url, clusterName, nreqs) } // Now create a leafnode on cluster A. sl, slOpts := createLNS(ca) defer sl.Shutdown() // Now connect to the leafnode server and run test. checkLeafDQ(slOpts, ca.name, 100) // Should have some for all 3 QS [ncA1, ncA2, ncA3] if lr := len(resp); lr != 3 { t.Fatalf("Expected all 3 queue subscribers to have received some messages, only got %d", lr) } // Now check that we have at least 10% for each subscriber. for _, r := range resp { if r < 10 { t.Fatalf("Got a subscriber with less than 10 responses: %d", r) } } } func TestLeafNodeDefaultPort(t *testing.T) { o := testDefaultOptionsForLeafNodes() o.LeafNode.Port = server.DEFAULT_LEAFNODE_PORT s := RunServer(o) defer s.Shutdown() conf := createConfFile(t, []byte(` port: -1 leaf { remotes = [ { url: "leafnode://127.0.0.1" } ] } `)) sl, _ := RunServerWithConfig(conf) defer sl.Shutdown() checkLeafNodeConnected(t, s) } // Since leafnode's are true interest only we need to make sure that we // register the proper interest with global routing $GR.xxxxxx._INBOX.> func TestLeafNodeAndGatewayGlobalRouting(t *testing.T) { server.SetGatewaysSolicitDelay(50 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) sl, slOpts := runSolicitLeafServer(ca.opts[1]) defer sl.Shutdown() checkLeafNodeConnected(t, ca.servers[1]) // Create a client on the leafnode. This will listen for requests. ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", slOpts.Host, slOpts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() ncl.Subscribe("foo", func(m *nats.Msg) { m.Respond([]byte("World")) }) ncl.Flush() // Since for leafnodes the account becomes interest-only mode, // let's make sure that the interest on "foo" has time to propagate // to cluster B. time.Sleep(250 * time.Millisecond) // Create a direct connect requestor. Try with all possible // servers in cluster B to make sure that we also receive the // reply when the accepting leafnode server does not have // its outbound GW connection to the requestor's server. for i := 0; i < 3; i++ { opts := cb.opts[i] url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // We don't use an INBOX here because we had a bug where // leafnode would subscribe to _GR_.*.*.*.> instead of // _GR_.>, and inbox masked that because of their number // of tokens. reply := nuid.Next() sub, err := nc.SubscribeSync(reply) if err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := nc.PublishRequest("foo", reply, []byte("Hello")); err != nil { t.Fatalf("Failed to send request: %v", err) } if _, err := sub.NextMsg(250 * time.Millisecond); err != nil { t.Fatalf("Did not get reply from server %d: %v", i, err) } nc.Close() } } func checkLeafNode2Connected(t *testing.T, s *server.Server) { t.Helper() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if nln := s.NumLeafNodes(); nln != 2 { return fmt.Errorf("Expected a connected leafnode for server %q, got none", s.ID()) } return nil }) } func TestLeafNodesStaggeredSubPub(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() sl1, sl1Opts := runSolicitLeafServer(opts) defer sl1.Shutdown() checkLeafNodeConnected(t, s) // Create a client on the leafnode and a subscription. ncl, err := nats.Connect(fmt.Sprintf("nats://%s:%d", sl1Opts.Host, sl1Opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() sub, err := ncl.SubscribeSync("foo") if err != nil { t.Fatalf("Failed to subscribe: %v", err) } sl2, sl2Opts := runSolicitLeafServer(opts) defer sl2.Shutdown() checkLeafNode2Connected(t, s) // Create a client on the second leafnode and publish a message. ncl2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", sl2Opts.Host, sl2Opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl2.Close() ncl2.Publish("foo", nil) checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != 1 { return fmt.Errorf("Did not receive the message: %v", err) } return nil }) } func TestLeafNodeMultipleRemoteURLs(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() content := ` port: -1 leafnodes { remotes = [ { urls: [nats-leaf://127.0.0.1:%d,nats-leaf://localhost:%d] } ] } ` config := fmt.Sprintf(content, opts.LeafNode.Port, opts.LeafNode.Port) conf := createConfFile(t, []byte(config)) sl, _ := RunServerWithConfig(conf) defer sl.Shutdown() checkLeafNodeConnected(t, s) } func runSolicitLeafCluster(t *testing.T, clusterName string, d1, d2 *cluster) *cluster { c := &cluster{servers: make([]*server.Server, 0, 2), opts: make([]*server.Options, 0, 2), name: clusterName} // Who we will solicit for server 1 ci := rand.Intn(len(d1.opts)) opts := d1.opts[ci] surl := fmt.Sprintf("nats-leaf://%s:%d", opts.LeafNode.Host, opts.LeafNode.Port) o := DefaultTestOptions o.Port = -1 rurl, _ := url.Parse(surl) o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} o.LeafNode.ReconnectInterval = 100 * time.Millisecond o.Cluster.Name = clusterName o.Cluster.Host = o.Host o.Cluster.Port = -1 s := RunServer(&o) checkLeafNodeConnected(t, d1.servers[ci]) c.servers = append(c.servers, s) c.opts = append(c.opts, &o) // Grab route info routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) curl, _ := url.Parse(routeAddr) // Who we will solicit for server 2 ci = rand.Intn(len(d2.opts)) opts = d2.opts[ci] surl = fmt.Sprintf("nats-leaf://%s:%d", opts.LeafNode.Host, opts.LeafNode.Port) // This is for the case were d1 == d2 and we select the same server. plfn := d2.servers[ci].NumLeafNodes() o2 := DefaultTestOptions o2.Port = -1 rurl, _ = url.Parse(surl) o2.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} o2.LeafNode.ReconnectInterval = 100 * time.Millisecond o2.Cluster.Name = clusterName o2.Cluster.Host = o.Host o2.Cluster.Port = -1 o2.Routes = []*url.URL{curl} s = RunServer(&o2) if plfn == 0 { checkLeafNodeConnected(t, d2.servers[ci]) } else { checkLeafNode2Connected(t, d2.servers[ci]) } c.servers = append(c.servers, s) c.opts = append(c.opts, &o2) checkClusterFormed(t, c.servers...) return c } func clientForCluster(t *testing.T, c *cluster) *nats.Conn { t.Helper() opts := c.opts[rand.Intn(len(c.opts))] url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url) if err != nil { t.Fatalf("Error on connect: %v", err) } return nc } func TestLeafNodeCycleWithSolicited(t *testing.T) { server.SetGatewaysSolicitDelay(10 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() // Accepting leafnode cluster, e.g. NGS ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) cb := createClusterWithName(t, "B", 3, ca) defer shutdownCluster(cb) // Create the responders. requestsReceived := int32(0) nc := clientForCluster(t, ca) defer nc.Close() nc.QueueSubscribe("request", "cycles", func(m *nats.Msg) { atomic.AddInt32(&requestsReceived, 1) m.Respond([]byte("22")) }) nc = clientForCluster(t, cb) defer nc.Close() nc.QueueSubscribe("request", "cycles", func(m *nats.Msg) { atomic.AddInt32(&requestsReceived, 1) m.Respond([]byte("33")) }) // Soliciting cluster, both solicited connected to the "A" cluster sc := runSolicitLeafCluster(t, "SC", ca, ca) defer shutdownCluster(sc) checkInterest := func(s *server.Server, subject string) bool { t.Helper() acc, _ := s.LookupAccount("$G") return acc.SubscriptionInterest(subject) } waitForInterest := func(subject string, servers ...*server.Server) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { for _, s := range servers { if !checkInterest(s, subject) { return fmt.Errorf("No interest") } } return nil }) } waitForInterest("request", sc.servers[0], sc.servers[1], ca.servers[0], ca.servers[1], ca.servers[2], cb.servers[0], cb.servers[1], cb.servers[2], ) // Connect a client to a random server in sc createClientAndRequest := func(c *cluster) (*nats.Conn, *nats.Subscription) { nc := clientForCluster(t, c) reply := nats.NewInbox() sub, err := nc.SubscribeSync(reply) if err != nil { t.Fatalf("Could not subscribe: %v", err) } if err := nc.PublishRequest("request", reply, []byte("fingers crossed")); err != nil { t.Fatalf("Error sending request: %v", err) } return nc, sub } verifyOneResponse := func(sub *nats.Subscription) { time.Sleep(250 * time.Millisecond) m, _, err := sub.Pending() if err != nil { t.Fatalf("Error calling Pending(): %v", err) } if m > 1 { t.Fatalf("Received more then one response, cycle indicated: %d", m) } } verifyRequestTotal := func(nre int32) { if nr := atomic.LoadInt32(&requestsReceived); nr != nre { t.Fatalf("Expected %d requests received, got %d", nre, nr) } } // This should pass to here, but if we have a cycle things will be spinning and we will receive // too many responses when it should only be 1. nc, rsub := createClientAndRequest(sc) defer nc.Close() verifyOneResponse(rsub) verifyRequestTotal(1) // Do a solicit across GW, so shut this one down. nc.Close() shutdownCluster(sc) // Soliciting cluster, connect to different clusters across a GW. sc = runSolicitLeafCluster(t, "SC", ca, cb) defer shutdownCluster(sc) nc, rsub = createClientAndRequest(sc) defer nc.Close() verifyOneResponse(rsub) verifyRequestTotal(2) // This is total since use same responders. } func TestLeafNodeNoRaceGeneratingNonce(t *testing.T) { opts := testDefaultOptionsForLeafNodes() opts.Cluster.Port = -1 opts.Cluster.Name = "xyz" s := RunServer(opts) defer s.Shutdown() quitCh := make(chan struct{}) wg := sync.WaitGroup{} wg.Add(2) go func() { defer wg.Done() for { lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) checkInfoMsg(t, lc) lc.Close() select { case <-quitCh: return case <-time.After(time.Millisecond): } } }() go func() { defer wg.Done() for { rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) checkInfoMsg(t, rc) rc.Close() select { case <-quitCh: return case <-time.After(time.Millisecond): } } }() // Let this run for a bit to see if we get data race time.Sleep(100 * time.Millisecond) close(quitCh) wg.Wait() } func runSolicitAndAcceptLeafServer(lso *server.Options) (*server.Server, *server.Options) { surl := fmt.Sprintf("nats-leaf://%s:%d", lso.LeafNode.Host, lso.LeafNode.Port) o := testDefaultOptionsForLeafNodes() o.Port = -1 rurl, _ := url.Parse(surl) o.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}} o.LeafNode.ReconnectInterval = 100 * time.Millisecond return RunServer(o), o } func TestLeafNodeDaisyChain(t *testing.T) { // To quickly enable trace and debug logging // doLog, doTrace, doDebug = true, true, true s1, opts1 := runLeafServer() defer s1.Shutdown() s2, opts2 := runSolicitAndAcceptLeafServer(opts1) defer s2.Shutdown() checkLeafNodeConnected(t, s1) s3, _ := runSolicitLeafServer(opts2) defer s3.Shutdown() checkLeafNodeConnections(t, s2, 2) // Make so we can tell the two apart since in same PID. if doLog { s1.SetLogger(logger.NewTestLogger("[S-1] - ", false), true, true) s2.SetLogger(logger.NewTestLogger("[S-2] - ", false), true, true) s3.SetLogger(logger.NewTestLogger("[S-3] - ", false), true, true) } nc1, err := nats.Connect(s1.ClientURL()) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc1.Close() nc1.Subscribe("ngs.usage", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) nc1.Flush() checkSubInterest(t, s3, "$G", "ngs.usage", time.Second) nc2, err := nats.Connect(s3.ClientURL()) if err != nil { t.Fatalf("Error creating client: %v", err) } defer nc2.Close() if _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } } // This will test failover to a server with a cert with only an IP after successfully connecting // to a server with a cert with both. func TestClusterTLSMixedIPAndDNS(t *testing.T) { confA := createConfFile(t, []byte(` listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-iponly.pem" key_file: "./configs/certs/server-key-iponly.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 } } cluster { listen: "127.0.0.1:-1" name: xyz } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() bConfigTemplate := ` listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" ca_file: "./configs/certs/ca.pem" timeout: 2 } } cluster { listen: "127.0.0.1:-1" name: xyz routes [ "nats://%s:%d" ] } ` confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.Cluster.Host, optsA.Cluster.Port))) srvB, optsB := RunServerWithConfig(confB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) // Solicit a leafnode server here. Don't use the helper since we need verification etc. o := DefaultTestOptions o.Port = -1 rurl, _ := url.Parse(fmt.Sprintf("nats-leaf://%s:%d", optsB.LeafNode.Host, optsB.LeafNode.Port)) o.LeafNode.ReconnectInterval = 10 * time.Millisecond remote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}} remote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} pool := x509.NewCertPool() rootPEM, err := os.ReadFile("./configs/certs/ca.pem") if err != nil || rootPEM == nil { t.Fatalf("Error loading or parsing rootCA file: %v", err) } ok := pool.AppendCertsFromPEM(rootPEM) if !ok { t.Fatalf("Failed to parse root certificate from %q", "./configs/certs/ca.pem") } remote.TLSConfig.RootCAs = pool o.LeafNode.Remotes = []*server.RemoteLeafOpts{remote} sl, _ := RunServer(&o), &o defer sl.Shutdown() checkLeafNodeConnected(t, srvB) // Now kill off srvB and force client to connect to srvA. srvB.Shutdown() // Make sure this works. checkLeafNodeConnected(t, srvA) } // This will test for a bug in stream export/import with leafnodes. // https://github.com/nats-io/nats-server/issues/1332 func TestStreamExportWithMultipleAccounts(t *testing.T) { confA := createConfFile(t, []byte(` listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() bConfigTemplate := ` listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" remotes = [ { url:"nats://127.0.0.1:%d" account:"EXTERNAL" } ] } accounts: { INTERNAL: { users: [ {user: good, password: pwd} ] imports: [ { stream: { account: EXTERNAL, subject: "foo"}, prefix: "bar" } ] }, EXTERNAL: { users: [ {user: bad, password: pwd} ] exports: [{stream: "foo"}] }, } ` confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port))) srvB, optsB := RunServerWithConfig(confB) defer srvB.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsB.Host, optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() wcsub, err := nc.SubscribeSync(">") if err != nil { t.Fatalf("Unexpected error: %v", err) } defer wcsub.Unsubscribe() nc.Flush() nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc2.Publish("foo", nil) msg, err := wcsub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Did not receive the message: %v", err) } if msg.Subject != "bar.foo" { t.Fatalf("Received on wrong subject: %q", msg.Subject) } } // This will test for a bug in service export/import with leafnodes. // https://github.com/nats-io/nats-server/issues/1336 func TestServiceExportWithMultipleAccounts(t *testing.T) { confA := createConfFile(t, []byte(` server_name: A listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" } `)) srvA, optsA := RunServerWithConfig(confA) defer srvA.Shutdown() bConfigTemplate := ` server_name: B listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" remotes = [ { url:"nats://127.0.0.1:%d" account:"EXTERNAL" } ] } accounts: { INTERNAL: { users: [ {user: good, password: pwd} ] imports: [ { service: { account: EXTERNAL subject: "foo" } } ] }, EXTERNAL: { users: [ {user: evil, password: pwd} ] exports: [{service: "foo"}] }, } ` confB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port))) srvB, optsB := RunServerWithConfig(confB) defer srvB.Shutdown() checkLeafNodeConnected(t, srvB) // connect to confA, and offer a service nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%d", optsA.Host, optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc2.Subscribe("foo", func(msg *nats.Msg) { if err := msg.Respond([]byte("world")); err != nil { t.Fatalf("Error on respond: %v", err) } }) nc2.Flush() checkSubInterest(t, srvB, "INTERNAL", "foo", time.Second) nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsB.Host, optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() resp, err := nc.Request("foo", []byte("hello"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { t.Fatal("Did not receive the correct message") } } // This will test for a bug in service export/import with leafnode restart. // https://github.com/nats-io/nats-server/issues/1344 func TestServiceExportWithLeafnodeRestart(t *testing.T) { confG := createConfFile(t, []byte(` server_name: G listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" authorization { account:"EXTERNAL" } } accounts: { INTERNAL: { users: [ {user: good, password: pwd} ] exports: [{service: "foo", response: singleton}] imports: [ { service: { account: EXTERNAL subject: "evilfoo" }, to: from_evilfoo } ] }, EXTERNAL: { users: [ {user: evil, password: pwd} ] exports: [{service: "evilfoo", response: singleton}] imports: [ { service: { account: INTERNAL subject: "foo" }, to: goodfoo } ] } } `)) srvG, optsG := RunServerWithConfig(confG) defer srvG.Shutdown() eConfigTemplate := ` server_name: E listen: 127.0.0.1:-1 leafnodes { listen: "127.0.0.1:-1" remotes = [ { url:"nats://127.0.0.1:%d" account:"EXTERNAL_GOOD" } ] } accounts: { INTERNAL_EVILINC: { users: [ {user: evil, password: pwd} ] exports: [{service: "foo", response: singleton}] imports: [ { service: { account: EXTERNAL_GOOD subject: "goodfoo" }, to: from_goodfoo } ] }, EXTERNAL_GOOD: { users: [ {user: good, password: pwd} ] exports: [{service: "goodfoo", response: singleton}] imports: [ { service: { account: INTERNAL_EVILINC subject: "foo" }, to: evilfoo } ] }, } ` confE := createConfFile(t, []byte(fmt.Sprintf(eConfigTemplate, optsG.LeafNode.Port))) srvE, optsE := RunServerWithConfig(confE) defer srvE.Shutdown() // connect to confE, and offer a service nc2, err := nats.Connect(fmt.Sprintf("nats://evil:pwd@%s:%d", optsE.Host, optsE.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc2.Subscribe("foo", func(msg *nats.Msg) { if err := msg.Respond([]byte("world")); err != nil { t.Fatalf("Error on respond: %v", err) } }) nc2.Flush() nc, err := nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsG.Host, optsG.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() resp, err := nc.Request("from_evilfoo", []byte("hello"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { t.Fatal("Did not receive the correct message") } // Now restart server E and requestor and replier. nc.Close() nc2.Close() srvE.Shutdown() srvE, optsE = RunServerWithConfig(confE) defer srvE.Shutdown() nc2, err = nats.Connect(fmt.Sprintf("nats://evil:pwd@%s:%d", optsE.Host, optsE.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc2.Subscribe("foo", func(msg *nats.Msg) { if err := msg.Respond([]byte("world")); err != nil { t.Fatalf("Error on respond: %v", err) } }) nc2.Flush() nc, err = nats.Connect(fmt.Sprintf("nats://good:pwd@%s:%d", optsG.Host, optsG.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() resp, err = nc.Request("from_evilfoo", []byte("hello"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { t.Fatal("Did not receive the correct message") } } func TestLeafNodeQueueSubscriberUnsubscribe(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) // Create a client on leaf server and create a queue sub c1 := createClientConn(t, opts.Host, opts.Port) defer c1.Close() send1, expect1 := setupConn(t, c1) send1("SUB foo bar 1\r\nPING\r\n") expect1(pongRe) // Leaf should receive an LS+ foo bar 1 leafExpect(lsubRe) // Create a second client on leaf server and create queue sub on same group. c2 := createClientConn(t, opts.Host, opts.Port) defer c2.Close() send2, expect2 := setupConn(t, c2) send2("SUB foo bar 1\r\nPING\r\n") expect2(pongRe) // Leaf should receive an LS+ foo bar 2 leafExpect(lsubRe) // Now close c1 c1.Close() // Leaf should receive an indication that the queue group went to 1. // Which means LS+ foo bar 1. buf := leafExpect(lsubRe) if matches := lsubRe.FindAllSubmatch(buf, -1); len(matches) != 1 { t.Fatalf("Expected only 1 LS+, got %v", len(matches)) } // Make sure that we did not get a LS- at the same time. if bytes.Contains(buf, []byte("LS-")) { t.Fatalf("Unexpected LS- in response: %q", buf) } // Make sure we receive nothing... expectNothing(t, lc) } func TestLeafNodeOriginClusterSingleHub(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() c1 := ` listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1 } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, opts.LeafNode.Port))) ln1, lopts1 := RunServerWithConfig(lconf1) defer ln1.Shutdown() c2 := ` listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, opts.LeafNode.Port))) ln2, _ := RunServerWithConfig(lconf2) defer ln2.Shutdown() ln3, _ := RunServerWithConfig(lconf2) defer ln3.Shutdown() checkClusterFormed(t, ln1, ln2, ln3) checkLeafNodeConnections(t, s, 3) // So now we are setup with 3 solicited leafnodes all connected to a hub. // We will create two clients, one on each leafnode server. nc1, err := nats.Connect(ln1.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() nc2, err := nats.Connect(ln2.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() checkInterest := func(s *server.Server, subject string) bool { t.Helper() acc, _ := s.LookupAccount("$G") return acc.SubscriptionInterest(subject) } waitForInterest := func(subject string, servers ...*server.Server) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { for _, s := range servers { if !checkInterest(s, subject) { return fmt.Errorf("No interest") } } return nil }) } subj := "foo.bar" sub, _ := nc2.SubscribeSync(subj) waitForInterest(subj, ln1, ln2, ln3, s) // Make sure we truncated the subscription bouncing through the hub and back to other leafnodes. for _, s := range []*server.Server{ln1, ln3} { acc, _ := s.LookupAccount("$G") if nms := acc.Interest(subj); nms != 1 { t.Fatalf("Expected only one active subscription, got %d", nms) } } // Send a message. nc1.Publish(subj, nil) nc1.Flush() // Wait to propagate time.Sleep(25 * time.Millisecond) // Make sure we only get it once. if n, _, _ := sub.Pending(); n != 1 { t.Fatalf("Expected only one message, got %d", n) } } func TestLeafNodeOriginCluster(t *testing.T) { ca := createClusterWithName(t, "A", 3) defer shutdownCluster(ca) c1 := ` server_name: L1 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1 } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, ca.opts[0].LeafNode.Port))) ln1, lopts1 := RunServerWithConfig(lconf1) defer ln1.Shutdown() c2 := ` server_name: L2 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, ca.opts[1].LeafNode.Port))) ln2, _ := RunServerWithConfig(lconf2) defer ln2.Shutdown() c3 := ` server_name: L3 listen: 127.0.0.1:-1 cluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } ` lconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, ca.opts[2].LeafNode.Port))) ln3, _ := RunServerWithConfig(lconf3) defer ln3.Shutdown() checkClusterFormed(t, ln1, ln2, ln3) checkLeafNodeConnections(t, ca.servers[0], 1) checkLeafNodeConnections(t, ca.servers[1], 1) checkLeafNodeConnections(t, ca.servers[2], 1) // So now we are setup with 3 solicited leafnodes connected to different servers in the hub cluster. // We will create two clients, one on each leafnode server. nc1, err := nats.Connect(ln1.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() nc2, err := nats.Connect(ln2.ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() checkInterest := func(s *server.Server, subject string) bool { t.Helper() acc, _ := s.LookupAccount("$G") return acc.SubscriptionInterest(subject) } waitForInterest := func(subject string, servers ...*server.Server) { t.Helper() checkFor(t, time.Second, 10*time.Millisecond, func() error { for _, s := range servers { if !checkInterest(s, subject) { return fmt.Errorf("No interest") } } return nil }) } subj := "foo.bar" sub, _ := nc2.SubscribeSync(subj) waitForInterest(subj, ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2]) // Make sure we truncated the subscription bouncing through the hub and back to other leafnodes. for _, s := range []*server.Server{ln1, ln3} { acc, _ := s.LookupAccount("$G") if nms := acc.Interest(subj); nms != 1 { t.Fatalf("Expected only one active subscription, got %d", nms) } } // Send a message. nc1.Publish(subj, nil) nc1.Flush() // Wait to propagate time.Sleep(25 * time.Millisecond) // Make sure we only get it once. if n, _, _ := sub.Pending(); n != 1 { t.Fatalf("Expected only one message, got %d", n) } // eat the msg sub.NextMsg(time.Second) // Now create interest on the hub side. This will draw the message from a leafnode // to the hub. We want to make sure that message does not bounce back to other leafnodes. nc3, err := nats.Connect(ca.servers[0].ClientURL()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc3.Close() wcSubj := "foo.*" wcsub, _ := nc3.SubscribeSync(wcSubj) // This is a placeholder that we can use to check all interest has propagated. nc3.SubscribeSync("bar") waitForInterest("bar", ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2]) // Send another message. m := nats.NewMsg(subj) m.Header.Add("Accept-Encoding", "json") m.Header.Add("Authorization", "s3cr3t") m.Data = []byte("Hello Headers!") nc1.PublishMsg(m) nc1.Flush() // Wait to propagate time.Sleep(25 * time.Millisecond) // Make sure we only get it once. if n, _, _ := sub.Pending(); n != 1 { t.Fatalf("Expected only one message, got %d", n) } // Also for wc if n, _, _ := wcsub.Pending(); n != 1 { t.Fatalf("Expected only one message, got %d", n) } // grab the msg msg, _ := sub.NextMsg(time.Second) if !bytes.Equal(m.Data, msg.Data) { t.Fatalf("Expected the payloads to match, wanted %q, got %q", m.Data, msg.Data) } if len(msg.Header) != 2 { t.Fatalf("Expected 2 header entries, got %d", len(msg.Header)) } if msg.Header.Get("Authorization") != "s3cr3t" { t.Fatalf("Expected auth header to match, wanted %q, got %q", "s3cr3t", msg.Header.Get("Authorization")) } } func TestLeafNodeAdvertiseInCluster(t *testing.T) { o1 := testDefaultOptionsForLeafNodes() o1.Cluster.Name = "abc" o1.Cluster.Host = "127.0.0.1" o1.Cluster.Port = -1 s1 := runGatewayServer(o1) defer s1.Shutdown() lc := createLeafConn(t, o1.LeafNode.Host, o1.LeafNode.Port) defer lc.Close() leafSend, leafExpect := setupLeaf(t, lc, 1) leafSend("PING\r\n") leafExpect(pongRe) o2 := testDefaultOptionsForLeafNodes() o2.Cluster.Name = "abc" o2.Cluster.Host = "127.0.0.1" o2.Cluster.Port = -1 o2.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o2.LeafNode.Advertise = "srvB:7222" s2 := runGatewayServer(o2) defer s2.Shutdown() checkClusterFormed(t, s1, s2) buf := leafExpect(infoRe) si := &server.Info{} json.Unmarshal(buf[5:], si) var ok bool for _, u := range si.LeafNodeURLs { if u == "srvB:7222" { ok = true break } } if !ok { t.Fatalf("Url srvB:7222 was not found: %q", si.GatewayURLs) } o3 := testDefaultOptionsForLeafNodes() o3.Cluster.Name = "abc" o3.Cluster.Host = "127.0.0.1" o3.Cluster.Port = -1 o3.Routes = server.RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", o1.Cluster.Port)) o3.LeafNode.Advertise = "srvB:7222" s3 := runGatewayServer(o3) defer s3.Shutdown() checkClusterFormed(t, s1, s2, s3) // Since it is the save srvB:7222 url, we should not get an update. expectNothing(t, lc) // Now shutdown s2 and make sure that we are not getting an update // with srvB:7222 missing. s2.Shutdown() expectNothing(t, lc) } func TestLeafNodeAndGatewaysStreamAndShadowSubs(t *testing.T) { server.SetGatewaysSolicitDelay(10 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() conf1 := createConfFile(t, []byte(` port: -1 system_account: SYS accounts { SYS {} A: { users: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }] exports: [{ stream: A.b.>, accounts: [B] }] }, B: { users: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}] imports: [{ stream: { account: A, subject: A.b.> } }] } } gateway { name: "A" port: -1 } leafnodes { port: -1 authorization: { users: [ {user: a, password: pwd, account: A} ] } } `)) s1, o1 := RunServerWithConfig(conf1) defer s1.Shutdown() conf2 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 system_account: SYS accounts { SYS {} A: { users: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }] exports: [{ stream: A.b.>, accounts: [B] }] }, B: { users: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}] imports: [{ stream: { account: A, subject: A.b.> } }] } } gateway { name: "B" port: -1 gateways [ { name: "A" urls: ["nats://127.0.0.1:%d"] } ] } `, o1.Gateway.Port))) s2, o2 := RunServerWithConfig(conf2) defer s2.Shutdown() waitForOutboundGateways(t, s1, 1, 2*time.Second) waitForOutboundGateways(t, s2, 1, 2*time.Second) nc, err := nats.Connect(fmt.Sprintf("nats://b:pwd@127.0.0.1:%d", o2.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() sub, err := nc.SubscribeSync("A.b.>") if err != nil { t.Fatalf("Error on subscibe: %v", err) } defer sub.Unsubscribe() conf3 := createConfFile(t, []byte(fmt.Sprintf(` port: -1 system_account: SYS accounts: { SYS {} C: { imports: [{ stream: { account: D, subject: b.> }, prefix: A }] } D: { users: [{ user: d, password: pwd, permissions: {publish: [ b.> ]} }] exports: [{ stream: b.>, accounts: [C] }] } } leafnodes { remotes [ { url: "nats://a:pwd@127.0.0.1:%d" account: C } ] } `, o1.LeafNode.Port))) s3, o3 := RunServerWithConfig(conf3) defer s3.Shutdown() checkLeafNodeConnected(t, s1) checkLeafNodeConnected(t, s3) ncl, err := nats.Connect(fmt.Sprintf("nats://d:pwd@127.0.0.1:%d", o3.Port)) if err != nil { t.Fatalf("Error connecting: %v", err) } defer ncl.Close() ncl.Publish("b.c", []byte("test")) if _, err := sub.NextMsg(time.Second); err != nil { t.Fatalf("Did not receive message: %v", err) } } func TestLeafnodeHeaders(t *testing.T) { srv, opts := runLeafServer() defer srv.Shutdown() leaf, _ := runSolicitLeafServer(opts) defer leaf.Shutdown() checkLeafNodeConnected(t, srv) checkLeafNodeConnected(t, leaf) snc, err := nats.Connect(srv.ClientURL()) if err != nil { t.Fatal(err.Error()) } defer snc.Close() lnc, err := nats.Connect(leaf.ClientURL()) if err != nil { t.Fatal(err.Error()) } defer lnc.Close() // Start with subscription on leaf so that we check that srv has the interest // (since we are going to publish from srv) lsub, err := lnc.SubscribeSync("test") if err != nil { t.Fatalf("subscribe failed: %s", err) } lnc.Flush() checkSubInterest(t, srv, "$G", "test", time.Second) ssub, err := snc.SubscribeSync("test") if err != nil { t.Fatalf("subscribe failed: %s", err) } msg := nats.NewMsg("test") msg.Header.Add("Test", "Header") if len(msg.Header) == 0 { t.Fatalf("msg header is empty") } err = snc.PublishMsg(msg) if err != nil { t.Fatal(err.Error()) } smsg, err := ssub.NextMsg(time.Second) if err != nil { t.Fatalf("next failed: %s", err) } if len(smsg.Header) == 0 { t.Fatalf("server msgs header is empty") } lmsg, err := lsub.NextMsg(time.Second) if err != nil { t.Fatalf("next failed: %s", err) } if len(lmsg.Header) == 0 { t.Fatalf("leaf msg header is empty") } } nats-server-2.10.27/test/log_test.go000066400000000000000000000055141477524627100173060ustar00rootroot00000000000000package test import ( "fmt" "os" "path/filepath" "runtime" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) func RunServerWithLogging(opts *server.Options) *server.Server { if opts == nil { opts = &DefaultTestOptions } opts.NoLog = false opts.Cluster.PoolSize = -1 opts.Cluster.Compression.Mode = server.CompressionOff opts.LeafNode.Compression.Mode = server.CompressionOff s, err := server.NewServer(opts) if err != nil || s == nil { panic(fmt.Sprintf("No NATS Server object returned: %v", err)) } s.ConfigureLogger() go s.Start() if !s.ReadyForConnections(10 * time.Second) { panic("Unable to start NATS Server in Go Routine") } return s } func TestLogMaxArchives(t *testing.T) { // With logfile_size_limit set to small 100 characters, plain startup rotates 8 times for _, test := range []struct { name string config string totEntriesExpected int }{ { "Default implicit, no max logs, expect 0 purged logs", ` port: -1 log_file: %s logfile_size_limit: 100 `, 9, }, { "Default explicit, no max logs, expect 0 purged logs", ` port: -1 log_file: %s logfile_size_limit: 100 logfile_max_num: 0 `, 9, }, { "Default explicit - negative val, no max logs, expect 0 purged logs", ` port: -1 log_file: %s logfile_size_limit: 100 logfile_max_num: -42 `, 9, }, { "1-max num, expect 8 purged logs", ` port: -1 log_file: %s logfile_size_limit: 100 logfile_max_num: 1 `, 1, }, { "5-max num, expect 4 purged logs; use opt alias", ` port: -1 log_file: %s log_size_limit: 100 log_max_num: 5 `, 5, }, { "100-max num, expect 0 purged logs", ` port: -1 log_file: %s logfile_size_limit: 100 logfile_max_num: 100 `, 9, }, } { t.Run(test.name, func(t *testing.T) { d, err := os.MkdirTemp("", "logtest") if err != nil { t.Fatalf("Error creating temp dir: %v", err) } content := fmt.Sprintf(test.config, filepath.Join(d, "nats-server.log")) // server config does not like plain windows backslash if runtime.GOOS == "windows" { content = filepath.ToSlash(content) } opts, err := server.ProcessConfigFile(createConfFile(t, []byte(content))) if err != nil { t.Fatalf("Error processing config file: %v", err) } s := RunServerWithLogging(opts) if s == nil { t.Fatalf("No NATS Server object returned") } s.Shutdown() // Windows filesystem can be a little pokey on the flush, so wait a bit after shutdown... time.Sleep(500 * time.Millisecond) entries, err := os.ReadDir(d) if err != nil { t.Fatalf("Error reading dir: %v", err) } if len(entries) != test.totEntriesExpected { t.Fatalf("Expected %d log files, got %d", test.totEntriesExpected, len(entries)) } }) } } nats-server-2.10.27/test/maxpayload_test.go000066400000000000000000000104641477524627100206640ustar00rootroot00000000000000// Copyright 2015-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "net" "runtime" "strings" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) func TestMaxPayload(t *testing.T) { srv, opts := RunServerWithConfig("./configs/override.conf") defer srv.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(fmt.Sprintf("nats://%s/", endpoint)) if err != nil { t.Fatalf("Could not connect to server: %v", err) } defer nc.Close() size := 4 * 1024 * 1024 big := sizedBytes(size) err = nc.Publish("foo", big) if err != nats.ErrMaxPayload { t.Fatalf("Expected a Max Payload error") } conn, err := net.DialTimeout("tcp", endpoint, nc.Opts.Timeout) if err != nil { t.Fatalf("Could not make a raw connection to the server: %v", err) } defer conn.Close() info := make([]byte, 512) _, err = conn.Read(info) if err != nil { t.Fatalf("Expected an info message to be sent by the server: %s", err) } pub := fmt.Sprintf("PUB bar %d\r\n", size) _, err = conn.Write([]byte(pub)) if err != nil { t.Fatalf("Could not publish event to the server: %s", err) } errMsg := make([]byte, 35) _, err = conn.Read(errMsg) if err != nil { t.Fatalf("Expected an error message to be sent by the server: %s", err) } if !strings.Contains(string(errMsg), "Maximum Payload Violation") { t.Errorf("Received wrong error message (%v)\n", string(errMsg)) } // Client proactively omits sending the message so server // does not close the connection. if nc.IsClosed() { t.Errorf("Expected connection to not be closed.") } // On the other hand client which did not proactively omitted // publishing the bytes following what is suggested by server // in the info message has its connection closed. _, err = conn.Write(big) if err == nil && runtime.GOOS != "windows" { t.Errorf("Expected error due to maximum payload transgression.") } // On windows, the previous write will not fail because the connection // is not fully closed at this stage. if runtime.GOOS == "windows" { // Issuing a PING and not expecting the PONG. _, err = conn.Write([]byte("PING\r\n")) if err == nil { conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) _, err = conn.Read(big) if err == nil { t.Errorf("Expected closed connection due to maximum payload transgression.") } } } } func TestMaxPayloadOverrun(t *testing.T) { opts := DefaultTestOptions opts.Port = -1 opts.MaxPayload = 10000 s := RunServer(&opts) defer s.Shutdown() // Overrun a int32 c := createClientConn(t, "127.0.0.1", opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PUB foo 199380988\r\n") expect(errRe) // Now overrun an int64, parseSize will have returned -1, // so we get disconnected. c = createClientConn(t, "127.0.0.1", opts.Port) defer c.Close() send, _ = setupConn(t, c) send("PUB foo 18446744073709551615123\r\n") expectDisconnect(t, c) } func TestAsyncInfoWithSmallerMaxPayload(t *testing.T) { s, opts := runOperatorServer(t) defer s.Shutdown() const testMaxPayload = 522 okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) nac.Limits.Payload = testMaxPayload ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } if err := s.AccountResolver().Store(apub, ajwt); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Flush() defer nc.Close() if mp := nc.MaxPayload(); mp != testMaxPayload { t.Fatalf("Expected MaxPayload of %d, got %d", testMaxPayload, mp) } } nats-server-2.10.27/test/monitor_test.go000066400000000000000000000566571477524627100202320ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "crypto/tls" "crypto/x509" "encoding/json" "fmt" "io" "net" "net/http" "os" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) const CLIENT_PORT = 11422 const MONITOR_PORT = 11522 func runMonitorServer() *server.Server { resetPreviousHTTPConnections() opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPPort = MONITOR_PORT opts.HTTPHost = "127.0.0.1" opts.NoSystemAccount = true return RunServer(&opts) } // Runs a clustered pair of monitor servers for testing the /routez endpoint func runMonitorServerClusteredPair(t *testing.T) (*server.Server, *server.Server) { resetPreviousHTTPConnections() opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPPort = MONITOR_PORT opts.HTTPHost = "127.0.0.1" opts.Cluster = server.ClusterOpts{Name: "M", Host: "127.0.0.1", Port: 10223} opts.Routes = server.RoutesFromStr("nats-route://127.0.0.1:10222") opts.NoSystemAccount = true s1 := RunServer(&opts) opts2 := DefaultTestOptions opts2.Port = CLIENT_PORT + 1 opts2.HTTPPort = MONITOR_PORT + 1 opts2.HTTPHost = "127.0.0.1" opts2.Cluster = server.ClusterOpts{Name: "M", Host: "127.0.0.1", Port: 10222} opts2.Routes = server.RoutesFromStr("nats-route://127.0.0.1:10223") opts2.NoSystemAccount = true s2 := RunServer(&opts2) checkClusterFormed(t, s1, s2) return s1, s2 } func runMonitorServerNoHTTPPort() *server.Server { resetPreviousHTTPConnections() opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPPort = 0 opts.NoSystemAccount = true return RunServer(&opts) } func resetPreviousHTTPConnections() { http.DefaultTransport.(*http.Transport).CloseIdleConnections() } // Make sure that we do not run the http server for monitoring unless asked. func TestNoMonitorPort(t *testing.T) { s := runMonitorServerNoHTTPPort() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) if resp, err := http.Get(url + "varz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } if resp, err := http.Get(url + "healthz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } if resp, err := http.Get(url + "connz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } } // testEndpointDataRace tests a monitoring endpoint for data races by polling // while client code acts to ensure statistics are updated. It is designed to // run under the -race flag to catch violations. The caller must start the // NATS server. func testEndpointDataRace(endpoint string, t *testing.T) { var doneWg sync.WaitGroup url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) // Poll as fast as we can, while creating connections, publishing, // and subscribing. clientDone := int64(0) doneWg.Add(1) go func() { for atomic.LoadInt64(&clientDone) == 0 { resp, err := http.Get(url + endpoint) if err != nil { t.Errorf("Expected no error: Got %v\n", err) } else { resp.Body.Close() } } doneWg.Done() }() // create connections, subscriptions, and publish messages to // update the monitor variables. var conns []net.Conn for i := 0; i < 50; i++ { cl := createClientConnSubscribeAndPublish(t) // keep a few connections around to test monitor variables. if i%10 == 0 { conns = append(conns, cl) } else { cl.Close() } } atomic.AddInt64(&clientDone, 1) // wait for the endpoint polling goroutine to exit doneWg.Wait() // cleanup the conns for _, cl := range conns { cl.Close() } } func TestEndpointDataRaces(t *testing.T) { // setup a small cluster to test /routez s1, s2 := runMonitorServerClusteredPair(t) defer s1.Shutdown() defer s2.Shutdown() // test all of our endpoints testEndpointDataRace("varz", t) testEndpointDataRace("connz", t) testEndpointDataRace("routez", t) testEndpointDataRace("subsz", t) testEndpointDataRace("stacksz", t) } func TestVarz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) resp, err := http.Get(url + "varz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } v := server.Varz{} if err := json.Unmarshal(body, &v); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Do some sanity checks on values if time.Since(v.Start) > 10*time.Second { t.Fatal("Expected start time to be within 10 seconds.") } cl := createClientConnSubscribeAndPublish(t) defer cl.Close() resp, err = http.Get(url + "varz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err = io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } if strings.Contains(string(body), "cluster_port") { t.Fatal("Varz body contains cluster information when no cluster is defined.") } v = server.Varz{} if err := json.Unmarshal(body, &v); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } if v.Connections != 1 { t.Fatalf("Expected Connections of 1, got %v\n", v.Connections) } if v.InMsgs != 1 { t.Fatalf("Expected InMsgs of 1, got %v\n", v.InMsgs) } if v.OutMsgs != 1 { t.Fatalf("Expected OutMsgs of 1, got %v\n", v.OutMsgs) } if v.InBytes != 5 { t.Fatalf("Expected InBytes of 5, got %v\n", v.InBytes) } if v.OutBytes != 5 { t.Fatalf("Expected OutBytes of 5, got %v\n", v.OutBytes) } if v.MaxPending != server.MAX_PENDING_SIZE { t.Fatalf("Expected MaxPending of %d, got %v\n", server.MAX_PENDING_SIZE, v.MaxPending) } if v.WriteDeadline != server.DEFAULT_FLUSH_DEADLINE { t.Fatalf("Expected WriteDeadline of %d, got %v\n", server.DEFAULT_FLUSH_DEADLINE, v.WriteDeadline) } } func TestConnz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) resp, err := http.Get(url + "connz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Test contents.. if c.NumConns != 0 { t.Fatalf("Expected 0 connections, got %d\n", c.NumConns) } if c.Total != 0 { t.Fatalf("Expected 0 live connections, got %d\n", c.Total) } if c.Conns == nil || len(c.Conns) != 0 { t.Fatalf("Expected 0 connections in array, got %p\n", c.Conns) } cl := createClientConnSubscribeAndPublish(t) defer cl.Close() resp, err = http.Get(url + "connz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err = io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } if c.NumConns != 1 { t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) } if c.Total != 1 { t.Fatalf("Expected 1 live connection, got %d\n", c.Total) } if c.Conns == nil || len(c.Conns) != 1 { t.Fatalf("Expected 1 connection in array, got %p\n", c.Conns) } if c.Limit != server.DefaultConnListSize { t.Fatalf("Expected limit of %d, got %v\n", server.DefaultConnListSize, c.Limit) } if c.Offset != 0 { t.Fatalf("Expected offset of 0, got %v\n", c.Offset) } // Test inside details of each connection ci := c.Conns[0] if ci.Cid == 0 { t.Fatalf("Expected non-zero cid, got %v\n", ci.Cid) } if ci.IP != "127.0.0.1" { t.Fatalf("Expected \"127.0.0.1\" for IP, got %v\n", ci.IP) } if ci.Port == 0 { t.Fatalf("Expected non-zero port, got %v\n", ci.Port) } if ci.NumSubs != 1 { t.Fatalf("Expected num_subs of 1, got %v\n", ci.NumSubs) } if len(ci.Subs) != 0 { t.Fatalf("Expected subs of 0, got %v\n", ci.Subs) } if ci.InMsgs != 1 { t.Fatalf("Expected InMsgs of 1, got %v\n", ci.InMsgs) } if ci.OutMsgs != 1 { t.Fatalf("Expected OutMsgs of 1, got %v\n", ci.OutMsgs) } if ci.InBytes != 5 { t.Fatalf("Expected InBytes of 1, got %v\n", ci.InBytes) } if ci.OutBytes != 5 { t.Fatalf("Expected OutBytes of 1, got %v\n", ci.OutBytes) } } func TestTLSConnz(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls.conf") defer srv.Shutdown() rootCAFile := "./configs/certs/ca.pem" clientCertFile := "./configs/certs/client-cert.pem" clientKeyFile := "./configs/certs/client-key.pem" // Test with secure connection endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) nc, err := nats.Connect(nurl, nats.RootCAs(rootCAFile)) if err != nil { t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) } defer nc.Close() ch := make(chan struct{}) nc.Subscribe("foo", func(m *nats.Msg) { ch <- struct{}{} }) nc.Publish("foo", []byte("Hello")) // Wait for message <-ch url := fmt.Sprintf("https://127.0.0.1:%d/", opts.HTTPSPort) tlsConfig := &tls.Config{} caCert, err := os.ReadFile(rootCAFile) if err != nil { t.Fatalf("Got error reading RootCA file: %s", err) } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig.RootCAs = caCertPool cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) if err != nil { t.Fatalf("Got error reading client certificates: %s", err) } tlsConfig.Certificates = []tls.Certificate{cert} transport := &http.Transport{TLSClientConfig: tlsConfig} httpClient := &http.Client{Transport: transport} resp, err := httpClient.Get(url + "connz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } if c.NumConns != 1 { t.Fatalf("Expected 1 connection, got %d\n", c.NumConns) } if c.Total != 1 { t.Fatalf("Expected 1 live connection, got %d\n", c.Total) } if c.Conns == nil || len(c.Conns) != 1 { t.Fatalf("Expected 1 connection in array, got %d\n", len(c.Conns)) } // Test inside details of each connection ci := c.Conns[0] if ci.Cid == 0 { t.Fatalf("Expected non-zero cid, got %v\n", ci.Cid) } if ci.IP != "127.0.0.1" { t.Fatalf("Expected \"127.0.0.1\" for IP, got %v\n", ci.IP) } if ci.Port == 0 { t.Fatalf("Expected non-zero port, got %v\n", ci.Port) } if ci.NumSubs != 1 { t.Fatalf("Expected num_subs of 1, got %v\n", ci.NumSubs) } if len(ci.Subs) != 0 { t.Fatalf("Expected subs of 0, got %v\n", ci.Subs) } if ci.InMsgs != 1 { t.Fatalf("Expected InMsgs of 1, got %v\n", ci.InMsgs) } if ci.OutMsgs != 1 { t.Fatalf("Expected OutMsgs of 1, got %v\n", ci.OutMsgs) } if ci.InBytes != 5 { t.Fatalf("Expected InBytes of 1, got %v\n", ci.InBytes) } if ci.OutBytes != 5 { t.Fatalf("Expected OutBytes of 1, got %v\n", ci.OutBytes) } if ci.Start.IsZero() { t.Fatalf("Expected Start to be valid\n") } if ci.Uptime == "" { t.Fatalf("Expected Uptime to be valid\n") } if ci.LastActivity.IsZero() { t.Fatalf("Expected LastActivity to be valid\n") } if ci.LastActivity.UnixNano() < ci.Start.UnixNano() { t.Fatalf("Expected LastActivity [%v] to be > Start [%v]\n", ci.LastActivity, ci.Start) } if ci.Idle == "" { t.Fatalf("Expected Idle to be valid\n") } } func TestConnzWithSubs(t *testing.T) { s := runMonitorServer() defer s.Shutdown() cl := createClientConnSubscribeAndPublish(t) defer cl.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) resp, err := http.Get(url + "connz?subs=1") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Test inside details of each connection ci := c.Conns[0] if len(ci.Subs) != 1 || ci.Subs[0] != "foo" { t.Fatalf("Expected subs of 1, got %v\n", ci.Subs) } } func TestConnzWithAuth(t *testing.T) { srv, opts := RunServerWithConfig("./configs/multi_user.conf") defer srv.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) curl := fmt.Sprintf("nats://%s:%s@%s/", opts.Users[0].Username, opts.Users[0].Password, endpoint) nc, err := nats.Connect(curl) if err != nil { t.Fatalf("Got an error on Connect: %+v\n", err) } defer nc.Close() ch := make(chan struct{}) nc.Subscribe("foo", func(m *nats.Msg) { ch <- struct{}{} }) nc.Publish("foo", []byte("Hello")) // Wait for message <-ch url := fmt.Sprintf("http://127.0.0.1:%d/", opts.HTTPPort) resp, err := http.Get(url + "connz?auth=1") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Test that we have authorized_user and its Alice. ci := c.Conns[0] if ci.AuthorizedUser != opts.Users[0].Username { t.Fatalf("Expected authorized_user to be %q, got %q\n", opts.Users[0].Username, ci.AuthorizedUser) } } func TestConnzWithAccounts(t *testing.T) { resetPreviousHTTPConnections() s, opts := RunServerWithConfig("./configs/multi_accounts.conf") defer s.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) // Connect all the users. Tests depend on knowing users, accounts. if len(opts.Users) != 6 { t.Fatalf("Expected 6 total users, got %d", len(opts.Users)) } if len(opts.Accounts) != 3 { t.Fatalf("Expected 3 total accounts, got %d", len(opts.Accounts)) } // Map from user to account name. utoa := make(map[string]string) conns := make([]*nats.Conn, len(opts.Users)) for _, u := range opts.Users { nc, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s/", u.Username, u.Password, endpoint)) if err != nil { t.Fatalf("Got an error on Connect: %+v\n", err) } defer nc.Close() utoa[u.Username] = u.Account.Name conns = append(conns, nc) } url := fmt.Sprintf("http://127.0.0.1:%d/", opts.HTTPPort) grabConnz := func(args string) *server.Connz { t.Helper() if args != "" { args = "&" + args } resp, err := http.Get(url + "connz?auth=1" + args) if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return &c } c := grabConnz("") if c.NumConns != 6 { t.Fatalf("Expected 6 connection entries, got %d", c.NumConns) } checkConn := func(ci *server.ConnInfo) { t.Helper() user := ci.AuthorizedUser account := utoa[user] if user == "" || account == "" { t.Fatalf("Empty user or account: %q - %q", user, account) } if ci.Account != account { t.Fatalf("Expected account of %q, got %q", account, ci.Account) } } for i, ci := range c.Conns { if ci.Cid != uint64(i+1) { t.Fatalf("Expected CID of %d, got %d", i+1, ci.Cid) } checkConn(ci) } // Now make sure we can pull connections by account and user pullByAccount := func(accName, state string) { t.Helper() c = grabConnz("acc=" + accName + "&state=" + state) if c.NumConns != 2 { t.Fatalf("Expected 2 connection entries, got %d", c.NumConns) } for _, ci := range c.Conns { if ci.Account != accName { t.Fatalf("Expected %q account, go %q", accName, ci.Account) } } } pullByUser := func(user, state string) { t.Helper() c = grabConnz("user=" + user + "&state=" + state) if c.NumConns != 1 { t.Fatalf("Expected 1 connection, got %d", c.NumConns) } if c.Conns[0].AuthorizedUser != user { t.Fatalf("Expected user %q, got %q", user, c.Conns[0].AuthorizedUser) } } pullByAccount("engineering", "open") pullByAccount("finance", "open") pullByAccount("legal", "open") pullByUser("alice", "open") pullByUser("bob", "open") pullByUser("john", "open") pullByUser("mary", "open") pullByUser("peter", "open") pullByUser("paul", "open") // Now closed and make sure these work on closed as well. for _, nc := range conns { nc.Close() } checkFor(t, time.Second, 10*time.Millisecond, func() error { if numClients := s.NumClients(); numClients != 0 { return fmt.Errorf("Number of client is %d", numClients) } return nil }) pullByAccount("engineering", "closed") pullByAccount("finance", "closed") pullByAccount("legal", "closed") pullByUser("alice", "closed") pullByUser("bob", "closed") pullByUser("john", "closed") pullByUser("mary", "closed") pullByUser("peter", "closed") pullByUser("paul", "closed") } func TestConnzWithOffsetAndLimit(t *testing.T) { s := runMonitorServer() defer s.Shutdown() cl1 := createClientConnSubscribeAndPublish(t) defer cl1.Close() cl2 := createClientConnSubscribeAndPublish(t) defer cl2.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) resp, err := http.Get(url + "connz?offset=1&limit=1") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } c := server.Connz{} if err := json.Unmarshal(body, &c); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } if c.Limit != 1 { t.Fatalf("Expected limit of 1, got %v\n", c.Limit) } if c.Offset != 1 { t.Fatalf("Expected offset of 1, got %v\n", c.Offset) } if len(c.Conns) != 1 { t.Fatalf("Expected conns of 1, got %v\n", len(c.Conns)) } } func TestSubsz(t *testing.T) { s := runMonitorServer() defer s.Shutdown() cl := createClientConnSubscribeAndPublish(t) defer cl.Close() url := fmt.Sprintf("http://127.0.0.1:%d/", MONITOR_PORT) resp, err := http.Get(url + "subscriptionsz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } su := server.Subsz{} if err := json.Unmarshal(body, &su); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } // Do some sanity checks on values if su.NumSubs != 1 { t.Fatalf("Expected num_subs of 1, got %v\n", su.NumSubs) } } func TestHTTPHost(t *testing.T) { s := runMonitorServer() defer s.Shutdown() // Grab non-127.0.0.1 address and try to use that to connect. // Should fail. var ip net.IP ifaces, _ := net.Interfaces() for _, i := range ifaces { addrs, _ := i.Addrs() for _, addr := range addrs { switch v := addr.(type) { case *net.IPNet: ip = v.IP case *net.IPAddr: ip = v.IP } // Skip loopback/127.0.0.1 or any ipv6 for now. if ip.IsLoopback() || ip.To4() == nil { ip = nil continue } break } if ip != nil { break } } if ip == nil { t.Fatalf("Could not find non-loopback IPV4 address") } url := fmt.Sprintf("http://%v:%d/", ip, MONITOR_PORT) if resp, err := http.Get(url + "varz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } } // Create a connection to test ConnInfo func createClientConnSubscribeAndPublish(t *testing.T) net.Conn { cl := createClientConn(t, "127.0.0.1", CLIENT_PORT) send, expect := setupConn(t, cl) expectMsgs := expectMsgsCommand(t, expect) send("SUB foo 1\r\nPUB foo 5\r\nhello\r\n") expectMsgs(1) return cl } func TestMonitorNoTLSConfig(t *testing.T) { opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPHost = "127.0.0.1" opts.HTTPSPort = MONITOR_PORT s := server.New(&opts) defer s.Shutdown() // Check with manually starting the monitoring, which should return an error if err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "TLS") { t.Fatalf("Expected error about missing TLS config, got %v", err) } // Also check by calling Start(), which should produce a fatal error dl := &dummyLogger{} s.SetLogger(dl, false, false) defer s.SetLogger(nil, false, false) s.Start() if !strings.Contains(dl.msg, "TLS") { t.Fatalf("Expected error about missing TLS config, got %v", dl.msg) } } func TestMonitorErrorOnListen(t *testing.T) { s := runMonitorServer() defer s.Shutdown() opts := DefaultTestOptions opts.Port = CLIENT_PORT + 1 opts.HTTPHost = "127.0.0.1" opts.HTTPPort = MONITOR_PORT s2 := server.New(&opts) defer s2.Shutdown() if err := s2.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "listen") { t.Fatalf("Expected error about not able to start listener, got %v", err) } } func TestMonitorBothPortsConfigured(t *testing.T) { opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPHost = "127.0.0.1" opts.HTTPPort = MONITOR_PORT opts.HTTPSPort = MONITOR_PORT + 1 s := server.New(&opts) defer s.Shutdown() if err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), "specify both") { t.Fatalf("Expected error about ports configured, got %v", err) } } func TestMonitorStop(t *testing.T) { resetPreviousHTTPConnections() opts := DefaultTestOptions opts.Port = CLIENT_PORT opts.HTTPHost = "127.0.0.1" opts.HTTPPort = MONITOR_PORT url := fmt.Sprintf("http://%v:%d/", opts.HTTPHost, MONITOR_PORT) // Create a server instance and start only the monitoring http server. s := server.New(&opts) if err := s.StartMonitoring(); err != nil { t.Fatalf("Error starting monitoring: %v", err) } // Make sure http server is started resp, err := http.Get(url + "varz") if err != nil { t.Fatalf("Error on http request: %v", err) } resp.Body.Close() // Although the server itself was not started (we did not call s.Start()), // Shutdown() should stop the http server. s.Shutdown() // HTTP request should now fail if resp, err := http.Get(url + "varz"); err == nil { t.Fatalf("Expected error: Got %+v\n", resp) } } nats-server-2.10.27/test/new_routes_test.go000066400000000000000000001521101477524627100207120ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "fmt" "net" "testing" "time" "github.com/nats-io/nats-server/v2/logger" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) func runNewRouteServer(t *testing.T) (*server.Server, *server.Options) { return RunServerWithConfig("./configs/new_cluster.conf") } func TestNewRouteInfoOnConnect(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() info := checkInfoMsg(t, rc) if info.Port != opts.Cluster.Port { t.Fatalf("Received wrong information for port, expected %d, got %d", info.Port, opts.Cluster.Port) } // Make sure we advertise new proto. if info.Proto < server.RouteProtoV2 { t.Fatalf("Expected routeProtoV2, got %d", info.Proto) } // New proto should always send nonce too. if info.Nonce == "" { t.Fatalf("Expected a non empty nonce in new route INFO") } // By default headers should be true. if !info.Headers { t.Fatalf("Expected to have headers on by default") } // Leafnode origin cluster support. if !info.LNOC { t.Fatalf("Expected to have leafnode origin cluster support") } if !info.LNOCU { t.Fatalf("Expected to have leafnode origin cluster in unsub protocol support") } } func TestNewRouteHeaderSupport(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupHeaderConn(t, clientA) sendA("SUB foo bar 22\r\n") sendA("PING\r\n") expectA(pongRe) if err := checkExpectedSubs(1, srvA, srvB); err != nil { t.Fatalf("%v", err) } sendB, expectB := setupHeaderConn(t, clientB) // Can not have \r\n in payload fyi for regex. sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) expectHeaderMsgs := expectHeaderMsgsCommand(t, expectA) matches := expectHeaderMsgs(1) checkHmsg(t, matches[0], "foo", "22", "reply", "12", "14", "K1:V1,K2:V2 ", "ok") } func TestNewRouteHeaderSupportOldAndNew(t *testing.T) { optsA := LoadConfig("./configs/srv_a.conf") optsA.NoHeaderSupport = true srvA := RunServer(optsA) defer srvA.Shutdown() srvB, optsB := RunServerWithConfig("./configs/srv_b.conf") defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupHeaderConn(t, clientA) sendA("SUB foo bar 22\r\n") sendA("PING\r\n") expectA(pongRe) if err := checkExpectedSubs(1, srvA, srvB); err != nil { t.Fatalf("%v", err) } sendB, expectB := setupHeaderConn(t, clientB) // Can not have \r\n in payload fyi for regex. sendB("HPUB foo reply 12 14\r\nK1:V1,K2:V2 ok\r\n") sendB("PING\r\n") expectB(pongRe) expectMsgs := expectMsgsCommand(t, expectA) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "22", "reply", "2", "ok") } func sendRouteInfo(t *testing.T, rc net.Conn, routeSend sendFun, routeID string) { info := checkInfoMsg(t, rc) info.ID = routeID info.Name = "" b, err := json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } routeSend(fmt.Sprintf("INFO %s\r\n", b)) } func TestNewRouteConnectSubs(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) // Create 10 normal subs and 10 queue subscribers. for i := 0; i < 10; i++ { send(fmt.Sprintf("SUB foo %d\r\n", i)) send(fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) } send("PING\r\n") expect(pongRe) // This client should not be considered active since no subscriptions or // messages have been published. rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:22" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) buf := routeExpect(rsubRe) matches := rsubRe.FindAllSubmatch(buf, -1) if len(matches) != 2 { t.Fatalf("Expected 2 results, got %d", len(matches)) } for _, m := range matches { if string(m[1]) != "$G" { t.Fatalf("Expected global account name of '$G', got %q", m[1]) } if string(m[2]) != "foo" { t.Fatalf("Expected subject of 'foo', got %q", m[2]) } if m[3] != nil { if string(m[3]) != "bar" { t.Fatalf("Expected group of 'bar', got %q", m[3]) } // Expect a weighted count for the queue group if len(m) != 5 { t.Fatalf("Expected a weight for the queue group") } if m[4] == nil || string(m[4]) != "10" { t.Fatalf("Expected Weight of '10', got %q", m[4]) } } } // Close the client connection, check the results. c.Close() // Expect 2 for numUnSubs := 0; numUnSubs != 2; { buf := routeExpect(runsubRe) numUnSubs += len(runsubRe.FindAllSubmatch(buf, -1)) } } func TestNewRouteConnectSubsWithAccount(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() accName := "$FOO" s.RegisterAccount(accName) c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConnWithAccount(t, s, c, accName) // Create 10 normal subs and 10 queue subscribers. for i := 0; i < 10; i++ { send(fmt.Sprintf("SUB foo %d\r\n", i)) send(fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) } send("PING\r\n") expect(pongRe) // This client should not be considered active since no subscriptions or // messages have been published. rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:22" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) buf := routeExpect(rsubRe) matches := rsubRe.FindAllSubmatch(buf, -1) if len(matches) != 2 { t.Fatalf("Expected 2 results, got %d", len(matches)) } for _, m := range matches { if string(m[1]) != accName { t.Fatalf("Expected global account name of %q, got %q", accName, m[1]) } if string(m[2]) != "foo" { t.Fatalf("Expected subject of 'foo', got %q", m[2]) } if m[3] != nil { if string(m[3]) != "bar" { t.Fatalf("Expected group of 'bar', got %q", m[3]) } // Expect the SID to be the total weighted count for the queue group if len(m) != 5 { t.Fatalf("Expected a weight for the queue group") } if m[4] == nil || string(m[4]) != "10" { t.Fatalf("Expected Weight of '10', got %q", m[4]) } } } // Close the client connection, check the results. c.Close() // Expect 2 for numUnSubs := 0; numUnSubs != 2; { buf := routeExpect(runsubRe) numUnSubs += len(runsubRe.FindAllSubmatch(buf, -1)) } } func TestNewRouteRSubs(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() foo, err := s.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account '$foo': %v", err) } bar, err := s.RegisterAccount("$bar") if err != nil { t.Fatalf("Error creating account '$bar': %v", err) } // Create a client an account foo. clientA := createClientConn(t, opts.Host, opts.Port) sendA, expectA := setupConnWithAccount(t, s, clientA, "$foo") defer clientA.Close() sendA("PING\r\n") expectA(pongRe) if foonc := foo.NumConnections(); foonc != 1 { t.Fatalf("Expected foo account to have 1 client, got %d", foonc) } if barnc := bar.NumConnections(); barnc != 0 { t.Fatalf("Expected bar account to have 0 clients, got %d", barnc) } // Create a routeConn rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:33" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) // Have the client listen on foo. sendA("SUB foo 1\r\nPING\r\n") expectA(pongRe) // Now create a new client for account $bar and have them subscribe. clientB := createClientConn(t, opts.Host, opts.Port) sendB, expectB := setupConnWithAccount(t, s, clientB, "$bar") defer clientB.Close() sendB("PING\r\n") expectB(pongRe) if foonc := foo.NumConnections(); foonc != 1 { t.Fatalf("Expected foo account to have 1 client, got %d", foonc) } if barnc := bar.NumConnections(); barnc != 1 { t.Fatalf("Expected bar account to have 1 client, got %d", barnc) } // Have the client listen on foo. sendB("SUB foo 1\r\nPING\r\n") expectB(pongRe) routeExpect(rsubRe) // Unsubscribe on clientA from foo subject. sendA("UNSUB 1\r\nPING\r\n") expectA(pongRe) // We should get an RUSUB here. routeExpect(runsubRe) // Now unsubscribe clientB, which should trigger an RS-. sendB("UNSUB 1\r\nPING\r\n") expectB(pongRe) // We should get an RUSUB here. routeExpect(runsubRe) // Now close down the clients. clientA.Close() sendB("SUB foo 2\r\nPING\r\n") expectB(pongRe) routeExpect(rsubRe) // Now close down client B. clientB.Close() // This should trigger an RS- routeExpect(runsubRe) } func TestNewRouteProgressiveNormalSubs(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:33" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) // For progressive we will expect to receive first normal sub but // not subsequent ones. send("SUB foo 1\r\nPING\r\n") expect(pongRe) routeExpect(rsubRe) send("SUB foo 2\r\nPING\r\n") expect(pongRe) expectNothing(t, rc) var buf []byte // Check that sid is showing us total number of subscriptions. checkQueueSub := func(n string) { matches := rsubRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatalf("Expected 1 result, got %d", len(matches)) } m := matches[0] if len(m) != 5 { t.Fatalf("Expected a SID for the queue group, only got %d elements", len(m)) } if string(m[4]) != n { t.Fatalf("Expected %q, got %q", n, m[4]) } } // We should always get the SUB info for QUEUES. send("SUB foo bar 3\r\nPING\r\n") expect(pongRe) buf = routeExpect(rsubRe) checkQueueSub("1") send("SUB foo bar 4\r\nPING\r\n") expect(pongRe) buf = routeExpect(rsubRe) checkQueueSub("2") send("SUB foo bar 5\r\nPING\r\n") expect(pongRe) buf = routeExpect(rsubRe) checkQueueSub("3") // Now walk them back down. // Again we should always get updates for queue subscribers. // And these will be RS+ protos walking the weighted count back down. send("UNSUB 5\r\nPING\r\n") expect(pongRe) buf = routeExpect(rsubRe) checkQueueSub("2") send("UNSUB 4\r\nPING\r\n") expect(pongRe) buf = routeExpect(rsubRe) checkQueueSub("1") // This one should send UNSUB send("UNSUB 3\r\nPING\r\n") expect(pongRe) routeExpect(runsubRe) // Now normal ones. send("UNSUB 1\r\nPING\r\n") expect(pongRe) expectNothing(t, rc) send("UNSUB 2\r\nPING\r\n") expect(pongRe) routeExpect(runsubRe) } func TestNewRouteClientClosedWithNormalSubscriptions(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:44" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) send("SUB foo 1\r\nPING\r\n") expect(pongRe) routeExpect(rsubRe) for i := 2; i < 100; i++ { send(fmt.Sprintf("SUB foo %d\r\n", i)) } send("PING\r\n") expect(pongRe) // Expect nothing from the route. expectNothing(t, rc) // Now close connection. c.Close() expectNothing(t, c) buf := routeExpect(runsubRe) matches := runsubRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) } } func TestNewRouteClientClosedWithQueueSubscriptions(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:44" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) for i := 0; i < 100; i++ { send(fmt.Sprintf("SUB foo bar %d\r\n", i)) } send("PING\r\n") expect(pongRe) // Queue subscribers will send all updates. for numRSubs := 0; numRSubs != 100; { buf := routeExpect(rsubRe) numRSubs += len(rsubRe.FindAllSubmatch(buf, -1)) } // Now close connection. c.Close() expectNothing(t, c) // We should only get one unsub for the queue subscription. matches := runsubRe.FindAllSubmatch(routeExpect(runsubRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) } } func TestNewRouteRUnsubAccountSpecific(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() // Create a routeConn rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:77" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) // Now create 500 subs on same subject but all different accounts. for i := 0; i < 500; i++ { account := fmt.Sprintf("$foo.account.%d", i) s.RegisterAccount(account) routeSend(fmt.Sprintf("RS+ %s foo\r\n", account)) } routeSend("PING\r\n") routeExpect(pongRe) routeSend("RS- $foo.account.22 foo\r\nPING\r\n") routeExpect(pongRe) // Do not expect a message on that account. c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConnWithAccount(t, s, c, "$foo.account.22") send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) c.Close() // But make sure we still receive on others c = createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect = setupConnWithAccount(t, s, c, "$foo.account.33") send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) matches := rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$foo.account.33", "foo", "", "2", "ok") } func TestNewRouteRSubCleanupOnDisconnect(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() // Create a routeConn rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:77" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) // Now create 100 subs on 3 different accounts. for i := 0; i < 100; i++ { subject := fmt.Sprintf("foo.%d", i) routeSend(fmt.Sprintf("RS+ $foo %s\r\n", subject)) routeSend(fmt.Sprintf("RS+ $bar %s\r\n", subject)) routeSend(fmt.Sprintf("RS+ $baz %s bar %d\r\n", subject, i+1)) } routeSend("PING\r\n") routeExpect(pongRe) rc.Close() checkFor(t, time.Second, 10*time.Millisecond, func() error { if ns := s.NumSubscriptions(); ns != 0 { return fmt.Errorf("Number of subscriptions is %d", ns) } return nil }) } func TestNewRouteSendSubsAndMsgs(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:44" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) // Now let's send in interest from the new protocol. // Normal Subscription routeSend("RS+ $G foo\r\n") // Make sure things were processed. routeSend("PING\r\n") routeExpect(pongRe) // Now create a client and send a message, make sure we receive it // over the route connection. c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) buf := routeExpect(rmsgRe) matches := rmsgRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") // Queue Subscription routeSend("RS+ $G foo bar 1\r\n") // Make sure things were processed. routeSend("PING\r\n") routeExpect(pongRe) send("PUB foo reply 2\r\nok\r\nPING\r\n") expect(pongRe) matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "+ reply bar", "2", "ok") // Another Queue Subscription routeSend("RS+ $G foo baz 1\r\n") // Make sure things were processed. routeSend("PING\r\n") routeExpect(pongRe) send("PUB foo reply 2\r\nok\r\nPING\r\n") expect(pongRe) matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") // Matching wildcard routeSend("RS+ $G *\r\n") // Make sure things were processed. routeSend("PING\r\n") routeExpect(pongRe) send("PUB foo reply 2\r\nok\r\nPING\r\n") expect(pongRe) matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") // No reply send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "| bar baz", "2", "ok") // Now unsubscribe from the queue group. routeSend("RS- $G foo baz\r\n") routeSend("RS- $G foo bar\r\n") routeSend("PING\r\n") routeExpect(pongRe) // Now send and make sure they are removed. We should still get the message. send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) if len(matches) != 1 { t.Fatalf("Expected only 1 msg, got %d", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") routeSend("RS- $G foo\r\n") routeSend("RS- $G *\r\n") routeSend("PING\r\n") routeExpect(pongRe) // Now we should not receive messages anymore. send("PUB foo 2\r\nok\r\nPING\r\n") expect(pongRe) expectNothing(t, rc) } func TestNewRouteProcessRoutedMsgs(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:55" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) // Create a client c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) // Normal sub to start send("SUB foo 1\r\nPING\r\n") expect(pongRe) routeExpect(rsubRe) expectMsgs := expectMsgsCommand(t, expect) // Now send in a RMSG to the route and make sure its delivered to the client. routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") // Now send in a RMSG to the route with a reply and make sure its delivered to the client. routeSend("RMSG $G foo reply 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches = expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") // Now add in a queue subscriber for the client. send("SUB foo bar 11\r\nPING\r\n") expect(pongRe) routeExpect(rsubRe) // Now add in another queue subscriber for the client. send("SUB foo baz 22\r\nPING\r\n") expect(pongRe) routeExpect(rsubRe) // If we send from a route with no queues. Should only get one message. routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches = expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") // Now send to a specific queue group. We should get multiple messages now. routeSend("RMSG $G foo | bar 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches = expectMsgs(2) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") // Now send to both queue groups. We should get all messages now. routeSend("RMSG $G foo | bar baz 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches = expectMsgs(3) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") // Make sure we do the right thing with reply. routeSend("RMSG $G foo + reply bar baz 2\r\nok\r\nPING\r\n") routeExpect(pongRe) matches = expectMsgs(3) checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") } func TestNewRouteQueueSubsDistribution(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendA, expectA := setupConn(t, clientA) sendB, expectB := setupConn(t, clientB) // Create 100 subscribers on each server. for i := 0; i < 100; i++ { sproto := fmt.Sprintf("SUB foo bar %d\r\n", i) sendA(sproto) sendB(sproto) } sendA("PING\r\n") expectA(pongRe) sendB("PING\r\n") expectB(pongRe) // Each server should have its 100 local subscriptions, plus 1 for the route. if err := checkExpectedSubs(101, srvA, srvB); err != nil { t.Fatal(err.Error()) } sender := createClientConn(t, optsA.Host, optsA.Port) defer sender.Close() send, expect := setupConn(t, sender) // Send 100 messages from Sender for i := 0; i < 100; i++ { send("PUB foo 2\r\nok\r\n") } send("PING\r\n") expect(pongRe) numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) numBReceived := len(msgRe.FindAllSubmatch(expectB(msgRe), -1)) // We may not be able to properly time all messages being ready. for numAReceived+numBReceived != 100 { if buf := peek(clientB); buf != nil { numBReceived += len(msgRe.FindAllSubmatch(buf, -1)) } if buf := peek(clientA); buf != nil { numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) } } // These should be close to 50/50 if numAReceived < 30 || numBReceived < 30 { t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numBReceived) } } // Since we trade interest in accounts now, we have a potential issue with a new client // connecting via a brand new account, publishing and properly doing a flush, then exiting. // If existing subscribers were present but on a remote server they may not get the message. func TestNewRouteSinglePublishOnNewAccount(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() srvA.RegisterAccount("$TEST22") srvB.RegisterAccount("$TEST22") // Create and establish a listener on foo for $TEST22 account. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") sendA("SUB foo 1\r\nPING\r\n") expectA(pongRe) if err := checkExpectedSubs(1, srvB); err != nil { t.Fatal(err.Error()) } clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() // Send a message, flush to make sure server processed and close connection. sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") sendB("PUB foo 2\r\nok\r\nPING\r\n") expectB(pongRe) clientB.Close() expectMsgs := expectMsgsCommand(t, expectA) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") } // Same as above but make sure it works for queue subscribers as well. func TestNewRouteSinglePublishToQueueSubscriberOnNewAccount(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() srvA.RegisterAccount("$TEST22") srvB.RegisterAccount("$TEST22") // Create and establish a listener on foo for $TEST22 account. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") sendA("SUB foo bar 1\r\nPING\r\n") expectA(pongRe) clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() // Send a message, flush to make sure server processed and close connection. sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") sendB("PUB foo bar 2\r\nok\r\nPING\r\n") expectB(pongRe) defer clientB.Close() expectMsgs := expectMsgsCommand(t, expectA) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") sendB("PUB foo bar 2\r\nok\r\nPING\r\n") expectB(pongRe) matches = expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") } // Same as above but make sure it works for queue subscribers over multiple routes as well. func TestNewRouteSinglePublishToMultipleQueueSubscriberOnNewAccount(t *testing.T) { srvA, srvB, srvC, optsA, optsB, optsC := runThreeServers(t) defer srvA.Shutdown() defer srvB.Shutdown() defer srvC.Shutdown() srvA.RegisterAccount("$TEST22") srvB.RegisterAccount("$TEST22") srvC.RegisterAccount("$TEST22") // Create and establish a listener on foo/bar for $TEST22 account. Do this on ClientA and ClientC. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$TEST22") sendA("SUB foo bar 11\r\nPING\r\n") expectA(pongRe) clientC := createClientConn(t, optsC.Host, optsC.Port) defer clientC.Close() sendC, expectC := setupConnWithAccount(t, srvC, clientC, "$TEST22") sendC("SUB foo bar 33\r\nPING\r\n") expectC(pongRe) sendA("PING\r\n") expectA(pongRe) sendC("PING\r\n") expectC(pongRe) clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() time.Sleep(100 * time.Millisecond) // Send a message, flush to make sure server processed and close connection. sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$TEST22") sendB("PUB foo 2\r\nok\r\nPING\r\n") expectB(pongRe) defer clientB.Close() // This should trigger either clientA or clientC, but not both.. bufA := peek(clientA) bufC := peek(clientC) if bufA != nil && bufC != nil { t.Fatalf("Expected one or the other, but got something on both") } numReceived := len(msgRe.FindAllSubmatch(bufA, -1)) numReceived += len(msgRe.FindAllSubmatch(bufC, -1)) if numReceived != 1 { t.Fatalf("Expected only 1 msg, got %d", numReceived) } // Now make sure that we are distributing correctly between A and C // Send 100 messages from Sender for i := 0; i < 100; i++ { sendB("PUB foo 2\r\nok\r\n") } sendB("PING\r\n") expectB(pongRe) numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) numCReceived := len(msgRe.FindAllSubmatch(expectC(msgRe), -1)) // We may not be able to properly time all messages being ready. for numAReceived+numCReceived != 100 { if buf := peek(clientC); buf != nil { numCReceived += len(msgRe.FindAllSubmatch(buf, -1)) } if buf := peek(clientA); buf != nil { numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) } } // These should be close to 50/50 if numAReceived < 30 || numCReceived < 30 { t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numCReceived) } } func registerAccounts(t *testing.T, s *server.Server) (*server.Account, *server.Account) { // Now create two accounts. f, err := s.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account '$foo': %v", err) } b, err := s.RegisterAccount("$bar") if err != nil { t.Fatalf("Error creating account '$bar': %v", err) } return f, b } func addStreamExport(subject string, authorized []*server.Account, targets ...*server.Account) { for _, acc := range targets { acc.AddStreamExport(subject, authorized) } } func addServiceExport(subject string, authorized []*server.Account, targets ...*server.Account) { for _, acc := range targets { acc.AddServiceExport(subject, authorized) } } var isPublic = []*server.Account(nil) func TestNewRouteStreamImport(t *testing.T) { testNewRouteStreamImport(t, false) } func testNewRouteStreamImport(t *testing.T, duplicateSub bool) { t.Helper() srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Do Accounts for the servers. fooA, _ := registerAccounts(t, srvA) fooB, barB := registerAccounts(t, srvB) // Add export to both. addStreamExport("foo", isPublic, fooA, fooB) // Add import abilities to server B's bar account from foo. barB.AddStreamImport(fooB, "foo", "") // clientA will be connected to srvA and be the stream producer. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") sendB("SUB foo 1\r\n") if duplicateSub { sendB("SUB foo 1\r\n") } sendB("PING\r\n") expectB(pongRe) // The subscription on "foo" for account $bar will also become // a subscription on "foo" for account $foo due to import. // So total of 2 subs. if err := checkExpectedSubs(2, srvA); err != nil { t.Fatal(err.Error()) } // Send on clientA sendA("PING\r\n") expectA(pongRe) sendA("PUB foo 2\r\nok\r\nPING\r\n") expectA(pongRe) expectMsgs := expectMsgsCommand(t, expectB) matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") // Send Again on clientA sendA("PUB foo 2\r\nok\r\nPING\r\n") expectA(pongRe) matches = expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") sendB("UNSUB 1\r\nPING\r\n") expectB(pongRe) if err := checkExpectedSubs(0, srvA); err != nil { t.Fatal(err.Error()) } sendA("PUB foo 2\r\nok\r\nPING\r\n") expectA(pongRe) expectNothing(t, clientA) } func TestNewRouteStreamImportLargeFanout(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Do Accounts for the servers. // This account will export a stream. fooA, err := srvA.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account '$foo': %v", err) } fooB, err := srvB.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account '$foo': %v", err) } // Add export to both. addStreamExport("foo", isPublic, fooA, fooB) // Now we will create 100 accounts who will all import from foo. fanout := 100 barA := make([]*server.Account, fanout) for i := 0; i < fanout; i++ { acc := fmt.Sprintf("$bar-%d", i) barA[i], err = srvB.RegisterAccount(acc) if err != nil { t.Fatalf("Error creating account %q: %v", acc, err) } // Add import abilities to server B's bar account from foo. barA[i].AddStreamImport(fooB, "foo", "") } // clientA will be connected to srvA and be the stream producer. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() // Now setup fanout clients on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := make([]net.Conn, fanout) sendB := make([]sendFun, fanout) expectB := make([]expectFun, fanout) for i := 0; i < fanout; i++ { clientB[i] = createClientConn(t, optsB.Host, optsB.Port) defer clientB[i].Close() sendB[i], expectB[i] = setupConnWithAccount(t, srvB, clientB[i], barA[i].Name) sendB[i]("SUB foo 1\r\nPING\r\n") expectB[i](pongRe) } // Since we do not shadow all the bar acounts on srvA they will be dropped // when they hit the other side, which means we could only have one sub for // all the imports on srvA, and srvB will have 2*fanout, one normal and one // that represents the import. checkFor(t, time.Second, 10*time.Millisecond, func() error { if ns := srvA.NumSubscriptions(); ns != uint32(1) { return fmt.Errorf("Number of subscriptions is %d", ns) } if ns := srvB.NumSubscriptions(); ns != uint32(2*fanout) { return fmt.Errorf("Number of subscriptions is %d", ns) } return nil }) } func TestNewRouteReservedReply(t *testing.T) { s, opts := runNewRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) // Test that clients can't send to reserved service import replies. send("PUB foo _R_.foo 2\r\nok\r\nPING\r\n") expect(errRe) } func TestNewRouteServiceImport(t *testing.T) { // To quickly enable trace and debug logging //doLog, doTrace, doDebug = true, true, true srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Make so we can tell the two apart since in same PID. if doLog { srvA.SetLogger(logger.NewTestLogger("[SRV-A] - ", false), true, true) srvB.SetLogger(logger.NewTestLogger("[SRV-B] - ", false), true, true) } // Do Accounts for the servers. fooA, barA := registerAccounts(t, srvA) fooB, barB := registerAccounts(t, srvB) // Add export to both. addServiceExport("test.request", isPublic, fooA, fooB) // Add import abilities to server B's bar account from foo. // Meaning that when a user sends a request on foo.request from account bar, // the request will be mapped to be received by the responder on account foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Do same on A. if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") sendA("SUB test.request 1\r\nPING\r\n") expectA(pongRe) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") sendB("SUB reply 1\r\nPING\r\n") expectB(pongRe) // Wait for all subs to be propagated. (1 on foo, 2 on bar) if err := checkExpectedSubs(3, srvA, srvB); err != nil { t.Fatal(err.Error()) } // Send the request from clientB on foo.request, sendB("PUB foo.request reply 2\r\nhi\r\nPING\r\n") expectB(pongRe) expectMsgsA := expectMsgsCommand(t, expectA) expectMsgsB := expectMsgsCommand(t, expectB) // Expect the request on A matches := expectMsgsA(1) reply := string(matches[0][replyIndex]) checkMsg(t, matches[0], "test.request", "1", reply, "2", "hi") if reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) expectA(pongRe) matches = expectMsgsB(1) checkMsg(t, matches[0], "reply", "1", "", "2", "ok") // This will be the responder and the wildcard for all service replies. if ts := fooA.TotalSubs(); ts != 2 { t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) } routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) r := routez.Routes[0] if r == nil { t.Fatalf("Expected 1 route, got none") } if r.NumSubs != 2 { t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) } } func TestNewRouteServiceExportWithWildcards(t *testing.T) { for _, test := range []struct { name string public bool }{ { name: "public", public: true, }, { name: "private", public: false, }, } { t.Run(test.name, func(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Do Accounts for the servers. fooA, barA := registerAccounts(t, srvA) fooB, barB := registerAccounts(t, srvB) var accs []*server.Account // Add export to both. if !test.public { accs = []*server.Account{barA} } addServiceExport("ngs.update.*", accs, fooA) if !test.public { accs = []*server.Account{barB} } addServiceExport("ngs.update.*", accs, fooB) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Do same on A. if err := barA.AddServiceImport(fooA, "ngs.update", "ngs.update.$bar"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") sendA("SUB ngs.update.* 1\r\nPING\r\n") expectA(pongRe) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") sendB("SUB reply 1\r\nPING\r\n") expectB(pongRe) // Wait for all subs to be propagated. (1 on foo, 2 on bar) if err := checkExpectedSubs(3, srvA, srvB); err != nil { t.Fatal(err.Error()) } // Send the request from clientB on foo.request, sendB("PUB ngs.update reply 2\r\nhi\r\nPING\r\n") expectB(pongRe) expectMsgsA := expectMsgsCommand(t, expectA) expectMsgsB := expectMsgsCommand(t, expectB) // Expect the request on A matches := expectMsgsA(1) reply := string(matches[0][replyIndex]) checkMsg(t, matches[0], "ngs.update.$bar", "1", reply, "2", "hi") if reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) expectA(pongRe) matches = expectMsgsB(1) checkMsg(t, matches[0], "reply", "1", "", "2", "ok") if ts := fooA.TotalSubs(); ts != 2 { t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) } routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) r := routez.Routes[0] if r == nil { t.Fatalf("Expected 1 route, got none") } if r.NumSubs != 2 { t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) } }) } } func TestNewRouteServiceImportQueueGroups(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Do Accounts for the servers. fooA, barA := registerAccounts(t, srvA) fooB, barB := registerAccounts(t, srvB) // Add export to both. addServiceExport("test.request", isPublic, fooA, fooB) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // Do same on A. if err := barA.AddServiceImport(fooA, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint and responder. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") sendA("SUB test.request QGROUP 1\r\nPING\r\n") expectA(pongRe) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") sendB("SUB reply QGROUP_TOO 1\r\nPING\r\n") expectB(pongRe) // Wait for all subs to be propagated. (1 on foo, 2 on bar) if err := checkExpectedSubs(3, srvA, srvB); err != nil { t.Fatal(err.Error()) } // Send the request from clientB on foo.request, sendB("PUB foo.request reply 2\r\nhi\r\nPING\r\n") expectB(pongRe) expectMsgsA := expectMsgsCommand(t, expectA) expectMsgsB := expectMsgsCommand(t, expectB) // Expect the request on A matches := expectMsgsA(1) reply := string(matches[0][replyIndex]) checkMsg(t, matches[0], "test.request", "1", reply, "2", "hi") if reply == "reply" { t.Fatalf("Expected randomized reply, but got original") } sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) expectA(pongRe) matches = expectMsgsB(1) checkMsg(t, matches[0], "reply", "1", "", "2", "ok") if ts := fooA.TotalSubs(); ts != 2 { t.Fatalf("Expected two subs to be left on fooA, but got %d", ts) } routez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) r := routez.Routes[0] if r == nil { t.Fatalf("Expected 1 route, got none") } if r.NumSubs != 2 { t.Fatalf("Expected 2 subs in the route connection, got %v", r.NumSubs) } } func TestNewRouteServiceImportDanglingRemoteSubs(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() // Do Accounts for the servers. fooA, _ := registerAccounts(t, srvA) fooB, barB := registerAccounts(t, srvB) // Add in the service export for the requests. Make it public. if err := fooA.AddServiceExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // Add export to both. addServiceExport("test.request", isPublic, fooA, fooB) // Add import abilities to server B's bar account from foo. if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { t.Fatalf("Error adding service import: %v", err) } // clientA will be connected to srvA and be the service endpoint, but will not send responses. clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() sendA, expectA := setupConnWithAccount(t, srvA, clientA, "$foo") // Express interest. sendA("SUB test.request 1\r\nPING\r\n") expectA(pongRe) // Now setup client B on srvB who will do a sub from account $bar // that should map account $foo's foo subject. clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() sendB, expectB := setupConnWithAccount(t, srvB, clientB, "$bar") sendB("SUB reply 1\r\nPING\r\n") expectB(pongRe) // Wait for all subs to be propagated (1 on foo and 1 on bar on srvA) // (note that srvA is not importing) if err := checkExpectedSubs(2, srvA); err != nil { t.Fatal(err.Error()) } // Wait for all subs to be propagated (1 on foo and 2 on bar) if err := checkExpectedSubs(3, srvB); err != nil { t.Fatal(err.Error()) } // Send 100 requests from clientB on foo.request, for i := 0; i < 100; i++ { sendB("PUB foo.request reply 2\r\nhi\r\n") } sendB("PING\r\n") expectB(pongRe) numRequests := 0 // Expect the request on A checkFor(t, time.Second, 10*time.Millisecond, func() error { buf := expectA(msgRe) matches := msgRe.FindAllSubmatch(buf, -1) numRequests += len(matches) if numRequests != 100 { return fmt.Errorf("Number of requests is %d", numRequests) } return nil }) expectNothing(t, clientB) // These reply subjects will be dangling off of $foo account on serverA. // Remove our service endpoint and wait for the dangling replies to go to zero. sendA("UNSUB 1\r\nPING\r\n") expectA(pongRe) checkFor(t, time.Second, 10*time.Millisecond, func() error { if ts := fooA.TotalSubs(); ts != 1 { return fmt.Errorf("Number of subs is %d, should be only 1", ts) } return nil }) } func TestNewRouteNoQueueSubscribersBounce(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) ncA, err := nats.Connect(urlA) if err != nil { t.Fatalf("Failed to create connection for ncA: %v\n", err) } defer ncA.Close() ncB, err := nats.Connect(urlB) if err != nil { t.Fatalf("Failed to create connection for ncB: %v\n", err) } defer ncB.Close() response := []byte("I will help you") // Create a lot of queue subscribers on A, and have one on B. ncB.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { ncB.Publish(m.Reply, response) }) for i := 0; i < 100; i++ { ncA.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { ncA.Publish(m.Reply, response) }) } ncB.Flush() ncA.Flush() // Send all requests from B numAnswers := 0 for i := 0; i < 500; i++ { if _, err := ncB.Request("foo.request", []byte("Help Me"), time.Second); err != nil { t.Fatalf("Received an error on Request test [%d]: %s", i, err) } numAnswers++ // After we have sent 20 close the ncA client. if i == 20 { ncA.Close() } } if numAnswers != 500 { t.Fatalf("Expect to get all 500 responses, got %d", numAnswers) } } func TestNewRouteLargeDistinctQueueSubscribers(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) ncA, err := nats.Connect(urlA) if err != nil { t.Fatalf("Failed to create connection for ncA: %v\n", err) } defer ncA.Close() ncB, err := nats.Connect(urlB) if err != nil { t.Fatalf("Failed to create connection for ncB: %v\n", err) } defer ncB.Close() const nqsubs = 100 qsubs := make([]*nats.Subscription, 100) // Create 100 queue subscribers on B all with different queue groups. for i := 0; i < nqsubs; i++ { qg := fmt.Sprintf("worker-%d", i) qsubs[i], _ = ncB.QueueSubscribeSync("foo", qg) } ncB.Flush() checkFor(t, time.Second, 10*time.Millisecond, func() error { if ns := srvA.NumSubscriptions(); ns != 100 { return fmt.Errorf("Number of subscriptions is %d", ns) } return nil }) // Send 10 messages. We should receive 1000 responses. for i := 0; i < 10; i++ { ncA.Publish("foo", nil) } ncA.Flush() checkFor(t, 2*time.Second, 10*time.Millisecond, func() error { for i := 0; i < nqsubs; i++ { if n, _, _ := qsubs[i].Pending(); n != 10 { return fmt.Errorf("Number of messages is %d", n) } } return nil }) } func TestNewRouteLeafNodeOriginSupport(t *testing.T) { content := ` listen: 127.0.0.1:-1 cluster { name: xyz, listen: 127.0.0.1:-1 } leafnodes { listen: 127.0.0.1:-1 } no_sys_acc: true ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() gacc, _ := s.LookupAccount("$G") lcontent := ` listen: 127.0.0.1:-1 cluster { name: ln1, listen: 127.0.0.1:-1 } leafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] } no_sys_acc: true ` lconf := createConfFile(t, []byte(fmt.Sprintf(lcontent, opts.LeafNode.Port))) ln, _ := RunServerWithConfig(lconf) defer ln.Shutdown() checkLeafNodeConnected(t, s) lgacc, _ := ln.LookupAccount("$G") rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "LNOC:22" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) pingPong := func() { t.Helper() routeSend("PING\r\n") routeExpect(pongRe) } info := checkInfoMsg(t, rc) info.ID = routeID info.Name = "" info.LNOC = true // Overwrite to false to check that we are getting LS- without origin // if we are an old server. info.LNOCU = false b, err := json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } routeSend(fmt.Sprintf("INFO %s\r\n", b)) routeExpect(rlsubRe) pingPong() sendLSProtosFromRoute := func(lnocu bool) { t.Helper() // Make sure it can process and LS+ routeSend("LS+ ln1 $G foo\r\n") pingPong() // Check interest is registered on remote server. if !gacc.SubscriptionInterest("foo") { t.Fatalf("Expected interest on \"foo\"") } // This should not have been sent to the leafnode since same origin cluster. time.Sleep(10 * time.Millisecond) if lgacc.SubscriptionInterest("foo") { t.Fatalf("Did not expect interest on \"foo\"") } // Now unsub. Either act as an old server that does not support origin // in the LS- or as a new server. if lnocu { routeSend("LS- ln1 $G foo\r\n") } else { routeSend("LS- $G foo\r\n") } pingPong() // Interest should be gone. if gacc.SubscriptionInterest("foo") { t.Fatalf("Expected no interest on \"foo\"") } // Make sure we did not incorrectly send an interest to the leaf. time.Sleep(10 * time.Millisecond) if lgacc.SubscriptionInterest("foo") { t.Fatalf("Did not expect interest on \"foo\"") } // Repeat with a queue. routeSend("LS+ ln1 $G foo bar 1\r\n") pingPong() if !gacc.SubscriptionInterest("foo") { t.Fatalf("Expected interest on \"foo\"") } // This should not have been sent to the leafnode since same origin cluster. time.Sleep(10 * time.Millisecond) if lgacc.SubscriptionInterest("foo") { t.Fatalf("Did not expect interest on \"foo\"") } // Now unsub. if lnocu { routeSend("LS- ln1 $G foo bar\r\n") } else { routeSend("LS- $G foo bar\r\n") } pingPong() // Subscription should be gone. if gacc.SubscriptionInterest("foo") { t.Fatalf("Expected no interest on \"foo\"") } // Make sure we did not incorrectly send an interest to the leaf. time.Sleep(10 * time.Millisecond) if lgacc.SubscriptionInterest("foo") { t.Fatalf("Did not expect interest on \"foo\"") } } // Check the LS+/- when not supporting origin in LS- sendLSProtosFromRoute(false) // Create a connection on the leafnode server. nc, err := nats.Connect(ln.ClientURL()) if err != nil { t.Fatalf("Unexpected error connecting %v", err) } defer nc.Close() sub, _ := nc.SubscribeSync("bar") // Let it propagate to the main server checkFor(t, time.Second, 10*time.Millisecond, func() error { if !gacc.SubscriptionInterest("bar") { return fmt.Errorf("No interest") } return nil }) // For "bar" routeExpect(rlsubRe) // Now pretend like we send a message to the main server over the // route but from the same origin cluster, should not be delivered // to the leafnode. // Make sure it can process and LMSG. // LMSG for routes is like HMSG with an origin cluster before the account. routeSend("LMSG ln1 $G bar 0 2\r\nok\r\n") pingPong() // Let it propagate if not properly truncated. time.Sleep(10 * time.Millisecond) if n, _, _ := sub.Pending(); n != 0 { t.Fatalf("Should not have received the message on bar") } // Try one with all the bells and whistles. routeSend("LMSG ln1 $G foo + reply bar baz 0 2\r\nok\r\n") pingPong() // Let it propagate if not properly truncated. time.Sleep(10 * time.Millisecond) if n, _, _ := sub.Pending(); n != 0 { t.Fatalf("Should not have received the message on bar") } // Now unsubscribe, we should receive an LS- without origin. sub.Unsubscribe() routeExpect(lunsubRe) // Quick check for queues sub, _ = nc.QueueSubscribeSync("baz", "bat") // Let it propagate to the main server checkFor(t, time.Second, 10*time.Millisecond, func() error { if !gacc.SubscriptionInterest("baz") { return fmt.Errorf("No interest") } return nil }) // For "baz" routeExpect(rlsubRe) sub.Unsubscribe() routeExpect(lunsubRe) // Restart our routed server, but this time indicate support // for LS- with origin cluster. rc.Close() rc = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeSend, routeExpect = setupRouteEx(t, rc, opts, routeID) info = checkInfoMsg(t, rc) info.ID = routeID info.Name = "" // These should be already set to true since the server that sends the // INFO has them enabled, but just be explicit. info.LNOC = true info.LNOCU = true b, err = json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } routeSend(fmt.Sprintf("INFO %s\r\n", b)) routeExpect(rlsubRe) pingPong() // Check the LS+/LS- sendLSProtosFromRoute(true) sub, _ = nc.SubscribeSync("bar") routeExpect(rlsubRe) sub.Unsubscribe() routeExpect(rlunsubRe) sub, _ = nc.QueueSubscribeSync("baz", "bat") routeExpect(rlsubRe) sub.Unsubscribe() routeExpect(rlunsubRe) } // Check that real duplicate subscription (that is, sent by client with same sid) // are ignored and do not register multiple shadow subscriptions. func TestNewRouteDuplicateSubscription(t *testing.T) { // This is same test than TestNewRouteStreamImport but calling "SUB foo 1" twice. testNewRouteStreamImport(t, true) opts := LoadConfig("./configs/new_cluster.conf") opts.DisableShortFirstPing = true s := RunServer(opts) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_DUPLICATE:22" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) sendRouteInfo(t, rc, routeSend, routeID) routeSend("PING\r\n") routeExpect(pongRe) c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, expect := setupConn(t, c) // Create a real duplicate subscriptions (same sid) send("SUB foo 1\r\nSUB foo 1\r\nPING\r\n") expect(pongRe) // Route should receive single RS+ routeExpect(rsubRe) // Unsubscribe. send("UNSUB 1\r\nPING\r\n") expect(pongRe) // Route should receive RS-. // With defect, only 1 subscription would be found during the unsubscribe, // however route map would have been updated twice when processing the // duplicate SUB, which means that the RS- would not be received because // the count would still be 1. routeExpect(runsubRe) } nats-server-2.10.27/test/norace_test.go000066400000000000000000000434671477524627100200050ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !race && !skip_no_race_tests // +build !race,!skip_no_race_tests package test import ( "context" crand "crypto/rand" "encoding/json" "fmt" "net" "net/url" "os" "regexp" "runtime" "strconv" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nuid" ) // IMPORTANT: Tests in this file are not executed when running with the -race flag. // The test name should be prefixed with TestNoRace so we can run only // those tests: go test -run=TestNoRace ... func TestNoRaceRouteSendSubs(t *testing.T) { template := ` port: -1 write_deadline: "2s" cluster { port: -1 pool_size: -1 compression: disabled %s } no_sys_acc: true ` cfa := createConfFile(t, []byte(fmt.Sprintf(template, ""))) srvA, optsA := RunServerWithConfig(cfa) srvA.Shutdown() optsA.DisableShortFirstPing = true srvA = RunServer(optsA) defer srvA.Shutdown() cfb := createConfFile(t, []byte(fmt.Sprintf(template, ""))) srvB, optsB := RunServerWithConfig(cfb) srvB.Shutdown() optsB.DisableShortFirstPing = true srvB = RunServer(optsB) defer srvB.Shutdown() clientA := createClientConn(t, optsA.Host, optsA.Port) defer clientA.Close() clientASend, clientAExpect := setupConn(t, clientA) clientASend("PING\r\n") clientAExpect(pongRe) clientB := createClientConn(t, optsB.Host, optsB.Port) defer clientB.Close() clientBSend, clientBExpect := setupConn(t, clientB) clientBSend("PING\r\n") clientBExpect(pongRe) // total number of subscriptions per server totalPerServer := 100000 for i := 0; i < totalPerServer/2; i++ { proto := fmt.Sprintf("SUB foo.%d %d\r\n", i, i*2+1) clientASend(proto) clientBSend(proto) proto = fmt.Sprintf("SUB bar.%d queue.%d %d\r\n", i, i, i*2+2) clientASend(proto) clientBSend(proto) } clientASend("PING\r\n") clientAExpect(pongRe) clientBSend("PING\r\n") clientBExpect(pongRe) if err := checkExpectedSubs(totalPerServer, srvA, srvB); err != nil { t.Fatal(err.Error()) } routes := fmt.Sprintf(` routes: [ "nats://%s:%d" ] `, optsA.Cluster.Host, optsA.Cluster.Port) if err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, routes)), 0600); err != nil { t.Fatalf("Error rewriting B's config file: %v", err) } if err := srvB.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } checkClusterFormed(t, srvA, srvB) if err := checkExpectedSubs(2*totalPerServer, srvA, srvB); err != nil { t.Fatal(err.Error()) } checkSlowConsumers := func(t *testing.T) { t.Helper() if srvB.NumSlowConsumers() != 0 || srvA.NumSlowConsumers() != 0 { t.Fatalf("Expected no slow consumers, got %d for srvA and %df for srvB", srvA.NumSlowConsumers(), srvB.NumSlowConsumers()) } } checkSlowConsumers(t) type sender struct { c net.Conn sf sendFun ef expectFun } var senders []*sender createSenders := func(t *testing.T, host string, port int) { t.Helper() for i := 0; i < 25; i++ { s := &sender{} s.c = createClientConn(t, host, port) s.sf, s.ef = setupConn(t, s.c) s.sf("PING\r\n") s.ef(pongRe) senders = append(senders, s) } } createSenders(t, optsA.Host, optsA.Port) createSenders(t, optsB.Host, optsB.Port) for _, s := range senders { defer s.c.Close() } // Now create SUBs on A and B for "ping.replies" and simulate // that there are thousands of replies being sent on // both sides. createSubOnReplies := func(t *testing.T, host string, port int) net.Conn { t.Helper() c := createClientConn(t, host, port) send, expect := setupConn(t, c) send("SUB ping.replies 123456789\r\nPING\r\n") expect(pongRe) return c } requestorOnA := createSubOnReplies(t, optsA.Host, optsA.Port) defer requestorOnA.Close() requestorOnB := createSubOnReplies(t, optsB.Host, optsB.Port) defer requestorOnB.Close() if err := checkExpectedSubs(2*totalPerServer+2, srvA, srvB); err != nil { t.Fatal(err.Error()) } totalReplies := 120000 payload := sizedBytes(400) expectedBytes := (len(fmt.Sprintf("MSG ping.replies 123456789 %d\r\n\r\n", len(payload))) + len(payload)) * totalReplies ch := make(chan error, 2) recvReplies := func(c net.Conn) { var buf [32 * 1024]byte for total := 0; total < expectedBytes; { n, err := c.Read(buf[:]) if err != nil { ch <- fmt.Errorf("read error: %v", err) return } total += n } ch <- nil } go recvReplies(requestorOnA) go recvReplies(requestorOnB) wg := sync.WaitGroup{} wg.Add(len(senders)) replyMsg := fmt.Sprintf("PUB ping.replies %d\r\n%s\r\n", len(payload), payload) for _, s := range senders { go func(s *sender, count int) { defer wg.Done() for i := 0; i < count; i++ { s.sf(replyMsg) } s.sf("PING\r\n") s.ef(pongRe) }(s, totalReplies/len(senders)) } for i := 0; i < 2; i++ { select { case e := <-ch: if e != nil { t.Fatalf("Error: %v", e) } case <-time.After(10 * time.Second): t.Fatalf("Did not receive all %v replies", totalReplies) } } checkSlowConsumers(t) wg.Wait() checkSlowConsumers(t) // Let's remove the route and do a config reload. // Otherwise, on test shutdown the client close // will cause the server to try to send unsubs and // this can delay the test. if err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, "")), 0600); err != nil { t.Fatalf("Error rewriting B's config file: %v", err) } if err := srvB.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } } func TestNoRaceDynamicResponsePermsMemory(t *testing.T) { srv, opts := RunServerWithConfig("./configs/authorization.conf") defer srv.Shutdown() // We will test the timeout to make sure that we are not showing excessive growth // when a reply subject is not utilized by the responder. // Alice can do anything, so she will be our requestor rc := createClientConn(t, opts.Host, opts.Port) defer rc.Close() expectAuthRequired(t, rc) doAuthConnect(t, rc, "", "alice", DefaultPass) expectResult(t, rc, okRe) // MY_STREAM_SERVICE has an expiration of 10ms for the response permissions. c := createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "svcb", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "SUB my.service.req 1\r\n") expectResult(t, c, okRe) var m runtime.MemStats runtime.GC() runtime.ReadMemStats(&m) pta := m.TotalAlloc // Need this so we do not blow the allocs on expectResult which makes 32k each time. expBuf := make([]byte, 32768) expect := func(c net.Conn, re *regexp.Regexp) { t.Helper() c.SetReadDeadline(time.Now().Add(2 * time.Second)) n, _ := c.Read(expBuf) c.SetReadDeadline(time.Time{}) buf := expBuf[:n] if !re.Match(buf) { t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'", buf, re) } } // Now send off some requests. We will not answer them and this will build up reply // permissions in the server. for i := 0; i < 10000; i++ { pub := fmt.Sprintf("PUB my.service.req resp.%d 2\r\nok\r\n", i) sendProto(t, rc, pub) expect(rc, okRe) expect(c, msgRe) } const max = 20 * 1024 * 1024 // 20MB checkFor(t, time.Second, 25*time.Millisecond, func() error { runtime.GC() runtime.ReadMemStats(&m) used := m.TotalAlloc - pta if used > max { return fmt.Errorf("Using too much memory, expect < 20MB, got %dMB", used/(1024*1024)) } return nil }) } func TestNoRaceLargeClusterMem(t *testing.T) { // Try to clean up. runtime.GC() var m runtime.MemStats runtime.ReadMemStats(&m) pta := m.TotalAlloc opts := func() *server.Options { o := DefaultTestOptions o.Host = "127.0.0.1" o.Port = -1 o.Cluster.Host = o.Host o.Cluster.Port = -1 return &o } var servers []*server.Server // Create seed first. o := opts() s := RunServer(o) servers = append(servers, s) // For connecting to seed server above. routeAddr := fmt.Sprintf("nats-route://%s:%d", o.Cluster.Host, o.Cluster.Port) rurl, _ := url.Parse(routeAddr) routes := []*url.URL{rurl} numServers := 15 for i := 1; i < numServers; i++ { o := opts() o.Routes = routes s := RunServer(o) servers = append(servers, s) } checkClusterFormed(t, servers...) // Calculate in MB what we are using now. const max = 80 * 1024 * 1024 // 80MB runtime.ReadMemStats(&m) used := m.TotalAlloc - pta if used > max { t.Fatalf("Cluster using too much memory, expect < 80MB, got %dMB", used/(1024*1024)) } for _, s := range servers { s.Shutdown() } } // Make sure we have the correct remote state when dealing with queue subscribers // across many client connections. func TestNoRaceQueueSubWeightOrderMultipleConnections(t *testing.T) { opts, err := server.ProcessConfigFile("./configs/new_cluster.conf") if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.DisableShortFirstPing = true s := RunServer(opts) defer s.Shutdown() // Create 100 connections to s url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) clients := make([]*nats.Conn, 0, 100) for i := 0; i < 100; i++ { nc, err := nats.Connect(url, nats.NoReconnect()) if err != nil { t.Fatalf("Error connecting: %v", err) } defer nc.Close() clients = append(clients, nc) } rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RTEST_NEW:22" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) info := checkInfoMsg(t, rc) info.ID = routeID info.Name = routeID b, err := json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } // Send our INFO and wait for a PONG. This will prevent a race // where server will have started processing queue subscriptions // and then processes the route's INFO (sending its current subs) // followed by updates. routeSend(fmt.Sprintf("INFO %s\r\nPING\r\n", b)) routeExpect(pongRe) start := make(chan bool) for _, nc := range clients { go func(nc *nats.Conn) { <-start // Now create 100 identical queue subscribers on each connection. for i := 0; i < 100; i++ { if _, err := nc.QueueSubscribe("foo", "bar", func(_ *nats.Msg) {}); err != nil { return } } nc.Flush() }(nc) } close(start) // We did have this where we wanted to get every update, but now with optimizations // we just want to make sure we always are increasing and that a previous update to // a lesser queue weight is never delivered for this test. maxExpected := 10000 updates := 0 for qw := 0; qw < maxExpected; { buf := routeExpect(rsubRe) matches := rsubRe.FindAllSubmatch(buf, -1) for _, m := range matches { if len(m) != 5 { t.Fatalf("Expected a weight for the queue group") } nqw, err := strconv.Atoi(string(m[4])) if err != nil { t.Fatalf("Got an error converting queue weight: %v", err) } // Make sure the new value only increases, ok to skip since we will // optimize this now, but needs to always be increasing. if nqw <= qw { t.Fatalf("Was expecting increasing queue weight after %d, got %d", qw, nqw) } qw = nqw updates++ } } if updates >= maxExpected { t.Fatalf("Was not expecting all %v updates to be received", maxExpected) } } func TestNoRaceClusterLeaksSubscriptions(t *testing.T) { srvA, srvB, optsA, optsB := runServers(t) defer srvA.Shutdown() defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) numResponses := 100 repliers := make([]*nats.Conn, 0, numResponses) var noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {} // Create 100 repliers for i := 0; i < 50; i++ { nc1, _ := nats.Connect(urlA) defer nc1.Close() nc1.SetErrorHandler(noOpErrHandler) nc2, _ := nats.Connect(urlB) defer nc2.Close() nc2.SetErrorHandler(noOpErrHandler) repliers = append(repliers, nc1, nc2) nc1.Subscribe("test.reply", func(m *nats.Msg) { m.Respond([]byte("{\"sender\": 22 }")) }) nc2.Subscribe("test.reply", func(m *nats.Msg) { m.Respond([]byte("{\"sender\": 33 }")) }) nc1.Flush() nc2.Flush() } servers := fmt.Sprintf("%s, %s", urlA, urlB) req := sizedBytes(8 * 1024) // Now run a requestor in a loop, creating and tearing down each time to // simulate running a modified nats-req. doReq := func() { msgs := make(chan *nats.Msg, 1) inbox := nats.NewInbox() grp := nuid.Next() // Create 8 queue Subscribers for responses. for i := 0; i < 8; i++ { nc, _ := nats.Connect(servers) nc.SetErrorHandler(noOpErrHandler) nc.ChanQueueSubscribe(inbox, grp, msgs) nc.Flush() defer nc.Close() } nc, _ := nats.Connect(servers) nc.SetErrorHandler(noOpErrHandler) nc.PublishRequest("test.reply", inbox, req) defer nc.Close() ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() var received int for { select { case <-msgs: received++ if received >= numResponses { return } case <-ctx.Done(): return } } } var wg sync.WaitGroup doRequests := func(n int) { for i := 0; i < n; i++ { doReq() } wg.Done() } concurrent := 10 wg.Add(concurrent) for i := 0; i < concurrent; i++ { go doRequests(10) } wg.Wait() // Close responders too, should have zero(0) subs attached to routes. for _, nc := range repliers { nc.Close() } // Make sure no clients remain. This is to make sure the test is correct and that // we have closed all the client connections. checkFor(t, time.Second, 10*time.Millisecond, func() error { v1, _ := srvA.Varz(nil) v2, _ := srvB.Varz(nil) if v1.Connections != 0 || v2.Connections != 0 { return fmt.Errorf("We have lingering client connections %d:%d", v1.Connections, v2.Connections) } return nil }) loadRoutez := func() (*server.Routez, *server.Routez) { v1, err := srvA.Routez(&server.RoutezOptions{Subscriptions: true}) if err != nil { t.Fatalf("Error getting Routez: %v", err) } v2, err := srvB.Routez(&server.RoutezOptions{Subscriptions: true}) if err != nil { t.Fatalf("Error getting Routez: %v", err) } return v1, v2 } checkFor(t, time.Second, 10*time.Millisecond, func() error { r1, r2 := loadRoutez() if r1.Routes[0].NumSubs != 0 { return fmt.Errorf("Leaked %d subs: %+v", r1.Routes[0].NumSubs, r1.Routes[0].Subs) } if r2.Routes[0].NumSubs != 0 { return fmt.Errorf("Leaked %d subs: %+v", r2.Routes[0].NumSubs, r2.Routes[0].Subs) } return nil }) } func TestNoRaceLeafNodeSmapUpdate(t *testing.T) { s, opts := runLeafServer() defer s.Shutdown() // Create a client on leaf server c := createClientConn(t, opts.Host, opts.Port) defer c.Close() csend, cexpect := setupConn(t, c) numSubs := make(chan int, 1) doneCh := make(chan struct{}, 1) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for i := 1; ; i++ { csend(fmt.Sprintf("SUB foo.%d %d\r\n", i, i)) select { case <-doneCh: numSubs <- i return default: } } }() time.Sleep(5 * time.Millisecond) // Create leaf node lc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port) defer lc.Close() setupConn(t, lc) checkLeafNodeConnected(t, s) close(doneCh) ns := <-numSubs csend("PING\r\n") cexpect(pongRe) wg.Wait() // Make sure we receive as many LS+ protocols (since all subs are unique). // But we also have to count for LDS subject. // There may be so many protocols and partials, that expectNumberOfProtos may // not work. Do a manual search here. checkLS := func(proto string, expected int) { t.Helper() p := []byte(proto) cur := 0 buf := make([]byte, 32768) for ls := 0; ls < expected; { lc.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err := lc.Read(buf) lc.SetReadDeadline(time.Time{}) if err == nil && n > 0 { for i := 0; i < n; i++ { if buf[i] == p[cur] { cur++ if cur == len(p) { ls++ cur = 0 } } else { cur = 0 } } } if err != nil || ls > expected { t.Fatalf("Expected %v %sgot %v, err: %v", expected, proto, ls, err) } } } checkLS("LS+ ", ns+1) // Now unsub all those subs... for i := 1; i <= ns; i++ { csend(fmt.Sprintf("UNSUB %d\r\n", i)) } csend("PING\r\n") cexpect(pongRe) // Expect that many LS- checkLS("LS- ", ns) } func TestNoRaceSlowProxy(t *testing.T) { t.Skip() opts := DefaultTestOptions opts.Port = -1 s := RunServer(&opts) defer s.Shutdown() rttTarget := 22 * time.Millisecond bwTarget := 10 * 1024 * 1024 / 8 // 10mbit sp := newSlowProxy(rttTarget, bwTarget, bwTarget, &opts) defer sp.stop() nc, err := nats.Connect(sp.clientURL()) if err != nil { t.Fatalf("Unexpected error: %v", err) } defer nc.Close() doRTT := func() time.Duration { t.Helper() const samples = 5 var total time.Duration for i := 0; i < samples; i++ { rtt, _ := nc.RTT() total += rtt } return total / samples } rtt := doRTT() if rtt < rttTarget || rtt > (rttTarget*3/2) { t.Fatalf("rtt is out of range, target of %v, actual %v", rttTarget, rtt) } // Now test send BW. const payloadSize = 64 * 1024 var payload [payloadSize]byte crand.Read(payload[:]) // 5MB total. bytesSent := (5 * 1024 * 1024) toSend := bytesSent / payloadSize start := time.Now() for i := 0; i < toSend; i++ { nc.Publish("z", payload[:]) } nc.Flush() tt := time.Since(start) bps := float64(bytesSent) / tt.Seconds() min, max := float64(bwTarget)*0.8, float64(bwTarget)*1.25 if bps < min || bps > max { t.Fatalf("bps is off, target is %v, actual is %v", bwTarget, bps) } } nats-server-2.10.27/test/ocsp_peer_test.go000066400000000000000000003347021477524627100205100ustar00rootroot00000000000000// Copyright 2023-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" "testing" "time" . "github.com/nats-io/nats-server/v2/internal/ocsp" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "golang.org/x/crypto/ocsp" ) func NewOCSPResponderRootCA(t *testing.T) *http.Server { t.Helper() respCertPEM := "configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:8888") } func NewOCSPResponderIntermediateCA1(t *testing.T) *http.Server { t.Helper() respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:18888") } func NewOCSPResponderIntermediateCA1Undelegated(t *testing.T) *http.Server { t.Helper() issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" issuerCertKey := "configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem" return NewOCSPResponderCustomAddress(t, issuerCertPEM, issuerCertKey, "127.0.0.1:18888") } func NewOCSPResponderBadDelegateIntermediateCA1(t *testing.T) *http.Server { t.Helper() // UserA2 is a cert issued by intermediate1, but intermediate1 did not add OCSP signing extension respCertPEM := "configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:18888") } func NewOCSPResponderIntermediateCA2(t *testing.T) *http.Server { t.Helper() respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem" return NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, "127.0.0.1:28888") } // TestOCSPPeerGoodClients is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) // and default ocsp_cache implementation and oscp_cache=false configuration func TestOCSPPeerGoodClients(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", ` port: -1 # default ocsp_cache since omitted tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration, non-default ca_timeout ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 2", ` port: -1 # default ocsp_cache since omitted tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "Explicit true cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", ` port: -1 # Short form configuration ocsp_cache: true tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } // TestOCSPPeerUnknownClient is test of NATS client that is OCSP status Unknown from its OCSP Responder func TestOCSPPeerUnknownClient(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) defer intermediateCA1Responder.Shutdown(ctx) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Default cache, mTLS OCSP peer check on inbound client connection, client unknown to intermediate CA 1", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() t.Errorf("Expected connection error, fell through") }) } } // TestOCSPPeerRevokedClient is test of NATS client that is OCSP status Revoked from its OCSP Responder func TestOCSPPeerRevokedClient(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so this revoked client should NOT be able to connect ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, { "Explicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Explicit disable of OCSP peer check ocsp_peer: false } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "Implicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Implicit disable of OCSP peer check (i.e. not configured) # ocsp_peer: false } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "Explicit disable (long form), mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Explicit disable of OCSP peer check, long form ocsp_peer: { verify: false } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerUnknownAndRevokedIntermediate test of NATS client that is OCSP good but either its intermediate is unknown or revoked func TestOCSPPeerUnknownAndRevokedIntermediate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Revoked) // No test OCSP status set on intermediate2, so unknown intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "mTLS OCSP peer check on inbound client connection, client's intermediate is revoked", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, { "mTLS OCSP peer check on inbound client connection, client's intermediate is unknown'", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() t.Errorf("Expected connection error, fell through") }) } } // TestOCSPPeerLeafGood tests Leaf Spoke peer checking Leaf Hub, Leaf Hub peer checking Leaf Spoke, and both peer checking func TestOCSPPeerLeafGood(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Good) for _, test := range []struct { name string hubconfig string spokeconfig string expected int }{ { "OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)", ` port: -1 # Cache configuration is default leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Short form configuration ocsp_peer: true } } ] } `, 1, }, { "OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)", ` port: -1 # Cache configuration is default leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } ] } `, 1, }, { "OCSP peer check bi-directionally", ` port: -1 # Cache configuration is default leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Short form configuration ocsp_peer: true } } ] } `, 1, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") hubcontent := test.hubconfig hubconf := createConfFile(t, []byte(hubcontent)) hub, _ := RunServerWithConfig(hubconf) defer hub.Shutdown() spokecontent := test.spokeconfig spokeconf := createConfFile(t, []byte(spokecontent)) spoke, _ := RunServerWithConfig(spokeconf) defer spoke.Shutdown() checkLeafNodeConnectedCount(t, hub, test.expected) }) } } // TestOCSPPeerLeafRejects tests rejected Leaf Hub, rejected Leaf Spoke, and both rejecting each other func TestOCSPPeerLeafReject(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Revoked) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Revoked) for _, test := range []struct { name string hubconfig string spokeconfig string expected int }{ { "OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)", ` port: -1 # Cache configuration is default leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Short form configuration ocsp_peer: true } } ] } `, 0, }, { "OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)", ` port: -1 leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } ] } `, 0, }, { "OCSP peer check bi-directionally", ` port: -1 leaf: { listen: 127.0.0.1:7444 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } } `, ` port: -1 leaf: { remotes: [ { url: "nats://127.0.0.1:7444", tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Short form configuration ocsp_peer: true } } ] } `, 0, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") hubcontent := test.hubconfig hubconf := createConfFile(t, []byte(hubcontent)) hub, _ := RunServerWithConfig(hubconf) defer hub.Shutdown() spokecontent := test.spokeconfig spokeconf := createConfFile(t, []byte(spokecontent)) spoke, _ := RunServerWithConfig(spokeconf) defer spoke.Shutdown() // Need to inject some time for leaf connection attempts to complete, could refine this to better // negative test time.Sleep(2000 * time.Millisecond) checkLeafNodeConnectedCount(t, hub, test.expected) }) } } func checkLeafNodeConnectedCount(t testing.TB, s *server.Server, lnCons int) { t.Helper() checkFor(t, 5*time.Second, 15*time.Millisecond, func() error { if nln := s.NumLeafNodes(); nln != lnCons { return fmt.Errorf("expected %d connected leafnode(s) for server %q, got %d", lnCons, s.ID(), nln) } return nil }) } // TestOCSPPeerGoodClientsNoneCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) // and ocsp cache type of none (no-op) func TestOCSPPeerGoodClientsNoneCache(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) deleteLocalStore(t, "") for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "None cache explicit long form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", ` port: -1 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } # Long form configuration ocsp_cache: { type: none } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "None cache explicit short form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1", ` port: -1 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } # Short form configuration ocsp_cache: false `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } // TestOCSPPeerGoodClientsLocalCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates) // and leveraging the local ocsp cache type func TestOCSPPeerGoodClientsLocalCache(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } # Short form configuration, local as default ocsp_cache: true `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "Local cache long form: mTLS OCSP peer check on inbound client connection, UserB1 client of intermediate CA 2", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } # Long form configuration ocsp_cache: { type: local } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { // Cleanup any previous test that saved a local cache deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } nc.Close() v := monitorGetVarzHelper(t, 8222) if v.OCSPResponseCache == nil { t.Fatalf("Expected OCSP statistics to be in varz") } if v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Responses != 2 { t.Errorf("Expected cache misses and cache items to be 2, got %d and %d", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses) } // Should get a cache hit now nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } v = monitorGetVarzHelper(t, 8222) if v.OCSPResponseCache == nil { t.Fatalf("Expected OCSP statistics to be in varz") } if v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Hits != 2 || v.OCSPResponseCache.Responses != 2 { t.Errorf("Expected cache misses, hits and cache items to be 2, got %d and %d and %d", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Hits, v.OCSPResponseCache.Responses) } }) } } func TestOCSPPeerMonitor(t *testing.T) { for _, test := range []struct { name string config string NATSClient bool WSClient bool MQTTClient bool LeafClient bool LeafRemotes bool NumTrueLeafRemotes int }{ { "Monitor peer config setting on NATS client", ` port: -1 http_port: 8222 # Default cache configuration tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } `, true, false, false, false, false, 0, }, { "Monitor peer config setting on Websockets client", ` port: -1 http_port: 8222 # Default cache configuration websocket: { port: 8443 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } } `, false, true, false, false, false, 0, }, { "Monitor peer config setting on MQTT client", ` port: -1 http_port: 8222 # Default cache configuration # Required for MQTT server_name: "my_mqtt_server" jetstream: { enabled: true } mqtt: { port: 1883 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } } `, false, false, true, false, false, 0, }, { "Monitor peer config setting on Leaf client", ` port: -1 http_port: 8222 # Default cache configuration leaf: { port: 7422 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } } `, false, false, false, true, false, 0, }, { "Monitor peer config on some Leaf Remotes as well as Leaf client", ` port: -1 http_port: 8222 # Default cache configuration leaf: { port: 7422 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } remotes: [ { url: "nats-leaf://bogus:7422" tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Long form configuration ocsp_peer: { verify: true } } }, { url: "nats-leaf://anotherbogus:7422" tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Short form configuration ocsp_peer: true } }, { url: "nats-leaf://yetanotherbogus:7422" tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 # Peer not configured (default false) } } ] } `, false, false, false, true, true, 2, }, } { t.Run(test.name, func(t *testing.T) { content := test.config conf := createConfFile(t, []byte(content)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() v := monitorGetVarzHelper(t, 8222) if test.NATSClient { if !v.TLSOCSPPeerVerify { t.Fatalf("Expected NATS Client TLSOCSPPeerVerify to be true, got false") } } if test.WSClient { if !v.Websocket.TLSOCSPPeerVerify { t.Fatalf("Expected WS Client TLSOCSPPeerVerify to be true, got false") } } if test.LeafClient { if !v.LeafNode.TLSOCSPPeerVerify { t.Fatalf("Expected Leaf Client TLSOCSPPeerVerify to be true, got false") } } if test.LeafRemotes { cnt := 0 for _, r := range v.LeafNode.Remotes { if r.TLSOCSPPeerVerify { cnt++ } } if cnt != test.NumTrueLeafRemotes { t.Fatalf("Expected %d Leaf Remotes with TLSOCSPPeerVerify true, got %d", test.NumTrueLeafRemotes, cnt) } } }) } } func TestOCSPResponseCacheMonitor(t *testing.T) { for _, test := range []struct { name string config string expect string }{ { "Monitor local cache enabled, explicit cache true", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } # Short form configuration ocsp_cache: true `, "local", }, { "Monitor local cache enabled, explicit cache type local", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } # Long form configuration ocsp_cache: { type: local } `, "local", }, { "Monitor local cache enabled, implicit default", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } # Short form configuration # ocsp_cache: true `, "local", }, { "Monitor none cache enabled, explicit cache false (short)", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } # Short form configuration ocsp_cache: false `, "", }, { "Monitor none cache enabled, explicit cache false (long)", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true } } # Long form configuration ocsp_cache: { type: none } `, "", }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") content := test.config conf := createConfFile(t, []byte(content)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() v := monitorGetVarzHelper(t, 8222) var ct string if v.OCSPResponseCache != nil { ct = v.OCSPResponseCache.Type } if ct != test.expect { t.Fatalf("Expected OCSP Response Cache to be %s, got %s", test.expect, ct) } }) } } func TestOCSPResponseCacheChangeAndReload(t *testing.T) { deleteLocalStore(t, "") // Start with ocsp cache set to none content := ` port: -1 http_port: 8222 tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } # Long form configuration ocsp_cache: { type: none } ` conf := createConfFile(t, []byte(content)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() v := monitorGetVarzHelper(t, 8222) var ct string if v.OCSPResponseCache != nil { ct = v.OCSPResponseCache.Type } if ct != "" { t.Fatalf("Expected OCSP Response Cache to have empty type in varz indicating none") } // Change to local cache content = ` port: -1 http_port: 8222 tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } # Long form configuration ocsp_cache: { type: local } ` if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } time.Sleep(2 * time.Second) v = monitorGetVarzHelper(t, 8222) ct = "" if v.OCSPResponseCache != nil { ct = v.OCSPResponseCache.Type } if ct != "local" { t.Fatalf("Expected OCSP Response Cache type to be local, got %q", ct) } } func deleteLocalStore(t *testing.T, dir string) { t.Helper() if dir == "" { // default dir = "_rc_" } if err := os.RemoveAll(dir); err != nil { t.Fatalf("Error cleaning up local store: %v", err) } } func monitorGetVarzHelper(t *testing.T, httpPort int) *server.Varz { t.Helper() url := fmt.Sprintf("http://127.0.0.1:%d/", httpPort) resp, err := http.Get(url + "varz") if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if resp.StatusCode != 200 { t.Fatalf("Expected a 200 response, got %d\n", resp.StatusCode) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Got an error reading the body: %v\n", err) } v := server.Varz{} if err := json.Unmarshal(body, &v); err != nil { t.Fatalf("Got an error unmarshalling the body: %v\n", err) } return &v } func writeCacheFile(dir string, content []byte) error { if dir == "" { dir = "_rc_" } err := os.MkdirAll(filepath.Join(dir), os.ModePerm) if err != nil { return err } return os.WriteFile(filepath.Join(dir, "cache.json"), content, os.ModePerm) } // TestOCSPPeerPreserveRevokedCacheItem is test of the preserve_revoked cache policy func TestOCSPPeerPreserveRevokedCacheItem(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option responses int64 revokes int64 goods int64 unknowns int64 err error rerr error clean bool }{ { "Test expired revoked cert not actually deleted", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so this revoked client should NOT be able to connect ocsp_peer: { verify: true ca_timeout: 0.5 } } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, 1, 1, 0, 0, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), true, }, { "Test expired revoked cert replaced by current good cert", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so this revoked client should NOT be able to connect ocsp_peer: true } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, 2, 0, 2, 0, nil, nil, false, }, } { t.Run(test.name, func(t *testing.T) { var intermediateCA1Responder *http.Server // clean slate starting the test and start the leaf CA responder for first run if test.clean { deleteLocalStore(t, "") // establish the revoked item (expired) in cache c := []byte(` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-05-29T17:56:45Z", "resp_status": "revoked", "resp_expires": "2023-05-29T17:56:49Z", "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`) err := writeCacheFile("", c) if err != nil { t.Fatal(err) } } else { intermediateCA1Responder = NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) defer intermediateCA1Responder.Shutdown(ctx) } content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() v := monitorGetVarzHelper(t, 8222) if v.OCSPResponseCache == nil { t.Fatalf("Expected OCSP statistics to be in varz") } responses := v.OCSPResponseCache.Responses revokes := v.OCSPResponseCache.Revokes goods := v.OCSPResponseCache.Goods unknowns := v.OCSPResponseCache.Unknowns if !(responses == test.responses && revokes == test.revokes && goods == test.goods && unknowns == test.unknowns) { t.Fatalf("Expected %d response, %d revoked, %d good, %d unknown; got [%d] and [%d] and [%d] and [%d]", test.responses, test.revokes, test.goods, test.unknowns, responses, revokes, goods, unknowns) } }) } } // TestOCSPStapleFeatureInterop is a test of a NATS client (AIA enabled at leaf and cert) connecting to a NATS Server // in which both ocsp_peer is enabled on NATS client connections (verify client) and the ocsp staple is enabled such // that the NATS Server will staple its own OCSP response and make available to the NATS client during handshake. func TestOCSPStapleFeatureInterop(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Interop: Both Good: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client", ` port: -1 ocsp_cache: true ocsp: { mode: always } tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration, non-default ca_timeout ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("expected OCSP staple to be present") } resp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1]) if err != nil || resp.Status != ocsp.Good { return fmt.Errorf("expected a valid GOOD stapled response") } return nil }, }), nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) }, }, { "Interop: Bad Client: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client", ` port: -1 ocsp_cache: true ocsp: { mode: always } tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration, non-default ca_timeout ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("expected OCSP staple to be present") } resp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1]) if err != nil || resp.Status != ocsp.Good { return fmt.Errorf("expected a valid GOOD stapled response") } return nil }, }), nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, fmt.Errorf("remote error: tls: bad certificate"), nil, func() { SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) }, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } // TestOCSPPeerWarnOnlyOption is test of NATS client that is OCSP Revoked status but allowed to pass with warn_only option func TestOCSPPeerWarnOnlyOption(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Revoked) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Revoked NATS client with warn_only explicitly set to false", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Enable OCSP peer but with warn_only option set to false ocsp_peer: { verify: true warn_only: false } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, { "Revoked NATS client with warn_only explicitly set to true", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Enable OCSP peer but with warn_only option set to true ocsp_peer: { verify: true warn_only: true } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerUnknownIsGoodOption is test of NATS client that is OCSP status Unknown from its OCSP Responder but we treat // status Unknown as "Good" func TestOCSPPeerUnknownIsGoodOption(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) defer intermediateCA1Responder.Shutdown(ctx) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "Unknown NATS client with no unknown_is_good option set (default false)", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Short form configuration ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, { "Unknown NATS client with unknown_is_good set to true", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true unknown_is_good: true } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerAllowWhenCAUnreachableOption is test of the allow_when_ca_unreachable peer option func TestOCSPPeerAllowWhenCAUnreachableOption(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option cachedResponse string err error rerr error }{ { "Expired Revoked response in cache for UserA1 -- should be rejected connection (expired revoke honored)", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, ` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-05-29T17:56:45Z", "resp_status": "revoked", "resp_expires": "2023-05-29T17:56:49Z", "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), }, { "Expired Good response in cache for UserA1 -- should be allowed connection (cached item irrelevant)", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, ` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-06-05T16:33:52Z", "resp_status": "good", "resp_expires": "2023-06-05T16:33:55Z", "resp": "/wYAAFMyc1R3TwBYBQBgpzMn1wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjMzMDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeAADZmABA1MVqgEToTAPRAATVaMA0GCSqGSIb3DQEBCwUAA4IBAQB9csJxA2VcpQYmC5lzI0dJJnEC1zcaP1oFbBm5VsZAbWh4mIGTqyEjMkucBqANTypnhWFRYcZE5nJE8tN8Aen0EBYkhk1wfEIhincaF3zs6x5RzigxQOqTPAanO55keKH5exYnfBcyCwAQiBbQaTXLJJdjerfqiRjqgsnLW8yl2IC8+kxTCfR4GHTK34mvqVWYYhP/xbaxTLJTp35t1vf1o78ct9R+z8W5sCSO4TsMNnExZlu4Ejeon/KivMR22j7nelTEaDCuaOl03WxKh9yhw2ix8V7lvR74wh5f4fjLZria6Y0+lUjwvvrZyBRwA62W/ihe3778q8KhLECFPQSaoIIENTCCBDEwggQtMIIDFaADAgECAhQp4VKN/aUqh+sd5B1HbOGKWGlzqzo/AQQwWlEk2jECCBowGEkxqBFJbnRlcm1lZGlhdGUgQ0EgMTAeFw0yMzA1MDExOTI4MzlaFw0zMzA0MjgND1GtFQBJDDCCASIuJwIAAQUA9EABDwAwggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMteUGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAUssDUDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEOGAQDAgeAMBYBHgAlARAQDDAKBgiJtrADCTA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ldVigxX2NybC5kZXIwMxFLHAEBBCcwJTAjEVsMMAGGF1pKAC4SAgALBQD0AQEBAEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`, nil, nil, }, { "Expired Unknown response in cache for UserA1 -- should be allowed connection (cached item irrelevant)", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, ` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-06-05T16:45:01Z", "resp_status": "unknown", "resp_expires": "2023-06-05T16:45:05Z", "resp": "/wYAAFMyc1R3TwBSBQBH1aW01wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjQ1MDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeCADpmAAwxWqAROhMA9EABNVowDQYJKoZIhvcNAQELBQADggEBAAzwED88ngiFj3hLzIejZ8DE/ppticX0vgAjUM8oXjDjwNXpSQ5xdXHsDk4RAYAVpNyiAfL2fapwz91g3JuZot6npp8smtZC5D0YMKK8iMjrx2aMqVyv+ai/33WG8PRWpBNzSTYaLhlBFjhUrx8HDu97ozNbmfgDWzRS1LqkJRa5YXvkyppqYTFSX73DV9R9tOVOwZ0x5WEKst9IJ+88mXsOBGyuye2Gh9RK6KsLgaOwiD9FBf18WRKIixeVM1Y/xHc/iwFPi8k3Z6hZ6gHX8NboQ/djCyzYVSWsUedTo/62uuagHPRWoYci4HQl4bSFXfcEO/EkWGnqkWBHfYZ4soigggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSTaMQIIGjAYSTGoEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUa0VAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSywNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm2sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`, nil, nil, }, { "No response in cache for UserA1 -- should be allowed connection", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, "", nil, nil, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") c := []byte(test.cachedResponse) err := writeCacheFile("", c) if err != nil { t.Fatal(err) } content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPResponseCacheLocalStoreOption is test of default and non-default local_store option func TestOCSPResponseCacheLocalStoreOpt(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option cachedResponse string err error rerr error storeLocation string }{ { "Test load from non-default local store _custom_; connect will reject only if cache file found and loaded", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local local_store: "_custom_" preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, ` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-05-29T17:56:45Z", "resp_status": "revoked", "resp_expires": "2023-05-29T17:56:49Z", "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), "_custom_", }, { "Test load from default local store when \"\" set; connect will reject only if cache file found and loaded", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check but allow when CA is unreachable ocsp_peer: { verify: true ca_timeout: 0.5 allow_when_ca_unreachable: true } } # preserve revoked true ocsp_cache: { type: local local_store: "" preserve_revoked: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, ` { "5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=": { "subject": "CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US", "cached_at": "2023-05-29T17:56:45Z", "resp_status": "revoked", "resp_expires": "2023-05-29T17:56:49Z", "resp": "/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=" } }`, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), "_rc_", }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, test.storeLocation) c := []byte(test.cachedResponse) err := writeCacheFile(test.storeLocation, c) if err != nil { t.Fatal(err) } content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerIncrementalSaveLocalCache is test of timer-based response cache save as new entries added func TestOCSPPeerIncrementalSaveLocalCache(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", ocsp.Good) var fi os.FileInfo var err error for _, test := range []struct { name string config string opts [][]nats.Option err error rerr error configure func() }{ { "Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } # Local cache with custom save_interval for testability ocsp_cache: { type: local # Save if dirty ever 1 second save_interval: 1 } `, [][]nats.Option{ { nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, { nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { // Cleanup any previous test that saved a local cache deleteLocalStore(t, "") fi, err = statCacheFile("") if err != nil && fi != nil && fi.Size() != 0 { t.Fatalf("Expected no local cache file, got a FileInfo with size %d", fi.Size()) } test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Connect with UserA1 client and get a CA Response nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts[0]...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } nc.Close() time.Sleep(2 * time.Second) fi, err = statCacheFile("") if err == nil && fi != nil && fi.Size() > 0 { // good } else { if err != nil { t.Fatalf("Expected an extant local cache file, got error: %v", err) } if fi != nil { t.Fatalf("Expected non-zero size local cache file, got a FileInfo with size %d", fi.Size()) } } firstFi := fi // Connect with UserB1 client and get another CA Response nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts[1]...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } nc.Close() time.Sleep(2 * time.Second) fi, err = statCacheFile("") if err == nil && fi != nil && fi.Size() > firstFi.Size() { // good } else { if err != nil { t.Fatalf("Expected an extant local cache file, got error: %v", err) } if fi != nil { t.Fatalf("Expected non-zero size local cache file with more bytes, got a FileInfo with size %d", fi.Size()) } } }) } } func statCacheFile(dir string) (os.FileInfo, error) { if dir == "" { dir = "_rc_" } return os.Stat(filepath.Join(dir, "cache.json")) } // TestOCSPPeerUndelegatedCAResponseSigner func TestOCSPPeerUndelegatedCAResponseSigner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1Undelegated(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so unvalidated clients can't connect ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerDelegatedCAResponseSigner func TestOCSPPeerDelegatedCAResponseSigner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so unvalidated clients can't connect ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerBadDelegatedCAResponseSigner func TestOCSPPeerBadDelegatedCAResponseSigner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) intermediateCA1Responder := NewOCSPResponderBadDelegateIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "mTLS OCSP peer check on inbound client connection, responder is not a legal delegate", ` port: -1 # Cache configuration is default tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Turn on CA OCSP check so unvalidated clients can't connect ocsp_peer: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, errors.New("remote error: tls: bad certificate"), errors.New("expect error"), func() {}, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() }) } } // TestOCSPPeerNextUpdateUnset is test of scenario when responder does not set NextUpdate and cache TTL option is used func TestOCSPPeerNextUpdateUnset(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" intermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, "127.0.0.1:18888", 0, "") intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error expectedMisses int64 configure func() }{ { "TTL set to 4 seconds with second client connection leveraging cache from first client connect", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 0 cache_ttl_when_next_update_unset: 4 } } # Short form configuration, local as default ocsp_cache: true `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, 2, func() {}, }, { "TTL set to 1 seconds with second client connection not leveraging cache items from first client connect", ` port: -1 http_port: 8222 tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 0 cache_ttl_when_next_update_unset: 1 } } # Short form configuration, local as default ocsp_cache: true `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, 3, func() {}, }, } { t.Run(test.name, func(t *testing.T) { // Cleanup any previous test that saved a local cache deleteLocalStore(t, "") test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } nc.Close() // Wait interval shorter than first test, and longer than second test time.Sleep(2 * time.Second) nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() v := monitorGetVarzHelper(t, 8222) if v.OCSPResponseCache == nil { t.Fatalf("Expected OCSP statistics to be in varz") } if v.OCSPResponseCache.Misses != test.expectedMisses || v.OCSPResponseCache.Responses != 2 { t.Errorf("Expected cache misses to be %d and cache items to be 2, got %d and %d", test.expectedMisses, v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses) } }) } } func TestOCSPMonitoringPort(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() rootCAResponder := NewOCSPResponderRootCA(t) rootCAResponderURL := fmt.Sprintf("http://%s", rootCAResponder.Addr) defer rootCAResponder.Shutdown(ctx) SetOCSPStatus(t, rootCAResponderURL, "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", ocsp.Good) respCertPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem" respKeyPEM := "configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem" issuerCertPEM := "configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem" intermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, "127.0.0.1:18888", 0, "") intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", ocsp.Good) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem", ocsp.Good) for _, test := range []struct { name string config string opts []nats.Option err error rerr error }{ { "https with ocsp_peer", ` net: 127.0.0.1 port: -1 https: -1 # Short form configuration ocsp_cache: true store_dir = %s tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true # Long form configuration ocsp_peer: { verify: true ca_timeout: 5 allowed_clockskew: 30 } } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, }, { "https with just ocsp", ` net: 127.0.0.1 port: -1 https: -1 ocsp { mode = always url = http://127.0.0.1:18888 } store_dir = %s tls: { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem"), nats.RootCAs("./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem"), nats.ErrorHandler(noOpErrHandler), }, nil, nil, }, } { t.Run(test.name, func(t *testing.T) { deleteLocalStore(t, "") content := test.config conf := createConfFile(t, []byte(fmt.Sprintf(content, t.TempDir()))) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } // Make request to the HTTPS port using the client cert. tlsConfig := &tls.Config{} clientCertFile := "./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem" clientKeyFile := "./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem" cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) if err != nil { t.Fatal(err) } tlsConfig.Certificates = []tls.Certificate{cert} caCertFile := "./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" caCert, err := os.ReadFile(caCertFile) if err != nil { t.Fatal(err) } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig.RootCAs = caCertPool hc := &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsConfig, }, } resp, err := hc.Get("https://" + s.MonitorAddr().String()) if err != nil { t.Fatal(err) } defer resp.Body.Close() if resp.StatusCode != 200 { t.Errorf("Unexpected status: %v", resp.Status) } }) } } nats-server-2.10.27/test/ocsp_test.go000066400000000000000000003306151477524627100174740ustar00rootroot00000000000000// Copyright 2021-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bytes" "context" "crypto/tls" "fmt" "os" "path/filepath" "testing" "time" . "github.com/nats-io/nats-server/v2/internal/ocsp" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "golang.org/x/crypto/ocsp" ) func TestOCSPAlwaysMustStapleAndShutdown(t *testing.T) { // Certs that have must staple will auto shutdown the server. const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) opts := server.Options{} opts.Host = "127.0.0.1" opts.NoLog = true opts.NoSigs = true opts.MaxControlLine = 4096 opts.Port = -1 opts.TLSCert = serverCert opts.TLSKey = serverKey opts.TLSCaCert = caCert opts.TLSTimeout = 5 tcOpts := &server.TLSConfigOpts{ CertFile: opts.TLSCert, KeyFile: opts.TLSKey, CaFile: opts.TLSCaCert, Timeout: opts.TLSTimeout, } tlsConf, err := server.GenTLSConfig(tcOpts) if err != nil { t.Fatal(err) } opts.TLSConfig = tlsConf opts.OCSPConfig = &server.OCSPConfig{ Mode: server.OCSPModeAlways, OverrideURLs: []string{addr}, } srv := RunServer(&opts) defer srv.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { return fmt.Errorf("invalid staple") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // The server will shutdown because the server becomes revoked // and the policy is to always must-staple. The OCSP Responder // instructs the NATS Server to fetch OCSP Staples every 2 seconds. time.Sleep(2 * time.Second) SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) time.Sleep(2 * time.Second) // Should be connection refused since server will abort now. _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nats.ErrNoServers { t.Errorf("Expected connection refused") } // Verify that the server finishes shutdown srv.WaitForShutdown() } func TestOCSPMustStapleShutdown(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-status-request-cert.pem" serverKey = "configs/certs/ocsp/server-status-request-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) opts := server.Options{} opts.Host = "127.0.0.1" opts.NoLog = true opts.NoSigs = true opts.MaxControlLine = 4096 opts.Port = -1 opts.TLSCert = serverCert opts.TLSKey = serverKey opts.TLSCaCert = caCert opts.TLSTimeout = 5 tlsConfigOpts := &server.TLSConfigOpts{ CertFile: opts.TLSCert, KeyFile: opts.TLSKey, CaFile: opts.TLSCaCert, Timeout: opts.TLSTimeout, } tlsConf, err := server.GenTLSConfig(tlsConfigOpts) if err != nil { t.Fatal(err) } opts.TLSConfig = tlsConf opts.OCSPConfig = &server.OCSPConfig{ Mode: server.OCSPModeMust, OverrideURLs: []string{addr}, } srv := RunServer(&opts) defer srv.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { return fmt.Errorf("invalid staple") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // The server will shutdown because the server becomes revoked // and the policy is to always must-staple. The OCSP Responder // instructs the NATS Server to fetch OCSP Staples every 2 seconds. time.Sleep(2 * time.Second) SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) time.Sleep(2 * time.Second) // Should be connection refused since server will abort now. _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nats.ErrNoServers { t.Errorf("Expected connection refused") } } func TestOCSPMustStapleAutoDoesNotShutdown(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) content := ` port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { t.Errorf("Expected valid OCSP staple status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // The server will shutdown because the server becomes revoked // and the policy is to always must-staple. The OCSP Responder // instructs the NATS Server to fetch OCSP Staples every 2 seconds. time.Sleep(2 * time.Second) SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) time.Sleep(2 * time.Second) // Should not be connection refused, the client will continue running and // be served the stale OCSP staple instead. nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Revoked { t.Errorf("Expected revoked status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() } func TestOCSPAutoWithoutMustStapleDoesNotShutdownOnRevoke(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) opts := server.Options{} opts.Host = "127.0.0.1" opts.NoLog = true opts.NoSigs = true opts.MaxControlLine = 4096 opts.Port = -1 opts.TLSCert = serverCert opts.TLSKey = serverKey opts.TLSCaCert = caCert opts.TLSTimeout = 5 tlsConfigOpts := &server.TLSConfigOpts{ CertFile: opts.TLSCert, KeyFile: opts.TLSKey, CaFile: opts.TLSCaCert, Timeout: opts.TLSTimeout, } tlsConf, err := server.GenTLSConfig(tlsConfigOpts) if err != nil { t.Fatal(err) } opts.TLSConfig = tlsConf opts.OCSPConfig = &server.OCSPConfig{ Mode: server.OCSPModeAuto, OverrideURLs: []string{addr}, } srv := RunServer(&opts) defer srv.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return fmt.Errorf("Unexpected OCSP staple for certificate") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // Revoke the client certificate, nothing will happens since does // not have MustStaple. time.Sleep(2 * time.Second) SetOCSPStatus(t, addr, serverCert, ocsp.Revoked) time.Sleep(2 * time.Second) // Should not be connection refused since server will continue running. nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Errorf("Unexpected error: %s", err) } nc.Close() } func TestOCSPClient(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) ocspURL := fmt.Sprintf("http://%s", ocspr.Addr) defer ocspr.Shutdown(ctx) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "OCSP Stapling makes server fail to boot if status is unknown", ` port: -1 # Enable OCSP stapling with policy to honor must staple if present. ocsp: true tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() {}, }, { "OCSP Stapling ignored by default if server without must staple status", ` port: -1 ocsp: true tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "OCSP Stapling honored by default if server has must staple status", ` port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) }, }, { "OCSP Stapling can be disabled even if server has must staple status", ` port: -1 ocsp: false tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Revoked) }, }, } { t.Run(test.name, func(t *testing.T) { test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestOCSPReloadRotateTLSCertWithNoURL(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" updatedServerCert = "configs/certs/ocsp/server-status-request-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) content := ` port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { t.Errorf("Expected valid OCSP staple status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // Change the contents with another that will fail to get a staple // since it does not have an URL. content = ` port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-cert.pem" key_file: "configs/certs/ocsp/server-status-request-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } // Reload show warning because of cert missing OCSP Url so cannot be used // with OCSP stapling. if err := s.Reload(); err != nil { t.Fatal(err) } expectedErr := fmt.Errorf("missing OCSP response") // The server will not shutdown because the reload will fail. _, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { // The new certificate does not have OCSP Staples since // it could not fetch one from a OCSP server. if s.OCSPResponse == nil { return expectedErr } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != expectedErr { t.Fatalf("Unexpected error: %s", expectedErr) } } func TestOCSPReloadRotateTLSCertDisableMustStaple(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" updatedServerCert = "configs/certs/ocsp/server-status-request-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) storeDir := t.TempDir() originalContent := ` port: -1 store_dir: '%s' tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` content := fmt.Sprintf(originalContent, storeDir) conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() var staple []byte nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { staple = s.OCSPResponse resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { t.Errorf("Expected valid OCSP staple status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() files := []string{} err = filepath.Walk(storeDir+"/ocsp/", func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } files = append(files, path) return nil }) if err != nil { t.Fatal(err) } found := false for _, file := range files { data, err := os.ReadFile(file) if err != nil { t.Error(err) } if bytes.Equal(staple, data) { found = true } } if !found { t.Error("Could not find OCSP Staple") } // Change the contents with another that has OCSP Stapling disabled. updatedContent := ` port: -1 store_dir: '%s' tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` content = fmt.Sprintf(updatedContent, storeDir) if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } // The new certificate does not have must staple so they will be missing. time.Sleep(4 * time.Second) nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return fmt.Errorf("unexpected OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() // Re-enable OCSP Stapling content = fmt.Sprintf(originalContent, storeDir) if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } var newStaple []byte nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { newStaple = s.OCSPResponse resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { t.Errorf("Expected valid OCSP staple status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() // Confirm that it got a new staple. files = []string{} err = filepath.Walk(storeDir+"/ocsp/", func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } files = append(files, path) return nil }) if err != nil { t.Fatal(err) } found = false for _, file := range files { data, err := os.ReadFile(file) if err != nil { t.Error(err) } if bytes.Equal(newStaple, data) { found = true } } if !found { t.Error("Could not find OCSP Staple") } if bytes.Equal(staple, newStaple) { t.Error("Expected new OCSP Staple") } } func TestOCSPReloadRotateTLSCertEnableMustStaple(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) // Start without OCSP Stapling MustStaple content := ` port: -1 tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return fmt.Errorf("unexpected OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // Change the contents with another that has OCSP Stapling enabled. content = ` port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } // The new certificate does not have must staple so they will be missing. time.Sleep(2 * time.Second) nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { t.Errorf("Expected valid OCSP staple status") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() } func TestOCSPCluster(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // Seed server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { name: AB host: "127.0.0.1" advertise: 127.0.0.1 port: -1 pool_size: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // The rest srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { name: AB host: "127.0.0.1" advertise: 127.0.0.1 port: -1 pool_size: -1 routes: [ nats://127.0.0.1:%d ] connect_retries: 30 tls { cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() checkClusterFormed(t, srvA, srvB) // Revoke the seed server cluster certificate, following servers will not be able to verify connection. SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) // Original set of servers still can communicate to each other, even though the cert has been revoked. // NOTE: Should we unplug from the cluster in case our server is revoke and OCSP policy is always or must? checkClusterFormed(t, srvA, srvB) // Wait for seed server to notice that its certificate has been revoked, // so that new routes can't connect to it. time.Sleep(6 * time.Second) // Start another server against the seed server that has an invalid OCSP Staple srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" tls { cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { name: AB host: "127.0.0.1" advertise: 127.0.0.1 port: -1 pool_size: -1 routes: [ nats://127.0.0.1:%d ] connect_retries: 30 tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Cluster.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() cB, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() // There should be no connectivity between the clients due to the revoked staple. _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Fatal(err) } cB.Flush() resp, err := cC.Request("foo", nil, 2*time.Second) if err == nil { t.Errorf("Unexpected success, response: %+v", resp) } resp, err = cC.Request("bar", nil, 2*time.Second) if err == nil { t.Errorf("Unexpected success, response: %+v", resp) } // Switch the certs from the seed server to new ones that are not revoked, // this should restart OCSP Stapling for the cluster routes. srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { port: -1 pool_size: -1 compression: "disabled" name: AB host: "127.0.0.1" advertise: 127.0.0.1 connect_retries: 30 tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } // Wait to get a new OCSP Staple. time.Sleep(10 * time.Second) checkClusterFormed(t, srvA, srvB, srvC) // Now clients connect to C can communicate with B and A. _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } } func TestOCSPLeaf(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // LeafNode server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 # Leaf connection must present certs. verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // LeafNode that has the original as a remote and running // without OCSP Stapling for the leaf remote. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { # Cert without OCSP Stapling enabled is able to connect. cert_file: "configs/certs/ocsp/client-cert.pem" key_file: "configs/certs/ocsp/client-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ] } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() checkLeafNodeConnected(t, srvA) // Revoke the seed server cluster certificate, following servers will not be able to verify connection. SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) // Original set of servers still can communicate to each other via leafnode, even though the staple // for the leaf server has been revoked. checkLeafNodeConnected(t, srvA) // Wait for seed server to notice that its certificate has been revoked. time.Sleep(6 * time.Second) // Start another server against the seed server that has an revoked OCSP Staple. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" tls { cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ] } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() // There should be connectivity between the clients even if there is a revoked staple // from a leafnode connection. _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Fatal(err) } cB.Flush() _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("Expected success, got: %+v", err) } _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("Expected success, got: %+v", err) } // Switch the certs from the leafnode server to new ones that are not revoked, // this should restart OCSP Stapling for the leafnode server. srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } time.Sleep(4 * time.Second) // A <-> A _, err = cA.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // B <-> A _, err = cB.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // C <-> A _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // C <-> B via leafnode A _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } } func TestOCSPLeafNoVerify(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // LeafNode server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" # for this test, explicitly disable compression because we do it # in RunServer but here we do a config reload... compression: off tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 # Leaf server does not require certs for clients. verify: false } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // LeafNode remote that will connect to A and will not present certs. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ] } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() checkLeafNodeConnected(t, srvA) // Revoke the seed server cluster certificate, following servers will not be able to verify connection. SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) // Original set of servers still can communicate to each other, even though the cert has been revoked. checkLeafNodeConnected(t, srvA) // Wait for seed server to notice that its certificate has been revoked. time.Sleep(6 * time.Second) // Start another server against the seed server that has an revoked OCSP Staple. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" tls { cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 timeout: 5 } } ] } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() // There should be connectivity between the clients even if there is a revoked staple // from a leafnode connection. _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Fatal(err) } cB.Flush() _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("Expected success, got: %+v", err) } _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("Expected success, got: %+v", err) } // Switch the certs from the leafnode server to new ones that are not revoked, // this should restart OCSP Stapling for the leafnode server. srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" compression: off tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } time.Sleep(4 * time.Second) // A <-> A _, err = cA.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // B <-> A _, err = cB.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // C <-> A _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } // C <-> B via leafnode A _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } } func TestOCSPLeafVerifyLeafRemote(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() // LeafNode server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // LeafNode remote that will connect to A and will not present certs. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ] } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, _ := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() // Should not have been able to connect. checkLeafNodeConnections(t, srvA, 0) } func TestOCSPLeafVerifyAndMapLeafRemote(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/client-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() // LeafNode server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify_and_map: true } store_dir: '%s' leafnodes { host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify_and_map: true } } accounts: { leaf: { users: [ {user: "C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost server-status-request-url-04"} ] } client: { users: [ {user: "C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost client"} ] } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // LeafNode remote that will connect to A and will not present certs. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' leafnodes { remotes: [ { url: "tls://127.0.0.1:%d" tls { cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ] } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, _ := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() checkLeafNodeConnections(t, srvA, 1) } func TestOCSPGateway(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // LeafNode that has the original as a remote. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: B host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A" url: "nats://127.0.0.1:%d" tls { cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } }] tls { cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() waitForOutboundGateways(t, srvB, 1, 5*time.Second) // Revoke the seed server cluster certificate, following servers will not be able to verify connection. SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Revoked) // Original set of servers still can communicate to each other, even though the cert has been revoked. waitForOutboundGateways(t, srvA, 1, 5*time.Second) waitForOutboundGateways(t, srvB, 1, 5*time.Second) // Wait for gateway A to notice that its certificate has been revoked, // so that new gateways can't connect to it. time.Sleep(6 * time.Second) // Start another server against the seed server that has an invalid OCSP Staple srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" tls { cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: C host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{name: "A", url: "nats://127.0.0.1:%d" }] tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() // There should be no connectivity between the clients due to the revoked staple. _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Fatal(err) } cB.Flush() // Gateway C was not able to mesh with Gateway A because of the revoked OCSP staple // so these requests to A and B should fail. resp, err := cC.Request("foo", nil, 2*time.Second) if err == nil { t.Errorf("Unexpected success, response: %+v", resp) } // Make request to B resp, err = cC.Request("bar", nil, 2*time.Second) if err == nil { t.Errorf("Unexpected success, response: %+v", resp) } // Switch the certs from the seed server to new ones that are not revoked, // this should restart OCSP Stapling for the cluster routes. srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" tls { cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } time.Sleep(4 * time.Second) waitForOutboundGateways(t, srvA, 2, 5*time.Second) waitForOutboundGateways(t, srvB, 2, 5*time.Second) waitForOutboundGateways(t, srvC, 2, 5*time.Second) // Now clients connect to C can communicate with B and A. _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } } func TestOCSPGatewayIntermediate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", ocsp.Good) // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" ocsp: { mode: always url: %s } gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, intermediateCA1ResponderURL) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" ocsp: { mode: always url: %s } gateway { name: B host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A" url: "nats://127.0.0.1:%d" }] tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } ` srvConfB = fmt.Sprintf(srvConfB, intermediateCA1ResponderURL, optsA.Gateway.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() waitForOutboundGateways(t, srvB, 1, 5*time.Second) cB, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() } func TestOCSPGatewayReload(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) // Node A SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) // Node B SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) // Node C SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) // Node A rotated certs SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "AAA" ocsp { mode = always } store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // Gateway B connects to Gateway A. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "BBB" ocsp { mode = always } store_dir: '%s' gateway { name: B host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A" url: "nats://127.0.0.1:%d" }] tls { cert_file: "configs/certs/ocsp/server-status-request-url-04-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-04-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() // Wait for connectivity between A and B. waitForOutboundGateways(t, srvB, 1, 5*time.Second) // Gateway C also connects to Gateway A. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "CCC" ocsp { mode = always } store_dir: '%s' gateway { name: C host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{name: "A", url: "nats://127.0.0.1:%d" }] tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() //////////////////////////////////////////////////////////////////////////// // // // A and B are connected at this point and C is starting with certs that // // will be rotated, in v2.10.8 on reload now all OCSP monitors are also // // always restarted. // // // //////////////////////////////////////////////////////////////////////////// cB, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() cC, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsC.Port), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond(nil) }) if err != nil { t.Fatal(err) } cB.Flush() ///////////////////////////////////////////////////////////////////////////////// // // // Switch all the certs from server A, all OCSP monitors should be restarted // // so it should have new staples. // // // ///////////////////////////////////////////////////////////////////////////////// srvConfA = ` host: "127.0.0.1" port: -1 server_name: "AAA" ocsp { mode = always } store_dir: '%s' gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) if err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := srvA.Reload(); err != nil { t.Fatal(err) } waitForOutboundGateways(t, srvA, 2, 5*time.Second) waitForOutboundGateways(t, srvB, 2, 5*time.Second) waitForOutboundGateways(t, srvC, 2, 5*time.Second) // Now clients connect to C can communicate with B and A. _, err = cC.Request("foo", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } _, err = cC.Request("bar", nil, 2*time.Second) if err != nil { t.Errorf("%v", err) } } func TestOCSPCustomConfig(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) ocspURL := fmt.Sprintf("http://%s", ocspr.Addr) defer ocspr.Shutdown(ctx) var ( errExpectedNoStaple = fmt.Errorf("expected no staple") errMissingStaple = fmt.Errorf("missing OCSP Staple from server") ) for _, test := range []struct { name string config string opts []nats.Option err error rerr error configure func() }{ { "OCSP Stapling in auto mode makes server fail to boot if status is revoked", ` port: -1 ocsp { mode: auto } tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return errExpectedNoStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Revoked) }, }, { "OCSP Stapling must staple ignored if disabled with ocsp: false", ` port: -1 ocsp: false tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return errExpectedNoStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) }, }, { "OCSP Stapling must staple ignored if disabled with ocsp mode never", ` port: -1 ocsp: { mode: never } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return errExpectedNoStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) }, }, { "OCSP Stapling in always mode fetches a staple even if cert does not have one", ` port: -1 ocsp { mode: always url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "OCSP Stapling in must staple mode does not fetch staple if there is no must staple flag", ` port: -1 ocsp { mode: must url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return errExpectedNoStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "OCSP Stapling in must staple mode fetches staple if there is a must staple flag", ` port: -1 ocsp { mode: must url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), }, nil, nil, func() { SetOCSPStatus(t, ocspURL, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) }, }, } { t.Run(test.name, func(t *testing.T) { test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestOCSPCustomConfigReloadDisable(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) // Start with server without OCSP Stapling MustStaple content := ` port: -1 ocsp: { mode: always, url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // Change and disable OCSP Stapling. content = ` port: -1 ocsp: { mode: never } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } // The new certificate has must staple but OCSP Stapling is disabled. time.Sleep(2 * time.Second) nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return fmt.Errorf("unexpected OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() } func TestOCSPCustomConfigReloadEnable(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" updatedServerCert = "configs/certs/ocsp/server-status-request-url-01-cert.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) SetOCSPStatus(t, addr, updatedServerCert, ocsp.Good) // Start with server without OCSP Stapling MustStaple content := ` port: -1 ocsp: { mode: never, url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse != nil { return fmt.Errorf("unexpected OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() // Change and disable OCSP Stapling. content = ` port: -1 ocsp: { mode: always, url: "http://127.0.0.1:8888" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } ` if err := os.WriteFile(conf, []byte(content), 0666); err != nil { t.Fatalf("Error writing config: %v", err) } if err := s.Reload(); err != nil { t.Fatal(err) } time.Sleep(2 * time.Second) nc, err = nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple!") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } nc.Close() } func TestOCSPTLSConfigNoLeafSet(t *testing.T) { o := DefaultTestOptions o.HTTPHost = "127.0.0.1" o.HTTPSPort = -1 o.TLSConfig = &tls.Config{ServerName: "localhost"} cert, err := tls.LoadX509KeyPair("configs/certs/server-cert.pem", "configs/certs/server-key.pem") if err != nil { t.Fatalf("Got error reading certificates: %s", err) } o.TLSConfig.Certificates = []tls.Certificate{cert} s := RunServer(&o) s.Shutdown() } func TestOCSPSuperCluster(t *testing.T) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponder(t, caCert, caKey) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-01-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-02-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-03-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-04-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-05-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-06-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-07-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-status-request-url-08-cert.pem", ocsp.Good) SetOCSPStatus(t, addr, "configs/certs/ocsp/server-cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() storeDirD := t.TempDir() // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "A" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { name: A host: "127.0.0.1" advertise: 127.0.0.1 port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // Server that has the original as a cluster. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "B" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-01-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-01-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' cluster { name: A host: "127.0.0.1" advertise: 127.0.0.1 port: -1 routes: [ nats://127.0.0.1:%d ] tls { cert_file: "configs/certs/ocsp/server-status-request-url-02-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-02-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } } gateway { name: A host: "127.0.0.1" advertise: "127.0.0.1" port: -1 tls { cert_file: "configs/certs/ocsp/server-status-request-url-03-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-03-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() // Start another server that will make connect as a gateway to cluster A. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "C" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-05-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-05-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: C host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A", urls: ["nats://127.0.0.1:%d"] tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } }] tls { cert_file: "configs/certs/ocsp/server-status-request-url-06-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-06-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() // Check that server is connected to any server from the other cluster. checkClusterFormed(t, srvA, srvB) waitForOutboundGateways(t, srvC, 1, 5*time.Second) // Start one more server that will become another gateway. srvConfD := ` host: "127.0.0.1" port: -1 server_name: "D" ocsp { mode: "auto", url: "%s" } tls { cert_file: "configs/certs/ocsp/server-status-request-url-07-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-07-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 } store_dir: '%s' gateway { name: D host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A", urls: ["nats://127.0.0.1:%d"] tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 }}, { name: "C", urls: ["nats://127.0.0.1:%d"] #################################################################### ## TEST NOTE: This cert does not have an OCSP Staple intentionally## #################################################################### tls { ca_file: "configs/certs/ocsp/ca-cert.pem" cert_file: "configs/certs/ocsp/server-cert.pem" key_file: "configs/certs/ocsp/server-key.pem" timeout: 5 }} ] tls { cert_file: "configs/certs/ocsp/server-status-request-url-08-cert.pem" key_file: "configs/certs/ocsp/server-status-request-url-08-key.pem" ca_file: "configs/certs/ocsp/ca-cert.pem" timeout: 5 verify: true } } ` srvConfD = fmt.Sprintf(srvConfD, addr, storeDirD, optsA.Gateway.Port, optsC.Gateway.Port) conf = createConfFile(t, []byte(srvConfD)) srvD, _ := RunServerWithConfig(conf) defer srvD.Shutdown() // There should be a single gateway here because one of the gateway connections does not have a OCSP staple. waitForOutboundGateways(t, srvD, 1, 10*time.Second) // Connect to cluster A using server B. cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() // Connects to cluster C using server C. cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond([]byte("From Server A")) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond([]byte("From Server B")) }) if err != nil { t.Fatal(err) } cB.Flush() // Confirm that a message from server C can flow back to server A via gateway.. var ( resp *nats.Msg lerr error ) for i := 0; i < 10; i++ { resp, lerr = cC.Request("foo", nil, 500*time.Millisecond) if lerr != nil { continue } got := string(resp.Data) expected := "From Server A" if got != expected { t.Fatalf("Expected %v, got: %v", expected, got) } // Make request to B resp, lerr = cC.Request("bar", nil, 500*time.Millisecond) if lerr != nil { continue } got = string(resp.Data) expected = "From Server B" if got != expected { t.Errorf("Expected %v, got: %v", expected, got) } lerr = nil break } if lerr != nil { t.Errorf("Unexpected error: %v", lerr) } if n := srvD.NumOutboundGateways(); n > 1 { t.Errorf("Expected single gateway, got: %v", n) } } func TestOCSPLocalIssuerDetermination(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) // Test constants ocspURL := intermediateCA1ResponderURL clientTrustBundle := "configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem" serverCert := "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" var ( errMissingStaple = fmt.Errorf("missing OCSP Staple from server") ) for _, test := range []struct { name string config string opts []nats.Option err error rerr error serverStart bool configure func() }{ { "Correct issuer configured in cert bundle", ` port: -1 ocsp { mode: always } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(clientTrustBundle), nats.ErrorHandler(noOpErrHandler), }, nil, nil, true, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "Wrong issuer configured in cert bundle, server no start", ` port: -1 ocsp { mode: always } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(clientTrustBundle), nats.ErrorHandler(noOpErrHandler), }, nil, nil, false, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "Issuer configured in CA bundle only, configuration 1", ` port: -1 ocsp { mode: always } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(clientTrustBundle), nats.ErrorHandler(noOpErrHandler), }, nil, nil, true, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "Issuer configured in CA bundle only, configuration 2", ` port: -1 ocsp { mode: always } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(clientTrustBundle), nats.ErrorHandler(noOpErrHandler), }, nil, nil, true, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, { "Issuer configured in CA bundle only, configuration 3", ` port: -1 ocsp { mode: always } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem" timeout: 5 } `, []nats.Option{ nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return errMissingStaple } return nil }, }), nats.ClientCert("./configs/certs/ocsp/client-cert.pem", "./configs/certs/ocsp/client-key.pem"), nats.RootCAs(clientTrustBundle), nats.ErrorHandler(noOpErrHandler), }, nil, nil, true, func() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) }, }, } { t.Run(test.name, func(t *testing.T) { defer func() { r := recover() if r != nil && test.serverStart { t.Fatalf("Expected server start, unexpected panic: %v", r) } if r == nil && !test.serverStart { t.Fatalf("Expected server to not start and panic thrown") } }() test.configure() content := test.config conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) // server may not start for some tests if s != nil { defer s.Shutdown() } nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.opts...) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestMixedCAOCSPSuperCluster(t *testing.T) { const ( caCert = "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() intermediateCA1Responder := NewOCSPResponderIntermediateCA1(t) intermediateCA1ResponderURL := fmt.Sprintf("http://%s", intermediateCA1Responder.Addr) defer intermediateCA1Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA1ResponderURL, "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", ocsp.Good) intermediateCA2Responder := NewOCSPResponderIntermediateCA2(t) intermediateCA2ResponderURL := fmt.Sprintf("http://%s", intermediateCA2Responder.Addr) defer intermediateCA2Responder.Shutdown(ctx) SetOCSPStatus(t, intermediateCA2ResponderURL, "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem", ocsp.Good) // Store Dirs storeDirA := t.TempDir() storeDirB := t.TempDir() storeDirC := t.TempDir() // Gateway server configuration srvConfA := ` host: "127.0.0.1" port: -1 server_name: "A" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } store_dir: '%s' cluster { name: A host: "127.0.0.1" advertise: 127.0.0.1 port: -1 tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } gateway { name: A host: "127.0.0.1" port: -1 advertise: "127.0.0.1" tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true } } ` srvConfA = fmt.Sprintf(srvConfA, storeDirA) sconfA := createConfFile(t, []byte(srvConfA)) srvA, optsA := RunServerWithConfig(sconfA) defer srvA.Shutdown() // Server that has the original as a cluster. srvConfB := ` host: "127.0.0.1" port: -1 server_name: "B" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } store_dir: '%s' cluster { name: A host: "127.0.0.1" advertise: 127.0.0.1 port: -1 routes: [ nats://127.0.0.1:%d ] tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } } gateway { name: A host: "127.0.0.1" advertise: "127.0.0.1" port: -1 tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true } } ` srvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port) conf := createConfFile(t, []byte(srvConfB)) srvB, optsB := RunServerWithConfig(conf) defer srvB.Shutdown() // Client connects to server A. cA, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsA.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cA.Close() // Start another server that will make connect as a gateway to cluster A but with different CA issuer. srvConfC := ` host: "127.0.0.1" port: -1 server_name: "C" ocsp { mode: "always" } tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } store_dir: '%s' gateway { name: C host: "127.0.0.1" advertise: "127.0.0.1" port: -1 gateways: [{ name: "A", urls: ["nats://127.0.0.1:%d"] tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 } }] tls { cert_file: "configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem" key_file: "configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem" ca_file: "configs/certs/ocsp_peer/mini-ca/root/root_cert.pem" timeout: 5 verify: true } } ` srvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port) conf = createConfFile(t, []byte(srvConfC)) srvC, optsC := RunServerWithConfig(conf) defer srvC.Shutdown() // Check that server is connected to any server from the other cluster. checkClusterFormed(t, srvA, srvB) waitForOutboundGateways(t, srvC, 1, 5*time.Second) // Connect to cluster A using server B. cB, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsB.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cB.Close() // Connects to cluster C using server C. cC, err := nats.Connect(fmt.Sprintf("tls://127.0.0.1:%d", optsC.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { if s.OCSPResponse == nil { return fmt.Errorf("missing OCSP Staple from server") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer cC.Close() _, err = cA.Subscribe("foo", func(m *nats.Msg) { m.Respond([]byte("From Server A")) }) if err != nil { t.Errorf("%v", err) } cA.Flush() _, err = cB.Subscribe("bar", func(m *nats.Msg) { m.Respond([]byte("From Server B")) }) if err != nil { t.Fatal(err) } cB.Flush() // Confirm that a message from server C can flow back to server A via gateway.. var ( resp *nats.Msg lerr error ) for i := 0; i < 10; i++ { resp, lerr = cC.Request("foo", nil, 500*time.Millisecond) if lerr != nil { continue } got := string(resp.Data) expected := "From Server A" if got != expected { t.Fatalf("Expected %v, got: %v", expected, got) } // Make request to B resp, lerr = cC.Request("bar", nil, 500*time.Millisecond) if lerr != nil { continue } got = string(resp.Data) expected = "From Server B" if got != expected { t.Errorf("Expected %v, got: %v", expected, got) } lerr = nil break } if lerr != nil { t.Errorf("Unexpected error: %v", lerr) } } func TestOCSPResponderHTTPMethods(t *testing.T) { t.Run("prefer get", func(t *testing.T) { testOCSPResponderHTTPMethods(t, "GET") }) t.Run("prefer post", func(t *testing.T) { testOCSPResponderHTTPMethods(t, "POST") }) t.Run("all methods failing", func(t *testing.T) { testOCSPResponderFailing(t, "TEST") }) } func testOCSPResponderHTTPMethods(t *testing.T, method string) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) // Add another responder that fails. badaddr := "http://127.0.0.1:8889" badocsp := NewOCSPResponderCustomAddress(t, caCert, caKey, badaddr) defer badocsp.Shutdown(ctx) opts := server.Options{} opts.Host = "127.0.0.1" opts.NoLog = true opts.NoSigs = true opts.MaxControlLine = 4096 opts.Port = -1 opts.TLSCert = serverCert opts.TLSKey = serverKey opts.TLSCaCert = caCert opts.TLSTimeout = 5 tcOpts := &server.TLSConfigOpts{ CertFile: opts.TLSCert, KeyFile: opts.TLSKey, CaFile: opts.TLSCaCert, Timeout: opts.TLSTimeout, } tlsConf, err := server.GenTLSConfig(tcOpts) if err != nil { t.Fatal(err) } opts.TLSConfig = tlsConf opts.OCSPConfig = &server.OCSPConfig{ Mode: server.OCSPModeAlways, OverrideURLs: []string{badaddr, addr}, } srv := RunServer(&opts) defer srv.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), nats.Secure(&tls.Config{ VerifyConnection: func(s tls.ConnectionState) error { resp, err := GetOCSPStatus(s) if err != nil { return err } if resp.Status != ocsp.Good { return fmt.Errorf("invalid staple") } return nil }, }), nats.RootCAs(caCert), nats.ErrorHandler(noOpErrHandler), ) if err != nil { t.Fatal(err) } defer nc.Close() sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatal(err) } nc.Publish("foo", []byte("hello world")) nc.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatal(err) } nc.Close() } func testOCSPResponderFailing(t *testing.T, method string) { const ( caCert = "configs/certs/ocsp/ca-cert.pem" caKey = "configs/certs/ocsp/ca-key.pem" serverCert = "configs/certs/ocsp/server-cert.pem" serverKey = "configs/certs/ocsp/server-key.pem" ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method) defer ocspr.Shutdown(ctx) addr := fmt.Sprintf("http://%s", ocspr.Addr) SetOCSPStatus(t, addr, serverCert, ocsp.Good) opts := server.Options{} opts.Host = "127.0.0.1" opts.NoLog = true opts.NoSigs = true opts.MaxControlLine = 4096 opts.Port = -1 opts.TLSCert = serverCert opts.TLSKey = serverKey opts.TLSCaCert = caCert opts.TLSTimeout = 5 tcOpts := &server.TLSConfigOpts{ CertFile: opts.TLSCert, KeyFile: opts.TLSKey, CaFile: opts.TLSCaCert, Timeout: opts.TLSTimeout, } tlsConf, err := server.GenTLSConfig(tcOpts) if err != nil { t.Fatal(err) } opts.TLSConfig = tlsConf opts.OCSPConfig = &server.OCSPConfig{ Mode: server.OCSPModeAlways, OverrideURLs: []string{addr}, } expected := "bad OCSP status update for certificate at 'configs/certs/ocsp/server-cert.pem': " expected += "exhausted ocsp servers: non-ok http status on POST request (reqlen=68): 400\nnon-ok http status on GET request (reqlen=92): 400" _, err = server.NewServer(&opts) if err == nil { t.Error("Unexpected success setting up server") } else if err.Error() != expected { t.Errorf("Expected %q, got: %q", expected, err.Error()) } } nats-server-2.10.27/test/operator_test.go000066400000000000000000000461611477524627100203630ustar00rootroot00000000000000// Copyright 2018-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bytes" "fmt" "os" "strings" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) const ( testOpConfig = "./configs/operator.conf" testOpInlineConfig = "./configs/operator_inline.conf" ) // This matches ./configs/nkeys_jwts/test.seed // Test operator seed. var oSeed = []byte("SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU") // This is a signing key seed. var skSeed = []byte("SOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ") func checkKeys(t *testing.T, opts *server.Options, opc *jwt.OperatorClaims, expected int) { // We should have filled in the TrustedKeys here. if len(opts.TrustedKeys) != expected { t.Fatalf("Should have %d trusted keys, got %d", expected, len(opts.TrustedKeys)) } // Check that we properly placed all keys from the opc into TrustedKeys chkMember := func(s string) { for _, c := range opts.TrustedKeys { if s == c { return } } t.Fatalf("Expected %q to be in TrustedKeys", s) } chkMember(opc.Issuer) for _, sk := range opc.SigningKeys { chkMember(sk) } } // This will test that we enforce certain restrictions when you use trusted operators. // Like auth is always true, can't define accounts or users, required to define an account resolver, etc. func TestOperatorRestrictions(t *testing.T) { opts, err := server.ProcessConfigFile(testOpConfig) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoSigs = true if _, err := server.NewServer(opts); err != nil { t.Fatalf("Expected to create a server successfully") } // TrustedKeys get defined when processing from above, trying again with // same opts should not work. if _, err := server.NewServer(opts); err == nil { t.Fatalf("Expected an error with TrustedKeys defined") } // Must wipe and rebuild to succeed. wipeOpts := func() { opts.TrustedKeys = nil opts.Accounts = nil opts.Users = nil opts.Nkeys = nil } wipeOpts() opts.Accounts = []*server.Account{{Name: "TEST"}} if _, err := server.NewServer(opts); err == nil { t.Fatalf("Expected an error with Accounts defined") } wipeOpts() opts.Users = []*server.User{{Username: "TEST"}} if _, err := server.NewServer(opts); err == nil { t.Fatalf("Expected an error with Users defined") } wipeOpts() opts.Nkeys = []*server.NkeyUser{{Nkey: "TEST"}} if _, err := server.NewServer(opts); err == nil { t.Fatalf("Expected an error with Nkey Users defined") } wipeOpts() opts.AccountResolver = nil if _, err := server.NewServer(opts); err == nil { t.Fatalf("Expected an error without an AccountResolver defined") } } func TestOperatorConfig(t *testing.T) { opts, err := server.ProcessConfigFile(testOpConfig) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoSigs = true // Check we have the TrustedOperators if len(opts.TrustedOperators) != 1 { t.Fatalf("Expected to load the operator") } _, err = server.NewServer(opts) if err != nil { t.Fatalf("Expected to create a server: %v", err) } // We should have filled in the public TrustedKeys here. // Our master public key (issuer) plus the signing keys (3). checkKeys(t, opts, opts.TrustedOperators[0], 4) } func TestOperatorConfigInline(t *testing.T) { opts, err := server.ProcessConfigFile(testOpInlineConfig) if err != nil { t.Fatalf("Error processing config file: %v", err) } opts.NoSigs = true // Check we have the TrustedOperators if len(opts.TrustedOperators) != 1 { t.Fatalf("Expected to load the operator") } _, err = server.NewServer(opts) if err != nil { t.Fatalf("Expected to create a server: %v", err) } // We should have filled in the public TrustedKeys here. // Our master public key (issuer) plus the signing keys (3). checkKeys(t, opts, opts.TrustedOperators[0], 4) } func runOperatorServer(t *testing.T) (*server.Server, *server.Options) { return RunServerWithConfig(testOpConfig) } func publicKeyFromKeyPair(t *testing.T, pair nkeys.KeyPair) (pkey string) { var err error if pkey, err = pair.PublicKey(); err != nil { t.Fatalf("Expected no error %v", err) } return } func createAccountForOperatorKey(t *testing.T, s *server.Server, seed []byte) nkeys.KeyPair { t.Helper() okp, _ := nkeys.FromSeed(seed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) jwt, _ := nac.Encode(okp) if err := s.AccountResolver().Store(pub, jwt); err != nil { t.Fatalf("Account Resolver returned an error: %v", err) } return akp } func createAccount(t *testing.T, s *server.Server) (acc *server.Account, akp nkeys.KeyPair) { t.Helper() akp = createAccountForOperatorKey(t, s, oSeed) if pub, err := akp.PublicKey(); err != nil { t.Fatalf("Expected this to pass, got %v", err) } else if acc, err = s.LookupAccount(pub); err != nil { t.Fatalf("Error looking up account: %v", err) } return acc, akp } func createUserCreds(t *testing.T, s *server.Server, akp nkeys.KeyPair) nats.Option { t.Helper() opt, _ := createUserCredsOption(t, s, akp) return opt } func createUserCredsOption(t *testing.T, s *server.Server, akp nkeys.KeyPair) (nats.Option, string) { t.Helper() kp, _ := nkeys.CreateUser() pub, _ := kp.PublicKey() nuc := jwt.NewUserClaims(pub) ujwt, err := nuc.Encode(akp) if err != nil { t.Fatalf("Error generating user JWT: %v", err) } userCB := func() (string, error) { return ujwt, nil } sigCB := func(nonce []byte) ([]byte, error) { sig, _ := kp.Sign(nonce) return sig, nil } return nats.UserJWT(userCB, sigCB), pub } func TestOperatorServer(t *testing.T) { s, opts := runOperatorServer(t) defer s.Shutdown() url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) if _, err := nats.Connect(url); err == nil { t.Fatalf("Expected to fail with no credentials") } _, akp := createAccount(t, s) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Close() // Now create an account from another operator, this should fail. okp, _ := nkeys.CreateOperator() seed, _ := okp.Seed() akp = createAccountForOperatorKey(t, s, seed) _, err = nats.Connect(url, createUserCreds(t, s, akp)) if err == nil { t.Fatalf("Expected error on connect") } // The account should not be in memory either if v, err := s.LookupAccount(publicKeyFromKeyPair(t, akp)); err == nil { t.Fatalf("Expected account to NOT be in memory: %v", v) } } func TestOperatorSystemAccount(t *testing.T) { s, _ := runOperatorServer(t) defer s.Shutdown() // Create an account from another operator, this should fail if used as a system account. okp, _ := nkeys.CreateOperator() seed, _ := okp.Seed() akp := createAccountForOperatorKey(t, s, seed) if err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err == nil { t.Fatalf("Expected this to fail") } if acc := s.SystemAccount(); acc != nil { t.Fatalf("Expected no account to be set for system account") } acc, _ := createAccount(t, s) if err := s.SetSystemAccount(acc.Name); err != nil { t.Fatalf("Expected this succeed, got %v", err) } if sysAcc := s.SystemAccount(); sysAcc != acc { t.Fatalf("Did not get matching account for system account") } } func TestOperatorSigningKeys(t *testing.T) { s, opts := runOperatorServer(t) defer s.Shutdown() // Create an account with a signing key, not the master key. akp := createAccountForOperatorKey(t, s, skSeed) // Make sure we can set system account. if err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err != nil { t.Fatalf("Expected this succeed, got %v", err) } // Make sure we can create users with it too. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, akp)) if err != nil { t.Fatalf("Error on connect: %v", err) } nc.Close() } func TestOperatorMemResolverPreload(t *testing.T) { s, opts := RunServerWithConfig("./configs/resolver_preload.conf") defer s.Shutdown() // Make sure we can look up the account. acc, _ := s.LookupAccount("ADM2CIIL3RWXBA6T2HW3FODNCQQOUJEHHQD6FKCPVAMHDNTTSMO73ROX") if acc == nil { t.Fatalf("Expected to properly lookup account") } sacc := s.SystemAccount() if sacc == nil { t.Fatalf("Expected to have system account registered") } if sacc.Name != opts.SystemAccount { t.Fatalf("System account does not match, wanted %q, got %q", opts.SystemAccount, sacc.Name) } } func TestOperatorConfigReloadDoesntKillNonce(t *testing.T) { s, _ := runOperatorServer(t) defer s.Shutdown() if !s.NonceRequired() { t.Fatalf("Error nonce should be required") } if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } if !s.NonceRequired() { t.Fatalf("Error nonce should still be required after reload") } } func createAccountForConfig(t *testing.T) (string, nkeys.KeyPair) { t.Helper() okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) jwt, _ := nac.Encode(okp) return jwt, akp } func TestReloadDoesNotWipeAccountsWithOperatorMode(t *testing.T) { // We will run an operator mode server that forms a cluster. We will // make sure that a reload does not wipe account information. // We will force reload of auth by changing cluster auth timeout. // Create two accounts, system and normal account. sysJWT, sysKP := createAccountForConfig(t) sysPub, _ := sysKP.PublicKey() accJWT, accKP := createAccountForConfig(t) accPub, _ := accKP.PublicKey() cf := ` listen: 127.0.0.1:-1 cluster { name: "A" listen: 127.0.0.1:-1 pool_size: -1 compression: "disabled" authorization { timeout: 2.2 } %s } operator = "./configs/nkeys/op.jwt" system_account = "%s" resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" } ` contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) conf := createConfFile(t, []byte(contents)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Create a new server and route to main one. routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) conf2 := createConfFile(t, []byte(contents2)) s2, opts2 := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s, s2) // Create a client on the first server and subscribe. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, accKP)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() ch := make(chan bool) nc.Subscribe("foo", func(m *nats.Msg) { ch <- true }) nc.Flush() // Use this to check for message. checkForMsg := func() { t.Helper() select { case <-ch: case <-time.After(2 * time.Second): t.Fatal("Timeout waiting for message across route") } } // Wait for "foo" interest to be propagated to s2's account `accPub` checkSubInterest(t, s2, accPub, "foo", 2*time.Second) // Create second client and send message from this one. Interest should be here. url2 := fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) nc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP)) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() // Check that we can send messages. nc2.Publish("foo", nil) checkForMsg() // Now shutdown nc2 and srvA. nc2.Close() s2.Shutdown() // Now change config and do reload which will do an auth change. b, err := os.ReadFile(conf) if err != nil { t.Fatal(err) } newConf := bytes.Replace(b, []byte("2.2"), []byte("3.3"), 1) err = os.WriteFile(conf, newConf, 0644) if err != nil { t.Fatal(err) } // This will cause reloadAuthorization to kick in and reprocess accounts. s.Reload() s2, opts2 = RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s, s2) checkSubInterest(t, s2, accPub, "foo", 2*time.Second) // Reconnect and make sure this works. If accounts blown away this will fail. url2 = fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) nc2, err = nats.Connect(url2, createUserCreds(t, s2, accKP)) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() // Check that we can send messages. nc2.Publish("foo", nil) checkForMsg() } func TestReloadDoesUpdateAccountsWithMemoryResolver(t *testing.T) { // We will run an operator mode server with a memory resolver. // Reloading should behave similar to configured accounts. // Create two accounts, system and normal account. sysJWT, sysKP := createAccountForConfig(t) sysPub, _ := sysKP.PublicKey() accJWT, accKP := createAccountForConfig(t) accPub, _ := accKP.PublicKey() cf := ` listen: 127.0.0.1:-1 cluster { name: "A" listen: 127.0.0.1:-1 pool_size: -1 authorization { timeout: 2.2 } %s } operator = "./configs/nkeys/op.jwt" system_account = "%s" resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" } ` contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) conf := createConfFile(t, []byte(contents)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Create a client on the first server and subscribe. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(url, createUserCreds(t, s, accKP)) if err != nil { t.Fatalf("Error on connect: %v", err) } asyncErr := make(chan error, 1) nc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }) defer nc.Close() nc.Subscribe("foo", func(m *nats.Msg) {}) nc.Flush() // Now update and remove normal account and make sure we get disconnected. accJWT2, accKP2 := createAccountForConfig(t) accPub2, _ := accKP2.PublicKey() contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub2, accJWT2), "\n\t", "\n", -1) err = os.WriteFile(conf, []byte(contents), 0644) if err != nil { t.Fatal(err) } // This will cause reloadAuthorization to kick in and reprocess accounts. s.Reload() select { case err := <-asyncErr: if err != nats.ErrAuthorization { t.Fatalf("Expected ErrAuthorization, got %v", err) } case <-time.After(2 * time.Second): // Give it up to 2 sec. t.Fatal("Expected connection to be disconnected") } // Make sure we can lool up new account and not old one. if _, err := s.LookupAccount(accPub2); err != nil { t.Fatalf("Error looking up account: %v", err) } if _, err := s.LookupAccount(accPub); err == nil { t.Fatalf("Expected error looking up old account") } } func TestReloadFailsWithBadAccountsWithMemoryResolver(t *testing.T) { // Create two accounts, system and normal account. sysJWT, sysKP := createAccountForConfig(t) sysPub, _ := sysKP.PublicKey() // Create an expired account by hand here. We want to make sure we start up correctly // with expired or otherwise accounts with validation issues. okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() apub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(apub) nac.IssuedAt = time.Now().Add(-10 * time.Second).Unix() nac.Expires = time.Now().Add(-2 * time.Second).Unix() ajwt, err := nac.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } cf := ` listen: 127.0.0.1:-1 cluster { name: "A" listen: 127.0.0.1:-1 pool_size: -1 authorization { timeout: 2.2 } %s } operator = "./configs/nkeys/op.jwt" system_account = "%s" resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" } ` contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, apub, ajwt), "\n\t", "\n", -1) conf := createConfFile(t, []byte(contents)) s, _ := RunServerWithConfig(conf) defer s.Shutdown() // Now add in bogus account for second item and make sure reload fails. contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, "foo", "bar"), "\n\t", "\n", -1) err = os.WriteFile(conf, []byte(contents), 0644) if err != nil { t.Fatal(err) } if err := s.Reload(); err == nil { t.Fatalf("Expected fatal error with bad account on reload") } // Put it back with a normal account and reload should succeed. accJWT, accKP := createAccountForConfig(t) accPub, _ := accKP.PublicKey() contents = strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) err = os.WriteFile(conf, []byte(contents), 0644) if err != nil { t.Fatal(err) } if err := s.Reload(); err != nil { t.Fatalf("Got unexpected error on reload: %v", err) } } func TestConnsRequestDoesNotLoadAccountCheckingConnLimits(t *testing.T) { // Create two accounts, system and normal account. sysJWT, sysKP := createAccountForConfig(t) sysPub, _ := sysKP.PublicKey() // Do this account by hand to add in connection limits okp, _ := nkeys.FromSeed(oSeed) accKP, _ := nkeys.CreateAccount() accPub, _ := accKP.PublicKey() nac := jwt.NewAccountClaims(accPub) nac.Limits.Conn = 10 accJWT, _ := nac.Encode(okp) cf := ` listen: 127.0.0.1:-1 cluster { name: "A" listen: 127.0.0.1:-1 authorization { timeout: 2.2 } %s } operator = "./configs/nkeys/op.jwt" system_account = "%s" resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" } ` contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) conf := createConfFile(t, []byte(contents)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Create a new server and route to main one. routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), "\n\t", "\n", -1) conf2 := createConfFile(t, []byte(contents2)) s2, _ := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s, s2) // Make sure that we do not have the account loaded. // Just SYS and $G if nla := s.NumLoadedAccounts(); nla != 2 { t.Fatalf("Expected only 2 loaded accounts, got %d", nla) } if nla := s2.NumLoadedAccounts(); nla != 2 { t.Fatalf("Expected only 2 loaded accounts, got %d", nla) } // Now connect to first server on accPub. nc, err := nats.Connect(s.ClientURL(), createUserCreds(t, s, accKP)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Just wait for the request for connections to move to S2 and cause a fetch. // This is what we want to fix. time.Sleep(100 * time.Millisecond) // We should have 3 here for sure. if nla := s.NumLoadedAccounts(); nla != 3 { t.Fatalf("Expected 3 loaded accounts, got %d", nla) } // Now make sure that we still only have 2 loaded accounts on server 2. if nla := s2.NumLoadedAccounts(); nla != 2 { t.Fatalf("Expected only 2 loaded accounts, got %d", nla) } } nats-server-2.10.27/test/opts_test.go000066400000000000000000000024151477524627100175070ustar00rootroot00000000000000// Copyright 2015-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "testing" ) func TestServerConfig(t *testing.T) { srv, opts := RunServerWithConfig("./configs/override.conf") defer srv.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() sinfo := checkInfoMsg(t, c) if sinfo.MaxPayload != opts.MaxPayload { t.Fatalf("Expected max_payload from server, got %d vs %d", opts.MaxPayload, sinfo.MaxPayload) } } func TestTLSConfig(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls.conf") defer srv.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() sinfo := checkInfoMsg(t, c) if !sinfo.TLSRequired { t.Fatal("Expected TLSRequired to be true when configured") } } nats-server-2.10.27/test/pedantic_test.go000066400000000000000000000046661477524627100203230ustar00rootroot00000000000000// Copyright 2012-2019 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "testing" "github.com/nats-io/nats-server/v2/server" ) func runPedanticServer() *server.Server { opts := DefaultTestOptions opts.NoLog = false opts.Trace = true opts.Port = PROTO_TEST_PORT return RunServer(&opts) } func TestPedanticSub(t *testing.T) { s := runPedanticServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send := sendCommand(t, c) expect := expectCommand(t, c) doConnect(t, c, false, true, false) // Ping should still be same send("PING\r\n") expect(pongRe) // Test malformed subjects for SUB // Sub can contain wildcards, but // subject must still be legit. // Empty terminal token send("SUB foo. 1\r\n") expect(errRe) // Empty beginning token send("SUB .foo. 1\r\n") expect(errRe) // Empty middle token send("SUB foo..bar 1\r\n") expect(errRe) // Bad non-terminal FWC send("SUB foo.>.bar 1\r\n") buf := expect(errRe) // Check that itr is 'Invalid Subject' matches := errRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatal("Wanted one overall match") } if string(matches[0][1]) != "'Invalid Subject'" { t.Fatalf("Expected 'Invalid Subject', got %s", string(matches[0][1])) } } func TestPedanticPub(t *testing.T) { s := runPedanticServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send := sendCommand(t, c) expect := expectCommand(t, c) doConnect(t, c, false, true, false) // Ping should still be same send("PING\r\n") expect(pongRe) // Test malformed subjects for PUB // PUB subjects can not have wildcards // This will error in pedantic mode send("PUB foo.* 2\r\nok\r\n") expect(errRe) send("PUB foo.> 2\r\nok\r\n") expect(errRe) send("PUB foo. 2\r\nok\r\n") expect(errRe) send("PUB .foo 2\r\nok\r\n") expect(errRe) send("PUB foo..* 2\r\nok\r\n") expect(errRe) } nats-server-2.10.27/test/pid_test.go000066400000000000000000000022001477524627100172660ustar00rootroot00000000000000// Copyright 2012-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "os" "testing" ) func TestPidFile(t *testing.T) { opts := DefaultTestOptions file := createTempFile(t, "nats-server:pid_") file.Close() opts.PidFile = file.Name() s := RunServer(&opts) s.Shutdown() buf, err := os.ReadFile(opts.PidFile) if err != nil { t.Fatalf("Could not read pid_file: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length pid_file") } pid := 0 fmt.Sscanf(string(buf), "%d", &pid) if pid != os.Getpid() { t.Fatalf("Expected pid to be %d, got %d\n", os.Getpid(), pid) } } nats-server-2.10.27/test/ping_test.go000066400000000000000000000126621477524627100174640ustar00rootroot00000000000000// Copyright 2012-2021 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "crypto/tls" "fmt" "net" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) const ( PING_TEST_PORT = 9972 PING_INTERVAL = 50 * time.Millisecond PING_MAX = 2 ) func runPingServer() *server.Server { opts := DefaultTestOptions opts.Port = PING_TEST_PORT opts.PingInterval = PING_INTERVAL opts.MaxPingsOut = PING_MAX return RunServer(&opts) } func TestPingSentToTLSConnection(t *testing.T) { opts := DefaultTestOptions opts.Port = PING_TEST_PORT opts.PingInterval = PING_INTERVAL opts.MaxPingsOut = PING_MAX opts.TLSCert = "configs/certs/server-cert.pem" opts.TLSKey = "configs/certs/server-key.pem" opts.TLSCaCert = "configs/certs/ca.pem" tc := server.TLSConfigOpts{} tc.CertFile = opts.TLSCert tc.KeyFile = opts.TLSKey tc.CaFile = opts.TLSCaCert opts.TLSConfig, _ = server.GenTLSConfig(&tc) opts.TLSTimeout = 5 s := RunServer(&opts) defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) defer c.Close() checkInfoMsg(t, c) c = tls.Client(c, &tls.Config{InsecureSkipVerify: true}) tlsConn := c.(*tls.Conn) tlsConn.Handshake() cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v}\r\n", false, false, true) sendProto(t, c, cs) expect := expectCommand(t, c) // Expect the max to be delivered correctly.. for i := 0; i < PING_MAX; i++ { time.Sleep(PING_INTERVAL / 2) expect(pingRe) } // We should get an error from the server time.Sleep(PING_INTERVAL) expect(errRe) // Server should close the connection at this point.. time.Sleep(PING_INTERVAL) c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) var err error for { _, err = c.Write([]byte("PING\r\n")) if err != nil { break } } c.SetWriteDeadline(time.Time{}) if err == nil { t.Fatal("No error: Expected to have connection closed") } if ne, ok := err.(net.Error); ok && ne.Timeout() { t.Fatal("timeout: Expected to have connection closed") } } func TestPingInterval(t *testing.T) { s := runPingServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) defer c.Close() doConnect(t, c, false, false, false) expect := expectCommand(t, c) // Expect the max to be delivered correctly.. for i := 0; i < PING_MAX; i++ { time.Sleep(PING_INTERVAL / 2) expect(pingRe) } // We should get an error from the server time.Sleep(PING_INTERVAL) expect(errRe) // Server should close the connection at this point.. time.Sleep(PING_INTERVAL) c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) var err error for { _, err = c.Write([]byte("PING\r\n")) if err != nil { break } } c.SetWriteDeadline(time.Time{}) if err == nil { t.Fatal("No error: Expected to have connection closed") } if ne, ok := err.(net.Error); ok && ne.Timeout() { t.Fatal("timeout: Expected to have connection closed") } } func TestUnpromptedPong(t *testing.T) { s := runPingServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) defer c.Close() doConnect(t, c, false, false, false) expect := expectCommand(t, c) // Send lots of PONGs in a row... for i := 0; i < 100; i++ { c.Write([]byte("PONG\r\n")) } // The server should still send the max number of PINGs and then // close the connection. for i := 0; i < PING_MAX; i++ { time.Sleep(PING_INTERVAL / 2) expect(pingRe) } // We should get an error from the server time.Sleep(PING_INTERVAL) expect(errRe) // Server should close the connection at this point.. c.SetWriteDeadline(time.Now().Add(PING_INTERVAL)) var err error for { _, err = c.Write([]byte("PING\r\n")) if err != nil { break } } c.SetWriteDeadline(time.Time{}) if err == nil { t.Fatal("No error: Expected to have connection closed") } if ne, ok := err.(net.Error); ok && ne.Timeout() { t.Fatal("timeout: Expected to have connection closed") } } func TestPingSuppresion(t *testing.T) { pingInterval := 100 * time.Millisecond highWater := 130 * time.Millisecond opts := DefaultTestOptions opts.Port = PING_TEST_PORT opts.PingInterval = pingInterval opts.DisableShortFirstPing = true s := RunServer(&opts) defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PING_TEST_PORT) defer c.Close() connectTime := time.Now() send, expect := setupConn(t, c) expect(pingRe) pingTime := time.Since(connectTime) send("PONG\r\n") // Should be > 100 but less then 120(ish) if pingTime < pingInterval { t.Fatalf("pingTime too low: %v", pingTime) } // +5 is just for fudging in case things are slow in the testing system. if pingTime > highWater { t.Fatalf("pingTime too high: %v", pingTime) } time.Sleep(pingInterval / 2) // Sending a PING should suppress. send("PING\r\n") expect(pongRe) // This will wait for the time period where a PING should have fired // and been delivered. We expect nothing here since it should be suppressed. expectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond)) } nats-server-2.10.27/test/port_test.go000066400000000000000000000027241477524627100175110ustar00rootroot00000000000000// Copyright 2014-2019 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "net" "strconv" "testing" "github.com/nats-io/nats-server/v2/server" ) func TestResolveRandomPort(t *testing.T) { opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoSigs: true} s := RunServer(opts) defer s.Shutdown() addr := s.Addr() _, port, err := net.SplitHostPort(addr.String()) if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } portNum, err := strconv.Atoi(port) if err != nil { t.Fatalf("Expected no error: Got %v\n", err) } if portNum == server.DEFAULT_PORT { t.Fatalf("Expected server to choose a random port\nGot: %d", server.DEFAULT_PORT) } if portNum == server.RANDOM_PORT { t.Fatalf("Expected server to choose a random port\nGot: %d", server.RANDOM_PORT) } if opts.Port != portNum { t.Fatalf("Options port (%d) should have been overridden by chosen random port (%d)", opts.Port, portNum) } } nats-server-2.10.27/test/ports_test.go000066400000000000000000000132111477524627100176650ustar00rootroot00000000000000// Copyright 2018-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) // waits until a calculated list of listeners is resolved or a timeout func waitForFile(path string, dur time.Duration) ([]byte, error) { end := time.Now().Add(dur) for time.Now().Before(end) { if _, err := os.Stat(path); os.IsNotExist(err) { time.Sleep(25 * time.Millisecond) continue } else { return os.ReadFile(path) } } return nil, errors.New("Timeout") } func portFile(dirname string) string { return filepath.Join(dirname, fmt.Sprintf("%s_%d.ports", filepath.Base(os.Args[0]), os.Getpid())) } func TestPortsFile(t *testing.T) { portFileDir := t.TempDir() opts := DefaultTestOptions opts.PortsFileDir = portFileDir opts.Port = -1 opts.HTTPPort = -1 opts.ProfPort = -1 opts.Cluster.Port = -1 opts.Websocket.Port = -1 tc := &server.TLSConfigOpts{ CertFile: "./configs/certs/server-cert.pem", KeyFile: "./configs/certs/server-key.pem", } opts.Websocket.TLSConfig, _ = server.GenTLSConfig(tc) s := RunServer(&opts) // this for test cleanup in case we fail - will be ignored if server already shutdown defer s.Shutdown() ports := s.PortsInfo(5 * time.Second) if ports == nil { t.Fatal("services failed to start in 5 seconds") } // the pid file should be portsFile := portFile(portFileDir) if portsFile == "" { t.Fatal("Expected a ports file") } // try to read a file here - the file should be a json buf, err := waitForFile(portsFile, 5*time.Second) if err != nil { t.Fatalf("Could not read ports file: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length ports file") } readPorts := server.Ports{} json.Unmarshal(buf, &readPorts) if len(readPorts.Nats) == 0 || !strings.HasPrefix(readPorts.Nats[0], "nats://") { t.Fatal("Expected at least one nats url") } if len(readPorts.Monitoring) == 0 || !strings.HasPrefix(readPorts.Monitoring[0], "http://") { t.Fatal("Expected at least one monitoring url") } if len(readPorts.Cluster) == 0 || !strings.HasPrefix(readPorts.Cluster[0], "nats://") { t.Fatal("Expected at least one cluster listen url") } if len(readPorts.Profile) == 0 || !strings.HasPrefix(readPorts.Profile[0], "http://") { t.Fatal("Expected at least one profile listen url") } if len(readPorts.WebSocket) == 0 || !strings.HasPrefix(readPorts.WebSocket[0], "wss://") { t.Fatal("Expected at least one ws listen url") } // testing cleanup s.Shutdown() // if we called shutdown, the cleanup code should have kicked if _, err := os.Stat(portsFile); os.IsNotExist(err) { // good } else { t.Fatalf("the port file %s was not deleted", portsFile) } } // makes a temp directory with two directories 'A' and 'B' // the location of the ports file is changed from dir A to dir B. func TestPortsFileReload(t *testing.T) { tempDir := t.TempDir() // make child temp dir A dirA := filepath.Join(tempDir, "A") os.MkdirAll(dirA, 0777) // write the config file with a reference to A config := fmt.Sprintf(` ports_file_dir: "%s" port: -1 `, dirA) config = strings.Replace(config, "\\", "/", -1) confPath := filepath.Join(tempDir, fmt.Sprintf("%d.conf", os.Getpid())) if err := os.WriteFile(confPath, []byte(config), 0666); err != nil { t.Fatalf("Error writing ports file (%s): %v", confPath, err) } opts, err := server.ProcessConfigFile(confPath) if err != nil { t.Fatalf("Error processing the configuration: %v", err) } s := RunServer(opts) defer s.Shutdown() ports := s.PortsInfo(5 * time.Second) if ports == nil { t.Fatal("services failed to start in 5 seconds") } // get the ports file path name portsFileInA := portFile(dirA) // the file should be in dirA if !strings.HasPrefix(portsFileInA, dirA) { t.Fatalf("expected ports file to be in [%s] but was in [%s]", dirA, portsFileInA) } // wait for it buf, err := waitForFile(portsFileInA, 5*time.Second) if err != nil { t.Fatalf("Could not read ports file: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length ports file") } // change the configuration for the ports file to dirB dirB := filepath.Join(tempDir, "B") os.MkdirAll(dirB, 0777) config = fmt.Sprintf(` ports_file_dir: "%s" port: -1 `, dirB) config = strings.Replace(config, "\\", "/", -1) if err := os.WriteFile(confPath, []byte(config), 0666); err != nil { t.Fatalf("Error writing ports file (%s): %v", confPath, err) } // reload the server if err := s.Reload(); err != nil { t.Fatalf("error reloading server: %v", err) } // wait for the new file to show up portsFileInB := portFile(dirB) buf, err = waitForFile(portsFileInB, 5*time.Second) if !strings.HasPrefix(portsFileInB, dirB) { t.Fatalf("expected ports file to be in [%s] but was in [%s]", dirB, portsFileInB) } if err != nil { t.Fatalf("Could not read ports file: %v", err) } if len(buf) <= 0 { t.Fatal("Expected a non-zero length ports file") } // the file in dirA should have deleted if _, err := os.Stat(portsFileInA); os.IsNotExist(err) { // good } else { t.Fatalf("the port file %s was not deleted", portsFileInA) } } nats-server-2.10.27/test/proto_test.go000066400000000000000000000154511477524627100176710ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) const PROTO_TEST_PORT = 9922 func runProtoServer() *server.Server { opts := DefaultTestOptions opts.Port = PROTO_TEST_PORT opts.MaxControlLine = 256 opts.NoSystemAccount = true return RunServer(&opts) } func TestProtoBasics(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) expectMsgs := expectMsgsCommand(t, expect) // Ping send("PING\r\n") expect(pongRe) // Single Msg send("SUB foo 1\r\nPUB foo 5\r\nhello\r\n") matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "5", "hello") // 2 Messages send("SUB * 2\r\nPUB foo 2\r\nok\r\n") matches = expectMsgs(2) // Could arrive in any order checkMsg(t, matches[0], "foo", "", "", "2", "ok") checkMsg(t, matches[1], "foo", "", "", "2", "ok") } func TestProtoErr(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) // Make sure we get an error on bad proto send("ZZZ") expect(errRe) } func TestUnsubMax(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) expectMsgs := expectMsgsCommand(t, expect) send("SUB foo 22\r\n") send("UNSUB 22 2\r\n") for i := 0; i < 100; i++ { send("PUB foo 2\r\nok\r\n") } time.Sleep(50 * time.Millisecond) matches := expectMsgs(2) checkMsg(t, matches[0], "foo", "22", "", "2", "ok") checkMsg(t, matches[1], "foo", "22", "", "2", "ok") } func TestQueueSub(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) expectMsgs := expectMsgsCommand(t, expect) sent := 100 send("SUB foo qgroup1 22\r\n") send("SUB foo qgroup1 32\r\n") for i := 0; i < sent; i++ { send("PUB foo 2\r\nok\r\n") } // Wait for responses time.Sleep(250 * time.Millisecond) matches := expectMsgs(sent) sids := make(map[string]int) for _, m := range matches { sids[string(m[sidIndex])]++ } if len(sids) != 2 { t.Fatalf("Expected only 2 sids, got %d\n", len(sids)) } for k, c := range sids { if c < 35 { t.Fatalf("Expected ~50 (+-15) msgs for sid:'%s', got %d\n", k, c) } } } func TestMultipleQueueSub(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) expectMsgs := expectMsgsCommand(t, expect) sent := 100 send("SUB foo g1 1\r\n") send("SUB foo g1 2\r\n") send("SUB foo g2 3\r\n") send("SUB foo g2 4\r\n") for i := 0; i < sent; i++ { send("PUB foo 2\r\nok\r\n") } // Wait for responses time.Sleep(250 * time.Millisecond) matches := expectMsgs(sent * 2) sids := make(map[string]int) for _, m := range matches { sids[string(m[sidIndex])]++ } if len(sids) != 4 { t.Fatalf("Expected 4 sids, got %d\n", len(sids)) } for k, c := range sids { if c < 35 { t.Fatalf("Expected ~50 (+-15) msgs for '%s', got %d\n", k, c) } } } func TestPubToArgState(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) send("PUBS foo 2\r\nok\r\n") expect(errRe) } func TestSubToArgState(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) send("SUBZZZ foo 1\r\n") expect(errRe) } // Issue #63 func TestProtoCrash(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := sendCommand(t, c), expectCommand(t, c) checkInfoMsg(t, c) send("CONNECT {\"verbose\":true,\"tls_required\":false,\"user\":\"test\",\"pedantic\":true,\"pass\":\"password\"}") time.Sleep(100 * time.Millisecond) send("\r\n") expect(okRe) } // Issue #136 func TestDuplicateProtoSub(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) send("PING\r\n") expect(pongRe) send("SUB foo 1\r\n") send("SUB foo 1\r\n") ns := 0 for i := 0; i < 5; i++ { ns = int(s.NumSubscriptions()) if ns == 0 { time.Sleep(50 * time.Millisecond) } else { break } } if ns != 1 { t.Fatalf("Expected 1 subscription, got %d\n", ns) } } func TestIncompletePubArg(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) size := 10000 goodBuf := "" for i := 0; i < size; i++ { goodBuf += "A" } goodBuf += "\r\n" badSize := 3371 badBuf := "" for i := 0; i < badSize; i++ { badBuf += "B" } // Message is corrupted and since we are still reading from client, // next PUB accidentally becomes part of the payload of the // incomplete message thus breaking the protocol. badBuf2 := "" for i := 0; i < size; i++ { badBuf2 += "C" } badBuf2 += "\r\n" pub := "PUB example 10000\r\n" send(pub + goodBuf + pub + goodBuf + pub + badBuf + pub + badBuf2) expect(errRe) } func TestControlLineMaximums(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() send, expect := setupConn(t, c) pubTooLong := "PUB foo " for i := 0; i < 32; i++ { pubTooLong += "2222222222" } send(pubTooLong) expect(errRe) } func TestServerInfoWithClientAdvertise(t *testing.T) { opts := DefaultTestOptions opts.Port = PROTO_TEST_PORT opts.ClientAdvertise = "me:1" s := RunServer(&opts) defer s.Shutdown() c := createClientConn(t, opts.Host, PROTO_TEST_PORT) defer c.Close() buf := expectResult(t, c, infoRe) js := infoRe.FindAllSubmatch(buf, 1)[0][1] var sinfo server.Info err := json.Unmarshal(js, &sinfo) if err != nil { t.Fatalf("Could not unmarshal INFO json: %v\n", err) } if sinfo.Host != "me" || sinfo.Port != 1 { t.Fatalf("Expected INFO Host:Port to be me:1, got %s:%d", sinfo.Host, sinfo.Port) } } nats-server-2.10.27/test/route_discovery_test.go000066400000000000000000000450421477524627100217520ustar00rootroot00000000000000// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bufio" "encoding/json" "fmt" "io" "net" "net/http" "runtime" "strconv" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) func runSeedServer(t *testing.T) (*server.Server, *server.Options) { return RunServerWithConfig("./configs/seed.conf") } func runAuthSeedServer(t *testing.T) (*server.Server, *server.Options) { return RunServerWithConfig("./configs/auth_seed.conf") } func TestSeedFirstRouteInfo(t *testing.T) { s, opts := runSeedServer(t) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() _, routeExpect := setupRoute(t, rc, opts) buf := routeExpect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if info.ID != s.ID() { t.Fatalf("Expected seed's ID %q, got %q", s.ID(), info.ID) } } func TestSeedMultipleRouteInfo(t *testing.T) { s, opts := runSeedServer(t) defer s.Shutdown() rc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc1.Close() rc1ID := "2222" rc1Port := 22 rc1Host := "127.0.0.1" routeSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID) route1Expect(infoRe) // register ourselves via INFO r1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port} b, _ := json.Marshal(r1Info) infoJSON := fmt.Sprintf(server.InfoProto, b) routeSend1(infoJSON) routeSend1("PING\r\n") route1Expect(pongRe) rc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc2.Close() rc2ID := "2224" rc2Port := 24 rc2Host := "127.0.0.1" routeSend2, route2Expect := setupRouteEx(t, rc2, opts, rc2ID) hp2 := fmt.Sprintf("nats-route://%s/", net.JoinHostPort(rc2Host, strconv.Itoa(rc2Port))) // register ourselves via INFO r2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port} b, _ = json.Marshal(r2Info) infoJSON = fmt.Sprintf(server.InfoProto, b) routeSend2(infoJSON) // Now read back the second INFO route1 should receive letting // it know about route2 buf := route1Expect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if info.ID != rc2ID { t.Fatalf("Expected info.ID to be %q, got %q", rc2ID, info.ID) } if info.IP == "" { t.Fatalf("Expected a IP for the implicit route") } if info.IP != hp2 { t.Fatalf("Expected IP Host of %s, got %s\n", hp2, info.IP) } route2Expect(infoRe) routeSend2("PING\r\n") route2Expect(pongRe) // Now let's do a third. rc3 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc3.Close() rc3ID := "2226" rc3Port := 26 rc3Host := "127.0.0.1" routeSend3, _ := setupRouteEx(t, rc3, opts, rc3ID) // register ourselves via INFO r3Info := server.Info{ID: rc3ID, Host: rc3Host, Port: rc3Port} b, _ = json.Marshal(r3Info) infoJSON = fmt.Sprintf(server.InfoProto, b) routeSend3(infoJSON) // Now read back out the info from the seed route buf = route1Expect(infoRe) info = server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if info.ID != rc3ID { t.Fatalf("Expected info.ID to be %q, got %q", rc3ID, info.ID) } // Now read back out the info from the seed route buf = route2Expect(infoRe) info = server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if info.ID != rc3ID { t.Fatalf("Expected info.ID to be %q, got %q", rc3ID, info.ID) } } func TestSeedSolicitWorks(t *testing.T) { s1, opts := runSeedServer(t) defer s1.Shutdown() // Create the routes string for others to connect to the seed. routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) // Run Server #2 s2Opts := nextServerOpts(opts) s2Opts.Routes = server.RoutesFromStr(routesStr) s2 := RunServer(s2Opts) defer s2.Shutdown() // Run Server #3 s3Opts := nextServerOpts(s2Opts) s3 := RunServer(s3Opts) defer s3.Shutdown() // Wait for a bit for graph to connect time.Sleep(500 * time.Millisecond) // Grab Routez from monitor ports, make sure we are fully connected. url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) rz := readHTTPRoutez(t, url) for _, route := range rz.Routes { if route.LastActivity.IsZero() { t.Error("Expected LastActivity to be valid\n") } } ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) if ris[s2.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } // Server 2 did solicit routes to Server 1. url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) rz = readHTTPRoutez(t, url) for _, route := range rz.Routes { if route.LastActivity.IsZero() { t.Error("Expected LastActivity to be valid") } } ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) if !ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) rz = readHTTPRoutez(t, url) for _, route := range rz.Routes { if route.LastActivity.IsZero() { t.Error("Expected LastActivity to be valid") } } ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) if !ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server to be configured\n") } if ris[s2.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } } type serverInfo struct { server *server.Server opts *server.Options } func checkConnected(t *testing.T, servers []serverInfo, current int, oneSeed bool) error { s := servers[current] // Grab Routez from monitor ports, make sure we are fully connected url := fmt.Sprintf("http://%s:%d/", s.opts.Host, s.opts.HTTPPort) rz := readHTTPRoutez(t, url) total := len(servers) var ids []string for i := 0; i < total; i++ { if i == current { continue } ids = append(ids, servers[i].server.ID()) } ris, err := expectRidsNoFatal(t, true, rz, ids) if err != nil { return err } for i := 0; i < total; i++ { if i == current { continue } s := servers[i] if current == 0 || ((oneSeed && i > 0) || (!oneSeed && (i != current-1))) { if ris[s.server.ID()].IsConfigured { return fmt.Errorf("Expected server %s:%d not to be configured", s.opts.Host, s.opts.Port) } } else if oneSeed || (i == current-1) { if !ris[s.server.ID()].IsConfigured { return fmt.Errorf("Expected server %s:%d to be configured", s.opts.Host, s.opts.Port) } } } return nil } func TestStressSeedSolicitWorks(t *testing.T) { s1, opts := runSeedServer(t) defer s1.Shutdown() // Create the routes string for others to connect to the seed. routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) s2Opts := nextServerOpts(opts) s2Opts.Routes = server.RoutesFromStr(routesStr) s3Opts := nextServerOpts(s2Opts) s4Opts := nextServerOpts(s3Opts) for i := 0; i < 10; i++ { func() { // Run these servers manually, because we want them to start and // connect to s1 as fast as possible. s2 := server.New(s2Opts) if s2 == nil { panic("No NATS Server object returned.") } defer s2.Shutdown() go s2.Start() s3 := server.New(s3Opts) if s3 == nil { panic("No NATS Server object returned.") } defer s3.Shutdown() go s3.Start() s4 := server.New(s4Opts) if s4 == nil { panic("No NATS Server object returned.") } defer s4.Shutdown() go s4.Start() serversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}} checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for j := 0; j < len(serversInfo); j++ { if err := checkConnected(t, serversInfo, j, true); err != nil { return err } } return nil }) }() checkNumRoutes(t, s1, 0) } } func TestChainedSolicitWorks(t *testing.T) { s1, opts := runSeedServer(t) defer s1.Shutdown() // Create the routes string for others to connect to the seed. routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) // Run Server #2 s2Opts := nextServerOpts(opts) s2Opts.Routes = server.RoutesFromStr(routesStr) s2 := RunServer(s2Opts) defer s2.Shutdown() // Run Server #3 s3Opts := nextServerOpts(s2Opts) // We will have s3 connect to s2, not the seed. routesStr = fmt.Sprintf("nats-route://%s:%d/", s2Opts.Cluster.Host, s2Opts.Cluster.Port) s3Opts.Routes = server.RoutesFromStr(routesStr) s3 := RunServer(s3Opts) defer s3.Shutdown() // Wait for a bit for graph to connect time.Sleep(500 * time.Millisecond) // Grab Routez from monitor ports, make sure we are fully connected url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) rz := readHTTPRoutez(t, url) ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) if ris[s2.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) rz = readHTTPRoutez(t, url) ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) if !ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) rz = readHTTPRoutez(t, url) ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) if !ris[s2.ID()].IsConfigured { t.Fatalf("Expected s2 server to be configured\n") } if ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server not to be configured\n") } } func TestStressChainedSolicitWorks(t *testing.T) { s1, opts := runSeedServer(t) defer s1.Shutdown() // Create the routes string for s2 to connect to the seed routesStr := fmt.Sprintf("nats-route://%s:%d/", opts.Cluster.Host, opts.Cluster.Port) s2Opts := nextServerOpts(opts) s2Opts.Routes = server.RoutesFromStr(routesStr) s3Opts := nextServerOpts(s2Opts) // Create the routes string for s3 to connect to s2 routesStr = fmt.Sprintf("nats-route://%s:%d/", s2Opts.Cluster.Host, s2Opts.Cluster.Port) s3Opts.Routes = server.RoutesFromStr(routesStr) s4Opts := nextServerOpts(s3Opts) // Create the routes string for s4 to connect to s3 routesStr = fmt.Sprintf("nats-route://%s:%d/", s3Opts.Cluster.Host, s3Opts.Cluster.Port) s4Opts.Routes = server.RoutesFromStr(routesStr) for i := 0; i < 10; i++ { func() { // Run these servers manually, because we want them to start and // connect to s1 as fast as possible. s2 := server.New(s2Opts) if s2 == nil { panic("No NATS Server object returned.") } defer s2.Shutdown() go s2.Start() s3 := server.New(s3Opts) if s3 == nil { panic("No NATS Server object returned.") } defer s3.Shutdown() go s3.Start() s4 := server.New(s4Opts) if s4 == nil { panic("No NATS Server object returned.") } defer s4.Shutdown() go s4.Start() serversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}} checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { for j := 0; j < len(serversInfo); j++ { if err := checkConnected(t, serversInfo, j, false); err != nil { return err } } return nil }) }() checkNumRoutes(t, s1, 0) } } func TestAuthSeedSolicitWorks(t *testing.T) { s1, opts := runAuthSeedServer(t) defer s1.Shutdown() // Create the routes string for others to connect to the seed. routesStr := fmt.Sprintf("nats-route://%s:%s@%s:%d/", opts.Cluster.Username, opts.Cluster.Password, opts.Cluster.Host, opts.Cluster.Port) // Run Server #2 s2Opts := nextServerOpts(opts) s2Opts.Routes = server.RoutesFromStr(routesStr) s2 := RunServer(s2Opts) defer s2.Shutdown() // Run Server #3 s3Opts := nextServerOpts(s2Opts) s3 := RunServer(s3Opts) defer s3.Shutdown() // Wait for a bit for graph to connect time.Sleep(500 * time.Millisecond) // Grab Routez from monitor ports, make sure we are fully connected url := fmt.Sprintf("http://%s:%d/", opts.Host, opts.HTTPPort) rz := readHTTPRoutez(t, url) ris := expectRids(t, rz, []string{s2.ID(), s3.ID()}) if ris[s2.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } url = fmt.Sprintf("http://%s:%d/", s2Opts.Host, s2Opts.HTTPPort) rz = readHTTPRoutez(t, url) ris = expectRids(t, rz, []string{s1.ID(), s3.ID()}) if !ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server to be configured\n") } if ris[s3.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } url = fmt.Sprintf("http://%s:%d/", s3Opts.Host, s3Opts.HTTPPort) rz = readHTTPRoutez(t, url) ris = expectRids(t, rz, []string{s1.ID(), s2.ID()}) if !ris[s1.ID()].IsConfigured { t.Fatalf("Expected seed server to be configured\n") } if ris[s2.ID()].IsConfigured { t.Fatalf("Expected server not to be configured\n") } } // Helper to check for correct route memberships func expectRids(t *testing.T, rz *server.Routez, rids []string) map[string]*server.RouteInfo { ri, err := expectRidsNoFatal(t, false, rz, rids) if err != nil { t.Fatalf("%v", err) } return ri } func expectRidsNoFatal(t *testing.T, direct bool, rz *server.Routez, rids []string) (map[string]*server.RouteInfo, error) { caller := 1 if !direct { caller++ } if len(rids) != rz.NumRoutes { _, fn, line, _ := runtime.Caller(caller) return nil, fmt.Errorf("[%s:%d] Expecting %d routes, got %d\n", fn, line, len(rids), rz.NumRoutes) } set := make(map[string]bool) for _, v := range rids { set[v] = true } // Make result map for additional checking ri := make(map[string]*server.RouteInfo) for _, r := range rz.Routes { if !set[r.RemoteID] { _, fn, line, _ := runtime.Caller(caller) return nil, fmt.Errorf("[%s:%d] Route with rid %s unexpected, expected %+v\n", fn, line, r.RemoteID, rids) } ri[r.RemoteID] = r } return ri, nil } // Helper to easily grab routez info. func readHTTPRoutez(t *testing.T, url string) *server.Routez { resetPreviousHTTPConnections() resp, err := http.Get(url + "routez") if err != nil { stackFatalf(t, "Expected no error: Got %v\n", err) } defer resp.Body.Close() if resp.StatusCode != 200 { stackFatalf(t, "Expected a 200 response, got %d\n", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { stackFatalf(t, "Got an error reading the body: %v\n", err) } r := server.Routez{} if err := json.Unmarshal(body, &r); err != nil { stackFatalf(t, "Got an error unmarshalling the body: %v\n", err) } return &r } func TestSeedReturnIPInInfo(t *testing.T) { s, opts := runSeedServer(t) defer s.Shutdown() rc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc1.Close() rc1ID := "2222" rc1Port := 22 rc1Host := "127.0.0.1" routeSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID) route1Expect(infoRe) // register ourselves via INFO r1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port} b, _ := json.Marshal(r1Info) infoJSON := fmt.Sprintf(server.InfoProto, b) routeSend1(infoJSON) routeSend1("PING\r\n") route1Expect(pongRe) rc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc2.Close() rc2ID := "2224" rc2Port := 24 rc2Host := "127.0.0.1" routeSend2, _ := setupRouteEx(t, rc2, opts, rc2ID) // register ourselves via INFO r2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port} b, _ = json.Marshal(r2Info) infoJSON = fmt.Sprintf(server.InfoProto, b) routeSend2(infoJSON) // Now read info that route1 should have received from the seed buf := route1Expect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if info.IP == "" { t.Fatal("Expected to have IP in INFO") } rip, _, err := net.SplitHostPort(strings.TrimPrefix(info.IP, "nats-route://")) if err != nil { t.Fatalf("Error parsing url: %v", err) } addr, ok := rc1.RemoteAddr().(*net.TCPAddr) if !ok { t.Fatal("Unable to get IP address from route") } s1 := strings.ToLower(addr.IP.String()) s2 := strings.ToLower(rip) if s1 != s2 { t.Fatalf("Expected IP %s, got %s", s1, s2) } } func TestImplicitRouteRetry(t *testing.T) { srvSeed, optsSeed := runSeedServer(t) defer srvSeed.Shutdown() optsA := nextServerOpts(optsSeed) optsA.Routes = server.RoutesFromStr(fmt.Sprintf("nats://%s:%d", optsSeed.Cluster.Host, optsSeed.Cluster.Port)) optsA.Cluster.ConnectRetries = 5 srvA := RunServer(optsA) defer srvA.Shutdown() optsB := nextServerOpts(optsA) rcb := createRouteConn(t, optsSeed.Cluster.Host, optsSeed.Cluster.Port) defer rcb.Close() rcbID := "ServerB" routeBSend, routeBExpect := setupRouteEx(t, rcb, optsB, rcbID) routeBExpect(infoRe) // register ourselves via INFO rbInfo := server.Info{ID: rcbID, Host: optsB.Cluster.Host, Port: optsB.Cluster.Port} b, _ := json.Marshal(rbInfo) infoJSON := fmt.Sprintf(server.InfoProto, b) routeBSend(infoJSON) routeBSend("PING\r\n") routeBExpect(pongRe) // srvA should try to connect. Wait to make sure that it fails. time.Sleep(1200 * time.Millisecond) // Setup a fake route listen for routeB rbListen, err := net.Listen("tcp", fmt.Sprintf("%s:%d", optsB.Cluster.Host, optsB.Cluster.Port)) if err != nil { t.Fatalf("Error during listen: %v", err) } defer rbListen.Close() c, err := rbListen.Accept() if err != nil { t.Fatalf("Error during accept: %v", err) } defer c.Close() br := bufio.NewReaderSize(c, 32768) // Consume CONNECT and INFO for i := 0; i < 2; i++ { c.SetReadDeadline(time.Now().Add(2 * time.Second)) buf, _, err := br.ReadLine() c.SetReadDeadline(time.Time{}) if err != nil { t.Fatalf("Error reading: %v", err) } if i == 0 { continue } buf = buf[len("INFO "):] info := &server.Info{} if err := json.Unmarshal(buf, info); err != nil { t.Fatalf("Error during unmarshal: %v", err) } // Check INFO is from server A. if info.ID != srvA.ID() { t.Fatalf("Expected CONNECT from %v, got CONNECT from %v", srvA.ID(), info.ID) } } } nats-server-2.10.27/test/routes_test.go000066400000000000000000000767641477524627100200650ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "fmt" "io" "net" "os" "runtime" "strconv" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/internal/testhelper" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) const clientProtoInfo = 1 func runRouteServer(t *testing.T) (*server.Server, *server.Options) { return RunServerWithConfig("./configs/cluster.conf") } func runRouteServerOverrides(t *testing.T, cbo func(*server.Options), cbs func(*server.Server)) (*server.Server, *server.Options) { return RunServerWithConfigOverrides("./configs/cluster.conf", cbo, cbs) } func TestRouterListeningSocket(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() // Check that the cluster socket is able to be connected. addr := fmt.Sprintf("%s:%d", opts.Cluster.Host, opts.Cluster.Port) checkSocket(t, addr, 2*time.Second) } func TestRouteGoServerShutdown(t *testing.T) { base := runtime.NumGoroutine() s, _ := runRouteServer(t) s.Shutdown() time.Sleep(50 * time.Millisecond) delta := (runtime.NumGoroutine() - base) if delta > 1 { t.Fatalf("%d Go routines still exist post Shutdown()", delta) } } func TestSendRouteInfoOnConnect(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeID := "RouteID" routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) buf := routeExpect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if !info.AuthRequired { t.Fatal("Expected to see AuthRequired") } if info.Port != opts.Cluster.Port { t.Fatalf("Received wrong information for port, expected %d, got %d", info.Port, opts.Cluster.Port) } // Need to send a different INFO than the one received, otherwise the server // will detect as a "cycle" and close the connection. info.ID = routeID info.Name = "" b, err := json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } infoJSON := fmt.Sprintf("INFO %s\r\n", b) routeSend(infoJSON) routeSend("PING\r\n") routeExpect(pongRe) } func TestRouteToSelf(t *testing.T) { l := testhelper.NewDummyLogger(100) s, opts := runRouteServerOverrides(t, nil, func(s *server.Server) { s.SetLogger(l, true, true) }) defer s.Shutdown() rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() routeSend, routeExpect := setupRouteEx(t, rc, opts, s.ID()) buf := routeExpect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } if !info.AuthRequired { t.Fatal("Expected to see AuthRequired") } if info.Port != opts.Cluster.Port { t.Fatalf("Received wrong information for port, expected %d, got %d", info.Port, opts.Cluster.Port) } // Now send it back and that should be detected as a route to self and the // connection closed. routeSend(string(buf)) routeSend("PING\r\n") rc.SetReadDeadline(time.Now().Add(2 * time.Second)) if _, err := rc.Read(buf); err == nil { t.Fatal("Expected route connection to be closed") } // This should have been removed by removePassFromTrace(), but we also check debug logs here l.CheckForProhibited(t, "route authorization password found", "top_secret") } func TestSendRouteSubAndUnsub(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() c := createClientConn(t, opts.Host, opts.Port) defer c.Close() send, _ := setupConn(t, c) // We connect to the route. rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer rc.Close() expectAuthRequired(t, rc) routeSend, routeExpect := setupRouteEx(t, rc, opts, "ROUTER:xyz") routeSend("INFO {\"server_id\":\"ROUTER:xyz\"}\r\n") routeSend("PING\r\n") routeExpect(pongRe) // Send SUB via client connection send("SUB foo 22\r\n") // Make sure the RS+ is broadcast via the route expectResult(t, rc, rsubRe) // Send UNSUB via client connection send("UNSUB 22\r\n") // Make sure the RS- is broadcast via the route expectResult(t, rc, runsubRe) // Explicitly shutdown the server, otherwise this test would // cause following test to fail. s.Shutdown() } func TestSendRouteSolicit(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() // Listen for a connection from the server on the first route. if len(opts.Routes) <= 0 { t.Fatalf("Need an outbound solicted route for this test") } rURL := opts.Routes[0] conn := acceptRouteConn(t, rURL.Host, server.DEFAULT_ROUTE_CONNECT) defer conn.Close() // We should receive a connect message right away due to auth. buf := expectResult(t, conn, connectRe) // Check INFO follows. Could be inline, with first result, if not // check follow-on buffer. if !infoRe.Match(buf) { expectResult(t, conn, infoRe) } } func TestRouteForwardsMsgFromClients(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() client := createClientConn(t, opts.Host, opts.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) route := acceptRouteConn(t, opts.Routes[0].Host, server.DEFAULT_ROUTE_CONNECT) defer route.Close() routeSend, routeExpect := setupRoute(t, route, opts) expectMsgs := expectMsgsCommand(t, routeExpect) // Eat the CONNECT and INFO protos buf := routeExpect(connectRe) if !infoRe.Match(buf) { routeExpect(infoRe) } // Send SUB via route connection, RS+ routeSend("RS+ $G foo\r\nPING\r\n") routeExpect(pongRe) // Send PUB via client connection clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) matches := expectMsgs(1) checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") } func TestRouteForwardsMsgToClients(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() client := createClientConn(t, opts.Host, opts.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) expectMsgs := expectMsgsCommand(t, clientExpect) route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() expectAuthRequired(t, route) routeSend, _ := setupRoute(t, route, opts) // Subscribe to foo clientSend("SUB foo 1\r\nPING\r\n") // Use ping roundtrip to make sure its processed. clientExpect(pongRe) // Send RMSG proto via route connection routeSend("RMSG $G foo 2\r\nok\r\n") matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") } func TestRouteOneHopSemantics(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() expectAuthRequired(t, route) routeSend, _ := setupRoute(t, route, opts) // Express interest on this route for foo. routeSend("RS+ $G foo\r\n") // Send MSG proto via route connection routeSend("RMSG foo 2\r\nok\r\n") // Make sure it does not come back! expectNothing(t, route) } func TestRouteOnlySendOnce(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() client := createClientConn(t, opts.Host, opts.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() expectAuthRequired(t, route) routeSend, routeExpect := setupRoute(t, route, opts) expectMsgs := expectMsgsCommand(t, routeExpect) // Express multiple interest on this route for foo. routeSend("RS+ $G foo\r\n") routeSend("RS+ $G foo\r\n") routeSend("PING\r\n") routeExpect(pongRe) // Send PUB via client connection clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) expectMsgs(1) routeSend("PING\r\n") routeExpect(pongRe) } func TestRouteQueueSemantics(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() client := createClientConn(t, opts.Host, opts.Port) clientSend, clientExpect := setupConn(t, client) clientExpectMsgs := expectMsgsCommand(t, clientExpect) defer client.Close() route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() expectAuthRequired(t, route) routeSend, routeExpect := setupRouteEx(t, route, opts, "ROUTER:xyz") routeSend("INFO {\"server_id\":\"ROUTER:xyz\"}\r\n") expectMsgs := expectRmsgsCommand(t, routeExpect) // Express multiple interest on this route for foo, queue group bar. routeSend("RS+ $G foo bar 1\r\n") routeSend("RS+ $G foo bar 2\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") routeExpect(pongRe) // Send PUB via client connection clientSend("PUB foo 2\r\nok\r\n") // Use ping roundtrip to make sure its processed. clientSend("PING\r\n") clientExpect(pongRe) // Only 1 matches := expectMsgs(1) checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") // Add normal Interest as well to route interest. routeSend("RS+ $G foo\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") routeExpect(pongRe) // Send PUB via client connection clientSend("PUB foo 2\r\nok\r\nPING\r\n") // Use ping roundtrip to make sure its processed. clientExpect(pongRe) // Should be 1 now for everything. Always receive 1 message. matches = expectMsgs(1) checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") // Now create a queue subscription for the client as well as a normal one. clientSend("SUB foo 1\r\n") // Use ping roundtrip to make sure its processed. clientSend("PING\r\n") clientExpect(pongRe) routeExpect(rsubRe) clientSend("SUB foo bar 2\r\nPING\r\n") // Use ping roundtrip to make sure its processed. clientExpect(pongRe) routeExpect(rsubRe) // Deliver a MSG from the route itself, make sure the client receives both. routeSend("RMSG $G foo | bar 2\r\nok\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") routeExpect(pongRe) // Should get 2 msgs. matches = clientExpectMsgs(2) // Expect first to be the normal subscriber, next will be the queue one. checkMsg(t, matches[0], "foo", "", "", "2", "ok") } func TestSolicitRouteReconnect(t *testing.T) { l := testhelper.NewDummyLogger(100) s, opts := runRouteServerOverrides(t, nil, func(s *server.Server) { s.SetLogger(l, true, true) }) defer s.Shutdown() rURL := opts.Routes[0] route := acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT) // Go ahead and close the Route. route.Close() // We expect to get called back.. route = acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT) route.Close() // Now we want to check for the debug logs when it tries to reconnect l.CheckForProhibited(t, "route authorization password found", ":bar") } func TestMultipleRoutesSameId(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() route1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route1.Close() expectAuthRequired(t, route1) route1Send, route1Expect := setupRouteEx(t, route1, opts, "ROUTE:2222") route2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route2.Close() expectAuthRequired(t, route2) route2Send, route2Expect := setupRouteEx(t, route2, opts, "ROUTE:2222") // Send SUB via route connections sub := "RS+ $G foo\r\nPING\r\n" route1Send(sub) route2Send(sub) route1Expect(pongRe) route2Expect(pongRe) // Make sure we do not get anything on a RMSG send to a router. // Send RMSG proto via route connection route1Send("RMSG $G foo 2\r\nok\r\n") expectNothing(t, route1) expectNothing(t, route2) // Setup a client client := createClientConn(t, opts.Host, opts.Port) clientSend, clientExpect := setupConn(t, client) defer client.Close() // Send PUB via client connection clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) // We should only receive on one route, not both. // Check both manually. route1.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) buf, _ := io.ReadAll(route1) route1.SetReadDeadline(time.Time{}) if len(buf) <= 0 { route2.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) buf, _ = io.ReadAll(route2) route2.SetReadDeadline(time.Time{}) if len(buf) <= 0 { t.Fatal("Expected to get one message on a route, received none.") } } matches := rmsgRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatalf("Expected 1 msg, got %d\n", len(matches)) } checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") } func TestRouteResendsLocalSubsOnReconnect(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() client := createClientConn(t, opts.Host, opts.Port) defer client.Close() clientSend, clientExpect := setupConn(t, client) // Setup a local subscription, make sure it reaches. clientSend("SUB foo 1\r\n") clientSend("PING\r\n") clientExpect(pongRe) route := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() routeSend, routeExpect := setupRouteEx(t, route, opts, "ROUTE:1234") // Expect to see the local sub echoed through after we send our INFO. time.Sleep(50 * time.Millisecond) buf := routeExpect(infoRe) // Generate our own INFO so we can send one to trigger the local subs. info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { t.Fatalf("Could not unmarshal route info: %v", err) } info.ID = "ROUTE:1234" info.Name = "" b, err := json.Marshal(info) if err != nil { t.Fatalf("Could not marshal test route info: %v", err) } infoJSON := fmt.Sprintf("INFO %s\r\n", b) // Trigger the send of local subs. routeSend(infoJSON) routeExpect(rsubRe) // Close and then re-open route.Close() // Give some time for the route close to be processed before trying to recreate. checkNumRoutes(t, s, 0) route = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route.Close() routeSend, routeExpect = setupRouteEx(t, route, opts, "ROUTE:1234") routeExpect(infoRe) routeSend(infoJSON) routeExpect(rsubRe) } type ignoreLogger struct{} func (l *ignoreLogger) Fatalf(f string, args ...any) {} func (l *ignoreLogger) Errorf(f string, args ...any) {} func TestRouteConnectOnShutdownRace(t *testing.T) { s, opts := runRouteServer(t) defer s.Shutdown() l := &ignoreLogger{} var wg sync.WaitGroup cQuit := make(chan bool, 1) wg.Add(1) go func() { defer wg.Done() for { route := createRouteConn(l, opts.Cluster.Host, opts.Cluster.Port) if route != nil { setupRouteEx(l, route, opts, "ROUTE:1234") route.Close() } select { case <-cQuit: return default: } } }() time.Sleep(5 * time.Millisecond) s.Shutdown() cQuit <- true wg.Wait() } func TestRouteSendAsyncINFOToClients(t *testing.T) { f := func(opts *server.Options) { s := RunServer(opts) defer s.Shutdown() clientURL := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) oldClient := createClientConn(t, opts.Host, opts.Port) defer oldClient.Close() oldClientSend, oldClientExpect := setupConn(t, oldClient) oldClientSend("PING\r\n") oldClientExpect(pongRe) newClient := createClientConn(t, opts.Host, opts.Port) defer newClient.Close() newClientSend, newClientExpect := setupConnWithProto(t, newClient, clientProtoInfo) newClientSend("PING\r\n") newClientExpect(pongRe) // Check that even a new client does not receive an async INFO at this point // since there is no route created yet. expectNothing(t, newClient) routeID := "Server-B" createRoute := func() (net.Conn, sendFun, expectFun) { rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) buf := routeExpect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { stackFatalf(t, "Could not unmarshal route info: %v", err) } if opts.Cluster.NoAdvertise { if len(info.ClientConnectURLs) != 0 { stackFatalf(t, "Expected ClientConnectURLs to be empty, got %v", info.ClientConnectURLs) } } else { if len(info.ClientConnectURLs) == 0 { stackFatalf(t, "Expected a list of URLs, got none") } if info.ClientConnectURLs[0] != clientURL { stackFatalf(t, "Expected ClientConnectURLs to be %q, got %q", clientURL, info.ClientConnectURLs[0]) } } return rc, routeSend, routeExpect } sendRouteINFO := func(routeSend sendFun, routeExpect expectFun, urls []string) { routeInfo := server.Info{} routeInfo.ID = routeID routeInfo.Cluster = "xyz" routeInfo.Host = "127.0.0.1" routeInfo.Port = 5222 routeInfo.ClientConnectURLs = urls b, err := json.Marshal(routeInfo) if err != nil { stackFatalf(t, "Could not marshal test route info: %v", err) } infoJSON := fmt.Sprintf("INFO %s\r\n", b) routeSend(infoJSON) routeSend("PING\r\n") routeExpect(pongRe) } checkClientConnectURLS := func(urls, expected []string) { // Order of array is not guaranteed. ok := false if len(urls) == len(expected) { m := make(map[string]struct{}, len(expected)) for _, url := range expected { m[url] = struct{}{} } ok = true for _, url := range urls { if _, present := m[url]; !present { ok = false break } } } if !ok { stackFatalf(t, "Expected ClientConnectURLs to be %v, got %v", expected, urls) } } checkINFOReceived := func(client net.Conn, clientExpect expectFun, expectedURLs []string) { if opts.Cluster.NoAdvertise { expectNothing(t, client) return } buf := clientExpect(infoRe) info := server.Info{} if err := json.Unmarshal(buf[4:], &info); err != nil { stackFatalf(t, "Could not unmarshal route info: %v", err) } checkClientConnectURLS(info.ClientConnectURLs, expectedURLs) } // Create a route rc, routeSend, routeExpect := createRoute() defer rc.Close() // Send an INFO with single URL routeClientConnectURLs := []string{"127.0.0.1:5222"} sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) // Expect nothing for old clients expectNothing(t, oldClient) // We expect to get the one from the server we connect to and the other route. expectedURLs := []string{clientURL, routeClientConnectURLs[0]} // Expect new client to receive an INFO (unless disabled) checkINFOReceived(newClient, newClientExpect, expectedURLs) // Disconnect the route rc.Close() // Expect nothing for old clients expectNothing(t, oldClient) // Expect new client to receive an INFO (unless disabled). // The content will now have the disconnected route ClientConnectURLs // removed from the INFO. So it should be the one from the server the // client is connected to. checkINFOReceived(newClient, newClientExpect, []string{clientURL}) // Reconnect the route. rc, routeSend, routeExpect = createRoute() defer rc.Close() // Resend the same route INFO json. The server will now send // the INFO since the disconnected route ClientConnectURLs was // removed in previous step. sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) // Expect nothing for old clients expectNothing(t, oldClient) // Expect new client to receive an INFO (unless disabled) checkINFOReceived(newClient, newClientExpect, expectedURLs) // Now stop the route and restart with an additional URL rc.Close() // On route disconnect, clients will receive an updated INFO expectNothing(t, oldClient) checkINFOReceived(newClient, newClientExpect, []string{clientURL}) rc, routeSend, routeExpect = createRoute() defer rc.Close() // Create a client not sending the CONNECT until after route is added clientNoConnect := createClientConn(t, opts.Host, opts.Port) defer clientNoConnect.Close() // Create a client that does not send the first PING yet clientNoPing := createClientConn(t, opts.Host, opts.Port) defer clientNoPing.Close() clientNoPingSend, clientNoPingExpect := setupConnWithProto(t, clientNoPing, clientProtoInfo) // The route now has an additional URL routeClientConnectURLs = append(routeClientConnectURLs, "127.0.0.1:7777") expectedURLs = append(expectedURLs, "127.0.0.1:7777") // This causes the server to add the route and send INFO to clients sendRouteINFO(routeSend, routeExpect, routeClientConnectURLs) // Expect nothing for old clients expectNothing(t, oldClient) // Expect new client to receive an INFO, and verify content as expected. checkINFOReceived(newClient, newClientExpect, expectedURLs) // Expect nothing yet for client that did not send the PING expectNothing(t, clientNoPing) // Now send the first PING clientNoPingSend("PING\r\n") // Should receive PONG followed by INFO // Receive PONG only first pongBuf := make([]byte, len("PONG\r\n")) clientNoPing.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err := clientNoPing.Read(pongBuf) clientNoPing.SetReadDeadline(time.Time{}) if n <= 0 && err != nil { t.Fatalf("Error reading from conn: %v\n", err) } if !pongRe.Match(pongBuf) { t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'\n", pongBuf, pongRe) } checkINFOReceived(clientNoPing, clientNoPingExpect, expectedURLs) // Have the client that did not send the connect do it now clientNoConnectSend, clientNoConnectExpect := setupConnWithProto(t, clientNoConnect, clientProtoInfo) // Send the PING clientNoConnectSend("PING\r\n") // Should receive PONG followed by INFO // Receive PONG only first clientNoConnect.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err = clientNoConnect.Read(pongBuf) clientNoConnect.SetReadDeadline(time.Time{}) if n <= 0 && err != nil { t.Fatalf("Error reading from conn: %v\n", err) } if !pongRe.Match(pongBuf) { t.Fatalf("Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'\n", pongBuf, pongRe) } checkINFOReceived(clientNoConnect, clientNoConnectExpect, expectedURLs) // Create a client connection and verify content of initial INFO contains array // (but empty if no advertise option is set) cli := createClientConn(t, opts.Host, opts.Port) defer cli.Close() buf := expectResult(t, cli, infoRe) js := infoRe.FindAllSubmatch(buf, 1)[0][1] var sinfo server.Info err = json.Unmarshal(js, &sinfo) if err != nil { t.Fatalf("Could not unmarshal INFO json: %v\n", err) } if opts.Cluster.NoAdvertise { if len(sinfo.ClientConnectURLs) != 0 { t.Fatalf("Expected ClientConnectURLs to be empty, got %v", sinfo.ClientConnectURLs) } } else { checkClientConnectURLS(sinfo.ClientConnectURLs, expectedURLs) } // Add a new route routeID = "Server-C" rc2, route2Send, route2Expect := createRoute() defer rc2.Close() // Send an INFO with single URL rc2ConnectURLs := []string{"127.0.0.1:8888"} sendRouteINFO(route2Send, route2Expect, rc2ConnectURLs) // This is the combined client connect URLs array totalConnectURLs := expectedURLs totalConnectURLs = append(totalConnectURLs, rc2ConnectURLs...) // Expect nothing for old clients expectNothing(t, oldClient) // Expect new client to receive an INFO (unless disabled) checkINFOReceived(newClient, newClientExpect, totalConnectURLs) // Make first route disconnect rc.Close() // Expect nothing for old clients expectNothing(t, oldClient) // Expect new client to receive an INFO (unless disabled) // The content should be the server client is connected to and the last route checkINFOReceived(newClient, newClientExpect, []string{"127.0.0.1:5242", "127.0.0.1:8888"}) } opts := LoadConfig("./configs/cluster.conf") // For this test, be explicit about listen spec. opts.Host = "127.0.0.1" opts.Port = 5242 opts.DisableShortFirstPing = true f(opts) opts.Cluster.NoAdvertise = true f(opts) } func TestRouteBasicPermissions(t *testing.T) { srvA, optsA := RunServerWithConfig("./configs/srv_a_perms.conf") defer srvA.Shutdown() srvB, optsB := RunServerWithConfig("./configs/srv_b.conf") defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) // Create a connection to server B ncb, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncb.Close() ch := make(chan bool, 1) cb := func(_ *nats.Msg) { ch <- true } // Subscribe on Server B on "bar" and "baz", which should be accepted by server A across the route // Due to allowing "*" subBbar, err := ncb.Subscribe("bar", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subBbar.Unsubscribe() subBbaz, err := ncb.Subscribe("baz", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subBbaz.Unsubscribe() ncb.Flush() if err := checkExpectedSubs(2, srvA, srvB); err != nil { t.Fatal(err.Error()) } // Create a connection to server A nca, err := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nca.Close() // Publish on bar and baz, messages should be received. if err := nca.Publish("bar", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } if err := nca.Publish("baz", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } for i := 0; i < 2; i++ { select { case <-ch: case <-time.After(time.Second): t.Fatal("Did not get the messages") } } // From B, start a subscription on "foo", which server A should drop since // it only exports on "bar" and "baz" subBfoo, err := ncb.Subscribe("foo", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subBfoo.Unsubscribe() ncb.Flush() // B should have now 3 subs if err := checkExpectedSubs(3, srvB); err != nil { t.Fatal(err.Error()) } // and A still 2. if err := checkExpectedSubs(2, srvA); err != nil { t.Fatal(err.Error()) } // So producing on "foo" from A should not be forwarded to B. if err := nca.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: t.Fatal("Message should not have been received") case <-time.After(100 * time.Millisecond): } // Now on A, create a subscription on something that A does not import, // like "bat". subAbat, err := nca.Subscribe("bat", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subAbat.Unsubscribe() nca.Flush() // A should have 3 subs if err := checkExpectedSubs(3, srvA); err != nil { t.Fatal(err.Error()) } // And from B, send a message on that subject and make sure it is not received. if err := ncb.Publish("bat", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: t.Fatal("Message should not have been received") case <-time.After(100 * time.Millisecond): } // Stop subscription on foo from B subBfoo.Unsubscribe() ncb.Flush() // Back to 2 subs on B if err := checkExpectedSubs(2, srvB); err != nil { t.Fatal(err.Error()) } // Create subscription on foo from A, this should be forwared to B. subAfoo, err := nca.Subscribe("foo", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subAfoo.Unsubscribe() // Create another one so that test the import permissions cache subAfoo2, err := nca.Subscribe("foo", cb) if err != nil { t.Fatalf("Error on subscribe: %v", err) } defer subAfoo2.Unsubscribe() nca.Flush() // A should have 5 subs if err := checkExpectedSubs(5, srvA); err != nil { t.Fatal(err.Error()) } // B should have 3 since we coalesce te two for 'foo' if err := checkExpectedSubs(3, srvB); err != nil { t.Fatal(err.Error()) } // Send a message from B and check that it is received. if err := ncb.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } for i := 0; i < 2; i++ { select { case <-ch: case <-time.After(time.Second): t.Fatal("Did not get the message") } } // Close connection from B, and restart server B too. // We want to make sure that ncb.Close() srvB.Shutdown() // Since B had 2 local subs, A should still only go from 4 to 3 if err := checkExpectedSubs(3, srvA); err != nil { t.Fatal(err.Error()) } // Restart server B srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") defer srvB.Shutdown() // Check that subs from A that can be sent to B are sent. // That would be 2 (the 2 subscriptions on foo) as one. if err := checkExpectedSubs(1, srvB); err != nil { t.Fatal(err.Error()) } // Connect to B and send on "foo" and make sure we receive ncb, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncb.Close() if err := ncb.Publish("foo", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } for i := 0; i < 2; i++ { select { case <-ch: case <-time.After(time.Second): t.Fatal("Did not get the message") } } // Send on "bat" and make sure that this is not received. if err := ncb.Publish("bat", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: t.Fatal("Message should not have been received") case <-time.After(100 * time.Millisecond): } nca.Close() ncb.Close() srvA.Shutdown() srvB.Shutdown() optsA.Cluster.Permissions.Export = nil srvA = RunServer(optsA) defer srvA.Shutdown() srvB = RunServer(optsB) defer srvB.Shutdown() checkClusterFormed(t, srvA, srvB) nca, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsA.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nca.Close() // Subscribe on "bar" which is not imported if _, err := nca.Subscribe("bar", cb); err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := checkExpectedSubs(1, srvA); err != nil { t.Fatal(err.Error()) } // Publish from B, should not be received ncb, err = nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", optsB.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncb.Close() if err := ncb.Publish("bar", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: t.Fatal("Message should not have been received") case <-time.After(100 * time.Millisecond): //ok } // Subscribe on "baz" on B if _, err := ncb.Subscribe("baz", cb); err != nil { t.Fatalf("Error on subscribe: %v", err) } if err := checkExpectedSubs(1, srvB); err != nil { t.Fatal(err.Error()) } if err := checkExpectedSubs(2, srvA); err != nil { t.Fatal(err.Error()) } // Publish from A, since there is no export restriction, message should be received. if err := nca.Publish("baz", []byte("hello")); err != nil { t.Fatalf("Error on publish: %v", err) } select { case <-ch: // ok case <-time.After(250 * time.Millisecond): t.Fatal("Message should have been received") } } func createConfFile(t testing.TB, content []byte) string { t.Helper() conf := createTempFile(t, "") fName := conf.Name() conf.Close() if err := os.WriteFile(fName, content, 0666); err != nil { t.Fatalf("Error writing conf file: %v", err) } return fName } func TestRoutesOnlyImportOrExport(t *testing.T) { contents := []string{ `import: "foo"`, `import: { allow: "foo" }`, `import: { deny: "foo" }`, `import: { allow: "foo" deny: "foo" }`, `export: "foo"`, `export: { allow: "foo" }`, `export: { deny: "foo" }`, `export: { allow: "foo" deny: "foo" }`, } f := func(c string) { cf := createConfFile(t, []byte(fmt.Sprintf(` port: -1 cluster { name: "Z" port: -1 authorization { user: ivan password: pwd permissions { %s } } } `, c))) s, _ := RunServerWithConfig(cf) s.Shutdown() } for _, c := range contents { f(c) } } nats-server-2.10.27/test/service_latency_test.go000066400000000000000000001541131477524627100217040ustar00rootroot00000000000000// Copyright 2019-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "encoding/json" "fmt" "math/rand" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/jwt/v2" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" ) // Used to setup superclusters for tests. type supercluster struct { t *testing.T clusters []*cluster } func (sc *supercluster) shutdown() { if sc == nil { return } for _, c := range sc.clusters { shutdownCluster(c) } } const digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" const base = 36 const cnlen = 8 func randClusterName() string { var name []byte rn := rand.Int63() for i := 0; i < cnlen; i++ { name = append(name, digits[rn%base]) rn /= base } return string(name[:cnlen]) } func createSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { clusters := []*cluster{} for i := 0; i < numClusters; i++ { // Pick cluster name and setup default accounts. c := createClusterEx(t, true, 5*time.Millisecond, true, randClusterName(), numServersPer, clusters...) clusters = append(clusters, c) } return &supercluster{t, clusters} } func (sc *supercluster) setResponseThreshold(t *testing.T, maxTime time.Duration) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { foo, err := s.LookupAccount("FOO") if err != nil { t.Fatalf("Error looking up account 'FOO': %v", err) } if err := foo.SetServiceExportResponseThreshold("ngs.usage.*", maxTime); err != nil { t.Fatalf("Error setting response threshold") } } } } func (sc *supercluster) setImportShare(t *testing.T) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { foo, err := s.LookupAccount("FOO") if err != nil { t.Fatalf("Error looking up account 'FOO': %v", err) } bar, err := s.LookupAccount("BAR") if err != nil { t.Fatalf("Error looking up account 'BAR': %v", err) } if err := bar.SetServiceImportSharing(foo, "ngs.usage.bar", true); err != nil { t.Fatalf("Error setting import sharing: %v", err) } } } } func (sc *supercluster) setupLatencyTracking(t *testing.T, p int) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { foo, err := s.LookupAccount("FOO") if err != nil { t.Fatalf("Error looking up account 'FOO': %v", err) } bar, err := s.LookupAccount("BAR") if err != nil { t.Fatalf("Error looking up account 'BAR': %v", err) } if err := foo.AddServiceExport("ngs.usage.*", nil); err != nil { t.Fatalf("Error adding service export to 'FOO': %v", err) } if err := foo.TrackServiceExportWithSampling("ngs.usage.*", "results", p); err != nil { t.Fatalf("Error adding latency tracking to 'FOO': %v", err) } if err := bar.AddServiceImport(foo, "ngs.usage", "ngs.usage.bar"); err != nil { t.Fatalf("Error adding service import to 'ngs.usage': %v", err) } } } } func (sc *supercluster) removeLatencyTracking(t *testing.T) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { foo, err := s.LookupAccount("FOO") if err != nil { t.Fatalf("Error looking up account 'FOO': %v", err) } foo.UnTrackServiceExport("ngs.usage.*") } } } func (sc *supercluster) totalSubs() int { totalSubs := 0 for _, c := range sc.clusters { totalSubs += c.totalSubs() } return totalSubs } func clientConnectWithName(t *testing.T, opts *server.Options, user, appname string) *nats.Conn { t.Helper() url := fmt.Sprintf("nats://%s:pass@%s:%d", user, opts.Host, opts.Port) nc, err := nats.Connect(url, nats.Name(appname)) if err != nil { t.Fatalf("Error on connect: %v", err) } return nc } func clientConnect(t *testing.T, opts *server.Options, user string) *nats.Conn { t.Helper() return clientConnectWithName(t, opts, user, "") } func clientConnectOldRequest(t *testing.T, opts *server.Options, user string) *nats.Conn { t.Helper() url := fmt.Sprintf("nats://%s:pass@%s:%d", user, opts.Host, opts.Port) nc, err := nats.Connect(url, nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } return nc } func checkServiceLatency(t *testing.T, sl server.ServiceLatency, start time.Time, serviceTime time.Duration) { t.Helper() if sl.Status != 200 { t.Fatalf("Bad status received, wanted 200 got %d", sl.Status) } serviceTime = serviceTime.Round(time.Millisecond) startDelta := sl.RequestStart.Sub(start) // Original test was 5ms, but got GitHub Action failure with "Bad start delta 5.033929ms", // and Travis will get something like: "Bad start delta 15.046059ms", so be more generous. if startDelta > 20*time.Millisecond { t.Fatalf("Bad start delta %v", startDelta) } // Since RTT during tests is estimate we remove from calculation. if (sl.ServiceLatency + sl.Responder.RTT) < time.Duration(float64(serviceTime)*0.8) { t.Fatalf("Bad service latency: %v (%v)", sl.ServiceLatency, serviceTime) } if sl.TotalLatency < sl.ServiceLatency { t.Fatalf("Bad total latency: %v (%v)", sl.TotalLatency, sl.ServiceLatency) } // We should have NATS latency here that is non-zero with real clients. if sl.Requestor.RTT == 0 { t.Fatalf("Expected non-zero NATS Requestor latency") } if sl.Responder.RTT == 0 { t.Fatalf("Expected non-zero NATS Requestor latency") } // Make sure they add up got := sl.TotalLatency expected := sl.ServiceLatency + sl.NATSTotalTime() if got != expected { t.Fatalf("Numbers do not add up: %+v,\ngot: %v\nexpected: %v", sl, got, expected) } } func extendedCheck(t *testing.T, lc *server.ClientInfo, eUser, appName, eServer string) { t.Helper() if lc.User != eUser { t.Fatalf("Expected user of %q, got %q", eUser, lc.User) } if appName != "" && appName != lc.Name { t.Fatalf("Expected appname of %q, got %q\n", appName, lc.Name) } if lc.Host == "" { t.Fatalf("Expected non-empty IP") } if lc.ID < 1 || lc.ID > 20 { t.Fatalf("Expected a ID in range, got %d", lc.ID) } if eServer != "" && eServer != lc.Server { t.Fatalf("Expected server of %q, got %q", eServer, lc.Server) } } func noShareCheck(t *testing.T, lc *server.ClientInfo) { t.Helper() if lc.Name != "" { t.Fatalf("appname should not have been shared, got %q", lc.Name) } if lc.User != "" { t.Fatalf("user should not have been shared, got %q", lc.User) } if lc.Host != "" { t.Fatalf("client ip should not have been shared, got %q", lc.Host) } if lc.ID != 0 { t.Fatalf("client id should not have been shared, got %d", lc.ID) } if lc.Server != "" { t.Fatalf("client' server should not have been shared, got %q", lc.Server) } } func TestServiceLatencySingleServerConnect(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) // Now we can setup and test, do single node only first. // This is the service provider. nc := clientConnectWithName(t, sc.clusters[0].opts[0], "foo", "service22") defer nc.Close() // The service listener. serviceTime := 25 * time.Millisecond nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") // Requestor nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") defer nc2.Close() // Send the request. start := time.Now() if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, _ := rsub.NextMsg(time.Second) json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) rs := sc.clusters[0].servers[0] extendedCheck(t, sl.Responder, "foo", "service22", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) // Now make sure normal use case works with old request style. nc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], "bar") defer nc3.Close() start = time.Now() if _, err := nc3.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } nc3.Close() checkServiceLatency(t, sl, start, serviceTime) extendedCheck(t, sl.Responder, "foo", "service22", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) } // If a client has a longer RTT that the effective RTT for NATS + responder // the requestor RTT will be marked as 0. This can happen quite often with // utility programs that are far away from a cluster like NGS but the service // response time has a shorter RTT. func TestServiceLatencyClientRTTSlowerVsServiceRTT(t *testing.T) { sc := createSuperCluster(t, 2, 2) defer sc.shutdown() // Now add in new service export to FOO and have BAR import that with tracking enabled. sc.setupLatencyTracking(t, 100) nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // The service listener. Mostly instant response. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { time.Sleep(time.Millisecond) msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") nc.Flush() // Requestor and processing requestAndCheck := func(cindex, sindex int) { t.Helper() sopts := sc.clusters[cindex].opts[sindex] if nmsgs, _, err := rsub.Pending(); err != nil || nmsgs != 0 { t.Fatalf("Did not expect any latency results, got %d", nmsgs) } rtt := 10 * time.Millisecond bw := 1024 * 1024 sp := newSlowProxy(rtt+5*time.Millisecond, bw, bw, sopts) defer sp.stop() nc2 := clientConnect(t, sp.opts(), "bar") defer nc2.Close() start := time.Now() nc2.Flush() // Check rtt for slow proxy if d := time.Since(start); d < rtt { t.Fatalf("Expected an rtt of at least %v, got %v", rtt, d) } // Send the request. _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second) if err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, err := rsub.NextMsg(time.Second) if err != nil || rmsg == nil { t.Fatalf("Did not receive latency results") } json.Unmarshal(rmsg.Data, &sl) if sl.Status != 200 { t.Fatalf("Expected a status 200 Ok, got [%d] %q", sl.Status, sl.Error) } // We want to test here that when the client requestor RTT is larger then the response time // we still deliver a requestor value > 0. // Now check that it is close to rtt. if sl.Requestor.RTT < rtt { t.Fatalf("Expected requestor latency to be > %v, got %v", rtt, sl.Requestor.RTT) } if sl.TotalLatency < rtt { t.Fatalf("Expected total latency to be > %v, got %v", rtt, sl.TotalLatency) } rs := sc.clusters[0].servers[0] extendedCheck(t, sl.Responder, "foo", "", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) // Check for trailing duplicates.. rmsg, err = rsub.NextMsg(100 * time.Millisecond) if err == nil { t.Fatalf("Duplicate metric result, %q", rmsg.Data) } } // Check same server. requestAndCheck(0, 0) // Check from remote server across GW. requestAndCheck(1, 1) // Same cluster but different server requestAndCheck(0, 1) } func connRTT(nc *nats.Conn) time.Duration { // Do 5x to flatten total := time.Duration(0) for i := 0; i < 5; i++ { start := time.Now() nc.Flush() total += time.Since(start) } return total / 5 } func TestServiceLatencyRemoteConnect(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) // Now we can setup and test, do single node only first. // This is the service provider. nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() subsBefore := int(sc.clusters[0].servers[0].NumSubscriptions()) // The service listener. serviceTime := 25 * time.Millisecond nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") nc.Flush() if err := checkExpectedSubs(subsBefore+2, sc.clusters[0].servers...); err != nil { t.Fatal(err.Error()) } // Same Cluster Requestor nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") defer nc2.Close() // Send the request. start := time.Now() _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second) if err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting latency measurement: %v", err) } json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) rs := sc.clusters[0].servers[0] extendedCheck(t, sl.Responder, "foo", "", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) // Lastly here, we need to make sure we are properly tracking the extra hops. // We will make sure that NATS latency is close to what we see from the outside in terms of RTT. if crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt { t.Fatalf("Not tracking second measurement for NATS latency across servers: %v vs %v", sl.NATSTotalTime(), crtt) } // Gateway Requestor nc2 = clientConnect(t, sc.clusters[1].opts[1], "bar") defer nc2.Close() // Send the request. start = time.Now() _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second) if err != nil { t.Fatalf("Expected a response") } rmsg, err = rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting latency measurement: %v", err) } json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) extendedCheck(t, sl.Responder, "foo", "", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) // Lastly here, we need to make sure we are properly tracking the extra hops. // We will make sure that NATS latency is close to what we see from the outside in terms of RTT. if crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt { t.Fatalf("Not tracking second measurement for NATS latency across servers: %v vs %v", sl.NATSTotalTime(), crtt) } // Now turn off and make sure we no longer receive updates. sc.removeLatencyTracking(t) _, err = nc2.Request("ngs.usage", []byte("1h"), time.Second) if err != nil { t.Fatalf("Expected a response") } _, err = rsub.NextMsg(100 * time.Millisecond) if err == nil { t.Fatalf("Did not expect to receive a latency metric") } } func TestServiceLatencySampling(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 50) // Now we can setup and test, do single node only first. // This is the service provider. nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) // Listen for metrics received := int32(0) nc.Subscribe("results", func(msg *nats.Msg) { atomic.AddInt32(&received, 1) }) // Same Cluster Requestor nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") defer nc2.Close() toSend := 1000 for i := 0; i < toSend; i++ { nc2.Request("ngs.usage", []byte("1h"), time.Second) } // Wait for results to flow in. time.Sleep(100 * time.Millisecond) mid := toSend / 2 delta := toSend / 10 // 10% got := int(atomic.LoadInt32(&received)) if got > mid+delta || got < mid-delta { t.Fatalf("Sampling number incorrect: %d vs %d", mid, got) } } func TestServiceLatencyNoSubsLeak(t *testing.T) { sc := createSuperCluster(t, 3, 3) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) nc := clientConnectWithName(t, sc.clusters[0].opts[1], "foo", "dlc22") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) nc.Flush() // Propagation of sub through super cluster. time.Sleep(100 * time.Millisecond) startSubs := sc.totalSubs() fooAcc, _ := sc.clusters[1].servers[1].LookupAccount("FOO") startNumSis := fooAcc.NumServiceImports() for i := 0; i < 100; i++ { nc := clientConnect(t, sc.clusters[1].opts[1], "bar") defer nc.Close() if _, err := nc.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Error on request: %v", err) } nc.Close() } // We are adding 3 here for the wildcard response subject for service replies. // we only have one but it will show in three places. startSubs += 3 checkFor(t, time.Second, 50*time.Millisecond, func() error { if numSubs := sc.totalSubs(); numSubs != startSubs { return fmt.Errorf("Leaked %d subs", numSubs-startSubs) } return nil }) // Now also check to make sure the service imports created for the request go away as well. checkFor(t, time.Second, 50*time.Millisecond, func() error { if numSis := fooAcc.NumServiceImports(); numSis != startNumSis { return fmt.Errorf("Leaked %d service imports", numSis-startNumSis) } return nil }) } func TestServiceLatencyWithName(t *testing.T) { sc := createSuperCluster(t, 1, 1) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) opts := sc.clusters[0].opts[0] nc := clientConnectWithName(t, opts, "foo", "dlc22") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") nc.Flush() nc2 := clientConnect(t, opts, "bar") defer nc2.Close() nc2.Request("ngs.usage", []byte("1h"), time.Second) var sl server.ServiceLatency rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Error getting message: %v", err) } json.Unmarshal(rmsg.Data, &sl) // Make sure we have AppName set. rs := sc.clusters[0].servers[0] extendedCheck(t, sl.Responder, "foo", "dlc22", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) } func TestServiceLatencyWithNameMultiServer(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) nc := clientConnectWithName(t, sc.clusters[0].opts[1], "foo", "dlc22") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") nc.Flush() nc2 := clientConnect(t, sc.clusters[1].opts[1], "bar") defer nc2.Close() nc2.Request("ngs.usage", []byte("1h"), time.Second) var sl server.ServiceLatency rmsg, _ := rsub.NextMsg(time.Second) json.Unmarshal(rmsg.Data, &sl) // Make sure we have AppName set. rs := sc.clusters[0].servers[1] extendedCheck(t, sl.Responder, "foo", "dlc22", rs.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) } func createAccountWithJWT(t *testing.T) (string, nkeys.KeyPair, *jwt.AccountClaims) { t.Helper() okp, _ := nkeys.FromSeed(oSeed) akp, _ := nkeys.CreateAccount() pub, _ := akp.PublicKey() nac := jwt.NewAccountClaims(pub) jwt, _ := nac.Encode(okp) return jwt, akp, nac } func TestServiceLatencyWithJWT(t *testing.T) { okp, _ := nkeys.FromSeed(oSeed) // Create three accounts, system, service and normal account. sysJWT, sysKP, _ := createAccountWithJWT(t) sysPub, _ := sysKP.PublicKey() _, svcKP, svcAcc := createAccountWithJWT(t) svcPub, _ := svcKP.PublicKey() // Add in the service export with latency tracking here. serviceExport := &jwt.Export{Subject: "req.*", Type: jwt.Service} svcAcc.Exports.Add(serviceExport) svcJWT, err := svcAcc.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } _, accKP, accAcc := createAccountWithJWT(t) accPub, _ := accKP.PublicKey() // Add in the import. serviceImport := &jwt.Import{Account: svcPub, Subject: "request", To: "req.echo", Type: jwt.Service} accAcc.Imports.Add(serviceImport) accJWT, err := accAcc.Encode(okp) if err != nil { t.Fatalf("Error generating account JWT: %v", err) } cf := ` listen: 127.0.0.1:-1 cluster { name: "A" listen: 127.0.0.1:-1 authorization { timeout: 2.2 } %s } operator = "./configs/nkeys/op.jwt" system_account = "%s" resolver = MEMORY resolver_preload = { %s : "%s" %s : "%s" %s : "%s" } ` contents := strings.Replace(fmt.Sprintf(cf, "", sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), "\n\t", "\n", -1) conf := createConfFile(t, []byte(contents)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() // Create a new server and route to main one. routeStr := fmt.Sprintf("\n\t\troutes = [nats-route://%s:%d]", opts.Cluster.Host, opts.Cluster.Port) contents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), "\n\t", "\n", -1) conf2 := createConfFile(t, []byte(contents2)) s2, opts2 := RunServerWithConfig(conf2) defer s2.Shutdown() checkClusterFormed(t, s, s2) // Create service provider. url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) copt, pubUser := createUserCredsOption(t, s, svcKP) nc, err := nats.Connect(url, copt, nats.Name("fooService")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // The service listener. serviceTime := 25 * time.Millisecond nc.Subscribe("req.echo", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond(msg.Data) }) // Listen for metrics rsub, _ := nc.SubscribeSync("results") nc.Flush() // Create second client and send request from this one. url2 := fmt.Sprintf("nats://%s:%d/", opts2.Host, opts2.Port) nc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP)) if err != nil { t.Fatalf("Error creating client: %v\n", err) } defer nc2.Close() // Send the request. _, err = nc2.Request("request", []byte("hello"), time.Second) if err != nil { t.Fatalf("Expected a response") } // We should not receive latency at this time. _, err = rsub.NextMsg(100 * time.Millisecond) if err == nil { t.Fatalf("Did not expect to receive a latency metric") } // Now turn it on.. updateAccount := func() { t.Helper() for _, s := range []*server.Server{s, s2} { svcAccount, err := s.LookupAccount(svcPub) if err != nil { t.Fatalf("Could not lookup service account from server %+v", s) } s.UpdateAccountClaims(svcAccount, svcAcc) } } serviceExport.Latency = &jwt.ServiceLatency{Sampling: 100, Results: "results"} updateAccount() // Grab the service responder's user. // Send the request. start := time.Now() _, err = nc2.Request("request", []byte("hello"), time.Second) if err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, err := rsub.NextMsg(time.Second) if err != nil || rmsg == nil { t.Fatalf("Did not receive a latency metric") } json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) extendedCheck(t, sl.Responder, pubUser, "fooService", s.Name()) // Normally requestor's don't share noShareCheck(t, sl.Requestor) // Now we will remove tracking. Do this by simulating a JWT update. serviceExport.Latency = nil updateAccount() // Now we should not get any tracking data. _, err = nc2.Request("request", []byte("hello"), time.Second) if err != nil { t.Fatalf("Expected a response") } _, err = rsub.NextMsg(100 * time.Millisecond) if err == nil { t.Fatalf("Did not expect to receive a latency metric") } } func TestServiceLatencyAdjustNegativeLatencyValues(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import // that with tracking enabled. sc.setupLatencyTracking(t, 100) // Now we can setup and test, do single node only first. // This is the service provider. nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, err := nc.SubscribeSync("results") if err != nil { t.Fatal(err) } nc.Flush() // Requestor nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") defer nc2.Close() // Send the request. totalSamples := 50 for i := 0; i < totalSamples; i++ { if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } } var sl server.ServiceLatency for i := 0; i < totalSamples; i++ { rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected to receive latency metric: %d, %s", i, err) } if err := json.Unmarshal(rmsg.Data, &sl); err != nil { t.Errorf("Unexpected error processing latency metric: %s", err) } if sl.ServiceLatency < 0 { t.Fatalf("Unexpected negative latency value: %v", sl) } } } func TestServiceLatencyRemoteConnectAdjustNegativeValues(t *testing.T) { sc := createSuperCluster(t, 3, 2) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) // Now we can setup and test, do single node only first. // This is the service provider. nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // The service listener. nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) }) // Listen for metrics rsub, err := nc.SubscribeSync("results") if err != nil { t.Fatal(err) } nc.Flush() // Same Cluster Requestor nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") defer nc2.Close() // Gateway Requestor nc3 := clientConnect(t, sc.clusters[1].opts[1], "bar") defer nc3.Close() // Send a few initial requests to ensure interest is propagated // both for cluster and gateway requestors. checkFor(t, 3*time.Second, time.Second, func() error { _, err1 := nc2.Request("ngs.usage", []byte("1h"), time.Second) _, err2 := nc3.Request("ngs.usage", []byte("1h"), time.Second) if err1 != nil || err2 != nil { return fmt.Errorf("Timed out waiting for super cluster to be ready") } return nil }) // Send the request. totalSamples := 20 for i := 0; i < totalSamples; i++ { if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } } for i := 0; i < totalSamples; i++ { if _, err := nc3.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } } var sl server.ServiceLatency for i := 0; i < totalSamples*2; i++ { rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected to receive latency metric: %d, %s", i, err) } if err = json.Unmarshal(rmsg.Data, &sl); err != nil { t.Errorf("Unexpected error processing latency metric: %s", err) } if sl.ServiceLatency < 0 { t.Fatalf("Unexpected negative service latency value: %v", sl) } if sl.SystemLatency < 0 { t.Fatalf("Unexpected negative system latency value: %v", sl) } } } func TestServiceLatencyFailureReportingSingleServer(t *testing.T) { sc := createSuperCluster(t, 1, 1) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) sc.setResponseThreshold(t, 20*time.Millisecond) nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // Listen for metrics rsub, err := nc.SubscribeSync("results") if err != nil { t.Fatal(err) } nc.Flush() getMetricResult := func() *server.ServiceLatency { t.Helper() rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected to receive latency metric: %v", err) } var sl server.ServiceLatency if err = json.Unmarshal(rmsg.Data, &sl); err != nil { t.Errorf("Unexpected error processing latency metric: %s", err) } return &sl } // Same server nc2 := clientConnect(t, sc.clusters[0].opts[0], "bar") defer nc2.Close() // Test a request with no reply subject nc2.Publish("ngs.usage", []byte("1h")) sl := getMetricResult() if sl.Status != 400 { t.Fatalf("Expected to get a bad request status [400], got %d", sl.Status) } // Proper request, but no responders. nc2.Request("ngs.usage", []byte("1h"), 20*time.Millisecond) sl = getMetricResult() if sl.Status != 503 { t.Fatalf("Expected to get a service unavailable status [503], got %d", sl.Status) } // The service listener. Make it slow. 20ms is respThreshold, so take 2X sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { time.Sleep(40 * time.Millisecond) msg.Respond([]byte("22 msgs")) }) nc.Flush() nc2.Request("ngs.usage", []byte("1h"), 20*time.Millisecond) sl = getMetricResult() if sl.Status != 504 { t.Fatalf("Expected to get a service timeout status [504], got %d", sl.Status) } // Make sure we do not get duplicates. if rmsg, err := rsub.NextMsg(50 * time.Millisecond); err == nil { t.Fatalf("Unexpected second response metric: %q\n", rmsg.Data) } // Now setup a responder that will respond under the threshold. sub.Unsubscribe() nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { time.Sleep(5 * time.Millisecond) msg.Respond([]byte("22 msgs")) }) nc.Flush() time.Sleep(100 * time.Millisecond) // Now create a responder using old request and we will do a short timeout. nc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], "bar") defer nc3.Close() nc3.Request("ngs.usage", []byte("1h"), time.Millisecond) sl = getMetricResult() if sl.Status != 408 { t.Fatalf("Expected to get a request timeout status [408], got %d", sl.Status) } } func TestServiceLatencyFailureReportingMultipleServers(t *testing.T) { sc := createSuperCluster(t, 3, 3) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) sc.setResponseThreshold(t, 10*time.Millisecond) nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // Listen for metrics rsub, err := nc.SubscribeSync("results") if err != nil { t.Fatal(err) } nc.Flush() getMetricResult := func() *server.ServiceLatency { t.Helper() rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected to receive latency metric: %v", err) } var sl server.ServiceLatency if err = json.Unmarshal(rmsg.Data, &sl); err != nil { t.Errorf("Unexpected error processing latency metric: %s", err) } return &sl } cases := []struct { ci, si int desc string }{ {0, 0, "same server"}, {0, 1, "same cluster, different server"}, {1, 1, "different cluster"}, } for _, cs := range cases { // Select the server to send request from. nc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], "bar") defer nc2.Close() // Test a request with no reply subject nc2.Publish("ngs.usage", []byte("1h")) sl := getMetricResult() if sl.Status != 400 { t.Fatalf("Test %q, Expected to get a bad request status [400], got %d", cs.desc, sl.Status) } // We wait here for the gateways to report no interest b/c optimistic mode. time.Sleep(50 * time.Millisecond) // Proper request, but no responders. nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) sl = getMetricResult() if sl.Status != 503 { t.Fatalf("Test %q, Expected to get a service unavailable status [503], got %d", cs.desc, sl.Status) } // The service listener. Make it slow. 10ms is respThreshold, so make 3X sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { time.Sleep(30 * time.Millisecond) msg.Respond([]byte("22 msgs")) }) defer sub.Unsubscribe() nc.Flush() // Wait to propagate. time.Sleep(200 * time.Millisecond) nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) sl = getMetricResult() if sl.Status != 504 { t.Fatalf("Test %q, Expected to get a service timeout status [504], got %d", cs.desc, sl.Status) } // Clean up subscriber and requestor nc2.Close() sub.Unsubscribe() nc.Flush() // Wait to propagate. time.Sleep(200 * time.Millisecond) } } // To test a bug rip@nats.io is seeing. func TestServiceLatencyOldRequestStyleSingleServer(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { SVC: { users: [ {user: svc, password: pass} ] exports: [ { service: "svc.echo" accounts: [CLIENT] latency: { sampling: 100% subject: latency.svc } } ] }, CLIENT: { users: [{user: client, password: pass} ] imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ] }, SYS: { users: [{user: admin, password: pass}] } } system_account: SYS `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Listen for metrics rsub, _ := nc.SubscribeSync("latency.svc") // Requestor nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Setup responder serviceTime := 25 * time.Millisecond sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond([]byte("world")) }) nc.Flush() defer sub.Unsubscribe() // Send a request start := time.Now() if _, err := nc2.Request("SVC", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, _ := rsub.NextMsg(time.Second) json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) extendedCheck(t, sl.Responder, "svc", "", srv.Name()) noShareCheck(t, sl.Requestor) } // To test a bug wally@nats.io is seeing. func TestServiceAndStreamStackOverflow(t *testing.T) { conf := createConfFile(t, []byte(` accounts { STATIC { users = [ { user: "static", pass: "foo" } ] exports [ { stream: > } { service: my.service } ] } DYN { users = [ { user: "foo", pass: "bar" } ] imports [ { stream { subject: >, account: STATIC } } { service { subject: my.service, account: STATIC } } ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder (just request sub) nc, err := nats.Connect(fmt.Sprintf("nats://static:foo@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() sub, _ := nc.SubscribeSync("my.service") nc.Flush() // Requestor nc2, err := nats.Connect(fmt.Sprintf("nats://foo:bar@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Send a single request. nc2.PublishRequest("my.service", "foo", []byte("hi")) checkFor(t, time.Second, 10*time.Millisecond, func() error { if nm, _, err := sub.Pending(); err != nil || nm != 1 { return fmt.Errorf("Expected one request, got %d", nm) } return nil }) // Make sure works for queue subscribers as well. sub.Unsubscribe() sub, _ = nc.QueueSubscribeSync("my.service", "prod") nc.Flush() // Send a single request. nc2.PublishRequest("my.service", "foo", []byte("hi")) checkFor(t, time.Second, 10*time.Millisecond, func() error { if nm, _, err := sub.Pending(); err != nil || nm != 1 { return fmt.Errorf("Expected one request, got %d", nm) } return nil }) // Now create an interest in the stream from nc2. that is a queue subscriber. sub2, _ := nc2.QueueSubscribeSync("my.service", "prod") defer sub2.Unsubscribe() nc2.Flush() // Send a single request. nc2.PublishRequest("my.service", "foo", []byte("hi")) time.Sleep(10 * time.Millisecond) checkFor(t, time.Second, 10*time.Millisecond, func() error { if nm, _, err := sub.Pending(); err != nil || nm != 2 { return fmt.Errorf("Expected two requests, got %d", nm) } return nil }) } // Check we get the proper detailed information for the requestor when allowed. func TestServiceLatencyRequestorSharesDetailedInfo(t *testing.T) { sc := createSuperCluster(t, 3, 3) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) sc.setResponseThreshold(t, 10*time.Millisecond) sc.setImportShare(t) nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() // Listen for metrics rsub, err := nc.SubscribeSync("results") if err != nil { t.Fatal(err) } nc.Flush() getMetricResult := func() *server.ServiceLatency { t.Helper() rmsg, err := rsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected to receive latency metric: %v", err) } var sl server.ServiceLatency if err = json.Unmarshal(rmsg.Data, &sl); err != nil { t.Errorf("Unexpected error processing latency metric: %s", err) } return &sl } cases := []struct { ci, si int desc string }{ {0, 0, "same server"}, {0, 1, "same cluster, different server"}, {1, 1, "different cluster"}, } for _, cs := range cases { // Select the server to send request from. nc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], "bar") defer nc2.Close() rs := sc.clusters[cs.ci].servers[cs.si] // Test a request with no reply subject nc2.Publish("ngs.usage", []byte("1h")) sl := getMetricResult() if sl.Status != 400 { t.Fatalf("Test %q, Expected to get a bad request status [400], got %d", cs.desc, sl.Status) } extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) // We wait here for the gateways to report no interest b/c optimistic mode. time.Sleep(50 * time.Millisecond) // Proper request, but no responders. nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) sl = getMetricResult() if sl.Status != 503 { t.Fatalf("Test %q, Expected to get a service unavailable status [503], got %d", cs.desc, sl.Status) } extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) // The service listener. Make it slow. 10ms is respThreshold, so take 2.5X sub, _ := nc.Subscribe("ngs.usage.bar", func(msg *nats.Msg) { time.Sleep(25 * time.Millisecond) msg.Respond([]byte("22 msgs")) }) defer sub.Unsubscribe() nc.Flush() // Wait to propagate. time.Sleep(200 * time.Millisecond) nc2.Request("ngs.usage", []byte("1h"), 10*time.Millisecond) sl = getMetricResult() if sl.Status != 504 { t.Fatalf("Test %q, Expected to get a service timeout status [504], got %d", cs.desc, sl.Status) } extendedCheck(t, sl.Requestor, "bar", "", rs.Name()) // Clean up subscriber and requestor nc2.Close() sub.Unsubscribe() nc.Flush() // Wait to propagate. time.Sleep(200 * time.Millisecond) } } func TestServiceLatencyRequestorSharesConfig(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { SVC: { users: [ {user: svc, password: pass} ] exports: [ { service: "svc.echo" accounts: [CLIENT] latency: { sampling: 100% subject: latency.svc } } ] }, CLIENT: { users: [{user: client, password: pass} ] imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ] }, SYS: { users: [{user: admin, password: pass}] } } system_account: SYS `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Listen for metrics rsub, _ := nc.SubscribeSync("latency.svc") // Requestor nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Setup responder serviceTime := 25 * time.Millisecond sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond([]byte("world")) }) nc.Flush() defer sub.Unsubscribe() // Send a request start := time.Now() if _, err := nc2.Request("SVC", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } var sl server.ServiceLatency rmsg, _ := rsub.NextMsg(time.Second) json.Unmarshal(rmsg.Data, &sl) checkServiceLatency(t, sl, start, serviceTime) extendedCheck(t, sl.Responder, "svc", "", srv.Name()) extendedCheck(t, sl.Requestor, "client", "", srv.Name()) // Check reload. newConf := []byte(` listen: 127.0.0.1:-1 accounts: { SVC: { users: [ {user: svc, password: pass} ] exports: [ { service: "svc.echo" accounts: [CLIENT] latency: { sampling: 100% subject: latency.svc } } ] }, CLIENT: { users: [{user: client, password: pass} ] imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ] }, SYS: { users: [{user: admin, password: pass}] } } system_account: SYS `) if err := os.WriteFile(conf, newConf, 0600); err != nil { t.Fatalf("Error rewriting server's config file: %v", err) } if err := srv.Reload(); err != nil { t.Fatalf("Error on server reload: %v", err) } if _, err = nc2.Request("SVC", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } var sl2 server.ServiceLatency rmsg, _ = rsub.NextMsg(time.Second) json.Unmarshal(rmsg.Data, &sl2) noShareCheck(t, sl2.Requestor) } func TestServiceLatencyLossTest(t *testing.T) { // assure that behavior with respect to requests timing out (and samples being reordered) is as expected. conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { SVC: { users: [ {user: svc, password: pass} ] exports: [ { service: "svc.echo" threshold: "500ms" accounts: [CLIENT] latency: { sampling: headers subject: latency.svc } } ] }, CLIENT: { users: [{user: client, password: pass} ] imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ] }, SYS: { users: [{user: admin, password: pass}] } } system_account: SYS `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder connection ncr, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port), nats.Name("responder")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncr.Close() ncl, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port), nats.Name("latency")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer ncl.Close() // Table of expected state for which message. // This also codifies that the first message, in respsonse to second request is ok. // Second message, in response to first request times out. expectedState := map[int]int{1: http.StatusOK, 2: http.StatusGatewayTimeout} msgCnt := 0 start := time.Now().Add(250 * time.Millisecond) var latErr []error // Listen for metrics wg := sync.WaitGroup{} wg.Add(2) rsub, _ := ncl.Subscribe("latency.svc", func(rmsg *nats.Msg) { defer wg.Done() var sl server.ServiceLatency json.Unmarshal(rmsg.Data, &sl) msgCnt++ if want := expectedState[msgCnt]; want != sl.Status { latErr = append(latErr, fmt.Errorf("Expected different status for msg #%d: %d != %d", msgCnt, want, sl.Status)) } if msgCnt > 1 { if start.Before(sl.RequestStart) { latErr = append(latErr, fmt.Errorf("start times should indicate reordering %v : %v", start, sl.RequestStart)) } } start = sl.RequestStart if strings.EqualFold(sl.RequestHeader.Get("Uber-Trace-Id"), fmt.Sprintf("msg-%d", msgCnt)) { latErr = append(latErr, fmt.Errorf("no header present")) } }) defer rsub.Unsubscribe() // Setup requestor nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle(), nats.Name("requestor")) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() respCnt := int64(0) reply := nc2.NewRespInbox() repSub, _ := nc2.Subscribe(reply, func(msg *nats.Msg) { atomic.AddInt64(&respCnt, 1) }) defer repSub.Unsubscribe() nc2.Flush() // use dedicated send that publishes requests using same reply subject send := func(msg string) { if err := nc2.PublishMsg(&nats.Msg{Subject: "SVC", Data: []byte(msg), Reply: reply, Header: nats.Header{"X-B3-Sampled": []string{"1"}}}); err != nil { t.Fatalf("Expected a response got: %v", err) } } // Setup responder that skips responding and triggers next request OR responds sub, _ := ncr.Subscribe("svc.echo", func(msg *nats.Msg) { if string(msg.Data) != "msg2" { msg.Respond([]byte("response")) } else { wg.Add(1) go func() { // second request (use go routine to not block in responders callback) defer wg.Done() time.Sleep(250 * time.Millisecond) send("msg1") // will cause the first latency measurement }() } }) ncr.Flush() ncl.Flush() nc2.Flush() defer sub.Unsubscribe() // Send first request, which is expected to timeout send("msg2") // wait till we got enough responses wg.Wait() if len(latErr) > 0 { t.Fatalf("Got errors %v", latErr) } if atomic.LoadInt64(&respCnt) != 1 { t.Fatalf("Expected only one message") } } func TestServiceLatencyHeaderTriggered(t *testing.T) { receiveAndTest := func(t *testing.T, rsub *nats.Subscription, shared bool, header nats.Header, status int, srvName string) server.ServiceLatency { t.Helper() var sl server.ServiceLatency rmsg, _ := rsub.NextMsg(time.Second) if rmsg == nil { t.Fatal("Expected message") return sl } json.Unmarshal(rmsg.Data, &sl) if sl.Status != status { t.Fatalf("Expected different status %d != %d", status, sl.Status) } if status == http.StatusOK { extendedCheck(t, sl.Responder, "svc", "", srvName) } if shared { extendedCheck(t, sl.Requestor, "client", "", srvName) } else { noShareCheck(t, sl.Requestor) } // header are always included if v := sl.RequestHeader.Get("Some-Other"); v != "" { t.Fatalf("Expected header to be gone") } for k, value := range header { if v := sl.RequestHeader.Get(k); v != value[0] { t.Fatalf("Expected header %q to be set", k) } } return sl } for _, v := range []struct { shared bool header nats.Header }{ {true, nats.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}}}, {true, nats.Header{"X-B3-Sampled": []string{"1"}}}, {true, nats.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}}}, {true, nats.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90"}}}, {true, nats.Header{"Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}}, {false, nats.Header{"Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}}}, {false, nats.Header{"X-B3-Sampled": []string{"1"}}}, {false, nats.Header{"X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}}}, {false, nats.Header{"B3": []string{"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90"}}}, {false, nats.Header{"Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}}, {false, nats.Header{ "X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, "X-B3-ParentSpanId": []string{"05e3ac9a4f6e3b90"}, "X-B3-SpanId": []string{"e457b5a2e4d86bd1"}, "X-B3-Sampled": []string{"1"}, }}, {false, nats.Header{ "X-B3-TraceId": []string{"80f198ee56343ba864fe8b2a57d3eff7"}, "X-B3-ParentSpanId": []string{"05e3ac9a4f6e3b90"}, "X-B3-SpanId": []string{"e457b5a2e4d86bd1"}, }}, {false, nats.Header{ "Uber-Trace-Id": []string{"479fefe9525eddb:479fefe9525eddb:0:1"}, "Uberctx-X": []string{"foo"}, }}, {false, nats.Header{ "Traceparent": []string{"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, "Tracestate": []string{"rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"}, }}, } { t.Run(fmt.Sprintf("%s_%t_%s", t.Name(), v.shared, v.header), func(t *testing.T) { conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 accounts: { SVC: { users: [ {user: svc, password: pass} ] exports: [ { service: "svc.echo" accounts: [CLIENT] latency: { sampling: headers subject: latency.svc } }] }, CLIENT: { users: [{user: client, password: pass} ] imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:%t} ] }, SYS: { users: [{user: admin, password: pass}] } } system_account: SYS `, v.shared))) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder nc, err := nats.Connect(fmt.Sprintf("nats://svc:pass@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Listen for metrics rsub, _ := nc.SubscribeSync("latency.svc") defer rsub.Unsubscribe() // Setup responder serviceTime := 25 * time.Millisecond sub, _ := nc.Subscribe("svc.echo", func(msg *nats.Msg) { time.Sleep(serviceTime) msg.Respond([]byte("world")) }) nc.Flush() defer sub.Unsubscribe() // Setup requestor nc2, err := nats.Connect(fmt.Sprintf("nats://client:pass@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Send a request start := time.Now() msg := &nats.Msg{ Subject: "SVC", Data: []byte("1h"), Header: make(nats.Header), } for k, v := range v.header { for _, val := range v { msg.Header.Add(k, val) } } msg.Header.Add("Some-Other", "value") if _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err != nil { t.Fatalf("Expected a response") } sl := receiveAndTest(t, rsub, v.shared, v.header, http.StatusOK, srv.Name()) checkServiceLatency(t, sl, start, serviceTime) // shut down responder to test various error scenarios sub.Unsubscribe() nc.Flush() // Send a request without responder if _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err == nil { t.Fatalf("Expected no response") } receiveAndTest(t, rsub, v.shared, v.header, http.StatusServiceUnavailable, srv.Name()) // send a message without a response msg.Reply = "" if err := nc2.PublishMsg(msg); err != nil { t.Fatalf("Expected no error got %v", err) } receiveAndTest(t, rsub, v.shared, v.header, http.StatusBadRequest, srv.Name()) }) } } // From a report by rip@nats.io on simple latency reporting missing in 2 server cluster setup. func TestServiceLatencyMissingResults(t *testing.T) { accConf := createConfFile(t, []byte(` accounts { one: { users = [ {user: one, password: password} ] imports = [ {service: {account: weather, subject: service.weather.requests.>}, to: service.weather.>, share: true} ] } weather: { users = [ {user: weather, password: password} ] exports = [ { service: service.weather.requests.> accounts: [one] latency: { sampling: 100%, subject: service.weather.latency } } ] } } `)) s1Conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: s1 cluster { port: -1 } include %q `, filepath.Base(accConf)))) // Link accConf for relative import from s1Conf os.Link(accConf, filepath.Join(filepath.Dir(s1Conf), filepath.Base(accConf))) s1, opts1 := RunServerWithConfig(s1Conf) defer s1.Shutdown() s2Conf := createConfFile(t, []byte(fmt.Sprintf(` listen: 127.0.0.1:-1 server_name: s2 cluster { port: -1 routes = [ nats-route://127.0.0.1:%d ] } include %q `, opts1.Cluster.Port, filepath.Base(accConf)))) // Link accConf for relative import from s2Conf os.Link(accConf, filepath.Join(filepath.Dir(s2Conf), filepath.Base(accConf))) s2, opts2 := RunServerWithConfig(s2Conf) defer s2.Shutdown() checkClusterFormed(t, s1, s2) nc1, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s:%d", "weather", "password", opts1.Host, opts1.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc1.Close() // Create responder sub, _ := nc1.Subscribe("service.weather.requests.>", func(msg *nats.Msg) { time.Sleep(25 * time.Millisecond) msg.Respond([]byte("sunny!")) }) defer sub.Unsubscribe() // Create sync listener for latency. latSubj := "service.weather.latency" lsub, _ := nc1.SubscribeSync(latSubj) defer lsub.Unsubscribe() nc1.Flush() // Make sure the subscription propagates to s2 server. checkSubInterest(t, s2, "weather", latSubj, time.Second) // Create requestor on s2. nc2, err := nats.Connect(fmt.Sprintf("nats://%s:%s@%s:%d", "one", "password", opts2.Host, opts2.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() nc2.Request("service.weather.los_angeles", nil, time.Second) lr, err := lsub.NextMsg(time.Second) if err != nil { t.Fatalf("Expected a latency result, got %v", err) } // Make sure we reported ok and have valid results for service, system and total. var sl server.ServiceLatency json.Unmarshal(lr.Data, &sl) if sl.Status != 200 { t.Fatalf("Expected a 200 status, got %d\n", sl.Status) } if sl.ServiceLatency == 0 || sl.SystemLatency == 0 || sl.TotalLatency == 0 { t.Fatalf("Received invalid tracking measurements, %d %d %d", sl.ServiceLatency, sl.SystemLatency, sl.TotalLatency) } } // To test a bug I was seeing. func TestServiceLatencyDoubleResponse(t *testing.T) { sc := createSuperCluster(t, 3, 1) defer sc.shutdown() // Now add in new service export to FOO and have bar import that with tracking enabled. sc.setupLatencyTracking(t, 100) // Responder nc := clientConnectWithName(t, sc.clusters[0].opts[0], "foo", "service22") defer nc.Close() // Setup responder sub, _ := nc.Subscribe("ngs.usage.*", func(msg *nats.Msg) { msg.Respond([]byte("22 msgs")) msg.Respond([]byte("boom")) }) nc.Flush() defer sub.Unsubscribe() // Listen for metrics rsub, _ := nc.SubscribeSync("latency.svc") // Requestor nc2 := clientConnect(t, sc.clusters[0].opts[2], "bar") defer nc2.Close() // Send a request if _, err := nc2.Request("ngs.usage", []byte("1h"), time.Second); err != nil { t.Fatalf("Expected a response") } rsub.NextMsg(time.Second) time.Sleep(time.Second) } nats-server-2.10.27/test/services_test.go000066400000000000000000000407531477524627100203540ustar00rootroot00000000000000// Copyright 2020-2022 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bytes" "fmt" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) var basicMASetupContents = []byte(` server_name: A listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [{service: "foo", response: stream}] }, B: { users: [{user: b, password: pwd} ] imports: [{ service: { account: A, subject: "foo"}, to: "foo_request" }] } } `) func TestServiceImportWithStreamed(t *testing.T) { conf := createConfFile(t, basicMASetupContents) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Limit max response maps here for the test. accB, err := srv.LookupAccount("B") if err != nil { t.Fatalf("Error looking up account: %v", err) } // connect and offer a service nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() nc.Subscribe("foo", func(msg *nats.Msg) { if err := msg.Respond([]byte("world")); err != nil { t.Fatalf("Error on respond: %v", err) } }) nc.Flush() nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() numRequests := 10 for i := 0; i < numRequests; i++ { resp, err := nc2.Request("foo_request", []byte("hello"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { t.Fatal("Did not receive the correct message") } } // Since we are using a new client that multiplexes, until the client itself goes away // we will have the full number of entries, even with the tighter ResponseEntriesPruneThreshold. accA, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } // These should always be the same now. if nre := accB.NumPendingReverseResponses(); nre != numRequests { t.Fatalf("Expected %d entries, got %d", numRequests, nre) } if nre := accA.NumPendingAllResponses(); nre != numRequests { t.Fatalf("Expected %d entries, got %d", numRequests, nre) } // Now kill of the client that was doing the requests. nc2.Close() checkFor(t, time.Second, 10*time.Millisecond, func() error { aNrssi := accA.NumPendingAllResponses() bNre := accB.NumPendingReverseResponses() if aNrssi != 0 || bNre != 0 { return fmt.Errorf("Response imports and response entries should all be 0, got %d %d", aNrssi, bNre) } return nil }) // Now let's test old style request and reply that uses a new inbox each time. This should work ok.. nc2, err = nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() for i := 0; i < numRequests; i++ { resp, err := nc2.Request("foo_request", []byte("hello"), 2*time.Second) if err != nil { t.Fatalf("Unexpected error: %v", err) } if resp == nil || strings.Compare("world", string(resp.Data)) != 0 { t.Fatal("Did not receive the correct message") } } checkFor(t, time.Second, 10*time.Millisecond, func() error { aNrssi := accA.NumPendingAllResponses() bNre := accB.NumPendingReverseResponses() if aNrssi != 0 || bNre != 0 { return fmt.Errorf("Response imports and response entries should all be 0, got %d %d", aNrssi, bNre) } return nil }) } func TestServiceImportWithStreamedResponseAndEOF(t *testing.T) { conf := createConfFile(t, basicMASetupContents) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() accA, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } accB, err := srv.LookupAccount("B") if err != nil { t.Fatalf("Error looking up account: %v", err) } nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // We will send four responses and then and nil message signaling EOF nc.Subscribe("foo", func(msg *nats.Msg) { // Streamed response. msg.Respond([]byte("world-1")) msg.Respond([]byte("world-2")) msg.Respond([]byte("world-3")) msg.Respond([]byte("world-4")) msg.Respond(nil) }) nc.Flush() // Now setup requester. nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() numRequests := 10 expectedResponses := 5 for i := 0; i < numRequests; i++ { // Create an inbox reply := nats.NewInbox() sub, _ := nc2.SubscribeSync(reply) defer sub.Unsubscribe() if err := nc2.PublishRequest("foo_request", reply, []byte("XOXO")); err != nil { t.Fatalf("Error sending request: %v", err) } // Wait and make sure we get all the responses. Should be five. checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expectedResponses { return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, expectedResponses) } return nil }) } if nre := accA.NumPendingAllResponses(); nre != 0 { t.Fatalf("Expected no entries, got %d", nre) } if nre := accB.NumPendingReverseResponses(); nre != 0 { t.Fatalf("Expected no entries, got %d", nre) } } func TestServiceExportsResponseFiltering(t *testing.T) { conf := createConfFile(t, []byte(` server_name: A listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [ {service: "foo"}, {service: "bar"} ] }, B: { users: [{user: b, password: pwd} ] imports: [ {service: { account: A, subject: "foo"}}, {service: { account: A, subject: "bar"}, to: "baz"} ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // If we do not subscribe the system is now smart enough to not setup the response service imports. nc.SubscribeSync("foo") nc.SubscribeSync("bar") nc.Flush() nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // We don't expect responses, so just do publishes. // 5 for foo sendFoo := 5 for i := 0; i < sendFoo; i++ { nc2.PublishRequest("foo", "reply", nil) } // 17 for bar sendBar := 17 for i := 0; i < sendBar; i++ { nc2.PublishRequest("baz", "reply", nil) } nc2.Flush() accA, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } sendTotal := sendFoo + sendBar if nre := accA.NumPendingAllResponses(); nre != sendTotal { t.Fatalf("Expected %d entries, got %d", sendTotal, nre) } if nre := accA.NumPendingResponses("foo"); nre != sendFoo { t.Fatalf("Expected %d entries, got %d", sendFoo, nre) } if nre := accA.NumPendingResponses("bar"); nre != sendBar { t.Fatalf("Expected %d entries, got %d", sendBar, nre) } } func TestServiceExportsAutoDirectCleanup(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [ {service: "foo"} ] }, B: { users: [{user: b, password: pwd} ] imports: [ {service: { account: A, subject: "foo"}} ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() acc, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } // Potential resonder. nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // Requestor nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() expectNone := func() { t.Helper() if nre := acc.NumPendingAllResponses(); nre != 0 { t.Fatalf("Expected no entries, got %d", nre) } } toSend := 10 // With no responders we should never register service import responses etc. for i := 0; i < toSend; i++ { nc2.PublishRequest("foo", "reply", nil) } nc2.Flush() expectNone() // Now register a responder. sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) { msg.Respond([]byte("world")) }) nc.Flush() defer sub.Unsubscribe() // With no reply subject on a request we should never register service import responses etc. for i := 0; i < toSend; i++ { nc2.Publish("foo", nil) } nc2.Flush() expectNone() // Create an old request style client. nc3, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port), nats.UseOldRequestStyle()) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc3.Close() // If the request loses interest before the response we should not queue up service import responses either. // This only works for old style requests at the moment where we can detect interest going away. delay := 25 * time.Millisecond sub.Unsubscribe() sub, _ = nc.Subscribe("foo", func(msg *nats.Msg) { time.Sleep(delay) msg.Respond([]byte("world")) }) nc.Flush() defer sub.Unsubscribe() for i := 0; i < toSend; i++ { nc3.Request("foo", nil, time.Millisecond) } nc3.Flush() time.Sleep(time.Duration(toSend) * delay * 2) expectNone() } // In some instances we do not have a forceful trigger that signals us to clean up. // Like a stream that does not send EOF or a responder who receives requests but does // not answer. For these we will have an expectation of a response threshold which // tells the system we should have seen a response by T, say 2 minutes, 30 seconds etc. func TestServiceExportsPruningCleanup(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [ {service: "foo", response: stream} ] }, B: { users: [{user: b, password: pwd} ] imports: [ {service: { account: A, subject: "foo"}} ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Potential resonder. nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // We will subscribe but not answer. sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) {}) nc.Flush() defer sub.Unsubscribe() acc, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } // Check on response thresholds. rt, err := acc.ServiceExportResponseThreshold("foo") if err != nil { t.Fatalf("Error retrieving response threshold, %v", err) } if rt != server.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD { t.Fatalf("Expected the response threshold to be %v, got %v", server.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD, rt) } // now set it newRt := 10 * time.Millisecond if err := acc.SetServiceExportResponseThreshold("foo", newRt); err != nil { t.Fatalf("Expected no error setting response threshold, got %v", err) } expectedPending := func(expected int) { t.Helper() // Caller is sleeping a bit before, but avoid flappers. checkFor(t, time.Second, 15*time.Millisecond, func() error { if nre := acc.NumPendingResponses("foo"); nre != expected { return fmt.Errorf("Expected %d entries, got %d", expected, nre) } return nil }) } // Requestor nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() toSend := 10 // This should register and they will dangle. Make sure we clean them up. for i := 0; i < toSend; i++ { nc2.PublishRequest("foo", "reply", nil) } nc2.Flush() expectedPending(10) time.Sleep(4 * newRt) expectedPending(0) // Do it again. for i := 0; i < toSend; i++ { nc2.PublishRequest("foo", "reply", nil) } nc2.Flush() expectedPending(10) time.Sleep(4 * newRt) expectedPending(0) } func TestServiceExportsResponseThreshold(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [ {service: "foo", response: stream, threshold: "1s"} ] }, } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Potential responder. nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() // We will subscribe but not answer. sub, _ := nc.Subscribe("foo", func(msg *nats.Msg) {}) nc.Flush() defer sub.Unsubscribe() acc, err := srv.LookupAccount("A") if err != nil { t.Fatalf("Error looking up account: %v", err) } // Check on response thresholds. rt, err := acc.ServiceExportResponseThreshold("foo") if err != nil { t.Fatalf("Error retrieving response threshold, %v", err) } if rt != 1*time.Second { t.Fatalf("Expected response threshold to be %v, got %v", 1*time.Second, rt) } } func TestServiceExportsResponseThresholdChunked(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd} ] exports: [ {service: "foo", response: chunked, threshold: "10ms"} ] }, B: { users: [{user: b, password: pwd} ] imports: [ {service: { account: A, subject: "foo"}} ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder. nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() numChunks := 10 // Respond with 5ms gaps for total response time for all chunks and EOF > 50ms. nc.Subscribe("foo", func(msg *nats.Msg) { // Streamed response. for i := 1; i <= numChunks; i++ { time.Sleep(5 * time.Millisecond) msg.Respond([]byte(fmt.Sprintf("chunk-%d", i))) } msg.Respond(nil) }) nc.Flush() // Now setup requester. nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() // Create an inbox reply := nats.NewInbox() sub, _ := nc2.SubscribeSync(reply) defer sub.Unsubscribe() if err := nc2.PublishRequest("foo", reply, nil); err != nil { t.Fatalf("Error sending request: %v", err) } checkFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error { if nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numChunks+1 { return fmt.Errorf("Did not receive correct number of chunks: %d vs %d", nmsgs, numChunks+1) } return nil }) } func TestServiceAllowResponsesPerms(t *testing.T) { conf := createConfFile(t, []byte(` listen: 127.0.0.1:-1 accounts: { A: { users: [ {user: a, password: pwd, permissions = {subscribe=foo, allow_responses=true}} ] exports: [ {service: "foo"} ] }, B: { users: [{user: b, password: pwd} ] imports: [ {service: { account: A, subject: "foo"}} ] } } `)) srv, opts := RunServerWithConfig(conf) defer srv.Shutdown() // Responder. nc, err := nats.Connect(fmt.Sprintf("nats://a:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc.Close() reply := []byte("Hello") // Respond with 5ms gaps for total response time for all chunks and EOF > 50ms. nc.Subscribe("foo", func(msg *nats.Msg) { msg.Respond(reply) }) nc.Flush() // Now setup requester. nc2, err := nats.Connect(fmt.Sprintf("nats://b:pwd@%s:%d", opts.Host, opts.Port)) if err != nil { t.Fatalf("Error on connect: %v", err) } defer nc2.Close() resp, err := nc2.Request("foo", []byte("help"), time.Second) if err != nil { t.Fatalf("Error expecting response %v", err) } if !bytes.Equal(resp.Data, reply) { t.Fatalf("Did not get correct response, %q vs %q", resp.Data, reply) } } nats-server-2.10.27/test/system_services_test.go000066400000000000000000000231041477524627100217470ustar00rootroot00000000000000// Copyright 2019-2023 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "fmt" "math/rand" "net/url" "strconv" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" ) const dbgSubs = "$SYS.DEBUG.SUBSCRIBERS" func (sc *supercluster) selectRandomServer() *server.Options { ci := rand.Int31n(int32(len(sc.clusters))) si := rand.Int31n(int32(len(sc.clusters[ci].servers))) return sc.clusters[ci].opts[si] } func (sc *supercluster) setupSystemServicesImports(t *testing.T, account string) { t.Helper() for _, c := range sc.clusters { for _, s := range c.servers { sysAcc := s.SystemAccount() if sysAcc == nil { t.Fatalf("System account not set") } acc, err := s.LookupAccount(account) if err != nil { t.Fatalf("Error looking up account '%s': %v", account, err) } if err := acc.AddServiceImport(sysAcc, dbgSubs, dbgSubs); err != nil { t.Fatalf("Error adding subscribers debug service to '%s': %v", account, err) } } } } func numSubs(t *testing.T, msg *nats.Msg) int { t.Helper() if msg == nil || msg.Data == nil { t.Fatalf("No response") } n, err := strconv.Atoi(string(msg.Data)) if err != nil { t.Fatalf("Got non-number response: %v", err) } return n } func checkDbgNumSubs(t *testing.T, nc *nats.Conn, subj string, expected int) { t.Helper() var n int for i := 0; i < 3; i++ { response, err := nc.Request(dbgSubs, []byte(subj), 250*time.Millisecond) if err != nil { continue } if n = numSubs(t, response); n == expected { return } } t.Fatalf("Expected %d subscribers, got %d", expected, n) } func TestSystemServiceSubscribers(t *testing.T) { numServers, numClusters := 3, 3 sc := createSuperCluster(t, numServers, numClusters) defer sc.shutdown() sc.setupSystemServicesImports(t, "FOO") nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() checkInterest := func(expected int) { t.Helper() checkDbgNumSubs(t, nc, "foo.bar", expected) } checkInterest(0) // Now add in local subscribers. for i := 0; i < 5; i++ { nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() nc.SubscribeSync("foo.bar") nc.SubscribeSync("foo.*") nc.Flush() } checkInterest(10) // Now create remote subscribers at random. for i := 0; i < 90; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() nc.SubscribeSync("foo.bar") nc.Flush() } checkInterest(100) } // Test that we can match wildcards. So sub may be foo.bar and we ask about foo.*, that should work. func TestSystemServiceSubscribersWildcards(t *testing.T) { numServers, numClusters := 3, 3 sc := createSuperCluster(t, numServers, numClusters) defer sc.shutdown() sc.setupSystemServicesImports(t, "FOO") nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() for i := 0; i < 50; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) nc.SubscribeSync(fmt.Sprintf("%d", i+1)) nc.Flush() } checkDbgNumSubs(t, nc, "foo.bar.*", 50) checkDbgNumSubs(t, nc, "foo.>", 50) checkDbgNumSubs(t, nc, "foo.bar.22", 1) response, _ := nc.Request(dbgSubs, []byte("_INBOX.*.*"), time.Second) hasInbox := numSubs(t, response) checkDbgNumSubs(t, nc, ">", 100+hasInbox) } // Test that we can match on queue groups as well. Separate request payload with any whitespace. func TestSystemServiceSubscribersQueueGroups(t *testing.T) { numServers, numClusters := 3, 3 sc := createSuperCluster(t, numServers, numClusters) defer sc.shutdown() sc.setupSystemServicesImports(t, "FOO") nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() for i := 0; i < 10; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() subj := fmt.Sprintf("foo.bar.%d", i+1) nc.QueueSubscribeSync(subj, "QG.11") nc.QueueSubscribeSync("foo.baz", "QG.33") nc.Flush() } for i := 0; i < 23; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() subj := fmt.Sprintf("foo.bar.%d", i+1) nc.QueueSubscribeSync(subj, "QG.22") nc.QueueSubscribeSync("foo.baz", "QG.22") nc.Flush() } checkDbgNumSubs(t, nc, "foo.bar.*", 33) checkDbgNumSubs(t, nc, "foo.bar.22 QG.22", 1) checkDbgNumSubs(t, nc, "foo.bar.2", 2) checkDbgNumSubs(t, nc, "foo.baz", 33) checkDbgNumSubs(t, nc, "foo.baz QG.22", 23) // Now check qfilters work on wildcards too. checkDbgNumSubs(t, nc, "foo.bar.> QG.11", 10) checkDbgNumSubs(t, nc, "*.baz QG.22", 23) checkDbgNumSubs(t, nc, "foo.*.22 QG.22", 1) } func TestSystemServiceSubscribersLeafNodesWithoutSystem(t *testing.T) { numServers, numClusters := 3, 3 sc := createSuperCluster(t, numServers, numClusters) defer sc.shutdown() sc.setupSystemServicesImports(t, "FOO") ci := rand.Int31n(int32(len(sc.clusters))) si := rand.Int31n(int32(len(sc.clusters[ci].servers))) s, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si] url := fmt.Sprintf("nats://%s:pass@%s:%d", "foo", opts.Host, opts.LeafNode.Port) ls, lopts := runSolicitLeafServerToURL(url) defer ls.Shutdown() checkLeafNodeConnected(t, s) // This is so we can test when the subs on a leafnode are flushed to the connected supercluster. fsubj := "__leaf.flush__" fc := clientConnect(t, opts, "foo") defer fc.Close() fc.Subscribe(fsubj, func(m *nats.Msg) { m.Respond(nil) }) lnc := clientConnect(t, lopts, "$G") defer lnc.Close() flushLeaf := func() { if _, err := lnc.Request(fsubj, nil, time.Second); err != nil { t.Fatalf("Did not flush through to the supercluster: %v", err) } } for i := 0; i < 10; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) nc.QueueSubscribeSync("foo.bar.baz", "QG.22") nc.Flush() } nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() checkDbgNumSubs(t, nc, "foo.bar.*", 20) checkDbgNumSubs(t, nc, "foo.bar.3", 1) lnc.SubscribeSync("foo.bar.3") lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") // We could flush here but that does not guarantee we have flushed through to the supercluster. flushLeaf() checkDbgNumSubs(t, nc, "foo.bar.3", 2) checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 11) lnc.SubscribeSync("foo.bar.3") lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") flushLeaf() // For now we do not see all the details behind a leafnode if the leafnode is not enabled. checkDbgNumSubs(t, nc, "foo.bar.3", 2) checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 12) } func runSolicitLeafServerWithSystemToURL(surl string) (*server.Server, *server.Options) { o := DefaultTestOptions o.Port = -1 fooAcc := server.NewAccount("FOO") o.Accounts = []*server.Account{server.NewAccount("$SYS"), fooAcc} o.SystemAccount = "$SYS" o.Users = []*server.User{ {Username: "foo", Password: "pass", Permissions: nil, Account: fooAcc}, } rurl, _ := url.Parse(surl) sysUrl, _ := url.Parse(strings.Replace(surl, rurl.User.Username(), "sys", -1)) o.LeafNode.Remotes = []*server.RemoteLeafOpts{ { URLs: []*url.URL{rurl}, LocalAccount: "FOO", }, { URLs: []*url.URL{sysUrl}, LocalAccount: "$SYS", }, } o.LeafNode.ReconnectInterval = 100 * time.Millisecond return RunServer(&o), &o } func TestSystemServiceSubscribersLeafNodesWithSystem(t *testing.T) { numServers, numClusters := 3, 3 sc := createSuperCluster(t, numServers, numClusters) defer sc.shutdown() sc.setupSystemServicesImports(t, "FOO") ci := rand.Int31n(int32(len(sc.clusters))) si := rand.Int31n(int32(len(sc.clusters[ci].servers))) s, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si] url := fmt.Sprintf("nats://%s:pass@%s:%d", "foo", opts.Host, opts.LeafNode.Port) ls, lopts := runSolicitLeafServerWithSystemToURL(url) defer ls.Shutdown() checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { if nln := s.NumLeafNodes(); nln != 2 { return fmt.Errorf("Expected a connected leafnode for server %q, got none", s.ID()) } return nil }) // This is so we can test when the subs on a leafnode are flushed to the connected supercluster. fsubj := "__leaf.flush__" fc := clientConnect(t, opts, "foo") defer fc.Close() fc.Subscribe(fsubj, func(m *nats.Msg) { m.Respond(nil) }) lnc := clientConnect(t, lopts, "foo") defer lnc.Close() flushLeaf := func() { if _, err := lnc.Request(fsubj, nil, time.Second); err != nil { t.Fatalf("Did not flush through to the supercluster: %v", err) } } for i := 0; i < 10; i++ { nc := clientConnect(t, sc.selectRandomServer(), "foo") defer nc.Close() nc.SubscribeSync(fmt.Sprintf("foo.bar.%d", i+1)) nc.QueueSubscribeSync("foo.bar.baz", "QG.22") nc.Flush() } nc := clientConnect(t, sc.clusters[0].opts[0], "foo") defer nc.Close() checkDbgNumSubs(t, nc, "foo.bar.3", 1) checkDbgNumSubs(t, nc, "foo.bar.*", 20) lnc.SubscribeSync("foo.bar.3") lnc.QueueSubscribeSync("foo.bar.baz", "QG.22") flushLeaf() // Since we are doing real tracking now on the other side, this will be off by 1 since we are counting // the leaf and the real sub. checkDbgNumSubs(t, nc, "foo.bar.3", 3) checkDbgNumSubs(t, nc, "foo.bar.baz QG.22", 12) } nats-server-2.10.27/test/test.go000066400000000000000000000467721477524627100164600ustar00rootroot00000000000000// Copyright 2012-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "crypto/rand" "encoding/hex" "encoding/json" "fmt" "io" "net" "os" "regexp" "runtime" "strconv" "strings" "testing" "time" "github.com/nats-io/nats-server/v2/server" ) // So we can pass tests and benchmarks.. type tLogger interface { Fatalf(format string, args ...any) Errorf(format string, args ...any) } // DefaultTestOptions are default options for the unit tests. var DefaultTestOptions = server.Options{ Host: "127.0.0.1", Port: 4222, NoLog: true, NoSigs: true, MaxControlLine: 4096, DisableShortFirstPing: true, } // RunDefaultServer starts a new Go routine based server using the default options func RunDefaultServer() *server.Server { return RunServer(&DefaultTestOptions) } func RunRandClientPortServer() *server.Server { opts := DefaultTestOptions opts.Port = -1 return RunServer(&opts) } // To turn on server tracing and debugging and logging which are // normally suppressed. var ( doLog = false doTrace = false doDebug = false ) // RunServer starts a new Go routine based server func RunServer(opts *server.Options) *server.Server { return RunServerCallback(opts, nil) } func RunServerCallback(opts *server.Options, callback func(*server.Server)) *server.Server { if opts == nil { opts = &DefaultTestOptions } // Optionally override for individual debugging of tests opts.NoLog = !doLog opts.Trace = doTrace opts.Debug = doDebug // For all tests in the "test" package, we will disable route pooling. opts.Cluster.PoolSize = -1 // Also disable compression for "test" package. opts.Cluster.Compression.Mode = server.CompressionOff opts.LeafNode.Compression.Mode = server.CompressionOff s, err := server.NewServer(opts) if err != nil || s == nil { panic(fmt.Sprintf("No NATS Server object returned: %v", err)) } if doLog { s.ConfigureLogger() } if callback != nil { callback(s) } // Run server in Go routine. go s.Start() // Wait for accept loop(s) to be started if !s.ReadyForConnections(10 * time.Second) { panic("Unable to start NATS Server in Go Routine") } return s } // LoadConfig loads a configuration from a filename func LoadConfig(configFile string) *server.Options { opts, err := server.ProcessConfigFile(configFile) if err != nil { panic(fmt.Sprintf("Error processing configuration file: %v", err)) } return opts } // RunServerWithConfig starts a new Go routine based server with a configuration file. func RunServerWithConfig(configFile string) (srv *server.Server, opts *server.Options) { opts = LoadConfig(configFile) srv = RunServer(opts) return } // RunServerWithConfigOverrides starts a new Go routine based server with a configuration file, // providing a callback to update the options configured. func RunServerWithConfigOverrides(configFile string, optsCallback func(*server.Options), svrCallback func(*server.Server)) (srv *server.Server, opts *server.Options) { opts = LoadConfig(configFile) if optsCallback != nil { optsCallback(opts) } srv = RunServerCallback(opts, svrCallback) return } func stackFatalf(t tLogger, f string, args ...any) { lines := make([]string, 0, 32) msg := fmt.Sprintf(f, args...) lines = append(lines, msg) // Ignore ourselves _, testFile, _, _ := runtime.Caller(0) // Generate the Stack of callers: for i := 0; true; i++ { _, file, line, ok := runtime.Caller(i) if !ok { break } if file == testFile { continue } msg := fmt.Sprintf("%d - %s:%d", i, file, line) lines = append(lines, msg) } t.Fatalf("%s", strings.Join(lines, "\n")) } func acceptRouteConn(t tLogger, host string, timeout time.Duration) net.Conn { l, e := net.Listen("tcp", host) if e != nil { stackFatalf(t, "Error listening for route connection on %v: %v", host, e) } defer l.Close() tl := l.(*net.TCPListener) tl.SetDeadline(time.Now().Add(timeout)) conn, err := l.Accept() tl.SetDeadline(time.Time{}) if err != nil { stackFatalf(t, "Did not receive a route connection request: %v", err) } return conn } func createRouteConn(t tLogger, host string, port int) net.Conn { return createClientConn(t, host, port) } func createClientConn(t tLogger, host string, port int) net.Conn { addr := fmt.Sprintf("%s:%d", host, port) c, err := net.DialTimeout("tcp", addr, 3*time.Second) if err != nil { stackFatalf(t, "Could not connect to server: %v\n", err) } return c } func checkSocket(t tLogger, addr string, wait time.Duration) { end := time.Now().Add(wait) for time.Now().Before(end) { conn, err := net.Dial("tcp", addr) if err != nil { // Retry after 50ms time.Sleep(50 * time.Millisecond) continue } conn.Close() // Wait a bit to give a chance to the server to remove this // "client" from its state, which may otherwise interfere with // some tests. time.Sleep(25 * time.Millisecond) return } // We have failed to bind the socket in the time allowed. t.Fatalf("Failed to connect to the socket: %q", addr) } func checkInfoMsg(t tLogger, c net.Conn) server.Info { buf := expectResult(t, c, infoRe) js := infoRe.FindAllSubmatch(buf, 1)[0][1] var sinfo server.Info err := json.Unmarshal(js, &sinfo) if err != nil { stackFatalf(t, "Could not unmarshal INFO json: %v\n", err) } return sinfo } func doHeadersConnect(t tLogger, c net.Conn, verbose, pedantic, ssl, headers bool) { checkInfoMsg(t, c) cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"headers\":%v}\r\n", verbose, pedantic, ssl, headers) sendProto(t, c, cs) } func doConnect(t tLogger, c net.Conn, verbose, pedantic, ssl bool) { doHeadersConnect(t, c, verbose, pedantic, ssl, false) } func doDefaultHeadersConnect(t tLogger, c net.Conn) { doHeadersConnect(t, c, false, false, false, true) } func doDefaultConnect(t tLogger, c net.Conn) { // Basic Connect doConnect(t, c, false, false, false) } const routeConnectProto = "CONNECT {\"verbose\":false,\"user\":\"%s\",\"pass\":\"%s\",\"name\":\"%s\",\"cluster\":\"xyz\"}\r\n" func doRouteAuthConnect(t tLogger, c net.Conn, user, pass, id string) { cs := fmt.Sprintf(routeConnectProto, user, pass, id) sendProto(t, c, cs) } func setupRouteEx(t tLogger, c net.Conn, opts *server.Options, id string) (sendFun, expectFun) { user := opts.Cluster.Username pass := opts.Cluster.Password doRouteAuthConnect(t, c, user, pass, id) return sendCommand(t, c), expectCommand(t, c) } func setupRoute(t tLogger, c net.Conn, opts *server.Options) (sendFun, expectFun) { u := make([]byte, 16) io.ReadFull(rand.Reader, u) id := fmt.Sprintf("ROUTER:%s", hex.EncodeToString(u)) return setupRouteEx(t, c, opts, id) } func setupHeaderConn(t tLogger, c net.Conn) (sendFun, expectFun) { doDefaultHeadersConnect(t, c) return sendCommand(t, c), expectCommand(t, c) } func setupConn(t tLogger, c net.Conn) (sendFun, expectFun) { doDefaultConnect(t, c) return sendCommand(t, c), expectCommand(t, c) } func setupConnWithProto(t tLogger, c net.Conn, proto int) (sendFun, expectFun) { checkInfoMsg(t, c) cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"protocol\":%d}\r\n", false, false, false, proto) sendProto(t, c, cs) return sendCommand(t, c), expectCommand(t, c) } func setupConnWithAccount(t tLogger, s *server.Server, c net.Conn, account string) (sendFun, expectFun) { info := checkInfoMsg(t, c) s.RegisterAccount(account) acc, err := s.LookupAccount(account) if err != nil { t.Fatalf("Unexpected Error: %v", err) } cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v}\r\n", false, false, false) sendProto(t, c, cs) send, expect := sendCommand(t, c), expectCommand(t, c) send("PING\r\n") expect(pongRe) nc := s.GetClient(info.CID) if nc == nil { t.Fatalf("Could not get client for CID:%d", info.CID) } nc.RegisterUser(&server.User{Account: acc}) return send, expect } func setupConnWithUserPass(t tLogger, c net.Conn, username, password string) (sendFun, expectFun) { checkInfoMsg(t, c) cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"protocol\":1,\"user\":%q,\"pass\":%q}\r\n", false, false, false, username, password) sendProto(t, c, cs) return sendCommand(t, c), expectLefMostCommand(t, c) } type sendFun func(string) type expectFun func(*regexp.Regexp) []byte // Closure version for easier reading func sendCommand(t tLogger, c net.Conn) sendFun { return func(op string) { sendProto(t, c, op) } } // Closure version for easier reading func expectCommand(t tLogger, c net.Conn) expectFun { return func(re *regexp.Regexp) []byte { return expectResult(t, c, re) } } // Closure version for easier reading func expectLefMostCommand(t tLogger, c net.Conn) expectFun { var buf []byte return func(re *regexp.Regexp) []byte { return expectLeftMostResult(t, c, re, &buf) } } // Send the protocol command to the server. func sendProto(t tLogger, c net.Conn, op string) { n, err := c.Write([]byte(op)) if err != nil { stackFatalf(t, "Error writing command to conn: %v\n", err) } if n != len(op) { stackFatalf(t, "Partial write: %d vs %d\n", n, len(op)) } } var ( anyRe = regexp.MustCompile(`.*`) infoRe = regexp.MustCompile(`INFO\s+([^\r\n]+)\r\n`) infoStartRe = regexp.MustCompile(`^INFO\s+([^\r\n]+)\r\n`) pingRe = regexp.MustCompile(`^PING\r\n`) pongRe = regexp.MustCompile(`^PONG\r\n`) hmsgRe = regexp.MustCompile(`(?:(?:HMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s+(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) msgRe = regexp.MustCompile(`(?:(?:MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) rawMsgRe = regexp.MustCompile(`(?:(?:MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n(.*?)))`) okRe = regexp.MustCompile(`\A\+OK\r\n`) errRe = regexp.MustCompile(`\A\-ERR\s+([^\r\n]+)\r\n`) connectRe = regexp.MustCompile(`CONNECT\s+([^\r\n]+)\r\n`) rsubRe = regexp.MustCompile(`RS\+\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) runsubRe = regexp.MustCompile(`RS\-\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) rmsgRe = regexp.MustCompile(`(?:(?:RMSG\s+([^\s]+)\s+([^\s]+)\s+(?:([|+]\s+([\w\s]+)|[^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) asubRe = regexp.MustCompile(`A\+\s+([^\r\n]+)\r\n`) aunsubRe = regexp.MustCompile(`A\-\s+([^\r\n]+)\r\n`) lsubRe = regexp.MustCompile(`LS\+\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) lunsubRe = regexp.MustCompile(`LS\-\s+([^\s]+)\s*([^\s]+)\s*([^\s]+)?\r\n`) lmsgRe = regexp.MustCompile(`(?:(?:LMSG\s+([^\s]+)\s+(?:([|+]\s+([\w\s]+)|[^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) rlsubRe = regexp.MustCompile(`LS\+\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) rlunsubRe = regexp.MustCompile(`LS\-\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) ) const ( // Regular Messages subIndex = 1 sidIndex = 2 replyIndex = 4 lenIndex = 5 msgIndex = 6 // Headers hlenIndex = 5 tlenIndex = 6 hmsgIndex = 7 // Routed Messages accIndex = 1 rsubIndex = 2 replyAndQueueIndex = 3 ) // Test result from server against regexp and return left most match func expectLeftMostResult(t tLogger, c net.Conn, re *regexp.Regexp, buf *[]byte) []byte { recv := func() []byte { expBuf := make([]byte, 32768) // Wait for commands to be processed and results queued for read c.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err := c.Read(expBuf) c.SetReadDeadline(time.Time{}) if n <= 0 && err != nil { stackFatalf(t, "Error reading from conn: %v\n", err) } return expBuf[:n] } if len(*buf) == 0 { *buf = recv() } emptyCnt := 0 for { result := re.Find(*buf) if result == nil { emptyCnt++ if emptyCnt > 5 { stackFatalf(t, "Reading empty data too often\n") } *buf = append(*buf, recv()...) } else { cutIdx := strings.Index(string(*buf), string(result)) + len(result) *buf = (*buf)[cutIdx:] return result } } } // Test result from server against regexp func expectResult(t tLogger, c net.Conn, re *regexp.Regexp) []byte { expBuf := make([]byte, 32768) // Wait for commands to be processed and results queued for read c.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err := c.Read(expBuf) c.SetReadDeadline(time.Time{}) if n <= 0 && err != nil { stackFatalf(t, "Error reading from conn: %v\n", err) } buf := expBuf[:n] if !re.Match(buf) { stackFatalf(t, "Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'", buf, re) } return buf } func peek(c net.Conn) []byte { expBuf := make([]byte, 32768) c.SetReadDeadline(time.Now().Add(50 * time.Millisecond)) n, err := c.Read(expBuf) c.SetReadDeadline(time.Time{}) if err != nil || n <= 0 { return nil } return expBuf } func expectDisconnect(t *testing.T, c net.Conn) { t.Helper() var b [8]byte c.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) _, err := c.Read(b[:]) c.SetReadDeadline(time.Time{}) if err != io.EOF { t.Fatalf("Expected a disconnect") } } func expectNothing(t tLogger, c net.Conn) { expectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond)) } func expectNothingTimeout(t tLogger, c net.Conn, dl time.Time) { expBuf := make([]byte, 32) c.SetReadDeadline(dl) n, err := c.Read(expBuf) c.SetReadDeadline(time.Time{}) if err == nil && n > 0 { stackFatalf(t, "Expected nothing, received: '%q'\n", expBuf[:n]) } } // This will check that we got what we expected from a normal message. func checkMsg(t tLogger, m [][]byte, subject, sid, reply, len, msg string) { if string(m[subIndex]) != subject { stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[subIndex]) } if sid != "" && string(m[sidIndex]) != sid { stackFatalf(t, "Did not get correct sid: expected '%s' got '%s'\n", sid, m[sidIndex]) } if string(m[replyIndex]) != reply { stackFatalf(t, "Did not get correct reply: expected '%s' got '%s'\n", reply, m[replyIndex]) } if string(m[lenIndex]) != len { stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex]) } if string(m[msgIndex]) != msg { stackFatalf(t, "Did not get correct msg: expected '%s' got '%s'\n", msg, m[msgIndex]) } } func checkRmsg(t tLogger, m [][]byte, account, subject, replyAndQueues, len, msg string) { if string(m[accIndex]) != account { stackFatalf(t, "Did not get correct account: expected '%s' got '%s'\n", account, m[accIndex]) } if string(m[rsubIndex]) != subject { stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[rsubIndex]) } if string(m[lenIndex]) != len { stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex]) } if string(m[replyAndQueueIndex]) != replyAndQueues { stackFatalf(t, "Did not get correct reply/queues: expected '%s' got '%s'\n", replyAndQueues, m[replyAndQueueIndex]) } } func checkLmsg(t tLogger, m [][]byte, subject, replyAndQueues, len, msg string) { if string(m[rsubIndex-1]) != subject { stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[rsubIndex-1]) } if string(m[lenIndex-1]) != len { stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex-1]) } if string(m[replyAndQueueIndex-1]) != replyAndQueues { stackFatalf(t, "Did not get correct reply/queues: expected '%s' got '%s'\n", replyAndQueues, m[replyAndQueueIndex-1]) } } // This will check that we got what we expected from a header message. func checkHmsg(t tLogger, m [][]byte, subject, sid, reply, hlen, len, hdr, msg string) { if string(m[subIndex]) != subject { stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[subIndex]) } if sid != "" && string(m[sidIndex]) != sid { stackFatalf(t, "Did not get correct sid: expected '%s' got '%s'\n", sid, m[sidIndex]) } if string(m[replyIndex]) != reply { stackFatalf(t, "Did not get correct reply: expected '%s' got '%s'\n", reply, m[replyIndex]) } if string(m[hlenIndex]) != hlen { stackFatalf(t, "Did not get correct header length: expected '%s' got '%s'\n", hlen, m[hlenIndex]) } if string(m[tlenIndex]) != len { stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[tlenIndex]) } // Extract the payload and break up the headers and msg. payload := string(m[hmsgIndex]) hi, _ := strconv.Atoi(hlen) rhdr, rmsg := payload[:hi], payload[hi:] if rhdr != hdr { stackFatalf(t, "Did not get correct headers: expected '%s' got '%s'\n", hdr, rhdr) } if rmsg != msg { stackFatalf(t, "Did not get correct msg: expected '%s' got '%s'\n", msg, rmsg) } } // Closure for expectMsgs func expectRmsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { return func(expected int) [][][]byte { buf := ef(rmsgRe) matches := rmsgRe.FindAllSubmatch(buf, -1) if len(matches) != expected { stackFatalf(t, "Did not get correct # routed msgs: %d vs %d\n", len(matches), expected) } return matches } } // Closure for expectHMsgs func expectHeaderMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { return func(expected int) [][][]byte { buf := ef(hmsgRe) matches := hmsgRe.FindAllSubmatch(buf, -1) if len(matches) != expected { stackFatalf(t, "Did not get correct # msgs: %d vs %d\n", len(matches), expected) } return matches } } // Closure for expectMsgs func expectMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { return func(expected int) [][][]byte { buf := ef(msgRe) matches := msgRe.FindAllSubmatch(buf, -1) if len(matches) != expected { stackFatalf(t, "Did not get correct # msgs: %d vs %d\n", len(matches), expected) } return matches } } // This will check that the matches include at least one of the sids. Useful for checking // that we received messages on a certain queue group. func checkForQueueSid(t tLogger, matches [][][]byte, sids []string) { seen := make(map[string]int, len(sids)) for _, sid := range sids { seen[sid] = 0 } for _, m := range matches { sid := string(m[sidIndex]) if _, ok := seen[sid]; ok { seen[sid]++ } } // Make sure we only see one and exactly one. total := 0 for _, n := range seen { total += n } if total != 1 { stackFatalf(t, "Did not get a msg for queue sids group: expected 1 got %d\n", total) } } // This will check that the matches include all of the sids. Useful for checking // that we received messages on all subscribers. func checkForPubSids(t tLogger, matches [][][]byte, sids []string) { seen := make(map[string]int, len(sids)) for _, sid := range sids { seen[sid] = 0 } for _, m := range matches { sid := string(m[sidIndex]) if _, ok := seen[sid]; ok { seen[sid]++ } } // Make sure we only see one and exactly one for each sid. for sid, n := range seen { if n != 1 { stackFatalf(t, "Did not get a msg for sid[%s]: expected 1 got %d\n", sid, n) } } } // Helper function to generate next opts to make sure no port conflicts etc. func nextServerOpts(opts *server.Options) *server.Options { nopts := opts.Clone() nopts.Port++ nopts.Cluster.Port++ nopts.HTTPPort++ return nopts } func createTempFile(t testing.TB, prefix string) *os.File { t.Helper() file, err := os.CreateTemp(t.TempDir(), prefix) if err != nil { t.Fatal(err) } return file } nats-server-2.10.27/test/test_test.go000066400000000000000000000071451477524627100175060ustar00rootroot00000000000000// Copyright 2016-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "context" "fmt" "net" "strconv" "strings" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/server" "golang.org/x/time/rate" ) func checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) { t.Helper() timeout := time.Now().Add(totalWait) var err error for time.Now().Before(timeout) { err = f() if err == nil { return } time.Sleep(sleepDur) } if err != nil { t.Fatal(err.Error()) } } // Slow Proxy - For introducing RTT and BW constraints. type slowProxy struct { listener net.Listener conns []net.Conn o *server.Options u string } func newSlowProxy(rtt time.Duration, up, down int, opts *server.Options) *slowProxy { saddr := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) hp := net.JoinHostPort("127.0.0.1", "0") l, e := net.Listen("tcp", hp) if e != nil { panic(fmt.Sprintf("Error listening on port: %s, %q", hp, e)) } port := l.Addr().(*net.TCPAddr).Port sp := &slowProxy{listener: l} go func() { client, err := l.Accept() if err != nil { return } server, err := net.DialTimeout("tcp", saddr, time.Second) if err != nil { panic("Can't connect to server") } sp.conns = append(sp.conns, client, server) go sp.loop(rtt, up, client, server) go sp.loop(rtt, down, server, client) }() sp.o = &server.Options{Host: "127.0.0.1", Port: port} sp.u = fmt.Sprintf("nats://%s:%d", sp.o.Host, sp.o.Port) return sp } func (sp *slowProxy) opts() *server.Options { return sp.o } //lint:ignore U1000 Referenced in norace_test.go func (sp *slowProxy) clientURL() string { return sp.u } func (sp *slowProxy) loop(rtt time.Duration, tbw int, r, w net.Conn) { delay := rtt / 2 const rbl = 1024 var buf [rbl]byte ctx := context.Background() rl := rate.NewLimiter(rate.Limit(tbw), rbl) for fr := true; ; { sr := time.Now() n, err := r.Read(buf[:]) if err != nil { return } // RTT delays if fr || time.Since(sr) > 2*time.Millisecond { fr = false time.Sleep(delay) } if err := rl.WaitN(ctx, n); err != nil { return } if _, err = w.Write(buf[:n]); err != nil { return } } } func (sp *slowProxy) stop() { if sp.listener != nil { sp.listener.Close() sp.listener = nil for _, c := range sp.conns { c.Close() } } } // Dummy Logger type dummyLogger struct { sync.Mutex msg string } func (d *dummyLogger) Fatalf(format string, args ...any) { d.Lock() d.msg = fmt.Sprintf(format, args...) d.Unlock() } func (d *dummyLogger) Errorf(format string, args ...any) { } func (d *dummyLogger) Debugf(format string, args ...any) { } func (d *dummyLogger) Tracef(format string, args ...any) { } func (d *dummyLogger) Noticef(format string, args ...any) { } func (d *dummyLogger) Warnf(format string, args ...any) { } func TestStackFatal(t *testing.T) { d := &dummyLogger{} stackFatalf(d, "test stack %d", 1) if !strings.HasPrefix(d.msg, "test stack 1") { t.Fatalf("Unexpected start of stack: %v", d.msg) } if !strings.Contains(d.msg, "test_test.go") { t.Fatalf("Unexpected stack: %v", d.msg) } } nats-server-2.10.27/test/tls_test.go000066400000000000000000001516371477524627100173370ustar00rootroot00000000000000// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "bufio" "crypto/tls" "crypto/x509" "errors" "fmt" "net" "net/url" "os" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats-server/v2/server" ) var noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {} func TestTLSConnection(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls.conf") defer srv.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) nc, err := nats.Connect(nurl) if err == nil { nc.Close() t.Fatalf("Expected error trying to connect to secure server") } // Do simple SecureConnect nc, err = nats.Connect(fmt.Sprintf("tls://%s/", endpoint)) if err == nil { nc.Close() t.Fatalf("Expected error trying to connect to secure server with no auth") } // Now do more advanced checking, verifying servername and using rootCA. nc, err = nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) if err != nil { t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) } defer nc.Close() subj := "foo-tls" sub, _ := nc.SubscribeSync(subj) nc.Publish(subj, []byte("We are Secure!")) nc.Flush() nmsgs, _, _ := sub.Pending() if nmsgs != 1 { t.Fatalf("Expected to receive a message over the TLS connection") } } // TestTLSInProcessConnection checks that even if TLS is enabled on the server, // that an in-process connection that does *not* use TLS still connects successfully. func TestTLSInProcessConnection(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls.conf") defer srv.Shutdown() nc, err := nats.Connect("", nats.InProcessServer(srv), nats.UserInfo(opts.Username, opts.Password)) if err != nil { t.Fatal(err) } defer nc.Close() if nc.TLSRequired() { t.Fatalf("Shouldn't have required TLS for in-process connection") } if _, err = nc.TLSConnectionState(); err == nil { t.Fatal("Should have got an error retrieving TLS connection state") } } func TestTLSClientCertificate(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tlsverify.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) _, err := nats.Connect(nurl) if err == nil { t.Fatalf("Expected error trying to connect to secure server without a certificate") } // Load client certificate to successfully connect. certFile := "./configs/certs/client-cert.pem" keyFile := "./configs/certs/client-key.pem" cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { t.Fatalf("error parsing X509 certificate/key pair: %v", err) } // Load in root CA for server verification rootPEM, err := os.ReadFile("./configs/certs/ca.pem") if err != nil || rootPEM == nil { t.Fatalf("failed to read root certificate") } pool := x509.NewCertPool() ok := pool.AppendCertsFromPEM([]byte(rootPEM)) if !ok { t.Fatalf("failed to parse root certificate") } config := &tls.Config{ Certificates: []tls.Certificate{cert}, ServerName: opts.Host, RootCAs: pool, MinVersion: tls.VersionTLS12, } copts := nats.GetDefaultOptions() copts.Url = nurl copts.Secure = true copts.TLSConfig = config nc, err := copts.Connect() if err != nil { t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) } nc.Flush() defer nc.Close() } func TestTLSClientCertificateHasUserID(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls_cert_id.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/client-id-auth-cert.pem", "./configs/certs/client-id-auth-key.pem"), nats.RootCAs("./configs/certs/ca.pem")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc.Close() } func TestTLSClientCertificateCheckWithAllowedConnectionTypes(t *testing.T) { conf := createConfFile(t, []byte( ` listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" timeout: 2 ca_file: "./configs/certs/ca.pem" verify_and_map: true } authorization { users = [ {user: derek@nats.io, permissions: { publish:"foo" }, allowed_connection_types: ["WEBSOCKET"]} ] } `)) s, o := RunServerWithConfig(conf) defer s.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", o.Host, o.Port) nc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/client-id-auth-cert.pem", "./configs/certs/client-id-auth-key.pem"), nats.RootCAs("./configs/certs/ca.pem")) if err == nil { if nc != nil { nc.Close() } t.Fatal("Expected connection to fail, it did not") } } func TestTLSClientCertificateCNBasedAuth(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls_cert_cn.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) errCh1 := make(chan error) errCh2 := make(chan error) // Using the default permissions nc1, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/tlsauth/client.pem", "./configs/certs/tlsauth/client-key.pem"), nats.RootCAs("./configs/certs/tlsauth/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { errCh1 <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc1.Close() // Admin permissions can publish to '>' nc2, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/tlsauth/client2.pem", "./configs/certs/tlsauth/client2-key.pem"), nats.RootCAs("./configs/certs/tlsauth/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { errCh2 <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc2.Close() err = nc1.Publish("foo.bar", []byte("hi")) if err != nil { t.Fatal(err) } _, err = nc1.SubscribeSync("foo.>") if err != nil { t.Fatal(err) } nc1.Flush() sub, err := nc2.SubscribeSync(">") if err != nil { t.Fatal(err) } nc2.Flush() err = nc2.Publish("hello", []byte("hi")) if err != nil { t.Fatal(err) } nc2.Flush() _, err = sub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // Wait for a couple of errors var count int Loop: for { select { case err := <-errCh1: if err != nil { count++ } if count == 2 { break Loop } case err := <-errCh2: if err != nil { t.Fatalf("Received unexpected auth error from client: %s", err) } case <-time.After(2 * time.Second): t.Fatalf("Timed out expecting auth errors") } } } func TestTLSClientCertificateSANsBasedAuth(t *testing.T) { // In this test we have 3 clients, one with permissions defined // for SAN 'app.nats.dev', other for SAN 'app.nats.prod' and another // one without the default permissions for the CN. srv, opts := RunServerWithConfig("./configs/tls_cert_san_auth.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) defaultErrCh := make(chan error) devErrCh := make(chan error) prodErrCh := make(chan error) // default: Using the default permissions (no SANs) // // Subject: CN = www.nats.io // defaultc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/client.pem", "./configs/certs/sans/client-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { defaultErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer defaultc.Close() // dev: Using SAN 'app.nats.dev' with permissions to its own sandbox. // // Subject: CN = www.nats.io // // X509v3 Subject Alternative Name: // DNS:app.nats.dev, DNS:*.app.nats.dev // devc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/dev.pem", "./configs/certs/sans/dev-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { devErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer devc.Close() // prod: Using SAN 'app.nats.prod' with all permissions. // // Subject: CN = www.nats.io // // X509v3 Subject Alternative Name: // DNS:app.nats.prod, DNS:*.app.nats.prod // prodc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/prod.pem", "./configs/certs/sans/prod-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { prodErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer prodc.Close() // No permissions to publish or subscribe on foo.> err = devc.Publish("foo.bar", []byte("hi")) if err != nil { t.Fatal(err) } _, err = devc.SubscribeSync("foo.>") if err != nil { t.Fatal(err) } devc.Flush() prodSub, err := prodc.SubscribeSync(">") if err != nil { t.Fatal(err) } prodc.Flush() err = prodc.Publish("hello", []byte("hi")) if err != nil { t.Fatal(err) } prodc.Flush() // prod: can receive message on wildcard subscription. _, err = prodSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // dev: enough permissions to publish to sandbox subject. devSub, err := devc.SubscribeSync("sandbox.>") if err != nil { t.Fatal(err) } devc.Flush() err = devc.Publish("sandbox.foo.bar", []byte("hi!")) if err != nil { t.Fatal(err) } // both dev and prod clients can receive message _, err = devSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } _, err = prodSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // default: no enough permissions _, err = defaultc.SubscribeSync("sandbox.>") if err != nil { t.Fatal(err) } defaultSub, err := defaultc.SubscribeSync("public.>") if err != nil { t.Fatal(err) } defaultc.Flush() err = devc.Publish("public.foo.bar", []byte("hi!")) if err != nil { t.Fatal(err) } _, err = defaultSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // Wait for a couple of errors var count int Loop: for { select { case err := <-defaultErrCh: if err != nil { count++ } if count == 3 { break Loop } case err := <-devErrCh: if err != nil { count++ } if count == 3 { break Loop } case err := <-prodErrCh: if err != nil { t.Fatalf("Received unexpected auth error from client: %s", err) } case <-time.After(2 * time.Second): t.Fatalf("Timed out expecting auth errors") } } } func TestTLSClientCertificateTLSAuthMultipleOptions(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls_cert_san_emails.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) defaultErrCh := make(chan error) devErrCh := make(chan error) prodErrCh := make(chan error) // default: Using the default permissions, there are SANs // present in the cert but they are not users in the NATS config // so the subject is used instead. // // Subject: CN = www.nats.io // // X509v3 Subject Alternative Name: // DNS:app.nats.dev, DNS:*.app.nats.dev // defaultc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/dev.pem", "./configs/certs/sans/dev-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { defaultErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer defaultc.Close() // dev: Using SAN to validate user, even if emails are present in the config. // // Subject: CN = www.nats.io // // X509v3 Subject Alternative Name: // DNS:app.nats.dev, email:admin@app.nats.dev, email:root@app.nats.dev // devc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/dev-email.pem", "./configs/certs/sans/dev-email-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { devErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer devc.Close() // prod: Using SAN '*.app.nats.prod' with all permissions, which is not the first SAN. // // Subject: CN = www.nats.io // // X509v3 Subject Alternative Name: // DNS:app.nats.prod, DNS:*.app.nats.prod // prodc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/sans/prod.pem", "./configs/certs/sans/prod-key.pem"), nats.RootCAs("./configs/certs/sans/ca.pem"), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { prodErrCh <- err }), ) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer prodc.Close() // No permissions to publish or subscribe on foo.> err = devc.Publish("foo.bar", []byte("hi")) if err != nil { t.Fatal(err) } _, err = devc.SubscribeSync("foo.>") if err != nil { t.Fatal(err) } devc.Flush() prodSub, err := prodc.SubscribeSync(">") if err != nil { t.Fatal(err) } prodc.Flush() err = prodc.Publish("hello", []byte("hi")) if err != nil { t.Fatal(err) } prodc.Flush() // prod: can receive message on wildcard subscription. _, err = prodSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // dev: enough permissions to publish to sandbox subject. devSub, err := devc.SubscribeSync("sandbox.>") if err != nil { t.Fatal(err) } devc.Flush() err = devc.Publish("sandbox.foo.bar", []byte("hi!")) if err != nil { t.Fatal(err) } // both dev and prod clients can receive message _, err = devSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } _, err = prodSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // default: no enough permissions _, err = defaultc.SubscribeSync("sandbox.>") if err != nil { t.Fatal(err) } defaultSub, err := defaultc.SubscribeSync("public.>") if err != nil { t.Fatal(err) } defaultc.Flush() err = devc.Publish("public.foo.bar", []byte("hi!")) if err != nil { t.Fatal(err) } _, err = defaultSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } // Wait for a couple of errors var count int Loop: for { select { case err := <-defaultErrCh: if err != nil { count++ } if count == 3 { break Loop } case err := <-devErrCh: if err != nil { count++ } if count == 3 { break Loop } case err := <-prodErrCh: if err != nil { t.Fatalf("Received unexpected auth error from client: %s", err) } case <-time.After(2 * time.Second): t.Fatalf("Timed out expecting auth errors") } } } func TestTLSRoutesCertificateCNBasedAuth(t *testing.T) { // Base config for the servers optsA := LoadConfig("./configs/tls_cert_cn_routes.conf") optsB := LoadConfig("./configs/tls_cert_cn_routes.conf") optsC := LoadConfig("./configs/tls_cert_cn_routes_invalid_auth.conf") // TLS map should have been enabled if !optsA.Cluster.TLSMap || !optsB.Cluster.TLSMap || !optsC.Cluster.TLSMap { t.Error("Expected Cluster TLS verify and map feature to be activated") } routeURLs := "nats://localhost:9935, nats://localhost:9936, nats://localhost:9937" optsA.Host = "127.0.0.1" optsA.Port = 9335 optsA.Cluster.Name = "xyz" optsA.Cluster.Host = optsA.Host optsA.Cluster.Port = 9935 optsA.Routes = server.RoutesFromStr(routeURLs) optsA.NoSystemAccount = true srvA := RunServer(optsA) defer srvA.Shutdown() optsB.Host = "127.0.0.1" optsB.Port = 9336 optsB.Cluster.Name = "xyz" optsB.Cluster.Host = optsB.Host optsB.Cluster.Port = 9936 optsB.Routes = server.RoutesFromStr(routeURLs) optsB.NoSystemAccount = true srvB := RunServer(optsB) defer srvB.Shutdown() optsC.Host = "127.0.0.1" optsC.Port = 9337 optsC.Cluster.Name = "xyz" optsC.Cluster.Host = optsC.Host optsC.Cluster.Port = 9937 optsC.Routes = server.RoutesFromStr(routeURLs) optsC.NoSystemAccount = true srvC := RunServer(optsC) defer srvC.Shutdown() // srvC is not connected to srvA and srvB due to wrong cert checkClusterFormed(t, srvA, srvB) nc1, err := nats.Connect(fmt.Sprintf("%s:%d", optsA.Host, optsA.Port), nats.Name("A")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc1.Close() nc2, err := nats.Connect(fmt.Sprintf("%s:%d", optsB.Host, optsB.Port), nats.Name("B")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc2.Close() // This client is partitioned from rest of cluster due to using wrong cert. nc3, err := nats.Connect(fmt.Sprintf("%s:%d", optsC.Host, optsC.Port), nats.Name("C")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc3.Close() // Allowed via permissions to broadcast to other members of cluster. publicSub, err := nc1.SubscribeSync("public.>") if err != nil { t.Fatal(err) } // Not part of cluster permissions so message is not forwarded. privateSub, err := nc1.SubscribeSync("private.>") if err != nil { t.Fatal(err) } // This ensures that srvA has received the sub, not srvB nc1.Flush() checkFor(t, time.Second, 15*time.Millisecond, func() error { if n := srvB.NumSubscriptions(); n != 1 { return fmt.Errorf("Expected 1 sub, got %v", n) } return nil }) // Not forwarded by cluster so won't be received. err = nc2.Publish("private.foo", []byte("private message on server B")) if err != nil { t.Fatal(err) } err = nc2.Publish("public.foo", []byte("public message from server B to server A")) if err != nil { t.Fatal(err) } nc2.Flush() err = nc3.Publish("public.foo", []byte("dropped message since unauthorized server")) if err != nil { t.Fatal(err) } nc3.Flush() // Message will be received since allowed via permissions _, err = publicSub.NextMsg(1 * time.Second) if err != nil { t.Fatalf("Error during wait for next message: %s", err) } msg, err := privateSub.NextMsg(500 * time.Millisecond) if err == nil { t.Errorf("Received unexpected message from private sub: %+v", msg) } msg, err = publicSub.NextMsg(500 * time.Millisecond) if err == nil { t.Errorf("Received unexpected message from private sub: %+v", msg) } } func TestTLSGatewaysCertificateCNBasedAuth(t *testing.T) { // Base config for the servers optsA := LoadConfig("./configs/tls_cert_cn_gateways.conf") optsB := LoadConfig("./configs/tls_cert_cn_gateways.conf") optsC := LoadConfig("./configs/tls_cert_cn_gateways_invalid_auth.conf") // TLS map should have been enabled if !optsA.Gateway.TLSMap || !optsB.Gateway.TLSMap || !optsC.Gateway.TLSMap { t.Error("Expected Cluster TLS verify and map feature to be activated") } gwA, err := url.Parse("nats://localhost:9995") if err != nil { t.Fatal(err) } gwB, err := url.Parse("nats://localhost:9996") if err != nil { t.Fatal(err) } gwC, err := url.Parse("nats://localhost:9997") if err != nil { t.Fatal(err) } optsA.Host = "127.0.0.1" optsA.Port = -1 optsA.Gateway.Name = "A" optsA.Gateway.Port = 9995 optsB.Host = "127.0.0.1" optsB.Port = -1 optsB.Gateway.Name = "B" optsB.Gateway.Port = 9996 optsC.Host = "127.0.0.1" optsC.Port = -1 optsC.Gateway.Name = "C" optsC.Gateway.Port = 9997 gateways := make([]*server.RemoteGatewayOpts, 3) gateways[0] = &server.RemoteGatewayOpts{ Name: optsA.Gateway.Name, URLs: []*url.URL{gwA}, } gateways[1] = &server.RemoteGatewayOpts{ Name: optsB.Gateway.Name, URLs: []*url.URL{gwB}, } gateways[2] = &server.RemoteGatewayOpts{ Name: optsC.Gateway.Name, URLs: []*url.URL{gwC}, } optsA.Gateway.Gateways = gateways optsB.Gateway.Gateways = gateways optsC.Gateway.Gateways = gateways server.SetGatewaysSolicitDelay(100 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() srvA := RunServer(optsA) defer srvA.Shutdown() srvB := RunServer(optsB) defer srvB.Shutdown() srvC := RunServer(optsC) defer srvC.Shutdown() // Because we need to use "localhost" in the gw URLs (to match // hostname in the user/CN), the server may try to connect to // a [::1], etc.. that may or may not work, so give a lot of // time for that to complete ok. waitForOutboundGateways(t, srvA, 1, 5*time.Second) waitForOutboundGateways(t, srvB, 1, 5*time.Second) nc1, err := nats.Connect(srvA.Addr().String(), nats.Name("A")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc1.Close() nc2, err := nats.Connect(srvB.Addr().String(), nats.Name("B")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc2.Close() nc3, err := nats.Connect(srvC.Addr().String(), nats.Name("C")) if err != nil { t.Fatalf("Expected to connect, got %v", err) } defer nc3.Close() received := make(chan *nats.Msg) _, err = nc1.Subscribe("foo", func(msg *nats.Msg) { select { case received <- msg: default: } }) if err != nil { t.Error(err) } _, err = nc3.Subscribe("help", func(msg *nats.Msg) { nc3.Publish(msg.Reply, []byte("response")) }) if err != nil { t.Error(err) } go func() { for range time.NewTicker(10 * time.Millisecond).C { if nc2.IsClosed() { return } nc2.Publish("foo", []byte("bar")) } }() select { case <-received: case <-time.After(2 * time.Second): t.Error("Timed out waiting for gateway messages") } msg, err := nc2.Request("help", []byte("should fail"), 100*time.Millisecond) if err == nil { t.Errorf("Expected to not receive any messages, got: %+v", msg) } } func TestTLSVerifyClientCertificate(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tlsverify_noca.conf") defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) // The client is configured properly, but the server has no CA // to verify the client certificate. Connection should fail. nc, err := nats.Connect(nurl, nats.ClientCert("./configs/certs/client-cert.pem", "./configs/certs/client-key.pem"), nats.RootCAs("./configs/certs/ca.pem")) if err == nil { nc.Close() t.Fatal("Expected failure to connect, did not") } } func TestTLSConnectionTimeout(t *testing.T) { opts := LoadConfig("./configs/tls.conf") opts.TLSTimeout = 0.25 srv := RunServer(opts) defer srv.Shutdown() // Dial with normal TCP endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) conn, err := net.Dial("tcp", endpoint) if err != nil { t.Fatalf("Could not connect to %q", endpoint) } defer conn.Close() // Read deadlines conn.SetReadDeadline(time.Now().Add(2 * time.Second)) // Read the INFO string. br := bufio.NewReader(conn) info, err := br.ReadString('\n') if err != nil { t.Fatalf("Failed to read INFO - %v", err) } if !strings.HasPrefix(info, "INFO ") { t.Fatalf("INFO response incorrect: %s\n", info) } wait := time.Duration(opts.TLSTimeout * float64(time.Second)) time.Sleep(wait) // Read deadlines conn.SetReadDeadline(time.Now().Add(2 * time.Second)) tlsErr, err := br.ReadString('\n') if err == nil && !strings.Contains(tlsErr, "-ERR 'Secure Connection - TLS Required") { t.Fatalf("TLS Timeout response incorrect: %q\n", tlsErr) } } // Ensure there is no race between authorization timeout and TLS handshake. func TestTLSAuthorizationShortTimeout(t *testing.T) { opts := LoadConfig("./configs/tls.conf") opts.AuthTimeout = 0.001 srv := RunServer(opts) defer srv.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) // Expect an error here (no CA) but not a TLS oversized record error which // indicates the authorization timeout fired too soon. _, err := nats.Connect(nurl) if err == nil { t.Fatal("Expected error trying to connect to secure server") } if strings.Contains(err.Error(), "oversized record") { t.Fatal("Corrupted TLS handshake:", err) } } func TestClientTLSAndNonTLSConnections(t *testing.T) { s, opts := RunServerWithConfig("./configs/tls_mixed.conf") defer s.Shutdown() surl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) nc, err := nats.Connect(surl, nats.RootCAs("./configs/certs/ca.pem")) if err != nil { t.Fatalf("Failed to connect with TLS: %v", err) } defer nc.Close() // Now also make sure we can connect through plain text. nurl := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port) nc2, err := nats.Connect(nurl) if err != nil { t.Fatalf("Failed to connect without TLS: %v", err) } defer nc2.Close() // Make sure they can go back and forth. sub, err := nc.SubscribeSync("foo") if err != nil { t.Fatalf("Error subscribing: %v", err) } sub2, err := nc2.SubscribeSync("bar") if err != nil { t.Fatalf("Error subscribing: %v", err) } nc.Flush() nc2.Flush() nmsgs := 100 for i := 0; i < nmsgs; i++ { nc2.Publish("foo", []byte("HELLO FROM PLAINTEXT")) nc.Publish("bar", []byte("HELLO FROM TLS")) } // Now wait for the messages. checkFor(t, time.Second, 10*time.Millisecond, func() error { if msgs, _, err := sub.Pending(); err != nil || msgs != nmsgs { return fmt.Errorf("Did not receive the correct number of messages: %d", msgs) } if msgs, _, err := sub2.Pending(); err != nil || msgs != nmsgs { return fmt.Errorf("Did not receive the correct number of messages: %d", msgs) } return nil }) } func stressConnect(t *testing.T, wg *sync.WaitGroup, errCh chan error, url string, index int) { defer wg.Done() subName := fmt.Sprintf("foo.%d", index) for i := 0; i < 33; i++ { nc, err := nats.Connect(url, nats.RootCAs("./configs/certs/ca.pem")) if err != nil { errCh <- fmt.Errorf("Unable to create TLS connection: %v\n", err) return } defer nc.Close() sub, err := nc.SubscribeSync(subName) if err != nil { errCh <- fmt.Errorf("Unable to subscribe on '%s': %v\n", subName, err) return } if err := nc.Publish(subName, []byte("secure data")); err != nil { errCh <- fmt.Errorf("Unable to send on '%s': %v\n", subName, err) } if _, err := sub.NextMsg(2 * time.Second); err != nil { errCh <- fmt.Errorf("Unable to get next message: %v\n", err) } nc.Close() } errCh <- nil } func TestTLSStressConnect(t *testing.T) { opts, err := server.ProcessConfigFile("./configs/tls.conf") if err != nil { panic(fmt.Sprintf("Error processing configuration file: %v", err)) } opts.NoSigs, opts.NoLog = true, true // For this test, remove the authorization opts.Username = "" opts.Password = "" // Increase ssl timeout opts.TLSTimeout = 2.0 srv := RunServer(opts) defer srv.Shutdown() nurl := fmt.Sprintf("tls://%s:%d", opts.Host, opts.Port) threadCount := 3 errCh := make(chan error, threadCount) var wg sync.WaitGroup wg.Add(threadCount) for i := 0; i < threadCount; i++ { go stressConnect(t, &wg, errCh, nurl, i) } wg.Wait() var lastError error for i := 0; i < threadCount; i++ { err := <-errCh if err != nil { lastError = err } } if lastError != nil { t.Fatalf("%v\n", lastError) } } func TestTLSBadAuthError(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls.conf") defer srv.Shutdown() endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, "NOT_THE_PASSWORD", endpoint) _, err := nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) if err == nil { t.Fatalf("Expected error trying to connect to secure server") } if strings.ToLower(err.Error()) != nats.ErrAuthorization.Error() { t.Fatalf("Expected and auth violation, got %v\n", err) } } func TestTLSConnectionCurvePref(t *testing.T) { srv, opts := RunServerWithConfig("./configs/tls_curve_pref.conf") defer srv.Shutdown() if len(opts.TLSConfig.CurvePreferences) != 1 { t.Fatal("Invalid curve preference loaded.") } if opts.TLSConfig.CurvePreferences[0] != tls.CurveP256 { t.Fatalf("Invalid curve preference loaded [%v].", opts.TLSConfig.CurvePreferences[0]) } endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port) nurl := fmt.Sprintf("tls://%s:%s@%s/", opts.Username, opts.Password, endpoint) nc, err := nats.Connect(nurl) if err == nil { nc.Close() t.Fatalf("Expected error trying to connect to secure server") } // Do simple SecureConnect nc, err = nats.Connect(fmt.Sprintf("tls://%s/", endpoint)) if err == nil { nc.Close() t.Fatalf("Expected error trying to connect to secure server with no auth") } // Now do more advanced checking, verifying servername and using rootCA. nc, err = nats.Connect(nurl, nats.RootCAs("./configs/certs/ca.pem")) if err != nil { t.Fatalf("Got an error on Connect with Secure Options: %+v\n", err) } defer nc.Close() subj := "foo-tls" sub, _ := nc.SubscribeSync(subj) nc.Publish(subj, []byte("We are Secure!")) nc.Flush() nmsgs, _, _ := sub.Pending() if nmsgs != 1 { t.Fatalf("Expected to receive a message over the TLS connection") } } type captureSlowConsumerLogger struct { dummyLogger ch chan string gotIt bool } func (l *captureSlowConsumerLogger) Noticef(format string, v ...any) { msg := fmt.Sprintf(format, v...) if strings.Contains(msg, "Slow Consumer") { l.Lock() if !l.gotIt { l.gotIt = true l.ch <- msg } l.Unlock() } } func TestTLSTimeoutNotReportSlowConsumer(t *testing.T) { oa, err := server.ProcessConfigFile("./configs/srv_a_tls.conf") if err != nil { t.Fatalf("Unable to load config file: %v", err) } // Override TLSTimeout to very small value so that handshake fails oa.Cluster.TLSTimeout = 0.0000001 sa := RunServer(oa) defer sa.Shutdown() ch := make(chan string, 1) cscl := &captureSlowConsumerLogger{ch: ch} sa.SetLogger(cscl, false, false) ob, err := server.ProcessConfigFile("./configs/srv_b_tls.conf") if err != nil { t.Fatalf("Unable to load config file: %v", err) } sb := RunServer(ob) defer sb.Shutdown() // Watch the logger for a bit and make sure we don't get any // Slow Consumer error. select { case e := <-ch: t.Fatalf("Unexpected slow consumer error: %s", e) case <-time.After(500 * time.Millisecond): // ok } } func TestTLSHandshakeFailureMemUsage(t *testing.T) { for i, test := range []struct { name string config string }{ { "connect to TLS client port", ` port: -1 %s `, }, { "connect to TLS route port", ` port: -1 cluster { port: -1 %s } `, }, { "connect to TLS gateway port", ` port: -1 gateway { name: "A" port: -1 %s } `, }, } { t.Run(test.name, func(t *testing.T) { content := fmt.Sprintf(test.config, ` tls { cert_file: "configs/certs/server-cert.pem" key_file: "configs/certs/server-key.pem" ca_file: "configs/certs/ca.pem" timeout: 5 } `) conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() var port int switch i { case 0: port = opts.Port case 1: port = s.ClusterAddr().Port case 2: port = s.GatewayAddr().Port } varz, _ := s.Varz(nil) base := varz.Mem buf := make([]byte, 1024*1024) for i := 0; i < 100; i++ { conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port)) if err != nil { t.Fatalf("Error on dial: %v", err) } conn.SetWriteDeadline(time.Now().Add(10 * time.Millisecond)) conn.Write(buf) conn.Close() } varz, _ = s.Varz(nil) newval := (varz.Mem - base) / (1024 * 1024) // May need to adjust that, but right now, with 100 clients // we are at about 20MB for client port, 11MB for route // and 6MB for gateways, so pick 50MB as the threshold if newval >= 50 { t.Fatalf("Consumed too much memory: %v MB", newval) } }) } } func TestTLSClientAuthWithRDNSequence(t *testing.T) { for _, test := range []struct { name string config string certs nats.Option err error rerr error }{ // To generate certs for these tests: // USE `regenerate_rdns_svid.sh` TO REGENERATE // // ``` // openssl req -newkey rsa:2048 -nodes -keyout client-$CLIENT_ID.key -subj "/C=US/ST=CA/L=Los Angeles/OU=NATS/O=NATS/CN=*.example.com/DC=example/DC=com" -addext extendedKeyUsage=clientAuth -out client-$CLIENT_ID.csr // openssl x509 -req -extfile <(printf "subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com") -days 1825 -in client-$CLIENT_ID.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client-$CLIENT_ID.pem // ``` // // To confirm subject from cert: // // ``` // openssl x509 -in client-$CLIENT_ID.pem -text | grep Subject: // ``` // { "connect with tls using full RDN sequence", ` port: -1 %s authorization { users = [ { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" } ] } `, // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), nil, nil, }, { "connect with tls using full RDN sequence in original order", ` port: -1 %s authorization { users = [ { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } ] } `, // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2 nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), nil, nil, }, { "connect with tls using partial RDN sequence has different permissions", ` port: -1 %s authorization { users = [ { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" }, { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" }, { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US", permissions = { subscribe = { deny = ">" }} } ] } `, // C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost nats.ClientCert("./configs/certs/rdns/client-b.pem", "./configs/certs/rdns/client-b.key"), nil, errors.New("nats: timeout"), }, { "connect with tls and RDN sequence partially matches", ` port: -1 %s authorization { users = [ { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" } { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US"}, ] } `, // Cert is: // // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 // // but it will actually match the user without DCs so will not get an error: // // C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost // nats.ClientCert("./configs/certs/rdns/client-c.pem", "./configs/certs/rdns/client-c.key"), nil, nil, }, { "connect with tls and RDN sequence does not match", ` port: -1 %s authorization { users = [ { user = "CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2" }, { user = "DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US" } ] } `, // C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4 // nats.ClientCert("./configs/certs/rdns/client-c.pem", "./configs/certs/rdns/client-c.key"), errors.New("nats: Authorization Violation"), nil, }, { "connect with tls and RDN sequence with space after comma not should matter", ` port: -1 %s authorization { users = [ { user = "DC=foo2, DC=foo1, CN=localhost, OU=NATS, O=NATS, L=Los Angeles, ST=CA, C=US" } ] } `, // C=US/ST=California/L=Los Angeles/O=NATS/OU=NATS/CN=localhost/DC=foo1/DC=foo2 // nats.ClientCert("./configs/certs/rdns/client-a.pem", "./configs/certs/rdns/client-a.key"), nil, nil, }, { "connect with tls and full RDN sequence respects order", ` port: -1 %s authorization { users = [ { user = "DC=com, DC=example, CN=*.example.com, O=NATS, OU=NATS, L=Los Angeles, ST=CA, C=US" } ] } `, // // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com // nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), nil, nil, }, { "connect with tls and full RDN sequence with added domainComponents and spaces also matches", ` port: -1 %s authorization { users = [ { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com" } ] } `, // // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com // nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), nil, nil, }, { "connect with tls and full RDN sequence with correct order takes precedence over others matches", ` port: -1 %s authorization { users = [ { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com", permissions = { subscribe = { deny = ">" }} } # This should take precedence since it is in the RFC2253 order. { user = "DC=com,DC=example,CN=*.example.com,O=NATS,OU=NATS,L=Los Angeles,ST=CA,C=US" } { user = "CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US", permissions = { subscribe = { deny = ">" }} } ] } `, // // C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com // nats.ClientCert("./configs/certs/rdns/client-d.pem", "./configs/certs/rdns/client-d.key"), nil, nil, }, { "connect with tls and RDN includes multiple CN elements", ` port: -1 %s authorization { users = [ { user = "DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe" } ] } `, // // OpenSSL: -subj "/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com" // Go: CN=jdoe,OU=Users+OU=Organic Units // RFC2253: DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe // nats.ClientCert("./configs/certs/rdns/client-e.pem", "./configs/certs/rdns/client-e.key"), nil, nil, }, { "connect with tls and DN includes a multi value RDN", ` port: -1 %s authorization { users = [ { user = "CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), nil, nil, }, { "connect with tls and DN includes a multi value RDN but there is no match", ` port: -1 %s authorization { users = [ { user = "CN=John Doe,DC=DEV,DC=OpenSSL,DC=org" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), errors.New("nats: Authorization Violation"), nil, }, { "connect with tls and DN includes a multi value RDN that are reordered", ` port: -1 %s authorization { users = [ { user = "CN=John Doe,O=users+DC=DEV,DC=OpenSSL,DC=org" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), nil, nil, }, } { t.Run(test.name, func(t *testing.T) { content := fmt.Sprintf(test.config, ` tls { cert_file: "configs/certs/rdns/server.pem" key_file: "configs/certs/rdns/server.key" ca_file: "configs/certs/rdns/ca.pem" timeout: 5 verify_and_map: true } `) conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.certs, nats.RootCAs("./configs/certs/rdns/ca.pem"), nats.ErrorHandler(noOpErrHandler), ) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestTLSClientAuthWithRDNSequenceReordered(t *testing.T) { for _, test := range []struct { name string config string certs nats.Option err error rerr error }{ { "connect with tls and DN includes a multi value RDN that are reordered", ` port: -1 %s authorization { users = [ { user = "DC=org,DC=OpenSSL,O=users+DC=DEV,CN=John Doe" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), nil, nil, }, { "connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs", ` port: -1 %s authorization { users = [ { user = "DC=org,DC=OpenSSL,O=users,CN=John Doe" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), errors.New("nats: Authorization Violation"), nil, }, { "connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs", ` port: -1 %s authorization { users = [ { user = "DC=OpenSSL, DC=org, O=users, CN=John Doe" } ] } `, // // OpenSSL: -subj "/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe" -multivalue-rdn // Go: CN=John Doe,O=users // RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org // nats.ClientCert("./configs/certs/rdns/client-f.pem", "./configs/certs/rdns/client-f.key"), errors.New("nats: Authorization Violation"), nil, }, } { t.Run(test.name, func(t *testing.T) { content := fmt.Sprintf(test.config, ` tls { cert_file: "configs/certs/rdns/server.pem" key_file: "configs/certs/rdns/server.key" ca_file: "configs/certs/rdns/ca.pem" timeout: 5 verify_and_map: true } `) conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.certs, nats.RootCAs("./configs/certs/rdns/ca.pem"), ) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestTLSClientSVIDAuth(t *testing.T) { for _, test := range []struct { name string config string certs nats.Option err error rerr error }{ { "connect with tls using certificate with URIs", ` port: -1 %s authorization { users = [ { user = "spiffe://localhost/my-nats-service/user-a" } ] } `, nats.ClientCert("./configs/certs/svid/client-a.pem", "./configs/certs/svid/client-a.key"), nil, nil, }, { "connect with tls using certificate with limited different permissions", ` port: -1 %s authorization { users = [ { user = "spiffe://localhost/my-nats-service/user-a" }, { user = "spiffe://localhost/my-nats-service/user-b" permissions = { subscribe = { deny = ">" }} } ] } `, nats.ClientCert("./configs/certs/svid/client-b.pem", "./configs/certs/svid/client-b.key"), nil, errors.New("nats: timeout"), }, { "connect with tls without URIs in permissions will still match SAN", ` port: -1 %s authorization { users = [ { user = "O=SPIRE,C=US" } ] } `, nats.ClientCert("./configs/certs/svid/client-b.pem", "./configs/certs/svid/client-b.key"), nil, nil, }, { "connect with tls but no permissions", ` port: -1 %s authorization { users = [ { user = "spiffe://localhost/my-nats-service/user-c" } ] } `, nats.ClientCert("./configs/certs/svid/client-a.pem", "./configs/certs/svid/client-a.key"), errors.New("nats: Authorization Violation"), nil, }, } { t.Run(test.name, func(t *testing.T) { content := fmt.Sprintf(test.config, ` tls { cert_file: "configs/certs/svid/server.pem" key_file: "configs/certs/svid/server.key" ca_file: "configs/certs/svid/ca.pem" timeout: 5 insecure: true verify_and_map: true } `) conf := createConfFile(t, []byte(content)) s, opts := RunServerWithConfig(conf) defer s.Shutdown() nc, err := nats.Connect(fmt.Sprintf("tls://localhost:%d", opts.Port), test.certs, nats.RootCAs("./configs/certs/svid/ca.pem"), nats.ErrorHandler(noOpErrHandler), ) if test.err == nil && err != nil { t.Errorf("Expected to connect, got %v", err) } else if test.err != nil && err == nil { t.Errorf("Expected error on connect") } else if test.err != nil && err != nil { // Error on connect was expected if test.err.Error() != err.Error() { t.Errorf("Expected error %s, got: %s", test.err, err) } return } defer nc.Close() nc.Subscribe("ping", func(m *nats.Msg) { m.Respond([]byte("pong")) }) nc.Flush() _, err = nc.Request("ping", []byte("ping"), 250*time.Millisecond) if test.rerr != nil && err == nil { t.Errorf("Expected error getting response") } else if test.rerr == nil && err != nil { t.Errorf("Expected response") } }) } } func TestTLSPinnedCertsClient(t *testing.T) { tmpl := ` host: localhost port: -1 tls { ca_file: "configs/certs/ca.pem" cert_file: "configs/certs/server-cert.pem" key_file: "configs/certs/server-key.pem" # Require a client certificate and map user id from certificate verify: true pinned_certs: ["%s"] }` confFileName := createConfFile(t, []byte(fmt.Sprintf(tmpl, "aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9"))) srv, o := RunServerWithConfig(confFileName) defer srv.Shutdown() if len(o.TLSPinnedCerts) != 1 { t.Fatal("expected one pinned cert") } opts := []nats.Option{ nats.RootCAs("configs/certs/ca.pem"), nats.ClientCert("./configs/certs/client-cert.pem", "./configs/certs/client-key.pem"), } nc, err := nats.Connect(srv.ClientURL(), opts...) if err == nil { nc.Close() t.Fatalf("Expected error trying to connect without a certificate in pinned_certs") } os.WriteFile(confFileName, []byte(fmt.Sprintf(tmpl, "bf6f821f09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9")), 0660) if err := srv.Reload(); err != nil { t.Fatalf("on Reload got %v", err) } // reload pinned to the certs used nc, err = nats.Connect(srv.ClientURL(), opts...) if err != nil { t.Fatalf("Expected no error, got: %v", err) } nc.Close() } type captureWarnLogger struct { dummyLogger receive chan string } func newCaptureWarnLogger() *captureWarnLogger { return &captureWarnLogger{ receive: make(chan string, 100), } } func (l *captureWarnLogger) Warnf(format string, v ...any) { l.receive <- fmt.Sprintf(format, v...) } func (l *captureWarnLogger) waitFor(expect string, timeout time.Duration) bool { for { select { case msg := <-l.receive: if strings.Contains(msg, expect) { return true } case <-time.After(timeout): return false } } } func TestTLSConnectionRate(t *testing.T) { config := ` listen: "127.0.0.1:-1" tls { cert_file: "./configs/certs/server-cert.pem" key_file: "./configs/certs/server-key.pem" connection_rate_limit: 3 } ` confFileName := createConfFile(t, []byte(config)) srv, _ := RunServerWithConfig(confFileName) logger := newCaptureWarnLogger() srv.SetLogger(logger, false, false) defer srv.Shutdown() var err error count := 0 for count < 10 { var nc *nats.Conn nc, err = nats.Connect(srv.ClientURL(), nats.RootCAs("./configs/certs/ca.pem")) if err != nil { break } nc.Close() count++ } if count != 3 { t.Fatalf("Expected 3 connections per second, got %d (%v)", count, err) } if !logger.waitFor("connections due to TLS rate limiting", time.Second) { t.Fatalf("did not log 'TLS rate limiting' warning") } } func TestTLSPinnedCertsRoute(t *testing.T) { tmplSeed := ` host: localhost port: -1 cluster { port: -1 pool_size: -1 tls { ca_file: "configs/certs/ca.pem" cert_file: "configs/certs/server-cert.pem" key_file: "configs/certs/server-key.pem" } }` // this server connects to seed, but is set up to not trust seeds cert tmplSrv := ` host: localhost port: -1 cluster { port: -1 routes = [nats-route://localhost:%d] tls { ca_file: "configs/certs/ca.pem" cert_file: "configs/certs/server-cert.pem" key_file: "configs/certs/server-key.pem" # Require a client certificate and map user id from certificate verify: true # expected to fail the seed server pinned_certs: ["%s"] } }` confSeed := createConfFile(t, []byte(tmplSeed)) srvSeed, o := RunServerWithConfig(confSeed) defer srvSeed.Shutdown() confSrv := createConfFile(t, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, "89386860ea1222698ea676fc97310bdf2bff6f7e2b0420fac3b3f8f5a08fede5"))) srv, _ := RunServerWithConfig(confSrv) defer srv.Shutdown() checkClusterFormed(t, srvSeed, srv) // this change will result in the server being and remaining disconnected os.WriteFile(confSrv, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, "aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9")), 0660) if err := srv.Reload(); err != nil { t.Fatalf("on Reload got %v", err) } checkNumRoutes(t, srvSeed, 0) checkNumRoutes(t, srv, 0) } func TestAllowNonTLSReload(t *testing.T) { tmpl := ` listen: "127.0.0.1:-1" ping_interval: "%s" tls { ca_file: "configs/certs/ca.pem" cert_file: "configs/certs/server-cert.pem" key_file: "configs/certs/server-key.pem" } allow_non_tls: true ` conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, "10s"))) s, o := RunServerWithConfig(conf) defer s.Shutdown() check := func() { t.Helper() nc := createClientConn(t, "127.0.0.1", o.Port) defer nc.Close() info := checkInfoMsg(t, nc) if !info.TLSAvailable { t.Fatal("TLSAvailable should be true, was false") } if info.TLSRequired { t.Fatal("TLSRequired should be false, was true") } } check() os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, "20s")), 0660) if err := s.Reload(); err != nil { t.Fatalf("Error on reload: %v", err) } check() } nats-server-2.10.27/test/user_authorization_test.go000066400000000000000000000220671477524627100224650ustar00rootroot00000000000000// Copyright 2016-2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "regexp" "testing" "time" ) const DefaultPass = "foo" var permErrRe = regexp.MustCompile(`\A\-ERR\s+'Permissions Violation([^\r\n]+)\r\n`) func TestUserAuthorizationProto(t *testing.T) { srv, opts := RunServerWithConfig("./configs/authorization.conf") defer srv.Shutdown() // Alice can do anything, check a few for OK result. c := createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "alice", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "PUB foo 2\r\nok\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB foo 1\r\n") expectResult(t, c, okRe) // Check that _ is ok sendProto(t, c, "PUB _ 2\r\nok\r\n") expectResult(t, c, okRe) c.Close() // Bob is a requestor only, e.g. req.foo, req.bar for publish, subscribe only to INBOXes. c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "bob", DefaultPass) expectResult(t, c, okRe) // These should error. sendProto(t, c, "SUB foo 1\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB foo 2\r\nok\r\n") expectResult(t, c, permErrRe) // These should work ok. sendProto(t, c, "SUB _INBOX.abcd 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB req.foo 2\r\nok\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB req.bar 2\r\nok\r\n") expectResult(t, c, okRe) c.Close() // Joe is a default user c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "joe", DefaultPass) expectResult(t, c, okRe) // These should error. sendProto(t, c, "SUB foo.bar.* 1\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB foo.bar.baz 2\r\nok\r\n") expectResult(t, c, permErrRe) // These should work ok. sendProto(t, c, "SUB _INBOX.abcd 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB PUBLIC.abcd 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB SANDBOX.foo 2\r\nok\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB SANDBOX.bar 2\r\nok\r\n") expectResult(t, c, okRe) // Since only PWC, this should fail (too many tokens). sendProto(t, c, "PUB SANDBOX.foo.bar 2\r\nok\r\n") expectResult(t, c, permErrRe) c.Close() // This is the new style permissions with allow and deny clauses. c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "ns", DefaultPass) expectResult(t, c, okRe) // These should work sendProto(t, c, "PUB SANDBOX.foo 2\r\nok\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB baz.bar 2\r\nok\r\n") expectResult(t, c, okRe) sendProto(t, c, "PUB baz.foo 2\r\nok\r\n") expectResult(t, c, okRe) // These should error. sendProto(t, c, "PUB foo 2\r\nok\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB bar 2\r\nok\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB foo.bar 2\r\nok\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB foo.bar.baz 2\r\nok\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "PUB SYS.1 2\r\nok\r\n") expectResult(t, c, permErrRe) // Subscriptions // These should work ok. sendProto(t, c, "SUB foo.bar 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB foo.foo 1\r\n") expectResult(t, c, okRe) // These should error. sendProto(t, c, "SUB foo 1\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "SUB foo.baz 1\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "SUB foo.baz 1\r\n") expectResult(t, c, permErrRe) sendProto(t, c, "SUB foo.baz 1\r\n") expectResult(t, c, permErrRe) // Deny clauses for subscriptions need to be able to allow subscriptions // on larger scoped wildcards, but prevent delivery of a message whose // subject matches a deny clause. // Clear old stuff c.Close() c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "ns", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "SUB foo.* 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB foo.* bar 2\r\n") expectResult(t, c, okRe) // Now send on foo.baz which should not be received on first client. // Joe is a default user nc := createClientConn(t, opts.Host, opts.Port) defer nc.Close() expectAuthRequired(t, nc) doAuthConnect(t, nc, "", "ns-pub", DefaultPass) expectResult(t, nc, okRe) sendProto(t, nc, "PUB foo.baz 2\r\nok\r\n") expectResult(t, nc, okRe) // Expect nothing from the wildcard subscription. expectNothing(t, c) sendProto(t, c, "PING\r\n") expectResult(t, c, pongRe) // Now create a queue sub on our ns-pub user. We want to test that // queue subscribers can be denied and delivery will route around. sendProto(t, nc, "SUB foo.baz bar 2\r\n") expectResult(t, nc, okRe) // Make sure we always get the message on our queue subscriber. // Do this several times since we should select the other subscriber // but get permission denied.. for i := 0; i < 20; i++ { sendProto(t, nc, "PUB foo.baz 2\r\nok\r\n") buf := expectResult(t, nc, okRe) if msgRe.Match(buf) { continue } else { expectResult(t, nc, msgRe) } } // Clear old stuff c.Close() c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "ns", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "SUB foo.bar 1\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB foo.bar.baz 2\r\n") expectResult(t, c, errRe) sendProto(t, c, "SUB > 3\r\n") expectResult(t, c, errRe) sendProto(t, c, "SUB SYS.> 4\r\n") expectResult(t, c, errRe) sendProto(t, c, "SUB SYS.TEST.foo 5\r\n") expectResult(t, c, okRe) sendProto(t, c, "SUB SYS.bar 5\r\n") expectResult(t, c, errRe) } func TestUserAuthorizationAllowResponses(t *testing.T) { srv, opts := RunServerWithConfig("./configs/authorization.conf") defer srv.Shutdown() // Alice can do anything, so she will be our requestor rc := createClientConn(t, opts.Host, opts.Port) defer rc.Close() expectAuthRequired(t, rc) doAuthConnect(t, rc, "", "alice", DefaultPass) expectResult(t, rc, okRe) // MY_SERVICE can subscribe to a single request subject but can // respond to any reply subject that it receives, but only // for one response. c := createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "svca", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "SUB my.service.req 1\r\n") expectResult(t, c, okRe) sendProto(t, rc, "PUB my.service.req resp.bar.22 2\r\nok\r\n") expectResult(t, rc, okRe) matches := msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.22", "2", "ok") // This should be allowed sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") expectResult(t, c, okRe) // This should not be allowed sendProto(t, c, "PUB resp.bar.33 2\r\nok\r\n") expectResult(t, c, errRe) // This should also not be allowed now since we already sent a response and max is 1. sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") expectResult(t, c, errRe) c.Close() // from MY_SERVICE // MY_STREAM_SERVICE can subscribe to a single request subject but can // respond to any reply subject that it receives, and send up to 10 responses. // Each permission for a response can last up to 10ms. c = createClientConn(t, opts.Host, opts.Port) defer c.Close() expectAuthRequired(t, c) doAuthConnect(t, c, "", "svcb", DefaultPass) expectResult(t, c, okRe) sendProto(t, c, "SUB my.service.req 1\r\n") expectResult(t, c, okRe) // Same rules as above. sendProto(t, rc, "PUB my.service.req resp.bar.22 2\r\nok\r\n") expectResult(t, rc, okRe) matches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.22", "2", "ok") // This should be allowed sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") expectResult(t, c, okRe) // This should not be allowed sendProto(t, c, "PUB resp.bar.33 2\r\nok\r\n") expectResult(t, c, errRe) // We should be able to send 9 more here since we are allowed 10 total for i := 0; i < 9; i++ { sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") expectResult(t, c, okRe) } // Now this should fail since we already sent 10 responses. sendProto(t, c, "PUB resp.bar.22 2\r\nok\r\n") expectResult(t, c, errRe) // Now test timeout. sendProto(t, rc, "PUB my.service.req resp.bar.11 2\r\nok\r\n") expectResult(t, rc, okRe) matches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1) checkMsg(t, matches[0], "my.service.req", "1", "resp.bar.11", "2", "ok") time.Sleep(100 * time.Millisecond) sendProto(t, c, "PUB resp.bar.11 2\r\nok\r\n") expectResult(t, c, errRe) } nats-server-2.10.27/test/verbose_test.go000066400000000000000000000033341477524627100201700ustar00rootroot00000000000000// Copyright 2012-2018 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package test import ( "testing" ) func TestVerbosePing(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() doConnect(t, c, true, false, false) send := sendCommand(t, c) expect := expectCommand(t, c) expect(okRe) // Ping should still be same send("PING\r\n") expect(pongRe) } func TestVerboseConnect(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() doConnect(t, c, true, false, false) send := sendCommand(t, c) expect := expectCommand(t, c) expect(okRe) // Connect send("CONNECT {\"verbose\":true,\"pedantic\":true,\"tls_required\":false}\r\n") expect(okRe) } func TestVerbosePubSub(t *testing.T) { s := runProtoServer() defer s.Shutdown() c := createClientConn(t, "127.0.0.1", PROTO_TEST_PORT) defer c.Close() doConnect(t, c, true, false, false) send := sendCommand(t, c) expect := expectCommand(t, c) expect(okRe) // Pub send("PUB foo 2\r\nok\r\n") expect(okRe) // Sub send("SUB foo 1\r\n") expect(okRe) // UnSub send("UNSUB 1\r\n") expect(okRe) } nats-server-2.10.27/util/000077500000000000000000000000001477524627100151305ustar00rootroot00000000000000nats-server-2.10.27/util/nats-server-hardened.service000066400000000000000000000042021477524627100225310ustar00rootroot00000000000000[Unit] Description=NATS Server After=network-online.target ntp.service # If you use a dedicated filesystem for JetStream data, then you might use something like: # ConditionPathIsMountPoint=/srv/jetstream # See also Service.ReadWritePaths [Service] Type=simple EnvironmentFile=-/etc/default/nats-server ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s SIGINT $MAINPID User=nats Group=nats Restart=always RestartSec=5 # The nats-server uses SIGUSR2 to trigger using Lame Duck Mode (LDM) shutdown KillSignal=SIGUSR2 # You might want to adjust TimeoutStopSec too. # Capacity Limits # JetStream requires 2 FDs open per stream. LimitNOFILE=800000 # Environment=GOMEMLIMIT=12GiB # You might find it better to set GOMEMLIMIT via /etc/default/nats-server, # so that you can change limits without needing a systemd daemon-reload. # Hardening CapabilityBoundingSet= LockPersonality=true MemoryDenyWriteExecute=true NoNewPrivileges=true PrivateDevices=true PrivateTmp=true PrivateUsers=true ProcSubset=pid ProtectClock=true ProtectControlGroups=true ProtectHome=true ProtectHostname=true ProtectKernelLogs=true ProtectKernelModules=true ProtectKernelTunables=true ProtectSystem=strict ReadOnlyPaths= RestrictAddressFamilies=AF_INET AF_INET6 RestrictNamespaces=true RestrictRealtime=true RestrictSUIDSGID=true SystemCallFilter=@system-service ~@privileged ~@resources UMask=0077 # Consider locking down all areas of /etc which hold machine identity keys, etc InaccessiblePaths=/etc/ssh # If you have systemd >= 247 ProtectProc=invisible # If you have systemd >= 248 PrivateIPC=true # Optional: writable directory for JetStream. # See also: Unit.ConditionPathIsMountPoint ReadWritePaths=/var/lib/nats # Optional: resource control. # Replace weights by values that make sense for your situation. # For a list of all options see: # https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html #CPUAccounting=true #CPUWeight=100 # of 10000 #IOAccounting=true #IOWeight=100 # of 10000 #MemoryAccounting=true #MemoryMax=1GB #IPAccounting=true [Install] WantedBy=multi-user.target Alias=nats.service nats-server-2.10.27/util/nats-server.service000066400000000000000000000006751477524627100207730ustar00rootroot00000000000000[Unit] Description=NATS Server After=network-online.target ntp.service [Service] PrivateTmp=true Type=simple ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s SIGINT $MAINPID User=nats Group=nats # The nats-server uses SIGUSR2 to trigger using Lame Duck Mode (LDM) shutdown KillSignal=SIGUSR2 # You might want to adjust TimeoutStopSec too. [Install] WantedBy=multi-user.target